datafile 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26b6312f588cc22d664d552ca2f01e6e11482f3d
-  data.tar.gz: 3afcbe6af0340685f5358a9f0713a4631a59e5ee
+  metadata.gz: 62443719632bb6ef1b75ce5b79ef146e1ddca9b1
+  data.tar.gz: 5d39b5537c4f280cb395d2be2e9abec702d910b4
 SHA512:
-  metadata.gz: 78dddc2a46795eb19b25819c9cc95e3d5202b81dcd11851ddbce92ac7efd883815495a6224546baaa53fbefaeb944abbdc686645b16ac3c9464bec7ccb491f5c
-  data.tar.gz: 48ae309c57d5a09b77d81d40e9542473427928c26ffabb28b9e4e7f18af941de1196f0f17a43440d9456a155111ddec61c9dfef7ae1854a3e8cb2c25ecbbc30c
+  metadata.gz: 04882e94add84a5c478025535375760c21ea006714a1f6e3d8b92660d98477d460065306082ac4ebf321621c7782d99beac8ae0ab7f75f28c3d430ef80658e84
+  data.tar.gz: 1ece9f87e091abf9fb494777aaf626a647188a788b4ede359bfb0a940eb53dffb50db90e6354d59cb128b383a980968928ccbfdfcc86eacd8fef766f11bf55b7

data/Manifest.txt

@@ -4,11 +4,25 @@ README.md
 Rakefile
 lib/datafile.rb
 lib/datafile/builder.rb
+lib/datafile/builder2.rb
 lib/datafile/datafile.rb
-lib/datafile/datasets/beer.rb
 lib/datafile/datasets/dataset.rb
-lib/datafile/datasets/football.rb
-lib/datafile/datasets/world.rb
 lib/datafile/version.rb
+lib/datafile/workers/dataset.rb
+lib/datafile/workers/file/dataset.rb
+lib/datafile/workers/file/registry.rb
+lib/datafile/workers/file/worker.rb
+lib/datafile/workers/zip/beer.rb
+lib/datafile/workers/zip/dataset.rb
+lib/datafile/workers/zip/football.rb
+lib/datafile/workers/zip/worker.rb
+lib/datafile/workers/zip/world.rb
+test/datafile/eurocup.rb
+test/datafile/stadiums.rb
+test/datafile/world.rb
+test/datafile2/at.rb
 test/helper.rb
 test/test_builder.rb
+test/test_builder2.rb
+test/test_file_dataset_registry.rb
+test/test_file_worker.rb

data/lib/datafile.rb

@@ -8,12 +8,24 @@ require 'logutils'
 require 'datafile/version'  # let it always go first
 
 require 'datafile/datasets/dataset'
-require 'datafile/datasets/football'
-require 'datafile/datasets/beer'
-require 'datafile/datasets/world'
+
+require 'datafile/workers/dataset'
+
+require 'datafile/workers/file/dataset'
+require 'datafile/workers/file/registry'
+require 'datafile/workers/file/worker'
+
+require 'datafile/workers/zip/dataset'
+require 'datafile/workers/zip/beer'
+require 'datafile/workers/zip/football'
+require 'datafile/workers/zip/world'
+require 'datafile/workers/zip/worker'
+
 require 'datafile/datafile'
 require 'datafile/builder'
+require 'datafile/builder2'
 
 
 # say hello
-puts Datafile.banner    if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG)
+puts Datafile.banner    if defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG
+

data/lib/datafile/builder.rb

@@ -2,15 +2,7 @@
 
 module Datafile
 
-class Builder
-
-  include LogUtils::Logging
-
-  attr_reader :datafile
-
-  def initialize()
-    @datafile = Datafile.new
-  end
+class Builder     ## "simple" builder (one file, one datafile)
 
   def self.load_file( path )
     code = File.read_utf8( path )
@@ -24,6 +16,14 @@ class Builder
   end
 
 
+  include LogUtils::Logging
+
+  def initialize
+    @datafile = Datafile.new
+  end
+
+  attr_reader :datafile
+
   def beer( name, opts={} )
     logger.info( "[builder] add beer-dataset '#{name}'" )
     @datafile.datasets << BeerDataset.new( name, opts )
@@ -41,3 +41,4 @@ class Builder
 
 end # class Builder
 end # module Datafile
+

data/lib/datafile/builder2.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+
+module Datafile
+
+
+###
+##  check/todo: ename to BatchBuilder, MultiBuilder,etc - find better name - why, why not??
+
+class BuilderEx
+
+  def self.load_file( path )
+    code = File.read_utf8( path )
+    self.load( code )
+  end
+
+  def self.load( code )
+    builder = BuilderEx.new
+    builder.instance_eval( code )
+    builder
+  end
+
+
+  include LogUtils::Logging
+
+  def initialize
+    @datafiles = []
+    @datafile = nil
+  end
+
+  attr_reader :datafiles
+
+
+  def task( arg )
+
+    logger.info( "[builder] add task '#{arg.inspect}' : #{arg.class.name}" )
+
+    if arg.kind_of?( String ) || arg.kind_of?( Symbol )   # e.g. 'at' or :at
+      name = arg.to_s
+      ## note: always default to FileWorker for now
+      ##  -- use file: true -- find better name e.g. worker/source: file - why? why not??
+      @datafile = Datafile.new( name: name, deps: [], file: true )
+      yield  ### execute block in context
+       ## b = Builder.new
+       ## block.call( b )  ## same as b.instance_eval( &block) ???
+       ## b.instance_eval( code )
+       ## b = Builder.load( &block )
+    elsif arg.kind_of?( Hash )  ## Hash  e.g. :at_calc => :at etc.
+      key   = arg.keys.first
+      value = arg[key]   ## todo: check if single value? always turn into array
+
+      name  = key.to_s   ## get first key (assume it's name)
+      if value.kind_of?( Array )
+        deps = value.map { |v| v.to_s }   ## convert to strings
+      else  ## assume single string/symbol -- convert to array
+        deps = [value.to_s]
+      end
+      @datafile = Datafile.new( name: name, deps: deps, file: true )  ## note: always default to FileWorker for now
+      yield   ### execute block in context
+      ## to be done
+    else
+      ## fix: report error: unknown type
+    end
+
+    @datafiles << @datafile
+  end
+
+  def calc( &block )
+    logger.info( "[builder] add script calc-block" )
+    @datafile.scripts << Script.new( block )
+  end
+
+  ################################
+  # "classic/standard" datasets
+  def beer( name, opts={} )
+    logger.info( "[builder] add beer-dataset '#{name}'" )
+    @datafile.datasets << BeerDataset.new( name, opts )
+  end
+
+  def football( name, opts={} )
+    logger.info( "[builder] add football-dataset '#{name}'" )
+    @datafile.datasets << FootballDataset.new( name, opts )
+  end
+
+  def world( name, opts={} )
+    logger.info( "[builder] add world-dataset '#{name}'" )
+    @datafile.datasets << WorldDataset.new( name, opts )
+  end
+
+end # class Builder2
+end # module Datafile

data/lib/datafile/datafile.rb

@@ -2,13 +2,26 @@
 
 module Datafile
 
-class Datafile
-
+class Script
   include LogUtils::Logging
 
-  attr_reader :datasets
+  def initialize( proc )
+    @proc = proc
+  end
+
+  def call
+    logger.info( "[script] calling calc block" )
+    @proc.call
+  end
+  
+  def dump
+    puts "  script: #{@proc.inspect}"
+  end
+end  ## class Script
 
 
+class Datafile
+
   ## convenience method - use like Datafile.load_file()
   def self.load_file( path='./Datafile' )
     code = File.read_utf8( path )
@@ -26,27 +39,67 @@ class Datafile
   end
 
 
-  def initialize
+  include LogUtils::Logging
+
+  def initialize( opts={} )
+    @opts     = opts
     @datasets = []
+    @scripts  = []   ## calculation scripts (calc blocks)
+
+    ## (target)name - return nil if noname (set/defined/assigned)
+    @name  = opts[:name] || nil
+    ## deps (dependencies) - note: always returns an array (empty array if no deps)
+    @deps  = opts[:deps] || []     
+
+    if opts[:file]
+      @worker  = FileWorker.new( self )
+    else
+      ## default to zip worker for now
+      @worker   = ZipWorker.new( self )
+    end
   end
 
+  attr_reader   :datasets
+  attr_reader   :scripts    ## calc(ulation) scripts (calc blocks)
+  attr_reader   :name
+  attr_reader   :deps       ## dep(endencies)
+
+  attr_accessor :worker      # lets you change worker - find a better way - how, why, why not??
+
 
   def run
     logger.info( "[datafile] begin - run" )
-    download()   # step 1 - download zips for datasets
-    read()       # step 2 - read in datasets from zips
+    download     # step 1 - download zips for datasets
+    read         # step 2 - read in datasets from zips
+    calc         # step 3 - run calc(ulations) scripts
     logger.info( "[datafile] end - run" )
   end
 
 
-
   def download
     logger.info( "[datafile] dowload" )
-    @datasets.each do |dataset|
-      dataset.download()
-    end
+    @worker.download
+    ## check: use @worker.download( @datasets) - why, why not??  link worker w/ datafile - why, why not??  
+  end
+
+  def read
+    logger.info( "[datafile] read" )
+    @worker.read
+  end
+
+  def calc
+    logger.info( "[datafile] calc" )
+    @worker.calc
   end
 
+  def dump
+    ## for debugging dump datasets (note: will/might also check if zip exits)
+    logger.info( "[datafile] dump datasets (for debugging)" )
+    @worker.dump
+  end
+
+
+=begin
   def download_world   ## only dl world datasets (skip all others)
     logger.info( "[datafile] dowload world datasets" )
     @datasets.each do |dataset|
@@ -79,21 +132,14 @@ class Datafile
       end
     end
   end
+=end
 
 
-
-
-  def read
-    logger.info( "[datafile] read" )
-    @datasets.each do |dataset|
-      dataset.read()
-    end
-  end
-
+=begin
   def read_world
     logger.info( "[datafile] read world datasets" )
     @datasets.each do |dataset|
-      if dataset.kind_of? WorldDataset
+      if dataset.kind_of?( WorldDataset )
         dataset.read()
       else
         # skip all others
@@ -104,7 +150,7 @@ class Datafile
   def read_beer
     logger.info( "[datafile] read beer datasets" )
     @datasets.each do |dataset|
-      if dataset.kind_of? BeerDataset
+      if dataset.kind_of?( BeerDataset )
         dataset.read()
       else
         # skip all others
@@ -115,23 +161,14 @@ class Datafile
   def read_football
     logger.info( "[datafile] read football datasets" )
     @datasets.each do |dataset|
-      if dataset.kind_of? FootballDataset
+      if dataset.kind_of?( FootballDataset )
         dataset.read()
       else
         # skip all others
       end
     end
   end
-
-
-  def dump
-    ## for debugging dump datasets (note: will/might also check if zip exits)
-    logger.info( "[datafile] dump datasets (for debugging)" )
-    @datasets.each do |dataset|
-      dataset.dump()
-    end
-  end
-
+=end
 
 end # class Datafile
 end # module Datafile

data/lib/datafile/datasets/dataset.rb

@@ -10,71 +10,42 @@ class Dataset
     @opts = opts
   end
 
+  attr_reader :name
+  attr_reader :opts
 
   def setup
     value = @opts[:setup] || 'all'
     "setups/#{value}"
   end
+end # class Dataset
 
-  def remote_zip_url  # remote zip url
-    ###  note: use http:// for now - lets us use (personal proxy NOT working w/ https) for now
-    ## "https://github.com/#{@name}/archive/master.zip"
-    "http://github.com/#{@name}/archive/master.zip"
-  end
-
-  def local_zip_name
-    ### note: replace / in name w/ --I--
-    ##  e.g. flatten the filename, that is, do NOT include any folders
-    @name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
-  end
 
-  def local_zip_root
-    "./tmp"
-  end
-
-  def local_zip_path  # local zip path
-    "#{local_zip_root}/#{local_zip_name}.zip"
+class WorldDataset < Dataset
+  def initialize( name, opts={} )
+    super( name, opts )  ## todo/check: just juse super (e.g. pass along all params - why? why not?)
   end
+  
+  def zip_worker()   WorldZipDataset.new( self );  end   ## check: change (rename) just use zip or use worker_zip?? - why, why not?
+  def file_worker()  WorldFileDataset.new( self ); end
+end  # class WorldDataset
 
-
-  def download
-    logger.info( "download dataset '#{@name}'" )
-    logger.info( "   from '#{remote_zip_url}'" )
-    logger.info( "   to '#{local_zip_path}'..." )
-
-    download_blob( remote_zip_url, local_zip_path )
+class FootballDataset < Dataset
+  def initialize( name, opts={} )
+    super( name, opts )
   end
 
+  def zip_worker()   FootballZipDataset.new( self );  end
+  def file_worker()  FootballFileDataset.new( self ); end
+end # class FootballDataset
 
-  def dump
-    ## for debuggin dump dataset (also check if zip exits)
-    puts "dataset '#{@name}' opts=#{@opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
-    puts "  local '#{local_zip_name}' (#{local_zip_path})"
-    if File.exist?( local_zip_path )
-      puts "    size: #{File.size(local_zip_path)} bytes"
-    else
-      puts "    (file not found)"
-    end
-    puts "  remote '#{remote_zip_url}'"
+class BeerDataset < Dataset
+  def initialize( name, opts={} )
+    super( name, opts )
   end
 
+  def zip_worker()   BeerZipDataset.new( self );  end
+  def file_worker()  BeerFileDataset.new( self ); end
+end  # class BeerDataset
 
-private
-  ####
-  # download tasks for zips
-  def download_blob( url, dest )
-    logger.info "downloading #{url} to #{dest}..."
-
-    ## make sure dest path exists
-    dest_p = File.dirname( dest )
-    FileUtils.mkdir_p( dest_p )  unless File.exists?( dest_p )   ## use Dir.exists?? why? why not??
-
-    worker = Fetcher::Worker.new
-    worker.copy( url, dest )
-    ## print some file stats
-    logger.debug "  size: #{File.size(dest)} bytes"
-  end
-
-end  # class Dataset
 
 end # module Datafile

data/lib/datafile/version.rb

@@ -3,8 +3,8 @@
 module Datafile
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 2
+  MINOR = 2
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/datafile/workers/dataset.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+
+module Datafile
+
+class Error < StandardError
+end
+
+####
+# todo/check:
+# rename to DatasetNotFound or similar??
+# use "common" error class - why? why not?
+class DatasetNotFoundError < Error
+  attr_reader :message
+
+  def initialize( message )
+    @message = message
+  end
+
+  def to_s
+    "datset not found => #{@message}"
+  end
+end
+
+
+class DatasetNode     ### find a better name (e.g. DatasetWorker, DatasetBase, DatasetRef, DatasetWrapper ???) ???
+
+  include LogUtils::Logging
+
+  def initialize( dataset )
+    @dataset = dataset
+  end
+
+  def name()  @dataset.name;  end
+  def opts()  @dataset.opts;  end
+  def setup() @dataset.setup; end
+
+end # class DatasetNode
+
+
+end # module Datafile

data/lib/datafile/workers/file/dataset.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+module Datafile
+
+class FileDataset < DatasetNode
+  ## read dataset from file(system)
+
+  @@registry = nil
+
+  def self.registry
+    ## use ||=  why, why not??  - add Registry as nested class, why, why not ??
+    if @@registry.nil?
+      @@registry = FileDatasetRegistry.new
+    end
+    @@registry
+  end
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def repo_dir     ### check: use (rename to) include dir (or local_repo_dir) - why, why not ??? 
+    registry.lookup( name )
+  end
+
+  def dump
+    ## for debuggin dump dataset  -- todo (also check if folder exits ??)
+    puts "dataset '#{name}' opts=#{opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
+    puts "  repo-dir '#{repo_dir}'"
+  end
+
+private
+  def registry    ## convenience method to access "static" shared class variable
+    FileDataset.registry     ## self.registry  not working?? - or self.registry() -why, why not??
+  end
+end # class FileDataset
+
+
+
+class FootballFileDataset < FileDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read()
+    logger.info( "read football-dataset (file) '#{name}', '#{setup}'" )
+
+    SportDb.read_setup( setup, repo_dir )
+  end
+end # class FootballFileDataset
+
+
+class WorldFileDataset < FileDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read()
+    logger.info( "read world-dataset (file) '#{name}', '#{setup}'" )
+
+    ## WorldDb.read_setup( 'setups/countries', WORLD_DB_INCLUDE_PATH, skip_tags: true )
+    WorldDb.read_setup( setup, repo_dir, skip_tags: true  )
+  end
+end # class WorldFileDataset
+
+class BeerFileDataset < FileDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read()
+    logger.info( "read beer-dataset (file) '#{name}', '#{setup}'" )
+
+    BeerDb.read_setup( setup, repo_dir )
+  end
+end # class BeerFileDataset
+
+
+end # module Datafile

data/lib/datafile/workers/file/registry.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+
+module Datafile
+
+class FileDatasetRegistry
+##  store mapping for (local) datasets e.g.
+##   map github handle e.g. openfootball/at-austria to local path
+
+  include LogUtils::Logging
+
+  def initialize
+    ## org rootpaths
+    @roots = {}
+    ## org defaults (use merge to overwrite for now)
+    @roots[:openmundi]    = '../../openmundi'   ## OPENMUNDI_ROOT    = "../../openmundi"
+    @roots[:openfootball] = '..'                ## OPENFOOTBALL_ROOT = ".."
+    @roots[:openbeer]     = '..'
+  end
+
+  def merge( hash )
+    ## todo: add support for merging project mappings too
+    ##   use merge_roots and merge_projects ?? why, why not??
+
+    @roots = @roots.merge( hash )
+  end
+
+  def lookup( name ) lookup_worker( name, false ); end  ## false=>return nil; do NOT fail w/ excep
+  def lookup!(name ) lookup_worker( name, true ); end  ## true=>throw except;
+
+private
+  def lookup_worker( name, fail_on_error )
+    ### fix: use lookup! version for exption and lookup (w/ returning nil) - why, why not??
+
+    ## split name in org/user + project (e.g. openfootball/at-austria)
+    parts = name.split( '/' )
+    ## check/todo: assert parts == 2 -- why, why not??
+    root = @roots[ parts[0].to_sym ]
+    if root.nil?
+      msg = "no mapping found for '#{parts[0]}' in '#{name}'"
+      logger.error( msg )
+      if fail_on_error
+        raise DatasetNotFoundError.new( msg )  ## throw exception FileNotFound / DatasetNotFound ??
+      else
+        return nil
+      end
+    end
+
+    path = "#{root}/#{parts[1]}"
+    ## check if folder/directory exists
+    unless File.exist?( path )
+      msg = "no file found for '#{name}'; expected '#{path}'"
+      logger.error( msg )
+      if fail_on_error
+        raise DatasetNotFoundError.new( msg )  ## throw exception FileNotFound / DatasetNotFound ??
+      else
+        return nil
+      end     
+    end
+    ### check for File.directory?( path ) too - why, why not???
+    path
+  end
+
+end  # class FileDatasetRegistry
+
+end # module Datafile

data/lib/datafile/workers/file/worker.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+module Datafile
+
+class FileWorker   ## check: rename to FileDatafileWorker?? or FileDatafile  -why, why not ??
+
+  include LogUtils::Logging
+
+  def initialize( datafile )
+    @datafile = datafile
+  end
+
+  def download
+    ## note: do NOTHING for now; assume repo already present (unpacked) on local filesystem
+  end
+
+  def read
+    @datafile.datasets.each do |dataset|
+      dataset.file_worker.read
+    end
+  end
+  
+  def calc
+    @datafile.scripts.each do |script|
+      script.call
+    end
+  end
+
+  def dump
+    @datafile.datasets.each do |dataset|
+      dataset.file_worker.dump
+    end
+
+    ## also dump scripts
+    @datafile.scripts.each do |script|
+      script.dump
+    end
+  end
+
+end  # class FileWorker
+
+end # module Datafile

data/lib/datafile/workers/zip/beer.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module Datafile
+
+class BeerZipDataset < ZipDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read
+    logger.info( "read beer-dataset (zip) '#{name}', '#{setup}'" )
+
+    BeerDb.read_setup_from_zip( local_zip_name, setup, local_zip_root )
+  end
+end  # class BeerZipDataset
+
+end # module Datafile

data/lib/datafile/workers/zip/dataset.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+
+module Datafile
+
+
+class ZipDataset < DatasetNode  ### use(rename to) ZipDatasetWorker - why, why not ???
+  ## read dataset from zip(archive)
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def remote_zip_url  # remote zip url
+    ###  note: use http:// for now - lets us use (personal proxy NOT working w/ https) for now
+    ## "https://github.com/#{@name}/archive/master.zip"
+    "http://github.com/#{name}/archive/master.zip"
+  end
+
+  def local_zip_name
+    ### note: replace / in name w/ --I--
+    ##  e.g. flatten the filename, that is, do NOT include any folders
+    name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
+  end
+
+  def local_zip_root
+    "./tmp"
+  end
+
+  def local_zip_path  # local zip path
+    "#{local_zip_root}/#{local_zip_name}.zip"
+  end
+
+
+  def download
+    logger.info( "download dataset '#{name}'" )
+    logger.info( "   from '#{remote_zip_url}'" )
+    logger.info( "   to '#{local_zip_path}'..." )
+
+    download_blob( remote_zip_url, local_zip_path )
+  end
+
+
+  def dump
+    ## for debuggin dump dataset (also check if zip exits)
+    puts "dataset '#{name}' opts=#{opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
+    puts "  local '#{local_zip_name}' (#{local_zip_path})"
+    if File.exist?( local_zip_path )
+      puts "    size: #{File.size(local_zip_path)} bytes"
+    else
+      puts "    (file not found)"
+    end
+    puts "  remote '#{remote_zip_url}'"
+  end
+
+
+private
+  ####
+  # download tasks for zips
+  def download_blob( url, dest )
+    logger.info "downloading #{url} to #{dest}..."
+
+    ## make sure dest path exists
+    dest_p = File.dirname( dest )
+    FileUtils.mkdir_p( dest_p )  unless File.exists?( dest_p )   ## use Dir.exists?? why? why not??
+
+    worker = Fetcher::Worker.new
+    worker.copy( url, dest )
+    ## print some file stats
+    logger.debug "  size: #{File.size(dest)} bytes"
+  end
+
+end  # class DatasetZip
+
+end # module Datafile

data/lib/datafile/workers/zip/football.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module Datafile
+
+class FootballZipDataset < ZipDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read
+    logger.info( "read football-dataset (zip) '#{name}', '#{setup}'" )
+
+    SportDb.read_setup_from_zip( local_zip_name, setup, local_zip_root )
+  end
+end  # class FootballZipDataset
+
+end # module Datafile

data/lib/datafile/workers/zip/worker.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+
+module Datafile
+
+class ZipWorker   ## check: rename to ZipDatafileWorker?? or ZipDatafile  -why, why not ??
+
+  include LogUtils::Logging
+
+  def initialize( datafile )
+    @datafile = datafile
+  end
+
+  def download
+    @datafile.datasets.each do |dataset|
+      dataset.zip_worker.download
+    end
+  end
+
+  def read
+    @datafile.datasets.each do |dataset|
+      dataset.zip_worker.read
+    end
+  end
+
+  def calc
+    @datafile.scripts.each do |script|
+      script.call
+    end
+  end
+
+  def dump
+    @datafile.datasets.each do |dataset|
+      dataset.zip_worker.dump
+    end
+    ## also dump scripts
+    @datafile.scripts.each do |script|
+      script.dump
+    end
+  end
+
+end  # class ZipWorker
+
+end # module Datafile

data/lib/datafile/workers/zip/world.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module Datafile
+
+class WorldZipDataset < ZipDataset
+
+  def initialize( dataset )
+    super( dataset )
+  end
+
+  def read
+    logger.info( "read world-dataset (zip) '#{name}', '#{setup}'" )
+
+    WorldDb.read_setup_from_zip( local_zip_name, setup, local_zip_root, { skip_tags: true } )
+  end
+end  # class WorldZipDataset
+
+end # module Datafile

data/test/datafile/eurocup.rb

@@ -0,0 +1,6 @@
+#####################
+# national teams
+
+football 'openfootball/national-teams'
+football 'openfootball/euro-cup'
+

data/test/datafile/stadiums.rb

@@ -0,0 +1,2 @@
+
+football 'openfootball/stadiums'   ## NOTE: default is setup: 'all'

data/test/datafile/world.rb

@@ -0,0 +1,3 @@
+
+world 'openmundi/world.db', setup: 'countries'
+

data/test/datafile2/at.rb

@@ -0,0 +1,51 @@
+
+puts "[eval] self in top = #{self.class.name}"
+
+task :at => :importbuiltin do
+  puts "[eval] self in data (enter) = #{self.class.name}"
+  football 'openfootball/at-austria'
+  puts "[eval] self in data (leave) = #{self.class.name}"
+end
+
+task :at_2014_15 => :importbuiltin do
+  football 'openfootball/at-austria', setup: '2014-15'
+end
+
+
+task :at_recalc => :at do
+  calc do
+    [['at.2012/13'],
+     ['at.2013/14'],
+     ['at.2014/15', 'at.2.2014/15']].each do |event_key|
+       recalc_standings( event_key, out_root: './build/at-autria' )
+    end
+  end
+end
+
+task :at_2014_15_recalc => :at_2014_15 do
+  calc do
+    recalc_standings( ['at.2014/15', 'at.2.2014/15'], out_root: './build/at-autria' )
+  end
+end
+
+
+task :test_at_recalc => :env  do
+  calc do
+    recalc_standings( ['at.2014/15', 'at.2.2014/15'], out_root: './build/at-autria' )
+    ## debug verison - write to ./build/at-austria 
+    ## recalc_standings( ['at.2014/15', 'at.2.2014/15'], out_root: './build/at-austria' )
+  end
+end
+
+
+## check if method def works too
+
+puts "[eval] another self in top = #{self.class.name}"
+
+
+def test_hello()
+  puts "[eval] self in method test_hello = #{self.class.name}"
+  puts "hello from test_hello"  
+end
+
+test_hello()

data/test/helper.rb

@@ -3,8 +3,9 @@
 require 'minitest/autorun'
 
 ## deps
-require 'worlddb'
-require 'sportdb'   # note: will include worlddb
+
+### require 'worlddb'
+require 'sportdb/models'   # note: will include worlddb
 
 
 ## our own code

data/test/test_builder.rb

@@ -34,6 +34,10 @@ EOS
     ## datafile.read
     
     datafile.dump
+    
+    ## change worker (defaults to ZipWorker)
+    datafile.worker = Datafile::FileWorker.new( datafile )
+    datafile.dump
 
     assert true  # if we get here - test success
   end

data/test/test_builder2.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_builder2.rb
+
+
+require 'helper'
+
+def recalc_standings( *args )
+  puts "[eval] self in method recal_standings = #{self.class.name}"
+  puts "hello from (global) recalc_standings"
+end
+
+
+class TestBuilder2 < MiniTest::Test
+
+  def test_builder
+
+    builder = Datafile::BuilderEx.load_file( "#{Datafile.root}/test/datafile2/at.rb" )
+
+    registry = Datafile::FileDataset.registry
+    registry.merge( openfootball: '../../openfootball' )
+
+    datafiles = builder.datafiles
+    datafiles.each do |datafile|
+      puts "=== datafile '#{datafile.name}' => #{datafile.deps.inspect}:"
+      puts "  #{datafile.datasets.size} datasets, #{datafile.scripts.size} scripts"
+      datafile.dump
+      datafile.calc  ## try calc (call dummy calculations for testing)
+    end
+
+    assert true  # if we get here - test success
+  end
+
+end # class TestBuilder2

data/test/test_file_dataset_registry.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_file_dataset_registry.rb
+
+
+require 'helper'
+
+class TestFileDatasetRegistry < MiniTest::Test
+
+  def test_lookup
+    registry = Datafile::FileDatasetRegistry.new
+    registry.merge( openfootball: '../../openfootball' )
+
+    assert_equal '../../openfootball/at-austria', registry.lookup( 'openfootball/at-austria')
+    assert_equal '../../openmundi/world.db',      registry.lookup( 'openmundi/world.db' )
+
+    assert_equal '../../openfootball/at-austria', registry.lookup!( 'openfootball/at-austria')
+    assert_equal '../../openmundi/world.db',      registry.lookup!( 'openmundi/world.db' )
+
+    assert_equal '../../openfootball/national-teams', registry.lookup( 'openfootball/national-teams')
+    assert_equal '../../openfootball/euro-cup',       registry.lookup( 'openfootball/euro-cup' )
+  end
+
+end # class TestFileDatasetRegistry
+

data/test/test_file_worker.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_file_worker.rb
+
+
+require 'helper'
+
+class TestFileWorker < MiniTest::Test
+
+  def test_eurocup
+
+    world_datafile = Datafile::Datafile.load_file( "#{Datafile.root}/test/datafile/world.rb" )
+    world_datafile.dump
+
+    eurocup_datafile = Datafile::Datafile.load_file( "#{Datafile.root}/test/datafile/eurocup.rb" )
+    eurocup_datafile.dump
+
+    # database setup 'n' config
+    ActiveRecord::Base.establish_connection( adapter:  'sqlite3',
+                                             database: ':memory:' )
+    SportDb.create_all
+    SportDb.read_builtin
+
+    ## change worker (defaults to ZipWorker)
+    world_datafile.worker = Datafile::FileWorker.new( world_datafile )
+    world_datafile.dump
+    world_datafile.read
+
+    registry = Datafile::FileDataset.registry
+    registry.merge( openfootball: '../../openfootball' )
+
+    eurocup_datafile.worker = Datafile::FileWorker.new( eurocup_datafile )
+    eurocup_datafile.dump
+    eurocup_datafile.read
+
+    assert true  # if we get here - test success
+  end
+
+end # class TestFileWorker

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datafile
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-29 00:00:00.000000000 Z
+date: 2015-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
@@ -96,14 +96,28 @@ files:
 - Rakefile
 - lib/datafile.rb
 - lib/datafile/builder.rb
+- lib/datafile/builder2.rb
 - lib/datafile/datafile.rb
-- lib/datafile/datasets/beer.rb
 - lib/datafile/datasets/dataset.rb
-- lib/datafile/datasets/football.rb
-- lib/datafile/datasets/world.rb
 - lib/datafile/version.rb
+- lib/datafile/workers/dataset.rb
+- lib/datafile/workers/file/dataset.rb
+- lib/datafile/workers/file/registry.rb
+- lib/datafile/workers/file/worker.rb
+- lib/datafile/workers/zip/beer.rb
+- lib/datafile/workers/zip/dataset.rb
+- lib/datafile/workers/zip/football.rb
+- lib/datafile/workers/zip/worker.rb
+- lib/datafile/workers/zip/world.rb
+- test/datafile/eurocup.rb
+- test/datafile/stadiums.rb
+- test/datafile/world.rb
+- test/datafile2/at.rb
 - test/helper.rb
 - test/test_builder.rb
+- test/test_builder2.rb
+- test/test_file_dataset_registry.rb
+- test/test_file_worker.rb
 homepage: https://github.com/rubylibs/datafile
 licenses:
 - Public Domain
@@ -131,4 +145,7 @@ signing_key:
 specification_version: 4
 summary: datafile - builder for downloading n reading datasets
 test_files:
+- test/test_builder2.rb
 - test/test_builder.rb
+- test/test_file_dataset_registry.rb
+- test/test_file_worker.rb

data/lib/datafile/datasets/beer.rb

@@ -1,18 +0,0 @@
-# encoding: utf-8
-
-module Datafile
-
-class BeerDataset < Dataset
-
-  def initialize( name, opts={} )
-    super( name, opts )
-  end
-
-  def read()
-    logger.info( "read beer-dataset '#{@name}', '#{setup}'" )
-
-    BeerDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root() )
-  end
-end  # class BeerDataset
-
-end # module Datafile

data/lib/datafile/datasets/football.rb

@@ -1,18 +0,0 @@
-# encoding: utf-8
-
-module Datafile
-
-class FootballDataset < Dataset
-
-  def initialize( name, opts={} )
-    super( name, opts )
-  end
-
-  def read()
-    logger.info( "read football-dataset '#{@name}', '#{setup}'" )
-
-    SportDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root() )
-  end
-end  # class FootballDataset
-
-end # module Datafile

data/lib/datafile/datasets/world.rb

@@ -1,19 +0,0 @@
-# encoding: utf-8
-
-module Datafile
-
-class WorldDataset < Dataset
-
-  def initialize( name, opts={} )
-    super( name, opts )
-  end
-
-  def read()
-    logger.info( "read world-dataset '#{@name}', '#{setup}'" )
-
-    WorldDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root(), { skip_tags: true } )
-  end
-end  # class WorldDataset
-
-end # module Datafile
-