datafile 0.2.5 → 0.3.0

data/lib/datafile/{datasets/dataset.rb → dataset.rb}

@@ -6,7 +6,7 @@
 
 def read_known_datasets( path )
     ary = []
-    lines = File.read( path )  ### fix: use File.read_utf8 ??
+    lines = File.open( path, 'r:utf-8' ).read
     lines.each_line do |line|
       ## skip blank and comments lines
       next   if /^\s*#/ =~ line || /^\s*$/ =~ line
@@ -27,13 +27,12 @@ class Dataset
     @opts = opts
   end
 
-  attr_reader :name
-  attr_reader :opts
+  attr_reader :name, :opts
+
+  ## convenience helpers for known opts
+  def setup()   @opts[:setup];           end    ## note: return nil if not found/set
+  def format()  @opts[:format] || 'txt'; end    ## note: assume default is txt (other formats incl. csv) for now - why? wh not?
 
-  def setup
-    value = @opts[:setup] || 'all'
-    "setups/#{value}"
-  end
 
   def file?                    # note: use file? (not exit? might use zip? later to check if zip exists? -why? why not?)
     ## hack/convenience shortcut:
@@ -49,7 +48,7 @@ class Dataset
     basename = parts[1]
     ##  e.g.
     ##  ./ (working folder) => at-austria
-    ##  openfootball/at-austria  
+    ##  openfootball/at-austria
     if File.basename( Dir.getwd ) == basename
       puts "  bingo!! working folder >#{basename}< matches dataset"
       true     ## return true
@@ -75,23 +74,18 @@ class WorldDataset < Dataset
 
     super( name, opts )  ## todo/check: just juse super (e.g. pass along all params - why? why not?)
   end
-  
-  def zip_worker()   WorldZipDataset.new( self );  end   ## check: change (rename) just use zip or use worker_zip?? - why, why not?
-  def file_worker()  WorldFileDataset.new( self ); end
 end  # class WorldDataset
 
 
 
 class FootballDataset < Dataset
 
-  @@known_football_datasets = nil
+  def self.build_known_datasets
+    read_known_datasets( "#{::Datafile.data_path}/football.txt" )
+  end
 
   def self.known_datasets
-    ## return array of known datasets
-    ### todo/fix - use \\= idiom - why, why not??
-    if @@known_football_datasets.nil?
-      @@known_football_datasets = read_known_datasets( "#{::Datafile.data_path}/football.txt" )
-    end
+    @@known_football_datasets ||= build_known_datasets
     @@known_football_datasets
   end
 
@@ -101,7 +95,11 @@ class FootballDataset < Dataset
     ## check if name include slash (e.g. /)
     ##  - if not auto-add openfootball/ (default)
     if name_easy.index( '/' ).nil?
-      name = "openfootball/#{name_easy}"
+      if opts[:format] == 'csv'
+        name = "footballcsv/#{name_easy}"
+      else
+        name = "openfootball/#{name_easy}"
+      end
     else
       name = name_easy   ## just pass through for now
     end
@@ -109,14 +107,11 @@ class FootballDataset < Dataset
     super( name, opts )
 
     ### check for known datasets; warn: if not known (might be a typo)
-    unless FootballDataset.known_datasets.include?( name )
+    unless self.class.known_datasets.include?( name )
       ## todo: use logger - why, why not??
       puts "*** warn: unknown football dataset '#{name}', typo ???"
     end
   end
-
-  def zip_worker()   FootballZipDataset.new( self );  end
-  def file_worker()  FootballFileDataset.new( self ); end
 end # class FootballDataset
 
 
@@ -133,9 +128,6 @@ class BeerDataset < Dataset
 
     super( name, opts )
   end
-
-  def zip_worker()   BeerZipDataset.new( self );  end
-  def file_worker()  BeerFileDataset.new( self ); end
 end  # class BeerDataset
 
 

data/lib/datafile/version.rb

@@ -3,8 +3,8 @@
 module Datafile
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 2
-  PATCH = 5
+  MINOR = 3
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version
@@ -17,7 +17,10 @@ module Datafile
 
   def self.root
     "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
-  end  
+  end
 
-end  # module Datafile
+  def self.data_path
+      "#{root}/data"
+  end
 
+end  # module Datafile

data/lib/datafile/workers/file/dataset.rb

@@ -2,28 +2,26 @@
 
 module Datafile
 
-class FileDataset < DatasetNode
+class FileDataset
   ## read dataset from file(system)
 
-  @@registry = nil
+  include LogUtils::Logging
+
 
   def self.registry
-    ## use ||=  why, why not??  - add Registry as nested class, why, why not ??
-    if @@registry.nil?
-      @@registry = FileDatasetRegistry.new
-    end
+    @@registry ||= FileDatasetRegistry.new
     @@registry
   end
 
   def initialize( dataset )
-    super( dataset )
+    @dataset = dataset
   end
 
-  def repo_dir     ### check: use (rename to) include dir (or local_repo_dir) - why, why not ??? 
+  def repo_dir     ### check: use (rename to) include dir (or local_repo_dir) - why, why not ???
     ## note: for easy testing allow "in situ" datasets
     ##   e.g.  ./ (e.g. mu-mauritius)  is openfootball/mu-mauritius
     ## split name in org/user + project (e.g. openfootball/at-austria)
-    parts = name.split( '/' )
+    parts = @dataset.name.split( '/' )
 
     basename = parts[1]
     if File.basename( Dir.getwd ) == basename
@@ -31,63 +29,28 @@ class FileDataset < DatasetNode
       return Dir.getwd     ## assume working directory/folder is repo dir
     end
 
-    registry.lookup( name )
+    registry.lookup( @dataset.name )
   end
 
   def dump
     ## for debuggin dump dataset  -- todo (also check if folder exits ??)
-    puts "dataset '#{name}' opts=#{opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
+    puts "dataset '#{@dataset.name}' opts=#{@dataset.opts.inspect}"     ## use opts.inspect instead of to_json - why? why not?
     puts "  repo-dir '#{repo_dir}'"
   end
 
-private
-  def registry    ## convenience method to access "static" shared class variable
-    FileDataset.registry     ## self.registry  not working?? - or self.registry() -why, why not??
-  end
-end # class FileDataset
-
-
-
-class FootballFileDataset < FileDataset
-
-  def initialize( dataset )
-    super( dataset )
-  end
-
-  def read()
-    logger.info( "read football-dataset (file) '#{name}', '#{setup}'" )
-
-    SportDb.read_setup( setup, repo_dir )
-  end
-end # class FootballFileDataset
-
+  def read
+    if @dataset.is_a?( FootballDataset )
+      logger.info( "read football dataset (file) '#{@dataset.name}', '#{@dataset.setup}'" )
 
-class WorldFileDataset < FileDataset
-
-  def initialize( dataset )
-    super( dataset )
-  end
-
-  def read()
-    logger.info( "read world-dataset (file) '#{name}', '#{setup}'" )
-
-    ## WorldDb.read_setup( 'setups/countries', WORLD_DB_INCLUDE_PATH, skip_tags: true )
-    WorldDb.read_setup( setup, repo_dir, skip_tags: true  )
-  end
-end # class WorldFileDataset
-
-class BeerFileDataset < FileDataset
-
-  def initialize( dataset )
-    super( dataset )
-  end
-
-  def read()
-    logger.info( "read beer-dataset (file) '#{name}', '#{setup}'" )
-
-    BeerDb.read_setup( setup, repo_dir )
+      pack = SportDb::DirPackage.new( repo_dir )
+      pack.read( season: @dataset.setup )   ##  note: pass on (optional) setup arg as season (filter) arg for now
+    else
+      logger.info( "TODO/FIX: read dataset (file) '#{@dataset.name}', '#{@dataset.setup}'; sorry" )
+    end
   end
-end # class BeerFileDataset
 
 
+private
+  def registry()  self.class.registry;  end    ## convenience method to access "static" shared class variable
+end # class FileDataset
 end # module Datafile

data/lib/datafile/workers/file/registry.rb

@@ -15,22 +15,30 @@ class FileDatasetRegistry
     @roots[:openmundi]    = '../../openmundi'   ## OPENMUNDI_ROOT    = "../../openmundi"
     @roots[:openfootball] = '..'                ## OPENFOOTBALL_ROOT = ".."
     @roots[:openbeer]     = '..'
+
+    @roots[:footballcsv]  = '..'
   end
 
   def merge( hash )
     ## todo: add support for merging project mappings too
     ##   use merge_roots and merge_projects ?? why, why not??
-
     @roots = @roots.merge( hash )
   end
 
-  def lookup( name ) lookup_worker( name, false ); end  ## false=>return nil; do NOT fail w/ excep
-  def lookup!(name ) lookup_worker( name, true ); end  ## true=>throw except;
+  def lookup( name )
+    path, _ = lookup_path( name )    ## note: ignore error message passed along in return
+    path
+  end
+
+  def lookup!( name )
+    path, error = lookup_path( name )
+    raise error  if error
+    path
+  end
 
-private
-  def lookup_worker( name, fail_on_error )
-    ### fix: use lookup! version for exption and lookup (w/ returning nil) - why, why not??
 
+private
+  def lookup_path( name )
     ## split name in org/user + project (e.g. openfootball/at-austria)
     parts = name.split( '/' )
     ## check/todo: assert parts == 2 -- why, why not??
@@ -38,11 +46,7 @@ private
     if root.nil?
       msg = "no mapping found for '#{parts[0]}' in '#{name}'"
       logger.error( msg )
-      if fail_on_error
-        raise DatasetNotFoundError.new( msg )  ## throw exception FileNotFound / DatasetNotFound ??
-      else
-        return nil
-      end
+      return [nil, DatasetNotFoundError.new( msg )]   ## throw exception FileNotFound / DatasetNotFound ??
     end
 
     path = "#{root}/#{parts[1]}"
@@ -50,16 +54,12 @@ private
     unless File.exist?( path )
       msg = "no file found for '#{name}'; expected '#{path}'"
       logger.error( msg )
-      if fail_on_error
-        raise DatasetNotFoundError.new( msg )  ## throw exception FileNotFound / DatasetNotFound ??
-      else
-        return nil
-      end     
+      return [nil, DatasetNotFoundError.new( msg )]   ## throw exception FileNotFound / DatasetNotFound ??
     end
+
     ### check for File.directory?( path ) too - why, why not???
-    path
+    [path, nil]    ## use go-style returns with error as second argument (as error as value)
   end
-
 end  # class FileDatasetRegistry
 
 end # module Datafile

data/lib/datafile/workers/file/worker.rb

@@ -15,38 +15,18 @@ class FileWorker   ## check: rename to FileDatafileWorker?? or FileDatafile  -wh
   end
 
   def read
-    ## note: also run inlines (setup script) before
-    @datafile.inlines.each do |inline|
-      inline.call
-    end
-
     @datafile.datasets.each do |dataset|
-      dataset.file_worker.read
-    end
-  end
-  
-  def calc
-    @datafile.scripts.each do |script|
-      script.call
+      f = FileDataset.new( dataset )
+      f.read
     end
   end
 
   def dump
-    ## also dump inlines
-    @datafile.inlines.each do |inline|
-      inline.dump
-    end
-
     @datafile.datasets.each do |dataset|
-      dataset.file_worker.dump
-    end
-
-    ## also dump scripts
-    @datafile.scripts.each do |script|
-      script.dump
+      f = FileDataset.new( dataset )
+      f.dump
     end
   end
 
 end  # class FileWorker
-
 end # module Datafile

data/lib/datafile/workers/zip/dataset.rb

@@ -3,23 +3,23 @@
 module Datafile
 
 
-class ZipDataset < DatasetNode  ### use(rename to) ZipDatasetWorker - why, why not ???
+class ZipDataset   ### use (rename to) ZipDatasetWorker/Helper/Wrapper/Fetcher/Downloader - why, why not ???
   ## read dataset from zip(archive)
 
+  include LogUtils::Logging
+
   def initialize( dataset )
-    super( dataset )
+    @dataset = dataset
   end
 
   def remote_zip_url  # remote zip url
-    ###  note: use http:// for now - lets us use (personal proxy NOT working w/ https) for now
-    ## "https://github.com/#{@name}/archive/master.zip"
-    "http://github.com/#{name}/archive/master.zip"
+    "https://github.com/#{@dataset.name}/archive/master.zip"
   end
 
   def local_zip_name
     ### note: replace / in name w/ --I--
     ##  e.g. flatten the filename, that is, do NOT include any folders
-    name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
+    @dataset.name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
   end
 
   def local_zip_root
@@ -32,7 +32,7 @@ class ZipDataset < DatasetNode  ### use(rename to) ZipDatasetWorker - why, why n
 
 
   def download
-    logger.info( "download dataset '#{name}'" )
+    logger.info( "download dataset '#{@dataset.name}'" )
     logger.info( "   from '#{remote_zip_url}'" )
     logger.info( "   to '#{local_zip_path}'..." )
 
@@ -42,7 +42,7 @@ class ZipDataset < DatasetNode  ### use(rename to) ZipDatasetWorker - why, why n
 
   def dump
     ## for debuggin dump dataset (also check if zip exits)
-    puts "dataset '#{name}' opts=#{opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
+    puts "dataset '#{@dataset.name}' opts=#{@dataset.opts.to_json}"     ## use opts.inspect instead of to_json - why? why not?
     puts "  local '#{local_zip_name}' (#{local_zip_path})"
     if File.exist?( local_zip_path )
       puts "    size: #{File.size(local_zip_path)} bytes"
@@ -52,6 +52,17 @@ class ZipDataset < DatasetNode  ### use(rename to) ZipDatasetWorker - why, why n
     puts "  remote '#{remote_zip_url}'"
   end
 
+  def read
+    if @dataset.is_a?( FootballDataset )
+      logger.info( "read football dataset (zip) '#{@dataset.name}', '#{@dataset.setup}'" )
+
+      pack = SportDb::ZipPackage.new( local_zip_path )
+      pack.read( season: @dataset.setup )   ##  note: pass on (optional) setup arg as season (filter) arg for now
+    else
+      logger.info( "TODO/FIX: read dataset (zip) '#{@dataset.name}', '#{@dataset.setup}'; sorry" )
+    end
+  end
+
 
 private
   ####

data/lib/datafile/workers/zip/worker.rb

@@ -12,38 +12,22 @@ class ZipWorker   ## check: rename to ZipDatafileWorker?? or ZipDatafile  -why,
 
   def download
     @datafile.datasets.each do |dataset|
-      dataset.zip_worker.download
+      z = ZipDataset.new( dataset )
+      z.download
     end
   end
 
   def read
-    ## note: also run inlines (setup script) before
-    @datafile.inlines.each do |inline|
-      inline.call
-    end
-
     @datafile.datasets.each do |dataset|
-      dataset.zip_worker.read
-    end
-  end
-
-  def calc
-    @datafile.scripts.each do |script|
-      script.call
+      z = ZipDataset.new( dataset )
+      z.read
     end
   end
 
   def dump
-    ## also dump inlines
-    @datafile.inlines.each do |inline|
-      inline.dump
-    end
     @datafile.datasets.each do |dataset|
-      dataset.zip_worker.dump
-    end
-    ## also dump scripts
-    @datafile.scripts.each do |script|
-      script.dump
+      z = ZipDataset.new( dataset )
+      z.dump
     end
   end