sportdb 1.9.10 → 1.9.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed1aa660c6f5bcc10d6197bbf48494c6fe6e0e3f
-  data.tar.gz: 6ab3685c798fff2e82e94af70b5d8c9e048d8db2
+  metadata.gz: 68f64d57578e9dffc8a1f676e0d7771fb486e7ef
+  data.tar.gz: 02d84695cdcb3b1fea7269db822292f9003e7305
 SHA512:
-  metadata.gz: a2f5811eb0d21e5f3ec58df3c9c087998017fc716a41b4105c96a31164fed79018e922367a14163dac30ced4151ba6bb93ed006c5bb8e270f5fd41458fd93043
-  data.tar.gz: dd6220cddf7565946b24b2e471efba17e93c29f47366abed585bd3bea4d3556190d557ebcbf00bdc64fe6597e98e5e3e53eee144bbb6772ea8f6809dfc94167c
+  metadata.gz: 1c1da39819f4b5921ea03598f0bce3b129e0219fdd2016ebad21626f9667ed50b0655302f8c251eda3a0c30b1c3dfbce7d113735f7db0bf9b0ae463c47a17ede
+  data.tar.gz: d3eedae5177bb1708a60bc03267f8672e5c290707d98ceb1338b509f144d2a4a1ba51c626ae2f72b536a9eb891f95aa953123599cfb508a29585891835f38645

data/Manifest.txt

@@ -13,6 +13,8 @@ config/fixtures/ro.yml
 data/seasons.txt
 data/setups/all.txt
 lib/sportdb.rb
+lib/sportdb/builder/builder.rb
+lib/sportdb/builder/dataset.rb
 lib/sportdb/calc.rb
 lib/sportdb/cli/main.rb
 lib/sportdb/cli/opts.rb
@@ -125,6 +127,7 @@ test/data/world-cup/teams_1962.txt
 test/data/world-cup/teams_1974.txt
 test/helper.rb
 test/test_assoc_reader.rb
+test/test_builder.rb
 test/test_changes.rb
 test/test_cursor.rb
 test/test_date.rb

data/lib/sportdb.rb

@@ -8,6 +8,9 @@
 # core and stlibs  (note: get included via worlddb gem; see worlddb gem/lib)
 
 
+require 'fileutils'
+
+
 # rubygems  / 3rd party libs
 
 require 'active_record'
@@ -110,6 +113,10 @@ require 'sportdb/deleter'
 require 'sportdb/stats'
 
 
+require 'sportdb/builder/dataset'
+require 'sportdb/builder/builder'
+
+
 module SportDb
 
   def self.config_path
@@ -152,6 +159,12 @@ module SportDb
     reader.load_setup( setup )
   end
 
+  def self.read_setup_from_zip( zip_name, setup, include_path, opts={} ) ## todo/check - use a better (shorter) name ??
+    reader = ZipReader.new( zip_name, include_path, opts )
+    reader.load_setup( setup )
+    reader.close
+  end
+
   def self.read_all( include_path )   # convenience helper
     read_setup( 'setups/all', include_path )
   end

data/lib/sportdb/builder/builder.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+
+module SportDb
+
+class Builder
+
+  include LogUtils::Logging
+
+  def initialize()
+    @datasets = []
+  end
+
+  def self.load_file( path )
+    code = File.read_utf8( path )
+    self.load( code )
+  end
+
+  def self.load( code )
+    builder = Builder.new
+    builder.instance_eval( code )
+    builder
+  end
+
+
+  def football( name, opts={} )
+    logger.info( "[builder] add football-dataset '#{@name}'" )
+    @datasets << FootballDataset.new( name, opts )
+  end
+
+  def world( name, opts={} )
+    logger.info( "[builder] add world-dataset '#{@name}'" )
+    @datasets << WorldDataset.new( name, opts )
+  end
+
+
+  def run()
+    logger.info( "[builder] begin - run" )
+    download()   # step 1 - download zips for datasets
+    read()       # step 2 - read in datasets from zips
+    logger.info( "[builder] end - run" )
+  end
+
+  def download()
+    logger.info( "[builder] dowload" )
+    @datasets.each do |dataset|
+      dataset.download()
+    end
+  end
+
+  def read()
+    logger.info( "[builder] read" )
+    @datasets.each do |dataset|
+      dataset.read()
+    end
+  end
+
+end # class Builder
+end # module SportDb

data/lib/sportdb/builder/dataset.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+
+module SportDb
+
+class Dataset
+  include LogUtils::Logging
+
+  def initialize( name, opts={} )
+    @name = name
+    @opts = opts
+  end
+
+  def setup()
+    value = @opts[:setup] || 'all'
+    "setups/#{value}"
+  end
+
+  def remote_zip_url()  # remote zip url
+    "https://github.com/#{@name}/archive/master.zip"
+  end
+
+  def local_zip_name()
+    ### note: replace / in name w/ --I--
+    ##  e.g. flatten the filename, that is, do NOT include any folders
+    @name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
+  end
+
+  def local_zip_root()
+    "./tmp"
+  end
+
+  def local_zip_path()  # local zip path
+    "#{local_zip_root}/#{local_zip_name}.zip"
+  end
+
+
+  def download()
+    logger.info( "[builder] download dataset '#{@name}'" )
+    logger.info( "[builder]     from '#{remote_zip_url}'" )
+    logger.info( "[builder]     to '#{local_zip_path}'..." )
+    
+    download_blob( remote_zip_url, local_zip_path )
+  end
+
+  ####
+  # download tasks for zips
+  def download_blob( url, dest )
+    logger.info "downloading #{url} to #{dest}..."
+
+    ## make sure dest path exists
+    dest_p = File.dirname( dest )
+    FileUtils.mkdir_p( dest_p )  unless File.exists?( dest_p )   ## use Dir.exists?? why? why not??
+
+    worker = Fetcher::Worker.new
+    worker.copy( url, dest )
+    ## print some file stats
+    logger.debug "  size: #{File.size(dest)} bytes"
+  end
+
+end  # class Dataset
+
+
+class WorldDataset < Dataset
+
+  def initialize( name, opts={} )
+    super( name, opts )
+  end
+
+  def read()
+    logger.info( "[builder] read world-dataset '#{@name}', '#{setup}'" )
+
+    WorldDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root(), { skip_tags: true } )
+  end
+end  # class WorldDataset
+
+class FootballDataset < Dataset
+
+  def initialize( name, opts={} )
+    super( name, opts )
+  end
+
+  def read()
+    logger.info( "[builder] read football-dataset '#{@name}', '#{setup}'" )
+
+    SportDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root() )
+  end
+end  # class FootballDataset
+
+
+end # module SportDb

data/lib/sportdb/cli/main.rb

@@ -108,6 +108,34 @@ command [:create] do |c|
 end # command create
 
 
+desc "Build DB (download/create/load); use ./Datafile - zips get downloaded to ./tmp"
+command [:build,:b] do |c|
+
+  c.action do |g,o,args|
+
+    builder = SportDb::Builder.load_file( './Datafile' )
+    builder.download  # builder step 1 - download all datasets/zips 
+
+    connect_to_db( opts )
+
+    LogDb.create
+    ConfDb.create
+    TagDb.create
+    WorldDb.create
+    PersonDb.create
+    SportDb.create
+
+    SportDb.read_builtin   # e.g. seasons.txt etc
+
+    builder.read  # builder step 2 - read all datasets
+  
+    puts 'Done.'
+  end # action
+end  # command setup
+
+
+
+
 desc "Create DB schema 'n' load all world and sports data"
 arg_name 'NAME'   # optional setup profile name
 command [:setup,:s] do |c|

data/lib/sportdb/reader_zip.rb

@@ -5,7 +5,160 @@ module SportDb
 
 class ZipReader < ReaderBase
 
+  def initialize( name, include_path, opts = {} )
 
+    ## todo/fix: make include_path an opts (included in opts?) - why? why not??
+
+    path = "#{include_path}/#{name}.zip"
+    ## todo: check if zip exists
+
+    @zip_file = Zip::File.open( path )   ## NOTE: do NOT create if file is missing; let it crash
+    
+    ### allow prefix (path) in name
+    ###    e.g. assume all files relative to setup manifest
+    ## e.g. at-austria-master/setups/all.txt or
+    ##      be-belgium-master/setups/all.txt
+    ##  for
+    ##    setups/all.txt
+    ###
+    ##  will get (re)set w/ fixture/setup reader
+    ##
+    ## todo/fix: change/rename to @relative_path ?? - why? why not? 
+    @zip_prefix = ''
+  end
+
+  def close
+    ## todo/check: add a close method - why? why not ???
+    @zip_file.close
+  end
+
+
+  def create_fixture_reader( name )
+    ## e.g. pass in =>  setups/all  or setups/test etc.  e.g. w/o .txt extension
+    query = "**/#{name}.txt"
+
+    ## note: returns an array of Zip::Entry
+    candidates = @zip_file.glob( query )
+    pp candidates
+
+    ## use first candidates entry as match
+    ## todo/fix: issue warning if more than one entries/matches!!
+
+    ## get fullpath e.g. at-austria-master/setups/all.txt
+    path = candidates[0].name
+    logger.debug "  zip entry path >>#{path}<<"
+
+    ## cut-off at-austria-master/    NOTE: includes trailing slash (if present)
+    ## logger.debug "  path.size #{path.size} >>#{path}<<"
+    ## logger.debug "  name.size #{name.size+4} >>#{name}<<"
+
+    ## note: add +4 for extension (.txt)
+    @zip_prefix = path[ 0...(path.size-(name.size+4)) ]
+    logger.debug "  zip entry prefix >>#{@zip_prefix}<<"
+
+    logger.info "parsing data in zip '#{name}' (#{path})..."
+
+    FixtureReader.from_zip( @zip_file, path )
+  end
+
+  def create_club_squad_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (club squad) '#{name}' (#{path})..."
+    ClubSquadReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_national_team_squad_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (national team squad) '#{name}' (#{path})..."
+    NationalTeamSquadReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_season_reader( name )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (season) '#{name}' (#{path})..."
+    SeasonReader.from_zip( @zip_file, path )
+  end
+
+  def create_assoc_reader( name )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (assoc) '#{name}' (#{path})..."
+    AssocReader.from_zip( @zip_file, path )
+  end
+
+  def create_ground_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (ground) '#{name}' (#{path})..."
+    GroundReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_league_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (league) '#{name}' (#{path})..."
+    LeagueReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_team_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+
+    logger.info "parsing data in zip (team) '#{name}' (#{path})..."
+    TeamReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_event_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name, '.yml' )   ## NOTE: use .yml extension
+
+    logger.info "parsing data in zip (event) '#{name}' (#{path})..."
+    EventReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_game_reader( name, more_attribs={} )
+    ## NOTE: pass in .yml as path (that is, event config!!!!)
+    path = name_to_zip_entry_path( name, '.yml' )     ## NOTE: use .yml extension
+    
+    logger.info "parsing data in zip (fixture) '#{name}' (#{path})..."
+    GameReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+
+  def create_person_reader( name, more_attribs={} )
+    ## fix-fix-fix: change to new format e.g. from_file, from_zip etc!!!
+    ## reader = PersonDb::PersonReader.new( include_path )
+    # reader.read( name, country_id: country.id )
+  end
+
+private
+
+  def path_to_real_path( path )
+    # map name to name_real_path
+    # name might include !/ for virtual path (gets cut off)
+    # e.g. at-austria!/w-wien/beers becomse w-wien/beers
+    pos = path.index( '!/')
+    if pos.nil?
+      path # not found; real path is the same as name
+    else
+      # cut off everything until !/ e.g.
+      # at-austria!/w-wien/beers becomes
+      # w-wien/beers
+      path[ (pos+2)..-1 ]
+    end
+  end
+
+  def name_to_zip_entry_path( name, extension='.txt' )
+    path = "#{name}#{extension}"
+
+    real_path = path_to_real_path( path )
+
+    # NOTE: add possible zip entry prefix path
+    #          (if present includes trailing slash e.g. /)
+    entry_path = "#{@zip_prefix}#{real_path}"
+    entry_path
+  end
 
 end # class ZipReader
 

data/lib/sportdb/readers/assoc.rb

@@ -11,8 +11,15 @@ class AssocReader
 #  e.g. lets you use Usage instead of Model::Usage
   include Models
 
+
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/event.rb

@@ -15,7 +15,15 @@ class EventReader
   attr_reader :fixtures        #  fixtures/sources entry from event config
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    config = File.basename( entry_path )  # name a of .yml file
+
+    self.from_string( text, config, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/game.rb

@@ -16,9 +16,46 @@ class GameReader
 
   include FixtureHelpers
 
-
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+
+    logger = LogKernel::Logger.root
+
+    reader = EventReader.from_zip( zip_file, entry_path )
+    reader.read()
+
+    event    = reader.event      ## was fetch_event( name )
+    fixtures = reader.fixtures   ## was fetch_event_fixtures( name )
+
+    if fixtures.empty?
+      ## logger.warn "no fixtures found for event - >#{name}<; assume fixture name is the same as event"
+      ## change extension from .yml to .txt
+      fixtures_with_path = [ entry_path.sub('.yml','.txt') ]
+    else
+      ## add path to fixtures (use path from event e.g)
+      #  - bl    + at-austria!/2012_13/bl  -> at-austria!/2012_13/bl
+      #  - bl_ii + at-austria!/2012_13/bl  -> at-austria!/2012_13/bl_ii
+
+      dir = File.dirname( entry_path ) # use dir for fixtures
+
+      fixtures_with_path = fixtures.map do |fx|
+        fx_new = "#{dir}/#{fx}.txt"   # add path upfront
+        logger.debug "fx: #{fx_new} | >#{fx}< + >#{dir}<"
+        fx_new
+      end
+    end
+
+    ## fix-fix-fix: change file extension to ??
+    text_ary = []
+    fixtures_with_path.each do |fixture_path|
+      entry = zip_file.find_entry( fixture_path )
+
+      text = entry.get_input_stream().read()
+      text = text.force_encoding( Encoding::UTF_8 )
+
+      text_ary << text
+    end
+
+    self.from_string( event, text_ary, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/ground.rb

@@ -12,7 +12,13 @@ class GroundReader
   include Models
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/league.rb

@@ -13,7 +13,13 @@ class LeagueReader
 
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/season.rb

@@ -13,7 +13,13 @@ class SeasonReader
 
 
   def self.from_zip( zip_file, entry_path )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text )
   end
 
   def self.from_file( path )

data/lib/sportdb/readers/squad_club.rb

@@ -23,7 +23,13 @@ class ClubSquadReader
 
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/squad_national_team.rb

@@ -20,7 +20,13 @@ class NationalTeamSquadReader
 
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/readers/team.rb

@@ -12,7 +12,13 @@ class TeamReader
   include Models
 
   def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## to be done
+    ## get text content from zip
+    entry = zip_file.find_entry( entry_path )
+
+    text = entry.get_input_stream().read()
+    text = text.force_encoding( Encoding::UTF_8 )
+
+    self.from_string( text, more_attribs )
   end
 
   def self.from_file( path, more_attribs={} )

data/lib/sportdb/version.rb

@@ -3,7 +3,7 @@ module SportDb
 
   MAJOR = 1    ## todo: namespace inside version or something - why? why not??
   MINOR = 9
-  PATCH = 10
+  PATCH = 11
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/test_builder.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_builder.rb
+
+require 'helper'
+
+class TestBuilder < MiniTest::Test
+
+  def test_builder
+    code =<<EOS
+## comments
+
+  world  'openmundi/world.db', setup: 'countries'
+
+  football 'openfootball/national-teams'      ## NOTE: default is setup: 'all'
+  
+  ### todo/fix: download archive only once(!!) even if included more than once
+  ## football 'openfootball/world-cup', setup: '2014_quali'
+
+  football 'openfootball/world-cup', setup: '2014'
+
+## more comments
+
+EOS
+
+    builder = SportDb::Builder.load( code )
+    ## builder.run
+    ## builder.download
+    builder.read
+
+    assert true  # if we get here - test success
+  end
+
+end # class TestBuilder

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sportdb
 version: !ruby/object:Gem::Version
-  version: 1.9.10
+  version: 1.9.11
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-22 00:00:00.000000000 Z
+date: 2014-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: props
@@ -260,6 +260,8 @@ files:
 - data/seasons.txt
 - data/setups/all.txt
 - lib/sportdb.rb
+- lib/sportdb/builder/builder.rb
+- lib/sportdb/builder/dataset.rb
 - lib/sportdb/calc.rb
 - lib/sportdb/cli/main.rb
 - lib/sportdb/cli/opts.rb
@@ -372,6 +374,7 @@ files:
 - test/data/world-cup/teams_1974.txt
 - test/helper.rb
 - test/test_assoc_reader.rb
+- test/test_builder.rb
 - test/test_changes.rb
 - test/test_cursor.rb
 - test/test_date.rb
@@ -435,6 +438,7 @@ test_files:
 - test/test_round_def.rb
 - test/test_utils.rb
 - test/test_reader_from_string.rb
+- test/test_builder.rb
 - test/test_round_header.rb
 - test/test_round_auto.rb
 - test/test_goals.rb