sportdb-readers 0.5.0 → 1.1.0

data/lib/sportdb/readers/match_reader.rb

@@ -2,120 +2,151 @@
 
 module SportDb
 
-class MatchReaderV2    ## todo/check: rename to MatchReaderV2 (use plural?) why? why not?
-
-  def self.config()  Import.config; end
-
-
+class MatchReader    ## todo/check: rename to MatchReaderV2 (use plural?) why? why not?
 
   def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
     parse( txt, season: season )
   end
 
   def self.parse( txt, season: nil )
-    recs = LeagueOutlineReader.parse( txt, season: season )
-    pp recs
+    new( txt ).parse( season: season )
+  end
+
 
-    recs.each do |rec|
-      league = Sync::League.find_or_create( rec[:league] )
-      season = Sync::Season.find_or_create( rec[:season] )
+  include Logging
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+  def parse( season: nil )
+    secs = LeagueOutlineReader.parse( @txt, season: season )
+    pp secs
+
+
+    ###
+    ## todo/check/fix:  move to LeagueOutlineReader for (re)use - why? why not?
+    ##                    use sec[:lang] or something?
+    langs = {    ## map country keys to lang codes
+      'de' => 'de', ## de - Deutsch (German)
+      'at' => 'de',
+      'fr' => 'fr', ## fr - French
+      'it' => 'it', ## it - Italian
+      'es' => 'es', ## es - Español (Spanish)
+      'mx' => 'es',
+      'pt' => 'pt', ## pt -  Português (Portuguese)
+      'br' => 'br'
+    }
+
+    secs.each do |sec|   ## sec(tion)s
+      season = sec[:season]
+      league = sec[:league]
+      stage  = sec[:stage]
+      lines  = sec[:lines]
 
       ## hack for now: switch lang
-      if ['de', 'at'].include?( league.country.key )
-        SportDb.lang.lang = 'de'
-        DateFormats.lang  = 'de'
-      elsif ['fr'].include?( league.country.key )
-        SportDb.lang.lang = 'fr'
-        DateFormats.lang  = 'fr'
-      elsif ['it'].include?( league.country.key )
-        SportDb.lang.lang = 'it'
-        DateFormats.lang  = 'it'
-      elsif ['es', 'mx'].include?( league.country.key )
-        SportDb.lang.lang = 'es'
-        DateFormats.lang  = 'es'
-      elsif ['pt', 'br'].include?( league.country.key )
-        SportDb.lang.lang = 'pt'
-        DateFormats.lang  = 'pt'
-      else
-        SportDb.lang.lang = 'en'
-        DateFormats.lang  = 'en'
+      ## todo/fix: set lang for now depending on league country!!!
+      if league.intl?   ## todo/fix: add intl? to ActiveRecord league!!!
+        Import.config.lang = 'en'
+      else  ## assume national/domestic
+        Import.config.lang = langs[ league.country.key ] || 'en'
       end
 
 
-      ## todo/fix:
-      ##    always auto create
-      ##   1) check for clubs count on event/stage - only if count == 0 use autoconf!!!
-      ##   2) add lang switch for date/lang too!!!!
+      start = if season.year?
+                Date.new( season.start_year, 1, 1 )
+              else
+                Date.new( season.start_year, 7, 1 )
+              end
 
-      event = Sync::Event.find_or_create( league: league, season: season )
+      auto_conf_teams, _ = AutoConfParser.parse( lines,
+                                                 start: start )
 
-      stage = if rec[:stage]
-        Sync::Stage.find_or_create( rec[:stage], event: event )
-      else
-        nil
-      end
+      ## step 1: map/find teams
 
+      ## note: loop over keys (holding the names); values hold the usage counter!! e.g. 'Arsenal' => 2, etc.
+      mods = nil
+      if league.clubs? && league.intl?    ## todo/fix: add intl? to ActiveRecord league!!!
+                ### quick hack mods for popular/known ambigious club names
+                ##    todo/fix: make more generic / reuseable!!!!
+                mods = {}
+                ## europa league uses same mods as champions league
+                mods[ 'uefa.el' ] = mods[ 'uefa.cl' ] = catalog.clubs.build_mods(
+                  { 'Liverpool | Liverpool FC' => 'Liverpool FC, ENG',
+                    'Arsenal  | Arsenal FC'    => 'Arsenal FC, ENG',
+                    'Barcelona'                => 'FC Barcelona, ESP',
+                    'Valencia'                 => 'Valencia CF, ESP'  })
+       end
 
-      auto_conf_clubs, _ = AutoConfParser.parse( rec[:lines],
-                                                 start: event.start_at )
+       teams = catalog.teams.find_by!( name:   auto_conf_teams.keys,
+                                       league: league,
+                                       mods:   mods )
 
-      ## step 1: map/find clubs
-      club_recs    = []   ## array of struct records
-      club_mapping = {}   ## name => database (ActiveRecord) record
+       ## build mapping - name => team struct record
+       team_mapping =  auto_conf_teams.keys.zip( teams ).to_h
 
-      ## note: loop over keys (holding the names); values hold the usage counter!! e.g. 'Arsenal' => 2, etc.
-      country = league.country
-      auto_conf_clubs.keys.each do |name|
-        club_rec = config.clubs.find_by!( name: name, country: country )
-        club_recs << club_rec
 
-        club = Sync::Club.find_or_create( club_rec )
-        club_mapping[ name ] = club
-      end
+      parser = MatchParser.new( lines,
+                                team_mapping,
+                                start )   ## note: keep season start_at date for now (no need for more specific stage date need for now)
 
+      matches, rounds, groups = parser.parse
 
-      ## todo/fix: check if all clubs are unique
-      ##   check if uniq works for club record (struct) - yes,no ??
-      clubs = club_mapping.values.uniq
+      pp rounds
+      pp groups
 
 
-      ## step 2: add to database
-      teams     =  stage ? stage.teams    : event.teams
-      team_ids  =  stage ? stage.team_ids : event.team_ids
+     ######################################################
+     ## step 2: add to database
 
-      clubs.each do |club|
-        ## add teams to event
-        ##   for now check if team is alreay included
-        ##   todo/fix: clear/destroy_all first - why? why not!!!
+     event_rec = Sync::Event.find_or_create_by( league: league,
+                                                season: season )
 
-        teams << club    unless team_ids.include?( club.id )
-      end
+     stage_rec = if stage
+                  Sync::Stage.find_or_create( stage, event: event_rec )
+                 else
+                   nil
+                 end
 
+     team_recs  =  stage_rec ? stage_rec.teams    : event_rec.teams
+     team_ids   =  stage_rec ? stage_rec.team_ids : event_rec.team_ids
 
+      ## todo/fix: check if all teams are unique
+      ##   check if uniq works for club record (struct) - yes,no ??
+      new_team_recs = Sync::Team.find_or_create( team_mapping.values.uniq )
 
-      ## todo/fix: set lang for now depending on league country!!!
-      parser = MatchParserSimpleV2.new( rec[:lines],
-                                club_mapping,
-                                event.start_at )   ## note: keep season start_at date for now (no need for more specific stage date need for now)
+     new_team_recs.each do |team_rec|
+       ## add teams to event
+       ##   for now check if team is alreay included
+       ##   todo/fix: clear/destroy_all first - why? why not!!!
+       team_recs << team_rec    unless team_ids.include?( team_rec.id )
+     end
 
-      match_recs, round_recs = parser.parse
 
-      pp round_recs
+      rounds.each do |round|
+        round_rec = Sync::Round.find_or_create( round, event: event_rec )  ## check: use/rename to EventRound why? why not?
+      end
 
-      round_recs.each do |round_rec|
-        ## quick hack:  if pos missing fill with dummy 999 for now
-        round_rec.pos = 999    if round_rec.pos.nil?
-        round = Sync::Round.find_or_create( round_rec, event: event )  ## check: use/rename to EventRound why? why not?
+      groups.each do |group|
+        group_rec = Sync::Group.find_or_create( group, event: event_rec )   ## check: use/rename to EventGroup why? why not?
       end
 
-      match_recs.each do |match_rec|
-        ## todo/fix: pass along stage (if present): stage  - optional!!!!
-        match = Sync::Match.create_or_update( match_rec, event: event )
+      matches.each do |match|
+        ## note: pass along stage (if present): stage  - optional from heading!!!!
+        match = match.update( stage: stage )   if stage
+        match_rec = Sync::Match.create_or_update( match, event: event_rec )
       end
     end
 
-    recs
-  end # method read
-end # class MatchReaderV2
+    true   ## success/ok
+  end # method parse
+
+
+  ######################
+  # (convenience) helpers
+
+  def catalog() Import.catalog; end
+
+end # class MatchReader
 end # module SportDb

data/lib/sportdb/readers/package.rb

@@ -2,130 +2,70 @@
 module SportDb
   class Package
 
-    CONF_RE        = Datafile::CONF_RE
-    CLUB_PROPS_RE  = Datafile::CLUB_PROPS_RE
-    LEAGUES_RE     = Datafile::LEAGUES_RE
-    CLUBS_RE       = Datafile::CLUBS_RE
-
-
-    ## note: if pattern includes directory add here (otherwise move to more "generic" datafile) - why? why not?
-    MATCH_RE = %r{ /(?: \d{4}-\d{2}        ## season folder e.g. /2019-20
-                      | \d{4}              ## season year-only folder e.g. /2019
-                    )
-                   /[a-z0-9_-]+\.txt$  ## txt e.g /1-premierleague.txt
-                }x
-
-
-    attr_reader :pack     ## allow access to embedded ("low-level") delegate package
-
-    def initialize( path_or_pack )
-      if path_or_pack.is_a?( Datafile::Package )
-        @pack = path_or_pack
-      else   ## assume it's a (string) path
-        path = path_or_pack
-        if !File.exist?( path )  ## file or directory
-          puts "** !!! ERROR !!! file NOT found >#{path}<; cannot open package"
-          exit 1
-        end
-
-        if File.directory?( path )
-          @pack = Datafile::DirPackage.new( path )     ## delegate to "generic" package
-        elsif File.file?( path ) && File.extname( path ) == '.zip'  # note: includes dot (.) eg .zip
-          @pack = Datafile::ZipPackage.new( path )
-        else
-          puts "** !!! ERROR !!! cannot open package - directory or file with .zip extension required"
-          exit 1
-        end
-      end
-    end
-
-    def each_conf( &blk )       @pack.each( pattern: CONF_RE, &blk ); end
-    def each_match( &blk )      @pack.each( pattern: MATCH_RE, &blk ); end
-    def each_club_props( &blk ) @pack.each( pattern: CLUB_PROPS_RE, &blk ); end
-
-    def each_leagues( &blk )    @pack.each( pattern: LEAGUES_RE, &blk ); end
-    def each_clubs( &blk )      @pack.each( pattern: CLUBS_RE, &blk ); end
-
+    ## note: add readers here; for full class def see the sourcein sportdb-formats!!!
 
     def read_leagues
-      each_leagues do |entry|
-        SportDb.parse_leagues( entry.read )
-      end
+      each_leagues { |entry| SportDb.parse_leagues( entry.read ) }
     end
 
     def read_clubs
-      each_clubs do |entry|
-        SportDb.parse_clubs( entry.read )
-      end
+      each_clubs { |entry| SportDb.parse_clubs( entry.read ) }
     end
 
-
-    def read_club_props( sync: true )   ## todo/fix: remove sync!! - why? why not?
-      each_club_props do |entry|
-        SportDb.parse_club_props( entry.read, sync: sync )
-      end
+    def read_club_props
+      each_club_props { |entry| SportDb.parse_club_props( entry.read ) }
     end
 
-    def read_conf( *names,
-                   season: nil, sync: true )
+
+    def read_conf( *names, season: nil )
       if names.empty?   ## no (entry) names passed in; read in all
         each_conf do |entry|
-          SportDb.parse_conf( entry.read, season: season, sync: sync )
+          SportDb.parse_conf( entry.read, season: season )
         end
       else
         names.each do |name|
           entry = @pack.find( name )
-          SportDb.parse_conf( entry.read, season: season, sync: sync )
+          SportDb.parse_conf( entry.read, season: season )
         end
       end
     end
 
-    def read_match( *names,
-                    season: nil, sync: true )
+    def read_match( *names, season: nil )
       if names.empty?   ## no (entry) names passed in; read in all
         each_match do |entry|
-          SportDb.parse_match( entry.read, season: season, sync: sync )
+          SportDb.parse_match( entry.read, season: season )
         end
       else
         names.each do |name|
           entry = @pack.find( name )
-          SportDb.parse_match( entry.read, season: season, sync: sync )
+          SportDb.parse_match( entry.read, season: season )
         end
       end
     end
 
 
-    def read( *names,
-              season: nil, sync: true )
+    def read( *names, season: nil )
       if names.empty?   ##  read all datafiles
         read_leagues()
         read_clubs()
-        read_club_props( sync: sync )   ## todo/fix: remove sync - why? why not?
-        read_conf( season: season, sync: sync )
-        read_match( season: season, sync: sync )
+        read_club_props()
+        ## note:  skip conf(iguration)s for now!!!!!!!
+        ## read_conf( season: season )
+        read_match( season: season )
       else
         names.each do |name|
           entry = @pack.find( name )
           ## fix/todo: add read_leagues, read_clubs too!!!
-          if Datafile.match_conf( name )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
-            SportDb.parse_conf( entry.read, season: season, sync: sync )
-          elsif Datafile.match_club_props( name )
-            SportDb.parse_club_props( entry.read, sync: sync )
-          else   ## assume "regular" match datafile
-            SportDb.parse_match( entry.read, season: season, sync: sync )
+          if match_conf?( name )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
+            SportDb.parse_conf( entry.read, season: season )
+          elsif match_club_props?( name )
+            SportDb.parse_club_props( entry.read )
+          else   ## assume "regular" match datafile or check pattern and report error on fail - why? why not?
+            SportDb.parse_match( entry.read, season: season )
           end
         end
       end
     end
   end   # class Package
 
-
-  class DirPackage < Package
-    def initialize( path )   super( Datafile::DirPackage.new( path ) ); end
-  end
-
-  class ZipPackage < Package
-    def initialize( path )   super( Datafile::ZipPackage.new( path ) ); end
-  end
-
 end   # module SportDb

data/lib/sportdb/readers/version.rb

@@ -4,8 +4,8 @@
 module SportDb
 module Readers
 
-  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 5
+  MAJOR = 1    ## todo: namespace inside version or something - why? why not??
+  MINOR = 1
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

data/test/helper.rb

@@ -1,19 +1,23 @@
-## $:.unshift(File.dirname(__FILE__))
+## note: use the local version of sportdb gems
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-formats/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-config/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-models/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-sync/lib' ))
+
 
 ## minitest setup
 require 'minitest/autorun'
 
 
-## note: use the local version of sportdb gems
-$LOAD_PATH.unshift( File.expand_path( '../sportdb-match-formats/lib' ))
-
 
 ## our own code
 require 'sportdb/readers'
 
-
-
-
 ## use (switch to) "external" datasets
 SportDb::Import.config.leagues_dir = "../../../openfootball/leagues"
 SportDb::Import.config.clubs_dir   = "../../../openfootball/clubs"
+
+
+COUNTRIES = SportDb::Import.catalog.countries
+LEAGUES   = SportDb::Import.catalog.leagues
+CLUBS     = SportDb::Import.catalog.clubs

data/test/test_conf_reader.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_conf_reader.rb
+
+
+require 'helper'
+
+
+class TestConfReader < MiniTest::Test
+
+  def setup
+    SportDb.connect( adapter:  'sqlite3',
+                     database: ':memory:' )
+    SportDb.create_all   ## build schema
+
+    ## turn on logging to console
+    ActiveRecord::Base.logger = Logger.new(STDOUT)
+  end
+
+  def test_read
+    # path = "../../../openfootball/austria/2018-19/.conf.txt"
+    path = "../../../openfootball/england/2015-16/.conf.txt"
+    # path = "../../../openfootball/england/2017-18/.conf.txt"
+    # path = "../../../openfootball/england/2018-19/.conf.txt"
+    # path = "../../../openfootball/england/2019-20/.conf.txt"
+    SportDb::ConfReader.read( path )
+  end  # method test_read
+
+
+  def test_read_champs
+txt =<<TXT
+= UEFA Champions League 2017/18
+
+Manchester United › ENG
+Liverpool         › ENG
+Chelsea           › ENG
+Manchester City   › ENG
+Tottenham Hotspur › ENG
+
+Atlético Madrid › ESP
+Barcelona       › ESP
+Sevilla         › ESP
+Real Madrid     › ESP
+
+Roma     › ITA
+Juventus › ITA
+Napoli   › ITA
+
+Bayern München    › GER
+Borussia Dortmund › GER
+RB Leipzig        › GER
+
+Benfica     › POR
+Sporting CP › POR
+Porto       › POR
+
+CSKA Moscow    › RUS
+Spartak Moscow › RUS
+
+Paris Saint-Germain › FRA
+Basel › SUI
+Celtic › SCO
+Anderlecht › BEL
+Qarabağ › AZE
+Olympiacos › GRE
+Maribor › SVN
+Shakhtar Donetsk › UKR
+Feyenoord › NED
+Beşiktaş › TUR
+Monaco › MCO
+APOEL › CYP
+TXT
+
+    SportDb::ConfReader.parse( txt )
+  end
+end  # class TestConfReader