sportdb-readers 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4a900ae50e920da9b842d72e975577d098dbedd
-  data.tar.gz: 9576ab392dcadb0d672535d59614c7d93ed29387
+  metadata.gz: 976e1c02d5e34f69d999581d0036b1dce641d27b
+  data.tar.gz: f84742df381b09fa244d963d6624d5a0137c3159
 SHA512:
-  metadata.gz: e1585ce4281e1725b0452194c1c34858714dc5a40ed4b25ffe598182be9878ce46f544ee66b3d43dc64ca5480b3e00195ae6c08de013a4f3c26962c1cb920484
-  data.tar.gz: fd04ee65cde3a82a144473b1d63eb8a2cba1cf63c5dc0d4e0a98122ece942f2bef980ef751a6752273acf01c7adaaab68e5202d35bc86c8dc09356dde39b1b84
+  metadata.gz: 040b202ef7acac2ce56a67f33b88c9c93fe961173cfb5ff28983dee540b643963ed5581273be2d6ee5c6f8d257f1559b62f7ea5c340fc8b02c7c279768add7f7
+  data.tar.gz: d0f3cfa4c66a261f5d4a30b24e7e5f86aef8ebc78730dd4782ffc33d00d44205e5d23a44275640664851fb0e7e10f83b5cc1a789293031c6b638dcf53de36b18

data/Manifest.txt

@@ -1,17 +1,19 @@
 CHANGELOG.md
 Manifest.txt
+NOTES.md
 README.md
 Rakefile
 lib/sportdb/readers.rb
-lib/sportdb/readers/conf_linter.rb
 lib/sportdb/readers/conf_reader.rb
-lib/sportdb/readers/league_outline_reader.rb
-lib/sportdb/readers/match_linter.rb
 lib/sportdb/readers/match_reader.rb
 lib/sportdb/readers/package.rb
 lib/sportdb/readers/version.rb
 test/helper.rb
+test/test_conf_reader.rb
+test/test_match_reader_champs.rb
 test/test_match_reader_eng.rb
+test/test_match_reader_euro.rb
 test/test_match_reader_mu.rb
 test/test_read.rb
 test/test_reader.rb
+test/test_reader_champs.rb

data/NOTES.md

@@ -0,0 +1,35 @@
+# Notes
+
+## Todos
+
+
+todo/fix:  use config / configurations; remove all globals e.g.:
+- COUNTRIES
+- CLUBS
+- LEAGUES !!!!!
+
+
+
+fix/todo: move find_league  to sportdb-league index use find_by! and find_by !!!!
+in sportdb-readers (and others?)
+
+```
+def self.find_league( name )
+  league = nil
+  m = config.leagues.match( name )
+  # pp m
+
+  if m.nil?
+    puts "** !!! ERROR !!! no league match found for >#{name}<, add to leagues table; sorry"
+    exit 1
+  elsif m.size > 1
+    puts "** !!! ERROR !!! ambigious league name; too many leagues (#{m.size}) found:"
+    pp m
+    exit 1
+  else
+    league = m[0]
+  end
+
+  league
+end
+```

data/Rakefile

@@ -20,9 +20,7 @@ Hoe.spec 'sportdb-readers' do
   self.licenses = ['Public Domain']
 
   self.extra_deps = [
-    ['sportdb-config',  '>= 0.9.0'],
-    ['sportdb-models',  '>= 1.18.6'],
-    ['sportdb-sync',    '>= 0.1.0'],
+    ['sportdb-sync',    '>= 1.0.0'],
   ]
 
   self.spec_extras = {

data/lib/sportdb/readers.rb

@@ -1,19 +1,13 @@
 # encoding: utf-8
 
-
-require 'sportdb/config'
-require 'sportdb/models'   ## add sql database support
 require 'sportdb/sync'
 
 
 ###
 # our own code
 require 'sportdb/readers/version' # let version always go first
-require 'sportdb/readers/league_outline_reader'
 require 'sportdb/readers/conf_reader'
-require 'sportdb/readers/conf_linter'
 require 'sportdb/readers/match_reader'
-require 'sportdb/readers/match_linter'
 require 'sportdb/readers/package'
 
 
@@ -22,76 +16,43 @@ require 'sportdb/readers/package'
 ##
 ##  add convenience shortcut helpers
 module SportDb
+  def self.read_conf( path, season: nil )  ConfReaderV2.read( path, season: season ); end
+  def self.parse_conf( txt, season: nil )  ConfReaderV2.parse( txt, season: season ); end
+
+  ### todo/check: add alias read_matches - why? why not?
+  def self.read_match( path, season: nil )  MatchReaderV2.read( path, season: season ); end
+  def self.parse_match( txt, season: nil )  MatchReaderV2.parse( txt, season: season ); end
+
+  def self.read_club_props( path )  Import::ClubPropsReader.read( path ); end
+  def self.parse_club_props( txt )  Import::ClubPropsReader.parse( txt ); end
+
+  def self.parse_leagues( txt ) recs = Import::LeagueReader.parse( txt ); Import::config.leagues.add( recs ); end
+  def self.parse_clubs( txt )   recs = Import::ClubReader.parse( txt ); Import::config.clubs.add( recs ); end
 
-  ## note: sync is dry run (for lint checking)
-  def self.read_conf( path, season: nil, sync: true )
-    sync ? ConfReaderV2.read( path, season: season )
-         : ConfLinter.read( path, season: season )
-  end
-  def self.parse_conf( txt, season: nil, sync: true )
-    sync ? ConfReaderV2.parse( txt, season: season )
-         : ConfLinter.parse( txt, season: season )
-  end
-
-  def self.read_match( path, season: nil, sync: true )  ### todo/check: add alias read_matches - why? why not?
-    sync ? MatchReaderV2.read( path, season: season )
-         : MatchLinter.read( path, season: season )
-  end
-  def self.parse_match( txt, season: nil, sync: true )  ### todo/check: add alias read_matches - why? why not?
-    sync ? MatchReaderV2.parse( txt, season: season )
-         : MatchLinter.parse( txt, season: season )
-  end
-
-  def self.read_club_props( path, sync: true )
-    ## note: for now run only if sync (e.g. run with db updates)
-    SportDb::Import::ClubPropsReader.read( path )   if sync
-  end
-  def self.parse_club_props( txt, sync: true )
-    ## note: for now run only if sync (e.g. run with db updates)
-    SportDb::Import::ClubPropsReader.parse( txt )   if sync
-  end
-
-
-  def self.parse_leagues( txt )
-    recs = SportDb::Import::LeagueReader.parse( txt )
-    Import::config.leagues.add( recs )
-  end
-
-  def self.parse_clubs( txt )
-    recs = SportDb::Import::ClubReader.parse( txt )
-    Import::config.clubs.add( recs )
-  end
-
-
-  def self.read( path, season: nil, sync: true )
+
+  def self.read( path, season: nil )
     pack = if File.directory?( path )          ## if directory assume "unzipped" package
               DirPackage.new( path )
-           elsif File.file?( path ) && Datafile.match_zip( path )  ## check if file is a .zip (archive) file
+           elsif File.file?( path ) && File.extname( path ) == '.zip'   ## check if file is a .zip (archive) file
               ZipPackage.new( path )
            else                                ## no package; assume single (standalone) datafile
              nil
            end
 
     if pack
-       pack.read( season: season, sync: sync )
+       pack.read( season: season )
     else
-      if Datafile.match_conf( path )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
-        read_conf( path, season: season, sync: sync )
-      elsif Datafile.match_club_props( path )
-        read_club_props( path, sync: sync )
+      if Package.conf?( path )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
+        read_conf( path, season: season )
+      elsif Package.club_props?( path )
+        read_club_props( path )
       else                                ## assume "regular" match datafile
-        read_match( path, season: season, sync: sync )
+        read_match( path, season: season )
       end
     end
   end  # method read
 
 
-
-  ## (more) convenience helpers for lint(ing)
-  def self.lint( path, season: nil )        read( path, season: season, sync: false ); end
-  def self.lint_conf( path, season: nil )   read_conf( path, season: season, sync: false ); end
-  def self.lint_match( path, season: nil )  read_match( path, season: season, sync: false ); end
-
 end # module SportDb
 
 

data/lib/sportdb/readers/conf_reader.rb

@@ -5,84 +5,96 @@ module SportDb
 
 class ConfReaderV2    ## todo/check: rename to EventsReaderV2 (use plural?) why? why not?
 
-  def self.config() Import.config; end    ## shortcut convenience helper
-
-
   def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
     parse( txt, season: season )
   end
 
   def self.parse( txt, season: nil )
-    recs = LeagueOutlineReader.parse( txt, season: season )
-    pp recs
-
-    ## pass 2 - check & map; replace inline (string with record)
-    recs.each do |rec|
-      league = rec[:league]
-      clubs = []    ## convert lines to clubs
-      rec[:lines].each do |line|
-
-        next if line =~ /^[ -]+$/   ## skip decorative lines with dash only (e.g. ---- or - - - -) etc.
-
-        scan = StringScanner.new( line )
-
-        if scan.check( /\d{1,2}[ ]+/ )    ## entry with standaning starts with ranking e.g. 1,2,3, etc.
-          puts "  table entry >#{line}<"
-          rank = scan.scan( /\d{1,2}[ ]+/ ).strip   # note: strip trailing spaces
-
-          ## note: uses look ahead scan until we hit at least two spaces
-          ##  or the end of string  (standing records for now optional)
-          name = scan.scan_until( /(?=\s{2})|$/ )
-          if scan.eos?
-            standing = nil
-          else
-            standing = scan.rest.strip   # note: strip leading and trailing spaces
-          end
-          puts "   rank: >#{rank}<, name: >#{name}<, standing: >#{standing}<"
+    new( txt ).parse( season: season )
+  end
 
-          ## note: rank and standing gets ignored (not used) for now
+
+  include Logging
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+  def parse( season: nil )
+    secs = LeagueOutlineReader.parse( @txt, season: season )
+    pp secs
+
+    ## pass 1 - check & map; replace inline (string with record)
+    secs.each do |sec|    # sec(tion)s
+
+      conf = ConfParser.parse( sec[:lines] )
+
+      league = sec[:league]
+      teams = []    ## convert lines to teams
+
+      if league.clubs?
+        if league.intl?
+          conf.each do |name, rec|
+            country_key = rec[:country]
+            teams << catalog.clubs.find_by!( name: name,
+                                            country: country_key )
+          end
         else
-          ## assume club is full line
-          name = line
+          conf.each do |name, _|
+            ## note: rank and standing gets ignored (not used) for now
+            teams << catalog.clubs.find_by!( name: name,
+                                            country: league.country )
+          end
+        end
+      else   ### assume national teams
+        conf.each do |name, _|
+          ## note: rank and standing gets ignored (not used) for now
+          teams << catalog.national_teams.find!( name )
         end
-
-        clubs << config.clubs.find_by( name: name, country: league.country )
       end
 
-      rec[:clubs] = clubs
-      rec.delete( :lines )   ## remove lines entry
+
+      sec[:teams] = teams
+
+      sec.delete( :lines )   ## remove lines entry
     end
 
-    ## pass 3 - import (insert/update) into db
-    recs.each do |rec|
-       league = Sync::League.find_or_create( rec[:league] )
-       season = Sync::Season.find_or_create( rec[:season] )
 
+    ## pass 2 - import (insert/update) into db
+    secs.each do |sec|   # sec(tion)s
+       ## todo/fix: always return Season struct record in LeagueReader - why? why not?
+       event_rec  = Sync::Event.find_or_create_by( league: sec[:league],
+                                                   season: sec[:season] )
 
-       event  = Sync::Event.find_or_create( league: league, season: season )
-       if rec[:stage]
-         stage = Sync::Stage.find_or_create( rec[:stage], event: event )
-       else
-         stage = nil
-       end
+       stage_rec  = if sec[:stage]
+                      Sync::Stage.find_or_create( sec[:stage], event: event_rec )
+                    else
+                      nil
+                    end
 
+       ## todo/fix: check if all teams are unique
+       ##   check if uniq works for club/national_team record (struct) - yes,no ??
+       teams = sec[:teams]
+       teams = teams.uniq
 
-       rec[:clubs].each do |club_rec|
-         club = Sync::Club.find_or_create( club_rec )
+       ## add to database
+       team_recs     =  stage_rec ? stage_rec.teams    : event_rec.teams
+       team_ids      =  stage_rec ? stage_rec.team_ids : event_rec.team_ids
+
+       new_team_recs = Sync::Team.find_or_create( teams )
+       new_team_recs.each do |team_rec|
          ## add teams to event
-         ##   todo/fix: check if team is alreay included?
-         ##    or clear/destroy_all first!!!
-         if stage
-           stage.teams << club
-         else
-           event.teams << club
-         end
+         ##   for now check if team is alreay included
+         ##   todo/fix: clear/destroy_all first - why? why not!!!
+         team_recs << team_rec    unless team_ids.include?( team_rec.id )
        end
     end
 
-    recs
+    true   ## todo/fix: return true/false or something
   end # method read
 
+  def catalog() Import.catalog; end  ## shortcut convenience helper
+
 end # class ConfReaderV2
 end # module SportDb

data/lib/sportdb/readers/match_reader.rb

@@ -4,118 +4,150 @@ module SportDb
 
 class MatchReaderV2    ## todo/check: rename to MatchReaderV2 (use plural?) why? why not?
 
-  def self.config()  Import.config; end
-
-
-
   def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
     parse( txt, season: season )
   end
 
   def self.parse( txt, season: nil )
-    recs = LeagueOutlineReader.parse( txt, season: season )
-    pp recs
-
-    recs.each do |rec|
-      league = Sync::League.find_or_create( rec[:league] )
-      season = Sync::Season.find_or_create( rec[:season] )
+    new( txt ).parse( season: season )
+  end
 
-      ## hack for now: switch lang
-      if ['de', 'at'].include?( league.country.key )
-        SportDb.lang.lang = 'de'
-        DateFormats.lang  = 'de'
-      elsif ['fr'].include?( league.country.key )
-        SportDb.lang.lang = 'fr'
-        DateFormats.lang  = 'fr'
-      elsif ['it'].include?( league.country.key )
-        SportDb.lang.lang = 'it'
-        DateFormats.lang  = 'it'
-      elsif ['es', 'mx'].include?( league.country.key )
-        SportDb.lang.lang = 'es'
-        DateFormats.lang  = 'es'
-      elsif ['pt', 'br'].include?( league.country.key )
-        SportDb.lang.lang = 'pt'
-        DateFormats.lang  = 'pt'
-      else
-        SportDb.lang.lang = 'en'
-        DateFormats.lang  = 'en'
-      end
 
+  include Logging
 
-      ## todo/fix:
-      ##    always auto create
-      ##   1) check for clubs count on event/stage - only if count == 0 use autoconf!!!
-      ##   2) add lang switch for date/lang too!!!!
+  def initialize( txt )
+    @txt = txt
+  end
 
-      event = Sync::Event.find_or_create( league: league, season: season )
+  def parse( season: nil )
+    secs = LeagueOutlineReader.parse( @txt, season: season )
+    pp secs
+
+
+    ###
+    ## todo/check/fix:  move to LeagueOutlineReader for (re)use - why? why not?
+    ##                    use sec[:lang] or something?
+    langs = {    ## map country keys to lang codes
+      'de' => 'de', ## de - Deutsch (German)
+      'at' => 'de',
+      'fr' => 'fr', ## fr - French
+      'it' => 'it', ## it - Italian
+      'es' => 'es', ## es - Español (Spanish)
+      'mx' => 'es',
+      'pt' => 'pt', ## pt -  Português (Portuguese)
+      'br' => 'br'
+    }
+
+    secs.each do |sec|   ## sec(tion)s
+      season = sec[:season]
+      league = sec[:league]
+      stage  = sec[:stage]
+      lines  = sec[:lines]
 
-      stage = if rec[:stage]
-        Sync::Stage.find_or_create( rec[:stage], event: event )
-      else
-        nil
+      ## hack for now: switch lang
+      ## todo/fix: set lang for now depending on league country!!!
+      if league.intl?   ## todo/fix: add intl? to ActiveRecord league!!!
+        Import.config.lang = 'en'
+      else  ## assume national/domestic
+        Import.config.lang = langs[ league.country.key ] || 'en'
       end
 
 
-      auto_conf_clubs, _ = AutoConfParser.parse( rec[:lines],
-                                                 start: event.start_at )
+      start = if season.year?
+                Date.new( season.start_year, 1, 1 )
+              else
+                Date.new( season.start_year, 7, 1 )
+              end
+
+      auto_conf_teams, _ = AutoConfParser.parse( lines,
+                                                 start: start )
 
-      ## step 1: map/find clubs
-      club_recs    = []   ## array of struct records
-      club_mapping = {}   ## name => database (ActiveRecord) record
+      ## step 1: map/find teams
 
       ## note: loop over keys (holding the names); values hold the usage counter!! e.g. 'Arsenal' => 2, etc.
-      country = league.country
-      auto_conf_clubs.keys.each do |name|
-        club_rec = config.clubs.find_by!( name: name, country: country )
-        club_recs << club_rec
+      mods = nil
+      if league.clubs? && league.intl?    ## todo/fix: add intl? to ActiveRecord league!!!
+                ### quick hack mods for popular/known ambigious club names
+                ##    todo/fix: make more generic / reuseable!!!!
+                mods = {}
+                ## europa league uses same mods as champions league
+                mods[ 'uefa.el' ] = mods[ 'uefa.cl' ] = catalog.clubs.build_mods(
+                  { 'Liverpool | Liverpool FC' => 'Liverpool FC, ENG',
+                    'Arsenal  | Arsenal FC'    => 'Arsenal FC, ENG',
+                    'Barcelona'                => 'FC Barcelona, ESP',
+                    'Valencia'                 => 'Valencia CF, ESP'  })
+       end
 
-        club = Sync::Club.find_or_create( club_rec )
-        club_mapping[ name ] = club
-      end
+       teams = catalog.teams.find_by!( name:   auto_conf_teams.keys,
+                                       league: league,
+                                       mods:   mods )
 
+       ## build mapping - name => team struct record
+       team_mapping =  auto_conf_teams.keys.zip( teams ).to_h
 
-      ## todo/fix: check if all clubs are unique
-      ##   check if uniq works for club record (struct) - yes,no ??
-      clubs = club_mapping.values.uniq
 
+      parser = MatchParserSimpleV2.new( lines,
+                                team_mapping,
+                                start )   ## note: keep season start_at date for now (no need for more specific stage date need for now)
 
-      ## step 2: add to database
-      teams     =  stage ? stage.teams    : event.teams
-      team_ids  =  stage ? stage.team_ids : event.team_ids
+      matches, rounds, groups = parser.parse
 
-      clubs.each do |club|
-        ## add teams to event
-        ##   for now check if team is alreay included
-        ##   todo/fix: clear/destroy_all first - why? why not!!!
+      pp rounds
+      pp groups
 
-        teams << club    unless team_ids.include?( club.id )
-      end
 
+     ######################################################
+     ## step 2: add to database
 
+     event_rec = Sync::Event.find_or_create_by( league: league,
+                                                season: season )
 
-      ## todo/fix: set lang for now depending on league country!!!
-      parser = MatchParserSimpleV2.new( rec[:lines],
-                                club_mapping,
-                                event.start_at )   ## note: keep season start_at date for now (no need for more specific stage date need for now)
+     stage_rec = if stage
+                  Sync::Stage.find_or_create( stage, event: event_rec )
+                 else
+                   nil
+                 end
+
+     team_recs  =  stage_rec ? stage_rec.teams    : event_rec.teams
+     team_ids   =  stage_rec ? stage_rec.team_ids : event_rec.team_ids
+
+      ## todo/fix: check if all teams are unique
+      ##   check if uniq works for club record (struct) - yes,no ??
+      new_team_recs = Sync::Team.find_or_create( team_mapping.values.uniq )
 
-      match_recs, round_recs = parser.parse
+     new_team_recs.each do |team_rec|
+       ## add teams to event
+       ##   for now check if team is alreay included
+       ##   todo/fix: clear/destroy_all first - why? why not!!!
+       team_recs << team_rec    unless team_ids.include?( team_rec.id )
+     end
 
-      pp round_recs
 
-      round_recs.each do |round_rec|
+      rounds.each do |round|
         ## quick hack:  if pos missing fill with dummy 999 for now
-        round_rec.pos = 999    if round_rec.pos.nil?
-        round = Sync::Round.find_or_create( round_rec, event: event )  ## check: use/rename to EventRound why? why not?
+        round.pos = 999    if round.pos.nil?
+        round_rec = Sync::Round.find_or_create( round, event: event_rec )  ## check: use/rename to EventRound why? why not?
       end
 
-      match_recs.each do |match_rec|
+      groups.each do |group|
+        group_rec = Sync::Group.find_or_create( group, event: event_rec )   ## check: use/rename to EventGroup why? why not?
+      end
+
+      matches.each do |match|
         ## todo/fix: pass along stage (if present): stage  - optional!!!!
-        match = Sync::Match.create_or_update( match_rec, event: event )
+        match_rec = Sync::Match.create_or_update( match, event: event_rec )
       end
     end
 
-    recs
-  end # method read
+    true   ## success/ok
+  end # method parse
+
+
+  ######################
+  # (convenience) helpers
+
+  def catalog() Import.catalog; end
+
 end # class MatchReaderV2
 end # module SportDb