sportdb-readers 0.5.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4a900ae50e920da9b842d72e975577d098dbedd
-  data.tar.gz: 9576ab392dcadb0d672535d59614c7d93ed29387
+  metadata.gz: f8c01094bf348c410e2d95343b8a5a3a660c84d4
+  data.tar.gz: 8983fe5aa77546edf2d4d31a64e9cd30a80d7320
 SHA512:
-  metadata.gz: e1585ce4281e1725b0452194c1c34858714dc5a40ed4b25ffe598182be9878ce46f544ee66b3d43dc64ca5480b3e00195ae6c08de013a4f3c26962c1cb920484
-  data.tar.gz: fd04ee65cde3a82a144473b1d63eb8a2cba1cf63c5dc0d4e0a98122ece942f2bef980ef751a6752273acf01c7adaaab68e5202d35bc86c8dc09356dde39b1b84
+  metadata.gz: 93f591316bb2a1d3e0111039757ada38294a744479388f0b868d81fb88e36b6f61f7036d0f4124dd49efb74679e8aee4d19bab9b2673649a955e6dc83757893a
+  data.tar.gz: 1de11267bcf199e67d82ccd07d40c2709413aeea985545b164131a0ae1a42851335133e4d0434f77c63fb6ab55b64ea4f77209118c6a8bc45117bd9148a14279

data/Manifest.txt

@@ -1,17 +1,19 @@
 CHANGELOG.md
 Manifest.txt
+NOTES.md
 README.md
 Rakefile
 lib/sportdb/readers.rb
-lib/sportdb/readers/conf_linter.rb
 lib/sportdb/readers/conf_reader.rb
-lib/sportdb/readers/league_outline_reader.rb
-lib/sportdb/readers/match_linter.rb
 lib/sportdb/readers/match_reader.rb
 lib/sportdb/readers/package.rb
 lib/sportdb/readers/version.rb
 test/helper.rb
+test/test_conf_reader.rb
+test/test_match_reader_champs.rb
 test/test_match_reader_eng.rb
+test/test_match_reader_euro.rb
 test/test_match_reader_mu.rb
 test/test_read.rb
 test/test_reader.rb
+test/test_reader_champs.rb

data/NOTES.md

@@ -0,0 +1,35 @@
+# Notes
+
+## Todos
+
+
+todo/fix:  use config / configurations; remove all globals e.g.:
+- COUNTRIES
+- CLUBS
+- LEAGUES !!!!!
+
+
+
+fix/todo: move find_league  to sportdb-league index use find_by! and find_by !!!!
+in sportdb-readers (and others?)
+
+```
+def self.find_league( name )
+  league = nil
+  m = config.leagues.match( name )
+  # pp m
+
+  if m.nil?
+    puts "** !!! ERROR !!! no league match found for >#{name}<, add to leagues table; sorry"
+    exit 1
+  elsif m.size > 1
+    puts "** !!! ERROR !!! ambigious league name; too many leagues (#{m.size}) found:"
+    pp m
+    exit 1
+  else
+    league = m[0]
+  end
+
+  league
+end
+```

data/README.md

@@ -31,50 +31,116 @@ Let's read in some leagues, seasons, clubs, and match schedules and results.
 Let's use the public domain football.db datasets for England (see [`openfootball/england`](https://github.com/openfootball/england)), as an example:
 
 
-``` ruby
-## turn on logging to console
-ActiveRecord::Base.logger = Logger.new( STDOUT )
+```
+= English Premier League 2015/16
+
+Matchday 1
+
+[Sat Aug 8]
+  Manchester United      1-0  Tottenham Hotspur
+  AFC Bournemouth        0-1  Aston Villa
+  Everton FC             2-2  Watford FC
+  Leicester City         4-2  Sunderland AFC
+  Norwich City           1-3  Crystal Palace
+  Chelsea FC             2-2  Swansea City
+[Sun Aug 9]
+  Arsenal FC             0-2  West Ham United
+  Newcastle United       2-2  Southampton FC
+  Stoke City             0-1  Liverpool FC
+[Mon Aug 10]
+  West Bromwich Albion   0-3  Manchester City
+
+...
+```
+
+(Source: [england/2015-16/1-premierleague-i.txt](https://github.com/openfootball/england/blob/master/2015-16/1-premierleague-i.txt))
 
+and let's try:
+
+``` ruby
 ## assumes football.db datasets for England in ./england directory
 ##   see github.com/openfootball/england
-SportDb::ConfReaderV2.read( './england/2015-16/.conf.txt' )
-SportDb::MatchReaderV2.read( './england/2015-16/1-premierleague-i.txt' )
-SportDb::MatchReaderV2.read( './england/2015-16/1-premierleague-ii.txt' )
+SportDb.read( './england/2015-16/1-premierleague-i.txt' )
+SportDb.read( './england/2015-16/1-premierleague-ii.txt' )
 
 ## let's try another season
-SportDb::ConfReaderV2.read( './england/2019-20/.conf.txt' )
-SportDb::MatchReaderV2.read( './england/2019-20/1-premierleague.txt' )
+SportDb.read( './england/2019-20/1-premierleague.txt' )
 ```
 
 All leagues, seasons, clubs, match days and rounds, match fixtures and results,
 and more are now in your (SQL) database of choice.
 
-
-Or as an alternative use the `read` convenience all-in-one shortcut helper:
+The proof of the pudding - Let's query the (SQL) database using the sport.db ActiveRecord models:
 
 ``` ruby
-## assumes football.db datasets for England in ./england directory
-##   see github.com/openfootball/england
-SportDb.read( './england/2015-16/.conf.txt' )
-SportDb.read( './england/2015-16/1-premierleague-i.txt' )
-SportDb.read( './england/2015-16/1-premierleague-ii.txt' )
+include SportDb::Models
 
-## let's try another season
-SportDb.read( './england/2019-20/.conf.txt' )
-SportDb.read( './england/2019-20/1-premierleague.txt' )
+pl_2015_16 = Event.find_by( key: 'eng.1.2015/16' )
+#=> SELECT * FROM events WHERE key = 'eng.1.2015/16' LIMIT 1
+
+pl_2015_16.teams.count  #=> 20
+#=> SELECT COUNT(*) FROM teams
+#      INNER JOIN events_teams ON teams.id = events_teams.team_id
+#      WHERE events_teams.event_id = 1
+
+pl_2015_16.games.count  #=> 380
+#=>  SELECT COUNT(*) FROM games
+#      INNER JOIN rounds ON games.round_id = rounds.id
+#      WHERE rounds.event_id = 1
+
+pl_2019_20 = Event.find_by( key: 'eng.1.2019/20' )
+pl_2015_16.teams.count  #=> 20
+pl_2015_16.games.count  #=> 380
+
+# -or-
+
+pl = League.find_by( key: 'eng.1' )
+#=> SELECT * FROM leagues WHERE key = 'eng.1' LIMIT 1
+
+pl.seasons.count   #=> 2
+#=> SELECT COUNT(*) FROM seasons
+#     INNER JOIN events ON seasons.id = events.season_id
+#     WHERE events.league_id = 1
+
+# and so on and so forth.
 ```
 
-Or as an alternative pass in the "package" directory and let `read` figure
-out what datafiles to read:
+Bonus: As an alternative pass in the "package" directory or a zip archive and let `read` figure
+out what datafiles to read in:
 
 ``` ruby
 ## assumes football.db datasets for England in ./england directory
 ##   see github.com/openfootball/england
 SportDb.read( './england' )
+## -or-   use a zip archive download
+SportDb.read( './england.zip' )
 ```
 
 That's it.
 
+## Frequently Asked Questions (F.A.Q.s) and Answers
+
+Q: What about reading in datasets in comma-separated values (CSV) format?
+Example:
+
+```
+Round, Date,              Team 1,               FT,  HT,  Team 2
+1,     (Fri)  9 Aug 2019, Liverpool FC,         4-1, 4-0, Norwich City FC
+1,     (Sat) 10 Aug 2019, West Ham United FC,   0-5, 0-1, Manchester City FC
+1,     (Sat) 10 Aug 2019, AFC Bournemouth,      1-1, 0-0, Sheffield United FC
+1,     (Sat) 10 Aug 2019, Burnley FC,           3-0, 0-0, Southampton FC
+1,     (Sat) 10 Aug 2019, Crystal Palace FC,    0-0, 0-0, Everton FC
+1,     (Sat) 10 Aug 2019, Watford FC,           0-3, 0-1, Brighton & Hove Albion FC
+1,     (Sat) 10 Aug 2019, Tottenham Hotspur FC, 3-1, 0-1, Aston Villa FC
+1,     (Sun) 11 Aug 2019, Leicester City FC,    0-0, 0-0, Wolverhampton Wanderers FC
+1,     (Sun) 11 Aug 2019, Newcastle United FC,  0-1, 0-0, Arsenal FC
+1,     (Sun) 11 Aug 2019, Manchester United FC, 4-0, 1-0, Chelsea FC
+...
+```
+(Source: [england/2019-20/eng.1.csv](https://github.com/footballcsv/england/blob/master/2010s/2019-20/eng.1.csv))
+
+Yes, you can. See the [sportdb-importers library / gem »](https://github.com/sportdb/sport.db/tree/master/sportdb-importers) 
+
 
 
 ## License

data/Rakefile

@@ -20,9 +20,7 @@ Hoe.spec 'sportdb-readers' do
   self.licenses = ['Public Domain']
 
   self.extra_deps = [
-    ['sportdb-config',  '>= 0.9.0'],
-    ['sportdb-models',  '>= 1.18.6'],
-    ['sportdb-sync',    '>= 0.1.0'],
+    ['sportdb-sync',    '>= 1.1.0'],
   ]
 
   self.spec_extras = {

data/lib/sportdb/readers.rb

@@ -1,19 +1,13 @@
 # encoding: utf-8
 
-
-require 'sportdb/config'
-require 'sportdb/models'   ## add sql database support
 require 'sportdb/sync'
 
 
 ###
 # our own code
 require 'sportdb/readers/version' # let version always go first
-require 'sportdb/readers/league_outline_reader'
 require 'sportdb/readers/conf_reader'
-require 'sportdb/readers/conf_linter'
 require 'sportdb/readers/match_reader'
-require 'sportdb/readers/match_linter'
 require 'sportdb/readers/package'
 
 
@@ -22,76 +16,43 @@ require 'sportdb/readers/package'
 ##
 ##  add convenience shortcut helpers
 module SportDb
+  def self.read_conf( path, season: nil )  ConfReader.read( path, season: season ); end
+  def self.parse_conf( txt, season: nil )  ConfReader.parse( txt, season: season ); end
+
+  ### todo/check: add alias read_matches - why? why not?
+  def self.read_match( path, season: nil )  MatchReader.read( path, season: season ); end
+  def self.parse_match( txt, season: nil )  MatchReader.parse( txt, season: season ); end
+
+  def self.read_club_props( path )  Import::ClubPropsReader.read( path ); end
+  def self.parse_club_props( txt )  Import::ClubPropsReader.parse( txt ); end
+
+  def self.parse_leagues( txt ) recs = Import::LeagueReader.parse( txt ); Import.catalog.leagues.add( recs ); end
+  def self.parse_clubs( txt )   recs = Import::ClubReader.parse( txt );   Import.catalog.clubs.add( recs ); end
 
-  ## note: sync is dry run (for lint checking)
-  def self.read_conf( path, season: nil, sync: true )
-    sync ? ConfReaderV2.read( path, season: season )
-         : ConfLinter.read( path, season: season )
-  end
-  def self.parse_conf( txt, season: nil, sync: true )
-    sync ? ConfReaderV2.parse( txt, season: season )
-         : ConfLinter.parse( txt, season: season )
-  end
-
-  def self.read_match( path, season: nil, sync: true )  ### todo/check: add alias read_matches - why? why not?
-    sync ? MatchReaderV2.read( path, season: season )
-         : MatchLinter.read( path, season: season )
-  end
-  def self.parse_match( txt, season: nil, sync: true )  ### todo/check: add alias read_matches - why? why not?
-    sync ? MatchReaderV2.parse( txt, season: season )
-         : MatchLinter.parse( txt, season: season )
-  end
-
-  def self.read_club_props( path, sync: true )
-    ## note: for now run only if sync (e.g. run with db updates)
-    SportDb::Import::ClubPropsReader.read( path )   if sync
-  end
-  def self.parse_club_props( txt, sync: true )
-    ## note: for now run only if sync (e.g. run with db updates)
-    SportDb::Import::ClubPropsReader.parse( txt )   if sync
-  end
-
-
-  def self.parse_leagues( txt )
-    recs = SportDb::Import::LeagueReader.parse( txt )
-    Import::config.leagues.add( recs )
-  end
-
-  def self.parse_clubs( txt )
-    recs = SportDb::Import::ClubReader.parse( txt )
-    Import::config.clubs.add( recs )
-  end
-
-
-  def self.read( path, season: nil, sync: true )
+
+  def self.read( path, season: nil )
     pack = if File.directory?( path )          ## if directory assume "unzipped" package
               DirPackage.new( path )
-           elsif File.file?( path ) && Datafile.match_zip( path )  ## check if file is a .zip (archive) file
+           elsif File.file?( path ) && File.extname( path ) == '.zip'   ## check if file is a .zip (archive) file
               ZipPackage.new( path )
            else                                ## no package; assume single (standalone) datafile
              nil
            end
 
     if pack
-       pack.read( season: season, sync: sync )
+       pack.read( season: season )
     else
-      if Datafile.match_conf( path )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
-        read_conf( path, season: season, sync: sync )
-      elsif Datafile.match_club_props( path )
-        read_club_props( path, sync: sync )
+      if Package.conf?( path )      ## check if datafile matches conf(iguration) naming (e.g. .conf.txt)
+        read_conf( path, season: season )
+      elsif Package.club_props?( path )
+        read_club_props( path )
       else                                ## assume "regular" match datafile
-        read_match( path, season: season, sync: sync )
+        read_match( path, season: season )
       end
     end
   end  # method read
 
 
-
-  ## (more) convenience helpers for lint(ing)
-  def self.lint( path, season: nil )        read( path, season: season, sync: false ); end
-  def self.lint_conf( path, season: nil )   read_conf( path, season: season, sync: false ); end
-  def self.lint_match( path, season: nil )  read_match( path, season: season, sync: false ); end
-
 end # module SportDb
 
 

data/lib/sportdb/readers/conf_reader.rb

@@ -3,86 +3,98 @@
 module SportDb
 
 
-class ConfReaderV2    ## todo/check: rename to EventsReaderV2 (use plural?) why? why not?
-
-  def self.config() Import.config; end    ## shortcut convenience helper
-
+class ConfReader    ## todo/check: rename to EventsReaderV2 (use plural?) why? why not?
 
   def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
     parse( txt, season: season )
   end
 
   def self.parse( txt, season: nil )
-    recs = LeagueOutlineReader.parse( txt, season: season )
-    pp recs
-
-    ## pass 2 - check & map; replace inline (string with record)
-    recs.each do |rec|
-      league = rec[:league]
-      clubs = []    ## convert lines to clubs
-      rec[:lines].each do |line|
-
-        next if line =~ /^[ -]+$/   ## skip decorative lines with dash only (e.g. ---- or - - - -) etc.
-
-        scan = StringScanner.new( line )
-
-        if scan.check( /\d{1,2}[ ]+/ )    ## entry with standaning starts with ranking e.g. 1,2,3, etc.
-          puts "  table entry >#{line}<"
-          rank = scan.scan( /\d{1,2}[ ]+/ ).strip   # note: strip trailing spaces
-
-          ## note: uses look ahead scan until we hit at least two spaces
-          ##  or the end of string  (standing records for now optional)
-          name = scan.scan_until( /(?=\s{2})|$/ )
-          if scan.eos?
-            standing = nil
-          else
-            standing = scan.rest.strip   # note: strip leading and trailing spaces
-          end
-          puts "   rank: >#{rank}<, name: >#{name}<, standing: >#{standing}<"
+    new( txt ).parse( season: season )
+  end
 
-          ## note: rank and standing gets ignored (not used) for now
+
+  include Logging
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+  def parse( season: nil )
+    secs = LeagueOutlineReader.parse( @txt, season: season )
+    pp secs
+
+    ## pass 1 - check & map; replace inline (string with record)
+    secs.each do |sec|    # sec(tion)s
+
+      conf = ConfParser.parse( sec[:lines] )
+
+      league = sec[:league]
+      teams = []    ## convert lines to teams
+
+      if league.clubs?
+        if league.intl?
+          conf.each do |name, rec|
+            country_key = rec[:country]
+            teams << catalog.clubs.find_by!( name: name,
+                                            country: country_key )
+          end
         else
-          ## assume club is full line
-          name = line
+          conf.each do |name, _|
+            ## note: rank and standing gets ignored (not used) for now
+            teams << catalog.clubs.find_by!( name: name,
+                                            country: league.country )
+          end
+        end
+      else   ### assume national teams
+        conf.each do |name, _|
+          ## note: rank and standing gets ignored (not used) for now
+          teams << catalog.national_teams.find!( name )
         end
-
-        clubs << config.clubs.find_by( name: name, country: league.country )
       end
 
-      rec[:clubs] = clubs
-      rec.delete( :lines )   ## remove lines entry
+
+      sec[:teams] = teams
+
+      sec.delete( :lines )   ## remove lines entry
     end
 
-    ## pass 3 - import (insert/update) into db
-    recs.each do |rec|
-       league = Sync::League.find_or_create( rec[:league] )
-       season = Sync::Season.find_or_create( rec[:season] )
 
+    ## pass 2 - import (insert/update) into db
+    secs.each do |sec|   # sec(tion)s
+       ## todo/fix: always return Season struct record in LeagueReader - why? why not?
+       event_rec  = Sync::Event.find_or_create_by( league: sec[:league],
+                                                   season: sec[:season] )
 
-       event  = Sync::Event.find_or_create( league: league, season: season )
-       if rec[:stage]
-         stage = Sync::Stage.find_or_create( rec[:stage], event: event )
-       else
-         stage = nil
-       end
+       stage_rec  = if sec[:stage]
+                      Sync::Stage.find_or_create( sec[:stage], event: event_rec )
+                    else
+                      nil
+                    end
 
+       ## todo/fix: check if all teams are unique
+       ##   check if uniq works for club/national_team record (struct) - yes,no ??
+       teams = sec[:teams]
+       teams = teams.uniq
 
-       rec[:clubs].each do |club_rec|
-         club = Sync::Club.find_or_create( club_rec )
+       ## add to database
+       team_recs     =  stage_rec ? stage_rec.teams    : event_rec.teams
+       team_ids      =  stage_rec ? stage_rec.team_ids : event_rec.team_ids
+
+       new_team_recs = Sync::Team.find_or_create( teams )
+       new_team_recs.each do |team_rec|
          ## add teams to event
-         ##   todo/fix: check if team is alreay included?
-         ##    or clear/destroy_all first!!!
-         if stage
-           stage.teams << club
-         else
-           event.teams << club
-         end
+         ##   for now check if team is alreay included
+         ##   todo/fix: clear/destroy_all first - why? why not!!!
+         team_recs << team_rec    unless team_ids.include?( team_rec.id )
        end
     end
 
-    recs
+    true   ## todo/fix: return true/false or something
   end # method read
 
-end # class ConfReaderV2
+  def catalog() Import.catalog; end  ## shortcut convenience helper
+
+end # class ConfReader
 end # module SportDb