sportdb-formats 1.1.0 → 1.1.5

data/lib/sportdb/formats/event/event_reader.rb

@@ -0,0 +1,183 @@
+
+module SportDb
+module Import
+
+
+class EventInfo
+    ##  "high level" info (summary) about event  (like a "wikipedia infobox")
+    ##    use for checking dataset imports; lets you check e.g.
+    ##    - dates within range
+    ##    - number of teams e.g. 20
+    ##    - matches played e.g. 380
+    ##    - goals scored e.g. 937
+    ##    etc.
+
+    attr_reader :league,
+                :season,
+                :teams,
+                :matches,
+                :goals,
+                :start_date,
+                :end_date
+
+    def initialize( league:, season:,
+                    start_date: nil, end_date: nil,
+                    teams:   nil,
+                    matches: nil,
+                    goals:   nil )
+
+      @league      = league
+      @season      = season
+
+      @start_date  = start_date
+      @end_date    = end_date
+
+      @teams       = teams    ## todo/check: rename/use teams_count ??
+      @matches     = matches  ## todo/check: rename/use match_count ??
+      @goals       = goals
+    end
+
+    def include?( date )
+       ## todo/fix: add options e.g.
+       ##  - add delta/off_by_one or such?
+       ##  - add strict (for) only return true if date range (really) defined (no generic auto-rules)
+
+      ### note: for now allow off by one error (via timezone/local time errors)
+      ##    todo/fix: issue warning if off by one!!!!
+      if @start_date && @end_date
+        date >= (@start_date-1) &&
+        date <= (@end_date+1)
+      else
+        if @season.year?
+           # assume generic rule
+           ## same year e.g. Jan 1 - Dec 31; always true for now
+           date.year == @season.start_year
+        else
+           # assume generic rule
+           ##  July 1 - June 30 (Y+1)
+           ##  - todo/check -start for some countries/leagues in June 1 or August 1 ????
+           date >= Date.new( @season.start_year, 7, 1 ) &&
+           date <= Date.new( @season.end_year, 6, 30 )
+        end
+      end
+    end  # method include?
+    alias_method :between?, :include?
+  end # class EventInfo
+
+
+  class EventInfoReader
+    def catalog() Import.catalog; end
+
+
+    def self.read( path )
+      txt = File.open( path, 'r:utf-8') {|f| f.read }
+      new( txt ).parse
+    end
+
+    def self.parse( txt )
+      new( txt ).parse
+    end
+
+    def initialize( txt )
+      @txt = txt
+    end
+
+    def parse
+      recs = []
+
+      parse_csv( @txt ).each do |row|
+        league_col = row['League']
+        season_col = row['Season'] || row['Year']
+        dates_col  = row['Dates']
+
+        season = Import::Season.parse( season_col )
+        league = catalog.leagues.find!( league_col )
+
+
+        dates = []
+        if dates_col.nil? || dates_col.empty?
+           ## do nothing; no dates - keep dates array empty
+        else
+          ## squish spaces
+          dates_col = dates_col.gsub( /[ ]{2,}/, ' ' )  ## squish/fold spaces
+
+          puts "#{league.name} (#{league.key}) | #{season.key} | #{dates_col}"
+
+          ### todo/check:  check what parts "Aug 15" return ???
+          ###                       short form for "Aug 15 -" - works?
+
+          ## todo/fix!!! - check EventInfo.include?
+          ##    now allow dates with only start_date too!! (WITHOUT end_date)
+          parts = dates_col.split( /[ ]*[–-][ ]*/ )
+          if parts.size == 1
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+          elsif parts.size == 2
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            dates << DateFormats.parse( parts[1], start: Date.new( season.end_year ? season.end_year : season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+
+            ## assert/check if period is less than 365 days for now
+            diff = dates[1].to_date.jd - dates[0].to_date.jd
+            puts "#{diff}d"
+            if diff > 365
+              puts "!! ERROR - date range / period assertion failed; expected diff < 365 days"
+              exit 1
+            end
+          else
+            puts "!! ERRROR - expected data range / period - one or two dates; got #{parts.size}:"
+            pp dates_col
+            pp parts
+            exit 1
+          end
+        end
+
+
+        teams_col    = row['Clubs'] || row['Teams']
+        goals_col    = row['Goals']
+
+        ## note: remove (and allow) all non-digits e.g. 370 goals, 20 clubs, etc.
+        teams_col    = teams_col.gsub( /[^0-9]/, '' )    if teams_col
+        goals_col    = goals_col.gsub( /[^0-9]/, '' )    if goals_col
+
+        teams  =   (teams_col.nil? || teams_col.empty?) ? nil : teams_col.to_i
+        goals  =   (goals_col.nil? || goals_col.empty?) ? nil : goals_col.to_i
+
+        matches_col  = row['Matches']
+        ## note: support additions in matches (played) e.g.
+        #   132 + 63 Play-off-Spiele
+        matches_col   = matches_col.gsub( /[^0-9+]/, '' )  if matches_col
+
+        matches =  if matches_col.nil? || matches_col.empty?
+                     nil
+                   else
+                     if matches_col.index( '+' )  ### check for calculations
+                        ## note: for now only supports additions
+                        matches_col.split( '+' ).reduce( 0 ) do |sum,str|
+                          sum + str.to_i
+                        end
+                     else            ## assume single (integer) number
+                        matches_col.to_i
+                     end
+                  end
+
+        rec = EventInfo.new( league:     league,
+                             season:     season,
+                             start_date: dates[0],
+                             end_date:   dates[1],
+                             teams:      teams,
+                             matches:    matches,
+                             goals:      goals
+                           )
+        recs << rec
+      end  # each row
+      recs
+    end  # method parse
+  end  # class EventInfoReader
+
+
+end ## module Import
+end ## module SportDb
+

data/lib/sportdb/formats/league/league_index.rb

@@ -95,36 +95,40 @@ class LeagueIndex
   end # method add
 
 
+  ## helper to always convert (possible) country key to existing country record
+  ##  todo: make private - why? why not?
+  def country( country )
+    if country.is_a?( String ) || country.is_a?( Symbol )
+      ## note:  use own "global" countries index setting for ClubIndex - why? why not?
+      rec = catalog.countries.find( country.to_s )
+      if rec.nil?
+        puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
+        exit 1
+      end
+      rec
+    else
+      country  ## (re)use country struct - no need to run lookup again
+    end
+  end
+
+
   def match( name )
-    ## todo/check: return empty array if no match!!! and NOT nil (add || []) - why? why not?
+    ## note: returns empty array if no match and NOT nil
     name = normalize( name )
-    @leagues_by_name[ name ]
+    @leagues_by_name[ name ] || []
   end
 
-
   def match_by( name:, country: )
     ## note: match must for now always include name
     m = match( name )
-    if m    ## filter by country
+    if country    ## filter by country
       ## note: country assumes / allows the country key or fifa code for now
-
       ## note: allow passing in of country struct too
-      country_rec = if country.is_a?( Country )
-                       country   ## (re)use country struct - no need to run lookup again
-                    else
-                       ## note:  use own "global" countries index setting for ClubIndex - why? why not?
-                       rec = catalog.countries.find( country )
-                       if rec.nil?
-                         puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
-                         exit 1
-                       end
-                       rec
-                    end
+      country_rec = country( country )
 
       ## note: also skip international leagues & cups (e.g. champions league etc.) for now - why? why not?
       m = m.select { |league| league.country &&
                               league.country.key == country_rec.key }
-      m = nil   if m.empty?     ## note: reset to nil if no more matches
     end
     m
   end
@@ -144,7 +148,7 @@ class LeagueIndex
     m = match( name )
     # pp m
 
-    if m.nil?
+    if m.empty?
       ## fall through/do nothing
     elsif m.size > 1
       puts "** !!! ERROR - ambigious league name; too many leagues (#{m.size}) found:"

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -65,7 +65,7 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
        filtered_secs = []
        filter = norm_seasons( season )
        secs.each do |sec|
-         if filter.include?( Import::Season.new( sec[:season] ).key )
+         if filter.include?( Season.parse( sec[:season] ).key )
            filtered_secs << sec
          else
            puts "  skipping season >#{sec[:season]}< NOT matched by filter"
@@ -76,7 +76,7 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
 
     ## pass 3 - check & map; replace inline (string with data struct record)
     secs.each do |sec|
-      sec[:season] = Import::Season.new( sec[:season ] )
+      sec[:season] = Season.parse( sec[:season ] )
       sec[:league] = catalog.leagues.find!( sec[:league] )
 
       check_stage( sec[:stage] )   if sec[:stage]   ## note: only check for now (no remapping etc.)
@@ -100,14 +100,18 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
          )
             $}x
 
+
   def norm_seasons( season_or_seasons )     ## todo/check: add alias norm_seasons - why? why not?
-      seasons = if season_or_seasons.is_a?( String )   ## wrap in array
+
+      seasons = if season_or_seasons.is_a?( Array )  # is it an array already
+                  season_or_seasons
+                elsif season_or_seasons.is_a?( Range )  # e.g. Season(1999)..Season(2001) or such
+                  season_or_seasons.to_a
+                else  ## assume - single entry - wrap in array
                   [season_or_seasons]
-                else  ## assume it's an array already
-                   season_or_seasons
                 end
 
-      seasons.map { |season| Import::Season.new( season ).key }
+      seasons.map { |season| Season( season ).key }
   end
 
 
@@ -127,7 +131,7 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     'Regular Season',
     'Regular Stage',
     'Championship Round',
-    'Championship Playoff',
+    'Championship Playoff',  # or Championship play-off
     'Relegation Round',
     'Relegation Playoff',
     'Play-offs',
@@ -140,6 +144,19 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     'EL Play-off',
     'Europa League Play-off',
     'Europa-League-Play-offs',
+    'Europa League Finals',
+    'Playoffs - Championship',
+    'Playoffs - Europa League',
+    'Playoffs - Europa League - Finals',
+    'Playoffs - Relegation',
+    'Playoffs - Challenger',
+    'Finals',
+
+    'Apertura',
+    'Apertura - Liguilla',
+    'Clausura',
+    'Clausura - Liguilla',
+
   ].map {|name| name.downcase.gsub( /[^a-z]/, '' ) }
 
 

data/lib/sportdb/formats/league/league_reader.rb

@@ -118,12 +118,18 @@ def parse
               alt_names_auto << "#{country.code}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
             end
             alt_names_auto << "#{country.name} #{league_key}"  if league_key =~ /^[0-9]+$/   ## if all numeric e.g. add Austria 1 etc.
+
+            ## auto-add with country prepended
+            ##   e.g. England Premier League, Austria Bundesliga etc.
+            ##  todo/check: also add variants with country alt name if present!!!
+            ##  todo/check: exclude cups or such from country + league name auto-add - why? why not?
+            alt_names_auto << "#{country.name} #{league_name}"
           else   ## assume int'l (no country) e.g. champions league, etc.
             ## only auto-add key (e.g. CL, EL, etc.)
             alt_names_auto << league_key.upcase.gsub('.', ' ')   ## note: no country code (prefix/leading) used
           end
 
-          pp alt_names_auto
+          ## pp alt_names_auto
 
           ## prepend country key/code if country present
           ##   todo/fix: only auto-prepend country if key/code start with a number (level) or incl. cup

data/lib/sportdb/formats/match/match_parser.rb

@@ -302,6 +302,11 @@ class MatchParser   ## simple match parser for team match schedules
     ScoreFormats.find!( line )
   end
 
+  def find_status!( line )
+    StatusParser.find!( line )
+  end
+
+
   def try_parse_game( line )
     # note: clone line; for possible test do NOT modify in place for now
     # note: returns true if parsed, false if no match
@@ -329,6 +334,10 @@ class MatchParser   ## simple match parser for team match schedules
       return false
     end
 
+    ## find (optional) match status e.g. [abandoned] or [replay] or [awarded]
+    ##                                   or [cancelled] or [postponed] etc.
+    status = find_status!( line )   ## todo/check: allow match status also in geo part (e.g. after @) - why? why not?
+
     ## pos = find_game_pos!( line )
 
       date      = find_date!( line, start: @start )
@@ -353,20 +362,23 @@ class MatchParser   ## simple match parser for team match schedules
     if @last_round
       round = @last_round
     else
-      ## find (first) matching round by date
-      @rounds.values.each do |round_rec|
-        ## note: convert date to date only (no time) with to_date!!!
-        if (round_rec.start_date && round_rec.end_date) &&
-           (date.to_date >= round_rec.start_date &&
-            date.to_date <= round_rec.end_date)
-          round = round_rec
-          break
+      ## find (first) matching round by date if rounds / matchdays defined
+      ##   if not rounds / matchdays defined - YES, allow matches WITHOUT rounds!!!
+      if @rounds.size > 0
+        @rounds.values.each do |round_rec|
+          ## note: convert date to date only (no time) with to_date!!!
+          if (round_rec.start_date && round_rec.end_date) &&
+             (date.to_date >= round_rec.start_date &&
+             date.to_date <= round_rec.end_date)
+            round = round_rec
+            break
+          end
+        end
+        if round.nil?
+          puts "!! ERROR - no matching round found for match date:"
+          pp date
+          exit 1
         end
-      end
-      if round.nil?
-        puts "!! ERROR - no matching round found for match date:"
-        pp date
-        exit 1
       end
     end
 
@@ -380,8 +392,8 @@ class MatchParser   ## simple match parser for team match schedules
                                    team2:   team2,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team2.name)
                                    score:   score,
                                    round:   round       ? round.name       : nil,   ## note: for now always use string (assume unique canonical name for event)
-                                   group:   @last_group ? @last_group.name : nil )  ## note: for now always use string (assume unique canonical name for event)
-
+                                   group:   @last_group ? @last_group.name : nil,   ## note: for now always use string (assume unique canonical name for event)
+                                   status:  status )
     ### todo: cache team lookups in hash?
 
 =begin
@@ -563,12 +575,29 @@ class MatchParser   ## simple match parser for team match schedules
 
     if date && team1.nil? && team2.nil?
       logger.debug( "date header line found: >#{line}<")
-      logger.debug( "    date: #{date}")
+      logger.debug( "    date: #{date} with start: #{@start}")
 
       @last_date = date   # keep a reference for later use
-      return true
+
+      ###  quick "corona" hack - support seasons going beyond 12 month (see swiss league 2019/20 and others!!)
+      ##    find a better way??
+      ##  set @start date to full year (e.g. 1.1.) if date.year  is @start.year+1
+      ##   todo/fix: add to linter to check for chronological dates!! - warn if NOT chronological
+      ###  todo/check: just turn on for 2019/20 season or always? why? why not?
+
+      ## todo/fix: add switch back to old @start_org
+      ##   if year is date.year == @start.year-1    -- possible when full date with year set!!!
+      if @start.month != 1
+         if date.year == @start.year+1
+           logger.debug( "!! hack - extending start date to full (next/end) year; assumes all dates are chronologigal - always moving forward" )
+           @start_org = @start   ## keep a copy of the original (old) start date - why? why not? - not used for now
+           @start = Date.new( @start.year+1, 1, 1 )
+         end
+      end
+
+      true
     else
-      return false
+      false
     end
   end
 

data/lib/sportdb/formats/package.rb

@@ -13,12 +13,22 @@ module SportDb
     ## leagues.txt or leagues_en.txt
     ##   remove support for en.leagues.txt - why? why not?
     LEAGUES_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
-       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.wiki.txt
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.leagues.txt
         leagues
           (?:_[a-z0-9_-]+)?
         \.txt$
     }x
 
+    ## seasons.txt or seasons_en.txt
+    ##   remove support for br.seasons.txt - why? why not?
+    SEASONS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.seasons.txt
+        seasons
+          (?:_[a-z0-9_-]+)?
+        \.txt$
+    }x
+
+
     ## clubs.txt or clubs_en.txt
     ##   remove support for en.clubs.txt - why? why not?
     CLUBS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -35,12 +45,22 @@ module SportDb
        \.wiki\.txt$
     }x
 
-    CLUB_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+    ## todo/fix: rename to CLUBS too e.g. CLUBS_PROPS to reflect filename - why? why not?
+    CLUBS_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.props.txt
         clubs
           (?:_[a-z0-9_-]+)?
         \.props\.txt$
     }x
+    CLUB_PROPS_RE = CLUBS_PROPS_RE  ## add alias for now (fix later - why? why not?)
+
+
+    CLUBS_HISTORY_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+      (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.history.txt
+        clubs
+          (?:_[a-z0-9_-]+)?
+        \.history\.txt$
+    }x
 
     ##  teams.txt or teams_history.txt
     TEAMS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -49,6 +69,8 @@ module SportDb
                       \.txt$
     }x
 
+
+    ###  todo/fix: change SEASON_RE to SEASON_KEY_RE (avoid confusion w/ SEASONS_RE for datafile?) - why? why not? !!!!!!!
     ### season folder:
     ##            e.g. /2019-20   or
     ##  year-only e.g. /2019      or
@@ -73,6 +95,10 @@ module SportDb
                        /[a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
                     }x
 
+    ### add "generic" pattern to find all csv datafiles
+    CSV_RE = %r{ (?: ^|/ )
+                   [a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
+               }x
 
 
     ## move class-level "static" finders to DirPackage (do NOT work for now for zip packages) - why? why not?
@@ -96,16 +122,22 @@ module SportDb
    def self.find_teams( path, pattern: TEAMS_RE )  find( path, pattern ); end
    def self.match_teams( path )  TEAMS_RE.match( path ); end
 
-   def self.find_clubs( path, pattern: CLUBS_RE )            find( path, pattern ); end
-   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )  find( path, pattern ); end
+   def self.find_clubs( path, pattern: CLUBS_RE )                 find( path, pattern ); end
+   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )       find( path, pattern ); end
+   def self.find_clubs_history( path, pattern: CLUBS_HISTORY_RE ) find( path, pattern ); end
 
-   def self.match_clubs( path )       CLUBS_RE.match( path ); end
-   def self.match_clubs_wiki( path )  CLUBS_WIKI_RE.match( path ); end
-   def self.match_club_props( path, pattern: CLUB_PROPS_RE ) pattern.match( path ); end
+   def self.match_clubs( path )         CLUBS_RE.match( path ); end
+   def self.match_clubs_wiki( path )    CLUBS_WIKI_RE.match( path ); end
+   def self.match_clubs_history( path ) CLUBS_HISTORY_RE.match( path); end
+   def self.match_clubs_props( path, pattern: CLUBS_PROPS_RE ) pattern.match( path ); end
 
    def self.find_leagues( path, pattern: LEAGUES_RE )  find( path, pattern ); end
    def self.match_leagues( path )  LEAGUES_RE.match( path ); end
 
+   def self.find_seasons( path, pattern: SEASONS_RE )  find( path, pattern ); end
+   def self.match_seasons( path )  SEASONS_RE.match( path ); end
+
+
    def self.find_conf( path, pattern: CONF_RE )  find( path, pattern ); end
    def self.match_conf( path )  CONF_RE.match( path ); end
 
@@ -118,6 +150,7 @@ module SportDb
    end
    ## add match_match and match_match_csv  - why? why not?
 
+
    class << self
      alias_method :match_teams?, :match_teams
      alias_method :teams?,       :match_teams
@@ -128,12 +161,21 @@ module SportDb
      alias_method :match_clubs_wiki?, :match_clubs_wiki
      alias_method :clubs_wiki?,       :match_clubs_wiki
 
-     alias_method :match_club_props?, :match_club_props
-     alias_method :club_props?,       :match_club_props
+     alias_method :match_clubs_history?, :match_clubs_history
+     alias_method :clubs_history?,       :match_clubs_history
+
+     alias_method :match_club_props,      :match_clubs_props
+     alias_method :match_club_props?,     :match_clubs_props
+     alias_method :club_props?,           :match_clubs_props
+     alias_method :match_clubs_props?,    :match_clubs_props
+     alias_method :clubs_props?,          :match_clubs_props
 
      alias_method :match_leagues?, :match_leagues
      alias_method :leagues?,       :match_leagues
 
+     alias_method :match_seasons?, :match_seasons
+     alias_method :seasons?,       :match_seasons
+
      alias_method :match_conf?, :match_conf
      alias_method :conf?,       :match_conf
    end
@@ -212,11 +254,17 @@ module SportDb
       end
     end
     def each_match_csv( &blk )  each( pattern: MATCH_CSV_RE, &blk ); end
+    def each_csv( &blk )        each( pattern: CSV_RE, &blk );       end
+
     def each_club_props( &blk ) each( pattern: CLUB_PROPS_RE, &blk ); end
 
     def each_leagues( &blk )    each( pattern: LEAGUES_RE, &blk ); end
     def each_clubs( &blk )      each( pattern: CLUBS_RE, &blk ); end
     def each_clubs_wiki( &blk ) each( pattern: CLUBS_WIKI_RE, &blk ); end
+    def each_clubs_history( &blk )  each( pattern: CLUBS_HISTORY_RE, &blk ); end
+
+    def each_seasons( &blk )    each( pattern: SEASONS_RE, &blk ); end
+
 
     ## return all match datafile entries
     def match( format: 'txt' )
@@ -287,13 +335,13 @@ module SportDb
       ##             filter.skip? filter.include? ( season_sason_key )?
       ##             fiteer.before?( season_key )  etc.
       ##              find some good method names!!!!
-      season_start = start ? Import::Season.new( start ) : nil
+      season_start = start ? Season( start ) : nil
 
       h = {}
       match( format: format ).each do |entry|
         ## note: assume last directory in datafile path is the season part/key
         season_q = File.basename( File.dirname( entry.name ))
-        season   = Import::Season.new( season_q )  ## normalize season
+        season   = Season.parse( season_q )  ## normalize season
 
         ## skip if start season before this season
         next if season_start && season_start.start_year > season.start_year