sportdb-formats 1.0.4 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ccd241256a964b430d9c9aa515901641ed5a0e2
-  data.tar.gz: 1d997241f388000d7b2c19e26a92c9ba39c8e172
+  metadata.gz: 89c3ae173bed0c7a68f30a897e4cf34da8d72d6e
+  data.tar.gz: f3898e8b03177b62075cced2a90be2502555bf63
 SHA512:
-  metadata.gz: 1d7d1e6b71009ec598f01f59ead597269ea3c4b48bc3e2f5be0a7257879114f2b3110937dfb23c153962cd7dd0a61e23d0050a8eb40c6652acf34a8a1720071c
-  data.tar.gz: 392a8eaf8c0d1f14b021a45ca39ce491f8edceac0cf9ba33ffde920e638a2cf1fd2adf836e596dc2100c233f5ca1fe8aadf47526fb1837de855acaf84974b19a
+  metadata.gz: 4c938af88f6dd86f1736532c9d8271a3e56f1769b955a31ce6d84a0b1fcf5a6eec7cfa42e0c019bd7072351d777a1988d6637c4297bee0dad9592bb8ec7450e8
+  data.tar.gz: a80df68f64e13eeb9a85931e2d2dd4381f204f94a4ea9254c89b4b28346db4c1542321aed4fd123aef5ddedc7e4cd2c08731dd340326fae5dc94b57fa6797835

data/Manifest.txt

@@ -8,6 +8,8 @@ lib/sportdb/formats/country/country_index.rb
 lib/sportdb/formats/country/country_reader.rb
 lib/sportdb/formats/datafile.rb
 lib/sportdb/formats/datafile_package.rb
+lib/sportdb/formats/event/event_index.rb
+lib/sportdb/formats/event/event_reader.rb
 lib/sportdb/formats/goals.rb
 lib/sportdb/formats/league/league_index.rb
 lib/sportdb/formats/league/league_outline_reader.rb
@@ -63,10 +65,12 @@ test/test_match.rb
 test/test_match_auto.rb
 test/test_match_auto_champs.rb
 test/test_match_auto_euro.rb
+test/test_match_auto_relegation.rb
 test/test_match_auto_worldcup.rb
 test/test_match_champs.rb
 test/test_match_eng.rb
 test/test_match_euro.rb
+test/test_match_start_date.rb
 test/test_match_worldcup.rb
 test/test_name_helper.rb
 test/test_outline_reader.rb

data/lib/sportdb/formats.rb

@@ -136,6 +136,20 @@ end   # module Import
 end   # module SportDb
 
 
+require 'sportdb/formats/event/event_reader'
+require 'sportdb/formats/event/event_index'
+
+## add convenience helper
+module SportDb
+module Import
+class EventInfo
+  def self.read( path )  EventInfoReader.read( path ); end
+  def self.parse( txt )  EventInfoReader.parse( txt ); end
+end   # class EventInfo
+end   # module Import
+end   # module SportDb
+
+
 
 
 

data/lib/sportdb/formats/country/country_reader.rb

@@ -57,7 +57,7 @@ def parse
 
          last_country = country = Country.new( name: "#{name} (-#{year})",
                                                code: code )
-         country.alt_names << name    ## note: do NOT forget - add name without year to alt_names!!!
+         ## country.alt_names << name    ## note: for now do NOT add name without year to alt_names - gets auto-add by index!!!
 
          countries << country
          ## todo/fix: add reference to country today (in parts[1] !!!!)

data/lib/sportdb/formats/event/event_index.rb

@@ -0,0 +1,143 @@
+module SportDb
+module Import
+
+
+class EventIndex
+
+    def self.build( path )
+      datafiles = Package.find_seasons( path )
+
+      puts
+      puts "#{datafiles.size} seasons datafile(s):"
+      pp datafiles
+
+      index = new
+      datafiles.each do |datafile|
+        recs = EventInfoReader.read( datafile )
+        # pp recs
+
+        index.add( recs )
+      end
+
+      index
+    end
+
+
+    attr_reader :events
+    def initialize
+      @events  = []
+      @leagues = {}
+    end
+
+    def add( recs )
+      @events += recs  ## add to "linear" records
+
+      recs.each do |rec|
+        league = rec.league
+        season = rec.season
+
+        seasons = @leagues[ league.key ] ||= {}
+        seasons[season.key] = rec
+      end
+      ## build search index by leagues (and season)
+    end
+
+    def find_by( league:, season: )
+      league_key = league.is_a?( String ) ? league : league.key
+      season_key = season.is_a?( String ) ? season : season.key
+
+      seasons = @leagues[ league_key ]
+      if seasons
+        seasons[ season_key ]
+      else
+        nil
+      end
+    end # method find_by
+end  ## class EventIndex
+
+
+
+class SeasonIndex
+    def initialize( *args )
+       @leagues = {}   ## use a league hash by years for now; change later
+
+       if args.size == 1 && args[0].is_a?( EventIndex )
+         ## convenience setup/hookup
+         ##  (auto-)add all events from event index
+         add( args[0].events )
+       else
+         pp args
+         raise ArgumentError.new( 'unsupported arguments' )
+       end
+    end
+
+    def add( recs )
+      ## use a lookup index by year for now
+      ##  todo - find something better/more generic for searching/matching date periods!!!
+      recs.each do |rec|
+        league = rec.league
+        season = rec.season
+
+        years = @leagues[ league.key ] ||= {}
+        if season.year?
+          years[season.start_year] ||= []
+          years[season.start_year] << rec
+        else
+          years[season.start_year] ||= []
+          years[season.end_year]   ||= []
+          years[season.start_year] << rec
+          years[season.end_year]   << rec
+        end
+      end
+    end # method add
+
+    def find_by( date:, league: )
+       date = Date.strptime( date, '%Y-%m-%d' )   if date.is_a?( String )
+       league_key = league.is_a?( String ) ? league : league.key
+
+       years = @leagues[ league_key ]
+       if years
+           year = years[ date.year ]
+           if year
+               season_key = nil
+               year.each do |event|
+                 ##  todo/check: rename/use between? instead of include? - why? why not?
+                 if event.include?( date )
+                   season_key = event.season.key
+                   break
+                 end
+               end
+               if season_key.nil?
+                 puts "!! WARN: date >#{date}< out-of-seasons for year #{date.year} in league #{league_key}:"
+                 year.each do |event|
+                   puts "  #{event.season.key} |  #{event.start_date} - #{event.end_date}"
+                 end
+                 ## retry again and pick season with "overflow" at the end (date is great end_date)
+                 year.each do |event|
+                   if date > event.end_date
+                     diff_in_days = date.to_date.jd - event.end_date.to_date.jd
+                     puts "    +#{diff_in_days} days - adding overflow to #{event.season.key} ending on #{event.end_date} ++ #{date}"
+                     season_key = event.season.key
+                     break
+                   end
+                 end
+                 ## exit now for sure - if still empty!!!!
+                 if season_key.nil?
+                   puts "!! ERROR: CANNOT auto-fix / (auto-)append date at the end of an event; check season setup - sorry"
+                   exit 1
+                 end
+               end
+               season_key
+           else
+             nil   ## no year defined / found for league
+           end
+       else
+          nil   ## no league defined / found
+       end
+   end  # method find
+
+end # class SeasonIndex
+
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/formats/event/event_reader.rb

@@ -0,0 +1,183 @@
+
+module SportDb
+module Import
+
+
+class EventInfo
+    ##  "high level" info (summary) about event  (like a "wikipedia infobox")
+    ##    use for checking dataset imports; lets you check e.g.
+    ##    - dates within range
+    ##    - number of teams e.g. 20
+    ##    - matches played e.g. 380
+    ##    - goals scored e.g. 937
+    ##    etc.
+
+    attr_reader :league,
+                :season,
+                :teams,
+                :matches,
+                :goals,
+                :start_date,
+                :end_date
+
+    def initialize( league:, season:,
+                    start_date: nil, end_date: nil,
+                    teams:   nil,
+                    matches: nil,
+                    goals:   nil )
+
+      @league      = league
+      @season      = season
+
+      @start_date  = start_date
+      @end_date    = end_date
+
+      @teams       = teams    ## todo/check: rename/use teams_count ??
+      @matches     = matches  ## todo/check: rename/use match_count ??
+      @goals       = goals
+    end
+
+    def include?( date )
+       ## todo/fix: add options e.g.
+       ##  - add delta/off_by_one or such?
+       ##  - add strict (for) only return true if date range (really) defined (no generic auto-rules)
+
+      ### note: for now allow off by one error (via timezone/local time errors)
+      ##    todo/fix: issue warning if off by one!!!!
+      if @start_date && @end_date
+        date >= (@start_date-1) &&
+        date <= (@end_date+1)
+      else
+        if @season.year?
+           # assume generic rule
+           ## same year e.g. Jan 1 - Dec 31; always true for now
+           date.year == @season.start_year
+        else
+           # assume generic rule
+           ##  July 1 - June 30 (Y+1)
+           ##  - todo/check -start for some countries/leagues in June 1 or August 1 ????
+           date >= Date.new( @season.start_year, 7, 1 ) &&
+           date <= Date.new( @season.end_year, 6, 30 )
+        end
+      end
+    end  # method include?
+    alias_method :between?, :include?
+  end # class EventInfo
+
+
+  class EventInfoReader
+    def catalog() Import.catalog; end
+
+
+    def self.read( path )
+      txt = File.open( path, 'r:utf-8') {|f| f.read }
+      new( txt ).parse
+    end
+
+    def self.parse( txt )
+      new( txt ).parse
+    end
+
+    def initialize( txt )
+      @txt = txt
+    end
+
+    def parse
+      recs = []
+
+      parse_csv( @txt ).each do |row|
+        league_col = row['League']
+        season_col = row['Season'] || row['Year']
+        dates_col  = row['Dates']
+
+        season = Import::Season.new( season_col )
+        league = catalog.leagues.find!( league_col )
+
+
+        dates = []
+        if dates_col.nil? || dates_col.empty?
+           ## do nothing; no dates - keep dates array empty
+        else
+          ## squish spaces
+          dates_col = dates_col.gsub( /[ ]{2,}/, ' ' )  ## squish/fold spaces
+
+          puts "#{league.name} (#{league.key}) | #{season.key} | #{dates_col}"
+
+          ### todo/check:  check what parts "Aug 15" return ???
+          ###                       short form for "Aug 15 -" - works?
+
+          ## todo/fix!!! - check EventInfo.include?
+          ##    now allow dates with only start_date too!! (WITHOUT end_date)
+          parts = dates_col.split( /[ ]*[–-][ ]*/ )
+          if parts.size == 1
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+          elsif parts.size == 2
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            dates << DateFormats.parse( parts[1], start: Date.new( season.end_year ? season.end_year : season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+
+            ## assert/check if period is less than 365 days for now
+            diff = dates[1].to_date.jd - dates[0].to_date.jd
+            puts "#{diff}d"
+            if diff > 365
+              puts "!! ERROR - date range / period assertion failed; expected diff < 365 days"
+              exit 1
+            end
+          else
+            puts "!! ERRROR - expected data range / period - one or two dates; got #{parts.size}:"
+            pp dates_col
+            pp parts
+            exit 1
+          end
+        end
+
+
+        teams_col    = row['Clubs'] || row['Teams']
+        goals_col    = row['Goals']
+
+        ## note: remove (and allow) all non-digits e.g. 370 goals, 20 clubs, etc.
+        teams_col    = teams_col.gsub( /[^0-9]/, '' )    if teams_col
+        goals_col    = goals_col.gsub( /[^0-9]/, '' )    if goals_col
+
+        teams  =   (teams_col.nil? || teams_col.empty?) ? nil : teams_col.to_i
+        goals  =   (goals_col.nil? || goals_col.empty?) ? nil : goals_col.to_i
+
+        matches_col  = row['Matches']
+        ## note: support additions in matches (played) e.g.
+        #   132 + 63 Play-off-Spiele
+        matches_col   = matches_col.gsub( /[^0-9+]/, '' )  if matches_col
+
+        matches =  if matches_col.nil? || matches_col.empty?
+                     nil
+                   else
+                     if matches_col.index( '+' )  ### check for calculations
+                        ## note: for now only supports additions
+                        matches_col.split( '+' ).reduce( 0 ) do |sum,str|
+                          sum + str.to_i
+                        end
+                     else            ## assume single (integer) number
+                        matches_col.to_i
+                     end
+                  end
+
+        rec = EventInfo.new( league:     league,
+                             season:     season,
+                             start_date: dates[0],
+                             end_date:   dates[1],
+                             teams:      teams,
+                             matches:    matches,
+                             goals:      goals
+                           )
+        recs << rec
+      end  # each row
+      recs
+    end  # method parse
+  end  # class EventInfoReader
+
+
+end ## module Import
+end ## module SportDb
+

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -121,14 +121,31 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     values
   end
 
-  def check_stage( name )
-    known_stages = ['regular season',
-                    'championship round',
-                    'relegation round',
-                    'play-offs'
-                   ]
 
-    if known_stages.include?( name.downcase )
+  # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
+  KNOWN_STAGES = [
+    'Regular Season',
+    'Regular Stage',
+    'Championship Round',
+    'Championship Playoff',
+    'Relegation Round',
+    'Relegation Playoff',
+    'Play-offs',
+    'Playoff Stage',
+    'Grunddurchgang',
+    'Finaldurchgang - Qualifikationsgruppe',
+    'Finaldurchgang - Qualifikation',
+    'Finaldurchgang - Meistergruppe',
+    'Finaldurchgang - Meister',
+    'EL Play-off',
+    'Europa League Play-off',
+    'Europa-League-Play-offs',
+  ].map {|name| name.downcase.gsub( /[^a-z]/, '' ) }
+
+
+  def check_stage( name )
+    # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
+    if KNOWN_STAGES.include?( name.downcase.gsub( /[^a-z]/, '' ) )
        ## everything ok
     else
       puts "** !!! ERROR - no (league) stage match found for >#{name}<, add to (builtin) stages table; sorry"

data/lib/sportdb/formats/match/mapper.rb

@@ -7,21 +7,21 @@ module SportDb
 ##   see https://github.com/textkit/textutils/blob/master/textutils/lib/textutils/title_mapper2.rb
 
 
-class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why not??
+class MapperV2      ## todo/check: rename to NameMapper ? why? why not??
 
   include Logging
 
-  attr_reader :known_titles   ## rename to mapping or mappings or just titles - why? why not?
+  attr_reader :known_names   ## rename to mapping or mappings or just names - why? why not?
 
   ########
   ##  key:      e.g. augsburg
-  ##  title:    e.g. FC Augsburg
-  ##  length (of title(!!) - not regex pattern):   e.g. 11   -- do not count dots (e.g. U.S.A. => 3 or 6) why? why not?
-  MappingStruct =  Struct.new( :key, :title, :length, :pattern)     ## todo/check: use (rename to) TitleStruct - why? why not??
+  ##  name:     e.g. FC Augsburg
+  ##  length (of name(!!) - not regex pattern):   e.g. 11   -- do not count dots (e.g. U.S.A. => 3 or 6) why? why not?
+  MappingStruct =  Struct.new( :key, :name, :length, :pattern)     ## todo/check: use (rename to) NameStruct - why? why not??
 
   ######
   ## convenience helper - (auto)build ActiveRecord-like team records/structs
-  Record = Struct.new( :key, :title, :synonyms )
+  Record = Struct.new( :key, :name, :alt_names )
   def build_records( txt_or_lines )
     recs = []
 
@@ -44,12 +44,12 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
       values = line.split( '|' )
       values = values.map { |value| value.strip }
 
-      title    = values[0]
+      name      = values[0]
       ## note: quick hack - auto-generate key, that is, remove all non-ascii chars and downcase
-      key      = title.downcase.gsub( /[^a-z]/, '' )
-      synonyms = values.size > 1 ? values[1..-1].join( '|' ) : nil
+      key       = name.downcase.gsub( /[^a-z]/, '' )
+      alt_names = values.size > 1 ? values[1..-1].join( '|' ) : nil
 
-      recs << Record.new( key, title, synonyms )
+      recs << Record.new( key, name, alt_names )
     end
     recs
   end
@@ -63,10 +63,10 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
                                                                   (records_or_mapping.is_a?( Array ) && records_or_mapping[0].is_a?( String ))
 
     ## build mapping lookup table
-    @known_titles =  if records_or_mapping.is_a?( Hash )  ## assume "custom" mapping hash table (title/name=>record)
-                        build_title_table_for_mapping( records_or_mapping )
+    @known_names =  if records_or_mapping.is_a?( Hash )  ## assume "custom" mapping hash table (name=>record)
+                        build_name_table_for_mapping( records_or_mapping )
                      else  ## assume array of records
-                        build_title_table_for_records( records_or_mapping )
+                        build_name_table_for_records( records_or_mapping )
                      end
 
     ## build lookup hash by record (e.g. team/club/etc.) key
@@ -85,9 +85,9 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
 
 
 
-  def map_titles!( line )   ## rename to just map! - why?? why not???
+  def map_names!( line )   ## rename to just map! - why?? why not???
     begin
-      found = map_title_for!( @tag, line, @known_titles )
+      found = map_name_for!( @tag, line, @known_names )
     end while found
   end
 
@@ -110,27 +110,27 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
 
 
 private
-  def build_title_table_for_mapping( mapping )
-    known_titles = []
+  def build_name_table_for_mapping( mapping )
+    known_names = []
 
-    mapping.each do |title, rec|
+    mapping.each do |name, rec|
       m = MappingStruct.new
       m.key     = rec.key
-      m.title   = title
-      m.length  = title.length
-      m.pattern = Regexp.escape( title )   ## note: just use "standard" regex escape (e.g. no extras for umlauts,accents,etc.)
+      m.name    = name
+      m.length  = name.length
+      m.pattern = Regexp.escape( name )   ## note: just use "standard" regex escape (e.g. no extras for umlauts,accents,etc.)
 
-      known_titles << m
+      known_names << m
     end
 
     ## note: sort here by length (largest goes first - best match)
-    known_titles = known_titles.sort { |l,r| r.length <=> l.length }
-    known_titles
+    known_names = known_names.sort { |l,r| r.length <=> l.length }
+    known_names
   end
 
-  def build_title_table_for_records( records )
+  def build_name_table_for_records( records )
 
-    ## build known tracks table w/ synonyms e.g.
+    ## build known tracks table w/ alt names e.g.
     #
     # [[ 'wolfsbrug', 'VfL Wolfsburg'],
     #  [ 'augsburg',  'FC Augsburg'],
@@ -138,65 +138,65 @@ private
     #  [ 'augsburg',  'Augi3' ],
     #  [ 'stuttgart', 'VfB Stuttgart']]
 
-    known_titles = []
+    known_names = []
 
     records.each_with_index do |rec,index|
 
-      title_candidates = []
-      title_candidates << rec.title
+      name_candidates = []
+      name_candidates << rec.name
 
-      title_candidates += rec.synonyms.split('|') if rec.synonyms && !rec.synonyms.empty?
+      name_candidates += rec.alt_names.split('|') if rec.alt_names && !rec.alt_names.empty?
 
 
-      ## check if title includes subtitle e.g. Grand Prix Japan (Suzuka Circuit)
-      #  make subtitle optional by adding title w/o subtitle e.g. Grand Prix Japan
+      ## check if name includes subname e.g. Grand Prix Japan (Suzuka Circuit)
+      #  make subname optional by adding name w/o subname e.g. Grand Prix Japan
 
-      titles = []
-      title_candidates.each do |t|
-        titles << t
+      names = []
+      name_candidates.each do |t|
+        names << t
         if t =~ /\(.+\)/
-          extra_title = t.gsub( /\(.+\)/, '' ) # remove/delete subtitles
+          extra_name = t.gsub( /\(.+\)/, '' ) # remove/delete subnames
           # note: strip leading n trailing withspaces too!
           #  -- todo: add squish or something if () is inline e.g. leaves two spaces?
-          extra_title.strip!
-          titles << extra_title
+          extra_name.strip!
+          names << extra_name
         end
       end
 
-      titles.each do |t|
+      names.each do |name|
         m = MappingStruct.new
         m.key     = rec.key
-        m.title   = t
-        m.length  = t.length
+        m.name    = name
+        m.length  = name.length
         ## note: escape for regex plus allow subs for special chars/accents
-        m.pattern = title_esc_regex( t )
+        m.pattern = name_esc_regex( name )
 
-        known_titles << m
+        known_names << m
       end
 
-      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{titles.join('|')}<"
+      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{names.join('|')}<"
 
       ## note: only include code field - if defined
       if rec.respond_to?(:code) && rec.code && !rec.code.empty?
         m = MappingStruct.new
         m.key     = rec.key
-        m.title   = rec.code
+        m.name    = rec.code
         m.length  = rec.code.length
         m.pattern = rec.code   ## note: use code for now as is (no variants allowed fow now)
 
-        known_titles << m
+        known_names << m
       end
     end
 
     ## note: sort here by length (largest goes first - best match)
       #  exclude code and key (key should always go last)
-    known_titles = known_titles.sort { |l,r| r.length <=> l.length }
-    known_titles
+    known_names = known_names.sort { |l,r| r.length <=> l.length }
+    known_names
   end
 
 
 
-  def map_title_for!( tag, line, mappings )
+  def map_name_for!( tag, line, mappings )
     mappings.each do |mapping|
       key     = mapping.key
       pattern = mapping.pattern
@@ -234,9 +234,9 @@ private
 
 
 ####
-# title helper cut-n-paste copy from TextUtils
+# name helper cut-n-paste copy from TextUtils
 ##  see https://github.com/textkit/textutils/blob/master/textutils/lib/textutils/helper/title_helper.rb
-def title_esc_regex( title_unescaped )
+def name_esc_regex( name_unescaped )
 
       ##  escape regex special chars e.g.
       #    . to \. and
@@ -257,16 +257,16 @@ def title_esc_regex( title_unescaped )
       # e.g. Club Atlético Colón (Santa Fe)
       # e.g. Bauer Anton (????)
 
-      ## NB: cannot use Regexp.escape! will escape space '' to '\ '
-      ## title = Regexp.escape( title_unescaped )
-      title = title_unescaped.gsub( '.', '\.' )
-      title = title.gsub( '(', '\(' )
-      title = title.gsub( ')', '\)' )
-      title = title.gsub( '?', '\?' )
-      title = title.gsub( '*', '\*' )
-      title = title.gsub( '+', '\+' )
-      title = title.gsub( '$', '\$' )
-      title = title.gsub( '^', '\^' )
+      ## note: cannot use Regexp.escape! will escape space '' to '\ '
+      ## name = Regexp.escape( name_unescaped )
+      name = name_unescaped.gsub( '.', '\.' )
+      name = name.gsub( '(', '\(' )
+      name = name.gsub( ')', '\)' )
+      name = name.gsub( '?', '\?' )
+      name = name.gsub( '*', '\*' )
+      name = name.gsub( '+', '\+' )
+      name = name.gsub( '$', '\$' )
+      name = name.gsub( '^', '\^' )
 
       ##  match accented char with or without accents
       ##  add (ü|ue) etc.
@@ -309,10 +309,10 @@ def title_esc_regex( title_unescaped )
       ##   collect some more (real-world) examples first!!!!!
 
       alternatives.each do |alt|
-        title = title.gsub( alt[0], alt[1] )
+        name = name.gsub( alt[0], alt[1] )
       end
 
-      title
+      name
   end
 
 end # class MapperV2