sportdb-formats 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89c3ae173bed0c7a68f30a897e4cf34da8d72d6e
-  data.tar.gz: f3898e8b03177b62075cced2a90be2502555bf63
+  metadata.gz: 666fc1f16808dddbc988aa073c0d3b47c08a5934
+  data.tar.gz: 97437435e1f37341c9f2cc15c8b4e67adf0efe61
 SHA512:
-  metadata.gz: 4c938af88f6dd86f1736532c9d8271a3e56f1769b955a31ce6d84a0b1fcf5a6eec7cfa42e0c019bd7072351d777a1988d6637c4297bee0dad9592bb8ec7450e8
-  data.tar.gz: a80df68f64e13eeb9a85931e2d2dd4381f204f94a4ea9254c89b4b28346db4c1542321aed4fd123aef5ddedc7e4cd2c08731dd340326fae5dc94b57fa6797835
+  metadata.gz: 9498c5e378feab9fa1e80a8287f33ab608e606ecde6ae914997483850faffbbbc7695ef07d2cea1e04a529e724b01612e8231fe8625c19d0af34f3fabffd0a14
+  data.tar.gz: 28146d5c35a061f6c128a001c190681eb76de5471cde8124435e77d93ccca090bf0a84a7a6a09ccb67372f8edef0198e8f09390126629d14072901ff5fcb7743

data/Manifest.txt

@@ -20,6 +20,7 @@ lib/sportdb/formats/match/mapper_teams.rb
 lib/sportdb/formats/match/match_parser.rb
 lib/sportdb/formats/match/match_parser_auto_conf.rb
 lib/sportdb/formats/match/match_parser_csv.rb
+lib/sportdb/formats/match/match_status_parser.rb
 lib/sportdb/formats/name_helper.rb
 lib/sportdb/formats/outline_reader.rb
 lib/sportdb/formats/package.rb
@@ -38,7 +39,9 @@ lib/sportdb/formats/structs/standings.rb
 lib/sportdb/formats/structs/team.rb
 lib/sportdb/formats/structs/team_usage.rb
 lib/sportdb/formats/team/club_index.rb
+lib/sportdb/formats/team/club_index_history.rb
 lib/sportdb/formats/team/club_reader.rb
+lib/sportdb/formats/team/club_reader_history.rb
 lib/sportdb/formats/team/club_reader_props.rb
 lib/sportdb/formats/team/national_team_index.rb
 lib/sportdb/formats/team/team_index.rb
@@ -46,14 +49,13 @@ lib/sportdb/formats/team/wiki_reader.rb
 lib/sportdb/formats/version.rb
 test/helper.rb
 test/test_club_index.rb
+test/test_club_index_history.rb
 test/test_club_reader.rb
+test/test_club_reader_history.rb
 test/test_club_reader_props.rb
 test/test_clubs.rb
-test/test_conf.rb
 test/test_country_index.rb
 test/test_country_reader.rb
-test/test_csv_match_parser.rb
-test/test_csv_match_parser_utils.rb
 test/test_csv_reader.rb
 test/test_datafile.rb
 test/test_datafile_package.rb
@@ -62,16 +64,7 @@ test/test_league_index.rb
 test/test_league_outline_reader.rb
 test/test_league_reader.rb
 test/test_match.rb
-test/test_match_auto.rb
-test/test_match_auto_champs.rb
-test/test_match_auto_euro.rb
-test/test_match_auto_relegation.rb
-test/test_match_auto_worldcup.rb
-test/test_match_champs.rb
-test/test_match_eng.rb
-test/test_match_euro.rb
-test/test_match_start_date.rb
-test/test_match_worldcup.rb
+test/test_match_status_parser.rb
 test/test_name_helper.rb
 test/test_outline_reader.rb
 test/test_package.rb

data/Rakefile

@@ -23,7 +23,7 @@ Hoe.spec 'sportdb-formats' do
     ['alphabets',     '>= 1.0.0'],
     ['date-formats',  '>= 1.0.1'],
     ['csvreader',     '>= 1.2.4'],
-    ['sportdb-langs', '>= 0.1.0'],
+    ['sportdb-langs', '>= 0.1.1'],
 
     ['rubyzip',       '>= 1.2.4' ],
   ]

data/lib/sportdb/formats.rb

@@ -75,10 +75,12 @@ require 'sportdb/formats/goals'
 
 require 'sportdb/formats/match/mapper'
 require 'sportdb/formats/match/mapper_teams'
+require 'sportdb/formats/match/match_status_parser'
 require 'sportdb/formats/match/match_parser'
 require 'sportdb/formats/match/match_parser_auto_conf'
 require 'sportdb/formats/match/conf_parser'
 
+
 require 'sportdb/formats/match/match_parser_csv'
 
 require 'sportdb/formats/country/country_reader'
@@ -119,6 +121,9 @@ require 'sportdb/formats/team/wiki_reader'
 require 'sportdb/formats/team/national_team_index'
 require 'sportdb/formats/team/team_index'
 
+require 'sportdb/formats/team/club_reader_history'
+require 'sportdb/formats/team/club_index_history'
+
 
 ###
 # add convenience helpers / shortcuts

data/lib/sportdb/formats/country/country_index.rb

@@ -109,12 +109,12 @@ class CountryIndex
     @countries_by_name[ name ]
   end
 
-  def []( key )
+  def find( key )
     country = find_by_code( key )
     country = find_by_name( key )  if country.nil?     ## try lookup / find by (normalized) name
     country
   end
-  alias_method :find, :[]
+  alias_method :[], :find
 
 
   ###

data/lib/sportdb/formats/event/event_index.rb

@@ -2,27 +2,25 @@ module SportDb
 module Import
 
 
+
 class EventIndex
 
     def self.build( path )
-      datafiles = Package.find_seasons( path )
-
-      puts
-      puts "#{datafiles.size} seasons datafile(s):"
-      pp datafiles
-
-      index = new
-      datafiles.each do |datafile|
-        recs = EventInfoReader.read( datafile )
-        # pp recs
+      pack = Package.new( path )   ## lets us use direcotry or zip archive
 
-        index.add( recs )
+      recs = []
+      pack.each_seasons do |entry|
+        recs += EventInfoReader.parse( entry.read )
       end
+      recs
 
+      index = new
+      index.add( recs )
       index
     end
 
 
+
     attr_reader :events
     def initialize
       @events  = []

data/lib/sportdb/formats/league/league_index.rb

@@ -95,36 +95,40 @@ class LeagueIndex
   end # method add
 
 
+  ## helper to always convert (possible) country key to existing country record
+  ##  todo: make private - why? why not?
+  def country( country )
+    if country.is_a?( String ) || country.is_a?( Symbol )
+      ## note:  use own "global" countries index setting for ClubIndex - why? why not?
+      rec = catalog.countries.find( country.to_s )
+      if rec.nil?
+        puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
+        exit 1
+      end
+      rec
+    else
+      country  ## (re)use country struct - no need to run lookup again
+    end
+  end
+
+
   def match( name )
-    ## todo/check: return empty array if no match!!! and NOT nil (add || []) - why? why not?
+    ## note: returns empty array if no match and NOT nil
     name = normalize( name )
-    @leagues_by_name[ name ]
+    @leagues_by_name[ name ] || []
   end
 
-
   def match_by( name:, country: )
     ## note: match must for now always include name
     m = match( name )
-    if m    ## filter by country
+    if country    ## filter by country
       ## note: country assumes / allows the country key or fifa code for now
-
       ## note: allow passing in of country struct too
-      country_rec = if country.is_a?( Country )
-                       country   ## (re)use country struct - no need to run lookup again
-                    else
-                       ## note:  use own "global" countries index setting for ClubIndex - why? why not?
-                       rec = catalog.countries.find( country )
-                       if rec.nil?
-                         puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
-                         exit 1
-                       end
-                       rec
-                    end
+      country_rec = country( country )
 
       ## note: also skip international leagues & cups (e.g. champions league etc.) for now - why? why not?
       m = m.select { |league| league.country &&
                               league.country.key == country_rec.key }
-      m = nil   if m.empty?     ## note: reset to nil if no more matches
     end
     m
   end
@@ -144,7 +148,7 @@ class LeagueIndex
     m = match( name )
     # pp m
 
-    if m.nil?
+    if m.empty?
       ## fall through/do nothing
     elsif m.size > 1
       puts "** !!! ERROR - ambigious league name; too many leagues (#{m.size}) found:"

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -127,7 +127,7 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     'Regular Season',
     'Regular Stage',
     'Championship Round',
-    'Championship Playoff',
+    'Championship Playoff',  # or Championship play-off
     'Relegation Round',
     'Relegation Playoff',
     'Play-offs',
@@ -140,6 +140,9 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     'EL Play-off',
     'Europa League Play-off',
     'Europa-League-Play-offs',
+    'Playoffs - Championship',
+    'Playoffs - Relegation',
+    'Finals',
   ].map {|name| name.downcase.gsub( /[^a-z]/, '' ) }
 
 

data/lib/sportdb/formats/league/league_reader.rb

@@ -118,12 +118,18 @@ def parse
               alt_names_auto << "#{country.code}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
             end
             alt_names_auto << "#{country.name} #{league_key}"  if league_key =~ /^[0-9]+$/   ## if all numeric e.g. add Austria 1 etc.
+
+            ## auto-add with country prepended
+            ##   e.g. England Premier League, Austria Bundesliga etc.
+            ##  todo/check: also add variants with country alt name if present!!!
+            ##  todo/check: exclude cups or such from country + league name auto-add - why? why not?
+            alt_names_auto << "#{country.name} #{league_name}"
           else   ## assume int'l (no country) e.g. champions league, etc.
             ## only auto-add key (e.g. CL, EL, etc.)
             alt_names_auto << league_key.upcase.gsub('.', ' ')   ## note: no country code (prefix/leading) used
           end
 
-          pp alt_names_auto
+          ## pp alt_names_auto
 
           ## prepend country key/code if country present
           ##   todo/fix: only auto-prepend country if key/code start with a number (level) or incl. cup

data/lib/sportdb/formats/match/match_parser.rb

@@ -302,6 +302,11 @@ class MatchParser   ## simple match parser for team match schedules
     ScoreFormats.find!( line )
   end
 
+  def find_status!( line )
+    StatusParser.find!( line )
+  end
+
+
   def try_parse_game( line )
     # note: clone line; for possible test do NOT modify in place for now
     # note: returns true if parsed, false if no match
@@ -329,6 +334,10 @@ class MatchParser   ## simple match parser for team match schedules
       return false
     end
 
+    ## find (optional) match status e.g. [abandoned] or [replay] or [awarded]
+    ##                                   or [cancelled] or [postponed] etc.
+    status = find_status!( line )   ## todo/check: allow match status also in geo part (e.g. after @) - why? why not?
+
     ## pos = find_game_pos!( line )
 
       date      = find_date!( line, start: @start )
@@ -353,20 +362,23 @@ class MatchParser   ## simple match parser for team match schedules
     if @last_round
       round = @last_round
     else
-      ## find (first) matching round by date
-      @rounds.values.each do |round_rec|
-        ## note: convert date to date only (no time) with to_date!!!
-        if (round_rec.start_date && round_rec.end_date) &&
-           (date.to_date >= round_rec.start_date &&
-            date.to_date <= round_rec.end_date)
-          round = round_rec
-          break
+      ## find (first) matching round by date if rounds / matchdays defined
+      ##   if not rounds / matchdays defined - YES, allow matches WITHOUT rounds!!!
+      if @rounds.size > 0
+        @rounds.values.each do |round_rec|
+          ## note: convert date to date only (no time) with to_date!!!
+          if (round_rec.start_date && round_rec.end_date) &&
+             (date.to_date >= round_rec.start_date &&
+             date.to_date <= round_rec.end_date)
+            round = round_rec
+            break
+          end
+        end
+        if round.nil?
+          puts "!! ERROR - no matching round found for match date:"
+          pp date
+          exit 1
         end
-      end
-      if round.nil?
-        puts "!! ERROR - no matching round found for match date:"
-        pp date
-        exit 1
       end
     end
 
@@ -380,8 +392,8 @@ class MatchParser   ## simple match parser for team match schedules
                                    team2:   team2,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team2.name)
                                    score:   score,
                                    round:   round       ? round.name       : nil,   ## note: for now always use string (assume unique canonical name for event)
-                                   group:   @last_group ? @last_group.name : nil )  ## note: for now always use string (assume unique canonical name for event)
-
+                                   group:   @last_group ? @last_group.name : nil,   ## note: for now always use string (assume unique canonical name for event)
+                                   status:  status )
     ### todo: cache team lookups in hash?
 
 =begin

data/lib/sportdb/formats/match/match_parser_csv.rb

@@ -101,9 +101,23 @@ module SportDb
          header_stage = find_header( headers, ['Stage'] )
          headers_mapping[:stage]  =  header_stage   if header_stage
 
+         header_group = find_header( headers, ['Group'] )
+         headers_mapping[:group]  =  header_group   if header_group
+
+
+         header_et = find_header( headers, ['ET', 'AET'] )   ## (after) extra time
+         headers_mapping[:score_et] = header_et   if header_et
+
+         header_p  = find_header( headers, ['P', 'PEN'] )    ## penalties
+         headers_mapping[:score_p] = header_p   if header_p
+
+         header_notes = find_header( headers, ['Notes', 'Comments'] )
+         headers_mapping[:notes]  =  header_notes   if header_notes
+
+
          header_league = find_header( headers, ['League'] )
          headers_mapping[:league] = header_league   if header_league
-      else
+        else
          ## else try footballdata.uk and others
          headers_mapping[:team1]  = find_header( headers, ['HomeTeam', 'HT', 'Home'] )
          headers_mapping[:team2]  = find_header( headers, ['AwayTeam', 'AT', 'Away'] )
@@ -229,7 +243,15 @@ module SportDb
       if headers_mapping[ :round ]
         col = row[ headers_mapping[ :round ]]
         ## todo: issue warning if not ? or - (and just empty string) why? why not
-        round = col.to_i  if col =~ /^\d{1,2}$/     # check format - e.g. ignore ? or - or such non-numbers for now
+        ## (old attic) was: round = col.to_i  if col =~ /^\d{1,2}$/     # check format - e.g. ignore ? or - or such non-numbers for now
+
+        ## note: make round always a string for now!!!! e.g. "1", "2" too!!
+        round = if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                 ## note: allow missing round for match / defaults to nil
+                 nil
+                else
+                  col
+                end
       end
 
 
@@ -258,25 +280,65 @@ module SportDb
         score2i = ht[1].to_i  if ht[1] =~ /^\d{1,2}$/
       end
 
+
       ## check for all-in-one full time scores?
       if headers_mapping[ :score ]
-        ft = row[ headers_mapping[ :score ] ]
-        if ft =~ /^\d{1,2}[\-:]\d{1,2}$/   ## sanity check scores format
-          scores = ft.split( /[\-:]/ )
-          score1 = scores[0].to_i
-          score2 = scores[1].to_i
+        col = row[ headers_mapping[ :score ]]
+        score = parse_score( col )
+        if score
+          score1 = score[0]
+          score2 = score[1]
+        else
+          puts "!! ERROR - invalid score (ft) format >#{col}<:"
+          pp row
+          exit 1
         end
-        ## todo/fix: issue warning if non-empty!!! and not matching format!!!!
       end
 
       if headers_mapping[ :scorei ]
-        ht = row[ headers_mapping[ :scorei ] ]
-        if ht =~ /^\d{1,2}[\-:]\d{1,2}$/   ## sanity check scores format
-          scores = ht.split( /[\-:]/)   ## allow 1-1 and 1:1
-          score1i = scores[0].to_i
-          score2i = scores[1].to_i
+        col = row[ headers_mapping[ :scorei ]]
+        score = parse_score( col )
+        if score
+          score1i = score[0]
+          score2i = score[1]
+        else
+          puts "!! ERROR - invalid score (ht) format >#{col}<:"
+          pp row
+          exit 1
+        end
+      end
+
+      ####
+      ## try optional score - extra time (et) and penalities (p/pen)
+      score1et  = nil
+      score2et  = nil
+      score1p   = nil
+      score2p   = nil
+
+      if headers_mapping[ :score_et ]
+        col = row[ headers_mapping[ :score_et ]]
+        score = parse_score( col )
+        if score
+          score1et = score[0]
+          score2et = score[1]
+        else
+          puts "!! ERROR - invalid score (et) format >#{col}<:"
+          pp row
+          exit 1
+        end
+      end
+
+      if headers_mapping[ :score_p ]
+        col = row[ headers_mapping[ :score_p ]]
+        score = parse_score( col )
+        if score
+          score1p = score[0]
+          score2p = score[1]
+        else
+          puts "!! ERROR - invalid score (p) format >#{col}<:"
+          pp row
+          exit 1
         end
-        ## todo/fix: issue warning if non-empty!!! and not matching format!!!!
       end
 
 
@@ -296,17 +358,51 @@ module SportDb
                  end
       end
 
+      group = nil
+      if headers_mapping[ :group ]
+        col = row[ headers_mapping[ :group ]]
+        ## todo/fix: check can col be nil e.g. col.nil? possible?
+        group =  if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                    ## note: allow missing stage for match / defaults to "regular"
+                    nil
+                 else
+                    col
+                 end
+      end
+
+      status = nil    ## e.g. AWARDED, CANCELLED, POSTPONED, etc.
+      if headers_mapping[ :notes ]
+        col = row[ headers_mapping[ :notes ]]
+        ## check for optional (match) status in notes / comments
+        status = if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                   nil
+                 else
+                   StatusParser.parse( col )  # note: returns nil if no (match) status found
+                 end
+      end
+
+
       league = nil
       league = row[ headers_mapping[ :league ]]   if headers_mapping[ :league ]
 
 
-      match = Import::Match.new( date:    date,
-                                 team1:   team1,   team2:   team2,
-                                 score1:  score1,  score2:  score2,
-                                 score1i: score1i, score2i: score2i,
-                                 round:   round,
-                                 stage:   stage,
-                                 league:  league )
+      ## puts 'match attributes:'
+      attributes = {
+        date:     date,
+        team1:    team1,    team2:    team2,
+        score1:   score1,   score2:   score2,
+        score1i:  score1i,  score2i:  score2i,
+        score1et: score1et, score2et: score2et,
+        score1p:  score1p,  score2p:  score2p,
+        round:    round,
+        stage:    stage,
+        group:    group,
+        status:   status,
+        league:   league
+      }
+      ## pp attributes
+
+      match = Import::Match.new( **attributes )
       matches << match
     end
 
@@ -326,6 +422,37 @@ module SportDb
     nil  ## no matching header  found!!!
   end
 
+########
+# more helpers
+#
+
+def parse_score( str )
+  if str.nil?    ## todo/check: remove nil case - possible? - why? why not?
+    [nil,nil]
+  else
+    ## remove (optional single) note/footnote/endnote markers
+    ##  e.g. (*) or (a), (b),
+    ##    or [*], [A], [1], etc.
+    ##  - allow (1) or maybe (*1) in the future - why? why not?
+    str = str.sub( /\( [a-z*] \)
+                        |
+                    \[ [1-9a-z*] \]
+                   /ix, '' ).strip
+
+    if str.empty? || str == '?' || str == '-' || str == 'n/a'
+      [nil,nil]
+    ### todo/check: use regex with named capture groups here - why? why not?
+    elsif str =~ /^\d{1,2}[:-]\d{1,2}$/   ## sanity check scores format
+      score  = str.split( /[:-]/ )
+      [score[0].to_i, score[1].to_i]
+    else
+      nil  ## note: returns nil if invalid / unparseable format!!!
+    end
+  end
+end # method parse_score
+
+
+
   end # class CsvMatchParser
   end # module SportDb