sportdb-formats 1.0.3 → 1.1.1

data/lib/sportdb/formats/match/match_parser_csv.rb

@@ -100,6 +100,9 @@ module SportDb
          ## optional headers - note: find_header returns nil if header NOT found
          header_stage = find_header( headers, ['Stage'] )
          headers_mapping[:stage]  =  header_stage   if header_stage
+
+         header_league = find_header( headers, ['League'] )
+         headers_mapping[:league] = header_league   if header_league
       else
          ## else try footballdata.uk and others
          headers_mapping[:team1]  = find_header( headers, ['HomeTeam', 'HT', 'Home'] )
@@ -167,7 +170,10 @@ module SportDb
 
 
       ## check if data present - if not skip (might be empty row)
-      if team1.nil? && team2.nil?
+      ##  note:  (old classic) csv reader returns nil for empty fields
+      ##         new modern csv reader ALWAYS returns strings (and empty strings for data not available (n/a))
+      if (team1.nil? || team1.empty?) &&
+         (team2.nil? || team2.empty?)
         puts "*** WARN: skipping empty? row[#{i}] - no teams found:"
         pp row
         next
@@ -182,9 +188,11 @@ module SportDb
       col = row[ headers_mapping[ :date ]]
       col = col.strip   # make sure not leading or trailing spaces left over
 
-      if col.empty? || col == '-' || col == '?'
-         ## note: allow missing / unknown date for match
-         date = nil
+      if col.empty? ||
+         col =~ /^-{1,}$/ ||      # e.g.  - or ---
+         col =~ /^\?{1,}$/        # e.g. ? or ???
+          ## note: allow missing / unknown date for match
+          date = nil
       else
         ## remove possible weekday or weeknumber  e.g. (Fri) (4) etc.
         col = col.sub( /\(W?\d{1,2}\)/, '' )  ## e.g. (W11), (4), (21) etc.
@@ -199,6 +207,8 @@ module SportDb
           date_fmt = '%Y-%m-%d'   # e.g. 1995-08-04
         elsif col =~ /^\d{1,2} \w{3} \d{4}$/
           date_fmt = '%d %b %Y'   # e.g. 8 Jul 2017
+        elsif col =~ /^\w{3} \w{3} \d{1,2} \d{4}$/
+          date_fmt = '%a %b %d %Y'   # e.g. Sat Aug 7 1993
         else
           puts "*** !!! wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
           ## todo/fix: add to errors/warns list - why? why not?
@@ -283,13 +293,17 @@ module SportDb
                  end
       end
 
+      league = nil
+      league = row[ headers_mapping[ :league ]]   if headers_mapping[ :league ]
+
 
       match = Import::Match.new( date:    date,
                                  team1:   team1,   team2:   team2,
                                  score1:  score1,  score2:  score2,
                                  score1i: score1i, score2i: score2i,
                                  round:   round,
-                                 stage:   stage )
+                                 stage:   stage,
+                                 league:  league )
       matches << match
     end
 

data/lib/sportdb/formats/package.rb

@@ -4,31 +4,50 @@ module SportDb
 
     ## todo/fix:   make all regexes case-insensitive with /i option - why? why not?
     ##                e.g. .TXT and .txt
+    ##   yes!! use /i option!!!!!
 
     CONF_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
         \.conf\.txt$
     }x
 
+    ## leagues.txt or leagues_en.txt
+    ##   remove support for en.leagues.txt - why? why not?
     LEAGUES_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
        (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.wiki.txt
-        leagues\.txt$
+        leagues
+          (?:_[a-z0-9_-]+)?
+        \.txt$
     }x
 
+    ## clubs.txt or clubs_en.txt
+    ##   remove support for en.clubs.txt - why? why not?
     CLUBS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
        (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.txt
-        clubs\.txt$
+        clubs
+          (?:_[a-z0-9_-]+)?
+        \.txt$
     }x
 
     CLUBS_WIKI_RE = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
         (?:[a-z]{1,4}\.)?   # optional country code/key e.g. eng.clubs.wiki.txt
-       clubs\.wiki\.txt$
+       clubs
+         (?:_[a-z0-9_-]+)?
+       \.wiki\.txt$
     }x
 
     CLUB_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.props.txt
-        clubs\.props\.txt$
+        clubs
+          (?:_[a-z0-9_-]+)?
+        \.props\.txt$
     }x
 
+    ##  teams.txt or teams_history.txt
+    TEAMS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+                      teams
+                        (?:_[a-z0-9_-]+)?
+                      \.txt$
+    }x
 
     ### season folder:
     ##            e.g. /2019-20   or
@@ -36,7 +55,7 @@ module SportDb
     ##                 /2016--france
     SEASON_RE = %r{ (?:
                        \d{4}-\d{2}
-                     | \d{4}(--[^/]+)?
+                     | \d{4}(--[a-z0-9_-]+)?
                     )
                   }x
     SEASON = SEASON_RE.source    ## "inline" helper for embedding in other regexes - keep? why? why not?
@@ -54,6 +73,10 @@ module SportDb
                        /[a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
                     }x
 
+    ### add "generic" pattern to find all csv datafiles
+    CSV_RE = %r{ (?: ^|/ )
+                   [a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
+               }x
 
 
     ## move class-level "static" finders to DirPackage (do NOT work for now for zip packages) - why? why not?
@@ -74,6 +97,8 @@ module SportDb
    end
 
 
+   def self.find_teams( path, pattern: TEAMS_RE )  find( path, pattern ); end
+   def self.match_teams( path )  TEAMS_RE.match( path ); end
 
    def self.find_clubs( path, pattern: CLUBS_RE )            find( path, pattern ); end
    def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )  find( path, pattern ); end
@@ -97,7 +122,11 @@ module SportDb
    end
    ## add match_match and match_match_csv  - why? why not?
 
+
    class << self
+     alias_method :match_teams?, :match_teams
+     alias_method :teams?,       :match_teams
+
      alias_method :match_clubs?, :match_clubs
      alias_method :clubs?,       :match_clubs
 
@@ -188,6 +217,8 @@ module SportDb
       end
     end
     def each_match_csv( &blk )  each( pattern: MATCH_CSV_RE, &blk ); end
+    def each_csv( &blk )        each( pattern: CSV_RE, &blk );       end
+
     def each_club_props( &blk ) each( pattern: CLUB_PROPS_RE, &blk ); end
 
     def each_leagues( &blk )    each( pattern: LEAGUES_RE, &blk ); end

data/lib/sportdb/formats/parser_helper.rb

@@ -18,10 +18,19 @@ module SportDb
 
 
   def is_round?( line )
-    ## note: =~ return nil if not match found, and 0,1, etc for match
-    (line =~ SportDb.lang.regex_round) != nil
+    ## note: =~ returns nil if not match found, and 0,1, etc for match
+
+    ##  note: allow "free standing" leg 1 and leg 2 too
+    ##         (e.g. Hinspiel, Rückspiel etc. used for now in Relegation, for example)
+    ##    note ONLY allowed if "free standing", that is, full line with nothing else
+    ##          use "custom" regex for special case for now
+    ##               avoids match HIN in PascHINg, for example (hin in german for leg 1)
+    line =~ SportDb.lang.regex_round    ||
+    line =~ /^(#{SportDb.lang.leg1})$/i ||
+    line =~ /^(#{SportDb.lang.leg2})$/i
   end
 
+
   def is_knockout_round?( line )
 
     ## todo: check for adding ignore case for regex (e.g. 1st leg/1st Leg)

data/lib/sportdb/formats/score/score_parser.rb

@@ -90,6 +90,12 @@ class ScoreParser
 
 
   def parse( line )
+
+     ##########
+     ## todo/fix/check: add unicode to regular dash conversion - why? why not?
+     ##  e.g. – becomes -  (yes, the letters a different!!!)
+     #############
+
     score = nil
     @formats.each do |format|
       re = format[0]

data/lib/sportdb/formats/structs/country.rb

@@ -11,11 +11,14 @@ class Country
 
   ## note: is read-only/immutable for now - why? why not?
   ##          add cities (array/list) - why? why not?
-  attr_reader   :key, :name, :fifa, :tags
+  attr_reader   :key, :name, :code, :tags
   attr_accessor :alt_names
 
-  def initialize( key:, name:, fifa:, tags: [] )
-    @key, @name, @fifa = key, name, fifa
+  def initialize( key: nil, name:, code:, tags: [] )
+    ## note: auto-generate key "on-the-fly" if missing for now - why? why not?
+    ## note: quick hack - auto-generate key, that is, remove all non-ascii chars and downcase
+    @key = key || name.downcase.gsub( /[^a-z]/, '' )
+    @name, @code = name, code
     @alt_names      = []
     @tags           = tags
   end

data/lib/sportdb/formats/structs/group.rb

@@ -2,20 +2,13 @@ module SportDb
   module Import
 
     class Group
-      attr_reader   :title, :pos, :teams
+      attr_reader :key, :name, :teams
 
-      ##
-      ##  todo:  change db schema
-      ##    make start and end date optional
-      ##    change pos to num - why? why not?
-      ##    make pos/num optional too
-      ##
-      ##    sort round by scheduled/planed start date
-      def initialize( title:,
-                      pos:,
+      def initialize( key: nil,
+                      name:,
                       teams: )
-        @title  = title
-        @pos    = pos
+        @key    = key    ## e.g. A,B,C  or 1,2,3, - note: always a string or nil
+        @name   = name
         @teams  = teams
       end
     end  # class Group

data/lib/sportdb/formats/structs/match.rb

@@ -20,7 +20,8 @@ class Match
               :group,
               :conf1,    :conf2,      ## special case for mls e.g. conference1, conference2 (e.g. west, east, central)
               :country1, :country2,    ## special case for champions league etc. - uses FIFA country code
-              :comments
+              :comments,
+              :league      ## (optinal) added as text for now (use struct?)
 
   def initialize( **kwargs )
     update( kwargs )  unless kwargs.empty?
@@ -47,6 +48,9 @@ class Match
     @group    = kwargs[:group]    if kwargs.has_key? :group
     @comments = kwargs[:comments] if kwargs.has_key? :comments
 
+    @league   = kwargs[:league]   if kwargs.has_key? :league
+
+
     if kwargs.has_key?( :score )   ## check all-in-one score struct for convenience!!!
       score = kwargs[:score]
       if score.nil?   ## reset all score attribs to nil!!

data/lib/sportdb/formats/structs/round.rb

@@ -2,24 +2,17 @@ module SportDb
   module Import
 
     class Round
-      attr_reader   :title, :start_date, :end_date, :knockout
-      attr_accessor :pos   # note: make read & writable
+      attr_reader   :name, :start_date, :end_date, :knockout
+      attr_accessor :num   # note: make read & writable - why? why not?
 
-      ##
-      ##  todo:  change db schema
-      ##    make start and end date optional
-      ##    change pos to num - why? why not?
-      ##    make pos/num optional too
-      ##
-      ##    sort round by scheduled/planed start date
-      def initialize( title:,
-                      pos: nil,
+      def initialize( name:,
+                      num: nil,
                       start_date: nil,
                       end_date: nil,
                       knockout: false,
                       auto: true )
-        @title      = title
-        @pos        = pos
+        @name       = name
+        @num        = num
         @start_date = start_date
         @end_date   = end_date
         @knockout   = knockout

data/lib/sportdb/formats/structs/standings.rb

@@ -12,7 +12,7 @@ module SportDb
 
 class Standings
 
-  class StandingsLine     ## nested class StandinsLine
+  class StandingsLine     ## nested class StandinsLine - todo/fix: change to Line - why? why not?
     attr_accessor  :rank, :team,
                    :played, :won, :lost, :drawn,                      ## -- total
                    :goals_for, :goals_against, :pts,
@@ -21,8 +21,13 @@ class Standings
                    :away_played, :away_won, :away_lost, :away_drawn,  ## -- away
                    :away_goals_for, :away_goals_against, :away_pts
 
+     alias_method :team_name, :team     ## note: team for now always a string
+     alias_method :pos,       :rank     ##  rename back to use pos instead of rank - why? why not?
+
+
     def initialize( team )
       @rank = nil # use 0? why? why not?
+      ## change rank back to pos - why? why not?
       @team = team
       @played = @home_played = @away_played = 0
       @won    = @home_won    = @away_won    = 0
@@ -49,7 +54,12 @@ class Standings
 
   def update( match_or_matches )
     ## convenience - update all matches at once
-    matches = match_or_matches.is_a?(Array) ? match_or_matches : [match_or_matches]
+    ## todo/check:  check for ActiveRecord_Associations_CollectionProxy and than use to_a (to "force" array) - why? why not?
+    matches = if match_or_matches.is_a?(Array)
+                match_or_matches
+              else
+                [match_or_matches]
+              end
 
     matches.each_with_index do |match,i| # note: index(i) starts w/ zero (0)
       update_match( match )
@@ -171,19 +181,30 @@ class Standings
 private
   def update_match( m )   ## add a match
 
-    ##  puts "   #{m.team1} - #{m.team2} #{m.score_str}"
+    ## note: always use team as string for now
+    ##   for now allow passing in of string OR struct - why? why not?
+    ##   todo/fix:  change to m.team1_name and m.team2_name - why? why not?
+    team1 = m.team1.is_a?( String ) ? m.team1 : m.team1.name
+    team2 = m.team2.is_a?( String ) ? m.team2 : m.team2.name
+
+    score = m.score_str
+
+    ##  puts "   #{team1} - #{team2} #{score}"
+
     unless m.over?
-      puts " !!!! skipping match - not yet over (play_at date in the future)"
+      puts " !!!! skipping match - not yet over (date in the future) => #{m.date}"
       return
     end
 
     unless m.complete?
-      puts "!!! [calc_standings] skipping match #{m.team1} - #{m.team2} - scores incomplete #{m.score_str}"
+      puts "!!! [calc_standings] skipping match #{team1} - #{team2} w/ past date #{m.date} - scores incomplete => #{score}"
       return
     end
 
-    line1 = @lines[ m.team1 ] || StandingsLine.new( m.team1 )
-    line2 = @lines[ m.team2 ] || StandingsLine.new( m.team2 )
+
+
+    line1 = @lines[ team1 ] || StandingsLine.new( team1 )
+    line2 = @lines[ team2 ] || StandingsLine.new( team2 )
 
     line1.played      += 1
     line1.home_played += 1
@@ -236,8 +257,8 @@ private
       puts "*** warn: [standings] skipping match with missing scores: #{m.inspect}"
     end
 
-    @lines[ m.team1 ] = line1
-    @lines[ m.team2 ] = line2
+    @lines[ team1 ] = line1
+    @lines[ team2 ] = line2
   end  # method update_match
 
 end  # class Standings

data/lib/sportdb/formats/structs/team.rb

@@ -11,10 +11,9 @@ class Team
   ##  todo: use just names for alt_names - why? why not?
   attr_accessor :key, :name, :alt_names,
                 :code,    ## code == abbreviation e.g. ARS etc.
-                :year, :year_end,   ## todo/fix: change year_end to end_year (like in season)!!!
+                :year, :year_end,   ## todo/fix: change year to start_year and year_end to end_year (like in season)!!!
                 :country
 
-  alias_method :title, :name  ## add alias/compat - why? why not
 
   def names
     ## todo/check: add alt_names_auto too? - why? why not?

data/lib/sportdb/formats/version.rb

@@ -6,8 +6,8 @@ module SportDb
 module Formats
 
   MAJOR = 1    ## todo: namespace inside version or something - why? why not??
-  MINOR = 0
-  PATCH = 3
+  MINOR = 1
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/helper.rb

@@ -1,5 +1,6 @@
 ## note: use the local version of gems
 $LOAD_PATH.unshift( File.expand_path( '../date-formats/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-langs/lib' ))
 
 
 ## minitest setup

data/test/test_country_index.rb

@@ -16,12 +16,12 @@ class TestCountryIndex < MiniTest::Test
     eng = countries[:eng]
     assert_equal 'eng',      eng.key
     assert_equal 'England',  eng.name
-    assert_equal 'ENG',      eng.fifa
+    assert_equal 'ENG',      eng.code
 
     at  = countries[:at]
     assert_equal 'at',                at.key
     assert_equal 'Austria',           at.name
-    assert_equal 'AUT',               at.fifa
+    assert_equal 'AUT',               at.code
     assert_equal ['Österreich [de]'], at.alt_names
 
     assert at == countries['AT']
@@ -48,13 +48,13 @@ class TestCountryIndex < MiniTest::Test
     assert at == countries.parse( 'Österreich • Austria' )
     assert at == countries.parse( 'Austria' )
     assert at == countries.parse( 'at' )    ## (iso alpha2) country code
-    assert at == countries.parse( 'AUT' )   ## fifa code
+    assert at == countries.parse( 'AUT' )   ## (fifa) country code
 
 
     de  = countries[:de]
     assert_equal 'de',                 de.key
     assert_equal 'Germany',            de.name
-    assert_equal 'GER',                de.fifa
+    assert_equal 'GER',                de.code
     assert_equal ['Deutschland [de]'], de.alt_names
 
     assert de == countries.parse( 'Deutschland (de) • Germany' )

data/test/test_country_reader.rb

@@ -16,12 +16,12 @@ class TestCountryReader < MiniTest::Test
     assert_equal 232, recs.size
 
     assert_equal 'Albania',        recs[0].name
-    assert_equal 'ALB',            recs[0].fifa
+    assert_equal 'ALB',            recs[0].code
     assert_equal 'al',             recs[0].key
     assert_equal ['fifa', 'uefa'], recs[0].tags
 
     assert_equal 'Andorra',        recs[1].name
-    assert_equal 'AND',            recs[1].fifa
+    assert_equal 'AND',            recs[1].code
     assert_equal 'ad',             recs[1].key
     assert_equal ['fifa', 'uefa'], recs[1].tags
   end
@@ -44,16 +44,46 @@ TXT
 
     assert_equal 4, recs.size
     assert_equal 'Afghanistan',    recs[0].name
-    assert_equal 'AFG',            recs[0].fifa
+    assert_equal 'AFG',            recs[0].code
     assert_equal 'af',             recs[0].key
     assert_equal [],               recs[0].alt_names
     assert_equal ['fifa', 'afc'],  recs[0].tags
 
     assert_equal 'American Samoa', recs[3].name
-    assert_equal 'ASA',            recs[3].fifa
+    assert_equal 'ASA',            recs[3].code
     assert_equal 'as',             recs[3].key
     assert_equal ['Am. Samoa'],    recs[3].alt_names
     assert_equal [],               recs[3].tags
   end
 
+  def test_parse_historic
+    recs = SportDb::Import::CountryReader.parse( <<TXT )
+###########################################
+# Former national teams
+#   with former FIFA country codes etc.
+-1992   Czechoslovakia, TCH      ⇒  Czech Republic
+
+-1991   Soviet Union,   URS      ⇒  Russia
+
+-1989   West Germany,   FRG      =>  Germany
+-1989   East Germany,   GDR      =>  Germany
+TXT
+
+    pp recs
+
+    assert_equal 4, recs.size
+    assert_equal 'Czechoslovakia (-1992)', recs[0].name
+    assert_equal 'TCH',                    recs[0].code
+    assert_equal 'czechoslovakia',         recs[0].key
+    assert_equal [],                       recs[0].alt_names
+    assert_equal [],                       recs[0].tags
+
+    assert_equal 'East Germany (-1989)',   recs[3].name
+    assert_equal 'GDR',                    recs[3].code
+    assert_equal 'eastgermany',            recs[3].key
+    assert_equal [],                       recs[3].alt_names
+    assert_equal [],                       recs[3].tags
+  end
+
+
 end # class TestCountryReader