sportdb-formats 1.0.5 → 1.1.3

data/lib/sportdb/formats/match/match_parser_csv.rb

@@ -95,12 +95,29 @@ module SportDb
          headers_mapping[:score]  = find_header( headers, ['FT'] )
          headers_mapping[:scorei] = find_header( headers, ['HT'] )
 
-         headers_mapping[:round]  = find_header( headers, ['Round'] )
+         headers_mapping[:round]  = find_header( headers, ['Round', 'Matchday'] )
 
          ## optional headers - note: find_header returns nil if header NOT found
          header_stage = find_header( headers, ['Stage'] )
          headers_mapping[:stage]  =  header_stage   if header_stage
-      else
+
+         header_group = find_header( headers, ['Group'] )
+         headers_mapping[:group]  =  header_group   if header_group
+
+
+         header_et = find_header( headers, ['ET', 'AET'] )   ## (after) extra time
+         headers_mapping[:score_et] = header_et   if header_et
+
+         header_p  = find_header( headers, ['P', 'PEN'] )    ## penalties
+         headers_mapping[:score_p] = header_p   if header_p
+
+         header_notes = find_header( headers, ['Notes', 'Comments'] )
+         headers_mapping[:notes]  =  header_notes   if header_notes
+
+
+         header_league = find_header( headers, ['League'] )
+         headers_mapping[:league] = header_league   if header_league
+        else
          ## else try footballdata.uk and others
          headers_mapping[:team1]  = find_header( headers, ['HomeTeam', 'HT', 'Home'] )
          headers_mapping[:team2]  = find_header( headers, ['AwayTeam', 'AT', 'Away'] )
@@ -167,7 +184,10 @@ module SportDb
 
 
       ## check if data present - if not skip (might be empty row)
-      if team1.nil? && team2.nil?
+      ##  note:  (old classic) csv reader returns nil for empty fields
+      ##         new modern csv reader ALWAYS returns strings (and empty strings for data not available (n/a))
+      if (team1.nil? || team1.empty?) &&
+         (team2.nil? || team2.empty?)
         puts "*** WARN: skipping empty? row[#{i}] - no teams found:"
         pp row
         next
@@ -182,9 +202,11 @@ module SportDb
       col = row[ headers_mapping[ :date ]]
       col = col.strip   # make sure not leading or trailing spaces left over
 
-      if col.empty? || col == '-' || col == '?'
-         ## note: allow missing / unknown date for match
-         date = nil
+      if col.empty? ||
+         col =~ /^-{1,}$/ ||      # e.g.  - or ---
+         col =~ /^\?{1,}$/        # e.g. ? or ???
+          ## note: allow missing / unknown date for match
+          date = nil
       else
         ## remove possible weekday or weeknumber  e.g. (Fri) (4) etc.
         col = col.sub( /\(W?\d{1,2}\)/, '' )  ## e.g. (W11), (4), (21) etc.
@@ -199,6 +221,8 @@ module SportDb
           date_fmt = '%Y-%m-%d'   # e.g. 1995-08-04
         elsif col =~ /^\d{1,2} \w{3} \d{4}$/
           date_fmt = '%d %b %Y'   # e.g. 8 Jul 2017
+        elsif col =~ /^\w{3} \w{3} \d{1,2} \d{4}$/
+          date_fmt = '%a %b %d %Y'   # e.g. Sat Aug 7 1993
         else
           puts "*** !!! wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
           ## todo/fix: add to errors/warns list - why? why not?
@@ -211,12 +235,23 @@ module SportDb
       end
 
 
+      ##
+      ## todo/fix:  round might not always be just a simple integer number!!!
+      ##             might be text such as Final | Leg 1 or such!!!!
       round   = nil
       ## check for (optional) round / matchday
       if headers_mapping[ :round ]
         col = row[ headers_mapping[ :round ]]
         ## todo: issue warning if not ? or - (and just empty string) why? why not
-        round = col.to_i  if col =~ /^\d{1,2}$/     # check format - e.g. ignore ? or - or such non-numbers for now
+        ## (old attic) was: round = col.to_i  if col =~ /^\d{1,2}$/     # check format - e.g. ignore ? or - or such non-numbers for now
+
+        ## note: make round always a string for now!!!! e.g. "1", "2" too!!
+        round = if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                 ## note: allow missing round for match / defaults to nil
+                 nil
+                else
+                  col
+                end
       end
 
 
@@ -245,25 +280,65 @@ module SportDb
         score2i = ht[1].to_i  if ht[1] =~ /^\d{1,2}$/
       end
 
+
       ## check for all-in-one full time scores?
       if headers_mapping[ :score ]
-        ft = row[ headers_mapping[ :score ] ]
-        if ft =~ /^\d{1,2}[\-:]\d{1,2}$/   ## sanity check scores format
-          scores = ft.split( /[\-:]/ )
-          score1 = scores[0].to_i
-          score2 = scores[1].to_i
+        col = row[ headers_mapping[ :score ]]
+        score = parse_score( col )
+        if score
+          score1 = score[0]
+          score2 = score[1]
+        else
+          puts "!! ERROR - invalid score (ft) format >#{col}<:"
+          pp row
+          exit 1
         end
-        ## todo/fix: issue warning if non-empty!!! and not matching format!!!!
       end
 
       if headers_mapping[ :scorei ]
-        ht = row[ headers_mapping[ :scorei ] ]
-        if ht =~ /^\d{1,2}[\-:]\d{1,2}$/   ## sanity check scores format
-          scores = ht.split( /[\-:]/)   ## allow 1-1 and 1:1
-          score1i = scores[0].to_i
-          score2i = scores[1].to_i
+        col = row[ headers_mapping[ :scorei ]]
+        score = parse_score( col )
+        if score
+          score1i = score[0]
+          score2i = score[1]
+        else
+          puts "!! ERROR - invalid score (ht) format >#{col}<:"
+          pp row
+          exit 1
+        end
+      end
+
+      ####
+      ## try optional score - extra time (et) and penalities (p/pen)
+      score1et  = nil
+      score2et  = nil
+      score1p   = nil
+      score2p   = nil
+
+      if headers_mapping[ :score_et ]
+        col = row[ headers_mapping[ :score_et ]]
+        score = parse_score( col )
+        if score
+          score1et = score[0]
+          score2et = score[1]
+        else
+          puts "!! ERROR - invalid score (et) format >#{col}<:"
+          pp row
+          exit 1
+        end
+      end
+
+      if headers_mapping[ :score_p ]
+        col = row[ headers_mapping[ :score_p ]]
+        score = parse_score( col )
+        if score
+          score1p = score[0]
+          score2p = score[1]
+        else
+          puts "!! ERROR - invalid score (p) format >#{col}<:"
+          pp row
+          exit 1
         end
-        ## todo/fix: issue warning if non-empty!!! and not matching format!!!!
       end
 
 
@@ -283,13 +358,51 @@ module SportDb
                  end
       end
 
+      group = nil
+      if headers_mapping[ :group ]
+        col = row[ headers_mapping[ :group ]]
+        ## todo/fix: check can col be nil e.g. col.nil? possible?
+        group =  if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                    ## note: allow missing stage for match / defaults to "regular"
+                    nil
+                 else
+                    col
+                 end
+      end
+
+      status = nil    ## e.g. AWARDED, CANCELLED, POSTPONED, etc.
+      if headers_mapping[ :notes ]
+        col = row[ headers_mapping[ :notes ]]
+        ## check for optional (match) status in notes / comments
+        status = if col.nil? || col.empty? || col == '-' || col == 'n/a'
+                   nil
+                 else
+                   StatusParser.parse( col )  # note: returns nil if no (match) status found
+                 end
+      end
+
+
+      league = nil
+      league = row[ headers_mapping[ :league ]]   if headers_mapping[ :league ]
 
-      match = Import::Match.new( date:    date,
-                                 team1:   team1,   team2:   team2,
-                                 score1:  score1,  score2:  score2,
-                                 score1i: score1i, score2i: score2i,
-                                 round:   round,
-                                 stage:   stage )
+
+      ## puts 'match attributes:'
+      attributes = {
+        date:     date,
+        team1:    team1,    team2:    team2,
+        score1:   score1,   score2:   score2,
+        score1i:  score1i,  score2i:  score2i,
+        score1et: score1et, score2et: score2et,
+        score1p:  score1p,  score2p:  score2p,
+        round:    round,
+        stage:    stage,
+        group:    group,
+        status:   status,
+        league:   league
+      }
+      ## pp attributes
+
+      match = Import::Match.new( **attributes )
       matches << match
     end
 
@@ -309,6 +422,37 @@ module SportDb
     nil  ## no matching header  found!!!
   end
 
+########
+# more helpers
+#
+
+def parse_score( str )
+  if str.nil?    ## todo/check: remove nil case - possible? - why? why not?
+    [nil,nil]
+  else
+    ## remove (optional single) note/footnote/endnote markers
+    ##  e.g. (*) or (a), (b),
+    ##    or [*], [A], [1], etc.
+    ##  - allow (1) or maybe (*1) in the future - why? why not?
+    str = str.sub( /\( [a-z*] \)
+                        |
+                    \[ [1-9a-z*] \]
+                   /ix, '' ).strip
+
+    if str.empty? || str == '?' || str == '-' || str == 'n/a'
+      [nil,nil]
+    ### todo/check: use regex with named capture groups here - why? why not?
+    elsif str =~ /^\d{1,2}[:-]\d{1,2}$/   ## sanity check scores format
+      score  = str.split( /[:-]/ )
+      [score[0].to_i, score[1].to_i]
+    else
+      nil  ## note: returns nil if invalid / unparseable format!!!
+    end
+  end
+end # method parse_score
+
+
+
   end # class CsvMatchParser
   end # module SportDb
 

data/lib/sportdb/formats/match/match_status_parser.rb

@@ -0,0 +1,86 @@
+#####################
+# helpers for parsing & finding match status e.g.
+#    - cancelled / canceled
+#    - awarded
+#    - abandoned
+#    - replay
+#    etc.
+
+
+module SportDb
+
+  class Status
+# note: use a class as an "enum"-like namespace for now - why? why not?
+#   move class into Match e.g. Match::Status  - why? why not?
+    CANCELLED = 'CANCELLED'   # canceled (US spelling), cancelled (UK spelling) - what to use?
+    AWARDED   = 'AWARDED'
+    POSTPONED = 'POSTPONED'
+    ABANDONED = 'ABANDONED'
+    REPLAY    = 'REPLAY'
+  end # class Status
+
+
+
+  class StatusParser
+
+    def self.parse( str )
+      ## note: returns nil if no match found
+      ## note: english usage - cancelled (in UK), canceled (in US)
+      if str =~ /^(cancelled|
+                   canceled|
+                   can\.
+                  )/xi
+        Status::CANCELLED
+      elsif str =~ /^(awarded|
+                       awd\.
+                      )/xi
+        Status::AWARDED
+      elsif str =~ /^(postponed
+                      )/xi
+        Status::POSTPONED
+      elsif str =~ /^(abandoned|
+                       abd\.
+                      )/xi
+        Status::ABANDONED
+      elsif str =~ /^(replay
+                      )/xi
+        Status::REPLAY
+      else
+        # no match
+        nil
+      end
+    end
+
+
+    RUN_RE = /\[
+                 (?<text>[^\]]+)
+               \]
+             /x
+    def self.find!( line )
+      ## for now check all "protected" text run blocks e.g. []
+      ##  puts "line: >#{line}<"
+
+      status = nil
+
+      str = line
+      while m = str.match( RUN_RE )
+        str = m.post_match  ## keep on processing rest of line/str (a.k.a. post match string)
+
+        ## check for status match
+        match_str = m[0]  ## keep a copy of the match string (for later sub)
+        text = m[:text].strip
+        ## puts "  text: >#{text}<"
+
+        status = parse( text )
+
+        if status
+           line.sub!( match_str, "[STATUS.#{status}]" )
+           break
+        end
+      end  # while match
+
+      status
+    end # method find!
+ end # class StatusParser
+
+end # module SportDb

data/lib/sportdb/formats/name_helper.rb

@@ -46,9 +46,12 @@ module SportDb
   ##   Estudiantes (LP) => Estudiantes LP
   ##   Saint Patrick’s Athletic FC => Saint Patricks Athletic FC
   ##   Myllykosken Pallo −47 => Myllykosken Pallo 47
+  ##
+  ##  add & too!!
+  ##   e.g. Brighton & Hove Albion => Brighton Hove Albion  -- and others in England
 
   NORM_RE =  %r{
-                    [.'’º/()_−-]
+                    [.'’º/()&_−-]
                   }x   # note: in [] dash (-) if last doesn't need to get escaped
   ## note: remove all dots (.), dash (-), ', º, /, etc.
   #   .  U+002E (46) - FULL STOP

data/lib/sportdb/formats/package.rb

@@ -13,12 +13,22 @@ module SportDb
     ## leagues.txt or leagues_en.txt
     ##   remove support for en.leagues.txt - why? why not?
     LEAGUES_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
-       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.wiki.txt
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.leagues.txt
         leagues
           (?:_[a-z0-9_-]+)?
         \.txt$
     }x
 
+    ## seasons.txt or seasons_en.txt
+    ##   remove support for br.seasons.txt - why? why not?
+    SEASONS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.seasons.txt
+        seasons
+          (?:_[a-z0-9_-]+)?
+        \.txt$
+    }x
+
+
     ## clubs.txt or clubs_en.txt
     ##   remove support for en.clubs.txt - why? why not?
     CLUBS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -35,12 +45,22 @@ module SportDb
        \.wiki\.txt$
     }x
 
-    CLUB_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+    ## todo/fix: rename to CLUBS too e.g. CLUBS_PROPS to reflect filename - why? why not?
+    CLUBS_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.props.txt
         clubs
           (?:_[a-z0-9_-]+)?
         \.props\.txt$
     }x
+    CLUB_PROPS_RE = CLUBS_PROPS_RE  ## add alias for now (fix later - why? why not?)
+
+
+    CLUBS_HISTORY_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+      (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.history.txt
+        clubs
+          (?:_[a-z0-9_-]+)?
+        \.history\.txt$
+    }x
 
     ##  teams.txt or teams_history.txt
     TEAMS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -49,6 +69,8 @@ module SportDb
                       \.txt$
     }x
 
+
+    ###  todo/fix: change SEASON_RE to SEASON_KEY_RE (avoid confusion w/ SEASONS_RE for datafile?) - why? why not? !!!!!!!
     ### season folder:
     ##            e.g. /2019-20   or
     ##  year-only e.g. /2019      or
@@ -73,6 +95,10 @@ module SportDb
                        /[a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
                     }x
 
+    ### add "generic" pattern to find all csv datafiles
+    CSV_RE = %r{ (?: ^|/ )
+                   [a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
+               }x
 
 
     ## move class-level "static" finders to DirPackage (do NOT work for now for zip packages) - why? why not?
@@ -96,16 +122,22 @@ module SportDb
    def self.find_teams( path, pattern: TEAMS_RE )  find( path, pattern ); end
    def self.match_teams( path )  TEAMS_RE.match( path ); end
 
-   def self.find_clubs( path, pattern: CLUBS_RE )            find( path, pattern ); end
-   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )  find( path, pattern ); end
+   def self.find_clubs( path, pattern: CLUBS_RE )                 find( path, pattern ); end
+   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )       find( path, pattern ); end
+   def self.find_clubs_history( path, pattern: CLUBS_HISTORY_RE ) find( path, pattern ); end
 
-   def self.match_clubs( path )       CLUBS_RE.match( path ); end
-   def self.match_clubs_wiki( path )  CLUBS_WIKI_RE.match( path ); end
-   def self.match_club_props( path, pattern: CLUB_PROPS_RE ) pattern.match( path ); end
+   def self.match_clubs( path )         CLUBS_RE.match( path ); end
+   def self.match_clubs_wiki( path )    CLUBS_WIKI_RE.match( path ); end
+   def self.match_clubs_history( path ) CLUBS_HISTORY_RE.match( path); end
+   def self.match_clubs_props( path, pattern: CLUBS_PROPS_RE ) pattern.match( path ); end
 
    def self.find_leagues( path, pattern: LEAGUES_RE )  find( path, pattern ); end
    def self.match_leagues( path )  LEAGUES_RE.match( path ); end
 
+   def self.find_seasons( path, pattern: SEASONS_RE )  find( path, pattern ); end
+   def self.match_seasons( path )  SEASONS_RE.match( path ); end
+
+
    def self.find_conf( path, pattern: CONF_RE )  find( path, pattern ); end
    def self.match_conf( path )  CONF_RE.match( path ); end
 
@@ -118,6 +150,7 @@ module SportDb
    end
    ## add match_match and match_match_csv  - why? why not?
 
+
    class << self
      alias_method :match_teams?, :match_teams
      alias_method :teams?,       :match_teams
@@ -128,12 +161,21 @@ module SportDb
      alias_method :match_clubs_wiki?, :match_clubs_wiki
      alias_method :clubs_wiki?,       :match_clubs_wiki
 
-     alias_method :match_club_props?, :match_club_props
-     alias_method :club_props?,       :match_club_props
+     alias_method :match_clubs_history?, :match_clubs_history
+     alias_method :clubs_history?,       :match_clubs_history
+
+     alias_method :match_club_props,      :match_clubs_props
+     alias_method :match_club_props?,     :match_clubs_props
+     alias_method :club_props?,           :match_clubs_props
+     alias_method :match_clubs_props?,    :match_clubs_props
+     alias_method :clubs_props?,          :match_clubs_props
 
      alias_method :match_leagues?, :match_leagues
      alias_method :leagues?,       :match_leagues
 
+     alias_method :match_seasons?, :match_seasons
+     alias_method :seasons?,       :match_seasons
+
      alias_method :match_conf?, :match_conf
      alias_method :conf?,       :match_conf
    end
@@ -212,11 +254,17 @@ module SportDb
       end
     end
     def each_match_csv( &blk )  each( pattern: MATCH_CSV_RE, &blk ); end
+    def each_csv( &blk )        each( pattern: CSV_RE, &blk );       end
+
     def each_club_props( &blk ) each( pattern: CLUB_PROPS_RE, &blk ); end
 
     def each_leagues( &blk )    each( pattern: LEAGUES_RE, &blk ); end
     def each_clubs( &blk )      each( pattern: CLUBS_RE, &blk ); end
     def each_clubs_wiki( &blk ) each( pattern: CLUBS_WIKI_RE, &blk ); end
+    def each_clubs_history( &blk )  each( pattern: CLUBS_HISTORY_RE, &blk ); end
+
+    def each_seasons( &blk )    each( pattern: SEASONS_RE, &blk ); end
+
 
     ## return all match datafile entries
     def match( format: 'txt' )