sportdb-formats 1.0.4 → 1.1.2

data/lib/sportdb/formats/match/mapper_teams.rb

@@ -16,7 +16,7 @@ class TeamMapper
   end
 
   def map_teams!( line )
-    @mapper.map_titles!( line )
+    @mapper.map_names!( line )
   end
 end # class TeamMapper
 

data/lib/sportdb/formats/match/match_parser.rb

@@ -82,15 +82,14 @@ class MatchParser   ## simple match parser for team match schedules
     #    team1 team2 - match  (will get new auto-matchday! not last round)
     @last_round     = nil
 
-    title, pos = find_group_title_and_pos!( line )
+    name = find_group_name!( line )
 
-    logger.debug "    title: >#{title}<"
-    logger.debug "    pos: >#{pos}<"
+    logger.debug "    name: >#{name}<"
     logger.debug "  line: >#{line}<"
 
-    group = @groups[ title ]
+    group = @groups[ name ]
     if group.nil?
-      puts "!! ERROR - no group def found for >#{title}<"
+      puts "!! ERROR - no group def found for >#{name}<"
       exit 1
     end
 
@@ -104,19 +103,19 @@ class MatchParser   ## simple match parser for team match schedules
     @mapper_teams.map_teams!( line )
     teams = @mapper_teams.find_teams!( line )
 
-    title, pos = find_group_title_and_pos!( line )
+    name = find_group_name!( line )
 
     logger.debug "  line: >#{line}<"
 
-    group = Import::Group.new( pos: pos,
-                               title: title,
-                               teams: teams.map {|team| team.title } )
+    ## todo/check/fix: add back group key - why? why not?
+    group = Import::Group.new( name:  name,
+                               teams: teams.map {|team| team.name } )
 
-    @groups[ title ] = group
+    @groups[ name ] = group
   end
 
 
-  def find_group_title_and_pos!( line )
+  def find_group_name!( line )
     ## group pos - for now support single digit e.g 1,2,3 or letter e.g. A,B,C or HEX
     ## nb:  (?:)  = is for non-capturing group(ing)
 
@@ -125,37 +124,25 @@ class MatchParser   ## simple match parser for team match schedules
 
     ## todo:
     ##   check if Group A:  or [Group A]  works e.g. : or ] get matched by \b ???
-    regex = /(?:Group|Gruppe|Grupo)\s+((?:\d{1}|[A-Z]{1,3}))\b/
+    regex = /\b
+              (?:
+                (Group | Gruppe | Grupo)
+                   [ ]+
+                (\d+ | [A-Z]+)
+              )
+            \b/x
 
     m = regex.match( line )
 
-    return [nil,nil] if m.nil?
-
-    pos = case m[1]
-          when 'A' then 1
-          when 'B' then 2
-          when 'C' then 3
-          when 'D' then 4
-          when 'E' then 5
-          when 'F' then 6
-          when 'G' then 7
-          when 'H' then 8
-          when 'I' then 9
-          when 'J' then 10
-          when 'K' then 11
-          when 'L' then 12
-          when 'HEX' then 666    # HEX for Hexagonal - todo/check: map to something else ??
-          else  m[1].to_i
-          end
-
-    title = m[0]
-
-    logger.debug "   title: >#{title}<"
-    logger.debug "   pos: >#{pos}<"
-
-    line.sub!( regex, '[GROUP.TITLE+POS]' )
-
-    [title,pos]
+    return nil    if m.nil?
+
+    name = m[0]
+
+    logger.debug "   name: >#{name}<"
+
+    line.sub!( name, '[GROUP.NAME]' )
+
+    name
   end
 
 
@@ -180,198 +167,130 @@ class MatchParser   ## simple match parser for team match schedules
     end_date   = end_date.to_date
 
 
-    pos   = find_round_pos!( line )
-    title = find_round_def_title!( line )
-    # NB: use extracted round title for knockout check
-    knockout_flag = is_knockout_round?( title )
+    name  = find_round_def_name!( line )
+    # NB: use extracted round name for knockout check
+    knockout_flag = is_knockout_round?( name )
 
 
     logger.debug "    start_date: #{start_date}"
     logger.debug "    end_date:   #{end_date}"
-    logger.debug "    pos:      #{pos}"
-    logger.debug "    title:    >#{title}<"
+    logger.debug "    name:    >#{name}<"
     logger.debug "    knockout_flag:   #{knockout_flag}"
 
     logger.debug "  line: >#{line}<"
 
-    #######################################
-    # todo/fix: add auto flag is false !!!! - why? why not?
-    round = Import::Round.new( pos:        pos,
-                               title:      title,
+    round = Import::Round.new( name:       name,
                                start_date: start_date,
                                end_date:   end_date,
                                knockout:   knockout_flag,
                                auto:       false )
 
-    @rounds[ title ] = round
+    @rounds[ name ] = round
   end
 
 
 
-  def find_round_pos!( line )
-    # pass #1) extract optional round pos from line
-    # e.g.  (1)   - must start line
-    regex_pos = /^[ \t]*\((\d{1,3})\)[ \t]+/
-
-    # pass #2) find free standing number  e.g. Matchday 3 or Round 5 or 3. Spieltag etc.
-    # note: /\b(\d{1,3})\b/
-    #   will match -12
-    #  thus, use space required - will NOT match  -2 e.g. Group-2 Play-off
-    #  note:  allow  1. Runde  n
-    #                1^ Giornata
-    regex_num = /(?:^|\s)(\d{1,3})(?:[.\^\s]|$)/
+  def find_round_def_name!( line )
+    # assume everything before pipe (\) is the round name
+    #  strip [ROUND.POS],  todo:?? [ROUND.NAME2]
 
-    if line =~ regex_pos
-      logger.debug "   pos: >#{$1}<"
-
-      line.sub!( regex_pos, '[ROUND.POS] ' )  ## NB: add back trailing space that got swallowed w/ regex -> [ \t]+
-      return $1.to_i
-    elsif line =~ regex_num
-      ## assume number in title is pos (e.g. Jornada 3, 3 Runde etc.)
-      ## NB: do NOT remove pos from string (will get removed by round title)
-
-      num = $1.to_i  # note: clone capture; keep a copy (another regex follows; will redefine $1)
-
-      #### fix:
-      #  use/make keywords required
-      #  e.g. Round of 16  -> should NOT match 16!
-      #    Spiel um Platz 3  (or 5) etc -> should NOT match 3!
-      #  Round 16 - ok
-      #  thus, check for required keywords
-
-      ## quick hack for round of 16
-      # todo: mask match e.g. Round of xxx ... and try again - might include something
-      #  reuse pattern for Group XX Replays for example
-      if line =~ /^\s*Round of \d{1,3}\b/
-         return nil
-      end
-
-      logger.debug "   pos: >#{num}<"
-      return num
-    else
-      ## fix: add logger.warn no round pos found in line
-      return nil
-    end
-  end # method find_round_pos!
-
-  def find_round_def_title!( line )
-    # assume everything before pipe (\) is the round title
-    #  strip [ROUND.POS],  todo:?? [ROUND.TITLE2]
-
-    # todo/fix: add title2 w/  // or /  why? why not?
+    # todo/fix: add name2 w/  // or /  why? why not?
     #  -- strip / or / chars
 
     buf = line.dup
-    logger.debug "  find_round_def_title! line-before: >>#{buf}<<"
+    logger.debug "  find_round_def_name! line-before: >>#{buf}<<"
 
     ## cut-off everything after (including) pipe (|)
     buf = buf[ 0...buf.index('|') ]
-
-    # e.g. remove [ROUND.POS], [ROUND.TITLE2], [GROUP.TITLE+POS] etc.
-    buf.gsub!( /\[[^\]]+\]/, '' )    ## fix: use helper for (re)use e.g. remove_match_placeholder/marker or similar?
-    # remove leading and trailing whitespace
     buf.strip!
 
-    logger.debug "  find_round_def_title! line-after: >>#{buf}<<"
+    logger.debug "  find_round_def_name! line-after: >>#{buf}<<"
 
-    logger.debug "   title: >>#{buf}<<"
-    line.sub!( buf, '[ROUND.TITLE]' )
+    logger.debug "   name: >>#{buf}<<"
+    line.sub!( buf, '[ROUND.NAME]' )
 
     buf
   end
 
-  def find_round_header_title!( line )
-    # assume everything left is the round title
-    #  extract all other items first (round title2, round pos, group title n pos, etc.)
 
-    ## todo/fix:
-    ##  cleanup method
-    ##   use  buf.index( '//' ) to split string (see found_round_def)
-    ##     why? simpler why not?
-    ##  - do we currently allow groups if title2 present? add example if it works?
+    ##  split by or || or |||
+    ##           or ++ or +++
+    ##           or -- or ---
+    ##           or // or ///
+    ##  note: allow Final | First Leg  as ONE name same as
+    ##              Final - First Leg or
+    ##              Final, First Leg
+    ##   for cut-off always MUST be more than two chars
+    ##
+    ##  todo/check: find a better name than HEADER_SEP(ARATOR) - why? why not?
+    ##   todo/fix: move to parser utils and add a method split_name or such?
+    HEADER_SEP_RE = /  [ ]*      ## allow (strip) leading spaces
+                      (?:\|{2,} |
+                          \+{2,} |
+                           -{2,} |
+                          \/{2,}
+                      )
+                      [ ]*       ## allow (strip) trailing spaces
+                  /x
+
+  def find_round_header_name!( line )
+    # assume everything left is the round name
+    #  extract all other items first (round name2, round pos, group name n pos, etc.)
 
     buf = line.dup
-    logger.debug "  find_round_header_title! line-before: >>#{buf}<<"
+    logger.debug "  find_round_header_name! line-before: >>#{buf}<<"
+
+
+    parts = buf.split( HEADER_SEP_RE )
+    buf = parts[0]
 
-    buf.gsub!( /\[[^\]]+\]/, '' )   # e.g. remove [ROUND.POS], [ROUND.TITLE2], [GROUP.TITLE+POS] etc.
     buf.strip!    # remove leading and trailing whitespace
 
-    logger.debug "  find_round_title! line-after: >>#{buf}<<"
+    logger.debug "  find_round_name! line-after: >>#{buf}<<"
 
-    ### bingo - assume what's left is the round title
+    ### bingo - assume what's left is the round name
 
-    logger.debug "   title: >>#{buf}<<"
-    line.sub!( buf, '[ROUND.TITLE]' )
+    logger.debug "   name: >>#{buf}<<"
+    line.sub!( buf, '[ROUND.NAME]' )
 
     buf
   end
 
+    ## quick hack- collect all "fillwords" by language!!!!
+    ##    change later  and add to sportdb-langs!!!!
+    ##
+    ##    strip all "fillwords" e.g.:
+    ##      Nachtrag/Postponed/Addition/Supplemento names
+    ##
+    ##  todo/change: find a better name for ROUND_EXTRA_WORDS - why? why not?
+    ROUND_EXTRA_WORDS_RE = /\b(?:
+                               Nachtrag |     ## de
+                               Postponed |    ## en
+                               Addition  |    ## en
+                               Supplemento    ## es
+                              )
+                             \b/ix
 
   def parse_round_header( line )
     logger.debug "parsing round header line: >#{line}<"
 
-    ## todo/check/fix:
-    #   make sure  Round of 16  will not return pos 16 -- how? possible?
-    #   add unit test too to verify
-    pos = find_round_pos!( line )
-
-    title = find_round_header_title!( line )
+    name = find_round_header_name!( line )
 
     logger.debug "  line: >#{line}<"
 
+    name = name.sub( ROUND_EXTRA_WORDS_RE, '' )
+    name = name.strip
 
-    round = @rounds[ title ]
+    round = @rounds[ name ]
     if round.nil?    ## auto-add / create if missing
-      round = Import::Round.new( pos:   pos,
-                                 title: title )
-      @rounds[ title ] = round
+      ## todo/check: add num (was pos) if present - why? why not?
+      round = Import::Round.new( name: name )
+      @rounds[ name ] = round
     end
 
     ## todo/check: if pos match (MUST always match for now)
     @last_round = round
     @last_group = nil   # note: reset group to no group - why? why not?
-
-
-    ## NB: dummy/placeholder start_at, end_at date
-    ##  replace/patch after adding all games for round
-
-=begin
-    round_attribs = {
-      title:  title,
-      title2: title2,
-      knockout: knockout_flag
-    }
-
-    if pos > 999000
-      # no pos (e.g. will get autonumbered later) - try match by title for now
-      #  e.g. lets us use title 'Group Replays', for example, multiple times
-      @round = Round.find_by_event_id_and_title( @event.id, title )
-    else
-      @round = Round.find_by_event_id_and_pos( @event.id, pos )
-    end
-
-    if @round.present?
-      logger.debug "update round #{@round.id}:"
-    else
-      logger.debug "create round:"
-      @round = Round.new
-
-      round_attribs = round_attribs.merge( {
-        event_id: @event.id,
-        pos:   pos,
-        start_at: Date.parse('1911-11-11'),
-        end_at:   Date.parse('1911-11-11')
-      })
-    end
-
-    logger.debug round_attribs.to_json
-
-    @round.update_attributes!( round_attribs )
-
-    @patch_round_ids_pos   << @round.id    if pos > 999000
-    ### store list of round ids for patching start_at/end_at at the end
-    @patch_round_ids_dates << @round.id   # todo/fix/check: check if round has definition (do NOT patch if definition (not auto-added) present)
-=end
   end
 
 
@@ -457,11 +376,11 @@ class MatchParser   ## simple match parser for team match schedules
     ## todo/check: pass along round and group refs or just string (canonical names) - why? why not?
 
     @matches << Import::Match.new( date:    date,
-                                   team1:   team1,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team1.title)
-                                   team2:   team2,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team2.title)
+                                   team1:   team1,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team1.name)
+                                   team2:   team2,  ## note: for now always use mapping value e.g. rec (NOT string e.g. team2.name)
                                    score:   score,
-                                   round:   round       ? round.title       : nil,   ## note: for now always use string (assume unique canonical name for event)
-                                   group:   @last_group ? @last_group.title : nil )  ## note: for now always use string (assume unique canonical name for event)
+                                   round:   round       ? round.name       : nil,   ## note: for now always use string (assume unique canonical name for event)
+                                   group:   @last_group ? @last_group.name : nil )  ## note: for now always use string (assume unique canonical name for event)
 
     ### todo: cache team lookups in hash?
 
@@ -517,7 +436,7 @@ class MatchParser   ## simple match parser for team match schedules
 
         round_attribs = {
           event_id: @event.id,
-          title: "Matchday #{date.to_date}",
+          name: "Matchday #{date.to_date}",
           pos: 999001+@patch_round_ids_pos.length,   # e.g. 999<count> - 999001,999002,etc.
           start_at:  date.to_date,
           end_at:    date.to_date
@@ -541,7 +460,7 @@ class MatchParser   ## simple match parser for team match schedules
       end
 
       ## note: will crash (round.pos) if round is nil
-      logger.debug( "  using round #{round.pos} >#{round.title}< start_at: #{round.start_at}, end_at: #{round.end_at}" )
+      logger.debug( "  using round #{round.pos} >#{round.name}< start_at: #{round.start_at}, end_at: #{round.end_at}" )
     else
       ## use round from last round header
       round = @round
@@ -644,12 +563,29 @@ class MatchParser   ## simple match parser for team match schedules
 
     if date && team1.nil? && team2.nil?
       logger.debug( "date header line found: >#{line}<")
-      logger.debug( "    date: #{date}")
+      logger.debug( "    date: #{date} with start: #{@start}")
 
       @last_date = date   # keep a reference for later use
-      return true
+
+      ###  quick "corona" hack - support seasons going beyond 12 month (see swiss league 2019/20 and others!!)
+      ##    find a better way??
+      ##  set @start date to full year (e.g. 1.1.) if date.year  is @start.year+1
+      ##   todo/fix: add to linter to check for chronological dates!! - warn if NOT chronological
+      ###  todo/check: just turn on for 2019/20 season or always? why? why not?
+
+      ## todo/fix: add switch back to old @start_org
+      ##   if year is date.year == @start.year-1    -- possible when full date with year set!!!
+      if @start.month != 1
+         if date.year == @start.year+1
+           logger.debug( "!! hack - extending start date to full (next/end) year; assumes all dates are chronologigal - always moving forward" )
+           @start_org = @start   ## keep a copy of the original (old) start date - why? why not? - not used for now
+           @start = Date.new( @start.year+1, 1, 1 )
+         end
+      end
+
+      true
     else
-      return false
+      false
     end
   end
 

data/lib/sportdb/formats/match/match_parser_csv.rb

@@ -95,11 +95,14 @@ module SportDb
          headers_mapping[:score]  = find_header( headers, ['FT'] )
          headers_mapping[:scorei] = find_header( headers, ['HT'] )
 
-         headers_mapping[:round]  = find_header( headers, ['Round'] )
+         headers_mapping[:round]  = find_header( headers, ['Round', 'Matchday'] )
 
          ## optional headers - note: find_header returns nil if header NOT found
          header_stage = find_header( headers, ['Stage'] )
          headers_mapping[:stage]  =  header_stage   if header_stage
+
+         header_league = find_header( headers, ['League'] )
+         headers_mapping[:league] = header_league   if header_league
       else
          ## else try footballdata.uk and others
          headers_mapping[:team1]  = find_header( headers, ['HomeTeam', 'HT', 'Home'] )
@@ -167,7 +170,10 @@ module SportDb
 
 
       ## check if data present - if not skip (might be empty row)
-      if team1.nil? && team2.nil?
+      ##  note:  (old classic) csv reader returns nil for empty fields
+      ##         new modern csv reader ALWAYS returns strings (and empty strings for data not available (n/a))
+      if (team1.nil? || team1.empty?) &&
+         (team2.nil? || team2.empty?)
         puts "*** WARN: skipping empty? row[#{i}] - no teams found:"
         pp row
         next
@@ -182,9 +188,11 @@ module SportDb
       col = row[ headers_mapping[ :date ]]
       col = col.strip   # make sure not leading or trailing spaces left over
 
-      if col.empty? || col == '-' || col == '?'
-         ## note: allow missing / unknown date for match
-         date = nil
+      if col.empty? ||
+         col =~ /^-{1,}$/ ||      # e.g.  - or ---
+         col =~ /^\?{1,}$/        # e.g. ? or ???
+          ## note: allow missing / unknown date for match
+          date = nil
       else
         ## remove possible weekday or weeknumber  e.g. (Fri) (4) etc.
         col = col.sub( /\(W?\d{1,2}\)/, '' )  ## e.g. (W11), (4), (21) etc.
@@ -199,6 +207,8 @@ module SportDb
           date_fmt = '%Y-%m-%d'   # e.g. 1995-08-04
         elsif col =~ /^\d{1,2} \w{3} \d{4}$/
           date_fmt = '%d %b %Y'   # e.g. 8 Jul 2017
+        elsif col =~ /^\w{3} \w{3} \d{1,2} \d{4}$/
+          date_fmt = '%a %b %d %Y'   # e.g. Sat Aug 7 1993
         else
           puts "*** !!! wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
           ## todo/fix: add to errors/warns list - why? why not?
@@ -211,6 +221,9 @@ module SportDb
       end
 
 
+      ##
+      ## todo/fix:  round might not always be just a simple integer number!!!
+      ##             might be text such as Final | Leg 1 or such!!!!
       round   = nil
       ## check for (optional) round / matchday
       if headers_mapping[ :round ]
@@ -283,13 +296,17 @@ module SportDb
                  end
       end
 
+      league = nil
+      league = row[ headers_mapping[ :league ]]   if headers_mapping[ :league ]
+
 
       match = Import::Match.new( date:    date,
                                  team1:   team1,   team2:   team2,
                                  score1:  score1,  score2:  score2,
                                  score1i: score1i, score2i: score2i,
                                  round:   round,
-                                 stage:   stage )
+                                 stage:   stage,
+                                 league:  league )
       matches << match
     end