sportdb-formats 1.0.6 → 1.1.4

data/lib/sportdb/formats/package.rb

@@ -13,12 +13,22 @@ module SportDb
     ## leagues.txt or leagues_en.txt
     ##   remove support for en.leagues.txt - why? why not?
     LEAGUES_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
-       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.wiki.txt
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.leagues.txt
         leagues
           (?:_[a-z0-9_-]+)?
         \.txt$
     }x
 
+    ## seasons.txt or seasons_en.txt
+    ##   remove support for br.seasons.txt - why? why not?
+    SEASONS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.seasons.txt
+        seasons
+          (?:_[a-z0-9_-]+)?
+        \.txt$
+    }x
+
+
     ## clubs.txt or clubs_en.txt
     ##   remove support for en.clubs.txt - why? why not?
     CLUBS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -35,12 +45,22 @@ module SportDb
        \.wiki\.txt$
     }x
 
-    CLUB_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+    ## todo/fix: rename to CLUBS too e.g. CLUBS_PROPS to reflect filename - why? why not?
+    CLUBS_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.props.txt
         clubs
           (?:_[a-z0-9_-]+)?
         \.props\.txt$
     }x
+    CLUB_PROPS_RE = CLUBS_PROPS_RE  ## add alias for now (fix later - why? why not?)
+
+
+    CLUBS_HISTORY_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+      (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.history.txt
+        clubs
+          (?:_[a-z0-9_-]+)?
+        \.history\.txt$
+    }x
 
     ##  teams.txt or teams_history.txt
     TEAMS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
@@ -49,6 +69,8 @@ module SportDb
                       \.txt$
     }x
 
+
+    ###  todo/fix: change SEASON_RE to SEASON_KEY_RE (avoid confusion w/ SEASONS_RE for datafile?) - why? why not? !!!!!!!
     ### season folder:
     ##            e.g. /2019-20   or
     ##  year-only e.g. /2019      or
@@ -73,6 +95,10 @@ module SportDb
                        /[a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
                     }x
 
+    ### add "generic" pattern to find all csv datafiles
+    CSV_RE = %r{ (?: ^|/ )
+                   [a-z0-9_.-]+\.csv$  ## note: allow dot (.) too e.g /eng.1.csv
+               }x
 
 
     ## move class-level "static" finders to DirPackage (do NOT work for now for zip packages) - why? why not?
@@ -96,16 +122,22 @@ module SportDb
    def self.find_teams( path, pattern: TEAMS_RE )  find( path, pattern ); end
    def self.match_teams( path )  TEAMS_RE.match( path ); end
 
-   def self.find_clubs( path, pattern: CLUBS_RE )            find( path, pattern ); end
-   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )  find( path, pattern ); end
+   def self.find_clubs( path, pattern: CLUBS_RE )                 find( path, pattern ); end
+   def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )       find( path, pattern ); end
+   def self.find_clubs_history( path, pattern: CLUBS_HISTORY_RE ) find( path, pattern ); end
 
-   def self.match_clubs( path )       CLUBS_RE.match( path ); end
-   def self.match_clubs_wiki( path )  CLUBS_WIKI_RE.match( path ); end
-   def self.match_club_props( path, pattern: CLUB_PROPS_RE ) pattern.match( path ); end
+   def self.match_clubs( path )         CLUBS_RE.match( path ); end
+   def self.match_clubs_wiki( path )    CLUBS_WIKI_RE.match( path ); end
+   def self.match_clubs_history( path ) CLUBS_HISTORY_RE.match( path); end
+   def self.match_clubs_props( path, pattern: CLUBS_PROPS_RE ) pattern.match( path ); end
 
    def self.find_leagues( path, pattern: LEAGUES_RE )  find( path, pattern ); end
    def self.match_leagues( path )  LEAGUES_RE.match( path ); end
 
+   def self.find_seasons( path, pattern: SEASONS_RE )  find( path, pattern ); end
+   def self.match_seasons( path )  SEASONS_RE.match( path ); end
+
+
    def self.find_conf( path, pattern: CONF_RE )  find( path, pattern ); end
    def self.match_conf( path )  CONF_RE.match( path ); end
 
@@ -118,6 +150,7 @@ module SportDb
    end
    ## add match_match and match_match_csv  - why? why not?
 
+
    class << self
      alias_method :match_teams?, :match_teams
      alias_method :teams?,       :match_teams
@@ -128,12 +161,21 @@ module SportDb
      alias_method :match_clubs_wiki?, :match_clubs_wiki
      alias_method :clubs_wiki?,       :match_clubs_wiki
 
-     alias_method :match_club_props?, :match_club_props
-     alias_method :club_props?,       :match_club_props
+     alias_method :match_clubs_history?, :match_clubs_history
+     alias_method :clubs_history?,       :match_clubs_history
+
+     alias_method :match_club_props,      :match_clubs_props
+     alias_method :match_club_props?,     :match_clubs_props
+     alias_method :club_props?,           :match_clubs_props
+     alias_method :match_clubs_props?,    :match_clubs_props
+     alias_method :clubs_props?,          :match_clubs_props
 
      alias_method :match_leagues?, :match_leagues
      alias_method :leagues?,       :match_leagues
 
+     alias_method :match_seasons?, :match_seasons
+     alias_method :seasons?,       :match_seasons
+
      alias_method :match_conf?, :match_conf
      alias_method :conf?,       :match_conf
    end
@@ -212,11 +254,17 @@ module SportDb
       end
     end
     def each_match_csv( &blk )  each( pattern: MATCH_CSV_RE, &blk ); end
+    def each_csv( &blk )        each( pattern: CSV_RE, &blk );       end
+
     def each_club_props( &blk ) each( pattern: CLUB_PROPS_RE, &blk ); end
 
     def each_leagues( &blk )    each( pattern: LEAGUES_RE, &blk ); end
     def each_clubs( &blk )      each( pattern: CLUBS_RE, &blk ); end
     def each_clubs_wiki( &blk ) each( pattern: CLUBS_WIKI_RE, &blk ); end
+    def each_clubs_history( &blk )  each( pattern: CLUBS_HISTORY_RE, &blk ); end
+
+    def each_seasons( &blk )    each( pattern: SEASONS_RE, &blk ); end
+
 
     ## return all match datafile entries
     def match( format: 'txt' )
@@ -287,13 +335,13 @@ module SportDb
       ##             filter.skip? filter.include? ( season_sason_key )?
       ##             fiteer.before?( season_key )  etc.
       ##              find some good method names!!!!
-      season_start = start ? Import::Season.new( start ) : nil
+      season_start = start ? Season( start ) : nil
 
       h = {}
       match( format: format ).each do |entry|
         ## note: assume last directory in datafile path is the season part/key
         season_q = File.basename( File.dirname( entry.name ))
-        season   = Import::Season.new( season_q )  ## normalize season
+        season   = Season.parse( season_q )  ## normalize season
 
         ## skip if start season before this season
         next if season_start && season_start.start_year > season.start_year

data/lib/sportdb/formats/parser_helper.rb

@@ -18,10 +18,19 @@ module SportDb
 
 
   def is_round?( line )
-    ## note: =~ return nil if not match found, and 0,1, etc for match
-    (line =~ SportDb.lang.regex_round) != nil
+    ## note: =~ returns nil if not match found, and 0,1, etc for match
+
+    ##  note: allow "free standing" leg 1 and leg 2 too
+    ##         (e.g. Hinspiel, Rückspiel etc. used for now in Relegation, for example)
+    ##    note ONLY allowed if "free standing", that is, full line with nothing else
+    ##          use "custom" regex for special case for now
+    ##               avoids match HIN in PascHINg, for example (hin in german for leg 1)
+    line =~ SportDb.lang.regex_round    ||
+    line =~ /^(#{SportDb.lang.leg1})$/i ||
+    line =~ /^(#{SportDb.lang.leg2})$/i
   end
 
+
   def is_knockout_round?( line )
 
     ## todo: check for adding ignore case for regex (e.g. 1st leg/1st Leg)

data/lib/sportdb/formats/team/club_index.rb

@@ -54,6 +54,7 @@ class ClubIndex
   ##       normalize( name )
 
   def strip_wiki( name )     # todo/check: rename to strip_wikipedia_en - why? why not?
+    ##  change/rename to strip_wiki_qualifier or such - why? why not?
     ## note: strip disambiguationn qualifier from wikipedia page name if present
     ##        note: only remove year and foot... for now
     ## e.g. FC Wacker Innsbruck (2002) => FC Wacker Innsbruck
@@ -178,22 +179,24 @@ class ClubIndex
   ## todo/fix/check: use rename to find_canon  or find_canonical() or something??
   ##  remove (getting used?) - why? why not?
   def []( name )    ## lookup by canoncial name only;  todo/fix: add find alias why? why not?
+    puts "WARN!! do not use ClubIndex#[] for lookup >#{name}< - will get removed!!!"
     @clubs[ name ]
   end
 
 
-  ## todo/fix/check: return empty array if no match!!!
-  ##     and NOT nil (add || []) - why? why not?
   def match( name )
+    # note: returns empty array (e.g. []) if no match and NOT nil
     name = normalize( name )
-    m = @clubs_by_name[ name ]
+    m = @clubs_by_name[ name ] || []
 
     ## no match - retry with unaccented variant if different
     ##    e.g. example is Preussen Münster  (with mixed accent and unaccented letters) that would go unmatched for now
     ##      Preussen Münster => preussenmünster (norm) => preussenmunster (norm+unaccent)
-    if m.nil?
+    if m.empty?
       name2 = unaccent( name )
-      m = @clubs_by_name[ name2 ]    if name2 != name
+      if name2 != name
+        m = @clubs_by_name[ name2 ] || []
+      end
     end
     m
   end
@@ -227,10 +230,8 @@ class ClubIndex
       country = country( country )
 
       ## note: match must for now always  include name
-      if m    ## filter by country
-        m = m.select { |club| club.country.key == country.key }
-        m = nil   if m.empty?     ## note: reset to nil if no more matches
-      end
+      ## filter by country
+      m = m.select { |club| club.country.key == country.key }
     end
     m
   end
@@ -263,7 +264,7 @@ class ClubIndex
 
       m = match_by( name: name, country: country )
 
-      if m.nil?
+      if m.empty?
         ## (re)try with second country - quick hacks for known leagues
         ##  todo/fix: add league flag to activate!!!  - why? why not
         m = match_by( name: name, country: 'wal' )  if country.key == 'eng'
@@ -272,6 +273,7 @@ class ClubIndex
         m = match_by( name: name, country: 'mc' )   if country.key == 'fr'
         m = match_by( name: name, country: 'li' )   if country.key == 'ch'
         m = match_by( name: name, country: 'ca' )   if country.key == 'us'
+        m = match_by( name: name, country: 'nz' )   if country.key == 'au'
       end
     else  ## try "global" search - no country passed in
       m = match( name )
@@ -279,7 +281,7 @@ class ClubIndex
 
 
     club = nil
-    if m.nil?
+    if m.empty?
       ## puts "** !!! WARN !!! no match for club >#{name}<"
     elsif m.size > 1
       puts "** !!! ERROR - too many matches (#{m.size}) for club >#{name}<:"

data/lib/sportdb/formats/team/club_index_history.rb

@@ -0,0 +1,134 @@
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+
+class ClubHistoryIndex
+
+  def self.build( path )
+    pack = Package.new( path )   ## lets us use direcotry or zip archive
+
+    recs = []
+    pack.each_clubs_history do |entry|
+      recs += ClubHistoryReader.parse( entry.read )
+    end
+    recs
+
+    index = new
+    index.add( recs )
+    index
+  end
+
+
+
+  def catalog() Import.catalog; end
+
+  ## note: keep name history for now separate from
+  ##          from club struct - why? why not?
+  ##       later yes, yes, yes, merge name history into club struct!!!!!
+  ##
+  ## for now the name history is experimental
+
+
+  def initialize
+    @clubs          = {}   ## clubs (indexed) by canonical name
+    @errors         = []
+  end
+
+  attr_reader :errors
+  def errors?() @errors.empty? == false; end
+
+  def mappings() @clubs; end   ## todo/check: rename to records or histories or something - why? why not?
+
+
+  def add_history( club_rec, keyword, season, args )
+    ## note use season obj for now (and NOT key) - why? why not?
+    rec = @clubs[ club_rec.name ] ||= []
+
+    rec << [season, [keyword, args]]
+
+    ## note: always keep records sorted by season_key for now
+    ##   check if 2010 and 2010/11 is in order using alpha sort?? (see argentina)
+    rec.sort! { |l,r| r[0] <=> l[0] }
+  end
+
+
+  def add( rec_or_recs )   ## add club record / alt_names
+    recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
+
+    recs.each do |rec|
+
+      keyword    = rec[0]
+      season_key = rec[1]
+      args       = rec[2..-1]   ## get rest of args e.g. one, two or more
+
+      ## note: for now only add (re)name history season records,
+      ##          that is, skip MERGE and BANKRUPT for now
+      ##           and incl. only RENAME, REFORM, MOVE for now
+      next if ['MERGE', 'BANKRUPT'].include?( keyword )
+
+
+      name_old = strip_geo( args[0][0] )  ## note: strip optional geo part from name
+      name_new = strip_geo( args[1][0] )
+
+      country_old = args[0][1]
+      country_new = args[1][1]
+
+      club_old = catalog.clubs.find_by!( name: name_old, country: country_old )
+      club_new = catalog.clubs.find_by!( name: name_new, country: country_new )
+
+      ## note use season obj for now (and NOT key) - why? why not?
+      season = Season.parse( season_key )
+
+      ## todo/check:
+      ##   check if  club_old and club_new reference different club record!!
+      ##    examples - RB II            -> Liefering ??  or
+      ##               FC Pasching      -> OOE Juniors ??
+      ##               Austria Salzburg -> RB Salburg ??
+      ##   for now always add name history to both - why? why not?
+
+      add_history( club_old, keyword, season, args )
+      ## note: allow for now different club references
+      ##    but maybe warn later - why? why not?
+      ## add history to both for now
+      add_history( club_new, keyword, season, args )  if club_old != club_new
+    end # each rec
+  end # method add
+
+
+  #### todo/check: move as method to club struct later - to always use club reference
+  ##  returns (simply) name as string for now or nil - why? why not?
+  #
+  #  history entry example
+  # Arsenal FC"=>
+  # [[1927/28, ["RENAME", [["The Arsenal FC, London", "eng"], ["Arsenal FC", "eng"]]]],
+  #  [1914/15, ["RENAME", [["Woolwich Arsenal FC, London", "eng"], ["The Arsenal FC", "eng"]]]],
+  #  [1892/93, ["RENAME", [["Royal Arsenal FC, London", "eng"], ["Woolwich Arsenal FC", "eng"]]]]],
+  def find_name_by( name:, season: )
+    recs = @clubs[ name ]
+    if recs
+      season = Season( season )   ## make sure season is a season obj (and NOT a string)
+      ## check season records for name; use linear search (assume only few records)
+      recs.each do |rec|
+        if season >= rec[0]
+           return strip_geo( rec[1][1][1][0] )  # use second arg
+        end
+      end
+      ## if we get here use last name
+      strip_geo( recs[-1][1][1][0][0] )   # use first arg
+    else
+      nil
+    end
+  end
+
+  ##################
+  ## helpers
+  def strip_geo( name )
+    ## e.g. Arsenal, London   =>   Arsenal
+    name.split(',')[0].strip
+  end
+end # class ClubHistoryIndex
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/formats/team/club_reader_history.rb

@@ -0,0 +1,203 @@
+# encoding: utf-8
+
+
+module SportDb
+module Import
+
+
+class ClubHistoryReader
+
+  def catalog() Import.catalog; end
+
+
+
+def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+  parse( txt )
+end
+
+def self.parse( txt )
+  new( txt ).parse
+end
+
+def initialize( txt )
+  @txt = txt
+end
+
+
+###
+## RENAME/RENAMED
+## MOVE/MOVED
+## BANKRUPT/BANKRUPTED
+## REFORM/REFORMED
+## MERGE/MERGED    - allow + or ++ or +++ or ; for "inline" - why? why not?
+
+
+KEYWORD_LINE_RE = %r{ ^(?<keyword>RENAMED?|
+                               MOVED?|
+                               BANKRUPT(?:ED)?|
+                               REFORM(?:ED)?|
+                               MERGED?
+                    )
+                      [ ]+
+                     (?<text>.*)    # rest of text
+                    $
+                  }x
+
+
+def parse
+  recs = []
+  last_rec  = nil
+
+  last_country = nil
+  last_season  = nil
+  last_keyword = nil
+  last_teams   = []
+
+  OutlineReader.parse( @txt ).each do |node|
+    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
+      heading_level  = node[0][1].to_i
+      heading        = node[1]
+
+      puts "heading #{heading_level} >#{heading}<"
+
+
+        if heading_level == 1
+            ## assume country in heading; allow all "formats" supported by parse e.g.
+            ##   Österreich • Austria (at)
+            ##   Österreich • Austria
+            ##   Austria
+            ##   Deutschland (de) • Germany
+            country = catalog.countries.parse( heading )
+            ## check country code - MUST exist for now!!!!
+            if country.nil?
+              puts "!!! error [club history reader] - unknown country >#{heading}< - sorry - add country to config to fix"
+              exit 1
+            end
+            puts "  country >#{heading}< => #{country.name}, #{country.key}"
+            last_country = country
+            last_season  = nil  ## reset "lower levels" - season & keyword
+            last_keyword = nil
+         elsif heading_level == 2
+            ## assume season
+            season = Season.parse( heading )
+            puts "  season >#{heading}< => #{season.key}"
+            last_season  = season  ## reset "lowwer levels" - keyword
+            last_keyword = nil
+         else
+            puts "!!! ERROR [club history reader] - for now only heading 1 & 2 supported; sorry"
+            exit 1
+         end
+
+    elsif node[0] == :p   ## paragraph with (text) lines
+      if last_country.nil?
+        puts "!!! ERROR [club history reader] - country heading 1 required, sorry"
+        exit 1
+      end
+      if last_season.nil?
+        puts "!!! ERROR [club history reader] - season heading 2 required, sorry"
+        exit 1
+      end
+
+      lines = node[1]
+      lines.each do |line|
+        if m=line.match(KEYWORD_LINE_RE)   ## extract keyword and continue
+          keyword = m[:keyword]
+          line    = m[:text].strip
+
+          puts "    keyword #{keyword}"
+          last_keyword = case keyword   ## "normalize" keywords
+                         when 'BANKRUPT', 'BANKRUPTED'
+                           'BANKRUPT'
+                         when 'RENAME', 'RENAMED'
+                           'RENAME'
+                         when 'REFORM', 'REFORMED'
+                           'REFORM'
+                         when 'MOVE',   'MOVED'
+                           'MOVE'
+                         when 'MERGE',  'MERGED'
+                           'MERGE'
+                         else
+                           puts "!!! ERROR [club history reader] - unexpected keyword >#{keyword}<; sorry - don't know how to normalize"
+                           exit 1
+                         end
+
+          last_teams   = []
+        end
+
+        if last_keyword.nil?
+          puts "!!! ERROR [club history reader] - line with keyword expected - got:"
+          puts line
+          exit 1
+        end
+
+        if    last_keyword == 'BANKRUPT'
+           ## requires / expects one team in one line
+           recs << [ last_keyword, last_season.key,
+                     [ squish(line), last_country.key ]
+                   ]
+        elsif last_keyword == 'RENAME' ||
+              last_keyword == 'REFORM' ||
+              last_keyword == 'MOVE'
+           ## requires / expects two teams in one line (separated by ⇒ or such)
+           teams = line.split( '⇒' )
+           if teams.size != 2
+            puts "!!! ERROR [club history reader] - expected two teams - got:"
+            pp teams
+            exit 1
+           end
+           teams = teams.map {|team| squish(team.strip) }  ## remove whitespaces
+           recs << [ last_keyword, last_season.key,
+                     [ teams[0], last_country.key ],
+                     [ teams[1], last_country.key ]
+                   ]
+        elsif last_keyword == 'MERGE'
+            ## check if line starts with separator
+            ## otherwise collect to be merged teams
+            if line.start_with?( '⇒' )
+              if last_teams.size < 2
+                puts "!!! ERROR [club history reader] - expected two or more teams for MERGE - got:"
+                pp last_teams
+                exit 1
+              end
+              ## auto-add country to all teams
+              teams = last_teams.map {|team| [team, last_country.key]}
+              recs << [ last_keyword, last_season.key,
+                        teams,
+                        [ squish(line.sub('⇒','').strip), last_country.key ]
+                      ]
+
+              last_teams = []
+            else
+              last_teams << squish(line)
+            end
+        else
+          puts "!!! ERROR [club history reader] - unknown keyword >#{last_keyword}<; cannot process; sorry"
+          exit 1
+        end
+      end  # each line (in paragraph)
+    else
+      puts "** !!! ERROR [club history reader] - unknown line type:"
+      pp node
+      exit 1
+    end
+  end
+
+  recs
+end  # method read
+
+
+###############
+## helper
+
+def squish( str )
+  ## colapse all whitespace to one
+  str.gsub( /[ ]+/,' ' )
+end
+
+
+end  # class ClubHistoryReader
+
+
+end ## module Import
+end ## module SportDb