sportdb-formats 1.0.0 → 1.0.1

data/lib/sportdb/formats/league/league_index.rb

@@ -0,0 +1,174 @@
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+class LeagueIndex
+
+  def self.build( path )
+    pack = Package.new( path )   ## lets us use direcotry or zip archive
+
+    recs = []
+    pack.each_leagues do |entry|
+      recs += League.parse( entry.read )
+    end
+    recs
+
+    leagues = new
+    leagues.add( recs )
+    leagues
+  end
+
+
+  def catalog() Import.catalog; end
+
+  def initialize
+    @leagues         = []   ## leagues by canonical name
+    @leagues_by_name = {}
+    @errors          = []
+  end
+
+  attr_reader :errors
+  def errors?() @errors.empty? == false; end
+
+  def mappings()   @leagues_by_name; end   ## todo/check: rename to index or something - why? why not?
+  def leagues()    @leagues.values;  end
+  alias_method :all, :leagues   ## use ActiveRecord-like alias for leagues
+
+
+  ## helpers from club - use a helper module for includes - why? why not?
+  include NameHelper
+  ## incl. strip_lang( name )
+  ##       normalize( name )
+
+
+  def add( rec_or_recs )   ## add club record / alt_names
+    recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
+
+    recs.each do |rec|
+      ## puts "adding:"
+      ## pp rec
+      ### step 1) add canonical name
+      @leagues << rec
+
+      ## step 2) add all names (canonical name + alt names + alt names (auto))
+      names = [rec.name] + rec.alt_names
+      ## check for duplicates - simple check for now - fix/improve
+      ## todo/fix: (auto)remove duplicates - why? why not?
+      count      = names.size
+      count_uniq = names.uniq.size
+      if count != count_uniq
+        puts "** !!! ERROR !!! - #{count-count_uniq} duplicate name(s):"
+        pp names
+        pp rec
+        exit 1
+      end
+
+      ## todo/fix: move alt_names_auto up for check unique names
+      ##    e.g. remove/avoid auto-generated duplicates ENG 1, AUT 1, etc
+      names += rec.alt_names_auto
+
+      names.each_with_index do |name,i|
+        ## check lang codes e.g. [en], [fr], etc.
+        ##  todo/check/fix:  move strip_lang up in the chain - check for duplicates (e.g. only lang code marker different etc.) - why? why not?
+        name = strip_lang( name )
+        norm = normalize( name )
+        alt_recs = @leagues_by_name[ norm ]
+        if alt_recs
+          ## check if include club rec already or is new club rec
+          if alt_recs.include?( rec )
+            ## note: do NOT include duplicate club record
+            msg = "** !!! WARN !!! - (norm) name conflict/duplicate for league - >#{name}< normalized to >#{norm}< already included >#{rec.name}, #{rec.country ? rec.country.key : '?'}<"
+            puts msg
+            @errors << msg
+          else
+            msg = "** !!! WARN !!! - name conflict/duplicate - >#{name}< will overwrite >#{alt_recs[0].name}, #{alt_recs[0].country ? alt_recs[0].country.key : '?'}< with >#{rec.name}, #{rec.country ? rec.country.key : '?'}<"
+            puts msg
+            @errors << msg
+            alt_recs << rec
+          end
+        else
+          @leagues_by_name[ norm ] = [rec]
+        end
+      end
+    end
+  end # method add
+
+
+  def match( name )
+    ## todo/check: return empty array if no match!!! and NOT nil (add || []) - why? why not?
+    name = normalize( name )
+    @leagues_by_name[ name ]
+  end
+
+
+  def match_by( name:, country: )
+    ## note: match must for now always include name
+    m = match( name )
+    if m    ## filter by country
+      ## note: country assumes / allows the country key or fifa code for now
+
+      ## note: allow passing in of country struct too
+      country_rec = if country.is_a?( Country )
+                       country   ## (re)use country struct - no need to run lookup again
+                    else
+                       ## note:  use own "global" countries index setting for ClubIndex - why? why not?
+                       rec = catalog.countries.find( country )
+                       if rec.nil?
+                         puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
+                         exit 1
+                       end
+                       rec
+                    end
+
+      ## note: also skip international leagues & cups (e.g. champions league etc.) for now - why? why not?
+      m = m.select { |league| league.country &&
+                              league.country.key == country_rec.key }
+      m = nil   if m.empty?     ## note: reset to nil if no more matches
+    end
+    m
+  end
+
+
+  def find!( name )
+    league = find( name )
+    if league.nil?
+      puts "** !!! ERROR - no league match found for >#{name}<, add to leagues table; sorry"
+      exit 1
+    end
+    league
+  end
+
+  def find( name )
+    league = nil
+    m = match( name )
+    # pp m
+
+    if m.nil?
+      ## fall through/do nothing
+    elsif m.size > 1
+      puts "** !!! ERROR - ambigious league name; too many leagues (#{m.size}) found:"
+      pp m
+      exit 1
+    else
+      league = m[0]
+    end
+
+    league
+  end
+
+
+
+
+  def dump_duplicates # debug helper - report duplicate club name records
+     @leagues_by_name.each do |name, leagues|
+       if leagues.size > 1
+         puts "#{leagues.size} matching leagues duplicates for >#{name}<:"
+         pp leagues
+       end
+     end
+  end
+end # class LeagueIndex
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -0,0 +1,141 @@
+# encoding: utf-8
+
+
+module SportDb
+
+## shared "higher-level" outline reader
+##  todo: add CountryOutlineReader - why? why not?
+
+class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeaguePageOutlineReader - why? why not?
+
+  def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
+    parse( txt, season: season )
+  end
+
+  def self.parse( txt, season: nil )
+    new( txt ).parse( season: season )
+  end
+
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+  def parse( season: nil )
+    secs=[]     # sec(tion)s
+    OutlineReader.parse( @txt ).each do |node|
+      if node[0] == :h1
+        ## check for league (and stage) and season
+        heading = node[1]
+        values = split_league( heading )
+        if m=values[0].match( LEAGUE_SEASON_HEADING_RE )
+          puts "league >#{m[:league]}<, season >#{m[:season]}<"
+
+           secs << { league: m[:league],
+                     season: m[:season],
+                     stage:  values[1],     ## note: defaults to nil if not present
+                     lines:  []
+                   }
+        else
+          puts "** !!! ERROR - cannot match league and season in heading; season missing?"
+          pp heading
+          exit 1
+        end
+      elsif node[0] == :p   ## paragraph with (text) lines
+        lines = node[1]
+        ## note: skip lines if no heading seen
+        if secs.empty?
+          puts "** !!! WARN - skipping lines (no heading):"
+          pp lines
+        else
+          ## todo/check: unroll paragraphs into lines or pass along paragraphs - why? why not?
+          secs[-1][:lines] += lines
+        end
+      else
+        puts "** !!! ERROR - unknown line type; for now only heading 1 for leagues supported; sorry:"
+        pp node
+        exit 1
+      end
+    end
+
+
+    ## pass 2 - filter seasons if filter present
+    if season
+       filtered_secs = []
+       filter = norm_seasons( season )
+       secs.each do |sec|
+         if filter.include?( Import::Season.new( sec[:season] ).key )
+           filtered_secs << sec
+         else
+           puts "  skipping season >#{sec[:season]}< NOT matched by filter"
+         end
+       end
+       secs = filtered_secs
+    end
+
+    ## pass 3 - check & map; replace inline (string with data struct record)
+    secs.each do |sec|
+      sec[:season] = Import::Season.new( sec[:season ] )
+      sec[:league] = catalog.leagues.find!( sec[:league] )
+
+      check_stage( sec[:stage] )   if sec[:stage]   ## note: only check for now (no remapping etc.)
+    end
+
+    secs
+  end # method parse
+
+
+
+  def catalog() Import.catalog; end    ## shortcut convenience helper
+
+  ## split into league + season
+  ##  e.g. Österr. Bundesliga 2015/16   ## or 2015-16
+  ##       World Cup 2018
+  LEAGUE_SEASON_HEADING_RE =  %r{^
+         (?<league>.+?)     ## non-greedy
+            \s+
+         (?<season>\d{4}
+            (?:[\/-]\d{1,4})?     ## optional 2nd year in season
+         )
+            $}x
+
+  def norm_seasons( season_or_seasons )     ## todo/check: add alias norm_seasons - why? why not?
+      seasons = if season_or_seasons.is_a?( String )   ## wrap in array
+                  [season_or_seasons]
+                else  ## assume it's an array already
+                   season_or_seasons
+                end
+
+      seasons.map { |season| Import::Season.new( season ).key }
+  end
+
+
+  def split_league( str )   ## todo/check: rename to parse_league(s) - why? why not?
+    ## split into league / stage / ... e.g.
+    ##  => Österr. Bundesliga 2018/19, Regular Season
+    ##  => Österr. Bundesliga 2018/19, Championship Round
+    ##  etc.
+    values = str.split( /[,<>‹›]/ )  ## note: allow , > < or › ‹ for now
+    values = values.map { |value| value.strip }   ## remove all whitespaces
+    values
+  end
+
+  def check_stage( name )
+    known_stages = ['regular season',
+                    'championship round',
+                    'relegation round',
+                    'play-offs'
+                   ]
+
+    if known_stages.include?( name.downcase )
+       ## everything ok
+    else
+      puts "** !!! ERROR - no (league) stage match found for >#{name}<, add to (builtin) stages table; sorry"
+      exit 1
+    end
+  end
+
+ end # class LeagueOutlineReader
+
+end # module SportDb

data/lib/sportdb/formats/league/league_reader.rb

@@ -0,0 +1,162 @@
+# encoding: utf-8
+
+
+module SportDb
+module Import
+
+
+class LeagueReader
+
+  def catalog() Import.catalog; end
+
+
+def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+  parse( txt )
+end
+
+def self.parse( txt )
+  new( txt ).parse
+end
+
+
+
+include Logging
+
+def initialize( txt )
+  @txt = txt
+end
+
+def parse
+  recs = []
+  last_rec = nil
+
+  country  = nil    # last country
+  intl     = false  # is international (league/tournament/cup/competition)
+  clubs    = true   # or clubs|national teams
+
+  OutlineReader.parse( @txt ).each do |node|
+    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
+      heading_level  = node[0][1].to_i
+      heading        = node[1]
+
+      logger.debug "heading #{heading_level} >#{heading}<"
+
+      if heading_level != 1
+        puts "** !!! ERROR !!! unsupported heading level; expected heading 1 for now only; sorry"
+        pp line
+        exit 1
+      else
+        logger.debug "heading (#{heading_level}) >#{heading}<"
+        last_heading = heading
+        ## map to country or international / int'l or national teams
+        if heading =~ /national team/i   ## national team tournament
+          country = nil
+          intl    = true
+          clubs   = false
+        elsif heading =~ /international|int'l/i  ## int'l club tournament
+          country = nil
+          intl    = true
+          clubs   = true
+        else
+          ## assume country in heading; allow all "formats" supported by parse e.g.
+          ##   Österreich • Austria (at)
+          ##   Österreich • Austria
+          ##   Austria
+          ##   Deutschland (de) • Germany
+          country = catalog.countries.parse( heading )
+          intl    = false
+          clubs   = true
+
+          ## check country code - MUST exist for now!!!!
+          if country.nil?
+            puts "!!! error [league reader] - unknown country >#{heading}< - sorry - add country to config to fix"
+            exit 1
+          end
+        end
+      end
+    elsif node[0] == :p   ## paragraph with (text) lines
+      lines = node[1]
+      lines.each do |line|
+
+      if line.start_with?( '|' )
+          ## assume continuation with line of alternative names
+          ##  note: skip leading pipe
+          values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
+          ## 1) strip (commerical) sponsor markers/tags e.g. $$ Liga $$BBV$$ MX
+          ## 2) strip and  squish (white)spaces
+          #   e.g. New York FC      (2011-)  => New York FC (2011-)
+          values = values.map { |value| value.gsub( '$', '' )
+                                             .gsub( /[ \t]+/, ' ' )
+                                             .strip  }
+          logger.debug "alt_names: #{values.join( '|' )}"
+
+          last_rec.alt_names += values
+      else
+        ## assume "regular" line
+        ##  check if starts with id  (todo/check: use a more "strict"/better regex capture pattern!!!)
+        if line =~ /^([a-z0-9][a-z0-9.]*)[ ]+(.+)$/
+          league_key  = $1
+          ## 1) strip (commercial) sponsor markers/tags e.g $$
+          ## 2) strip and squish (white)spaces
+          league_name = $2.gsub( '$', '' )
+                          .gsub( /[ \t]+/, ' ' )
+                          .strip
+
+          logger.debug "key: >#{league_key}<, name: >#{league_name}<"
+
+
+          alt_names_auto = []
+          if country
+            alt_names_auto << "#{country.key.upcase} #{league_key.upcase.gsub('.', ' ')}"
+            ## todo/check: add "hack" for cl (chile) and exclude?
+            ##             add a list of (auto-)excluded country codes with conflicts? why? why not?
+            ##                 cl - a) Chile  b) Champions League
+            alt_names_auto << "#{country.key.upcase}"   if league_key == '1'   ## add shortcut for top level 1 (just country key)
+            if country.key.upcase != country.fifa
+              alt_names_auto << "#{country.fifa} #{league_key.upcase.gsub('.', ' ')}"
+              alt_names_auto << "#{country.fifa}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
+            end
+            alt_names_auto << "#{country.name} #{league_key}"  if league_key =~ /^[0-9]+$/   ## if all numeric e.g. add Austria 1 etc.
+          else   ## assume int'l (no country) e.g. champions league, etc.
+            ## only auto-add key (e.g. CL, EL, etc.)
+            alt_names_auto << league_key.upcase.gsub('.', ' ')   ## note: no country code (prefix/leading) used
+          end
+
+          pp alt_names_auto
+
+          ## prepend country key/code if country present
+          ##   todo/fix: only auto-prepend country if key/code start with a number (level) or incl. cup
+          ##    why? lets you "overwrite" key if desired - use it - why? why not?
+          if country
+            league_key = "#{country.key}.#{league_key}"
+          end
+
+          rec = League.new( key:            league_key,
+                            name:           league_name,
+                            alt_names_auto: alt_names_auto,
+                            country:        country,
+                            intl:           intl,
+                            clubs:          clubs)
+          recs << rec
+          last_rec = rec
+        else
+          puts "** !!! ERROR !!! missing key for (canonical) league name"
+          exit 1
+        end
+      end
+      end  # each line
+    else
+      puts "** !!! ERROR !!! [league reader] - unknown line type:"
+      pp node
+      exit 1
+    end
+    ## pp line
+  end
+  recs
+end # method parse
+
+end # class LeagueReader
+
+end ## module Import
+end ## module SportDb