sportdb-formats 2.0.2 → 2.1.0

data/lib/sportdb/formats/event/event_reader.rb

@@ -1,119 +0,0 @@
-
-module SportDb
-module Import
-  class EventInfoReader
-    def catalog() Import.catalog; end
-
-
-    def self.read( path )
-      txt = File.open( path, 'r:utf-8') {|f| f.read }
-      new( txt ).parse
-    end
-
-    def self.parse( txt )
-      new( txt ).parse
-    end
-
-    def initialize( txt )
-      @txt = txt
-    end
-
-    def parse
-      recs = []
-
-      parse_csv( @txt ).each do |row|
-        league_col = row['League']
-        season_col = row['Season'] || row['Year']
-        dates_col  = row['Dates']
-
-        season = Season.parse( season_col )
-        league = catalog.leagues.find!( league_col )
-
-
-        dates = []
-        if dates_col.nil? || dates_col.empty?
-           ## do nothing; no dates - keep dates array empty
-        else
-          ## squish spaces
-          dates_col = dates_col.gsub( /[ ]{2,}/, ' ' )  ## squish/fold spaces
-
-          puts "#{league.name} (#{league.key}) | #{season.key} | #{dates_col}"
-
-          ### todo/check:  check what parts "Aug 15" return ???
-          ###                       short form for "Aug 15 -" - works?
-
-          ## todo/fix!!! - check EventInfo.include?
-          ##    now allow dates with only start_date too!! (WITHOUT end_date)
-          parts = dates_col.split( /[ ]*[–-][ ]*/ )
-          if parts.size == 1
-            pp parts
-            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
-            pp dates
-          elsif parts.size == 2
-            pp parts
-            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
-            dates << DateFormats.parse( parts[1], start: Date.new( season.end_year ? season.end_year : season.start_year, 1, 1 ), lang: 'en' )
-            pp dates
-
-            ## assert/check if period is less than 365 days for now
-            diff = dates[1].to_date.jd - dates[0].to_date.jd
-            puts "#{diff}d"
-            if diff > 365
-              puts "!! ERROR - date range / period assertion failed; expected diff < 365 days"
-              exit 1
-            end
-          else
-            puts "!! ERRROR - expected data range / period - one or two dates; got #{parts.size}:"
-            pp dates_col
-            pp parts
-            exit 1
-          end
-        end
-
-
-        teams_col    = row['Clubs'] || row['Teams']
-        goals_col    = row['Goals']
-
-        ## note: remove (and allow) all non-digits e.g. 370 goals, 20 clubs, etc.
-        teams_col    = teams_col.gsub( /[^0-9]/, '' )    if teams_col
-        goals_col    = goals_col.gsub( /[^0-9]/, '' )    if goals_col
-
-        teams  =   (teams_col.nil? || teams_col.empty?) ? nil : teams_col.to_i
-        goals  =   (goals_col.nil? || goals_col.empty?) ? nil : goals_col.to_i
-
-        matches_col  = row['Matches']
-        ## note: support additions in matches (played) e.g.
-        #   132 + 63 Play-off-Spiele
-        matches_col   = matches_col.gsub( /[^0-9+]/, '' )  if matches_col
-
-        matches =  if matches_col.nil? || matches_col.empty?
-                     nil
-                   else
-                     if matches_col.index( '+' )  ### check for calculations
-                        ## note: for now only supports additions
-                        matches_col.split( '+' ).reduce( 0 ) do |sum,str|
-                          sum + str.to_i
-                        end
-                     else            ## assume single (integer) number
-                        matches_col.to_i
-                     end
-                  end
-
-        rec = EventInfo.new( league:     league,
-                             season:     season,
-                             start_date: dates[0],
-                             end_date:   dates[1],
-                             teams:      teams,
-                             matches:    matches,
-                             goals:      goals
-                           )
-        recs << rec
-      end  # each row
-      recs
-    end  # method parse
-  end  # class EventInfoReader
-
-
-end ## module Import
-end ## module SportDb
-

data/lib/sportdb/formats/ground/ground_reader.rb

@@ -1,289 +0,0 @@
-###
-#  todo -   based on ClubReader
-#               share GeoReader or BaseReader or such for both
-#                  plus maybe for PlayerReader too!!!
-#
-#  fix/todo/cleanup  - move alt_names_auto from reader to indexer!!!!
-#                          indexer now handles unaccent (variants) etc.
-
-module SportDb
-module Import
-
-
-class GroundReader
-
-  def world() Import.world; end
-
-
-def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
-  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
-  parse( txt )
-end
-
-def self.parse( txt )
-  new( txt ).parse
-end
-
-def initialize( txt )
-  @txt = txt
-end
-
-
-## pattern for checking for address line e.g.
-##    use just one style / syntax - why? why not?
-##  Fischhofgasse 12 ~ 1100 Wien or
-##  Fischhofgasse 12 // 1100 Wien   or Fischhofgasse 12 /// 1100 Wien
-##  Fischhofgasse 12 ++ 1100 Wien   or Fischhofgasse 12 +++ 1100 Wien
-ADDR_MARKER_RE =  %r{ (?: ^|[ ] )                # space or beginning of line
-                        (?: ~ | /{2,} | \+{2,} )
-                      (?: [ ]|$)                 # space or end of line
-                       }x
-
-
-def parse
-  recs = []
-  last_rec  = nil
-  headings = []   ## headings stack
-
-  OutlineReader.parse( @txt ).each do |node|
-    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
-      heading_level  = node[0][1].to_i
-      heading        = node[1]
-
-      puts "heading #{heading_level} >#{heading}<"
-
-      ## 1) first pop headings if present
-      while headings.size+1 > heading_level
-        headings.pop
-      end
-
-      ## 2) add missing (hierarchy) level if
-      while headings.size+1 < heading_level
-        ##  todo/fix: issue warning about "skipping" hierarchy level
-        puts "!!! warn [ground reader] - skipping hierarchy level in headings "
-        headings.push( nil )
-      end
-
-      if heading =~ /^\?+$/    ## note: use ? or ?? or ?? to reset level to nil
-        ## keep level empty
-      else
-        ## note: if level is 1 assume country for now
-        if heading_level == 1
-            ## assume country in heading; allow all "formats" supported by parse e.g.
-            ##   Österreich • Austria (at)
-            ##   Österreich • Austria
-            ##   Austria
-            ##   Deutschland (de) • Germany
-            country = world.countries.parse( heading )
-            ## check country code - MUST exist for now!!!!
-            if country.nil?
-              puts "!!! error [ground reader] - unknown country >#{heading}< - sorry - add country to config to fix"
-              exit 1
-            end
-
-            headings.push( country.key )
-        else
-         ## quick hack:
-         ##   remove known fill/dummy words incl:
-         ##     Provincia San Juan  =>  San Juan   (see argentina, for example)
-         ##
-         ##   use geo tree long term with alternative names - why? why not?
-          words = ['Provincia']
-          words.each { |word| heading = heading.gsub( word, '' ) }
-          heading = heading.strip
-
-          headings.push( heading )
-        end
-
-        ## assert that hierarchy level is ok
-        if headings.size != heading_level
-          puts "!!! error - headings hierarchy/stack out of order - #{heading.size}<=>#{heading_level}"
-          exit 1
-        end
-      end
-
-      pp headings
-
-    elsif node[0] == :p   ## paragraph with (text) lines
-      lines = node[1]
-      lines.each do |line|
-      if line.start_with?( '|' )
-        ## assume continuation with line of alternative names
-        ##  note: skip leading pipe
-        values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
-        values = values.map {|value| _norm(value) }  ## squish/strip etc.
-
-        last_rec.alt_names += values
-
-       ## check for address line e.g.
-       ##    use just one style / syntax - why? why not?
-       ##  Fischhofgasse 12 ~ 1100 Wien or
-       ##  Fischhofgasse 12 // 1100 Wien or Fischhofgasse 12 /// 1100 Wien
-       ##  Fischhofgasse 12 ++ 1100 Wien or Fischhofgasse 12 +++ 1100 Wien
-      elsif line =~ ADDR_MARKER_RE
-        ## squish line here - why? why not?
-        last_rec.address = _squish( line )
-      else
-        values = line.split( ',' )
-
-        rec = Ground.new
-
-        col  = values.shift    ## get first item
-        ## note: allow optional alt names for convenience with required canoncial name
-        names = col.split( '|' )   # team names - allow/use pipe(|)
-        names = names.map {|name| _norm(name) }  ## squish/strip etc.
-
-        value     = names[0]         ## canonical name
-        alt_names = names[1..-1]     ## optional (inline) alt names
-
-        rec.name = value            # canoncial name (global unique "beautiful/long" name)
-        ## note: add optional (inline) alternate names if present
-        rec.alt_names += alt_names    if alt_names.size > 0
-
-        ## note:
-        ##   check/todo!!!!!!!!!!!!!!!!!-
-        ##  strip year if to present e.g. (2011-)
-        ##
-        ##  do NOT strip for defunct / historic clubs e.g.
-        ##    (1899-1910)
-        ## or (-1914) or (-2011) etc.
-
-        ###
-        ##  todo: move year out of canonical team name - why? why not?
-
-        ## check if canonical name include (2011-) or similar in name
-        ##   if yes, remove (2011-) and add to (alt) names
-        ##   e.g. New York FC (2011) => New York FC
-        if rec.name =~ /\(.+?\)/   ## note: use non-greedy (?) match
-          name = rec.name.gsub( /\(.+?\)/, '' ).strip
-
-          if rec.name =~ /\(([0-9]{4})-\)/            ## e.g. (2014-)
-            rec.year     = $1.to_i
-          elsif  rec.name =~ /\(-([0-9]{4})\)/            ## e.g. (-2014)
-            rec.year_end = $1.to_i
-          elsif  rec.name =~ /\(([0-9]{4})-([0-9]{4})\)/  ## e.g. (2011-2014)
-            rec.year     = $1.to_i
-            rec.year_end = $2.to_i
-          else
-            ## todo/check: warn about unknown year format
-          end
-        end
-
-        ##  todo/check - check for unknown format values
-        ##    e.g. too many values, duplicate years, etc.
-        ##         check for overwritting, etc.
-
-        ##  strip and squish (white)spaces
-        #   e.g. León     › Guanajuato     => León › Guanajuato
-        values = values.map {|value| _squish(value) }
-
-        while values.size > 0
-          value = values.shift
-          if value =~/^\d{4}$/   # e.g 1904
-            ## todo/check: issue warning if year is already set!!!!!!!
-            if rec.year
-              puts "!!! error - year already set to #{rec.year} - CANNOT overwrite with #{value}:"
-              pp rec
-              exit 1
-            end
-            rec.year  = value.to_i
-          elsif value =~/^[0-9_]+$/   # e.g 1904
-            ##  skip capacity for now
-          else
-            ## assume city / geo tree
-            ## split into geo tree
-            geos = split_geo( value )
-            city = geos[0]
-            ## check for "embedded" district e.g. London (Fulham) or Hamburg (St. Pauli) etc.
-            if city =~ /\((.+?)\)/   ## note: use non-greedy (?) match
-              rec.district  = $1.strip
-              city          = city.gsub( /\(.+?\)/, '' ).strip
-            end
-            rec.city = city
-
-            if geos.size > 1
-               ## cut-off city and keep the rest (of geo tree)
-               rec.geos = geos[1..-1]
-            end
-          end
-        end  ## while values
-
-
-        ###############
-        ## use headings text for geo tree
-
-        ## 1) add country if present
-        if headings.size > 0 && headings[0]
-          country = world.countries.find( headings[0] )
-          rec.country = country
-        else
-          ## make it an error - why? why not?
-          puts "!!! error - country missing in headings hierarchy - sorry - add to quicklist"
-          exit 1
-        end
-
-        ## 2) check geo tree with headings hierarchy
-        if headings.size > 1 && headings[1]
-           geos = split_geo( headings[1] )
-           if rec.geos
-             if rec.geos[0] != geos[0]
-               puts "!!! error - geo tree - headings mismatch >#{rec.geos[0]}< <=> >#{geos[0]}<"
-               exit 1
-             end
-             if rec.geos[1] && rec.geos[1] != geos[1]   ## check optional 2nd level too
-               puts "!!! error - geo tree - headings mismatch >#{rec.geos[1]}< <=> >#{geos[1]}<"
-               exit 1
-             end
-           else
-             ## add missing region (state/province) from headings hierarchy
-             rec.geos = geos
-           end
-        end
-
-        last_rec = rec
-
-        recs << rec
-      end
-      end  # each line (in paragraph)
-    else
-      puts "** !!! ERROR !!! [ground reader] - unknown line type:"
-      pp node
-      exit 1
-    end
-  end
-
-  recs
-end  # method read
-
-#######################
-###  helpers
-
-def split_geo( str )
-  ## assume city / geo tree
-  ##  strip and squish (white)spaces
-  #   e.g. León     › Guanajuato     => León › Guanajuato
-  str = _squish( str )
-
-  ## split into geo tree
-  geos = str.split( /[<>‹›]/ )   ## note: allow > < or › ‹
-  geos = geos.map { |geo| geo.strip }   ## remove all whitespaces
-  geos
-end
-
-## norm(alize) helper  - squish (spaces) 
-##                      and remove dollars ($$$)
-##                      and remove leading and trailing spaces
-def _norm( str )
-  ## only extra clean-up of dollars for now ($$$)
-  _squish( str.gsub( '$', '' ) )
-end
-
-def _squish( str )
-  str.gsub( /[ \t\u00a0]+/, ' ' ).strip
-end
-
-end  # class GroundReader
-
-
-end ## module Import
-end ## module SportDb

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -1,176 +0,0 @@
-# encoding: utf-8
-
-
-module SportDb
-
-## shared "higher-level" outline reader
-##  todo: add CountryOutlineReader - why? why not?
-
-class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeaguePageOutlineReader - why? why not?
-
-  def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
-    parse( txt, season: season )
-  end
-
-  def self.parse( txt, season: nil )
-    new( txt ).parse( season: season )
-  end
-
-
-  def initialize( txt )
-    @txt = txt
-  end
-
-  def parse( season: nil )
-    secs=[]     # sec(tion)s
-    OutlineReader.parse( @txt ).each do |node|
-      if node[0] == :h1
-        ## check for league (and stage) and season
-        heading = node[1]
-        values = split_league( heading )
-        if m=values[0].match( LEAGUE_SEASON_HEADING_RE )
-          puts "league >#{m[:league]}<, season >#{m[:season]}<"
-
-           secs << { league: m[:league],
-                     season: m[:season],
-                     stage:  values[1],     ## note: defaults to nil if not present
-                     lines:  []
-                   }
-        else
-          puts "** !!! ERROR - cannot match league and season in heading; season missing?"
-          pp heading
-          exit 1
-        end
-      elsif node[0] == :p   ## paragraph with (text) lines
-        lines = node[1]
-        ## note: skip lines if no heading seen
-        if secs.empty?
-          puts "** !!! WARN - skipping lines (no heading):"
-          pp lines
-        else
-          ## todo/check: unroll paragraphs into lines or pass along paragraphs - why? why not?
-          secs[-1][:lines] += lines
-        end
-      else
-        puts "** !!! ERROR - unknown line type; for now only heading 1 for leagues supported; sorry:"
-        pp node
-        exit 1
-      end
-    end
-
-
-    ## pass 2 - filter seasons if filter present
-    if season
-       filtered_secs = []
-       filter = norm_seasons( season )
-       secs.each do |sec|
-         if filter.include?( Season.parse( sec[:season] ).key )
-           filtered_secs << sec
-         else
-           puts "  skipping season >#{sec[:season]}< NOT matched by filter"
-         end
-       end
-       secs = filtered_secs
-    end
-
-    ## pass 3 - check & map; replace inline (string with data struct record)
-    secs.each do |sec|
-      sec[:season] = Season.parse( sec[:season ] )
-      sec[:league] = catalog.leagues.find!( sec[:league] )
-
-      check_stage( sec[:stage] )   if sec[:stage]   ## note: only check for now (no remapping etc.)
-    end
-
-    secs
-  end # method parse
-
-
-
-  def catalog() Import.catalog; end    ## shortcut convenience helper
-
-  ## split into league + season
-  ##  e.g. Österr. Bundesliga 2015/16   ## or 2015-16
-  ##       World Cup 2018
-  LEAGUE_SEASON_HEADING_RE =  %r{^
-         (?<league>.+?)     ## non-greedy
-            \s+
-         (?<season>\d{4}
-            (?:[\/-]\d{1,4})?     ## optional 2nd year in season
-         )
-            $}x
-
-
-  def norm_seasons( season_or_seasons )     ## todo/check: add alias norm_seasons - why? why not?
-
-      seasons = if season_or_seasons.is_a?( Array )  # is it an array already
-                  season_or_seasons
-                elsif season_or_seasons.is_a?( Range )  # e.g. Season(1999)..Season(2001) or such
-                  season_or_seasons.to_a
-                else  ## assume - single entry - wrap in array
-                  [season_or_seasons]
-                end
-
-      seasons.map { |season| Season( season ).key }
-  end
-
-
-  def split_league( str )   ## todo/check: rename to parse_league(s) - why? why not?
-    ## split into league / stage / ... e.g.
-    ##  => Österr. Bundesliga 2018/19, Regular Season
-    ##  => Österr. Bundesliga 2018/19, Championship Round
-    ##  etc.
-    values = str.split( /[,<>‹›]/ )  ## note: allow , > < or › ‹ for now
-    values = values.map { |value| value.strip }   ## remove all whitespaces
-    values
-  end
-
-
-  # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
-  KNOWN_STAGES = [
-    'Regular Season',
-    'Regular Stage',
-    'Championship Round',
-    'Championship Playoff',  # or Championship play-off
-    'Relegation Round',
-    'Relegation Playoff',
-    'Play-offs',
-    'Playoff Stage',
-    'Grunddurchgang',
-    'Finaldurchgang - Qualifikationsgruppe',
-    'Finaldurchgang - Qualifikation',
-    'Finaldurchgang - Meistergruppe',
-    'Finaldurchgang - Meister',
-    'EL Play-off',
-    'Europa League Play-off',
-    'Europa-League-Play-offs',
-    'Europa League Finals',
-    'Playoffs - Championship',
-    'Playoffs - Europa League',
-    'Playoffs - Europa League - Finals',
-    'Playoffs - Relegation',
-    'Playoffs - Challenger',
-    'Finals',
-    'Match 6th Place',  # e.g. Super League Greece 2012/13
-
-    'Apertura',
-    'Apertura - Liguilla',
-    'Clausura',
-    'Clausura - Liguilla',
-
-  ].map {|name| name.downcase.gsub( /[^a-z]/, '' ) }
-
-
-  def check_stage( name )
-    # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
-    if KNOWN_STAGES.include?( name.downcase.gsub( /[^a-z]/, '' ) )
-       ## everything ok
-    else
-      puts "** !!! ERROR - no (league) stage match found for >#{name}<, add to (builtin) stages table; sorry"
-      exit 1
-    end
-  end
-
- end # class LeagueOutlineReader
-
-end # module SportDb