sportdb-formats 1.1.6 → 1.2.0

data/lib/sportdb/formats/ground/ground_reader.rb

@@ -0,0 +1,289 @@
+###
+#  todo -   based on ClubReader
+#               share GeoReader or BaseReader or such for both
+#                  plus maybe for PlayerReader too!!!
+#
+#  fix/todo/cleanup  - move alt_names_auto from reader to indexer!!!!
+#                          indexer now handles unaccent (variants) etc.
+
+module SportDb
+module Import
+
+
+class GroundReader
+
+  def world() Import.world; end
+
+
+def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+  parse( txt )
+end
+
+def self.parse( txt )
+  new( txt ).parse
+end
+
+def initialize( txt )
+  @txt = txt
+end
+
+
+## pattern for checking for address line e.g.
+##    use just one style / syntax - why? why not?
+##  Fischhofgasse 12 ~ 1100 Wien or
+##  Fischhofgasse 12 // 1100 Wien   or Fischhofgasse 12 /// 1100 Wien
+##  Fischhofgasse 12 ++ 1100 Wien   or Fischhofgasse 12 +++ 1100 Wien
+ADDR_MARKER_RE =  %r{ (?: ^|[ ] )                # space or beginning of line
+                        (?: ~ | /{2,} | \+{2,} )
+                      (?: [ ]|$)                 # space or end of line
+                       }x
+
+
+def parse
+  recs = []
+  last_rec  = nil
+  headings = []   ## headings stack
+
+  OutlineReader.parse( @txt ).each do |node|
+    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
+      heading_level  = node[0][1].to_i
+      heading        = node[1]
+
+      puts "heading #{heading_level} >#{heading}<"
+
+      ## 1) first pop headings if present
+      while headings.size+1 > heading_level
+        headings.pop
+      end
+
+      ## 2) add missing (hierarchy) level if
+      while headings.size+1 < heading_level
+        ##  todo/fix: issue warning about "skipping" hierarchy level
+        puts "!!! warn [ground reader] - skipping hierarchy level in headings "
+        headings.push( nil )
+      end
+
+      if heading =~ /^\?+$/    ## note: use ? or ?? or ?? to reset level to nil
+        ## keep level empty
+      else
+        ## note: if level is 1 assume country for now
+        if heading_level == 1
+            ## assume country in heading; allow all "formats" supported by parse e.g.
+            ##   Österreich • Austria (at)
+            ##   Österreich • Austria
+            ##   Austria
+            ##   Deutschland (de) • Germany
+            country = world.countries.parse( heading )
+            ## check country code - MUST exist for now!!!!
+            if country.nil?
+              puts "!!! error [ground reader] - unknown country >#{heading}< - sorry - add country to config to fix"
+              exit 1
+            end
+
+            headings.push( country.key )
+        else
+         ## quick hack:
+         ##   remove known fill/dummy words incl:
+         ##     Provincia San Juan  =>  San Juan   (see argentina, for example)
+         ##
+         ##   use geo tree long term with alternative names - why? why not?
+          words = ['Provincia']
+          words.each { |word| heading = heading.gsub( word, '' ) }
+          heading = heading.strip
+
+          headings.push( heading )
+        end
+
+        ## assert that hierarchy level is ok
+        if headings.size != heading_level
+          puts "!!! error - headings hierarchy/stack out of order - #{heading.size}<=>#{heading_level}"
+          exit 1
+        end
+      end
+
+      pp headings
+
+    elsif node[0] == :p   ## paragraph with (text) lines
+      lines = node[1]
+      lines.each do |line|
+      if line.start_with?( '|' )
+        ## assume continuation with line of alternative names
+        ##  note: skip leading pipe
+        values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
+        values = values.map {|value| _norm(value) }  ## squish/strip etc.
+
+        last_rec.alt_names += values
+
+       ## check for address line e.g.
+       ##    use just one style / syntax - why? why not?
+       ##  Fischhofgasse 12 ~ 1100 Wien or
+       ##  Fischhofgasse 12 // 1100 Wien or Fischhofgasse 12 /// 1100 Wien
+       ##  Fischhofgasse 12 ++ 1100 Wien or Fischhofgasse 12 +++ 1100 Wien
+      elsif line =~ ADDR_MARKER_RE
+        ## squish line here - why? why not?
+        last_rec.address = _squish( line )
+      else
+        values = line.split( ',' )
+
+        rec = Ground.new
+
+        col  = values.shift    ## get first item
+        ## note: allow optional alt names for convenience with required canoncial name
+        names = col.split( '|' )   # team names - allow/use pipe(|)
+        names = names.map {|name| _norm(name) }  ## squish/strip etc.
+
+        value     = names[0]         ## canonical name
+        alt_names = names[1..-1]     ## optional (inline) alt names
+
+        rec.name = value            # canoncial name (global unique "beautiful/long" name)
+        ## note: add optional (inline) alternate names if present
+        rec.alt_names += alt_names    if alt_names.size > 0
+
+        ## note:
+        ##   check/todo!!!!!!!!!!!!!!!!!-
+        ##  strip year if to present e.g. (2011-)
+        ##
+        ##  do NOT strip for defunct / historic clubs e.g.
+        ##    (1899-1910)
+        ## or (-1914) or (-2011) etc.
+
+        ###
+        ##  todo: move year out of canonical team name - why? why not?
+
+        ## check if canonical name include (2011-) or similar in name
+        ##   if yes, remove (2011-) and add to (alt) names
+        ##   e.g. New York FC (2011) => New York FC
+        if rec.name =~ /\(.+?\)/   ## note: use non-greedy (?) match
+          name = rec.name.gsub( /\(.+?\)/, '' ).strip
+
+          if rec.name =~ /\(([0-9]{4})-\)/            ## e.g. (2014-)
+            rec.year     = $1.to_i
+          elsif  rec.name =~ /\(-([0-9]{4})\)/            ## e.g. (-2014)
+            rec.year_end = $1.to_i
+          elsif  rec.name =~ /\(([0-9]{4})-([0-9]{4})\)/  ## e.g. (2011-2014)
+            rec.year     = $1.to_i
+            rec.year_end = $2.to_i
+          else
+            ## todo/check: warn about unknown year format
+          end
+        end
+
+        ##  todo/check - check for unknown format values
+        ##    e.g. too many values, duplicate years, etc.
+        ##         check for overwritting, etc.
+
+        ##  strip and squish (white)spaces
+        #   e.g. León     › Guanajuato     => León › Guanajuato
+        values = values.map {|value| _squish(value) }
+
+        while values.size > 0
+          value = values.shift
+          if value =~/^\d{4}$/   # e.g 1904
+            ## todo/check: issue warning if year is already set!!!!!!!
+            if rec.year
+              puts "!!! error - year already set to #{rec.year} - CANNOT overwrite with #{value}:"
+              pp rec
+              exit 1
+            end
+            rec.year  = value.to_i
+          elsif value =~/^[0-9_]+$/   # e.g 1904
+            ##  skip capacity for now
+          else
+            ## assume city / geo tree
+            ## split into geo tree
+            geos = split_geo( value )
+            city = geos[0]
+            ## check for "embedded" district e.g. London (Fulham) or Hamburg (St. Pauli) etc.
+            if city =~ /\((.+?)\)/   ## note: use non-greedy (?) match
+              rec.district  = $1.strip
+              city          = city.gsub( /\(.+?\)/, '' ).strip
+            end
+            rec.city = city
+
+            if geos.size > 1
+               ## cut-off city and keep the rest (of geo tree)
+               rec.geos = geos[1..-1]
+            end
+          end
+        end  ## while values
+
+
+        ###############
+        ## use headings text for geo tree
+
+        ## 1) add country if present
+        if headings.size > 0 && headings[0]
+          country = world.countries.find( headings[0] )
+          rec.country = country
+        else
+          ## make it an error - why? why not?
+          puts "!!! error - country missing in headings hierarchy - sorry - add to quicklist"
+          exit 1
+        end
+
+        ## 2) check geo tree with headings hierarchy
+        if headings.size > 1 && headings[1]
+           geos = split_geo( headings[1] )
+           if rec.geos
+             if rec.geos[0] != geos[0]
+               puts "!!! error - geo tree - headings mismatch >#{rec.geos[0]}< <=> >#{geos[0]}<"
+               exit 1
+             end
+             if rec.geos[1] && rec.geos[1] != geos[1]   ## check optional 2nd level too
+               puts "!!! error - geo tree - headings mismatch >#{rec.geos[1]}< <=> >#{geos[1]}<"
+               exit 1
+             end
+           else
+             ## add missing region (state/province) from headings hierarchy
+             rec.geos = geos
+           end
+        end
+
+        last_rec = rec
+
+        recs << rec
+      end
+      end  # each line (in paragraph)
+    else
+      puts "** !!! ERROR !!! [ground reader] - unknown line type:"
+      pp node
+      exit 1
+    end
+  end
+
+  recs
+end  # method read
+
+#######################
+###  helpers
+
+def split_geo( str )
+  ## assume city / geo tree
+  ##  strip and squish (white)spaces
+  #   e.g. León     › Guanajuato     => León › Guanajuato
+  str = _squish( str )
+
+  ## split into geo tree
+  geos = str.split( /[<>‹›]/ )   ## note: allow > < or › ‹
+  geos = geos.map { |geo| geo.strip }   ## remove all whitespaces
+  geos
+end
+
+## norm(alize) helper  - squish (spaces) 
+##                      and remove dollars ($$$)
+##                      and remove leading and trailing spaces
+def _norm( str )
+  ## only extra clean-up of dollars for now ($$$)
+  _squish( str.gsub( '$', '' ) )
+end
+
+def _squish( str )
+  str.gsub( /[ \t\u00a0]+/, ' ' ).strip
+end
+
+end  # class GroundReader
+
+
+end ## module Import
+end ## module SportDb

data/lib/sportdb/formats/league/league_reader.rb

@@ -1,168 +1,152 @@
-# encoding: utf-8
-
-
-module SportDb
-module Import
-
-
-class LeagueReader
-
-  def catalog() Import.catalog; end
-
-
-def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
-  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
-  parse( txt )
-end
-
-def self.parse( txt )
-  new( txt ).parse
-end
-
-
-
-include Logging
-
-def initialize( txt )
-  @txt = txt
-end
-
-def parse
-  recs = []
-  last_rec = nil
-
-  country  = nil    # last country
-  intl     = false  # is international (league/tournament/cup/competition)
-  clubs    = true   # or clubs|national teams
-
-  OutlineReader.parse( @txt ).each do |node|
-    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
-      heading_level  = node[0][1].to_i
-      heading        = node[1]
-
-      logger.debug "heading #{heading_level} >#{heading}<"
-
-      if heading_level != 1
-        puts "** !!! ERROR !!! unsupported heading level; expected heading 1 for now only; sorry"
-        pp line
-        exit 1
-      else
-        logger.debug "heading (#{heading_level}) >#{heading}<"
-        last_heading = heading
-        ## map to country or international / int'l or national teams
-        if heading =~ /national team/i   ## national team tournament
-          country = nil
-          intl    = true
-          clubs   = false
-        elsif heading =~ /international|int'l/i  ## int'l club tournament
-          country = nil
-          intl    = true
-          clubs   = true
-        else
-          ## assume country in heading; allow all "formats" supported by parse e.g.
-          ##   Österreich • Austria (at)
-          ##   Österreich • Austria
-          ##   Austria
-          ##   Deutschland (de) • Germany
-          country = catalog.countries.parse( heading )
-          intl    = false
-          clubs   = true
-
-          ## check country code - MUST exist for now!!!!
-          if country.nil?
-            puts "!!! error [league reader] - unknown country >#{heading}< - sorry - add country to config to fix"
-            exit 1
-          end
-        end
-      end
-    elsif node[0] == :p   ## paragraph with (text) lines
-      lines = node[1]
-      lines.each do |line|
-
-      if line.start_with?( '|' )
-          ## assume continuation with line of alternative names
-          ##  note: skip leading pipe
-          values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
-          ## 1) strip (commerical) sponsor markers/tags e.g. $$ Liga $$BBV$$ MX
-          ## 2) strip and  squish (white)spaces
-          #   e.g. New York FC      (2011-)  => New York FC (2011-)
-          values = values.map { |value| value.gsub( '$', '' )
-                                             .gsub( /[ \t]+/, ' ' )
-                                             .strip  }
-          logger.debug "alt_names: #{values.join( '|' )}"
-
-          last_rec.alt_names += values
-      else
-        ## assume "regular" line
-        ##  check if starts with id  (todo/check: use a more "strict"/better regex capture pattern!!!)
-        if line =~ /^([a-z0-9][a-z0-9.]*)[ ]+(.+)$/
-          league_key  = $1
-          ## 1) strip (commercial) sponsor markers/tags e.g $$
-          ## 2) strip and squish (white)spaces
-          league_name = $2.gsub( '$', '' )
-                          .gsub( /[ \t]+/, ' ' )
-                          .strip
-
-          logger.debug "key: >#{league_key}<, name: >#{league_name}<"
-
-
-          alt_names_auto = []
-          if country
-            alt_names_auto << "#{country.key.upcase} #{league_key.upcase.gsub('.', ' ')}"
-            ## todo/check: add "hack" for cl (chile) and exclude?
-            ##             add a list of (auto-)excluded country codes with conflicts? why? why not?
-            ##                 cl - a) Chile  b) Champions League
-            alt_names_auto << "#{country.key.upcase}"   if league_key == '1'   ## add shortcut for top level 1 (just country key)
-            if country.key.upcase != country.code
-              alt_names_auto << "#{country.code} #{league_key.upcase.gsub('.', ' ')}"
-              alt_names_auto << "#{country.code}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
-            end
-            alt_names_auto << "#{country.name} #{league_key}"  if league_key =~ /^[0-9]+$/   ## if all numeric e.g. add Austria 1 etc.
-
-            ## auto-add with country prepended
-            ##   e.g. England Premier League, Austria Bundesliga etc.
-            ##  todo/check: also add variants with country alt name if present!!!
-            ##  todo/check: exclude cups or such from country + league name auto-add - why? why not?
-            alt_names_auto << "#{country.name} #{league_name}"
-          else   ## assume int'l (no country) e.g. champions league, etc.
-            ## only auto-add key (e.g. CL, EL, etc.)
-            alt_names_auto << league_key.upcase.gsub('.', ' ')   ## note: no country code (prefix/leading) used
-          end
-
-          ## pp alt_names_auto
-
-          ## prepend country key/code if country present
-          ##   todo/fix: only auto-prepend country if key/code start with a number (level) or incl. cup
-          ##    why? lets you "overwrite" key if desired - use it - why? why not?
-          if country
-            league_key = "#{country.key}.#{league_key}"
-          end
-
-          rec = League.new( key:            league_key,
-                            name:           league_name,
-                            alt_names_auto: alt_names_auto,
-                            country:        country,
-                            intl:           intl,
-                            clubs:          clubs)
-          recs << rec
-          last_rec = rec
-        else
-          puts "** !!! ERROR !!! missing key for (canonical) league name"
-          exit 1
-        end
-      end
-      end  # each line
-    else
-      puts "** !!! ERROR !!! [league reader] - unknown line type:"
-      pp node
-      exit 1
-    end
-    ## pp line
-  end
-  recs
-end # method parse
-
-end # class LeagueReader
-
-end ## module Import
-end ## module SportDb
+
+module SportDb
+module Import
+
+
+class LeagueReader
+
+  def world() Import.world; end
+
+
+def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+  parse( txt )
+end
+
+def self.parse( txt )
+  new( txt ).parse
+end
+
+
+
+include Logging
+
+def initialize( txt )
+  @txt = txt
+end
+
+def parse
+  recs = []
+  last_rec = nil
+
+  country  = nil    # last country
+  intl     = false  # is international (league/tournament/cup/competition)
+  clubs    = true   # or clubs|national teams
+
+  OutlineReader.parse( @txt ).each do |node|
+    if [:h1,:h2,:h3,:h4,:h5,:h6].include?( node[0] )
+      heading_level  = node[0][1].to_i
+      heading        = node[1]
+
+      logger.debug "heading #{heading_level} >#{heading}<"
+
+      if heading_level != 1
+        puts "** !!! ERROR !!! unsupported heading level; expected heading 1 for now only; sorry"
+        pp line
+        exit 1
+      else
+        logger.debug "heading (#{heading_level}) >#{heading}<"
+        last_heading = heading
+        ## map to country or international / int'l or national teams
+        if heading =~ /national team/i   ## national team tournament
+          country = nil
+          intl    = true
+          clubs   = false
+        elsif heading =~ /international|int'l/i  ## int'l club tournament
+          country = nil
+          intl    = true
+          clubs   = true
+        else
+          ## assume country in heading; allow all "formats" supported by parse e.g.
+          ##   Österreich • Austria (at)
+          ##   Österreich • Austria
+          ##   Austria
+          ##   Deutschland (de) • Germany
+          country = world.countries.parse( heading )
+          intl    = false
+          clubs   = true
+
+          ## check country code - MUST exist for now!!!!
+          if country.nil?
+            puts "!!! error [league reader] - unknown country >#{heading}< - sorry - add country to config to fix"
+            exit 1
+          end
+        end
+      end
+    elsif node[0] == :p   ## paragraph with (text) lines
+      lines = node[1]
+      lines.each do |line|
+
+      if line.start_with?( '|' )
+          ## assume continuation with line of alternative names
+          ##  note: skip leading pipe
+          values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
+          values = values.map {|value| _norm(value) }  ## squish/strip etc.
+
+          logger.debug "alt_names: #{values.join( '|' )}"
+
+          last_rec.alt_names += values
+      else
+        ## assume "regular" line
+        ##  check if starts with id  (todo/check: use a more "strict"/better regex capture pattern!!!)
+        if line =~ /^([a-z0-9][a-z0-9.]*)[ ]+(.+)$/
+          league_key  = $1
+          ## 1) strip (commercial) sponsor markers/tags e.g $$
+          ## 2) strip and squish (white)spaces
+          league_name = _norm( $2 )
+
+          logger.debug "key: >#{league_key}<, name: >#{league_name}<"
+
+
+          ## prepend country key/code if country present
+          ##   todo/fix: only auto-prepend country if key/code start with a number (level) or incl. cup
+          ##    why? lets you "overwrite" key if desired - use it - why? why not?
+          if country
+            league_key = "#{country.key}.#{league_key}"
+          end
+
+          rec = League.new( key:            league_key,
+                            name:           league_name,
+                            country:        country,
+                            intl:           intl,
+                            clubs:          clubs)
+          recs << rec
+          last_rec = rec
+        else
+          puts "** !!! ERROR !!! missing key for (canonical) league name"
+          exit 1
+        end
+      end
+      end  # each line
+    else
+      puts "** !!! ERROR !!! [league reader] - unknown line type:"
+      pp node
+      exit 1
+    end
+    ## pp line
+  end
+  recs
+end # method parse
+
+
+
+#######################
+###  helpers
+
+## norm(alize) helper  - squish (spaces) 
+##                      and remove dollars ($$$)
+##                      and remove leading and trailing spaces
+def _norm( str )
+  ## only extra clean-up of dollars for now ($$$)
+  _squish( str.gsub( '$', '' ) )
+end
+
+def _squish( str )
+  str.gsub( /[ \t\u00a0]+/, ' ' ).strip
+end
+
+
+end # class LeagueReader
+
+end ## module Import
+end ## module SportDb

data/lib/sportdb/formats/lines_reader.rb

@@ -0,0 +1,47 @@
+
+module SportDb
+
+class LinesReader   ## change to LinesEnumerator - why? why not?
+        def initialize( lines )
+            @iter = lines.each  ## get (external) enumerator (same as to_enum)
+            @lineno = 0
+        end
+    
+        def each( &blk )
+        ## note - StopIteration is rescued (automagically) by Kernel#loop.
+        ##            no need to rescue ourselves here
+            loop do
+              line = @iter.next   ## note - raises StopIteration  
+              blk.call( line )
+            end
+        end
+    
+        def each_with_index( &blk )
+            ## note - StopIteration is rescued (automagically) by Kernel#loop.
+                loop do
+                  line = @iter.next   ## note - raises StopIteration 
+                  blk.call( line, @lineno )
+                  @lineno += 1
+                end
+        end
+        
+        def peek
+            begin  
+              @iter.peek
+            rescue StopIteration
+                nil
+            end
+        end
+    
+        def next
+            ## todo/check - do NOT catch StopIteration for next - why? why not?
+            begin
+                line = @iter.next
+                @lineno += 1
+                line
+            rescue StopIteration
+                nil
+            end
+        end    
+end  # class LinesReader  
+end # module SportDb