sportdb-config 0.6.0 → 0.7.0

data/lib/sportdb/config/club.rb

@@ -1,147 +0,0 @@
-# encoding: utf-8
-
-module SportDb
-  module Import
-
-##
-#  note: use our own (internal) club struct for now - why? why not?
-#    - check that shape/structure/fields/attributes match
-#      the Team struct in sportdb-text (in SportDb::Struct::Team)  !!!!
-class Club
-  ##  todo: use just names for alt_names - why? why not?
-  attr_accessor :name, :alt_names,
-                :year, :ground, :city
-
-  ## more attribs - todo/fix - also add "upstream" to struct & model!!!!!
-  attr_accessor :district, :geos, :year_end, :country
-
-  ## special import only attribs
-  attr_accessor :alt_names_auto    ## auto-generated alt names
-  attr_accessor :wikipedia   # wikipedia page name (for english (en))
-
-  def historic?()  @year_end ? true : false; end
-  alias_method  :past?, :historic?
-
-
-  def wikipedia?()  @wikipedia; end
-  def wikipedia_url
-    if @wikipedia
-      ##  note: replace spaces with underscore (-)
-      ##  e.g. Club Brugge KV => Club_Brugge_KV
-      ##  todo/check/fix:
-      ##    check if "plain" dash (-) needs to get replaced with typographic dash??
-      "https://en.wikipedia.org/wiki/#{@wikipedia.gsub(' ','_')}"
-    else
-      nil
-    end
-  end
-
-
-  def initialize
-    @alt_names      = []
-    @alt_names_auto = []
-  end
-
-
-  ## helper methods for import only
-  ## check for duplicates
-  def duplicates?
-    names = [name] + alt_names + alt_names_auto
-    names = names.map { |name| normalize( sanitize(name) ) }
-
-    names.size != names.uniq.size
-  end
-
-  def duplicates
-    names = [name] + alt_names + alt_names_auto
-
-    ## calculate (count) frequency and select if greater than one
-    names.reduce( Hash.new ) do |h,name|
-       norm = normalize( sanitize(name) )
-       h[norm] ||= []
-       h[norm] << name; h
-    end.select { |norm,names| names.size > 1 }
-  end
-
-  def add_variants( name_or_names )
-    names = name_or_names.is_a?(Array) ? name_or_names : [name_or_names]
-    names.each do |name|
-      name = sanitize( name )
-      self.alt_names_auto += variants( name )
-    end
-  end
-
-   ###################################
-   # "global" helper - move to ___ ? why? why not?
-
-   YEAR_REGEX = /\([0-9,\- ]+?\)/
-   def self.strip_year( name )
-     ## check for year(s) e.g. (1887-1911), (-2013),
-     ##                        (1946-2001, 2013-) etc.
-     name.gsub( YEAR_REGEX, '' ).strip
-   end
-
-   def self.has_year?( name ) name =~ YEAR_REGEX; end
-
-   LANG_REGEX = /\[[a-z]{1,2}\]/   ## note also allow [a] or [d] or [e] - why? why not?
-   def self.strip_lang( name )
-     name.gsub( LANG_REGEX, '' ).strip
-   end
-
-   def self.has_lang?( name ) name =~ LANG_REGEX; end
-
-  def self.sanitize( name )
-    ## check for year(s) e.g. (1887-1911), (-2013),
-    ##                        (1946-2001,2013-) etc.
-    name = strip_year( name )
-    ## check lang codes e.g. [en], [fr], etc.
-    name = strip_lang( name )
-    name
-  end
-
-
-  NORM_REGEX =  /[.'º\-\/]/
-  ## note: remove all dots (.), dash (-), ', º, /, etc.
-  ##         for norm(alizing) names
-  def self.strip_norm( name )
-    name.gsub( NORM_REGEX, '' )
-  end
-
-  def self.normalize( name )
-    # note: do NOT call sanitize here (keep normalize "atomic" for reuse)
-
-    ## remove all dots (.), dash (-), º, /, etc.
-    name = strip_norm( name )
-    name = name.gsub( ' ', '' )  # note: also remove all spaces!!!
-
-    ## todo/fix: use our own downcase - why? why not?
-    name = downcase_i18n( name )     ## do NOT care about upper and lowercase for now
-    name
-  end
-
-
-  def self.strip_wiki( name )     # todo/check: rename to strip_wikipedia_en - why? why not?
-    ## note: strip disambiguationn qualifier from wikipedia page name if present
-    ##        note: only remove year and foot... for now
-    ## e.g. FC Wacker Innsbruck (2002) => FC Wacker Innsbruck
-    ##      Willem II (football club)  => Willem II
-    ##
-    ## e.g. do NOT strip others !! e.g.
-    ##   América Futebol Clube (MG)
-    ##  only add more "special" cases on demand (that, is) if we find more
-    name = name.gsub( /\([12][^\)]+?\)/, '' ).strip  ## starting with a digit 1 or 2 (assuming year)
-    name = name.gsub( /\(foot[^\)]+?\)/, '' ).strip  ## starting with foot (assuming football ...)
-    name
-  end
-
-
-private
-  ## private "shortcut" convenience helpers
-  def sanitize( name )    self.class.sanitize( name ); end
-  def normalize( name )   self.class.normalize( name ); end
-
-  def variants( name )  Variant.find( name ); end
-end # class Club
-
-end   # module Import
-end   # module SportDb

data/lib/sportdb/config/club_index.rb

@@ -1,212 +0,0 @@
-# encoding: utf-8
-
-module SportDb
-  module Import
-
-
-class ClubIndex
-
-  def self.build( path )
-    recs = []
-    datafiles = Configuration.find_datafiles_clubs( path )
-    datafiles.each do |datafile|
-      recs += ClubReader.read( datafile )
-    end
-    recs
-
-    clubs = self.new
-    clubs.add( recs )
-
-    ## add wiki(pedia) anchored links
-    recs = []
-    datafiles = Configuration.find_datafiles_clubs_wiki( path )
-    datafiles.each do |datafile|
-       recs += WikiReader.read( datafile )
-    end
-
-    pp recs
-    clubs.add_wiki( recs )
-    clubs
-  end
-
-
-
-  def initialize
-    @clubs          = {}   ## clubs (indexed) by canonical name
-    @clubs_by_name  = {}
-    @errors         = []
-  end
-
-  attr_reader :errors
-  def errors?() @errors.empty? == false; end
-
-  def mappings() @clubs_by_name; end   ## todo/check: rename to index or something - why? why not?
-  def clubs()    @clubs.values;  end
-
-
-  ## helpers from club - use a helper module for includes - why? why not?
-  def strip_year( name ) Club.strip_year( name ); end
-  def has_year?( name)   Club.has_year?( name ); end
-  def strip_lang( name ) Club.strip_lang( name ); end
-  def strip_wiki( name ) Club.strip_wiki( name ); end
-  def normalize( name )  Club.normalize( name ); end
-
-
-  def add_wiki( rec_or_recs )   ## add wiki(pedia club record / links
-    recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
-
-    recs.each do |rec|
-      ## note: strip qualifier () from wikipedia page name if present
-      ## e.g. FC Wacker Innsbruck (2002) => FC Wacker Innsbruck
-      ##      Willem II (football club)  => Willem II
-      ##
-      ## e.g. do NOT strip others !! e.g.
-      ##   América Futebol Clube (MG)
-      ##  only add more "special" cases on demand (that, is) if we find more
-      name = strip_wiki( rec.name )
-
-      m = match_by( name: name, country: rec.country )
-      if m.nil?
-        puts "** !!! ERROR !!! - no matching club found for wiki(pedia) name >#{name}, #{rec.country.name} (#{rec.country.key})<; sorry - to fix add name to clubs"
-        exit 1
-      end
-      if m.size > 1
-        puts "** !!! ERROR !!! - too many (greater than one) matching clubs found for wiki(pedia) name >#{name}, #{rec.country.name} (#{rec.country.key})<"
-        pp m
-        exit 1
-      end
-      club = m[0]
-      club.wikipedia = rec.name
-    end
-  end  # method add_wiki
-
-
-  def add( rec_or_recs )   ## add club record / alt_names
-    recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
-
-    recs.each do |rec|
-      ## puts "adding:"
-      ## pp rec
-      ### step 1) add canonical name
-      old_rec = @clubs[ rec.name ]
-      if old_rec
-        puts "** !!! ERROR !!! - (canonical) name conflict - duplicate - >#{rec.name}< will overwrite >#{old_rec.name}<:"
-        pp old_rec
-        pp rec
-        exit 1
-      else
-        @clubs[ rec.name ] = rec
-      end
-
-      ## step 2) add all names (canonical name + alt names + alt names (auto))
-      names = [rec.name] + rec.alt_names
-      more_names = []
-      ## check "hand-typed" names for year (auto-add)
-      ## check for year(s) e.g. (1887-1911), (-2013),
-      ##                        (1946-2001,2013-) etc.
-      names.each do |name|
-        if has_year?( name )
-          more_names <<  strip_year( name )
-        end
-      end
-
-      names += more_names
-      ## check for duplicates - simple check for now - fix/improve
-      ## todo/fix: (auto)remove duplicates - why? why not?
-      count      = names.size
-      count_uniq = names.uniq.size
-      if count != count_uniq
-        puts "** !!! ERROR !!! - #{count-count_uniq} duplicate name(s):"
-        pp names
-        pp rec
-        exit 1
-      end
-
-      ## check with auto-names just warn for now and do not exit
-      names += rec.alt_names_auto
-      count      = names.size
-      count_uniq = names.uniq.size
-      if count != count_uniq
-        puts "** !!! WARN !!! - #{count-count_uniq} duplicate name(s):"
-        pp names
-        pp rec
-      end
-
-
-      names.each_with_index do |name,i|
-        ## check lang codes e.g. [en], [fr], etc.
-        ##  todo/check/fix:  move strip_lang up in the chain - check for duplicates (e.g. only lang code marker different etc.) - why? why not?
-        name = strip_lang( name )
-        norm = normalize( name )
-        alt_recs = @clubs_by_name[ norm ]
-        if alt_recs
-          ## check if include club rec already or is new club rec
-          if alt_recs.include?( rec )
-            ## note: do NOT include duplicate club record
-            msg = "** !!! WARN !!! - (norm) name conflict/duplicate for club - >#{name}< normalized to >#{norm}< already included >#{rec.name}, #{rec.country.name}<"
-            puts msg
-            @errors << msg
-          else
-            msg = "** !!! WARN !!! - name conflict/duplicate - >#{name}< will overwrite >#{alt_recs[0].name}, #{alt_recs[0].country.name}< with >#{rec.name}, #{rec.country.name}<"
-            puts msg
-            @errors << msg
-            alt_recs << rec
-          end
-        else
-          @clubs_by_name[ norm ] = [rec]
-        end
-      end
-    end
-  end # method add
-
-
-  def []( name )    ## lookup by canoncial name only
-    @clubs[ name ]
-  end
-
-  def match( name )
-    ## todo/check: return empty array if no match!!! and NOT nil (add || []) - why? why not?
-    name = normalize( name )
-    @clubs_by_name[ name ]
-  end
-
-
-  def match_by( name:, country: )
-    ## note: match must for now always  include name
-    m = match( name )
-    if m    ## filter by country
-      ## note: country assumes / allows the country key or fifa code for now
-
-      ## note: allow passing in of country struct too
-      country_rec = if country.is_a?( SportDb::Import::Country )
-                       country   ## (re)use country struct - no need to run lookup again
-                    else
-                       rec = SportDb::Import.config.countries[ country ]
-                       if rec.nil?
-                         puts "** !!! ERROR !!! - unknown country >#{country}< - no match found, sorry - add to world/countries.txt in config"
-                         exit 1
-                       end
-                       rec
-                    end
-
-      m = m.select { |club| club.country.key == country_rec.key }
-      m = nil   if m.empty?     ## note: reset to nil if no more matches
-    end
-    m
-  end
-
-
-
-  def dump_duplicates # debug helper - report duplicate club name records
-     @clubs_by_name.each do |name, clubs|
-       if clubs.size > 1
-         puts "#{clubs.size} matching club duplicates for >#{name}<:"
-         pp clubs
-       end
-     end
-  end
-end # class ClubIndex
-
-
-end   # module Import
-end   # module SportDb

data/lib/sportdb/config/club_reader.rb

@@ -1,278 +0,0 @@
-# encoding: utf-8
-
-
-module SportDb
-module Import
-
-
-class ClubReader
-
-
-def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
-  txt = File.open( path, 'r:utf-8' ).read
-  parse( txt )
-end
-
-
-def self.parse( txt )
-  recs = []
-  last_rec  = nil
-  headings = []   ## headings stack
-
-  txt.each_line do |line|
-    line = line.strip
-
-    next if line.empty?
-    next if line.start_with?( '#' )   ## skip comments too
-
-    ## strip inline (until end-of-line) comments too
-    ##  e.g Eupen        => KAS Eupen,    ## [de]
-    ##   => Eupen        => KAS Eupen,
-    line = line.sub( /#.*/, '' ).strip
-    pp line
-
-
-    next if line =~ /^={1,}$/          ## skip "decorative" only heading e.g. ========
-
-     ## note: like in wikimedia markup (and markdown) all optional trailing ==== too
-     ##  todo/check:  allow ===  Text  =-=-=-=-=-=   too - why? why not?
-    if line =~ /^(={1,})       ## leading ======
-                 ([^=]+?)      ##  text   (note: for now no "inline" = allowed)
-                 =*            ## (optional) trailing ====
-                 $/x
-       heading_marker = $1
-       heading_level  = $1.length   ## count number of = for heading level
-       heading        = $2.strip
-
-       puts "heading #{heading_level} >#{heading}<"
-
-       ## 1) first pop headings if present
-       while headings.size+1 > heading_level
-         headings.pop
-       end
-
-       ## 2) add missing (hierarchy) level if
-       while headings.size+1 < heading_level
-         ##  todo/fix: issue warning about "skipping" hierarchy level
-         puts "!!! warn [team reader] - skipping hierarchy level in headings "
-         headings.push( nil )
-       end
-
-       if heading =~ /^\?+$/    ## note: use ? or ?? or ?? to reset level to nil
-         ## keep level empty
-       else
-
-         ## quick hack:   if level is 1 assume country for now
-         ##                 and extract country code e.g.
-         ##                    Austria (at) => at
-         ##  todo/fix:  allow code only e.g. at or aut without enclosing () too - why? why not?
-         if heading_level == 1
-             if heading =~ /\(([a-z]{2,3})\)/i    ## note allow (at) or (AUT) too
-               country_code = $1
-
-               ## check country code - MUST exist for now!!!!
-               country = SportDb::Import.config.countries[ country_code ]
-               if country.nil?
-                 puts "!!! error [team reader] - unknown country with code >#{country_code}< - sorry - add country to config to fix"
-                 exit 1
-               end
-
-               headings.push( country.key )
-             else
-               puts "!!! error - heading level 1 - missing country code - >#{heading}<"
-               exit 1
-             end
-         else
-          ## quick hack:
-          ##   remove known fill/dummy words incl:
-          ##     Provincia San Juan  =>  San Juan   (see argentina, for example)
-          ##
-          ##   use geo tree long term with alternative names - why? why not?
-           words = ['Provincia']
-           words.each { |word| heading = heading.gsub( word, '' ) }
-           heading = heading.strip
-
-           headings.push( heading )
-         end
-
-         ## assert that hierarchy level is ok
-         if headings.size != heading_level
-           puts "!!! error - headings hierarchy/stack out of order - #{heading.size}<=>#{heading_level}"
-           exit 1
-         end
-       end
-
-       pp headings
-
-    elsif line.start_with?( '|' )
-      ## assume continuation with line of alternative names
-      ##  note: skip leading pipe
-      values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
-      ## strip and  squish (white)spaces
-      #   e.g. New York FC      (2011-)  => New York FC (2011-)
-      values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
-      last_rec.alt_names += values
-      last_rec.add_variants( values ) # auto-add (possible) auto-generated variant names
-
-      ## check for duplicates
-      if last_rec.duplicates?
-        duplicates = last_rec.duplicates
-        puts "*** !!! WARN !!! - #{duplicates.size} duplicate alt name mapping(s):"
-        pp duplicates
-        pp last_rec
-        ##
-        ##  todo/fix:  make it only an error with exit 1
-        ##               if (not normalized) names are the same (not unique/uniq)
-        ##                  e.g. don't exit on  A.F.C. == AFC etc.
-        ## exit 1
-      end
-    else
-      values = line.split( ',' )
-
-      rec = Club.new
-      value = values.shift    ## get first item
-      ## strip and  squish (white)spaces
-      #   e.g. New York FC      (2011-)  => New York FC (2011-)
-      value = value.strip.gsub( /[ \t]+/, ' ' )
-      rec.name = value            # canoncial name (global unique "beautiful/long" name)
-      rec.add_variants( value )   # auto-add (possible) auto-generated variant names
-
-      ## note:
-      ##   check/todo!!!!!!!!!!!!!!!!!-
-      ##  strip year if to present e.g. (2011-)
-      ##
-      ##  do NOT strip for defunct / historic clubs e.g.
-      ##    (1899-1910)
-      ## or (-1914) or (-2011) etc.
-
-      ###
-      ##  todo: move year out of canonical team name - why? why not?
-
-      ## check if canonical name include (2011-) or similar in name
-      ##   if yes, remove (2011-) and add to (alt) names
-      ##   e.g. New York FC (2011) => New York FC
-      if rec.name =~ /\(.+?\)/   ## note: use non-greedy (?) match
-        name = rec.name.gsub( /\(.+?\)/, '' ).strip
-
-        if rec.name =~ /\(([0-9]{4})-\)/            ## e.g. (2014-)
-          rec.year     = $1.to_i
-        elsif  rec.name =~ /\(-([0-9]{4})\)/            ## e.g. (-2014)
-          rec.year_end = $1.to_i
-        elsif  rec.name =~ /\(([0-9]{4})-([0-9]{4})\)/  ## e.g. (2011-2014)
-          rec.year     = $1.to_i
-          rec.year_end = $2.to_i
-        else
-          ## todo/check: warn about unknown year format
-        end
-      end
-
-      ##  todo/check - check for unknown format values
-      ##    e.g. too many values, duplicate years, etc.
-      ##         check for overwritting, etc.
-      while values.size > 0
-        value = values.shift
-        ##  strip and squish (white)spaces
-        #   e.g. León     › Guanajuato     => León › Guanajuato
-        value = value.strip.gsub( /[ \t]+/, ' ' )
-        if value =~/^\d{4}$/   # e.g 1904
-          ## todo/check: issue warning if year is already set!!!!!!!
-          if rec.year
-            puts "!!! error - year already set to #{rec.year} - CANNOT overwrite with #{value}:"
-            pp rec
-            exit 1
-          end
-          rec.year  = value.to_i
-        elsif value.start_with?( '@' )   # e.g. @ Anfield
-          ## cut-off leading @ and spaces
-          rec.ground  = value[1..-1].strip
-        else
-          ## assume city / geo tree
-          ## split into geo tree
-          geos = split_geo( value )
-          city = geos[0]
-          ## check for "embedded" district e.g. London (Fulham) or Hamburg (St. Pauli) etc.
-          if city =~ /\((.+?)\)/   ## note: use non-greedy (?) match
-            rec.district  = $1.strip
-            city          = city.gsub( /\(.+?\)/, '' ).strip
-          end
-          rec.city = city
-
-          if geos.size > 1
-             ## cut-off city and keep the rest (of geo tree)
-             rec.geos = geos[1..-1]
-          end
-        end
-      end  ## while values
-
-
-      ###############
-      ## use headings text for geo tree
-
-      ## 1) add country if present
-      if headings.size > 0 && headings[0]
-        country = SportDb::Import.config.countries[ headings[0] ]
-        rec.country = country
-      else
-        ## make it an error - why? why not?
-        puts "!!! error - country missing in headings hierarchy - sorry - add to quicklist"
-        exit 1
-      end
-
-      ## 2) check geo tree with headings hierarchy
-      if headings.size > 1 && headings[1]
-         geos = split_geo( headings[1] )
-         if rec.geos
-           if rec.geos[0] != geos[0]
-             puts "!!! error - geo tree - headings mismatch >#{rec.geos[0]}< <=> >#{geos[0]}<"
-             exit 1
-           end
-           if rec.geos[1] && rec.geos[1] != geos[1]   ## check optional 2nd level too
-             puts "!!! error - geo tree - headings mismatch >#{rec.geos[1]}< <=> >#{geos[1]}<"
-             exit 1
-           end
-         else
-           ## add missing region (state/province) from headings hierarchy
-           rec.geos = geos
-         end
-      end
-
-      last_rec = rec
-
-
-    ### todo/fix:
-    ##  auto-add alt name with dots stripped - why? why not?
-    ##    e.g.  D.C. United    => DC United
-    ##    e.g.  Liverpool F.C. => Liverpool FC
-    ##    e.g.  St. Albin       => St Albin etc.
-    ##    e.g.  1. FC Köln     => 1 FC Köln  -- make special case for 1. - why? why not?
-
-    ##
-    ## todo/fix:  unify mapping entries
-    ##   always lowercase !!!!  (case insensitive)
-    ##   always strip (2011-) !!!
-    ##   always strip dots (e.g. St., F.C, etc.)
-
-      recs << rec
-    end
-  end  # each_line
-  recs
-end  # method read
-
-###  helpers
-def self.split_geo( str )
-  ## assume city / geo tree
-  ##  strip and squish (white)spaces
-  #   e.g. León     › Guanajuato     => León › Guanajuato
-  str = str.strip.gsub( /[ \t]+/, ' ' )
-
-  ## split into geo tree
-  geos = str.split( /[<>‹›]/ )   ## note: allow > < or › ‹
-  geos = geos.map { |geo| geo.strip }   ## remove all whitespaces
-  geos
-end
-
-end  # class ClubReader
-
-
-end ## module Import
-end ## module SportDb