sportdb-config 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f264abfc487652f687a3e4d6dc4388594a587896
-  data.tar.gz: 32f24cc478d3db058ef5aeb943a279019e2d68d2
+  metadata.gz: e017d063fd47d049d93b49763c2549bf87412172
+  data.tar.gz: 84d75edb2bb440805ae792c07ac732342f7f6828
 SHA512:
-  metadata.gz: 4e5a329a6027121d68c81d51dd683befec81eb348ba4ef4606437ca5b77bfd1346ae31e8781a4e5c086c93f5cdd053a417a359e9edefa440fbe580ee0345970d
-  data.tar.gz: a3b7f2e23f0cd8f9755a8c594ef7c520be6b0d131762dbefff81f5c6acee0dbb15ed80128fa292f72ae704e471af28b16152cd1fd71564cc38c4f0b5efbcdf82
+  metadata.gz: c0c746998dd8bf6da6628174dbcd684a60d6813a85968eed939f056f2fc44df5f035a2c1680902886091d33f26143060d1518e48fbb1ba9a2af9661189ffbbbb
+  data.tar.gz: 1a1ec84bf35b66b891fd7162529db514cbf2975769e7f62deb203263a5b272a25df83527128119086a8134e349e18c57a4c0c3f39a56a5266ddc1f87da205244

data/Manifest.txt

@@ -13,8 +13,9 @@ config/world/countries.txt
 config/world/de.txt
 config/world/eng.txt
 lib/sportdb/config.rb
+lib/sportdb/config/club.rb
+lib/sportdb/config/club_index.rb
 lib/sportdb/config/club_reader.rb
-lib/sportdb/config/clubs.rb
 lib/sportdb/config/config.rb
 lib/sportdb/config/countries.rb
 lib/sportdb/config/league.rb
@@ -23,6 +24,7 @@ lib/sportdb/config/league_utils.rb
 lib/sportdb/config/season_utils.rb
 lib/sportdb/config/variants.rb
 lib/sportdb/config/version.rb
+lib/sportdb/config/wiki_index.rb
 lib/sportdb/config/wiki_reader.rb
 test/helper.rb
 test/test_club_index.rb
@@ -34,4 +36,5 @@ test/test_league_reader.rb
 test/test_league_utils.rb
 test/test_season_utils.rb
 test/test_variants.rb
+test/test_wiki_index.rb
 test/test_wiki_reader.rb

data/lib/sportdb/config.rb

@@ -24,9 +24,11 @@ require 'sportdb/config/league_reader'
 
 require 'sportdb/config/variants'
 require 'sportdb/config/countries'
-require 'sportdb/config/clubs'
+require 'sportdb/config/club'
 require 'sportdb/config/club_reader'
+require 'sportdb/config/club_index'
 require 'sportdb/config/wiki_reader'
+require 'sportdb/config/wiki_index'
 require 'sportdb/config/config'
 
 

data/lib/sportdb/config/club.rb

@@ -0,0 +1,130 @@
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+##
+#  note: use our own (internal) club struct for now - why? why not?
+#    - check that shape/structure/fields/attributes match
+#      the Team struct in sportdb-text (in SportDb::Struct::Team)  !!!!
+class Club
+  ##  todo: use just names for alt_names - why? why not?
+  attr_accessor :name, :alt_names,
+                :year, :ground, :city
+
+  ## more attribs - todo/fix - also add "upstream" to struct & model!!!!!
+  attr_accessor :district, :geos, :year_end, :country
+
+  ## special import only attribs
+  attr_accessor :alt_names_auto    ## auto-generated alt names
+  attr_accessor :wikipedia   # wikipedia page name (for english (en))
+
+  def historic?()  @year_end ? true : false; end
+  alias_method  :past?, :historic?
+
+
+  def wikipedia?()  @wikipedia; end
+  def wikipedia_url
+    if @wikipedia
+      ##  note: replace spaces with underscore (-)
+      ##  e.g. Club Brugge KV => Club_Brugge_KV
+      ##  todo/check/fix:
+      ##    check if "plain" dash (-) needs to get replaced with typographic dash??
+      "https://en.wikipedia.org/wiki/#{@wikipedia.gsub(' ','_')}"
+    else
+      nil
+    end
+  end
+
+
+  def initialize
+    @alt_names      = []
+    @alt_names_auto = []
+  end
+
+
+  ## helper methods for import only
+  ## check for duplicates
+  def duplicates?
+    names = [name] + alt_names + alt_names_auto
+    names = names.map { |name| normalize( name ) }
+
+    names.size != names.uniq.size
+  end
+
+  def duplicates
+    names = [name] + alt_names + alt_names_auto
+
+    ## calculate (count) frequency and select if greater than one
+    names.reduce( Hash.new ) do |h,name|
+       norm = normalize( name )
+       h[norm] ||= []
+       h[norm] << name; h
+    end.select { |norm,names| names.size > 1 }
+  end
+
+  def add_variants( name_or_names )
+    names = name_or_names.is_a?(Array) ? name_or_names : [name_or_names]
+    names.each do |name|
+      name = sanitize( name )
+      self.alt_names_auto += variants( name )
+    end
+  end
+
+   ###################################
+   # "global" helper - move to ___ ? why? why not?
+
+   YEAR_REGEX = /\([0-9,\- ]+?\)/
+   def self.strip_year( name )
+     ## check for year(s) e.g. (1887-1911), (-2013),
+     ##                        (1946-2001, 2013-) etc.
+     name.gsub( YEAR_REGEX, '' ).strip
+   end
+
+   def self.has_year?( name ) name =~ YEAR_REGEX; end
+
+   LANG_REGEX = /\[[a-z]{2}\]/
+   def self.strip_lang( name )
+     name.gsub( LANG_REGEX, '' ).strip
+   end
+
+   def self.has_lang?( name ) name =~ LANG_REGEX; end
+
+   NORM_REGEX =  /[.'º\-\/]/
+   ## note: remove all dots (.), dash (-), ', º, /, etc.
+   ##         for norm(alizing) names
+   def self.strip_norm( name )
+     name.gsub( NORM_REGEX, '' )
+   end
+
+   def strip_year( name ) self.class.strip_year( name ); end
+   def strip_lang( name ) self.class.strip_lang( name ); end
+   def strip_norm( name ) self.class.strip_norm( name ); end
+
+private
+  def sanitize( name )
+    ## check for year(s) e.g. (1887-1911), (-2013),
+    ##                        (1946-2001,2013-) etc.
+    name = strip_year( name )
+    ## check lang codes e.g. [en], [fr], etc.
+    name = strip_lang( name )
+    name
+  end
+
+  def normalize( name )
+    name = sanitize( name )
+
+    ## remove all dots (.), dash (-), º, /, etc.
+    name = strip_norm( name )
+    name = name.gsub( ' ', '' )  # note: also remove all spaces!!!
+
+    ## todo/fix: use our own downcase - why? why not?
+    name = name.downcase     ## do NOT care about upper and lowercase for now
+    name
+  end
+
+  def variants( name )  Variant.find( name ); end
+end # class Club
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/config/{clubs.rb → club_index.rb}

@@ -3,134 +3,34 @@
 module SportDb
   module Import
 
-##
-#  note: use our own (internal) club struct for now - why? why not?
-#    - check that shape/structure/fields/attributes match
-#      the Team struct in sportdb-text (in SportDb::Struct::Team)  !!!!
-class Club
-  ##  todo: use just names for alt_names - why? why not?
-  attr_accessor :name, :alt_names,
-                :year, :ground, :city
-
-  ## more attribs - todo/fix - also add "upstream" to struct & model!!!!!
-  attr_accessor :district, :geos, :year_end, :country
-
-  ## special import only attribs
-  attr_accessor :alt_names_auto    ## auto-generated alt names
-  attr_accessor :wikipedia   # wikipedia page name (for english (en))
-
-  def historic?()  @year_end ? true : false; end
-  alias_method  :past?, :historic?
-
-
-  def wikipedia?()  @wikipedia; end
-  def wikipedia_url
-    if @wikipedia
-      ##  note: replace spaces with underscore (-)
-      ##  e.g. Club Brugge KV => Club_Brugge_KV
-      ##  todo/check/fix:
-      ##    check if "plain" dash (-) needs to get replaced with typographic dash??
-      "https://en.wikipedia.org/wiki/#{@wikipedia.gsub(' ','_')}"
-    else
-      nil
-    end
-  end
-
-
-  def initialize
-    @alt_names      = []
-    @alt_names_auto = []
-  end
-
-
-  ## helper methods for import only
-  ## check for duplicates
-  def duplicates?
-    names = [name] + alt_names + alt_names_auto
-    names = names.map { |name| normalize( name ) }
-
-    names.size != names.uniq.size
-  end
-
-  def duplicates
-    names = [name] + alt_names + alt_names_auto
 
-    ## calculate (count) frequency and select if greater than one
-    names.reduce( Hash.new ) do |h,name|
-       norm = normalize( name )
-       h[norm] ||= []
-       h[norm] << name; h
-    end.select { |norm,names| names.size > 1 }
-  end
+class ClubIndex
 
-  def add_variants( name_or_names )
-    names = name_or_names.is_a?(Array) ? name_or_names : [name_or_names]
-    names.each do |name|
-      name = sanitize( name )
-      self.alt_names_auto += variants( name )
+  def self.build( path )
+    recs = []
+    datafiles = Configuration.find_datafiles_clubs( path )
+    datafiles.each do |datafile|
+      recs += ClubReader.read( datafile )
     end
-  end
-
-   ###################################
-   # "global" helper - move to ___ ? why? why not?
-
-   YEAR_REGEX = /\([0-9,\- ]+?\)/
-   def self.strip_year( name )
-     ## check for year(s) e.g. (1887-1911), (-2013),
-     ##                        (1946-2001, 2013-) etc.
-     name.gsub( YEAR_REGEX, '' ).strip
-   end
-
-   def self.has_year?( name ) name =~ YEAR_REGEX; end
-
-   LANG_REGEX = /\[[a-z]{2}\]/
-   def self.strip_lang( name )
-     name.gsub( LANG_REGEX, '' ).strip
-   end
+    recs
 
-   def self.has_lang?( name ) name =~ LANG_REGEX; end
+    clubs = self.new
+    clubs.add( recs )
 
-   NORM_REGEX =  /[.'º\-\/]/
-   ## note: remove all dots (.), dash (-), ', º, /, etc.
-   ##         for norm(alizing) names
-   def self.strip_norm( name )
-     name.gsub( NORM_REGEX, '' )
-   end
-
-   def strip_year( name ) self.class.strip_year( name ); end
-   def strip_lang( name ) self.class.strip_lang( name ); end
-   def strip_norm( name ) self.class.strip_norm( name ); end
-
-private
-  def sanitize( name )
-    ## check for year(s) e.g. (1887-1911), (-2013),
-    ##                        (1946-2001,2013-) etc.
-    name = strip_year( name )
-    ## check lang codes e.g. [en], [fr], etc.
-    name = strip_lang( name )
-    name
-  end
-
-  def normalize( name )
-    name = sanitize( name )
-
-    ## remove all dots (.), dash (-), º, /, etc.
-    name = strip_norm( name )
-    name = name.gsub( ' ', '' )  # note: also remove all spaces!!!
+    ## add wiki(pedia) anchored links
+    recs = []
+    datafiles = Configuration.find_datafiles_clubs_wiki( path )
+    datafiles.each do |datafile|
+       recs += WikiReader.read( datafile )
+    end
 
-    ## todo/fix: use our own downcase - why? why not?
-    name = name.downcase     ## do NOT care about upper and lowercase for now
-    name
+    pp recs
+    clubs.add_wiki( recs )
+    clubs
   end
 
-  def variants( name )  Variant.find( name ); end
-end # class Club
 
 
-
-
-class ClubIndex
-
   def initialize
     @clubs          = {}   ## clubs (indexed) by canonical name
     @clubs_by_name  = {}
@@ -155,13 +55,24 @@ class ClubIndex
     recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
 
     recs.each do |rec|
-      m = match_by( name: rec.name, country: rec.country )
+      ## note: strip qualifier () from wikipedia page name if present
+      ## e.g. FC Wacker Innsbruck (2002) => FC Wacker Innsbruck
+      ##      Willem II (football club)  => Willem II
+      ##
+      ## e.g. do NOT strip others !! e.g.
+      ##   América Futebol Clube (MG)
+      ##  only add more "special" cases on demand (that, is) if we find more
+      name = rec.name
+      name = name.gsub( /\([12][^\)]+?\)/, '' ).strip  ## starting with a digit 1 or 2 (assuming year)
+      name = name.gsub( /\(foot[^\)]+?\)/, '' ).strip  ## starting with foot (assuming football ...)
+
+      m = match_by( name: name, country: rec.country )
       if m.nil?
-        puts "** !!! ERROR !!! - no matching club found for wiki(pedia) name >#{rec.name}, #{rec.country.name} (#{rec.country.key})<; sorry - to fix add name to clubs"
+        puts "** !!! ERROR !!! - no matching club found for wiki(pedia) name >#{name}, #{rec.country.name} (#{rec.country.key})<; sorry - to fix add name to clubs"
         exit 1
       end
       if m.size > 1
-        puts "** !!! ERROR !!! - too many (greater than one) matching clubs found for wiki(pedia) name >#{rec.name}, #{rec.country.name} (#{rec.country.key})<"
+        puts "** !!! ERROR !!! - too many (greater than one) matching clubs found for wiki(pedia) name >#{name}, #{rec.country.name} (#{rec.country.key})<"
         pp m
         exit 1
       end

data/lib/sportdb/config/config.rb

@@ -47,7 +47,7 @@ class Configuration
                        }x
 
 
-  def find_clubs_datafiles( path, pattern )
+  def self.find_datafiles( path, pattern )
      datafiles = []   ## note: [country, path] pairs for now
 
      ## check all txt files as candidates  (MUST include country code for now)
@@ -60,38 +60,17 @@ class Configuration
      pp datafiles
      datafiles
   end
+  def self.find_datafiles_clubs( path )       find_datafiles( path, CLUBS_REGEX ); end
+  def self.find_datafiles_clubs_wiki( path )  find_datafiles( path, CLUBS_WIKI_REGEX ); end
 
 
   def build_club_index
     ## unify team names; team (builtin/known/shared) name mappings
     ## cleanup team names - use local ("native") name with umlaut etc.
-    recs = []
-
-    ## todo/fix: pass along / use country code too
-    ##   note: country code no longer needed in path (is now expected as heading inside the file)
-
     ## todo/fix: add to teamreader
     ##              check that name and alt_names for a club are all unique (not duplicates)
-    datafiles = find_clubs_datafiles( clubs_dir, CLUBS_REGEX )
-    datafiles.each do |datafile|
-       recs += ClubReader.read( datafile )
-    end
-
-
-    clubs  = ClubIndex.new
-    clubs.add( recs )
-
-    ## add wiki(pedia) anchored links
-    recs = []
-    datafiles = find_clubs_datafiles( clubs_dir, CLUBS_WIKI_REGEX )
-    datafiles.each do |datafile|
-       recs += WikiReader.read( datafile )
-    end
-
-    pp recs
-    clubs.add_wiki( recs )
-
 
+    clubs = ClubIndex.build( clubs_dir )
     if clubs.errors?
       puts ""
       puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
@@ -122,8 +101,6 @@ end # class Configuration
 
 
 
-
-
 ## lets you use
 ##   SportDb::Import.configure do |config|
 ##      config.hello = 'World'

data/lib/sportdb/config/version.rb

@@ -8,7 +8,7 @@ module Boot      ## note: use a different module than Config to avoid confusion
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 5
-  PATCH = 1
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/sportdb/config/wiki_index.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+
+class WikiIndex
+
+  def self.build( path )
+    recs = []
+    datafiles = Configuration.find_datafiles_clubs_wiki( path )
+    datafiles.each do |datafile|
+        recs += WikiReader.read( datafile )
+    end
+    recs
+
+    self.new( recs )
+  end
+
+  def initialize( recs )
+    @pages_by_country = {}
+
+    ## todo/fix:
+    ##   check for duplicate recs - report and exit on dupliate!!!!!!
+    recs.each do |rec|
+      h = @pages_by_country[ rec.country.key ] ||= {}
+      h[ normalize(rec.name) ] = rec
+    end
+  end
+
+
+  def normalize( name )
+    ## todo/fix:  (re)use normalize from Club!!!!
+    name = name.gsub( /[\-\.]/, '' )
+    name = name.gsub( ' ', '' )    ## remove spaces too
+    name = name.downcase
+    name
+  end
+
+
+
+  def find_by( club: )    ## todo/check: use find_by_club - why? why not?
+    find_by_club( club )
+  end
+
+  def find_by_club( club )
+    rec = nil
+
+    ## get query params from club
+    names   = [club.name]+club.alt_names
+    country_key = club.country.key
+
+    h = @pages_by_country[ country_key ]
+    if h
+      ## todo/check: sort names ?
+      ##   sort by longest first (for best match)
+      names.each do |name|
+        ## todo/fix:  name - remove/string year and lang e.g. (1946-2001), [en]!!!!
+        rec = h[ normalize( name ) ]
+        break if rec   ## bingo!! found - break on first match
+      end
+    end
+
+    rec  ## note: return nil if nothing found
+  end
+end  # class WikiIndex
+
+
+end   # module Import
+end   # module SportDb

data/test/test_wiki_index.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_wiki_index.rb
+
+
+require 'helper'
+
+class TestWikiIndex < MiniTest::Test
+
+  def test_clubs
+    wiki = SportDb::Import::WikiIndex.build( SportDb::Import.config.clubs_dir )
+    ## pp wiki
+
+    ##############################################
+    ## test wikipedia names and links/urls
+    be = SportDb::Import.config.countries[ 'be' ]
+
+    club = SportDb::Import::Club.new
+    club.name    = 'Club Brugge KV'
+    club.country = be
+
+    rec = wiki.find_by( club: club )
+    assert_equal 'Club Brugge KV', rec.name
+
+
+    club = SportDb::Import::Club.new
+    club.name    = 'RSC Anderlecht'
+    club.country = be
+
+    rec = wiki.find_by( club: club )
+    assert_equal 'R.S.C. Anderlecht', rec.name
+  end
+
+end # class TestWikiIndex

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sportdb-config
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-03 00:00:00.000000000 Z
+date: 2019-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csvreader
@@ -86,8 +86,9 @@ files:
 - config/world/de.txt
 - config/world/eng.txt
 - lib/sportdb/config.rb
+- lib/sportdb/config/club.rb
+- lib/sportdb/config/club_index.rb
 - lib/sportdb/config/club_reader.rb
-- lib/sportdb/config/clubs.rb
 - lib/sportdb/config/config.rb
 - lib/sportdb/config/countries.rb
 - lib/sportdb/config/league.rb
@@ -96,6 +97,7 @@ files:
 - lib/sportdb/config/season_utils.rb
 - lib/sportdb/config/variants.rb
 - lib/sportdb/config/version.rb
+- lib/sportdb/config/wiki_index.rb
 - lib/sportdb/config/wiki_reader.rb
 - test/helper.rb
 - test/test_club_index.rb
@@ -107,6 +109,7 @@ files:
 - test/test_league_utils.rb
 - test/test_season_utils.rb
 - test/test_variants.rb
+- test/test_wiki_index.rb
 - test/test_wiki_reader.rb
 homepage: https://github.com/sportdb/sport.db
 licenses: