sportdb-config 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04f6537ca02571da4f1e429f3d4d2454097fda8c
-  data.tar.gz: b6135751ae7890b58dce9b5b367ac5a5a943ddc3
+  metadata.gz: 064b7f927038794c7d1efecde27bbba3d7ac78fa
+  data.tar.gz: 0dc83a80b70db77bca25e9921182fc4a1aea0466
 SHA512:
-  metadata.gz: e896c77e205beaa15ba43190d03f3b2da044ce2686d471b87afd5a21bf7f8b9feef354e46ddde9738f1f288be3734fd35df9087cf2f6102f4b0ff9bdfe0eeab3
-  data.tar.gz: 15ebeac9008cce8f1dcffc27d7ef765075151cbd29ff61dc88901dd120f5ebeb6427324bf73f503a9831b7d32fef4cf58d4f954db3add33d9f2212b7fa67c17d
+  metadata.gz: 75d1416b8976f6bd619de196265a7ba1ac3f88c020b28489a4302732272f0e1b686f76a2dcf877e40364e240d148bf0853ba643466ff86d454060e45c9dd4da3
+  data.tar.gz: 1fdc13cde4020f2bc2a21da04b7df6ce25810f4f282a477862adf2af789b17d6fa009e08163fc163579478aed0ca9210588aafb09f728d2472a9711bdec90c1a

data/Manifest.txt

@@ -17,8 +17,10 @@ lib/sportdb/config/league.rb
 lib/sportdb/config/league_reader.rb
 lib/sportdb/config/league_utils.rb
 lib/sportdb/config/season_utils.rb
+lib/sportdb/config/variants.rb
 lib/sportdb/config/version.rb
 test/helper.rb
+test/test_club_index.rb
 test/test_club_reader.rb
 test/test_clubs.rb
 test/test_config.rb
@@ -26,3 +28,4 @@ test/test_countries.rb
 test/test_league_reader.rb
 test/test_league_utils.rb
 test/test_season_utils.rb
+test/test_variants.rb

data/lib/sportdb/config/club_reader.rb

@@ -112,16 +112,18 @@ def self.parse( txt )
       #   e.g. New York FC      (2011-)  => New York FC (2011-)
       values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
       last_rec.alt_names += values
+      last_rec.add_variants( values ) # auto-add (possible) auto-generated variant names
 
-      ## check for duplicates - simple check for now - fix/improve
-      ## todo/fix: (auto)remove duplicates - why? why not?
-      ##  todo/fix: add canonical name too!! might get duplicated in alt names!!!
-      count      = last_rec.alt_names.size
-      count_uniq = last_rec.alt_names.uniq.size
-      if count != count_uniq
-        puts
-        puts "*** !!! WARN !!! - #{count-count_uniq} duplicate alt name(s):"
+      ## check for duplicates
+      if last_rec.duplicates?
+        duplicates = last_rec.duplicates
+        puts "*** !!! WARN !!! - #{duplicates.size} duplicate alt name mapping(s):"
+        pp duplicates
         pp last_rec
+        ##
+        ##  todo/fix:  make it only an error with exit 1
+        ##               if (not normalized) names are the same (not unique/uniq)
+        ##                  e.g. don't exit on  A.F.C. == AFC etc.
         ## exit 1
       end
     else
@@ -132,7 +134,8 @@ def self.parse( txt )
       ## strip and  squish (white)spaces
       #   e.g. New York FC      (2011-)  => New York FC (2011-)
       value = value.strip.gsub( /[ \t]+/, ' ' )
-      rec.name = value   # canoncial_name
+      rec.name = value            # canoncial name (global unique "beautiful/long" name)
+      rec.add_variants( value )   # auto-add (possible) auto-generated variant names
 
       ## note:
       ##   check/todo!!!!!!!!!!!!!!!!!-
@@ -150,7 +153,6 @@ def self.parse( txt )
       ##   e.g. New York FC (2011) => New York FC
       if rec.name =~ /\(.+?\)/   ## note: use non-greedy (?) match
         name = rec.name.gsub( /\(.+?\)/, '' ).strip
-        rec.alt_names << name
 
         if rec.name =~ /\(([0-9]{4})-\)/            ## e.g. (2014-)
           rec.year     = $1.to_i

data/lib/sportdb/config/clubs.rb

@@ -9,21 +9,79 @@ module SportDb
 #      the Team struct in sportdb-text (in SportDb::Struct::Team)  !!!!
 class Club
   ##  todo: use just names for alt_names - why? why not?
-  attr_accessor :name, :alt_names, :year, :ground, :city
+  attr_accessor :name, :alt_names,
+                :year, :ground, :city
 
   ## more attribs - todo/fix - also add "upstream" to struct & model!!!!!
   attr_accessor :district, :geos, :year_end, :country
 
+  ## special import only attribs
+  attr_accessor :alt_names_auto    ## auto-generated alt names
+
   def historic?()  @year_end ? true : false; end
   alias_method  :past?, :historic?
 
   def initialize
-    @alt_names = []
+    @alt_names      = []
+    @alt_names_auto = []
+  end
+
+
+  ## helper methods for import only
+  ## check for duplicates
+  def duplicates?
+    names = [name] + alt_names + alt_names_auto
+    names = names.map { |name| normalize( name ) }
+
+    names.size != names.uniq.size
+  end
+
+  def duplicates
+    names = [name] + alt_names + alt_names_auto
+
+    ## calculate (count) frequency and select if greater than one
+    names.reduce( Hash.new ) do |h,name|
+       norm = normalize( name )
+       h[norm] ||= []
+       h[norm] << name; h
+    end.select { |norm,names| names.size > 1 }
+  end
+
+  def add_variants( name_or_names )
+    names = name_or_names.is_a?(Array) ? name_or_names : [name_or_names]
+    names.each do |name|
+      name = sanitize( name )
+      self.alt_names_auto += variants( name )
+    end
   end
+
+private
+  def sanitize( name )
+    ## check for year(s) e.g. (1887-1911), (-2013) etc.
+    name = name.gsub( /\([0-9\- ]+?\)/, '' ).strip
+    ## check lang codes e.g. [en], [fr], etc.
+    name = name.gsub( /\[[a-z]{2}\]/, '' ).strip
+    name
+  end
+
+  def normalize( name )
+    name = sanitize( name)
+
+    name = name.gsub( '.', '' )    # remove all dots
+    ## don't report duplicates that only differ with dash (-)
+    ##    e.g. Al Ahly or Al-Ahly for now - why? why not?
+    name = name.gsub( '-', ' ' )   # replace - with space (e.g. - same as space)
+
+    name = name.downcase     ## do NOT care about upper and lowercase for now
+    name
+  end
+
+  def variants( name )  Variant.find( name ); end
 end # class Club
 
 
 
+
 class ClubIndex
 
   def initialize
@@ -53,28 +111,60 @@ class ClubIndex
         @clubs[ rec.name ] = rec
       end
 
-      ## step 2) add all names (canonical name + alt names)
+      ## step 2) add all names (canonical name + alt names + alt names (auto))
       names = [rec.name] + rec.alt_names
+      more_names = []
+      ## check "hand-typed" names for year (auto-add)
+      ## check for year(s) e.g. (1887-1911), (-2013) etc.
+      names.each do |name|
+        if name =~ /\([0-9\- ]+?\)/
+          more_names << name.gsub( /\([0-9\- ]+?\)/, '' ).strip
+        end
+      end
 
+      names += more_names
       ## check for duplicates - simple check for now - fix/improve
       ## todo/fix: (auto)remove duplicates - why? why not?
       count      = names.size
       count_uniq = names.uniq.size
       if count != count_uniq
         puts "** !!! ERROR !!! - #{count-count_uniq} duplicate name(s):"
+        pp names
         pp rec
         exit 1
       end
 
+      ## check with auto-names just warn for now and do not exit
+      names += rec.alt_names_auto
+      count      = names.size
+      count_uniq = names.uniq.size
+      if count != count_uniq
+        puts "** !!! WARN !!! - #{count-count_uniq} duplicate name(s):"
+        pp names
+        pp rec
+      end
+
+
       names.each_with_index do |name,i|
-        alt_recs = @clubs_by_name[ name ]
+        ## check lang codes e.g. [en], [fr], etc.
+        name = name.gsub( /\[[a-z]{2}\]/, '' ).strip
+        norm = normalize( name )
+        alt_recs = @clubs_by_name[ norm ]
         if alt_recs
-          msg = "** !!! WARN !!! - name conflict/duplicate - >#{name}< will overwrite >#{alt_recs[0].name}, #{alt_recs[0].country.name}< with >#{rec.name}, #{rec.country.name}<"
-          puts msg
-          @errors << msg
-          alt_recs << rec
+          ## check if include club rec already or is new club rec
+          if alt_recs.include?( rec )
+            ## note: do NOT include duplicate club record
+            msg = "** !!! WARN !!! - (norm) name conflict/duplicate for club - >#{name}< normalized to >#{norm}< already included >#{rec.name}, #{rec.country.name}<"
+            puts msg
+            @errors << msg
+          else
+            msg = "** !!! WARN !!! - name conflict/duplicate - >#{name}< will overwrite >#{alt_recs[0].name}, #{alt_recs[0].country.name}< with >#{rec.name}, #{rec.country.name}<"
+            puts msg
+            @errors << msg
+            alt_recs << rec
+          end
         else
-          @clubs_by_name[ name ] = [rec]
+          @clubs_by_name[ norm ] = [rec]
         end
       end
     end
@@ -87,6 +177,7 @@ class ClubIndex
 
   def match( name )
     ## todo/check: return empty array if no match!!! and NOT nil (add || []) - why? why not?
+    name = normalize( name )
     @clubs_by_name[ name ]
   end
 
@@ -111,14 +202,34 @@ class ClubIndex
 
 
   def dump_duplicates # debug helper - report duplicate club name records
+     @clubs.values.each do |club|
+       if club.duplicates?
+         duplicates = club.duplicates
+         puts "#{duplicates.size} (norm) name duplicate(s) for #{club.name}, #{club.country.name}:"
+         pp duplicates
+       end
+     end
+
      @clubs_by_name.each do |name, clubs|
        if clubs.size > 1
-         puts "#{clubs.size} duplicates for >#{name}<:"
+         puts "#{clubs.size} matching club duplicates for >#{name}<:"
          pp clubs
        end
      end
   end
 
+
+
+private
+  def normalize( name )
+    name = name.gsub( '.', '' )    # remove all dots
+    ## don't report duplicates that only differ with dash (-)
+    ##    e.g. Al Ahly or Al-Ahly for now - why? why not?
+    name = name.gsub( '-', ' ' )   # replace - with space (e.g. - same as space)
+
+    name = name.downcase     ## do NOT care about upper and lowercase for now
+    name
+  end
 end # class ClubIndex
 
 

data/lib/sportdb/config/config.rb

@@ -6,12 +6,6 @@ module SportDb
 
 class Configuration
 
-  ####
-  #  todo/fix:  find a better way to configure shared test datasets
-  attr_accessor   :test_data_dir   ## todo/check:  use test_dir - why? why not?
-  def test_data_dir()   @test_data_dir ||= './datasets'; end
-
-
   ##
   ##  todo: allow configure of countries_dir like clubs_dir
   ##         "fallback" and use a default built-in world/countries.txt

data/lib/sportdb/config/variants.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+
+
+class Variant    ## (spelling) variant finder / builder for names
+
+
+def self.frequency_table( name )   ## todo/check: use/rename to char_frequency_table
+  ## calculate the frequency table of letters, digits, etc.
+  freq = Hash.new(0)
+  name.each_char do |ch|
+     freq[ch] += 1
+  end
+  freq
+end
+
+ALPHA_SPECIALS = %w[
+  Ä Ö Ü
+  ä ö ü ß
+]
+
+##  "simple" translation
+SUB_ALPHA_SPECIALS = {
+  'Ä'=>'A',  'ä'=>'a',
+  'Ö'=>'O',  'ö'=>'o',
+  'Ü'=>'U',  'ü'=>'u',
+             'ß'=>'ss',
+}
+
+##  de,at,ch translation for umlauts
+SUB_ALPHA_SPECIALS_DE = {
+  'Ä'=>'Ae',  'ä'=>'ae',
+  'Ö'=>'Oe',  'ö'=>'oe',
+  'Ü'=>'Ue',  'ü'=>'ue',
+              'ß'=>'ss',
+}
+
+
+def self.alpha_specials_count( freq )
+  ALPHA_SPECIALS.reduce(0) do |count,ch|
+    count += freq[ch]
+    count
+  end
+end
+
+def self.tr( name, mapping )
+  buf = String.new
+  name.each_char do |ch|
+    buf << if mapping[ch]
+              mapping[ch]
+            else
+              ch
+            end
+  end
+  buf
+end
+
+
+
+def self.find( name )
+  alt_names = []
+
+  freq = frequency_table( name )
+
+  if alpha_specials_count( freq ) > 0    # check if includes äöü etc.
+    alt_names <<  tr( name, SUB_ALPHA_SPECIALS )
+    alt_names <<  tr( name, SUB_ALPHA_SPECIALS_DE )
+  end
+
+  ## todo - make uniq  e.g. Preußen is Preussen, Preussen 2x
+  alt_names = alt_names.uniq
+  alt_names
+end
+end   # Variant
+
+
+end ## module Import
+end ## module SportDb

data/lib/sportdb/config/version.rb

@@ -7,8 +7,8 @@ module Boot      ## note: use a different module than Config to avoid confusion
                  ##   maybe rename later gem itself to sportdb-boot - why? why not?
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 3
-  PATCH = 6
+  MINOR = 4
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/sportdb/config.rb

@@ -22,6 +22,7 @@ require 'sportdb/config/league_utils'
 require 'sportdb/config/league'
 require 'sportdb/config/league_reader'
 
+require 'sportdb/config/variants'
 require 'sportdb/config/countries'
 require 'sportdb/config/clubs'
 require 'sportdb/config/club_reader'
@@ -29,4 +30,20 @@ require 'sportdb/config/config'
 
 
 
+
+## let's put test configuration in its own namespace / module
+module SportDb
+
+class Test    ## todo/check: works with module too? use a module - why? why not?
+
+  ####
+  #  todo/fix:  find a better way to configure shared test datasets - why? why not?
+  def self.data_dir()        @data_dir ||= './datasets'; end
+  def self.data_dir=( path ) @data_dir = path; end
+end
+
+end   # module SportDb
+
+
+
 puts SportDb::Boot.banner   # say hello

data/test/test_club_index.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_club_index.rb
+
+
+require 'helper'
+
+class TestClubIndex < MiniTest::Test
+
+  def test_clubs
+    pp SportDb::Import.config.clubs.errors
+
+    SportDb::Import.config.clubs.dump_duplicates
+
+    m = SportDb::Import.config.clubs.match( 'Rapid Wien' )
+    assert_equal 'SK Rapid Wien', m[0].name
+    assert_equal 'Austria',       m[0].country.name
+    assert_equal 'Wien',          m[0].city
+
+    m = SportDb::Import.config.clubs.match( 'rapid wien' )
+    assert_equal 'SK Rapid Wien', m[0].name
+    assert_equal 'Austria',       m[0].country.name
+    assert_equal 'Wien',          m[0].city
+
+    ## note: all dots (.) get always removed
+    m = SportDb::Import.config.clubs.match( '...r.a.p.i.d w.i.e.n...' )
+    assert_equal 'SK Rapid Wien', m[0].name
+    assert_equal 'Austria',       m[0].country.name
+    assert_equal 'Wien',          m[0].city
+
+    m = SportDb::Import.config.clubs.match( 'RAPID WIEN' )
+    assert_equal 'SK Rapid Wien', m[0].name
+    assert_equal 'Austria',       m[0].country.name
+    assert_equal 'Wien',          m[0].city
+
+
+    c = SportDb::Import.config.clubs[ 'SK Rapid Wien' ]   ## check canoncial name match (only)
+    assert_equal 'SK Rapid Wien', c.name
+    assert_equal 'Austria',       c.country.name
+    assert_equal 'Wien',          c.city
+
+
+    m = SportDb::Import.config.clubs.match( 'Arsenal' )
+    assert_equal 3, m.size
+
+    m = SportDb::Import.config.clubs.match( 'ARSENAL' )
+    assert_equal 3, m.size
+
+    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'eng' )
+    assert_equal 1, m.size
+    assert_equal 'Arsenal FC', m[0].name
+    assert_equal 'England',    m[0].country.name
+    assert_equal 'London',     m[0].city
+
+    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'ar' )
+    assert_equal 1, m.size
+    assert_equal 'Arsenal de Sarandí', m[0].name
+    assert_equal 'Argentina',          m[0].country.name
+    assert_equal 'Sarandí',            m[0].city
+
+    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'ru' )
+    assert_equal 1, m.size
+    assert_equal 'Arsenal Tula', m[0].name
+    assert_equal 'Russia',       m[0].country.name
+    assert_equal 'Tula',         m[0].city
+
+
+    m = SportDb::Import.config.clubs.match( 'Arsenal FC' )
+    assert_equal 2, m.size
+
+    m = SportDb::Import.config.clubs.match( 'Arsenal F.C.' )
+    assert_equal 2, m.size
+
+    m = SportDb::Import.config.clubs.match( '...A.r.s.e.n.a.l... F.C...' )
+    assert_equal 2, m.size
+  end
+
+end # class TestClubIndex

data/test/test_clubs.rb

@@ -9,42 +9,22 @@ require 'helper'
 
 class TestClubs < MiniTest::Test
 
-  def test_clubs
-    pp SportDb::Import.config.clubs.errors
-
-    SportDb::Import.config.clubs.dump_duplicates
-
-    m = SportDb::Import.config.clubs.match( 'Rapid Wien' )
-    assert_equal 'SK Rapid Wien', m[0].name
-    assert_equal 'Austria',       m[0].country.name
-    assert_equal 'Wien',          m[0].city
-
-    c = SportDb::Import.config.clubs[ 'SK Rapid Wien' ]   ## check canoncial name match (only)
-    assert_equal 'SK Rapid Wien', c.name
-    assert_equal 'Austria',       c.country.name
-    assert_equal 'Wien',          c.city
-
-
-    m = SportDb::Import.config.clubs.match( 'Arsenal' )
-    assert_equal 3, m.size
-
-    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'eng' )
-    assert_equal 1, m.size
-    assert_equal 'Arsenal FC', m[0].name
-    assert_equal 'England',    m[0].country.name
-    assert_equal 'London',     m[0].city
-
-    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'ar' )
-    assert_equal 1, m.size
-    assert_equal 'Arsenal de Sarandí', m[0].name
-    assert_equal 'Argentina',          m[0].country.name
-    assert_equal 'Sarandí',            m[0].city
-
-    m = SportDb::Import.config.clubs.match_by( name: 'Arsenal', country: 'ru' )
-    assert_equal 1, m.size
-    assert_equal 'Arsenal Tula', m[0].name
-    assert_equal 'Russia',       m[0].country.name
-    assert_equal 'Tula',         m[0].city
+  def test_duplicats
+    club = SportDb::Import::Club.new
+    club.name = "Rapid Wien"
+
+    assert_equal false, club.duplicates?
+    duplicates = {}
+    assert_equal duplicates,  club.duplicates
+
+    club.alt_names_auto += ['Rapid', 'Rapid Wien', 'SK Rapid Wien']
+
+    pp club
+
+    assert_equal true,              club.duplicates?
+    duplicates = {'rapid wien'=>['Rapid Wien','Rapid Wien']}
+    pp club.duplicates
+    assert_equal duplicates, club.duplicates
   end
 
 end # class TestClubs

data/test/test_variants.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_variants.rb
+
+
+require 'helper'
+
+class TestVariants < MiniTest::Test
+
+  def variants( name )  SportDb::Import::Variant.find( name ); end
+
+
+  def test_de
+    assert_equal [],                             variants( 'Augsburg' )
+
+    assert_equal ['Koln', 'Koeln'],              variants( 'Köln' )
+    assert_equal ['1. FC Koln', '1. FC Koeln'],  variants( '1. FC Köln' )
+
+    assert_equal ['Bayern Munchen', 'Bayern Muenchen'], variants( 'Bayern München' )
+    assert_equal ['F. Dusseldorf', 'F. Duesseldorf'],   variants( 'F. Düsseldorf' )
+    assert_equal ['Preussen'], variants( 'Preußen' )
+    assert_equal ['Munster Preussen', 'Muenster Preussen'], variants( 'Münster Preußen' )
+    assert_equal ['Rot-Weiss Oberhausen'], variants( 'Rot-Weiß Oberhausen' )
+
+    assert_equal ['St. Polten', 'St. Poelten'], variants( 'St. Pölten' )
+  end
+
+
+end # class TestVariants

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sportdb-config
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-28 00:00:00.000000000 Z
+date: 2019-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csvreader
@@ -86,8 +86,10 @@ files:
 - lib/sportdb/config/league_reader.rb
 - lib/sportdb/config/league_utils.rb
 - lib/sportdb/config/season_utils.rb
+- lib/sportdb/config/variants.rb
 - lib/sportdb/config/version.rb
 - test/helper.rb
+- test/test_club_index.rb
 - test/test_club_reader.rb
 - test/test_clubs.rb
 - test/test_config.rb
@@ -95,6 +97,7 @@ files:
 - test/test_league_reader.rb
 - test/test_league_utils.rb
 - test/test_season_utils.rb
+- test/test_variants.rb
 homepage: https://github.com/sportdb/sport.db
 licenses:
 - Public Domain