sportdb-config 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e017d063fd47d049d93b49763c2549bf87412172
-  data.tar.gz: 84d75edb2bb440805ae792c07ac732342f7f6828
+  metadata.gz: 43d84070a1a63efe77220f0be88991956c6696a7
+  data.tar.gz: 4b3c0a3a728065b648f52083f32eba1ebaeab43f
 SHA512:
-  metadata.gz: c0c746998dd8bf6da6628174dbcd684a60d6813a85968eed939f056f2fc44df5f035a2c1680902886091d33f26143060d1518e48fbb1ba9a2af9661189ffbbbb
-  data.tar.gz: 1a1ec84bf35b66b891fd7162529db514cbf2975769e7f62deb203263a5b272a25df83527128119086a8134e349e18c57a4c0c3f39a56a5266ddc1f87da205244
+  metadata.gz: 2461e383c70749bfc398de2561a057ab087a1d366644b4cf20d11530c19858d70974e0852b96db71041a6a1f7ebdabb34492a5874785227beea0140911f48f77
+  data.tar.gz: 72e12603145d651a9a2b6266755d1838d962513be478d3f1dbcfb2025b649a8bf4adcea875cdb9bc9424e95cdcb8511c433493125d90f1e5ce1c7e49dfec9993

data/lib/sportdb/config/club.rb

@@ -47,7 +47,7 @@ class Club
   ## check for duplicates
   def duplicates?
     names = [name] + alt_names + alt_names_auto
-    names = names.map { |name| normalize( name ) }
+    names = names.map { |name| normalize( sanitize(name) ) }
 
     names.size != names.uniq.size
   end
@@ -57,7 +57,7 @@ class Club
 
     ## calculate (count) frequency and select if greater than one
     names.reduce( Hash.new ) do |h,name|
-       norm = normalize( name )
+       norm = normalize( sanitize(name) )
        h[norm] ||= []
        h[norm] << name; h
     end.select { |norm,names| names.size > 1 }
@@ -83,26 +83,14 @@ class Club
 
    def self.has_year?( name ) name =~ YEAR_REGEX; end
 
-   LANG_REGEX = /\[[a-z]{2}\]/
+   LANG_REGEX = /\[[a-z]{1,2}\]/   ## note also allow [a] or [d] or [e] - why? why not?
    def self.strip_lang( name )
      name.gsub( LANG_REGEX, '' ).strip
    end
 
    def self.has_lang?( name ) name =~ LANG_REGEX; end
 
-   NORM_REGEX =  /[.'º\-\/]/
-   ## note: remove all dots (.), dash (-), ', º, /, etc.
-   ##         for norm(alizing) names
-   def self.strip_norm( name )
-     name.gsub( NORM_REGEX, '' )
-   end
-
-   def strip_year( name ) self.class.strip_year( name ); end
-   def strip_lang( name ) self.class.strip_lang( name ); end
-   def strip_norm( name ) self.class.strip_norm( name ); end
-
-private
-  def sanitize( name )
+  def self.sanitize( name )
     ## check for year(s) e.g. (1887-1911), (-2013),
     ##                        (1946-2001,2013-) etc.
     name = strip_year( name )
@@ -111,18 +99,47 @@ private
     name
   end
 
-  def normalize( name )
-    name = sanitize( name )
+
+  NORM_REGEX =  /[.'º\-\/]/
+  ## note: remove all dots (.), dash (-), ', º, /, etc.
+  ##         for norm(alizing) names
+  def self.strip_norm( name )
+    name.gsub( NORM_REGEX, '' )
+  end
+
+  def self.normalize( name )
+    # note: do NOT call sanitize here (keep normalize "atomic" for reuse)
 
     ## remove all dots (.), dash (-), º, /, etc.
     name = strip_norm( name )
     name = name.gsub( ' ', '' )  # note: also remove all spaces!!!
 
     ## todo/fix: use our own downcase - why? why not?
-    name = name.downcase     ## do NOT care about upper and lowercase for now
+    name = downcase_i18n( name )     ## do NOT care about upper and lowercase for now
     name
   end
 
+
+  def self.strip_wiki( name )     # todo/check: rename to strip_wikipedia_en - why? why not?
+    ## note: strip disambiguationn qualifier from wikipedia page name if present
+    ##        note: only remove year and foot... for now
+    ## e.g. FC Wacker Innsbruck (2002) => FC Wacker Innsbruck
+    ##      Willem II (football club)  => Willem II
+    ##
+    ## e.g. do NOT strip others !! e.g.
+    ##   América Futebol Clube (MG)
+    ##  only add more "special" cases on demand (that, is) if we find more
+    name = name.gsub( /\([12][^\)]+?\)/, '' ).strip  ## starting with a digit 1 or 2 (assuming year)
+    name = name.gsub( /\(foot[^\)]+?\)/, '' ).strip  ## starting with foot (assuming football ...)
+    name
+  end
+
+
+private
+  ## private "shortcut" convenience helpers
+  def sanitize( name )    self.class.sanitize( name ); end
+  def normalize( name )   self.class.normalize( name ); end
+
   def variants( name )  Variant.find( name ); end
 end # class Club
 

data/lib/sportdb/config/club_index.rb

@@ -48,7 +48,8 @@ class ClubIndex
   def strip_year( name ) Club.strip_year( name ); end
   def has_year?( name)   Club.has_year?( name ); end
   def strip_lang( name ) Club.strip_lang( name ); end
-  def strip_norm( name ) Club.strip_norm( name ); end
+  def strip_wiki( name ) Club.strip_wiki( name ); end
+  def normalize( name )  Club.normalize( name ); end
 
 
   def add_wiki( rec_or_recs )   ## add wiki(pedia club record / links
@@ -62,9 +63,7 @@ class ClubIndex
       ## e.g. do NOT strip others !! e.g.
       ##   América Futebol Clube (MG)
       ##  only add more "special" cases on demand (that, is) if we find more
-      name = rec.name
-      name = name.gsub( /\([12][^\)]+?\)/, '' ).strip  ## starting with a digit 1 or 2 (assuming year)
-      name = name.gsub( /\(foot[^\)]+?\)/, '' ).strip  ## starting with foot (assuming football ...)
+      name = strip_wiki( rec.name )
 
       m = match_by( name: name, country: rec.country )
       if m.nil?
@@ -136,6 +135,7 @@ class ClubIndex
 
       names.each_with_index do |name,i|
         ## check lang codes e.g. [en], [fr], etc.
+        ##  todo/check/fix:  move strip_lang up in the chain - check for duplicates (e.g. only lang code marker different etc.) - why? why not?
         name = strip_lang( name )
         norm = normalize( name )
         alt_recs = @clubs_by_name[ norm ]
@@ -205,18 +205,6 @@ class ClubIndex
        end
      end
   end
-
-
-
-private
-  def normalize( name )
-    name = strip_norm( name )
-    name = name.gsub( ' ', '' )   # remove all spaces
-
-    ## todo/fix: use our own downcase - why? why not?
-    name = name.downcase     ## do NOT care about upper and lowercase for now
-    name
-  end
 end # class ClubIndex
 
 

data/lib/sportdb/config/variants.rb

@@ -21,33 +21,72 @@ end
 ##  "simple" translation
 ALPHA_SPECIALS = {
   'Ä'=>'A',  'ä'=>'a',
-             'á'=>'a',
+  'Á'=>'A',  'á'=>'a',
              'à'=>'a',
              'ã'=>'a',
              'â'=>'a',
+  'Å'=>'A',  'å'=>'a',
+             'æ'=>'ae',
+             'ā'=>'a',
+             'ă'=>'a',
+             'ą'=>'a',
 
-             'ç'=>'c',
+  'Ç' =>'C', 'ç'=>'c',
+             'ć'=>'c',
+  'Č'=>'C',  'č'=>'c',
 
   'É'=>'E',  'é'=>'e',
              'è'=>'e',
              'ê'=>'e',
+             'ë'=>'e',
+             'ė'=>'e',
+             'ę'=>'e',
 
-             'í'=>'i',
+             'ğ'=>'g',
+
+  'İ'=>'I',
+  'Í'=>'I',  'í'=>'i',
              'î'=>'i',
+             'ī'=>'i',
+             'ı'=>'i',
+
+  'Ł'=>'L', 'ł'=>'l',
 
              'ñ'=>'n',
+             'ń'=>'n',
+             'ň'=>'n',
 
   'Ö'=>'O',  'ö'=>'o',
              'ó'=>'o',
              'õ'=>'o',
              'ô'=>'o',
+             'ø'=>'o',
+             'ő'=>'o',
 
-  'Ü'=>'U',  'ü'=>'u',
-             'ú'=>'u',
+              'ř'=>'r',
 
+  'Ś'=>'S',
+  'Ş'=>'S',  'ş'=>'s',
+  'Š'=>'S',  'š'=>'s',
+             'ș'=>'s',  ## U+0219
              'ß'=>'ss',
+
+             'ţ'=>'t',  ## U+0163
+             'ț'=>'t',  ## U+021B
+             'þ'=>'th',
+
+  'Ü'=>'U',  'ü'=>'u',
+  'Ú'=>'U',  'ú'=>'u',
+             'ū'=>'u',
+
+             'ý'=>'y',
+
+             'ź'=>'z',
+             'ż'=>'z',
+  'Ž'=>'Z',  'ž'=>'z',
 }
 
+
 ##  de,at,ch translation for umlauts
 ALPHA_SPECIALS_DE = {
   'Ä'=>'Ae',  'ä'=>'ae',
@@ -59,6 +98,35 @@ ALPHA_SPECIALS_DE = {
 ## add ALPHA_SPECIALS_ES - why? why not?  is Espanyol catalan spelling or spanish (castillian)?
 # 'ñ'=>'ny',    ## e.g. Español => Espanyol
 
+ALPHA_DOWNCASE = %w[A B C D E F G H I J K L M N O P Q R S T U V W X Y Z].reduce({}) do |h,ch|
+  h[ch] = ch.downcase
+  h
+end.merge(
+  'Ä'=>'ä',
+  'Á'=>'á',
+  'Å'=>'å',
+
+  'Ç'=>'ç',
+  'Č'=>'č',
+
+  'É'=>'é',
+
+  'İ'=>'?',   ## fix - add lowercase
+  'Í'=>'í',
+
+  'Ł'=>'ł',
+
+  'Ö'=>'ö',
+
+  'Ś'=>'?',   ## fix - add lowercase
+  'Ş'=>'ş',
+  'Š'=>'š',
+
+  'Ü'=>'ü',
+  'Ú'=>'ú',
+
+  'Ž'=>'ž',
+)
 
 
 def self.alpha_specials_count( freq, mapping )
@@ -99,8 +167,19 @@ def self.find( name )
   alt_names = alt_names.uniq
   alt_names
 end
-end   # Variant
 
+def self.downcase_i18n( name )    ## our very own downcase for int'l characters / letters
+  tr( name, ALPHA_DOWNCASE )
+end
+
+end  # class Variant
 
 end ## module Import
 end ## module SportDb
+
+
+
+## "global" convenience helper
+def downcase_i18n( name )
+  SportDb::Import::Variant.downcase_i18n( name )
+end   # Variant

data/lib/sportdb/config/version.rb

@@ -8,7 +8,7 @@ module Boot      ## note: use a different module than Config to avoid confusion
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 5
-  PATCH = 2
+  PATCH = 3
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/sportdb/config/wiki_index.rb

@@ -17,6 +17,15 @@ class WikiIndex
     self.new( recs )
   end
 
+
+
+  ## helpers from club - use a helper module for includes - why? why not?
+  def strip_lang( name ) Club.strip_lang( name ); end
+  def strip_year( name ) Club.strip_year( name ); end
+  def normalize( name )  Club.normalize( name ); end
+  def strip_wiki( name)  Club.strip_wiki( name ); end
+
+
   def initialize( recs )
     @pages_by_country = {}
 
@@ -24,21 +33,11 @@ class WikiIndex
     ##   check for duplicate recs - report and exit on dupliate!!!!!!
     recs.each do |rec|
       h = @pages_by_country[ rec.country.key ] ||= {}
-      h[ normalize(rec.name) ] = rec
+      h[ normalize( strip_wiki( rec.name )) ] = rec
     end
   end
 
 
-  def normalize( name )
-    ## todo/fix:  (re)use normalize from Club!!!!
-    name = name.gsub( /[\-\.]/, '' )
-    name = name.gsub( ' ', '' )    ## remove spaces too
-    name = name.downcase
-    name
-  end
-
-
-
   def find_by( club: )    ## todo/check: use find_by_club - why? why not?
     find_by_club( club )
   end
@@ -55,8 +54,8 @@ class WikiIndex
       ## todo/check: sort names ?
       ##   sort by longest first (for best match)
       names.each do |name|
-        ## todo/fix:  name - remove/string year and lang e.g. (1946-2001), [en]!!!!
-        rec = h[ normalize( name ) ]
+        ## note: normalize AND sanitize (e.g. remove/string year and lang e.g. (1946-2001), [en] too)
+        rec = h[ normalize( strip_year( strip_lang( name ))) ]
         break if rec   ## bingo!! found - break on first match
       end
     end

data/test/test_variants.rb

@@ -12,9 +12,10 @@ class TestVariants < MiniTest::Test
   def variants( name )  SportDb::Import::Variant.find( name ); end
 
 
-  def test_downcase
-    ## todo/fix: use our own downcase for normalize - why? why not?
-    ## assert_equal 'äöü',  'ÄÖÜ'.downcase
+  def test_downcase_i18n
+    assert_equal 'abcdefghijklmnopqrstuvwxyz',  downcase_i18n( 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' )
+    assert_equal 'äöü',  downcase_i18n( 'ÄÖÜ' )
+    assert_equal 'köln', downcase_i18n( 'KÖLN' )
   end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sportdb-config
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-05 00:00:00.000000000 Z
+date: 2019-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csvreader