sportdb-config 0.4.1 → 0.5.0

data/lib/sportdb/config/clubs.rb

@@ -55,23 +55,54 @@ class Club
     end
   end
 
+   ###################################
+   # "global" helper - move to ___ ? why? why not?
+
+   YEAR_REGEX = /\([0-9,\- ]+?\)/
+   def self.strip_year( name )
+     ## check for year(s) e.g. (1887-1911), (-2013),
+     ##                        (1946-2001, 2013-) etc.
+     name.gsub( YEAR_REGEX, '' ).strip
+   end
+
+   def self.has_year?( name ) name =~ YEAR_REGEX; end
+
+   LANG_REGEX = /\[[a-z]{2}\]/
+   def self.strip_lang( name )
+     name.gsub( LANG_REGEX, '' ).strip
+   end
+
+   def self.has_lang?( name ) name =~ LANG_REGEX; end
+
+   NORM_REGEX =  /[.'º\-\/]/
+   ## note: remove all dots (.), dash (-), ', º, /, etc.
+   ##         for norm(alizing) names
+   def self.strip_norm( name )
+     name.gsub( NORM_REGEX, '' )
+   end
+
+   def strip_year( name ) self.class.strip_year( name ); end
+   def strip_lang( name ) self.class.strip_lang( name ); end
+   def strip_norm( name ) self.class.strip_norm( name ); end
+
 private
   def sanitize( name )
-    ## check for year(s) e.g. (1887-1911), (-2013) etc.
-    name = name.gsub( /\([0-9\- ]+?\)/, '' ).strip
+    ## check for year(s) e.g. (1887-1911), (-2013),
+    ##                        (1946-2001,2013-) etc.
+    name = strip_year( name )
     ## check lang codes e.g. [en], [fr], etc.
-    name = name.gsub( /\[[a-z]{2}\]/, '' ).strip
+    name = strip_lang( name )
     name
   end
 
   def normalize( name )
-    name = sanitize( name)
+    name = sanitize( name )
 
-    name = name.gsub( '.', '' )    # remove all dots
-    ## don't report duplicates that only differ with dash (-)
-    ##    e.g. Al Ahly or Al-Ahly for now - why? why not?
-    name = name.gsub( '-', ' ' )   # replace - with space (e.g. - same as space)
+    ## remove all dots (.), dash (-), º, /, etc.
+    name = strip_norm( name )
+    name = name.gsub( ' ', '' )  # note: also remove all spaces!!!
 
+    ## todo/fix: use our own downcase - why? why not?
     name = name.downcase     ## do NOT care about upper and lowercase for now
     name
   end
@@ -97,6 +128,13 @@ class ClubIndex
   def clubs()    @clubs.values;  end
 
 
+  ## helpers from club - use a helper module for includes - why? why not?
+  def strip_year( name ) Club.strip_year( name ); end
+  def has_year?( name)   Club.has_year?( name ); end
+  def strip_lang( name ) Club.strip_lang( name ); end
+  def strip_norm( name ) Club.strip_norm( name ); end
+
+
 
   def add( rec_or_recs )   ## add club record / alt_names
     recs = rec_or_recs.is_a?( Array ) ? rec_or_recs : [rec_or_recs]      ## wrap (single) rec in array
@@ -119,10 +157,11 @@ class ClubIndex
       names = [rec.name] + rec.alt_names
       more_names = []
       ## check "hand-typed" names for year (auto-add)
-      ## check for year(s) e.g. (1887-1911), (-2013) etc.
+      ## check for year(s) e.g. (1887-1911), (-2013),
+      ##                        (1946-2001,2013-) etc.
       names.each do |name|
-        if name =~ /\([0-9\- ]+?\)/
-          more_names << name.gsub( /\([0-9\- ]+?\)/, '' ).strip
+        if has_year?( name )
+          more_names <<  strip_year( name )
         end
       end
 
@@ -151,7 +190,7 @@ class ClubIndex
 
       names.each_with_index do |name,i|
         ## check lang codes e.g. [en], [fr], etc.
-        name = name.gsub( /\[[a-z]{2}\]/, '' ).strip
+        name = strip_lang( name )
         norm = normalize( name )
         alt_recs = @clubs_by_name[ norm ]
         if alt_recs
@@ -206,17 +245,6 @@ class ClubIndex
 
 
   def dump_duplicates # debug helper - report duplicate club name records
-
-    ## todo/fix: remove club.duplicates - alreay included in reports -see TeamDuplicatePart
-    ##             more a "feature" of Clubs than ClubIndex class - why? why not?
-     @clubs.values.each do |club|
-       if club.duplicates?
-         duplicates = club.duplicates
-         puts "#{duplicates.size} (norm) name duplicate(s) for #{club.name}, #{club.country.name}:"
-         pp duplicates
-       end
-     end
-
      @clubs_by_name.each do |name, clubs|
        if clubs.size > 1
          puts "#{clubs.size} matching club duplicates for >#{name}<:"
@@ -229,11 +257,10 @@ class ClubIndex
 
 private
   def normalize( name )
-    name = name.gsub( '.', '' )    # remove all dots
-    ## don't report duplicates that only differ with dash (-)
-    ##    e.g. Al Ahly or Al-Ahly for now - why? why not?
-    name = name.gsub( '-', ' ' )   # replace - with space (e.g. - same as space)
+    name = strip_norm( name )
+    name = name.gsub( ' ', '' )   # remove all spaces
 
+    ## todo/fix: use our own downcase - why? why not?
     name = name.downcase     ## do NOT care about upper and lowercase for now
     name
   end

data/lib/sportdb/config/config.rb

@@ -1,123 +1,123 @@
-# encoding: utf-8
-
-module SportDb
-  module Import
-
-
-class Configuration
-
-  ##
-  ##  todo: allow configure of countries_dir like clubs_dir
-  ##         "fallback" and use a default built-in world/countries.txt
-
-  ## todo/check:  rename to country_mappings/index - why? why not?
-  ##    or countries_by_code or countries_by_key
-  def countries
-    @countries ||= build_country_index
-    @countries
-  end
-
-  def build_country_index    ## todo/check: rename to setup_country_index or read_country_index - why? why not?
-    recs = read_csv( "#{SportDb::Boot.data_dir}/world/countries.txt" )
-    CountryIndex.new( recs )
-  end
-
-
-
-  def clubs
-    @clubs  ||= build_club_index
-    @clubs
-  end
-
-  ####
-  #  todo/fix:  find a better way to configure club / team datasets
-  attr_accessor   :clubs_dir
-  def clubs_dir()   @clubs_dir ||= './clubs'; end
-
-
-  CLUBS_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
-                       (?:[a-z]{1,3}\.)?   # optional country code/key e.g. eng.clubs.txt
-                       clubs\.txt$
-                   }x
-
-  def find_clubs_datafiles( path )
-     datafiles = []   ## note: [country, path] pairs for now
-
-     ## check all txt files as candidates  (MUST include country code for now)
-     candidates = Dir.glob( "#{path}/**/*.txt" )
-     pp candidates
-     candidates.each do |candidate|
-       datafiles << candidate    if CLUBS_REGEX.match( candidate )
-     end
-
-     pp datafiles
-     datafiles
-  end
-
-
-  def build_club_index
-    ## unify team names; team (builtin/known/shared) name mappings
-    ## cleanup team names - use local ("native") name with umlaut etc.
-    recs = []
-
-    ## todo/fix: pass along / use country code too
-    ##   note: country code no longer needed in path (is now expected as heading inside the file)
-
-    ## todo/fix: add to teamreader
-    ##              check that name and alt_names for a club are all unique (not duplicates)
-    datafiles = find_clubs_datafiles( clubs_dir )
-    datafiles.each do |datafile|
-       recs += ClubReader.read( datafile )
-    end
-
-
-    clubs  = ClubIndex.new
-    clubs.add( recs )
-
-    if clubs.errors?
-      puts ""
-      puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
-      puts " #{clubs.errors.size} errors:"
-      pp clubs.errors
-      ## exit 1
-    end
-
-    clubs
-end # method build_club_index
-
-
-
-
-def leagues
-  read_leagues()      if @leagues.nil?
-  @leagues
-end
-
-def read_leagues
-    #####
-    # add / read-in leagues config
-    @leagues = LeagueConfig.new
-
-    self  ## return self for chaining
-  end
-end # class Configuration
-
-
-
-
-
-## lets you use
-##   SportDb::Import.configure do |config|
-##      config.hello = 'World'
-##   end
-
-def self.configure
-  yield( config )
-end
-
-def self.config
-  @config ||= Configuration.new
-end
-
-end   # module Import
-end   # module SportDb
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+
+class Configuration
+
+  ##
+  ##  todo: allow configure of countries_dir like clubs_dir
+  ##         "fallback" and use a default built-in world/countries.txt
+
+  ## todo/check:  rename to country_mappings/index - why? why not?
+  ##    or countries_by_code or countries_by_key
+  def countries
+    @countries ||= build_country_index
+    @countries
+  end
+
+  def build_country_index    ## todo/check: rename to setup_country_index or read_country_index - why? why not?
+    recs = read_csv( "#{SportDb::Boot.data_dir}/world/countries.txt" )
+    CountryIndex.new( recs )
+  end
+
+
+
+  def clubs
+    @clubs  ||= build_club_index
+    @clubs
+  end
+
+  ####
+  #  todo/fix:  find a better way to configure club / team datasets
+  attr_accessor   :clubs_dir
+  def clubs_dir()   @clubs_dir ||= './clubs'; end
+
+
+  CLUBS_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
+                       (?:[a-z]{1,3}\.)?   # optional country code/key e.g. eng.clubs.txt
+                       clubs\.txt$
+                   }x
+
+  def find_clubs_datafiles( path )
+     datafiles = []   ## note: [country, path] pairs for now
+
+     ## check all txt files as candidates  (MUST include country code for now)
+     candidates = Dir.glob( "#{path}/**/*.txt" )
+     pp candidates
+     candidates.each do |candidate|
+       datafiles << candidate    if CLUBS_REGEX.match( candidate )
+     end
+
+     pp datafiles
+     datafiles
+  end
+
+
+  def build_club_index
+    ## unify team names; team (builtin/known/shared) name mappings
+    ## cleanup team names - use local ("native") name with umlaut etc.
+    recs = []
+
+    ## todo/fix: pass along / use country code too
+    ##   note: country code no longer needed in path (is now expected as heading inside the file)
+
+    ## todo/fix: add to teamreader
+    ##              check that name and alt_names for a club are all unique (not duplicates)
+    datafiles = find_clubs_datafiles( clubs_dir )
+    datafiles.each do |datafile|
+       recs += ClubReader.read( datafile )
+    end
+
+
+    clubs  = ClubIndex.new
+    clubs.add( recs )
+
+    if clubs.errors?
+      puts ""
+      puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      puts " #{clubs.errors.size} errors:"
+      pp clubs.errors
+      ## exit 1
+    end
+
+    clubs
+end # method build_club_index
+
+
+
+
+def leagues
+  read_leagues()      if @leagues.nil?
+  @leagues
+end
+
+def read_leagues
+    #####
+    # add / read-in leagues config
+    @leagues = LeagueConfig.new
+
+    self  ## return self for chaining
+  end
+end # class Configuration
+
+
+
+
+
+## lets you use
+##   SportDb::Import.configure do |config|
+##      config.hello = 'World'
+##   end
+
+def self.configure
+  yield( config )
+end
+
+def self.config
+  @config ||= Configuration.new
+end
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/config/league.rb

@@ -1,118 +1,118 @@
-# encoding: utf-8
-
-module SportDb
-  module Import
-
-
-class LeagueConfig   ## use LeagueInfo or LeagueMap or LeagueHash or similar
-
-def initialize
-
-  ## just use leagues without latest for latest - why? why not?
-  @leagues_latest = {
-   'es' => { '1' => 'liga',  # spanish liga 1
-             '2' => 'liga2',  # spanish liga 2
-           },
-   'it' => { '1' => 'seriea',  # italian serie a
-             '2' => 'serieb',  # italian serie b
-           },
-   'de' => { '1' => 'bundesliga',  # german bundesliga
-             '2' => 'bundesliga2', # german 2. bundesliga
-           },
-  'nl'  =>  { '1' => 'eredivisie' },   # dutch eredivisie
-  'be'  =>  { '1' => 'proleague' },    # belgian pro league
-  'pt'  =>  { '1' => 'liga' },         # portugese Primeira Liga
-  'tr'  =>  { '1' => 'superlig' },     # turkish Süper Lig
-
-#  note: eng now read from txt
-#     'eng'  => { '1' => 'premierleague',  # english premier league
-#                 '2' => 'championship',  # english championship league
-#                 '3' => 'league1',  # english league 1
-#               },
-  }
-
-  ## change history to past or changes/changelog something - why? why not?
-  @leagues_history = {
-
-#  note: eng now read from txt
-#    'eng' => {
-#               ## until (including) 2003-04 season
-#               '2003-04' => { '1' => 'premierleague', # english premier league
-#                              '2' => 'division1',     # english division 1
-#                            },
-#               ## until (including) 1991-92} season
-#               '1991-92' => { '1' => 'division1', # english division 1
-#                              '2' => 'division2', # english division 2
-#                            }
-#             }
-  }
-
-  pp @leagues_latest
-  pp @leagues_history
-
-  %w(eng sco fr gr).each do |country|
-    hash = LeagueReader.read( "#{Boot.data_dir}/leagues/#{country}.txt" )
-    pp hash
-
-    hash.each do |season,league_hash|
-      if season == '*'  ## assume latest / default season
-        @leagues_latest[ country ] = league_hash
-      else
-        @leagues_history[ country ] ||= {}
-        @leagues_history[ country ][ season ]  = league_hash
-      end
-    end
-  end
-
-  pp @leagues_latest
-  pp @leagues_history
-end
-
-
-
-def basename( league, country:, season: )
-  ## todo/check: rename league: to key: - why? why not?
-
-  if country.include?( '-' )  ## assume package name e.g. eng-england etc.
-    ## cut off country code from package name
-    cc = country.split( '-' )[0]   # use first part
-  else
-    cc = country
-  end
-
-  if season
-    puts "  checking season >#{season}<"
-    ## check history if season is provided / supplied / known
-    history = @leagues_history[ cc ]
-    if history
-       season_start_year = SeasonUtils.start_year( season ).to_i
-       ##
-       ##  todo: sorty season keys - why? why not?  -- assume reverse chronological order for now
-       history.keys.reverse.each do |key|
-          history_season_start_year = SeasonUtils.start_year( key ).to_i
-          puts "  #{season_start_year} <= #{history_season_start_year} - #{season_start_year <= history_season_start_year}"
-          if season_start_year <= history_season_start_year
-            result = history[ key ][ league ]
-            if result
-              return "#{league}-#{result}"
-            else
-              return nil
-            end
-          end
-       end
-    end
-  end
-
-  latest  = @leagues_latest[ cc ]
-  if latest
-     result = latest[ league ]
-     return "#{league}-#{result}"   if result
-  end
-
-  nil
-end # method basename
-end # class LeagueConfig
-
-
-end ## module Import
-end ## module SportDb
+# encoding: utf-8
+
+module SportDb
+  module Import
+
+
+class LeagueConfig   ## use LeagueInfo or LeagueMap or LeagueHash or similar
+
+def initialize
+
+  ## just use leagues without latest for latest - why? why not?
+  @leagues_latest = {
+   'es' => { '1' => 'liga',  # spanish liga 1
+             '2' => 'liga2',  # spanish liga 2
+           },
+   'it' => { '1' => 'seriea',  # italian serie a
+             '2' => 'serieb',  # italian serie b
+           },
+   'de' => { '1' => 'bundesliga',  # german bundesliga
+             '2' => 'bundesliga2', # german 2. bundesliga
+           },
+  'nl'  =>  { '1' => 'eredivisie' },   # dutch eredivisie
+  'be'  =>  { '1' => 'proleague' },    # belgian pro league
+  'pt'  =>  { '1' => 'liga' },         # portugese Primeira Liga
+  'tr'  =>  { '1' => 'superlig' },     # turkish Süper Lig
+
+#  note: eng now read from txt
+#     'eng'  => { '1' => 'premierleague',  # english premier league
+#                 '2' => 'championship',  # english championship league
+#                 '3' => 'league1',  # english league 1
+#               },
+  }
+
+  ## change history to past or changes/changelog something - why? why not?
+  @leagues_history = {
+
+#  note: eng now read from txt
+#    'eng' => {
+#               ## until (including) 2003-04 season
+#               '2003-04' => { '1' => 'premierleague', # english premier league
+#                              '2' => 'division1',     # english division 1
+#                            },
+#               ## until (including) 1991-92} season
+#               '1991-92' => { '1' => 'division1', # english division 1
+#                              '2' => 'division2', # english division 2
+#                            }
+#             }
+  }
+
+  pp @leagues_latest
+  pp @leagues_history
+
+  %w(eng sco fr gr).each do |country|
+    hash = LeagueReader.read( "#{Boot.data_dir}/leagues/#{country}.txt" )
+    pp hash
+
+    hash.each do |season,league_hash|
+      if season == '*'  ## assume latest / default season
+        @leagues_latest[ country ] = league_hash
+      else
+        @leagues_history[ country ] ||= {}
+        @leagues_history[ country ][ season ]  = league_hash
+      end
+    end
+  end
+
+  pp @leagues_latest
+  pp @leagues_history
+end
+
+
+
+def basename( league, country:, season: )
+  ## todo/check: rename league: to key: - why? why not?
+
+  if country.include?( '-' )  ## assume package name e.g. eng-england etc.
+    ## cut off country code from package name
+    cc = country.split( '-' )[0]   # use first part
+  else
+    cc = country
+  end
+
+  if season
+    puts "  checking season >#{season}<"
+    ## check history if season is provided / supplied / known
+    history = @leagues_history[ cc ]
+    if history
+       season_start_year = SeasonUtils.start_year( season ).to_i
+       ##
+       ##  todo: sorty season keys - why? why not?  -- assume reverse chronological order for now
+       history.keys.reverse.each do |key|
+          history_season_start_year = SeasonUtils.start_year( key ).to_i
+          puts "  #{season_start_year} <= #{history_season_start_year} - #{season_start_year <= history_season_start_year}"
+          if season_start_year <= history_season_start_year
+            result = history[ key ][ league ]
+            if result
+              return "#{league}-#{result}"
+            else
+              return nil
+            end
+          end
+       end
+    end
+  end
+
+  latest  = @leagues_latest[ cc ]
+  if latest
+     result = latest[ league ]
+     return "#{league}-#{result}"   if result
+  end
+
+  nil
+end # method basename
+end # class LeagueConfig
+
+
+end ## module Import
+end ## module SportDb