sportdb-formats 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b1daae3f04961c06c34b98e84438ee39a80c5ca
-  data.tar.gz: 872154cb339bb6b6c8f440a392f250dc7d7387d9
+  metadata.gz: 216e15369403af5707b3d2c6d82fcd6c72e2712d
+  data.tar.gz: 5c73e0e7ce7309b2c8fb86e38d90d1d566f80688
 SHA512:
-  metadata.gz: 853b0362fde6f9d8b17db7036ae954768500ab681acff7bf859b8cbf59b7d8bc4bec9d153f21e3c7ac069b6486b10386ce99fe151d4229df815c006a27dccbe0
-  data.tar.gz: e356061bfb0f15ff8274651ce107ec3be00818996d0c29a2ffb26bd5b71c385a3bd1464eea42e2e11ab60dc24ce96663a701e28fa6a8aebd27d434525c71a1f4
+  metadata.gz: 8033e4db4d7b047b37ec8b3be61657d15df72b7a6c6295317ca8c9cd19c0f578f2ecd75c25cf11b0dcb9218c1ff3e8d2a8e0efb0c59ae235b568b353ef55c9d6
+  data.tar.gz: e5b510b45d4c28c6d83d6baa43ef954e78ae64af116c8305a983b5c54d2fda486425a793b48c8332dcf8cf5c441da37f7183602d50a7e07066049995b5940f0f

data/Manifest.txt

@@ -17,6 +17,7 @@ lib/sportdb/formats/match/mapper.rb
 lib/sportdb/formats/match/mapper_teams.rb
 lib/sportdb/formats/match/match_parser.rb
 lib/sportdb/formats/match/match_parser_auto_conf.rb
+lib/sportdb/formats/match/match_parser_csv.rb
 lib/sportdb/formats/name_helper.rb
 lib/sportdb/formats/outline_reader.rb
 lib/sportdb/formats/package.rb
@@ -49,8 +50,11 @@ test/test_clubs.rb
 test/test_conf.rb
 test/test_country_index.rb
 test/test_country_reader.rb
+test/test_csv_match_parser.rb
+test/test_csv_match_parser_utils.rb
 test/test_csv_reader.rb
 test/test_datafile.rb
+test/test_datafile_package.rb
 test/test_goals.rb
 test/test_league_index.rb
 test/test_league_outline_reader.rb
@@ -59,6 +63,7 @@ test/test_match.rb
 test/test_match_auto.rb
 test/test_match_auto_champs.rb
 test/test_match_auto_euro.rb
+test/test_match_auto_relegation.rb
 test/test_match_auto_worldcup.rb
 test/test_match_champs.rb
 test/test_match_eng.rb

data/Rakefile

@@ -3,7 +3,7 @@ require './lib/sportdb/formats/version.rb'
 
 Hoe.spec 'sportdb-formats' do
 
-  self.version = SportDb::Formats::VERSION
+  self.version = SportDb::Module::Formats::VERSION
 
   self.summary = "sportdb-formats - sport.db format and text utilities"
   self.description = summary
@@ -21,7 +21,7 @@ Hoe.spec 'sportdb-formats' do
 
   self.extra_deps = [
     ['alphabets',     '>= 1.0.0'],
-    ['date-formats',  '>= 1.0.0'],
+    ['date-formats',  '>= 1.0.1'],
     ['csvreader',     '>= 1.2.4'],
     ['sportdb-langs', '>= 0.1.0'],
 

data/lib/sportdb/formats.rb

@@ -10,15 +10,26 @@ require 'zip'     ## todo/check: if zip is alreay included in a required module
 
 
 
-def read_csv( path )
-  CsvHash.read( path, :header_converters => :symbol )
+def read_csv( path, sep:             nil,
+                    symbolize_names: nil )
+  opts = {}
+  opts[:sep]               = sep         if sep
+  opts[:header_converters] = :symbol     if symbolize_names
+
+  CsvHash.read( path, **opts )
 end
 
-def parse_csv( txt )
-  CsvHash.parse( txt, :header_converters => :symbol )
+def parse_csv( txt, sep:             nil,
+                    symbolize_names: nil )
+  opts = {}
+  opts[:sep]               = sep         if sep
+  opts[:header_converters] = :symbol     if symbolize_names
+
+  CsvHash.parse( txt, **opts )
 end
 
 
+
 ## more sportdb libs/gems
 require 'sportdb/langs'
 
@@ -68,6 +79,7 @@ require 'sportdb/formats/match/match_parser'
 require 'sportdb/formats/match/match_parser_auto_conf'
 require 'sportdb/formats/match/conf_parser'
 
+require 'sportdb/formats/match/match_parser_csv'
 
 require 'sportdb/formats/country/country_reader'
 require 'sportdb/formats/country/country_index'
@@ -147,4 +159,4 @@ end   # module SportDb
 
 
 
-puts SportDb::Formats.banner   # say hello
+puts SportDb::Module::Formats.banner   # say hello

data/lib/sportdb/formats/country/country_index.rb

@@ -30,11 +30,11 @@ class CountryIndex
     ## auto-fill countries
     ## pp recs
     recs.each do |rec|
-      ## rec e.g. { key:'af', fifa:'AFG', name:'Afghanistan'}
+      ## rec e.g. { key:'af', code:'AFG', name:'Afghanistan'}
 
       @countries << rec
 
-      ## add codes lookups - key, fifa, ...
+      ## add codes lookups - key, code, ...
       if @countries_by_code[ rec.key ]
         puts "** !! ERROR !! country code (key) >#{rec.key}< already exits!!"
         exit 1
@@ -42,13 +42,13 @@ class CountryIndex
         @countries_by_code[ rec.key ] = rec
       end
 
-      ## add fifa code (only) if different from key
-      if rec.key != rec.fifa.downcase
-        if @countries_by_code[ rec.fifa.downcase ]
-          puts "** !! ERROR !! country code (fifa) >#{rec.fifa.downcase}< already exits!!"
+      ## add  code (only) if different from key
+      if rec.key != rec.code.downcase
+        if @countries_by_code[ rec.code.downcase ]
+          puts "** !! ERROR !! country code  >#{rec.code.downcase}< already exits!!"
           exit 1
         else
-          @countries_by_code[ rec.fifa.downcase ] = rec
+          @countries_by_code[ rec.code.downcase ] = rec
         end
       end
 

data/lib/sportdb/formats/country/country_reader.rb

@@ -43,11 +43,31 @@ def parse
         #   e.g. East Germany        (-1989)  => East Germany (-1989)
         values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
         last_country.alt_names += values
+      elsif line =~ /^-[ ]*(\d{4})
+                        [ ]+
+                       (.+)$
+                    /x     ## check for historic lines e.g. -1989
+         year   = $1.to_i
+         parts  = $2.split( /=>|⇒/ )
+         values = parts[0].split( ',' )
+         values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
+
+         name = values[0]
+         code = values[1]
+
+         last_country = country = Country.new( name: "#{name} (-#{year})",
+                                               code: code )
+         ## country.alt_names << name    ## note: for now do NOT add name without year to alt_names - gets auto-add by index!!!
+
+         countries << country
+         ## todo/fix: add reference to country today (in parts[1] !!!!)
       else
         ## assume "regular" line
         ##  check if starts with id  (todo/check: use a more "strict"/better regex capture pattern!!!)
         ##   note: allow country codes upto 4 (!!) e.g. Northern Cyprus
-        if line =~ /^([a-z]{2,4})[ ]+(.+)$/
+        if line =~ /^([a-z]{2,4})
+                        [ ]+
+                       (.+)$/x
           key    = $1
           values = $2.split( ',' )
           ## strip and squish (white)spaces
@@ -60,14 +80,14 @@ def parse
           geos = split_geo( values[0] )
           name = geos[0]    ## note: ignore all other geos for now
 
-          ##   note: allow fifa country codes upto 4 (!!) e.g. Northern Cyprus
-          fifa = if values[1] && values[1] =~ /^[A-Z]{3,4}$/   ## note: also check format
+          ##   note: allow country codes up to 4 (!!) e.g. Northern Cyprus
+          code = if values[1] && values[1] =~ /^[A-Z]{3,4}$/   ## note: also check format
                    values[1]
                  else
                    if values[1]
-                     puts "** !!! ERROR !!! wrong fifa code format >#{values[1]}<; expected three (or four)-letter all up-case"
+                     puts "** !!! ERROR !!! wrong code format >#{values[1]}<; expected three (or four)-letter all up-case"
                    else
-                     puts "** !!! ERROR !!! missing fifa code for (canonical) country name"
+                     puts "** !!! ERROR !!! missing code for (canonical) country name"
                    end
                    exit 1
                  end
@@ -80,7 +100,7 @@ def parse
 
           last_country = country = Country.new( key: key,
                                                 name: name,
-                                                fifa: fifa,
+                                                code: code,
                                                 tags: tags )
           countries << country
         else

data/lib/sportdb/formats/datafile_package.rb

@@ -30,7 +30,7 @@ class Entry
     @name = path[ pack.path.length+1..-1 ]
   end
   def name()  @name; end
-  def read()  File.open( @path, 'r:utf-8' ).read; end
+  def read()  File.open( @path, 'r:utf-8' ) {|f| f.read }; end
 end  # class DirPackage::Entry
 
 
@@ -44,17 +44,20 @@ end  # class DirPackage::Entry
     @name = basename
   end
 
-  def each( pattern:,  extension: 'txt' )    ## todo/check: rename to glob or something - why? why not?
+  ## todo/check:  change pattern: to re:  - why? why not?
+  def each( pattern: )    ## todo/check: rename to glob or something - why? why not?
     ##   use just .* for extension or remove and check if File.file? and skip File.directory? - why? why not?
     ## note: incl. files starting with dot (.)) as candidates (normally excluded with just *)
-    Dir.glob( "#{@path}/**/{*,.*}.#{extension}" ).each do |path|
-      ## todo/fix: (auto) skip and check for directories
-      if EXCLUDE_RE.match( path )
-        ## note: skip dot dirs (e.g. .build/, .git/, etc.)
+    ##   todo/check/fix:   is there a better (simpler) glob pattern?  yes? no?
+    Dir.glob( "#{@path}/**/{*,.*}.*" ).each do |path|
+      if File.directory?( path )
+         ## always skip directories / folders
+      elsif EXCLUDE_RE.match( path )
+         ## note: skip dot dirs (e.g. .build/, .git/, etc.)
       elsif pattern.match( path )
         yield( Entry.new( self, path ))
       else
-        ## puts "  skipping >#{path}<"
+         ## puts "  skipping >#{path}<"
       end
     end
   end

data/lib/sportdb/formats/league/league_outline_reader.rb

@@ -121,14 +121,31 @@ class LeagueOutlineReader   ## todo/check - rename to LeaguePageReader / LeagueP
     values
   end
 
-  def check_stage( name )
-    known_stages = ['regular season',
-                    'championship round',
-                    'relegation round',
-                    'play-offs'
-                   ]
 
-    if known_stages.include?( name.downcase )
+  # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
+  KNOWN_STAGES = [
+    'Regular Season',
+    'Regular Stage',
+    'Championship Round',
+    'Championship Playoff',
+    'Relegation Round',
+    'Relegation Playoff',
+    'Play-offs',
+    'Playoff Stage',
+    'Grunddurchgang',
+    'Finaldurchgang - Qualifikationsgruppe',
+    'Finaldurchgang - Qualifikation',
+    'Finaldurchgang - Meistergruppe',
+    'Finaldurchgang - Meister',
+    'EL Play-off',
+    'Europa League Play-off',
+    'Europa-League-Play-offs',
+  ].map {|name| name.downcase.gsub( /[^a-z]/, '' ) }
+
+
+  def check_stage( name )
+    # note: normalize names e.g. downcase and remove all non a-z chars (e.g. space, dash, etc.)
+    if KNOWN_STAGES.include?( name.downcase.gsub( /[^a-z]/, '' ) )
        ## everything ok
     else
       puts "** !!! ERROR - no (league) stage match found for >#{name}<, add to (builtin) stages table; sorry"

data/lib/sportdb/formats/league/league_reader.rb

@@ -113,9 +113,9 @@ def parse
             ##             add a list of (auto-)excluded country codes with conflicts? why? why not?
             ##                 cl - a) Chile  b) Champions League
             alt_names_auto << "#{country.key.upcase}"   if league_key == '1'   ## add shortcut for top level 1 (just country key)
-            if country.key.upcase != country.fifa
-              alt_names_auto << "#{country.fifa} #{league_key.upcase.gsub('.', ' ')}"
-              alt_names_auto << "#{country.fifa}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
+            if country.key.upcase != country.code
+              alt_names_auto << "#{country.code} #{league_key.upcase.gsub('.', ' ')}"
+              alt_names_auto << "#{country.code}"    if league_key == '1'   ## add shortcut for top level 1 (just country key)
             end
             alt_names_auto << "#{country.name} #{league_key}"  if league_key =~ /^[0-9]+$/   ## if all numeric e.g. add Austria 1 etc.
           else   ## assume int'l (no country) e.g. champions league, etc.

data/lib/sportdb/formats/match/mapper.rb

@@ -7,21 +7,21 @@ module SportDb
 ##   see https://github.com/textkit/textutils/blob/master/textutils/lib/textutils/title_mapper2.rb
 
 
-class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why not??
+class MapperV2      ## todo/check: rename to NameMapper ? why? why not??
 
   include Logging
 
-  attr_reader :known_titles   ## rename to mapping or mappings or just titles - why? why not?
+  attr_reader :known_names   ## rename to mapping or mappings or just names - why? why not?
 
   ########
   ##  key:      e.g. augsburg
-  ##  title:    e.g. FC Augsburg
-  ##  length (of title(!!) - not regex pattern):   e.g. 11   -- do not count dots (e.g. U.S.A. => 3 or 6) why? why not?
-  MappingStruct =  Struct.new( :key, :title, :length, :pattern)     ## todo/check: use (rename to) TitleStruct - why? why not??
+  ##  name:     e.g. FC Augsburg
+  ##  length (of name(!!) - not regex pattern):   e.g. 11   -- do not count dots (e.g. U.S.A. => 3 or 6) why? why not?
+  MappingStruct =  Struct.new( :key, :name, :length, :pattern)     ## todo/check: use (rename to) NameStruct - why? why not??
 
   ######
   ## convenience helper - (auto)build ActiveRecord-like team records/structs
-  Record = Struct.new( :key, :title, :synonyms )
+  Record = Struct.new( :key, :name, :alt_names )
   def build_records( txt_or_lines )
     recs = []
 
@@ -44,12 +44,12 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
       values = line.split( '|' )
       values = values.map { |value| value.strip }
 
-      title    = values[0]
+      name      = values[0]
       ## note: quick hack - auto-generate key, that is, remove all non-ascii chars and downcase
-      key      = title.downcase.gsub( /[^a-z]/, '' )
-      synonyms = values.size > 1 ? values[1..-1].join( '|' ) : nil
+      key       = name.downcase.gsub( /[^a-z]/, '' )
+      alt_names = values.size > 1 ? values[1..-1].join( '|' ) : nil
 
-      recs << Record.new( key, title, synonyms )
+      recs << Record.new( key, name, alt_names )
     end
     recs
   end
@@ -63,10 +63,10 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
                                                                   (records_or_mapping.is_a?( Array ) && records_or_mapping[0].is_a?( String ))
 
     ## build mapping lookup table
-    @known_titles =  if records_or_mapping.is_a?( Hash )  ## assume "custom" mapping hash table (title/name=>record)
-                        build_title_table_for_mapping( records_or_mapping )
+    @known_names =  if records_or_mapping.is_a?( Hash )  ## assume "custom" mapping hash table (name=>record)
+                        build_name_table_for_mapping( records_or_mapping )
                      else  ## assume array of records
-                        build_title_table_for_records( records_or_mapping )
+                        build_name_table_for_records( records_or_mapping )
                      end
 
     ## build lookup hash by record (e.g. team/club/etc.) key
@@ -85,9 +85,9 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
 
 
 
-  def map_titles!( line )   ## rename to just map! - why?? why not???
+  def map_names!( line )   ## rename to just map! - why?? why not???
     begin
-      found = map_title_for!( @tag, line, @known_titles )
+      found = map_name_for!( @tag, line, @known_names )
     end while found
   end
 
@@ -110,27 +110,27 @@ class MapperV2      ## todo/check: rename to NameMapper/TitleMapper ? why? why n
 
 
 private
-  def build_title_table_for_mapping( mapping )
-    known_titles = []
+  def build_name_table_for_mapping( mapping )
+    known_names = []
 
-    mapping.each do |title, rec|
+    mapping.each do |name, rec|
       m = MappingStruct.new
       m.key     = rec.key
-      m.title   = title
-      m.length  = title.length
-      m.pattern = Regexp.escape( title )   ## note: just use "standard" regex escape (e.g. no extras for umlauts,accents,etc.)
+      m.name    = name
+      m.length  = name.length
+      m.pattern = Regexp.escape( name )   ## note: just use "standard" regex escape (e.g. no extras for umlauts,accents,etc.)
 
-      known_titles << m
+      known_names << m
     end
 
     ## note: sort here by length (largest goes first - best match)
-    known_titles = known_titles.sort { |l,r| r.length <=> l.length }
-    known_titles
+    known_names = known_names.sort { |l,r| r.length <=> l.length }
+    known_names
   end
 
-  def build_title_table_for_records( records )
+  def build_name_table_for_records( records )
 
-    ## build known tracks table w/ synonyms e.g.
+    ## build known tracks table w/ alt names e.g.
     #
     # [[ 'wolfsbrug', 'VfL Wolfsburg'],
     #  [ 'augsburg',  'FC Augsburg'],
@@ -138,65 +138,65 @@ private
     #  [ 'augsburg',  'Augi3' ],
     #  [ 'stuttgart', 'VfB Stuttgart']]
 
-    known_titles = []
+    known_names = []
 
     records.each_with_index do |rec,index|
 
-      title_candidates = []
-      title_candidates << rec.title
+      name_candidates = []
+      name_candidates << rec.name
 
-      title_candidates += rec.synonyms.split('|') if rec.synonyms && !rec.synonyms.empty?
+      name_candidates += rec.alt_names.split('|') if rec.alt_names && !rec.alt_names.empty?
 
 
-      ## check if title includes subtitle e.g. Grand Prix Japan (Suzuka Circuit)
-      #  make subtitle optional by adding title w/o subtitle e.g. Grand Prix Japan
+      ## check if name includes subname e.g. Grand Prix Japan (Suzuka Circuit)
+      #  make subname optional by adding name w/o subname e.g. Grand Prix Japan
 
-      titles = []
-      title_candidates.each do |t|
-        titles << t
+      names = []
+      name_candidates.each do |t|
+        names << t
         if t =~ /\(.+\)/
-          extra_title = t.gsub( /\(.+\)/, '' ) # remove/delete subtitles
+          extra_name = t.gsub( /\(.+\)/, '' ) # remove/delete subnames
           # note: strip leading n trailing withspaces too!
           #  -- todo: add squish or something if () is inline e.g. leaves two spaces?
-          extra_title.strip!
-          titles << extra_title
+          extra_name.strip!
+          names << extra_name
         end
       end
 
-      titles.each do |t|
+      names.each do |name|
         m = MappingStruct.new
         m.key     = rec.key
-        m.title   = t
-        m.length  = t.length
+        m.name    = name
+        m.length  = name.length
         ## note: escape for regex plus allow subs for special chars/accents
-        m.pattern = title_esc_regex( t )
+        m.pattern = name_esc_regex( name )
 
-        known_titles << m
+        known_names << m
       end
 
-      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{titles.join('|')}<"
+      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{names.join('|')}<"
 
       ## note: only include code field - if defined
       if rec.respond_to?(:code) && rec.code && !rec.code.empty?
         m = MappingStruct.new
         m.key     = rec.key
-        m.title   = rec.code
+        m.name    = rec.code
         m.length  = rec.code.length
         m.pattern = rec.code   ## note: use code for now as is (no variants allowed fow now)
 
-        known_titles << m
+        known_names << m
       end
     end
 
     ## note: sort here by length (largest goes first - best match)
       #  exclude code and key (key should always go last)
-    known_titles = known_titles.sort { |l,r| r.length <=> l.length }
-    known_titles
+    known_names = known_names.sort { |l,r| r.length <=> l.length }
+    known_names
   end
 
 
 
-  def map_title_for!( tag, line, mappings )
+  def map_name_for!( tag, line, mappings )
     mappings.each do |mapping|
       key     = mapping.key
       pattern = mapping.pattern
@@ -234,9 +234,9 @@ private
 
 
 ####
-# title helper cut-n-paste copy from TextUtils
+# name helper cut-n-paste copy from TextUtils
 ##  see https://github.com/textkit/textutils/blob/master/textutils/lib/textutils/helper/title_helper.rb
-def title_esc_regex( title_unescaped )
+def name_esc_regex( name_unescaped )
 
       ##  escape regex special chars e.g.
       #    . to \. and
@@ -257,16 +257,16 @@ def title_esc_regex( title_unescaped )
       # e.g. Club Atlético Colón (Santa Fe)
       # e.g. Bauer Anton (????)
 
-      ## NB: cannot use Regexp.escape! will escape space '' to '\ '
-      ## title = Regexp.escape( title_unescaped )
-      title = title_unescaped.gsub( '.', '\.' )
-      title = title.gsub( '(', '\(' )
-      title = title.gsub( ')', '\)' )
-      title = title.gsub( '?', '\?' )
-      title = title.gsub( '*', '\*' )
-      title = title.gsub( '+', '\+' )
-      title = title.gsub( '$', '\$' )
-      title = title.gsub( '^', '\^' )
+      ## note: cannot use Regexp.escape! will escape space '' to '\ '
+      ## name = Regexp.escape( name_unescaped )
+      name = name_unescaped.gsub( '.', '\.' )
+      name = name.gsub( '(', '\(' )
+      name = name.gsub( ')', '\)' )
+      name = name.gsub( '?', '\?' )
+      name = name.gsub( '*', '\*' )
+      name = name.gsub( '+', '\+' )
+      name = name.gsub( '$', '\$' )
+      name = name.gsub( '^', '\^' )
 
       ##  match accented char with or without accents
       ##  add (ü|ue) etc.
@@ -309,10 +309,10 @@ def title_esc_regex( title_unescaped )
       ##   collect some more (real-world) examples first!!!!!
 
       alternatives.each do |alt|
-        title = title.gsub( alt[0], alt[1] )
+        name = name.gsub( alt[0], alt[1] )
       end
 
-      title
+      name
   end
 
 end # class MapperV2