sportdb-formats 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1188cc0edf55cd83ccdee10638662c64b302a44e
-  data.tar.gz: 06b6b09902513298357a9f47deef83cdf6962b9e
+  metadata.gz: 29715e2e61cd99fe3520e861b1d84c4614055650
+  data.tar.gz: e8109a80c7f79926c271560fd63f8503a44fabd2
 SHA512:
-  metadata.gz: 337fdb0f298625e847e40a29e47f2ee2e6c638e1ed8a4e72e340008b16de6d10ea5907923530594c0e4739de1a477dd4fe847f10d062372a64c5e4103ef3c06b
-  data.tar.gz: 72c0281e3f32a74750f8eb494aafd26b25a3c5f497842f389ea714bcf430f41451346b9aec7aac005dba7b8e8ab283945aca58a49ef7ecc8df6f8116d87b6370
+  metadata.gz: 8cb97f1cd4ae2d56e3b81282a7f921f99b2dcd325bafcf739e9d4c8a8bf9139fa99652a1a2804cf33604949b36b15694e7da68d851cef6d91b9b7dd727144bcf
+  data.tar.gz: 5967f5b9558d963cd9a6974be5b53ad343b8862c84ae4640bb6c3bd6975f80ab1a687946aa2a91aeeeecb9e5908537f530eb6e837553fccfc2d92f43db082edc

data/Manifest.txt

@@ -4,9 +4,14 @@ README.md
 Rakefile
 lib/sportdb/formats.rb
 lib/sportdb/formats/config.rb
+lib/sportdb/formats/country/country_index.rb
+lib/sportdb/formats/country/country_reader.rb
 lib/sportdb/formats/datafile.rb
 lib/sportdb/formats/datafile_package.rb
 lib/sportdb/formats/goals.rb
+lib/sportdb/formats/league/league_index.rb
+lib/sportdb/formats/league/league_outline_reader.rb
+lib/sportdb/formats/league/league_reader.rb
 lib/sportdb/formats/match/conf_parser.rb
 lib/sportdb/formats/match/mapper.rb
 lib/sportdb/formats/match/mapper_teams.rb
@@ -29,13 +34,27 @@ lib/sportdb/formats/structs/season.rb
 lib/sportdb/formats/structs/standings.rb
 lib/sportdb/formats/structs/team.rb
 lib/sportdb/formats/structs/team_usage.rb
+lib/sportdb/formats/team/club_index.rb
+lib/sportdb/formats/team/club_reader.rb
+lib/sportdb/formats/team/club_reader_props.rb
+lib/sportdb/formats/team/national_team_index.rb
+lib/sportdb/formats/team/team_index.rb
+lib/sportdb/formats/team/wiki_reader.rb
 lib/sportdb/formats/version.rb
 test/helper.rb
+test/test_club_index.rb
+test/test_club_reader.rb
+test/test_club_reader_props.rb
 test/test_clubs.rb
 test/test_conf.rb
+test/test_country_index.rb
+test/test_country_reader.rb
 test/test_csv_reader.rb
 test/test_datafile.rb
 test/test_goals.rb
+test/test_league_index.rb
+test/test_league_outline_reader.rb
+test/test_league_reader.rb
 test/test_match.rb
 test/test_match_auto.rb
 test/test_match_auto_champs.rb
@@ -49,5 +68,7 @@ test/test_name_helper.rb
 test/test_outline_reader.rb
 test/test_package.rb
 test/test_package_match.rb
+test/test_regex.rb
 test/test_scores.rb
 test/test_season.rb
+test/test_wiki_reader.rb

data/lib/sportdb/formats.rb

@@ -69,6 +69,69 @@ require 'sportdb/formats/match/match_parser_auto_conf'
 require 'sportdb/formats/match/conf_parser'
 
 
+require 'sportdb/formats/country/country_reader'
+require 'sportdb/formats/country/country_index'
+
+
+## add convenience helper
+module SportDb
+module Import
+class Country
+  def self.read( path )  CountryReader.read( path ); end
+  def self.parse( txt )  CountryReader.parse( txt ); end
+end   # class Country
+end   # module Import
+end   # module SportDb
+
+
+require 'sportdb/formats/league/league_reader'
+require 'sportdb/formats/league/league_index'
+require 'sportdb/formats/league/league_outline_reader'
+
+##
+## add convenience helper / short-cuts
+module SportDb
+module Import
+class League
+  def self.read( path ) LeagueReader.read( path ); end
+  def self.parse( txt ) LeagueReader.parse( txt ); end
+end   # class League
+end   # module Import
+end   # module SportDb
+
+
+require 'sportdb/formats/team/club_reader'
+require 'sportdb/formats/team/club_reader_props'
+require 'sportdb/formats/team/club_index'
+require 'sportdb/formats/team/wiki_reader'
+require 'sportdb/formats/team/national_team_index'
+require 'sportdb/formats/team/team_index'
+
+
+###
+# add convenience helpers / shortcuts
+module SportDb
+module Import
+class Club
+  def self.read( path )  ClubReader.read( path ); end
+  def self.parse( txt )  ClubReader.parse( txt ); end
+
+  def self.read_props( path )  ClubPropsReader.read( path ); end
+  def self.parse_props( txt )  ClubPropsReader.parse( txt ); end
+  ##  todo/check: use ClubProps.read and ClubProps.parse convenience alternate shortcuts - why? why not?
+end # class Club
+end   # module Import
+end   # module SportDb
+
+
+
+
+
+
+
+
+
+
 ## let's put test configuration in its own namespace / module
 module SportDb
 class Test    ## todo/check: works with module too? use a module - why? why not?

data/lib/sportdb/formats/country/country_index.rb

@@ -0,0 +1,192 @@
+# encoding: utf-8
+
+module SportDb
+module Import
+
+## built-in countries for (quick starter) auto-add
+class CountryIndex
+
+  attr_reader :countries     ## all country records
+
+  def initialize( recs )
+    @countries         = []
+    @countries_by_code = {}
+    @countries_by_name = {}
+
+    add( recs )
+  end
+
+
+  ## helpers from country - use a helper module for includes (share with clubs etc.) - why? why not?
+  include NameHelper
+  ## incl. strip_year( name )
+  ##       has_year?( name)
+  ##       strip_lang( name )
+  ##       normalize( name )
+
+
+  def add( recs )
+    ###########################################
+    ## auto-fill countries
+    ## pp recs
+    recs.each do |rec|
+      ## rec e.g. { key:'af', fifa:'AFG', name:'Afghanistan'}
+
+      @countries << rec
+
+      ## add codes lookups - key, fifa, ...
+      if @countries_by_code[ rec.key ]
+        puts "** !! ERROR !! country code (key) >#{rec.key}< already exits!!"
+        exit 1
+      else
+        @countries_by_code[ rec.key ] = rec
+      end
+
+      ## add fifa code (only) if different from key
+      if rec.key != rec.fifa.downcase
+        if @countries_by_code[ rec.fifa.downcase ]
+          puts "** !! ERROR !! country code (fifa) >#{rec.fifa.downcase}< already exits!!"
+          exit 1
+        else
+          @countries_by_code[ rec.fifa.downcase ] = rec
+        end
+      end
+
+
+      ##  add all names (canonical name + alt names
+      names = [rec.name] + rec.alt_names
+      more_names = []
+      ## check "hand-typed" names for year (auto-add)
+      ## check for year(s) e.g. (1887-1911), (-2013),
+      ##                        (1946-2001,2013-) etc.
+      names.each do |name|
+        if has_year?( name )
+          more_names <<  strip_year( name )
+        end
+      end
+
+      names += more_names
+      ## check for duplicates - simple check for now - fix/improve
+      ## todo/fix: (auto)remove duplicates - why? why not?
+      count      = names.size
+      count_uniq = names.uniq.size
+      if count != count_uniq
+        puts "** !!! ERROR !!! - #{count-count_uniq} duplicate name(s):"
+        pp names
+        pp rec
+        exit 1
+      end
+
+      names.each_with_index do |name,i|
+        ## check lang codes e.g. [en], [fr], etc.
+        ##  todo/check/fix:  move strip_lang up in the chain - check for duplicates (e.g. only lang code marker different etc.) - why? why not?
+        name = strip_lang( name )
+        norm = normalize( name )
+        old_rec = @countries_by_name[ norm ]
+        if old_rec
+          ## check if country name already is included or is new country rec
+            msg = "** !!! ERROR !!! - name conflict/duplicate - >#{name}< will overwrite >#{old_rec.name}< with >#{rec.name}<"
+            puts msg
+            exit 1
+        else
+          @countries_by_name[ norm ] = rec
+        end
+      end
+
+    end  ## each record
+  end # method initialize
+
+
+
+  ##  fix/todo: add  find_by (alias for find_by_name/find_by_code)
+  def find_by_code( code )
+    code = code.to_s.downcase   ## allow symbols (and always downcase e.g. AUT to aut etc.)
+    @countries_by_code[ code ]
+  end
+
+  def find_by_name( name )
+    name = normalize( name.to_s )  ## allow symbols too (e.g. use to.s first)
+    @countries_by_name[ name ]
+  end
+
+  def []( key )
+    country = find_by_code( key )
+    country = find_by_name( key )  if country.nil?     ## try lookup / find by (normalized) name
+    country
+  end
+  alias_method :find, :[]
+
+
+  ###
+  ##   split/parse country line
+  ##
+  ##  split on bullet e.g.
+  ##   split into name and code with regex - make code optional
+  ##
+  ##  Examples:
+  ##    Österreich • Austria (at)
+  ##    Österreich • Austria
+  ##    Austria
+  ##    Deutschland (de) • Germany
+  ##
+  ##   todo/check: support more formats - why? why not?
+  ##       e.g.  Austria, AUT  (e.g. with comma - why? why not?)
+  def parse( line )
+    values = line.split( '•' )   ## use/support multi-lingual separator
+    country = nil
+    values.each do |value|
+       value = value.strip
+       ## check for trailing country code e.g. (at), (eng), etc.
+       if value =~ /[ ]+\((?<code>[a-z]{1,4})\)$/  ## e.g. Austria (at)
+         code =  $~[:code]
+         name = value[0...(value.size-code.size-2)].strip  ## note: add -2 for brackets
+         candidates = [ find_by_code( code ), find_by_name( name ) ]
+         if candidates[0].nil?
+           puts "** !!! ERROR !!! country - unknown code >#{code}< in line: #{line}"
+           pp line
+           exit 1
+         end
+         if candidates[1].nil?
+           puts "** !!! ERROR !!! country - unknown name >#{code}< in line: #{line}"
+           pp line
+           exit 1
+         end
+         if candidates[0] != candidates[1]
+           puts "** !!! ERROR !!! country - name and code do NOT match the same country:"
+           pp line
+           pp candidates
+           exit 1
+         end
+         if country && country != candidates[0]
+           puts "** !!! ERROR !!! country - names do NOT match the same country:"
+           pp line
+           pp country
+           pp candidates
+           exit 1
+         end
+         country = candidates[0]
+       else
+         ## just assume value is name or code
+         candidate = find( value )
+         if candidate.nil?
+           puts "** !!! ERROR !!! country - unknown name or code >#{value}< in line: #{line}"
+           pp line
+           exit 1
+         end
+         if country && country != candidate
+           puts "** !!! ERROR !!! country - names do NOT match the same country:"
+           pp line
+           pp country
+           pp candidate
+           exit 1
+         end
+         country = candidate
+       end
+    end
+    country
+  end # method parse
+end # class CountryIndex
+
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/formats/country/country_reader.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+
+
+module SportDb
+module Import
+
+
+class CountryReader
+
+
+def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+  txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+  parse( txt )
+end
+
+def self.parse( txt )
+  new( txt ).parse
+end
+
+
+def initialize( txt )
+  @txt = txt
+end
+
+def parse
+  countries    = []
+  last_country = nil    ## note/check/fix: use countries[-1] - why? why not?
+
+  OutlineReader.parse( @txt ).each do |node|
+
+     node_type = node[0]
+
+     if [:h1, :h2].include?( node_type )
+       ## skip headings (and headings) for now too
+     elsif node_type == :p    ## paragraph
+      lines = node[1]
+      lines.each do |line|
+      if line.start_with?( '|' )
+        ## assume continuation with line of alternative names
+        ##  note: skip leading pipe
+        values = line[1..-1].split( '|' )   # team names - allow/use pipe(|)
+        ## strip and squish (white)spaces
+        #   e.g. East Germany        (-1989)  => East Germany (-1989)
+        values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
+        last_country.alt_names += values
+      else
+        ## assume "regular" line
+        ##  check if starts with id  (todo/check: use a more "strict"/better regex capture pattern!!!)
+        ##   note: allow country codes upto 4 (!!) e.g. Northern Cyprus
+        if line =~ /^([a-z]{2,4})[ ]+(.+)$/
+          key    = $1
+          values = $2.split( ',' )
+          ## strip and squish (white)spaces
+          #   e.g. East Germany        (-1989)  => East Germany (-1989)
+          values = values.map { |value| value.strip.gsub( /[ \t]+/, ' ' ) }
+
+          ## note: remove "overlords" from geo-tree marked territories  e.g. UK, US, etc. from name
+          ##    e.g. England › UK      => England
+          ##         Puerto Rico › US  => Puerto Rico
+          geos = split_geo( values[0] )
+          name = geos[0]    ## note: ignore all other geos for now
+
+          ##   note: allow fifa country codes upto 4 (!!) e.g. Northern Cyprus
+          fifa = if values[1] && values[1] =~ /^[A-Z]{3,4}$/   ## note: also check format
+                   values[1]
+                 else
+                   if values[1]
+                     puts "** !!! ERROR !!! wrong fifa code format >#{values[1]}<; expected three (or four)-letter all up-case"
+                   else
+                     puts "** !!! ERROR !!! missing fifa code for (canonical) country name"
+                   end
+                   exit 1
+                 end
+
+          tags = if values[2]   ## check if tags presents
+                   split_tags( values[2] )
+                 else
+                   []
+                 end
+
+          last_country = country = Country.new( key: key,
+                                                name: name,
+                                                fifa: fifa,
+                                                tags: tags )
+          countries << country
+        else
+          puts "** !! ERROR - missing key for (canonical) country name"
+          exit 1
+        end
+      end
+      end  # each line
+    else
+      puts "** !! ERROR - unknown node type / (input) source line:"
+      pp node
+      exit 1
+    end
+  end    # each node
+
+  countries
+end  # method parse
+
+
+
+#######################################
+##  helpers
+def split_tags( str )
+  tags = str.split( /[|<>‹›]/ )   ## allow pipe (|) and (<>‹›) as divider for now - add more? why? why not?
+  tags = tags.map { |tag| tag.strip }
+  tags
+end
+
+def split_geo( str )   ## todo/check: rename to parse_geo(s) - why? why not?
+  ## split into geo tree
+  geos = str.split( /[<>‹›]/ )          ## note: allow > < or › ‹ for now
+  geos = geos.map { |geo| geo.strip }   ## remove all whitespaces
+  geos
+end
+
+end  # class CountryReader
+
+end   # module Import
+end   # module SportDb