sportdb-formats 1.0.6 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35a7be066d82d59b2cf35c8dc5676c0d5118479b
-  data.tar.gz: eeceea13624ae5c795bc2b2709e7bef6727dd184
+  metadata.gz: bcac09da9cbe9b0e7d703df7b6b5075025979655
+  data.tar.gz: 71b00a95c66920bd25c1d92d0b4a29a2ec4436e3
 SHA512:
-  metadata.gz: 732d53cf84db4d87449c9fe86e4c0b1109a277b8442a7fd15cf61a5318555e6a53c61c08ab1b66187429fc770f89e4a72ef48b9bde616e2736a96bb8d1a47324
-  data.tar.gz: 60b46bf871ffd180bec2a3a96403ecceebc5689dde1d0c6181ecf1b22fdaf9b8aaf49bd8e15a8ba886cf9751e62fc89658376b27d983e8871003f498d6498c0e
+  metadata.gz: 81b387f334f672dbe493884bde8661dfd83c892a834bec422c95efc5231141f556397d5aacdfd7665e935589f5cffcedcbeab1f0993a414e8678ae56c89e80ab
+  data.tar.gz: ba6546d68cafa1198c39dec94d91481ec6652d051f3e9887b1366a1319a9a6b35667313390fac1bbae3cc46056468aff0d3bc41a8e128d429f0f9a16d2c7f998

data/Manifest.txt

@@ -3,11 +3,12 @@ Manifest.txt
 README.md
 Rakefile
 lib/sportdb/formats.rb
-lib/sportdb/formats/config.rb
 lib/sportdb/formats/country/country_index.rb
 lib/sportdb/formats/country/country_reader.rb
 lib/sportdb/formats/datafile.rb
 lib/sportdb/formats/datafile_package.rb
+lib/sportdb/formats/event/event_index.rb
+lib/sportdb/formats/event/event_reader.rb
 lib/sportdb/formats/goals.rb
 lib/sportdb/formats/league/league_index.rb
 lib/sportdb/formats/league/league_outline_reader.rb
@@ -17,26 +18,13 @@ lib/sportdb/formats/match/mapper.rb
 lib/sportdb/formats/match/mapper_teams.rb
 lib/sportdb/formats/match/match_parser.rb
 lib/sportdb/formats/match/match_parser_auto_conf.rb
-lib/sportdb/formats/match/match_parser_csv.rb
-lib/sportdb/formats/name_helper.rb
 lib/sportdb/formats/outline_reader.rb
 lib/sportdb/formats/package.rb
 lib/sportdb/formats/parser_helper.rb
-lib/sportdb/formats/score/score_formats.rb
-lib/sportdb/formats/score/score_parser.rb
-lib/sportdb/formats/season_utils.rb
-lib/sportdb/formats/structs/country.rb
-lib/sportdb/formats/structs/group.rb
-lib/sportdb/formats/structs/league.rb
-lib/sportdb/formats/structs/match.rb
-lib/sportdb/formats/structs/matchlist.rb
-lib/sportdb/formats/structs/round.rb
-lib/sportdb/formats/structs/season.rb
-lib/sportdb/formats/structs/standings.rb
-lib/sportdb/formats/structs/team.rb
-lib/sportdb/formats/structs/team_usage.rb
 lib/sportdb/formats/team/club_index.rb
+lib/sportdb/formats/team/club_index_history.rb
 lib/sportdb/formats/team/club_reader.rb
+lib/sportdb/formats/team/club_reader_history.rb
 lib/sportdb/formats/team/club_reader_props.rb
 lib/sportdb/formats/team/national_team_index.rb
 lib/sportdb/formats/team/team_index.rb
@@ -44,35 +32,20 @@ lib/sportdb/formats/team/wiki_reader.rb
 lib/sportdb/formats/version.rb
 test/helper.rb
 test/test_club_index.rb
+test/test_club_index_history.rb
 test/test_club_reader.rb
+test/test_club_reader_history.rb
 test/test_club_reader_props.rb
-test/test_clubs.rb
-test/test_conf.rb
 test/test_country_index.rb
 test/test_country_reader.rb
-test/test_csv_match_parser.rb
-test/test_csv_match_parser_utils.rb
-test/test_csv_reader.rb
 test/test_datafile.rb
 test/test_datafile_package.rb
 test/test_goals.rb
 test/test_league_index.rb
 test/test_league_outline_reader.rb
 test/test_league_reader.rb
-test/test_match.rb
-test/test_match_auto.rb
-test/test_match_auto_champs.rb
-test/test_match_auto_euro.rb
-test/test_match_auto_worldcup.rb
-test/test_match_champs.rb
-test/test_match_eng.rb
-test/test_match_euro.rb
-test/test_match_worldcup.rb
-test/test_name_helper.rb
 test/test_outline_reader.rb
 test/test_package.rb
 test/test_package_match.rb
 test/test_regex.rb
-test/test_scores.rb
-test/test_season.rb
 test/test_wiki_reader.rb

data/Rakefile

@@ -20,12 +20,9 @@ Hoe.spec 'sportdb-formats' do
   self.licenses = ['Public Domain']
 
   self.extra_deps = [
-    ['alphabets',     '>= 1.0.0'],
-    ['date-formats',  '>= 1.0.1'],
-    ['csvreader',     '>= 1.2.4'],
-    ['sportdb-langs', '>= 0.1.0'],
+    ['sportdb-structs', '>= 0.1.0'],
 
-    ['rubyzip',       '>= 1.2.4' ],
+    ['rubyzip',         '>= 1.2.4' ],
   ]
 
   self.spec_extras = {

data/lib/sportdb/formats.rb

@@ -1,43 +1,9 @@
-# encoding: utf-8
-
-
 ## 3rd party gems
-require 'alphabets'      # downcase_i18n, unaccent, variants, ...
-require 'date/formats'   # DateFormats.parse, find!, ...
-require 'csvreader'
-
-require 'zip'     ## todo/check: if zip is alreay included in a required module
-
-
-
-def read_csv( path, sep:             nil,
-                    symbolize_names: nil )
-  opts = {}
-  opts[:sep]               = sep         if sep
-  opts[:header_converters] = :symbol     if symbolize_names
-
-  CsvHash.read( path, **opts )
-end
-
-def parse_csv( txt, sep:             nil,
-                    symbolize_names: nil )
-  opts = {}
-  opts[:sep]               = sep         if sep
-  opts[:header_converters] = :symbol     if symbolize_names
+require 'sportdb/structs'
 
-  CsvHash.parse( txt, **opts )
-end
-
-
-
-## more sportdb libs/gems
-require 'sportdb/langs'
 
+require 'zip'     ## todo/check: if zip is alreay included in a required module
 
-## todo/fix: move shortcut to sportdb/langs!!!
-module SportDb
-  Logging = LogUtils::Logging     ## logging machinery shortcut; use LogUtils for now
-end
 
 
 
@@ -45,41 +11,59 @@ end
 # our own code
 require 'sportdb/formats/version' # let version always go first
 
-require 'sportdb/formats/config'  # let "global" config "framework" go next - why? why not?
-
 require 'sportdb/formats/outline_reader'
 require 'sportdb/formats/datafile'
 require 'sportdb/formats/datafile_package'
 require 'sportdb/formats/package'
-require 'sportdb/formats/season_utils'
 
-require 'sportdb/formats/name_helper'
+
 require 'sportdb/formats/parser_helper'
 
-require 'sportdb/formats/structs/country'
-require 'sportdb/formats/structs/season'
-require 'sportdb/formats/structs/league'
-require 'sportdb/formats/structs/team'
-require 'sportdb/formats/structs/round'
-require 'sportdb/formats/structs/group'
-require 'sportdb/formats/structs/match'
-require 'sportdb/formats/structs/matchlist'
-require 'sportdb/formats/structs/standings'
-require 'sportdb/formats/structs/team_usage'
+
+module SportDb
+  module Import
+    Season = ::Season   ## add a convenience alias for top-level Season class
+
+    ## add "old" convenience aliases for structs - why? why not?
+    ##   todo/check: just use include Sports !!!!
+    Country      = ::Sports::Country
+    League       = ::Sports::League
+    Group        = ::Sports::Group
+    Round        = ::Sports::Round
+    Match        = ::Sports::Match
+    Matchlist    = ::Sports::Matchlist
+    Goal         = ::Sports::Goal
+    Team         = ::Sports::Team
+    NationalTeam = ::Sports::NationalTeam
+    Club         = ::Sports::Club
+    Standings    = ::Sports::Standings
+    TeamUsage    = ::Sports::TeamUsage
+
+
+    class Team
+      ## add convenience lookup helper / method for name by season for now
+      ##   use clubs history - for now kept separate from struct - why? why not?
+      def name_by_season( season )
+        ## note: returns / fallback to "regular" name if no records found in history
+        SportDb::Import.catalog.clubs_history.find_name_by( name: name, season: season ) || name
+      end
+    end  # class Team
+
+  end   # module Import
+end     # module SportDb
 
 
-require 'sportdb/formats/score/score_formats'
-require 'sportdb/formats/score/score_parser'
 require 'sportdb/formats/goals'
 
 
+
 require 'sportdb/formats/match/mapper'
 require 'sportdb/formats/match/mapper_teams'
 require 'sportdb/formats/match/match_parser'
 require 'sportdb/formats/match/match_parser_auto_conf'
 require 'sportdb/formats/match/conf_parser'
 
-require 'sportdb/formats/match/match_parser_csv'
+
 
 require 'sportdb/formats/country/country_reader'
 require 'sportdb/formats/country/country_index'
@@ -119,11 +103,15 @@ require 'sportdb/formats/team/wiki_reader'
 require 'sportdb/formats/team/national_team_index'
 require 'sportdb/formats/team/team_index'
 
+require 'sportdb/formats/team/club_reader_history'
+require 'sportdb/formats/team/club_index_history'
+
 
 ###
 # add convenience helpers / shortcuts
 module SportDb
-module Import
+  module Import
+
 class Club
   def self.read( path )  ClubReader.read( path ); end
   def self.parse( txt )  ClubReader.parse( txt ); end
@@ -132,31 +120,27 @@ class Club
   def self.parse_props( txt )  ClubPropsReader.parse( txt ); end
   ##  todo/check: use ClubProps.read and ClubProps.parse convenience alternate shortcuts - why? why not?
 end # class Club
+
 end   # module Import
 end   # module SportDb
 
 
+require 'sportdb/formats/event/event_reader'
+require 'sportdb/formats/event/event_index'
 
-
-
-
-
-
-
-
-## let's put test configuration in its own namespace / module
+## add convenience helper
 module SportDb
-class Test    ## todo/check: works with module too? use a module - why? why not?
-
-  ####
-  #  todo/fix:  find a better way to configure shared test datasets - why? why not?
-  #    note: use one-up (..) directory for now as default - why? why not?
-  def self.data_dir()        @data_dir ||= '../test'; end
-  def self.data_dir=( path ) @data_dir = path; end
-end
+module Import
+class EventInfo
+  def self.read( path )  EventInfoReader.read( path ); end
+  def self.parse( txt )  EventInfoReader.parse( txt ); end
+end   # class EventInfo
+end   # module Import
 end   # module SportDb
 
 
 
 
+
+
 puts SportDb::Module::Formats.banner   # say hello

data/lib/sportdb/formats/country/country_index.rb

@@ -109,12 +109,12 @@ class CountryIndex
     @countries_by_name[ name ]
   end
 
-  def []( key )
+  def find( key )
     country = find_by_code( key )
     country = find_by_name( key )  if country.nil?     ## try lookup / find by (normalized) name
     country
   end
-  alias_method :find, :[]
+  alias_method :[], :find
 
 
   ###

data/lib/sportdb/formats/event/event_index.rb

@@ -0,0 +1,141 @@
+module SportDb
+module Import
+
+
+
+class EventIndex
+
+    def self.build( path )
+      pack = Package.new( path )   ## lets us use direcotry or zip archive
+
+      recs = []
+      pack.each_seasons do |entry|
+        recs += EventInfoReader.parse( entry.read )
+      end
+      recs
+
+      index = new
+      index.add( recs )
+      index
+    end
+
+
+
+    attr_reader :events
+    def initialize
+      @events  = []
+      @leagues = {}
+    end
+
+    def add( recs )
+      @events += recs  ## add to "linear" records
+
+      recs.each do |rec|
+        league = rec.league
+        season = rec.season
+
+        seasons = @leagues[ league.key ] ||= {}
+        seasons[season.key] = rec
+      end
+      ## build search index by leagues (and season)
+    end
+
+    def find_by( league:, season: )
+      league_key = league.is_a?( String ) ? league : league.key
+      season_key = season.is_a?( String ) ? season : season.key
+
+      seasons = @leagues[ league_key ]
+      if seasons
+        seasons[ season_key ]
+      else
+        nil
+      end
+    end # method find_by
+end  ## class EventIndex
+
+
+
+class SeasonIndex
+    def initialize( *args )
+       @leagues = {}   ## use a league hash by years for now; change later
+
+       if args.size == 1 && args[0].is_a?( EventIndex )
+         ## convenience setup/hookup
+         ##  (auto-)add all events from event index
+         add( args[0].events )
+       else
+         pp args
+         raise ArgumentError.new( 'unsupported arguments' )
+       end
+    end
+
+    def add( recs )
+      ## use a lookup index by year for now
+      ##  todo - find something better/more generic for searching/matching date periods!!!
+      recs.each do |rec|
+        league = rec.league
+        season = rec.season
+
+        years = @leagues[ league.key ] ||= {}
+        if season.year?
+          years[season.start_year] ||= []
+          years[season.start_year] << rec
+        else
+          years[season.start_year] ||= []
+          years[season.end_year]   ||= []
+          years[season.start_year] << rec
+          years[season.end_year]   << rec
+        end
+      end
+    end # method add
+
+    def find_by( date:, league: )
+       date = Date.strptime( date, '%Y-%m-%d' )   if date.is_a?( String )
+       league_key = league.is_a?( String ) ? league : league.key
+
+       years = @leagues[ league_key ]
+       if years
+           year = years[ date.year ]
+           if year
+               season_key = nil
+               year.each do |event|
+                 ##  todo/check: rename/use between? instead of include? - why? why not?
+                 if event.include?( date )
+                   season_key = event.season.key
+                   break
+                 end
+               end
+               if season_key.nil?
+                 puts "!! WARN: date >#{date}< out-of-seasons for year #{date.year} in league #{league_key}:"
+                 year.each do |event|
+                   puts "  #{event.season.key} |  #{event.start_date} - #{event.end_date}"
+                 end
+                 ## retry again and pick season with "overflow" at the end (date is great end_date)
+                 year.each do |event|
+                   if date > event.end_date
+                     diff_in_days = date.to_date.jd - event.end_date.to_date.jd
+                     puts "    +#{diff_in_days} days - adding overflow to #{event.season.key} ending on #{event.end_date} ++ #{date}"
+                     season_key = event.season.key
+                     break
+                   end
+                 end
+                 ## exit now for sure - if still empty!!!!
+                 if season_key.nil?
+                   puts "!! ERROR: CANNOT auto-fix / (auto-)append date at the end of an event; check season setup - sorry"
+                   exit 1
+                 end
+               end
+               season_key
+           else
+             nil   ## no year defined / found for league
+           end
+       else
+          nil   ## no league defined / found
+       end
+   end  # method find
+
+end # class SeasonIndex
+
+
+end   # module Import
+end   # module SportDb

data/lib/sportdb/formats/event/event_reader.rb

@@ -0,0 +1,183 @@
+
+module SportDb
+module Import
+
+
+class EventInfo
+    ##  "high level" info (summary) about event  (like a "wikipedia infobox")
+    ##    use for checking dataset imports; lets you check e.g.
+    ##    - dates within range
+    ##    - number of teams e.g. 20
+    ##    - matches played e.g. 380
+    ##    - goals scored e.g. 937
+    ##    etc.
+
+    attr_reader :league,
+                :season,
+                :teams,
+                :matches,
+                :goals,
+                :start_date,
+                :end_date
+
+    def initialize( league:, season:,
+                    start_date: nil, end_date: nil,
+                    teams:   nil,
+                    matches: nil,
+                    goals:   nil )
+
+      @league      = league
+      @season      = season
+
+      @start_date  = start_date
+      @end_date    = end_date
+
+      @teams       = teams    ## todo/check: rename/use teams_count ??
+      @matches     = matches  ## todo/check: rename/use match_count ??
+      @goals       = goals
+    end
+
+    def include?( date )
+       ## todo/fix: add options e.g.
+       ##  - add delta/off_by_one or such?
+       ##  - add strict (for) only return true if date range (really) defined (no generic auto-rules)
+
+      ### note: for now allow off by one error (via timezone/local time errors)
+      ##    todo/fix: issue warning if off by one!!!!
+      if @start_date && @end_date
+        date >= (@start_date-1) &&
+        date <= (@end_date+1)
+      else
+        if @season.year?
+           # assume generic rule
+           ## same year e.g. Jan 1 - Dec 31; always true for now
+           date.year == @season.start_year
+        else
+           # assume generic rule
+           ##  July 1 - June 30 (Y+1)
+           ##  - todo/check -start for some countries/leagues in June 1 or August 1 ????
+           date >= Date.new( @season.start_year, 7, 1 ) &&
+           date <= Date.new( @season.end_year, 6, 30 )
+        end
+      end
+    end  # method include?
+    alias_method :between?, :include?
+  end # class EventInfo
+
+
+  class EventInfoReader
+    def catalog() Import.catalog; end
+
+
+    def self.read( path )
+      txt = File.open( path, 'r:utf-8') {|f| f.read }
+      new( txt ).parse
+    end
+
+    def self.parse( txt )
+      new( txt ).parse
+    end
+
+    def initialize( txt )
+      @txt = txt
+    end
+
+    def parse
+      recs = []
+
+      parse_csv( @txt ).each do |row|
+        league_col = row['League']
+        season_col = row['Season'] || row['Year']
+        dates_col  = row['Dates']
+
+        season = Import::Season.parse( season_col )
+        league = catalog.leagues.find!( league_col )
+
+
+        dates = []
+        if dates_col.nil? || dates_col.empty?
+           ## do nothing; no dates - keep dates array empty
+        else
+          ## squish spaces
+          dates_col = dates_col.gsub( /[ ]{2,}/, ' ' )  ## squish/fold spaces
+
+          puts "#{league.name} (#{league.key}) | #{season.key} | #{dates_col}"
+
+          ### todo/check:  check what parts "Aug 15" return ???
+          ###                       short form for "Aug 15 -" - works?
+
+          ## todo/fix!!! - check EventInfo.include?
+          ##    now allow dates with only start_date too!! (WITHOUT end_date)
+          parts = dates_col.split( /[ ]*[–-][ ]*/ )
+          if parts.size == 1
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+          elsif parts.size == 2
+            pp parts
+            dates << DateFormats.parse( parts[0], start: Date.new( season.start_year, 1, 1 ), lang: 'en' )
+            dates << DateFormats.parse( parts[1], start: Date.new( season.end_year ? season.end_year : season.start_year, 1, 1 ), lang: 'en' )
+            pp dates
+
+            ## assert/check if period is less than 365 days for now
+            diff = dates[1].to_date.jd - dates[0].to_date.jd
+            puts "#{diff}d"
+            if diff > 365
+              puts "!! ERROR - date range / period assertion failed; expected diff < 365 days"
+              exit 1
+            end
+          else
+            puts "!! ERRROR - expected data range / period - one or two dates; got #{parts.size}:"
+            pp dates_col
+            pp parts
+            exit 1
+          end
+        end
+
+
+        teams_col    = row['Clubs'] || row['Teams']
+        goals_col    = row['Goals']
+
+        ## note: remove (and allow) all non-digits e.g. 370 goals, 20 clubs, etc.
+        teams_col    = teams_col.gsub( /[^0-9]/, '' )    if teams_col
+        goals_col    = goals_col.gsub( /[^0-9]/, '' )    if goals_col
+
+        teams  =   (teams_col.nil? || teams_col.empty?) ? nil : teams_col.to_i
+        goals  =   (goals_col.nil? || goals_col.empty?) ? nil : goals_col.to_i
+
+        matches_col  = row['Matches']
+        ## note: support additions in matches (played) e.g.
+        #   132 + 63 Play-off-Spiele
+        matches_col   = matches_col.gsub( /[^0-9+]/, '' )  if matches_col
+
+        matches =  if matches_col.nil? || matches_col.empty?
+                     nil
+                   else
+                     if matches_col.index( '+' )  ### check for calculations
+                        ## note: for now only supports additions
+                        matches_col.split( '+' ).reduce( 0 ) do |sum,str|
+                          sum + str.to_i
+                        end
+                     else            ## assume single (integer) number
+                        matches_col.to_i
+                     end
+                  end
+
+        rec = EventInfo.new( league:     league,
+                             season:     season,
+                             start_date: dates[0],
+                             end_date:   dates[1],
+                             teams:      teams,
+                             matches:    matches,
+                             goals:      goals
+                           )
+        recs << rec
+      end  # each row
+      recs
+    end  # method parse
+  end  # class EventInfoReader
+
+
+end ## module Import
+end ## module SportDb
+