footballdata-api 0.2.0 → 0.3.1

data/lib/footballdata/convert.rb

@@ -1,32 +1,81 @@
 
+module Footballdata
 
 
-module Footballdata
+def self.convert_score( score )
+  ## duration: REGULAR  · PENALTY_SHOOTOUT  · EXTRA_TIME
+  ft, ht, et, pen = ["","","",""]
+
+  if score['duration'] == 'REGULAR'
+    ft  = "#{score['fullTime']['home']}-#{score['fullTime']['away']}"
+    ht  = "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+  elsif score['duration'] == 'EXTRA_TIME'
+    et  =  "#{score['regularTime']['home']+score['extraTime']['home']}"
+    et << "-"
+    et << "#{score['regularTime']['away']+score['extraTime']['away']}"
+
+    ft =  "#{score['regularTime']['home']}-#{score['regularTime']['away']}"
+    ht =  "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+  elsif score['duration'] == 'PENALTY_SHOOTOUT'
+    if score['extraTime']
+      ## quick & dirty hack - calc et via regulartime+extratime
+      pen = "#{score['penalties']['home']}-#{score['penalties']['away']}"
+      et  = "#{score['regularTime']['home']+score['extraTime']['home']}"
+      et << "-"
+      et << "#{score['regularTime']['away']+score['extraTime']['away']}"
+
+      ft = "#{score['regularTime']['home']}-#{score['regularTime']['away']}"
+      ht = "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+    else  ### south american-style (no extra time)
+        ## quick & dirty hacke - calc ft via fullTime-penalties
+        pen =  "#{score['penalties']['home']}-#{score['penalties']['away']}"
+        ft  =  "#{score['fullTime']['home']-score['penalties']['home']}"
+        ft << "-"
+        ft << "#{score['fullTime']['away']-score['penalties']['away']}"
+        ht  = "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+    end
+  else
+    puts "!! unknown score duration:"
+    pp score
+    exit 1
+  end
+
+  [ft,ht,et,pen]
+end
+
 
+#######
+##  map round-like to higher-level stages
+STAGES = {
+  'REGULAR_SEASON'          => ['Regular'],
 
-  TIMEZONES = {
-    'eng.1' => 'Europe/London',  
-    'eng.2' => 'Europe/London',
-  
-    'es.1'  => 'Europe/Madrid',
-  
-    'de.1'  => 'Europe/Berlin',
-    'fr.1'  => 'Europe/Paris', 
-    'it.1'  => 'Europe/Rome',
-    'nl.1'  => 'Europe/Amsterdam',
-  
-    'pt.1'  => 'Europe/Lisbon',   
+  'PRELIMINARY_ROUND'       => ['Qualifying', 'Preliminary Round' ],
+  'PRELIMINARY_SEMI_FINALS' => ['Qualifying', 'Preliminary Semifinals' ],
+  'PRELIMINARY_FINAL'       => ['Qualifying', 'Preliminary Final' ],
+  '1ST_QUALIFYING_ROUND'    => ['Qualifying', 'Qual. Round 1' ],
+  '2ND_QUALIFYING_ROUND'    => ['Qualifying', 'Qual. Round 2' ],
+  '3RD_QUALIFYING_ROUND'    => ['Qualifying', 'Qual. Round 3' ],
+  'QUALIFICATION_ROUND_1'   => ['Qualifying', 'Qual. Round 1' ],
+  'QUALIFICATION_ROUND_2'   => ['Qualifying', 'Qual. Round 2' ],
+  'QUALIFICATION_ROUND_3'   => ['Qualifying', 'Qual. Round 3' ],
+  'ROUND_1'                 => ['Qualifying', 'Round 1'],    ##  use Qual. Round 1 - why? why not?
+  'ROUND_2'                 => ['Qualifying', 'Round 2'],
+  'ROUND_3'                 => ['Qualifying', 'Round 3'],
+  'PLAY_OFF_ROUND'          => ['Qualifying', 'Playoff Round'],
+  'PLAYOFF_ROUND_1'         => ['Qualifying', 'Playoff Round 1'],
+
+  'LEAGUE_STAGE'            => ['League'],
+  'GROUP_STAGE'             => ['Group'],
+  'PLAYOFFS'                => ['Playoffs'],
+
+  'ROUND_OF_16'             => ['Finals',     'Round of 16'],
+  'LAST_16'                 => ['Finals',     'Round of 16'],  ## use Last 16 - why? why not?
+  'QUARTER_FINALS'          => ['Finals',     'Quarterfinals'],
+  'SEMI_FINALS'             => ['Finals',     'Semifinals'],
+  'FINAL'                   => ['Finals',     'Final'],
+}
 
-    ## todo/fix - pt.1
-    ##  one team in madeira!!! check for different timezone??
-    ##  CD Nacional da Madeira 
 
-    'br.1'  => 'America/Sao_Paulo',
-    ## todo/fix - brazil has 4 timezones
-    ##           really only two in use for clubs
-    ##             west and east (amazonas et al)
-    ##           for now use west for all - why? why not?
-  }
 
 
 def self.convert( league:, season: )
@@ -40,7 +89,7 @@ def self.convert( league:, season: )
 
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league_code = LEAGUES[league.downcase]
+  league_code = find_league!( league )
 
   matches_url = Metal.competition_matches_url( league_code, season.start_year )
   teams_url   = Metal.competition_teams_url(   league_code, season.start_year )
@@ -48,15 +97,11 @@ def self.convert( league:, season: )
   data           = Webcache.read_json( matches_url )
   data_teams     = Webcache.read_json( teams_url )
 
-  
+
+
   ## check for time zone
-  tz_name = TIMEZONES[ league.downcase ]
-  if tz_name.nil?
-    puts "!! ERROR - sorry no timezone configured for league #{league}"
-    exit 1
-  end
-  
-  tz  = TZInfo::Timezone.get( tz_name )
+  tz  = find_zone!( league: league,
+                    season: season )
   pp tz
 
   ## build a (reverse) team lookup by name
@@ -67,9 +112,9 @@ def self.convert( league:, season: )
      h
   end
 
-  pp teams_by_name.keys
+  ## pp teams_by_name.keys
+
 
-  
 
 mods = MODS[ league.downcase ] || {}
 
@@ -81,22 +126,62 @@ teams = Hash.new( 0 )
 
 # stat  =  Stat.new
 
-#  track stati counts 
-stati = Hash.new(0)
+  ## track stage, match status et
+  stats = { 'status'    => Hash.new(0),
+            'stage'     => Hash.new(0),
+           }
+
+
 
 
 matches = data[ 'matches']
 matches.each do |m|
   # stat.update( m )
 
-  team1 = m['homeTeam']['name']
-  team2 = m['awayTeam']['name']
+  ## use ? or N.N. or ? for nil  - why? why not?
+  team1 = m['homeTeam']['name'] || 'N.N.'
+  team2 = m['awayTeam']['name'] || 'N.N.'
 
   score = m['score']
 
 
+  stage_key = m['stage']
+
+  stats['stage'][ stage_key ] += 1   ## track stage counts
+
+  ## map stage to stage + round
+  stage, stage_round  =  STAGES[ stage_key ]
+
+  if stage.nil?
+      puts "!! ERROR - no stage mapping found for stage >#{stage_key}<"
+      exit 1
+  end
+
+  matchday_num = m['matchday']
+  matchday_num = nil   if matchday_num == 0   ## change 0 to nil (empty) too
+
+  if stage_round.nil?  ## e.g. Regular, League, Group, Playoffs
+     ## keep/assume matchday number is matchday .e.g
+     ##   matchday 1, 2 etc.
+     matchday = matchday_num.to_s
+  else
+    ## note - if matchday defined; assume leg e.g. 1|2
+    ##     skip if different than one or two for now
+    matchday = String.new
+    matchday << stage_round
+    matchday << " | Leg #{matchday_num}"  if matchday_num &&
+                                         (matchday_num == 1 || matchday_num == 2)
+  end
+
+
+
+  group = m['group'] || ''
+  ## GROUP_A
+  ##  shorten group to A|B|C etc.
+  group = group.sub( /^GROUP_/, '' )
+
+
 
-  if m['stage'] == 'REGULAR_SEASON'
     teams[ team1 ] += 1
     teams[ team2 ] += 1
 
@@ -107,31 +192,36 @@ matches.each do |m|
     end
 
 
+   ## auto-fix copa.l 2024
+   ##  !! ERROR: unsupported match status >IN_PLAY< - sorry:
+   if m['status'] == 'IN_PLAY' &&
+      team1 == 'Club Aurora' && team2 == 'FBC Melgar'
+        m['status'] = 'FINISHED'
+   end
+
 
     comments = ''
     ft       = ''
     ht       = ''
+    et       = ''
+    pen      = ''
 
-    stati[m['status']] += 1  ## track stati counts for logs
+    stats['status'][m['status']]  += 1  ## track status counts
 
     case m['status']
     when 'SCHEDULED', 'TIMED'   ## , 'IN_PLAY'
       ft = ''
       ht = ''
     when 'FINISHED'
-      ## todo/fix: assert duration == "REGULAR"
-      assert( score['duration'] == 'REGULAR', 'score.duration REGULAR expected' ) 
-      ft = "#{score['fullTime']['home']}-#{score['fullTime']['away']}"
-      ht = "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+      ft, ht, et, pen = convert_score( score )
     when 'AWARDED'
-      ## todo/fix: assert duration == "REGULAR"
-      assert( score['duration'] == 'REGULAR', 'score.duration REGULAR expected' ) 
+      assert( score['duration'] == 'REGULAR', 'score.duration REGULAR expected' )
       ft = "#{score['fullTime']['home']}-#{score['fullTime']['away']}"
       ft << ' (*)'
       ht = ''
       comments = 'awarded'
     when 'CANCELLED'
-      ## note cancelled might have scores!!
+      ## note cancelled might have scores!! -- add/fix later!!!
       ##   ht only or ft+ht!!!  (see fr 2021/22)
       ft = '(*)'
       ht = ''
@@ -157,55 +247,49 @@ matches.each do |m|
     ## utc   = ## tz_utc.strptime( m['utcDate'], '%Y-%m-%dT%H:%M:%SZ' )
     ##  note:  DateTime.strptime  is supposed to be unaware of timezones!!!
     ##            use to parse utc
-    utc = DateTime.strptime( m['utcDate'], '%Y-%m-%dT%H:%M:%SZ' ).to_time.utc 
+    utc = UTC.strptime( m['utcDate'], '%Y-%m-%dT%H:%M:%SZ' )
     assert( utc.strftime( '%Y-%m-%dT%H:%M:%SZ' ) == m['utcDate'], 'utc time mismatch' )
-    
-    local = tz.to_local( utc )
-   
+
+
+    ## assume NOT valid utc time if 00:00
+    ##   do
+    if utc.hour == 0 && utc.min == 0 &&
+       ['SCHEDULED','POSTPONED'].include?( m['status'] )
+       date     = utc.strftime( '%Y-%m-%d' )
+       time     = ''
+       timezone = ''
+    else
+      local    = tz.to_local( utc )
+      date     = local.strftime( '%Y-%m-%d' )
+      time     = local.strftime( '%H:%M' )
+      timezone = local.strftime( '%Z/%z' )
+    end
+
 
     ## do NOT add time if status is SCHEDULED
     ##                        or POSTPONED for now
     ##   otherwise assume time always present - why? why not?
-  
+
+
 
     ## todo/fix: assert matchday is a number e.g. 1,2,3, etc.!!!
-    recs << [m['matchday'].to_s,   ## note: convert integer to string!!!
-             local.strftime( '%Y-%m-%d' ),
-             ['SCHEDULED','POSTPONED'].include?( m['status'] ) ? '' : local.strftime( '%H:%M' ),
-             local.strftime( '%Z / UTC%z' ), 
+    recs << [stage,
+             group,
+             matchday,
+             date,
+             time,
+             timezone,
              team1,
              ft,
              ht,
              team2,
+             et,
+             pen,
              comments,
              ## add more columns e.g. utc date, status
              m['status'],  # e.g. FINISHED, TIMED, etc.
              m['utcDate'],
             ]
-
-
-    print '%2s' % m['matchday']
-    print ' - '
-    print '%-26s' % team1
-    print '  '
-    print ft
-    print ' '
-    print "(#{ht})"    unless ht.empty?
-    print '  '
-    print '%-26s' % team2
-    print '  '
-    print comments
-    print ' | '
-    ## print date.to_date  ## strip time
-    print utc.strftime( '%a %b %-d %Y' )
-    print ' -- '
-    print utc
-    print "\n"
-  else
-    puts "!!! unexpected stage:"
-    puts "-- skipping #{m['stage']}"
-    # exit 1
-  end
 end # each match
 
 
@@ -227,8 +311,8 @@ dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
 
 buf = ''
 buf << "#{season.key} (#{dates}) - "
-buf << "#{teams.keys.size} clubs, "
-# buf << "#{stat[:regular_season][:matches]} matches, "
+buf << "#{teams.keys.size} teams, "
+buf << "#{recs.size} matches"
 # buf << "#{stat[:regular_season][:goals]} goals"
 buf << "\n"
 
@@ -248,7 +332,7 @@ puts buf
 
    File.open( './logs.txt', 'a:utf-8' ) do |f|
      f.write "====  #{league} #{season.key}  =============\n"
-     f.write "  match stati: #{stati.inspect}\n"
+     f.write "  #{stats.inspect}\n"
    end
 
 =begin
@@ -283,26 +367,30 @@ puts buf
 
 
 ## reformat date / beautify e.g. Sat Aug 7 1993
-recs = recs.map do |rec| 
-           rec[1] = Date.strptime( rec[1], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) 
+recs = recs.map do |rec|
+           rec[3] = Date.strptime( rec[3], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
            rec
        end
 
+## pp recs
+
+## check if all status colums
+###  are FINISHED
+###   if yes, set all to empty (for vacuum)
+
+if stats['status'].keys.size == 1 && stats['status'].keys[0] == 'FINISHED'
+  recs = recs.map { |rec| rec[-2] = ''; rec }
+end
+
+if stats['stage'].keys.size == 1 && stats['stage'].keys[0] == 'REGULAR_SEASON'
+  recs = recs.map { |rec| rec[0] = ''; rec }
+end
+
+
+recs, headers = vacuum( recs )
+
+
 
-headers = [
-  'Matchday',
-  'Date',
-  'Time',
-  'Timezone',  ## move back column - why? why not?
-  'Team 1',
-  'FT',
-  'HT',
-  'Team 2',
-  'Comments',
-  ##
-  'Status', # e.g. 
-  'UTC',    # date utc
-]
 
 ## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
   write_csv( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
@@ -319,14 +407,89 @@ teams.each do |name, count|
     print " › #{rec['area']['name']}"
     print "  - #{rec['address']}"
   else
-    puts "!! ERROR  - no team record found in teams.json for #{name}"
-    exit 1
+    if name == 'N.N.'
+      ## ignore missing record
+    else
+      puts "!! ERROR  - no team record found in teams.json for #{name}"
+      exit 1
+    end
   end
   print "\n"
 end
 
 ## pp stat
 end   # method convert
+
+
+
+MAX_HEADERS = [
+  'Stage',    # 0
+  'Group',    # 1
+  'Matchday', # 2
+  'Date',     # 3
+  'Time',     # 4
+  'Timezone',   # 5 ## move back column - why? why not?
+  'Team 1',     # 6
+  'FT',         # 7
+  'HT',         # 8
+  'Team 2',     # 9
+  'ET',         # 10     # extra: incl. extra time
+  'P',          # 11     # extra: incl. penalties
+  'Comments',   # 12
+  'Status',     # 13 / -2  # e.g.
+  'UTC',        # 14 / -1 # date utc
+]
+
+MIN_HEADERS = [   ## always keep even if all empty
+'Date',
+'Team 1',
+'FT',
+'Team 2'
+]
+
+
+
+def self.vacuum( rows, headers: MAX_HEADERS, fixed_headers: MIN_HEADERS )
+  ## check for unused columns and strip/remove
+  counter = Array.new( MAX_HEADERS.size, 0 )
+  rows.each do |row|
+     row.each_with_index do |col, idx|
+       counter[idx] += 1  unless col.nil? || col.empty?
+     end
+  end
+
+  ## pp counter
+
+  ## check empty columns
+  headers       = []
+  indices       = []
+  empty_headers = []
+  empty_indices = []
+
+  counter.each_with_index do |num, idx|
+     header = MAX_HEADERS[ idx ]
+     if num > 0 || (num == 0 && fixed_headers.include?( header ))
+       headers << header
+       indices << idx
+     else
+       empty_headers << header
+       empty_indices << idx
+     end
+  end
+
+  if empty_indices.size > 0
+    rows = rows.map do |row|
+             row_vacuumed = []
+             row.each_with_index do |col, idx|
+               ## todo/fix: use values or such??
+               row_vacuumed << col   unless empty_indices.include?( idx )
+             end
+             row_vacuumed
+         end
+    end
+
+  [rows, headers]
+end
 end #  module Footballdata
 
 

data/lib/footballdata/download.rb

@@ -6,7 +6,7 @@ module Footballdata
 def self.schedule( league:, season: )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league_code = LEAGUES[ league.downcase ]
+  league_code = find_league!( league )
   puts "  mapping league >#{league}< to >#{league_code}<"
 
   Metal.teams(   league_code, season.start_year )
@@ -17,7 +17,7 @@ end
 def self.matches( league:, season: )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league_code = LEAGUES[ league.downcase ]
+  league_code = find_league!( league )
   puts "  mapping league >#{league}< to >#{league_code}<"
   Metal.matches( league_code, season.start_year )
 end
@@ -26,7 +26,7 @@ end
 def self.teams( league:, season: )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league_code = LEAGUES[ league.downcase ]
+  league_code = find_league!( league )
   puts "  mapping league >#{league}< to >#{league_code}<"
   Metal.teams(   league_code, season.start_year )
 end
@@ -38,7 +38,7 @@ end
 
 class Metal
 
-  def self.get( url, 
+  def self.get( url,
                   auth:    true,
                   headers: {} )
 
@@ -84,19 +84,19 @@ class Metal
   def self.competition_teams_url( code, year )     "#{BASE_URL}/competitions/#{code}/teams?season=#{year}";     end
   def self.competition_standings_url( code, year ) "#{BASE_URL}/competitions/#{code}/standings?season=#{year}"; end
   def self.competition_scorers_url( code, year )   "#{BASE_URL}/competitions/#{code}/scorers?season=#{year}"; end
-    
+
   def self.matches( code, year,
-                    headers: {} )  
-      get( competition_matches_url( code, year ), 
-           headers: headers ) 
+                    headers: {} )
+      get( competition_matches_url( code, year ),
+           headers: headers )
   end
 
-  def self.todays_matches_url( date=Date.today )   
+  def self.todays_matches_url( date=Date.today )
     "#{BASE_URL}/matches?"+
     "dateFrom=#{date.strftime('%Y-%m-%d')}&"+
-    "dateTo=#{(date+1).strftime('%Y-%m-%d')}"   
+    "dateTo=#{(date+1).strftime('%Y-%m-%d')}"
   end
-  def self.todays_matches( date=Date.today )    ## use/rename to matches_today or such - why? why not? 
+  def self.todays_matches( date=Date.today )    ## use/rename to matches_today or such - why? why not?
       get( todays_matches_url( date ) )
   end
 

data/lib/footballdata/leagues.rb

@@ -1,59 +1,24 @@
 module Footballdata
 
-  LEAGUES = {
-    'eng.1' => 'PL',     # incl. team(s) from wales
-    'eng.2' => 'ELC',
-     # PL  - Premier League        , England        27 seasons | 2019-08-09 - 2020-07-25 / matchday 31
-     #  ELC - Championship          , England         3 seasons | 2019-08-02 - 2020-07-22 / matchday 38
-     #
-     # 2019 => 2019/20
-     # 2018 => 2018/19
-     # 2017 => xxx 2017-18 - requires subscription !!!
-
-    'es.1'  => 'PD',
-    # PD  - Primera Division      , Spain          27 seasons | 2019-08-16 - 2020-07-19 / matchday 31
-
-    'pt.1'  => 'PPL',
-    # PPL - Primeira Liga         , Portugal        9 seasons | 2019-08-10 - 2020-07-26 / matchday 28
-
-    'de.1'  => 'BL1',
-    # BL1 - Bundesliga            , Germany        24 seasons | 2019-08-16 - 2020-06-27 / matchday 34
-
-    'nl.1'  => 'DED',
-    # DED - Eredivisie            , Netherlands    10 seasons | 2019-08-09 - 2020-03-08 / matchday 34
-
-    'fr.1'  => 'FL1',    # incl. team(s) monaco
-    # FL1 - Ligue 1, France
-    #   9 seasons | 2019-08-09 - 2020-05-31 / matchday 38
-    #
-    # 2019 => 2019/20
-    # 2018 => 2018/19
-    # 2017 => xxx 2017-18 - requires subscription !!!
-
-    'it.1'  => 'SA',
-    # SA  - Serie A               , Italy          15 seasons | 2019-08-24 - 2020-08-02 / matchday 27
-
-    'br.1'  => 'BSA',
-    # BSA - Série A, Brazil
-    #   4 seasons | 2020-05-03 - 2020-12-06 / matchday 10
-    #
-    #  2020 => 2020
-    #  2019 => 2019
-    #  2018 => 2018
-    #  2017 => xxx 2017 - requires subscription !!!
-
-    ## todo/check: use champs and NOT cl - why? why not?
-    'uefa.cl'        => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
-    ##  was europe.cl / cl  
-    
-    ## Copa Libertadores
-    'copa.l'  => 'CLI',
- 
-    ############
-    ## national teams
-    'euro'  => 'EC',
-    'world' => 'WC',
-
-  }
+  def self.find_league!( league )
+     @leagues ||= begin
+                     recs = read_csv( "#{FootballdataApi.root}/config/leagues.csv" )
+                     leagues = {}
+                     recs.each do |rec|
+                        leagues[ rec['key'] ] = rec['code']
+                     end
+                     leagues
+                  end
+
+     key = league.downcase
+     code =  @leagues[ key ]
+     if code.nil?
+        puts "!! ERROR - no code/mapping found for league >#{league}<"
+        puts "     mappings include:"
+        pp @leagues
+        exit 1
+     end
+     code
+  end
 end   #  module Footballdata
 

data/lib/footballdata/mods.rb

@@ -6,6 +6,8 @@ module Footballdata
   # Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
   # AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
 
+
+
   MODS = {
     'br.1' => {
        'América FC' => 'América Mineiro',  # in year 2018 ??