footballdata-api 0.2.0

data/lib/footballdata/convert.rb

@@ -0,0 +1,332 @@
+
+
+
+module Footballdata
+
+
+  TIMEZONES = {
+    'eng.1' => 'Europe/London',  
+    'eng.2' => 'Europe/London',
+  
+    'es.1'  => 'Europe/Madrid',
+  
+    'de.1'  => 'Europe/Berlin',
+    'fr.1'  => 'Europe/Paris', 
+    'it.1'  => 'Europe/Rome',
+    'nl.1'  => 'Europe/Amsterdam',
+  
+    'pt.1'  => 'Europe/Lisbon',   
+
+    ## todo/fix - pt.1
+    ##  one team in madeira!!! check for different timezone??
+    ##  CD Nacional da Madeira 
+
+    'br.1'  => 'America/Sao_Paulo',
+    ## todo/fix - brazil has 4 timezones
+    ##           really only two in use for clubs
+    ##             west and east (amazonas et al)
+    ##           for now use west for all - why? why not?
+  }
+
+
+def self.convert( league:, season: )
+
+  ### note/fix: cl (champions league for now is a "special" case)
+  # if league.downcase == 'cl'
+  #   convert_cl( league: league,
+  #              season: season )
+  #  return
+  # end
+
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league_code = LEAGUES[league.downcase]
+
+  matches_url = Metal.competition_matches_url( league_code, season.start_year )
+  teams_url   = Metal.competition_teams_url(   league_code, season.start_year )
+
+  data           = Webcache.read_json( matches_url )
+  data_teams     = Webcache.read_json( teams_url )
+
+  
+  ## check for time zone
+  tz_name = TIMEZONES[ league.downcase ]
+  if tz_name.nil?
+    puts "!! ERROR - sorry no timezone configured for league #{league}"
+    exit 1
+  end
+  
+  tz  = TZInfo::Timezone.get( tz_name )
+  pp tz
+
+  ## build a (reverse) team lookup by name
+  puts "#{data_teams['teams'].size} teams"
+
+  teams_by_name = data_teams['teams'].reduce( {} ) do |h,rec|
+     h[ rec['name'] ] = rec
+     h
+  end
+
+  pp teams_by_name.keys
+
+  
+
+mods = MODS[ league.downcase ] || {}
+
+
+recs = []
+
+teams = Hash.new( 0 )
+
+
+# stat  =  Stat.new
+
+#  track stati counts 
+stati = Hash.new(0)
+
+
+matches = data[ 'matches']
+matches.each do |m|
+  # stat.update( m )
+
+  team1 = m['homeTeam']['name']
+  team2 = m['awayTeam']['name']
+
+  score = m['score']
+
+
+
+  if m['stage'] == 'REGULAR_SEASON'
+    teams[ team1 ] += 1
+    teams[ team2 ] += 1
+
+    ### mods - rename club names
+    unless mods.nil? || mods.empty?
+      team1 = mods[ team1 ]      if mods[ team1 ]
+      team2 = mods[ team2 ]      if mods[ team2 ]
+    end
+
+
+
+    comments = ''
+    ft       = ''
+    ht       = ''
+
+    stati[m['status']] += 1  ## track stati counts for logs
+
+    case m['status']
+    when 'SCHEDULED', 'TIMED'   ## , 'IN_PLAY'
+      ft = ''
+      ht = ''
+    when 'FINISHED'
+      ## todo/fix: assert duration == "REGULAR"
+      assert( score['duration'] == 'REGULAR', 'score.duration REGULAR expected' ) 
+      ft = "#{score['fullTime']['home']}-#{score['fullTime']['away']}"
+      ht = "#{score['halfTime']['home']}-#{score['halfTime']['away']}"
+    when 'AWARDED'
+      ## todo/fix: assert duration == "REGULAR"
+      assert( score['duration'] == 'REGULAR', 'score.duration REGULAR expected' ) 
+      ft = "#{score['fullTime']['home']}-#{score['fullTime']['away']}"
+      ft << ' (*)'
+      ht = ''
+      comments = 'awarded'
+    when 'CANCELLED'
+      ## note cancelled might have scores!!
+      ##   ht only or ft+ht!!!  (see fr 2021/22)
+      ft = '(*)'
+      ht = ''
+      comments  = 'canceled'   ## us eng ? -> canceled, british eng. cancelled ?
+    when 'POSTPONED'
+      ft = '(*)'
+      ht = ''
+      comments = 'postponed'
+    else
+      puts "!! ERROR: unsupported match status >#{m['status']}< - sorry:"
+      pp m
+      exit 1
+    end
+
+
+    ##
+    ##  add time, timezone(tz)
+    ##    2023-08-18T18:30:00Z
+    ## e.g. "utcDate": "2020-05-09T00:00:00Z",
+    ##      "utcDate": "2023-08-18T18:30:00Z",
+
+    ## -- todo - make sure / assert it's always utc - how???
+    ## utc   = ## tz_utc.strptime( m['utcDate'], '%Y-%m-%dT%H:%M:%SZ' )
+    ##  note:  DateTime.strptime  is supposed to be unaware of timezones!!!
+    ##            use to parse utc
+    utc = DateTime.strptime( m['utcDate'], '%Y-%m-%dT%H:%M:%SZ' ).to_time.utc 
+    assert( utc.strftime( '%Y-%m-%dT%H:%M:%SZ' ) == m['utcDate'], 'utc time mismatch' )
+    
+    local = tz.to_local( utc )
+   
+
+    ## do NOT add time if status is SCHEDULED
+    ##                        or POSTPONED for now
+    ##   otherwise assume time always present - why? why not?
+  
+
+    ## todo/fix: assert matchday is a number e.g. 1,2,3, etc.!!!
+    recs << [m['matchday'].to_s,   ## note: convert integer to string!!!
+             local.strftime( '%Y-%m-%d' ),
+             ['SCHEDULED','POSTPONED'].include?( m['status'] ) ? '' : local.strftime( '%H:%M' ),
+             local.strftime( '%Z / UTC%z' ), 
+             team1,
+             ft,
+             ht,
+             team2,
+             comments,
+             ## add more columns e.g. utc date, status
+             m['status'],  # e.g. FINISHED, TIMED, etc.
+             m['utcDate'],
+            ]
+
+
+    print '%2s' % m['matchday']
+    print ' - '
+    print '%-26s' % team1
+    print '  '
+    print ft
+    print ' '
+    print "(#{ht})"    unless ht.empty?
+    print '  '
+    print '%-26s' % team2
+    print '  '
+    print comments
+    print ' | '
+    ## print date.to_date  ## strip time
+    print utc.strftime( '%a %b %-d %Y' )
+    print ' -- '
+    print utc
+    print "\n"
+  else
+    puts "!!! unexpected stage:"
+    puts "-- skipping #{m['stage']}"
+    # exit 1
+  end
+end # each match
+
+
+
+## note: get season from first match
+##   assert - all other matches include the same season
+## e.g.
+# "season": {
+#  "id": 154,
+#  "startDate": "2018-08-03",
+#  "endDate": "2019-05-05",
+#  "currentMatchday": 46
+# }
+
+start_date = Date.strptime( matches[0]['season']['startDate'], '%Y-%m-%d' )
+end_date   = Date.strptime( matches[0]['season']['endDate'],   '%Y-%m-%d' )
+
+dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
+
+buf = ''
+buf << "#{season.key} (#{dates}) - "
+buf << "#{teams.keys.size} clubs, "
+# buf << "#{stat[:regular_season][:matches]} matches, "
+# buf << "#{stat[:regular_season][:goals]} goals"
+buf << "\n"
+
+puts buf
+
+
+=begin
+   ## note: warn if stage is greater one and not regular season!!
+   File.open( './errors.txt' , 'a:utf-8' ) do |f|
+     if stat[:all][:stage].keys != ['REGULAR_SEASON']
+      f.write "!! WARN - league: #{league}, season: #{season.key} includes non-regular stage(s):\n"
+      f.write "   #{stat[:all][:stage].keys.inspect}\n"
+     end
+   end
+=end
+
+
+   File.open( './logs.txt', 'a:utf-8' ) do |f|
+     f.write "====  #{league} #{season.key}  =============\n"
+     f.write "  match stati: #{stati.inspect}\n"
+   end
+
+=begin
+    f.write "\n================================\n"
+     f.write "====  #{league}  =============\n"
+     f.write buf
+     f.write "  match status: #{stat[:regular_season][:status].inspect}\n"
+     f.write "  match duration: #{stat[:regular_season][:duration].inspect}\n"
+
+     f.write "#{teams.keys.size} teams:\n"
+     teams.each do |name, count|
+        rec = teams_by_name[ name ]
+        f.write "  #{count}x  #{name}"
+        if rec
+          f.write " | #{rec['shortName']} "   if name != rec['shortName']
+          f.write " › #{rec['area']['name']}"
+          f.write "  - #{rec['address']}"
+        else
+          puts "!! ERROR - no team record found in teams.json for >#{name}<"
+          exit 1
+        end
+        f.write "\n"
+     end
+   end
+=end
+
+
+##
+##  sort buy utc date ??? - why? why not?
+
+# recs = recs.sort { |l,r| l[1] <=> r[1] }
+
+
+## reformat date / beautify e.g. Sat Aug 7 1993
+recs = recs.map do |rec| 
+           rec[1] = Date.strptime( rec[1], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) 
+           rec
+       end
+
+
+headers = [
+  'Matchday',
+  'Date',
+  'Time',
+  'Timezone',  ## move back column - why? why not?
+  'Team 1',
+  'FT',
+  'HT',
+  'Team 2',
+  'Comments',
+  ##
+  'Status', # e.g. 
+  'UTC',    # date utc
+]
+
+## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
+  write_csv( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
+                             recs,
+                             headers: headers )
+
+
+teams.each do |name, count|
+  rec = teams_by_name[ name ]
+  print "  #{count}x  "
+  print name
+  if rec
+    print " | #{rec['shortName']} "   if name != rec['shortName']
+    print " › #{rec['area']['name']}"
+    print "  - #{rec['address']}"
+  else
+    puts "!! ERROR  - no team record found in teams.json for #{name}"
+    exit 1
+  end
+  print "\n"
+end
+
+## pp stat
+end   # method convert
+end #  module Footballdata
+
+

data/lib/footballdata/download.rb

@@ -0,0 +1,131 @@
+module Footballdata
+
+
+#################
+##  porcelain "api"
+def self.schedule( league:, season: )
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league_code = LEAGUES[ league.downcase ]
+  puts "  mapping league >#{league}< to >#{league_code}<"
+
+  Metal.teams(   league_code, season.start_year )
+  Metal.matches( league_code, season.start_year )
+end
+
+
+def self.matches( league:, season: )
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league_code = LEAGUES[ league.downcase ]
+  puts "  mapping league >#{league}< to >#{league_code}<"
+  Metal.matches( league_code, season.start_year )
+end
+
+
+def self.teams( league:, season: )
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league_code = LEAGUES[ league.downcase ]
+  puts "  mapping league >#{league}< to >#{league_code}<"
+  Metal.teams(   league_code, season.start_year )
+end
+
+
+
+##################
+##  plumbing metal "helpers"
+
+class Metal
+
+  def self.get( url, 
+                  auth:    true,
+                  headers: {} )
+
+    token = ENV['FOOTBALLDATA']
+    ## note: because of public workflow log - do NOT output token
+    ## puts token
+
+    request_headers = {}
+    request_headers['X-Auth-Token'] = token    if auth && token
+    request_headers['User-Agent']   = 'ruby'
+    request_headers['Accept']       = '*/*'
+
+    request_headers = request_headers.merge( headers )   unless headers.empty?
+
+
+    ## note: add format: 'json' for pretty printing json (before) save in cache
+    response = Webget.call( url, headers: request_headers )
+
+    ## for debugging print pretty printed json first 400 chars
+    puts response.json.pretty_inspect[0..400]
+
+    exit 1  if response.status.nok?   # e.g. HTTP status code != 200
+
+    response.json
+  end
+
+
+
+  BASE_URL = 'http://api.football-data.org/v4'
+
+
+  def self.competitions_url
+    "#{BASE_URL}/competitions"
+  end
+
+  def self.competitions( auth: false )
+    get( competitions_url, auth: auth )
+  end
+
+
+  ## just use matches_url - why? why not?
+  def self.competition_matches_url( code, year )   "#{BASE_URL}/competitions/#{code}/matches?season=#{year}";   end
+  def self.competition_teams_url( code, year )     "#{BASE_URL}/competitions/#{code}/teams?season=#{year}";     end
+  def self.competition_standings_url( code, year ) "#{BASE_URL}/competitions/#{code}/standings?season=#{year}"; end
+  def self.competition_scorers_url( code, year )   "#{BASE_URL}/competitions/#{code}/scorers?season=#{year}"; end
+    
+  def self.matches( code, year,
+                    headers: {} )  
+      get( competition_matches_url( code, year ), 
+           headers: headers ) 
+  end
+
+  def self.todays_matches_url( date=Date.today )   
+    "#{BASE_URL}/matches?"+
+    "dateFrom=#{date.strftime('%Y-%m-%d')}&"+
+    "dateTo=#{(date+1).strftime('%Y-%m-%d')}"   
+  end
+  def self.todays_matches( date=Date.today )    ## use/rename to matches_today or such - why? why not? 
+      get( todays_matches_url( date ) )
+  end
+
+
+  def self.teams( code, year )    get( competition_teams_url( code, year )); end
+  def self.standings( code, year ) get( competition_standings_url( code, year )); end
+  def self.scorers( code, year ) get( competition_scorers_url( code, year )); end
+
+ ################
+ ## more
+ def self.competition( code )
+   get( "#{BASE_URL}/competitions/#{code}" )
+ end
+
+ def self.team( id )
+   get( "#{BASE_URL}/teams/#{id}" )
+ end
+
+ def self.match( id )
+   get( "#{BASE_URL}/matches/#{id}" )
+ end
+
+ def self.person( id )
+  get( "#{BASE_URL}/persons/#{id}" )
+ end
+
+ def self.areas
+     get( "#{BASE_URL}/areas" )
+  end
+end  # class Metal
+end # module Footballdata
+

data/lib/footballdata/generator.rb

@@ -0,0 +1,33 @@
+
+##
+### check - change Generator to Writer 
+##                   and write( league:, season: ) - why? why not?
+
+module Footballdata
+class Generator
+
+
+
+###########    
+## always download for now - why? why not?
+##      support cache - why? why not?
+def generate( league:, season: )
+
+  ## for testing use cached version always - why? why not?    
+  ## step 1 - download
+  Footballdata.schedule( league: league, season: season )
+
+  ## step 2 - convert (to .csv)
+
+  ## todo/fix - convert in-memory and return matches
+  Footballdata.convert( league: league, season: season )
+
+  source_dir = Footballdata.config.convert.out_dir
+
+  Writer.write( league: league, 
+                season: season,
+                source: source_dir )
+end  # def generate
+
+end # class Generator
+end # module Footballdata

data/lib/footballdata/leagues.rb

@@ -0,0 +1,59 @@
+module Footballdata
+
+  LEAGUES = {
+    'eng.1' => 'PL',     # incl. team(s) from wales
+    'eng.2' => 'ELC',
+     # PL  - Premier League        , England        27 seasons | 2019-08-09 - 2020-07-25 / matchday 31
+     #  ELC - Championship          , England         3 seasons | 2019-08-02 - 2020-07-22 / matchday 38
+     #
+     # 2019 => 2019/20
+     # 2018 => 2018/19
+     # 2017 => xxx 2017-18 - requires subscription !!!
+
+    'es.1'  => 'PD',
+    # PD  - Primera Division      , Spain          27 seasons | 2019-08-16 - 2020-07-19 / matchday 31
+
+    'pt.1'  => 'PPL',
+    # PPL - Primeira Liga         , Portugal        9 seasons | 2019-08-10 - 2020-07-26 / matchday 28
+
+    'de.1'  => 'BL1',
+    # BL1 - Bundesliga            , Germany        24 seasons | 2019-08-16 - 2020-06-27 / matchday 34
+
+    'nl.1'  => 'DED',
+    # DED - Eredivisie            , Netherlands    10 seasons | 2019-08-09 - 2020-03-08 / matchday 34
+
+    'fr.1'  => 'FL1',    # incl. team(s) monaco
+    # FL1 - Ligue 1, France
+    #   9 seasons | 2019-08-09 - 2020-05-31 / matchday 38
+    #
+    # 2019 => 2019/20
+    # 2018 => 2018/19
+    # 2017 => xxx 2017-18 - requires subscription !!!
+
+    'it.1'  => 'SA',
+    # SA  - Serie A               , Italy          15 seasons | 2019-08-24 - 2020-08-02 / matchday 27
+
+    'br.1'  => 'BSA',
+    # BSA - Série A, Brazil
+    #   4 seasons | 2020-05-03 - 2020-12-06 / matchday 10
+    #
+    #  2020 => 2020
+    #  2019 => 2019
+    #  2018 => 2018
+    #  2017 => xxx 2017 - requires subscription !!!
+
+    ## todo/check: use champs and NOT cl - why? why not?
+    'uefa.cl'        => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
+    ##  was europe.cl / cl  
+    
+    ## Copa Libertadores
+    'copa.l'  => 'CLI',
+ 
+    ############
+    ## national teams
+    'euro'  => 'EC',
+    'world' => 'WC',
+
+  }
+end   #  module Footballdata
+

data/lib/footballdata/mods.rb

@@ -0,0 +1,22 @@
+module Footballdata
+
+  #########
+  ## Mods
+  # e.g.
+  # Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
+  # AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
+
+  MODS = {
+    'br.1' => {
+       'América FC' => 'América Mineiro',  # in year 2018 ??
+    },
+    'pt.1' => {
+      'Vitória SC' => 'Vitória Guimarães',  ## avoid easy confusion with Vitória SC <=> Vitória FC
+      'Vitória FC' => 'Vitória Setúbal',
+    },
+  }
+
+end  # module Footballdata
+
+
+