football-sources 0.0.1

data/lib/football-sources/apis/convert_cl.rb

@@ -0,0 +1,267 @@
+##########
+##  todo/fix:
+##    use a more generic name?
+##      might also works for euro or worldcup - check?
+
+
+module Footballdata
+
+
+STAGE_TO_STAGE = {
+  'PRELIMINARY_SEMI_FINALS' => 'Qualifying',
+  'PRELIMINARY_FINAL'       => 'Qualifying',
+  '1ST_QUALIFYING_ROUND'    => 'Qualifying',
+  '2ND_QUALIFYING_ROUND'    => 'Qualifying',
+  '3RD_QUALIFYING_ROUND'    => 'Qualifying',
+  'PLAY_OFF_ROUND'          => 'Qualifying',
+  'ROUND_OF_16'             => 'Knockout',
+  'QUARTER_FINALS'          => 'Knockout',
+  'SEMI_FINALS'             => 'Knockout',
+  'FINAL'                   => 'Knockout',
+}
+
+STAGE_TO_ROUND = {
+  'PRELIMINARY_SEMI_FINALS' => 'Preliminary Semifinals',
+  'PRELIMINARY_FINAL'       => 'Preliminary Final',
+  '1ST_QUALIFYING_ROUND'    => 'Qual. Round 1',
+  '2ND_QUALIFYING_ROUND'    => 'Qual. Round 2',
+  '3RD_QUALIFYING_ROUND'    => 'Qual. Round 3',
+  'PLAY_OFF_ROUND'          => 'Playoff Round',
+  'ROUND_OF_16'             => 'Round of 16',
+  'QUARTER_FINALS'          => 'Quarterfinals',
+  'SEMI_FINALS'             => 'Semifinals',
+  'FINAL'                   => 'Final',
+}
+
+
+
+
+def self.convert_cl( league:, season: )
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  data           = Webcache.read_json( Metal.competition_matches_url( LEAGUES[league.downcase], season.start_year ))
+  data_teams     = Webcache.read_json( Metal.competition_teams_url(   LEAGUES[league.downcase], season.start_year ))
+
+
+  ## build a (reverse) team lookup by name
+  puts "#{data_teams['teams'].size} teams"
+
+  teams_by_name = data_teams['teams'].reduce( {} ) do |h,rec|
+     h[ rec['name'] ] = rec
+     h
+  end
+
+  pp teams_by_name.keys
+
+
+
+mods = MODS[ league.downcase ] || {}
+
+
+recs = []
+
+teams = Hash.new( 0 )
+stat = Stat.new
+
+matches = data[ 'matches' ]
+matches.each do |m|
+  stat.update( m )
+
+  team1 = m['homeTeam']['name']
+  team2 = m['awayTeam']['name']
+
+
+  score = m['score']
+
+
+
+  if m['stage'] == 'GROUP_STAGE'
+    stage = 'Group'
+    round = "Matchday #{m['matchday']}"
+    if m['group'] =~ /Group ([A-Z])/
+      group = $1
+    else
+      puts "!! ERROR - no group name found for group >#{m['group']}<"
+      exit 1
+    end
+  else
+    stage = STAGE_TO_STAGE[ m['stage'] ]
+    if stage.nil?
+      puts "!! ERROR - no stage mapping found for stage >#{m['stage']}<"
+      exit 1
+    end
+    round = STAGE_TO_ROUND[ m['stage'] ]
+    if round.nil?
+      puts "!! ERROR - no round mapping found for stage >#{m['stage']}<"
+      exit 1
+    end
+    group = ''
+  end
+
+
+    teams[ team1 ] += 1
+    teams[ team2 ] += 1
+
+    ### mods - rename club names
+    unless mods.nil? || mods.empty?
+      team1 = mods[ team1 ]      if mods[ team1 ]
+      team2 = mods[ team2 ]      if mods[ team2 ]
+    end
+
+
+    ## e.g. "utcDate": "2020-05-09T00:00:00Z",
+    date_str = m['utcDate']
+    date = DateTime.strptime( date_str, '%Y-%m-%dT%H:%M:%SZ' )
+
+
+    comments = ''
+    ft       = ''
+    ht       = ''
+    et       = ''
+    pen      = ''
+
+    case m['status']
+    when 'SCHEDULED', 'IN_PLAY'
+      ft  = ''
+      ht  = ''
+      et  = ''
+      pen = ''
+    when 'AWARDED'
+      ## todo/fix: assert duration == "REGULAR"
+      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
+      ft << ' (*)'
+      ht = ''
+      comments = 'awarded'
+    when 'FINISHED'
+      ## note: if extraTime present
+      ## than fullTime is extraTime score!!
+      ##   AND   fullTime - extraTime is fullTime score!!
+      ## double check in other season too??
+      ##   - checked in cl 2018/19
+
+      if score['extraTime']['homeTeam'] && score['extraTime']['awayTeam']
+        et = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
+        ft = "#{score['fullTime']['homeTeam']-score['extraTime']['homeTeam']}-#{score['fullTime']['awayTeam']-score['extraTime']['awayTeam']}"
+      else
+        ft  = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
+      end
+
+      ht  = "#{score['halfTime']['homeTeam']}-#{score['halfTime']['awayTeam']}"
+
+      pen = if score['penalties']['homeTeam'] && score['penalties']['awayTeam']
+              "#{score['penalties']['homeTeam']}-#{score['penalties']['awayTeam']}"
+            else
+              ''
+            end
+    else
+      puts "!! ERROR: unsupported match status >#{m['status']}< - sorry:"
+      pp m
+      exit 1
+    end
+
+
+    recs << [stage,
+             round,
+             group,
+             date.to_date.strftime( '%Y-%m-%d' ),
+             team1,
+             ft,
+             ht,
+             team2,
+             et,
+             pen,
+             comments
+            ]
+
+
+    print '%2s' % m['matchday']
+    print ' - '
+    print '%-24s' % team1
+    print '  '
+    print ft
+    print ' '
+    print "(#{ht})"    unless ht.empty?
+    print '  '
+    print '%-24s' % team2
+    print '  '
+    print comments
+    print ' | '
+    ## print date.to_date  ## strip time
+    print date.to_date.strftime( '%a %b %-d %Y' )
+    print ' -- '
+    print date
+    print "\n"
+
+end # each match
+
+
+## note: get season from first match
+##   assert - all other matches include the same season
+## e.g.
+# "season": {
+#  "id": 154,
+#  "startDate": "2018-08-03",
+#  "endDate": "2019-05-05",
+#  "currentMatchday": 46
+# }
+
+start_date = Date.strptime( matches[0]['season']['startDate'], '%Y-%m-%d' )
+end_date   = Date.strptime( matches[0]['season']['endDate'],   '%Y-%m-%d' )
+
+dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
+
+buf = ''
+buf << "#{season.key} (#{dates}) - "
+buf << "#{teams.keys.size} clubs, "
+buf << "#{stat[:all][:matches]} matches, "
+buf << "#{stat[:all][:goals]} goals"
+buf << "\n"
+
+puts buf
+
+
+
+# recs = recs.sort { |l,r| l[1] <=> r[1] }
+## reformat date / beautify e.g. Sat Aug 7 1993
+recs.each { |rec| rec[3] = Date.strptime( rec[3], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+
+headers = [
+  'Stage',
+  'Round',
+  'Group',
+  'Date',
+  'Team 1',
+  'FT',
+  'HT',
+  'Team 2',
+  'ET',
+  'P',
+  'Comments'
+]
+
+## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
+Cache::CsvMatchWriter.write( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
+                             recs,
+                             headers: headers )
+
+
+teams.each do |name, count|
+  rec = teams_by_name[ name ]
+  print "  #{count}x  "
+  print name
+  if rec
+    print " | #{rec['shortName']} "   if name != rec['shortName']
+    print " › #{rec['area']['name']}"
+    print "  - #{rec['address']}"
+  else
+    print "!! ERROR - no team record found in teams.json for #{name}"
+    exit 1
+  end
+  print "\n"
+end
+
+pp stat
+end   # method convert_cl
+end #  module Footballdata
+
+

data/lib/football-sources/apis/download.rb

@@ -0,0 +1,11 @@
+
+
+module Footballdata
+
+  def self.schedule( league:, season: )
+    season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+    Metal.competition( LEAGUES[ league.downcase ], season.start_year )
+  end
+
+end  # module Footballdata

data/lib/football-sources/apis/stat.rb

@@ -0,0 +1,59 @@
+module Footballdata
+
+class Stat      ## rename to match stat or something why? why not?
+  def initialize
+    @data = {}
+  end
+
+  def [](key) @data[ key ]; end
+
+  def update( match )
+     ## keep track of some statistics
+     stat = @data[:all] ||= { stage:    Hash.new( 0 ),
+                              duration: Hash.new( 0 ),
+                              status:   Hash.new( 0 ),
+                              group:    Hash.new( 0 ),
+                              matchday: Hash.new( 0 ),
+
+                              matches:  0,
+                              goals:    0,
+                            }
+
+     stat[:stage][ match['stage'] ]   += 1
+     stat[:group][ match['group'] ]  += 1
+     stat[:status][ match['status'] ]  += 1
+     stat[:matchday][ match['matchday'] ]  += 1
+
+     score = match['score']
+
+     stat[:duration][ score['duration'] ] += 1   ## track - assert always REGULAR
+
+     stat[:matches] += 1
+     stat[:goals]   += score['fullTime']['homeTeam'].to_i  if score['fullTime']['homeTeam']
+     stat[:goals]   += score['fullTime']['awayTeam'].to_i  if score['fullTime']['awayTeam']
+
+
+     stage_key = match['stage'].downcase.to_sym  # e.g. :regular_season
+     stat = @data[ stage_key ] ||= { duration: Hash.new( 0 ),
+                                     status:   Hash.new( 0 ),
+                                     group:    Hash.new( 0 ),
+                                     matchday: Hash.new( 0 ),
+
+                                     matches:  0,
+                                     goals:    0,
+                                  }
+     stat[:group][ match['group'] ]  += 1
+     stat[:status][ match['status'] ]  += 1
+     stat[:matchday][ match['matchday'] ]  += 1
+
+     stat[:duration][ score['duration'] ] += 1   ## track - assert always REGULAR
+
+     stat[:matches] += 1
+     stat[:goals]   += score['fullTime']['homeTeam'].to_i  if score['fullTime']['homeTeam']
+     stat[:goals]   += score['fullTime']['awayTeam'].to_i  if score['fullTime']['awayTeam']
+  end
+end  # class Stat
+end  # module Footballdata
+
+
+

data/lib/football-sources/version.rb

@@ -0,0 +1,19 @@
+
+module FootballSources
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
+  PATCH = 1
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "football-sources/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
+  end
+end   # module FootballSources

data/lib/football-sources/worldfootball.rb

@@ -0,0 +1,24 @@
+###########################
+#  note: split code in two parts
+#    metal  - "bare" basics - no ref to sportdb
+#    and rest / convert  with sportdb references / goodies
+
+
+## our own code
+require_relative 'worldfootball/leagues'
+
+require_relative 'worldfootball/config'
+
+require_relative 'worldfootball/download'
+
+require_relative 'worldfootball/mods'
+require_relative 'worldfootball/vacuum'
+require_relative 'worldfootball/build'
+require_relative 'worldfootball/convert'
+require_relative 'worldfootball/convert_reports'
+
+
+require_relative 'worldfootball/tool'
+
+
+

data/lib/football-sources/worldfootball/build.rb

@@ -0,0 +1,245 @@
+
+module Worldfootball
+
+
+ROUND_TO_EN = {
+  '1. Runde'      => 'Round 1',
+  '2. Runde'      => 'Round 2',
+  '3. Runde'      => 'Round 3',
+  '4. Runde'      => 'Round 4',
+  'Achtelfinale'  => 'Round of 16',
+  'Viertelfinale' => 'Quarterfinals',
+  'Halbfinale'    => 'Semifinals',
+  'Finale'        => 'Final',
+}
+
+
+## todo/check:  english league cup/trophy has NO ET - also support - make more flexible!!!
+
+## build "standard" match records from "raw" table rows
+def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such - why? why not?
+   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+   raise ArgumentError, "league key as string expected"  unless league.is_a?(String)  ## note: do NOT pass in league struct! pass in key (string)
+
+   print "  #{rows.size} rows - build #{league} #{season}"
+   print " - #{stage}" unless stage.empty?
+   print "\n"
+
+
+   ## note: use only first part from key for lookup
+   ##    e.g. at.1  => at
+   ##         eng.1 => eng
+   ##     and so on
+   mods = MODS[ league.split('.')[0] ] || {}
+
+   score_errors = SCORE_ERRORS[ league ] || {}
+
+
+   i = 0
+   recs = []
+   rows.each do |row|
+     i += 1
+
+
+  if row[:round] =~ /Spieltag/
+    puts
+    print '[%03d] ' % (i+1)
+    print row[:round]
+
+    if m = row[:round].match( /([0-9]+)\. Spieltag/ )
+      ## todo/check: always use a string even if number (as a string eg. '1' etc.)
+      round = m[1]  ## note: keep as string (NOT number)
+      print " => #{round}"
+    else
+      puts "!! ERROR: cannot find matchday number"
+      exit 1
+    end
+    print "\n"
+  elsif row[:round] =~ /[1-9]\.[ ]Runde|
+                          Achtelfinale|
+                          Viertelfinale|
+                          Halbfinale|
+                          Finale
+                          /x
+    puts
+    print '[%03d] ' % (i+1)
+    print row[:round]
+
+
+    ## do NOT translate rounds (to english) - keep in german / deutsch (de)
+    if ['at.cup', 'at.1',    ## at.1 - incl. europa league playoff
+        'de.cup'].include?( league )
+      round = row[:round]
+    else
+      round = ROUND_TO_EN[ row[:round] ]
+      if round.nil?
+        puts "!! ERROR: no mapping for round to english (en) found >#{row[:round]}<:"
+        pp row
+        exit 1
+      end
+      print " => #{round}"
+    end
+    print "\n"
+  else
+    puts "!! ERROR: unknown round >#{row[:round]}< for league >#{league}<:"
+    pp row
+    exit 1
+  end
+
+
+    date_str  = row[:date]
+    time_str  = row[:time]
+    team1_str = row[:team1]
+    team2_str = row[:team2]
+    score_str = row[:score]
+
+    ## convert date from string e.g. 2019-25-10
+    date = Date.strptime( date_str, '%Y-%m-%d' )
+
+
+    ### check for score_error; first (step 1) lookup by date
+    score_error = score_errors[ date.strftime('%Y-%m-%d') ]
+    if score_error
+      if team1_str == score_error[0] &&
+         team2_str == score_error[1]
+         ## check if team names match too; if yes, apply fix/patch!!
+         if score_str != score_error[2][0]
+           puts "!! WARN - score fix changed? - expected #{score_error[2][0]}, got #{score_str} - fixing to #{score_error[2][1]}"
+           pp row
+         end
+         puts "FIX - applying score error fix - from #{score_error[2][0]} to => #{score_error[2][1]}"
+         score_str = score_error[2][1]
+      end
+    end
+
+
+    print '[%03d]    ' % (i+1)
+    print "%-10s | " % date_str
+    print "%-5s | "  % time_str
+    print "%-22s | " % team1_str
+    print "%-22s | " % team2_str
+    print score_str
+    print "\n"
+
+
+    ## check for 0:3 Wert.   - change Wert. to awd.  (awarded)
+    score_str = score_str.sub( /Wert\./i, 'awd.' )
+
+    ## clean team name (e.g. remove (old))
+    ##   and asciify (e.g. ’ to ' )
+    team1_str = norm_team( team1_str )
+    team2_str = norm_team( team2_str )
+
+    team1_str = mods[ team1_str ]   if mods[ team1_str ]
+    team2_str = mods[ team2_str ]   if mods[ team2_str ]
+
+
+
+
+    ht, ft, et, pen, comments = parse_score( score_str )
+
+
+
+    recs <<  [stage,
+              round,
+              date.strftime( '%Y-%m-%d' ),
+              time_str,
+              team1_str,
+              ft,
+              ht,
+              team2_str,
+              et,              # extra: incl. extra time
+              pen,             # extra: incl. penalties
+              comments]
+   end  # each row
+   recs
+end  # build
+
+
+
+def self.parse_score( score_str )
+  comments = String.new( '' )     ## check - rename to/use status or such - why? why not?
+
+  ## split score
+  ft  = ''
+  ht  = ''
+  et  = ''
+  pen = ''
+  if score_str == '---'   ## in the future (no score yet) - was -:-
+    ft = ''
+    ht = ''
+  elsif score_str == 'n.gesp.' ||   ## cancelled (british) / canceled (us)
+        score_str == 'ausg.'   ||   ## todo/check: change to some other status ????
+        score_str == 'annull.'      ## todo/check: change to some other status (see ie 2012) ????
+    ft = '(*)'
+    ht = ''
+    comments = 'cancelled'
+  elsif score_str == 'abgebr.'  ## abandoned  -- waiting for replay?
+    ft = '(*)'
+    ht = ''
+    comments = 'abandoned'
+  elsif score_str == 'verl.'   ## postponed
+    ft = ''
+    ht = ''
+    comments = 'postponed'
+  # 5-4 (0-0, 1-1, 2-2) i.E.
+  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*
+                      \(([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*,[ ]*
+                        ([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*,[ ]*
+                       ([0-9]+) [ ]*-[ ]* ([0-9]+)\)
+                          [ ]*
+                       i\.E\.
+                     /x
+    pen = "#{$1}-#{$2}"
+    ht  = "#{$3}-#{$4}"
+    ft  = "#{$5}-#{$6}"
+    et  = "#{$7}-#{$8}"
+  # 2-1 (1-0, 1-1) n.V
+  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
+                      [ ]*
+                    \(([0-9]+) [ ]*-[ ]* ([0-9]+)
+                       [ ]*,[ ]*
+                      ([0-9]+) [ ]*-[ ]* ([0-9]+)
+                      \)
+                       [ ]*
+                       n\.V\.
+                     /x
+    et  = "#{$1}-#{$2}"
+    ht  = "#{$3}-#{$4}"
+    ft  = "#{$5}-#{$6}"
+  elsif score_str =~ /([0-9]+)
+                          [ ]*-[ ]*
+                      ([0-9]+)
+                          [ ]*
+                      \(([0-9]+)
+                          [ ]*-[ ]*
+                        ([0-9]+)
+                      \)
+                     /x
+    ft = "#{$1}-#{$2}"
+    ht = "#{$3}-#{$4}"
+  elsif  score_str =~ /([0-9]+)
+                         [ ]*-[ ]*
+                       ([0-9]+)
+                         [ ]*
+                        ([a-z.]+)
+                       /x
+    ft = "#{$1}-#{$2} (*)"
+    ht = ''
+    comments = $3
+  elsif score_str =~ /^([0-9]+)-([0-9]+)$/
+     ft = "#{$1}-#{$2}"     ## e.g. see luxemburg and others
+     ht = ''
+  else
+     puts "!! ERROR - unsupported score format >#{score_str}< - sorry; maybe add a score error fix/patch"
+     exit 1
+  end
+
+  [ht, ft, et, pen, comments]
+end
+
+end # module Worldfootball