football-sources 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0d80d7220f2414773d0547c6aa69d15722d414bb
+  data.tar.gz: eae20eaed5e9fd9efdb33602cc2f3125b1334797
+SHA512:
+  metadata.gz: cd4a9b5dbd4c86bd9ec20bebda0b56979bb6c5ce3d4fddf6fdfb947188b08f7dca68a931baf4bd199ff66f5518f326bc10bde273d43cd2c957c9e40c9f4b4cec
+  data.tar.gz: d8be6c1a8246cd61f77aa4b63f0b382d9cde0da408c83e0e7626ad34ed9daf567b5735702f05dbbf9a0dd8035baea51f9ff83e67a0cb14dc02e261659781e121

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+### 0.0.1 / 2020-11-09
+
+* Everything is new. First release.
+

data/Manifest.txt

@@ -0,0 +1,36 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+lib/football-sources.rb
+lib/football-sources/apis.rb
+lib/football-sources/apis/config.rb
+lib/football-sources/apis/convert.rb
+lib/football-sources/apis/convert_cl.rb
+lib/football-sources/apis/download.rb
+lib/football-sources/apis/stat.rb
+lib/football-sources/version.rb
+lib/football-sources/worldfootball.rb
+lib/football-sources/worldfootball/build.rb
+lib/football-sources/worldfootball/config.rb
+lib/football-sources/worldfootball/convert.rb
+lib/football-sources/worldfootball/convert_reports.rb
+lib/football-sources/worldfootball/download.rb
+lib/football-sources/worldfootball/leagues.rb
+lib/football-sources/worldfootball/leagues/asia.rb
+lib/football-sources/worldfootball/leagues/europe--british_isles.rb
+lib/football-sources/worldfootball/leagues/europe--central.rb
+lib/football-sources/worldfootball/leagues/europe--eastern.rb
+lib/football-sources/worldfootball/leagues/europe--northern.rb
+lib/football-sources/worldfootball/leagues/europe--southern.rb
+lib/football-sources/worldfootball/leagues/europe--western.rb
+lib/football-sources/worldfootball/leagues/europe.rb
+lib/football-sources/worldfootball/leagues/north_america.rb
+lib/football-sources/worldfootball/leagues/pacific.rb
+lib/football-sources/worldfootball/leagues/south_america.rb
+lib/football-sources/worldfootball/mods.rb
+lib/football-sources/worldfootball/tool.rb
+lib/football-sources/worldfootball/vacuum.rb
+lib/football/sources.rb
+test/helper.rb
+test/test_version.rb

data/README.md

@@ -0,0 +1,28 @@
+# football-sources - get football data via web pages or web api (json) calls
+
+
+* home  :: [github.com/sportdb/sport.db](https://github.com/sportdb/sport.db)
+* bugs  :: [github.com/sportdb/sport.db/issues](https://github.com/sportdb/sport.db/issues)
+* gem   :: [rubygems.org/gems/football-sources](https://rubygems.org/gems/football-sources)
+* rdoc  :: [rubydoc.info/gems/football-sources](http://rubydoc.info/gems/football-sources)
+* forum :: [groups.google.com/group/opensport](https://groups.google.com/group/opensport)
+
+
+## Usage
+
+TBD
+
+
+## License
+
+![](https://publicdomainworks.github.io/buttons/zero88x31.png)
+
+The `football-sources` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+
+## Questions? Comments?
+
+Send them along to the
+[Open Sports & Friends Forum/Mailing List](http://groups.google.com/group/opensport).
+Thanks!

data/Rakefile

@@ -0,0 +1,31 @@
+require 'hoe'
+require './lib/football-sources/version.rb'
+
+Hoe.spec 'football-sources' do
+
+  self.version = FootballSources::VERSION
+
+  self.summary = 'football-sources - get football data via web pages or web api (json) calls'
+  self.description = summary
+
+  self.urls = { home: 'https://github.com/sportdb/sport.db' }
+
+  self.author = 'Gerald Bauer'
+  self.email  = 'opensport@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.extra_deps = [
+    ['webget-football',  '>= 0.0.1'],
+    ['sportdb-catalogs', '>= 1.0.0'],
+  ]
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+   required_ruby_version: '>= 2.2.2'
+  }
+
+end

data/lib/football-sources.rb

@@ -0,0 +1,46 @@
+require 'webget/football'
+
+
+# require 'sportdb/formats'   ## for Season etc.
+require 'sportdb/catalogs'    ## note: incl. deps csvreader etc.
+
+
+
+#############
+## todo/fix:  reuse  a "original" CsvMatchWriter
+##                  how? why? why not?
+###############
+module Cache
+class CsvMatchWriter
+
+    def self.write( path, recs, headers: )
+
+      ## for convenience - make sure parent folders/directories exist
+      FileUtils.mkdir_p( File.dirname( path ))  unless Dir.exist?( File.dirname( path ))
+
+        File.open( path, 'w:utf-8' ) do |f|
+          f.write headers.join(',')   ## e.g. Date,Team 1,FT,HT,Team 2
+          f.write "\n"
+          recs.each do |rec|
+              f.write rec.join(',')
+              f.write "\n"
+          end
+        end
+    end
+
+end # class CsvMatchWriter
+end # module Cache
+
+
+
+
+###
+# our own code
+require 'football-sources/version' # let version always go first
+
+require 'football-sources/apis'
+require 'football-sources/worldfootball'
+
+
+
+puts FootballSources.banner   # say hello

data/lib/football-sources/apis.rb

@@ -0,0 +1,86 @@
+###########################
+#  note: split code in two parts
+#    metal  - "bare" basics - no ref to sportdb
+#    and rest / convert  with sportdb references / goodies
+
+
+module Footballdata
+
+LEAGUES = {
+  'eng.1' => 'PL',     # incl. team(s) from wales
+  'eng.2' => 'ELC',
+   # PL  - Premier League        , England        27 seasons | 2019-08-09 - 2020-07-25 / matchday 31
+   #  ELC - Championship          , England         3 seasons | 2019-08-02 - 2020-07-22 / matchday 38
+   #
+   # 2019 => 2019/20
+   # 2018 => 2018/19
+   # 2017 => xxx 2017-18 - requires subscription !!!
+
+  'es.1'  => 'PD',
+  # PD  - Primera Division      , Spain          27 seasons | 2019-08-16 - 2020-07-19 / matchday 31
+
+  'pt.1'  => 'PPL',
+  # PPL - Primeira Liga         , Portugal        9 seasons | 2019-08-10 - 2020-07-26 / matchday 28
+
+  'de.1'  => 'BL1',
+  # BL1 - Bundesliga            , Germany        24 seasons | 2019-08-16 - 2020-06-27 / matchday 34
+
+  'nl.1'  => 'DED',
+  # DED - Eredivisie            , Netherlands    10 seasons | 2019-08-09 - 2020-03-08 / matchday 34
+
+  'fr.1'  => 'FL1',    # incl. team(s) monaco
+  # FL1 - Ligue 1, France
+  #   9 seasons | 2019-08-09 - 2020-05-31 / matchday 38
+  #
+  # 2019 => 2019/20
+  # 2018 => 2018/19
+  # 2017 => xxx 2017-18 - requires subscription !!!
+
+  'it.1'  => 'SA',
+  # SA  - Serie A               , Italy          15 seasons | 2019-08-24 - 2020-08-02 / matchday 27
+
+  'br.1'  => 'BSA',
+  # BSA - Série A, Brazil
+  #   4 seasons | 2020-05-03 - 2020-12-06 / matchday 10
+  #
+  #  2020 => 2020
+  #  2019 => 2019
+  #  2018 => 2018
+  #  2017 => xxx 2017 - requires subscription !!!
+
+  ## todo/check: use champs and NOT cl - why? why not?
+  'cl'        => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
+  'europe.cl' => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
+  # CL  - UEFA Champions League , Europe         19 seasons | 2019-06-25 - 2020-05-30 / matchday 6
+}
+
+
+#########
+## Mods
+# e.g.
+# Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
+# AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
+
+
+MODS = {
+  'br.1' => {
+         'América FC' => 'América MG',   # in year 2018
+            },
+  'pt.1'  => {
+         'Vitória SC' => 'Vitória Guimarães',  ## avoid easy confusion with Vitória SC <=> Vitória FC
+         'Vitória FC' => 'Vitória Setúbal',
+           },
+}
+
+end  # module Footballdata
+
+
+###########################
+## our own code
+require_relative 'apis/config'
+require_relative 'apis/stat'
+require_relative 'apis/download'
+
+require_relative 'apis/convert'
+require_relative 'apis/convert_cl'
+

data/lib/football-sources/apis/config.rb

@@ -0,0 +1,17 @@
+
+module Footballdata
+
+### add some more config options / settings
+  class Configuration
+    #########
+    ## nested configuration classes - use - why? why not?
+    class Convert
+       def out_dir()       @out_dir || './o'; end
+       def out_dir=(value) @out_dir = value; end
+    end
+
+   def convert()  @convert ||= Convert.new; end
+  end # class Configuration
+
+
+end # module Footballdata

data/lib/football-sources/apis/convert.rb

@@ -0,0 +1,239 @@
+
+
+module Footballdata
+
+
+def self.convert( league:, season: )
+
+  ### note/fix: cl (champions league for now is a "special" case)
+  if league.downcase == 'cl'
+    convert_cl( league: league,
+                season: season )
+    return
+  end
+
+
+
+  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+  data           = Webcache.read_json( Metal.competition_matches_url( LEAGUES[league.downcase], season.start_year ))
+  data_teams     = Webcache.read_json( Metal.competition_teams_url(   LEAGUES[league.downcase], season.start_year ))
+
+
+  ## build a (reverse) team lookup by name
+  puts "#{data_teams['teams'].size} teams"
+
+  teams_by_name = data_teams['teams'].reduce( {} ) do |h,rec|
+     h[ rec['name'] ] = rec
+     h
+  end
+
+  pp teams_by_name.keys
+
+
+
+mods = MODS[ league.downcase ] || {}
+
+
+recs = []
+
+teams = Hash.new( 0 )
+
+stat  =  Stat.new
+
+matches = data[ 'matches']
+matches.each do |m|
+  stat.update( m )
+
+  team1 = m['homeTeam']['name']
+  team2 = m['awayTeam']['name']
+
+  score = m['score']
+
+
+
+  if m['stage'] == 'REGULAR_SEASON'
+    teams[ team1 ] += 1
+    teams[ team2 ] += 1
+
+    ### mods - rename club names
+    unless mods.nil? || mods.empty?
+      team1 = mods[ team1 ]      if mods[ team1 ]
+      team2 = mods[ team2 ]      if mods[ team2 ]
+    end
+
+
+    ## e.g. "utcDate": "2020-05-09T00:00:00Z",
+    date_str = m['utcDate']
+    date = DateTime.strptime( date_str, '%Y-%m-%dT%H:%M:%SZ' )
+
+
+    comments = ''
+    ft       = ''
+    ht       = ''
+
+    case m['status']
+    when 'SCHEDULED', 'IN_PLAY'
+      ft = ''
+      ht = ''
+    when 'FINISHED'
+      ## todo/fix: assert duration == "REGULAR"
+      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
+      ht = "#{score['halfTime']['homeTeam']}-#{score['halfTime']['awayTeam']}"
+    when 'AWARDED'
+      ## todo/fix: assert duration == "REGULAR"
+      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
+      ft << ' (*)'
+      ht = ''
+      comments = 'awarded'
+    when 'CANCELLED'
+      ft = '(*)'
+      ht = ''
+      comments  = 'canceled'   ## us eng ? -> canceled, british eng. cancelled ?
+    when 'POSTPONED'
+      ft = '(*)'
+      ht = ''
+      comments = 'postponed'
+    else
+      puts "!! ERROR: unsupported match status >#{m['status']}< - sorry:"
+      pp m
+      exit 1
+    end
+
+
+    ## todo/fix: assert matchday is a number e.g. 1,2,3, etc.!!!
+    recs << [m['matchday'],
+             date.to_date.strftime( '%Y-%m-%d' ),
+             team1,
+             ft,
+             ht,
+             team2,
+             comments
+            ]
+
+
+    print '%2s' % m['matchday']
+    print ' - '
+    print '%-24s' % team1
+    print '  '
+    print ft
+    print ' '
+    print "(#{ht})"    unless ht.empty?
+    print '  '
+    print '%-24s' % team2
+    print '  '
+    print comments
+    print ' | '
+    ## print date.to_date  ## strip time
+    print date.to_date.strftime( '%a %b %-d %Y' )
+    print ' -- '
+    print date
+    print "\n"
+  else
+    puts "-- skipping #{m['stage']}"
+  end
+end # each match
+
+
+
+## note: get season from first match
+##   assert - all other matches include the same season
+## e.g.
+# "season": {
+#  "id": 154,
+#  "startDate": "2018-08-03",
+#  "endDate": "2019-05-05",
+#  "currentMatchday": 46
+# }
+
+start_date = Date.strptime( matches[0]['season']['startDate'], '%Y-%m-%d' )
+end_date   = Date.strptime( matches[0]['season']['endDate'],   '%Y-%m-%d' )
+
+dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
+
+buf = ''
+buf << "#{season.key} (#{dates}) - "
+buf << "#{teams.keys.size} clubs, "
+buf << "#{stat[:regular_season][:matches]} matches, "
+buf << "#{stat[:regular_season][:goals]} goals"
+buf << "\n"
+
+puts buf
+
+
+
+   ## note: warn if stage is greater one and not regular season!!
+   File.open( './errors.txt' , 'a:utf-8' ) do |f|
+     if stat[:all][:stage].keys != ['REGULAR_SEASON']
+      f.write "!! WARN - league: #{league}, season: #{season.key} includes non-regular stage(s):\n"
+      f.write "   #{stat[:all][:stage].keys.inspect}\n"
+     end
+   end
+
+
+   File.open( './logs.txt', 'a:utf-8' ) do |f|
+     f.write "\n================================\n"
+     f.write "====  #{league}  =============\n"
+     f.write buf
+     f.write "  match status: #{stat[:regular_season][:status].inspect}\n"
+     f.write "  match duration: #{stat[:regular_season][:duration].inspect}\n"
+
+     f.write "#{teams.keys.size} teams:\n"
+     teams.each do |name, count|
+        rec = teams_by_name[ name ]
+        f.write "  #{count}x  #{name}"
+        if rec
+          f.write " | #{rec['shortName']} "   if name != rec['shortName']
+          f.write " › #{rec['area']['name']}"
+          f.write "  - #{rec['address']}"
+        else
+          puts "!! ERROR - no team record found in teams.json for >#{name}<"
+          exit 1
+        end
+        f.write "\n"
+     end
+   end
+
+
+
+
+# recs = recs.sort { |l,r| l[1] <=> r[1] }
+## reformat date / beautify e.g. Sat Aug 7 1993
+recs.each { |rec| rec[1] = Date.strptime( rec[1], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+
+headers = [
+  'Matchday',
+  'Date',
+  'Team 1',
+  'FT',
+  'HT',
+  'Team 2',
+  'Comments'
+]
+
+## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
+Cache::CsvMatchWriter.write( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
+                             recs,
+                             headers: headers )
+
+
+teams.each do |name, count|
+  rec = teams_by_name[ name ]
+  print "  #{count}x  "
+  print name
+  if rec
+    print " | #{rec['shortName']} "   if name != rec['shortName']
+    print " › #{rec['area']['name']}"
+    print "  - #{rec['address']}"
+  else
+    puts "!! ERROR  - no team record found in teams.json for #{name}"
+    exit 1
+  end
+  print "\n"
+end
+
+pp stat
+end   # method convert
+end #  module Footballdata
+
+