football-sources 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d80d7220f2414773d0547c6aa69d15722d414bb
-  data.tar.gz: eae20eaed5e9fd9efdb33602cc2f3125b1334797
+  metadata.gz: b068838936cfdba0eae277fe5f44c957f2281655
+  data.tar.gz: da3b4be77ac9661aaaa469580d38d37e68f3ae08
 SHA512:
-  metadata.gz: cd4a9b5dbd4c86bd9ec20bebda0b56979bb6c5ce3d4fddf6fdfb947188b08f7dca68a931baf4bd199ff66f5518f326bc10bde273d43cd2c957c9e40c9f4b4cec
-  data.tar.gz: d8be6c1a8246cd61f77aa4b63f0b382d9cde0da408c83e0e7626ad34ed9daf567b5735702f05dbbf9a0dd8035baea51f9ff83e67a0cb14dc02e261659781e121
+  metadata.gz: 9179c04e869df95a9b473aeabe1397f19c29108a1eac6e6d3fe149f2cb3b5f6967002d459e4ba013bfae26a22748724030805ebc591c333a389e53eb5d05576a
+  data.tar.gz: 31c7a7a673ea694996ddcf4ae550aa0904c2ba5391f5af3bfe09c9bf1f27f2af2ba0d8ddee0389da0b11a739b7bedb9e35055486a834a338072049d5a0f758e8

data/Manifest.txt

@@ -7,29 +7,20 @@ lib/football-sources/apis.rb
 lib/football-sources/apis/config.rb
 lib/football-sources/apis/convert.rb
 lib/football-sources/apis/convert_cl.rb
-lib/football-sources/apis/download.rb
+lib/football-sources/apis/mods.rb
 lib/football-sources/apis/stat.rb
+lib/football-sources/fbref.rb
+lib/football-sources/fbref/build.rb
+lib/football-sources/fbref/config.rb
+lib/football-sources/fbref/convert.rb
 lib/football-sources/version.rb
 lib/football-sources/worldfootball.rb
 lib/football-sources/worldfootball/build.rb
 lib/football-sources/worldfootball/config.rb
 lib/football-sources/worldfootball/convert.rb
 lib/football-sources/worldfootball/convert_reports.rb
-lib/football-sources/worldfootball/download.rb
-lib/football-sources/worldfootball/leagues.rb
-lib/football-sources/worldfootball/leagues/asia.rb
-lib/football-sources/worldfootball/leagues/europe--british_isles.rb
-lib/football-sources/worldfootball/leagues/europe--central.rb
-lib/football-sources/worldfootball/leagues/europe--eastern.rb
-lib/football-sources/worldfootball/leagues/europe--northern.rb
-lib/football-sources/worldfootball/leagues/europe--southern.rb
-lib/football-sources/worldfootball/leagues/europe--western.rb
-lib/football-sources/worldfootball/leagues/europe.rb
-lib/football-sources/worldfootball/leagues/north_america.rb
-lib/football-sources/worldfootball/leagues/pacific.rb
-lib/football-sources/worldfootball/leagues/south_america.rb
+lib/football-sources/worldfootball/jobs.rb
 lib/football-sources/worldfootball/mods.rb
-lib/football-sources/worldfootball/tool.rb
 lib/football-sources/worldfootball/vacuum.rb
 lib/football/sources.rb
 test/helper.rb

data/README.md

@@ -8,10 +8,151 @@
 * forum :: [groups.google.com/group/opensport](https://groups.google.com/group/opensport)
 
 
+
 ## Usage
 
-TBD
 
+### Source #1 - `football-data.org` - ur src for machine readable football data
+
+[Daniel Freitag](https://www.football-data.org/about)'s dev-friendly football API
+offers a free to use plan
+for 12 leagues (API key sign-up and use required).
+See [`football-data.org` »](https://www.football-data.org)
+
+
+**Step 0 - Setup Secrets**
+
+Set the API key / token in the env(ironement).
+Example:
+
+```
+set FOOTBALLDATA=1234567890abcdef1234567890abcdef
+```
+
+
+**Step 1 - Download Match Schedules**
+
+Download the match schedules (in json) via api calls
+to your (local) web cache (in `~/.cache`).
+Note: The free trier has a 10 request/minute limit,
+thus, sleep/wait 10 secs after every request
+(should result in ~6 requests/minute).
+
+
+``` ruby
+require 'football/sources'
+
+
+# download up (ongoing) 2020 or 2020/21 seasons
+Webget.config.sleep  = 10
+
+Footballdata.schedule( league: 'eng.1', season: '2020/21' )
+Footballdata.schedule( league: 'eng.2', season: '2020/21' )
+
+Footballdata.schedule( league: 'de.1', season: '2020/21' )
+Footballdata.schedule( league: 'es.1', season: '2020/21' )
+
+Footballdata.schedule( league: 'fr.1', season: '2020/21' )
+Footballdata.schedule( league: 'it.1', season: '2020/21' )
+
+Footballdata.schedule( league: 'nl.1', season: '2020/21' )
+Footballdata.schedule( league: 'pt.1', season: '2020/21' )
+
+Footballdata.schedule( league: 'cl',   season: '2020/21' )
+
+# note: Brasileirão - season is a calendar year (NOT an academic year)
+Footballdata.schedule( league: 'br.1', season: '2020' )
+```
+
+Note: You can find all downloaded match schedules
+in your (local) web cache (in `~/.cache/api.football-data.org`) as pretty printed json documents.
+
+
+
+
+**Step 2 - Convert (Cached) Match Schedules to Records**
+
+Convert the (cached) match schedules
+in JSON to the one-line, one-match & one-file, one-season
+"standard" [Football.CSV format](https://github.com/footballcsv). Example:
+
+``` ruby
+require 'football/sources'
+
+['eng.1', 'eng.2',
+ 'de.1',
+ 'es.1',
+ 'fr.1',
+ 'it.1',
+ 'nl.1',
+ 'pt.1',
+ 'cl',
+].each do |league|
+  Footballdata.convert( league: league, season: '2020/21' )
+end
+
+Footballdata.convert( league: 'br.1', season: '2020' )
+```
+
+Note: By default all datasets get written into the `./o`
+directory.  Use `Footballdata.config.convert.out_dir`
+to change the output directory.
+
+The English Premier League (`eng.1`) results in `./o/2020-21/eng.1.csv`:
+
+```
+Matchday,Date,Team 1,FT,HT,Team 2,Comments
+1,Sun Sep 13 2020,Manchester City FC,(*),,Aston Villa FC,postponed
+1,Sun Sep 13 2020,Burnley FC,(*),,Manchester United FC,postponed
+1,Sat Sep 12 2020,Fulham FC,0-3,0-1,Arsenal FC,
+1,Sat Sep 12 2020,Crystal Palace FC,1-0,1-0,Southampton FC,
+1,Sat Sep 12 2020,Liverpool FC,4-3,3-2,Leeds United FC,
+1,Sat Sep 12 2020,West Ham United FC,0-2,0-0,Newcastle United FC,
+1,Sun Sep 13 2020,West Bromwich Albion FC,0-3,0-0,Leicester City FC,
+1,Sun Sep 13 2020,Tottenham Hotspur FC,0-1,0-0,Everton FC,
+1,Mon Sep 14 2020,Sheffield United FC,0-2,0-2,Wolverhampton Wanderers FC,
+1,Mon Sep 14 2020,Brighton & Hove Albion FC,1-3,0-1,Chelsea FC,
+2,Sat Sep 19 2020,Everton FC,5-2,2-1,West Bromwich Albion FC,
+2,Sat Sep 19 2020,Leeds United FC,4-3,2-1,Fulham FC,
+2,Sat Sep 19 2020,Manchester United FC,1-3,0-1,Crystal Palace FC,
+2,Sat Sep 19 2020,Arsenal FC,2-1,1-1,West Ham United FC,
+2,Sun Sep 20 2020,Southampton FC,2-5,1-1,Tottenham Hotspur FC,
+2,Sun Sep 20 2020,Newcastle United FC,0-3,0-2,Brighton & Hove Albion FC,
+2,Sun Sep 20 2020,Chelsea FC,0-2,0-0,Liverpool FC,
+2,Sun Sep 20 2020,Leicester City FC,4-2,1-1,Burnley FC,
+2,Mon Sep 21 2020,Aston Villa FC,1-0,0-0,Sheffield United FC,
+2,Mon Sep 21 2020,Wolverhampton Wanderers FC,1-3,0-2,Manchester City FC,
+...
+```
+
+Or the Brasileirão (`br.1`) in  `./o/2020/br.1.csv`:
+
+```
+Matchday,Date,Team 1,FT,HT,Team 2,Comments
+1,Sat Aug 8 2020,Fortaleza EC,0-2,0-2,CA Paranaense,
+1,Sat Aug 8 2020,Coritiba FBC,0-1,0-0,SC Internacional,
+1,Sun Aug 9 2020,SC Recife,3-2,3-1,Ceará SC,
+1,Sun Aug 9 2020,Santos FC,1-1,0-0,RB Bragantino,
+1,Sun Aug 9 2020,CR Flamengo,0-1,0-1,CA Mineiro,
+1,Sun Aug 9 2020,Goiás EC,(*),,São Paulo FC,postponed
+1,Sun Aug 9 2020,Grêmio FBPA,1-0,1-0,Fluminense FC,
+1,Sun Aug 9 2020,SE Palmeiras,(*),,CR Vasco da Gama,postponed
+2,Wed Aug 12 2020,CA Mineiro,3-2,0-2,SC Corinthians Paulista,
+2,Wed Aug 12 2020,CA Paranaense,2-1,1-1,Goiás EC,
+2,Wed Aug 12 2020,RB Bragantino,1-1,1-0,Botafogo FR,
+2,Wed Aug 12 2020,AC Goianiense,3-0,2-0,CR Flamengo,
+2,Wed Aug 12 2020,EC Bahia,1-0,1-0,Coritiba FBC,
+2,Thu Aug 13 2020,Fluminense FC,1-1,1-1,SE Palmeiras,
+2,Thu Aug 13 2020,Ceará SC,1-1,1-0,Grêmio FBPA,
+2,Thu Aug 13 2020,São Paulo FC,1-0,1-0,Fortaleza EC,
+2,Thu Aug 13 2020,SC Internacional,2-0,0-0,Santos FC,
+2,Thu Aug 13 2020,CR Vasco da Gama,2-0,2-0,SC Recife,
+...
+```
+
+
+
+That's it for now. More sources upcoming.
 
 ## License
 

data/Rakefile

@@ -18,7 +18,7 @@ Hoe.spec 'football-sources' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['webget-football',  '>= 0.0.1'],
+    ['webget-football',  '>= 0.1.1'],
     ['sportdb-catalogs', '>= 1.0.0'],
   ]
 

data/lib/football-sources.rb

@@ -13,21 +13,32 @@ require 'sportdb/catalogs'    ## note: incl. deps csvreader etc.
 module Cache
 class CsvMatchWriter
 
-    def self.write( path, recs, headers: )
-
-      ## for convenience - make sure parent folders/directories exist
-      FileUtils.mkdir_p( File.dirname( path ))  unless Dir.exist?( File.dirname( path ))
-
-        File.open( path, 'w:utf-8' ) do |f|
-          f.write headers.join(',')   ## e.g. Date,Team 1,FT,HT,Team 2
-          f.write "\n"
-          recs.each do |rec|
-              f.write rec.join(',')
-              f.write "\n"
-          end
-        end
+  def self.csv_encode( values )
+    ## quote values that incl. a comma
+    values.map do |value|
+      if value.index(',')
+        puts "** rec with field with comma >#{value}< in:"
+        pp values
+        %Q{"#{value}"}
+      else
+        value
+      end
+    end.join( ',' )
+  end
+
+  def self.write( path, recs, headers: )
+    ## for convenience - make sure parent folders/directories exist
+    FileUtils.mkdir_p( File.dirname( path ))  unless Dir.exist?( File.dirname( path ))
+
+    File.open( path, 'w:utf-8' ) do |f|
+      f.write( headers.join(','))   ## e.g. Date,Team 1,FT,HT,Team 2
+      f.write( "\n" )
+      recs.each do |values|
+        f.write( csv_encode( values ))
+        f.write( "\n" )
+      end
     end
-
+  end
 end # class CsvMatchWriter
 end # module Cache
 
@@ -40,7 +51,7 @@ require 'football-sources/version' # let version always go first
 
 require 'football-sources/apis'
 require 'football-sources/worldfootball'
-
+require 'football-sources/fbref'
 
 
 puts FootballSources.banner   # say hello

data/lib/football-sources/apis.rb

@@ -1,86 +1,10 @@
-###########################
-#  note: split code in two parts
-#    metal  - "bare" basics - no ref to sportdb
-#    and rest / convert  with sportdb references / goodies
-
-
-module Footballdata
-
-LEAGUES = {
-  'eng.1' => 'PL',     # incl. team(s) from wales
-  'eng.2' => 'ELC',
-   # PL  - Premier League        , England        27 seasons | 2019-08-09 - 2020-07-25 / matchday 31
-   #  ELC - Championship          , England         3 seasons | 2019-08-02 - 2020-07-22 / matchday 38
-   #
-   # 2019 => 2019/20
-   # 2018 => 2018/19
-   # 2017 => xxx 2017-18 - requires subscription !!!
-
-  'es.1'  => 'PD',
-  # PD  - Primera Division      , Spain          27 seasons | 2019-08-16 - 2020-07-19 / matchday 31
-
-  'pt.1'  => 'PPL',
-  # PPL - Primeira Liga         , Portugal        9 seasons | 2019-08-10 - 2020-07-26 / matchday 28
-
-  'de.1'  => 'BL1',
-  # BL1 - Bundesliga            , Germany        24 seasons | 2019-08-16 - 2020-06-27 / matchday 34
-
-  'nl.1'  => 'DED',
-  # DED - Eredivisie            , Netherlands    10 seasons | 2019-08-09 - 2020-03-08 / matchday 34
-
-  'fr.1'  => 'FL1',    # incl. team(s) monaco
-  # FL1 - Ligue 1, France
-  #   9 seasons | 2019-08-09 - 2020-05-31 / matchday 38
-  #
-  # 2019 => 2019/20
-  # 2018 => 2018/19
-  # 2017 => xxx 2017-18 - requires subscription !!!
-
-  'it.1'  => 'SA',
-  # SA  - Serie A               , Italy          15 seasons | 2019-08-24 - 2020-08-02 / matchday 27
-
-  'br.1'  => 'BSA',
-  # BSA - Série A, Brazil
-  #   4 seasons | 2020-05-03 - 2020-12-06 / matchday 10
-  #
-  #  2020 => 2020
-  #  2019 => 2019
-  #  2018 => 2018
-  #  2017 => xxx 2017 - requires subscription !!!
-
-  ## todo/check: use champs and NOT cl - why? why not?
-  'cl'        => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
-  'europe.cl' => 'CL',    ## note: cl is country code for chile!! - use champs - why? why not?
-  # CL  - UEFA Champions League , Europe         19 seasons | 2019-06-25 - 2020-05-30 / matchday 6
-}
-
-
-#########
-## Mods
-# e.g.
-# Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
-# AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
-
-
-MODS = {
-  'br.1' => {
-         'América FC' => 'América MG',   # in year 2018
-            },
-  'pt.1'  => {
-         'Vitória SC' => 'Vitória Guimarães',  ## avoid easy confusion with Vitória SC <=> Vitória FC
-         'Vitória FC' => 'Vitória Setúbal',
-           },
-}
-
-end  # module Footballdata
 
 
 ###########################
 ## our own code
 require_relative 'apis/config'
+require_relative 'apis/mods'
 require_relative 'apis/stat'
-require_relative 'apis/download'
-
 require_relative 'apis/convert'
 require_relative 'apis/convert_cl'
 

data/lib/football-sources/apis/mods.rb

@@ -0,0 +1,20 @@
+module Footballdata
+
+  #########
+  ## Mods
+  # e.g.
+  # Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
+  # AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
+
+
+  MODS = {
+    'br.1' => {
+           'América FC' => 'América MG',   # in year 2018
+              },
+    'pt.1'  => {
+           'Vitória SC' => 'Vitória Guimarães',  ## avoid easy confusion with Vitória SC <=> Vitória FC
+           'Vitória FC' => 'Vitória Setúbal',
+             },
+  }
+
+end  # module Footballdata

data/lib/football-sources/fbref.rb

@@ -0,0 +1,4 @@
+
+require_relative 'fbref/config'
+require_relative 'fbref/build'
+require_relative 'fbref/convert'

data/lib/football-sources/fbref/build.rb

@@ -0,0 +1,96 @@
+
+module Fbref
+
+def self.build( rows, league:, season: )
+  season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+  raise ArgumentError, "league key as string expected"  unless league.is_a?(String)  ## note: do NOT pass in league struct! pass in key (string)
+
+  print "  #{rows.size} rows - build #{league} #{season}"
+  print "\n"
+
+
+  recs = []
+  rows.each do |row|
+
+    stage  =  row[:stage] || ''
+
+    ## todo/check:  assert that only matchweek or round can be present NOT both!!
+    round  =  if row[:matchweek] && row[:matchweek].size > 0
+                row[:matchweek]
+              elsif row[:round] && row[:round].size > 0
+                row[:round]
+              else
+                ''
+              end
+
+    date_str  = row[:date]
+    time_str  = row[:time]
+    team1_str = row[:team1]
+    team2_str = row[:team2]
+    score_str = row[:score]
+
+    ## convert date from string e.g. 2019-25-10
+    date = Date.strptime( date_str, '%Y-%m-%d' )
+
+    comments = row[:comments]
+    ht, ft, et, pen, comments = parse_score( score_str, comments )
+
+
+    venue_str =      row[:venue]
+    attendance_str = row[:attendance]
+
+
+    recs <<  [stage,
+              round,
+              date.strftime( '%Y-%m-%d' ),
+              time_str,
+              team1_str,
+              ft,
+              ht,
+              team2_str,
+              et,              # extra: incl. extra time
+              pen,             # extra: incl. penalties
+              venue_str,
+              attendance_str,
+              comments]
+  end
+
+  recs
+end
+
+
+def self.parse_score( score_str, comments )
+
+  ## split score
+  ft  = ''
+  ht  = ''
+  et  = ''
+  pen = ''
+
+  if score_str.size > 0
+    ## note: replace unicode "fancy" dash with ascii-dash
+    #  check other columns too - possible in teams?
+    score_str = score_str.gsub( /[–]/, '-' ).strip
+
+    if score_str =~ /^\(([0-9]+)\)
+                        [ ]+ ([0-9]+) - ([0-9+]) [ ]+
+                      \(([0-9]+)\)$/x
+      ft  = '?'
+      et  = "#{$2}-#{$3}"
+      pen = "#{$1}-#{$4}"
+    else  ## assume "regular" score e.g. 0-0
+          ## check if notes include extra time otherwise assume regular time
+      if comments =~ /extra time/i
+        ft = '?'
+        et = score_str
+      else
+        ft = score_str
+      end
+    end
+  end
+
+  [ht, ft, et, pen, comments]
+end
+
+end # module Fbref