football-sources 0.1.0 → 0.2.0

metadata

@@ -1,43 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: football-sources
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-21 00:00:00.000000000 Z
+date: 2024-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: webget-football
+  name: sportdb-catalogs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: sportdb-catalogs
+  name: sportdb-writers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: footballdata-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: worldfootball
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -64,18 +92,19 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '4.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
-description: football-sources - get football data via web pages or web api (json)
-  calls
-email: opensport@googlegroups.com
-executables: []
+        version: '4.1'
+description: football-sources - get football match data (leagues, cups & more) via
+  web pages or web api (json) calls
+email: gerald.bauer@gmail.com
+executables:
+- fbgen
 extensions: []
 extra_rdoc_files:
 - CHANGELOG.md
@@ -86,34 +115,16 @@ files:
 - Manifest.txt
 - README.md
 - Rakefile
+- bin/fbgen
 - lib/football-sources.rb
-- lib/football-sources/apis.rb
-- lib/football-sources/apis/config.rb
-- lib/football-sources/apis/convert.rb
-- lib/football-sources/apis/convert_cl.rb
-- lib/football-sources/apis/mods.rb
-- lib/football-sources/apis/stat.rb
-- lib/football-sources/fbref.rb
-- lib/football-sources/fbref/build.rb
-- lib/football-sources/fbref/config.rb
-- lib/football-sources/fbref/convert.rb
+- lib/football-sources/process.rb
 - lib/football-sources/version.rb
-- lib/football-sources/worldfootball.rb
-- lib/football-sources/worldfootball/build.rb
-- lib/football-sources/worldfootball/config.rb
-- lib/football-sources/worldfootball/convert.rb
-- lib/football-sources/worldfootball/convert_reports.rb
-- lib/football-sources/worldfootball/jobs.rb
-- lib/football-sources/worldfootball/mods.rb
-- lib/football-sources/worldfootball/vacuum.rb
 - lib/football/sources.rb
-- test/helper.rb
-- test/test_version.rb
 homepage: https://github.com/sportdb/sport.db
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -130,9 +141,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
-summary: football-sources - get football data via web pages or web api (json) calls
+summary: football-sources - get football match data (leagues, cups & more) via web
+  pages or web api (json) calls
 test_files: []

data/lib/football-sources/apis/config.rb

@@ -1,17 +0,0 @@
-
-module Footballdata
-
-### add some more config options / settings
-  class Configuration
-    #########
-    ## nested configuration classes - use - why? why not?
-    class Convert
-       def out_dir()       @out_dir || './o'; end
-       def out_dir=(value) @out_dir = value; end
-    end
-
-   def convert()  @convert ||= Convert.new; end
-  end # class Configuration
-
-
-end # module Footballdata

data/lib/football-sources/apis/convert.rb

@@ -1,239 +0,0 @@
-
-
-module Footballdata
-
-
-def self.convert( league:, season: )
-
-  ### note/fix: cl (champions league for now is a "special" case)
-  if league.downcase == 'cl'
-    convert_cl( league: league,
-                season: season )
-    return
-  end
-
-
-
-  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
-
-  data           = Webcache.read_json( Metal.competition_matches_url( LEAGUES[league.downcase], season.start_year ))
-  data_teams     = Webcache.read_json( Metal.competition_teams_url(   LEAGUES[league.downcase], season.start_year ))
-
-
-  ## build a (reverse) team lookup by name
-  puts "#{data_teams['teams'].size} teams"
-
-  teams_by_name = data_teams['teams'].reduce( {} ) do |h,rec|
-     h[ rec['name'] ] = rec
-     h
-  end
-
-  pp teams_by_name.keys
-
-
-
-mods = MODS[ league.downcase ] || {}
-
-
-recs = []
-
-teams = Hash.new( 0 )
-
-stat  =  Stat.new
-
-matches = data[ 'matches']
-matches.each do |m|
-  stat.update( m )
-
-  team1 = m['homeTeam']['name']
-  team2 = m['awayTeam']['name']
-
-  score = m['score']
-
-
-
-  if m['stage'] == 'REGULAR_SEASON'
-    teams[ team1 ] += 1
-    teams[ team2 ] += 1
-
-    ### mods - rename club names
-    unless mods.nil? || mods.empty?
-      team1 = mods[ team1 ]      if mods[ team1 ]
-      team2 = mods[ team2 ]      if mods[ team2 ]
-    end
-
-
-    ## e.g. "utcDate": "2020-05-09T00:00:00Z",
-    date_str = m['utcDate']
-    date = DateTime.strptime( date_str, '%Y-%m-%dT%H:%M:%SZ' )
-
-
-    comments = ''
-    ft       = ''
-    ht       = ''
-
-    case m['status']
-    when 'SCHEDULED', 'IN_PLAY'
-      ft = ''
-      ht = ''
-    when 'FINISHED'
-      ## todo/fix: assert duration == "REGULAR"
-      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
-      ht = "#{score['halfTime']['homeTeam']}-#{score['halfTime']['awayTeam']}"
-    when 'AWARDED'
-      ## todo/fix: assert duration == "REGULAR"
-      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
-      ft << ' (*)'
-      ht = ''
-      comments = 'awarded'
-    when 'CANCELLED'
-      ft = '(*)'
-      ht = ''
-      comments  = 'canceled'   ## us eng ? -> canceled, british eng. cancelled ?
-    when 'POSTPONED'
-      ft = '(*)'
-      ht = ''
-      comments = 'postponed'
-    else
-      puts "!! ERROR: unsupported match status >#{m['status']}< - sorry:"
-      pp m
-      exit 1
-    end
-
-
-    ## todo/fix: assert matchday is a number e.g. 1,2,3, etc.!!!
-    recs << [m['matchday'],
-             date.to_date.strftime( '%Y-%m-%d' ),
-             team1,
-             ft,
-             ht,
-             team2,
-             comments
-            ]
-
-
-    print '%2s' % m['matchday']
-    print ' - '
-    print '%-24s' % team1
-    print '  '
-    print ft
-    print ' '
-    print "(#{ht})"    unless ht.empty?
-    print '  '
-    print '%-24s' % team2
-    print '  '
-    print comments
-    print ' | '
-    ## print date.to_date  ## strip time
-    print date.to_date.strftime( '%a %b %-d %Y' )
-    print ' -- '
-    print date
-    print "\n"
-  else
-    puts "-- skipping #{m['stage']}"
-  end
-end # each match
-
-
-
-## note: get season from first match
-##   assert - all other matches include the same season
-## e.g.
-# "season": {
-#  "id": 154,
-#  "startDate": "2018-08-03",
-#  "endDate": "2019-05-05",
-#  "currentMatchday": 46
-# }
-
-start_date = Date.strptime( matches[0]['season']['startDate'], '%Y-%m-%d' )
-end_date   = Date.strptime( matches[0]['season']['endDate'],   '%Y-%m-%d' )
-
-dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
-
-buf = ''
-buf << "#{season.key} (#{dates}) - "
-buf << "#{teams.keys.size} clubs, "
-buf << "#{stat[:regular_season][:matches]} matches, "
-buf << "#{stat[:regular_season][:goals]} goals"
-buf << "\n"
-
-puts buf
-
-
-
-   ## note: warn if stage is greater one and not regular season!!
-   File.open( './errors.txt' , 'a:utf-8' ) do |f|
-     if stat[:all][:stage].keys != ['REGULAR_SEASON']
-      f.write "!! WARN - league: #{league}, season: #{season.key} includes non-regular stage(s):\n"
-      f.write "   #{stat[:all][:stage].keys.inspect}\n"
-     end
-   end
-
-
-   File.open( './logs.txt', 'a:utf-8' ) do |f|
-     f.write "\n================================\n"
-     f.write "====  #{league}  =============\n"
-     f.write buf
-     f.write "  match status: #{stat[:regular_season][:status].inspect}\n"
-     f.write "  match duration: #{stat[:regular_season][:duration].inspect}\n"
-
-     f.write "#{teams.keys.size} teams:\n"
-     teams.each do |name, count|
-        rec = teams_by_name[ name ]
-        f.write "  #{count}x  #{name}"
-        if rec
-          f.write " | #{rec['shortName']} "   if name != rec['shortName']
-          f.write " › #{rec['area']['name']}"
-          f.write "  - #{rec['address']}"
-        else
-          puts "!! ERROR - no team record found in teams.json for >#{name}<"
-          exit 1
-        end
-        f.write "\n"
-     end
-   end
-
-
-
-
-# recs = recs.sort { |l,r| l[1] <=> r[1] }
-## reformat date / beautify e.g. Sat Aug 7 1993
-recs.each { |rec| rec[1] = Date.strptime( rec[1], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
-
-headers = [
-  'Matchday',
-  'Date',
-  'Team 1',
-  'FT',
-  'HT',
-  'Team 2',
-  'Comments'
-]
-
-## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
-Cache::CsvMatchWriter.write( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
-                             recs,
-                             headers: headers )
-
-
-teams.each do |name, count|
-  rec = teams_by_name[ name ]
-  print "  #{count}x  "
-  print name
-  if rec
-    print " | #{rec['shortName']} "   if name != rec['shortName']
-    print " › #{rec['area']['name']}"
-    print "  - #{rec['address']}"
-  else
-    puts "!! ERROR  - no team record found in teams.json for #{name}"
-    exit 1
-  end
-  print "\n"
-end
-
-pp stat
-end   # method convert
-end #  module Footballdata
-
-

data/lib/football-sources/apis/convert_cl.rb

@@ -1,267 +0,0 @@
-##########
-##  todo/fix:
-##    use a more generic name?
-##      might also works for euro or worldcup - check?
-
-
-module Footballdata
-
-
-STAGE_TO_STAGE = {
-  'PRELIMINARY_SEMI_FINALS' => 'Qualifying',
-  'PRELIMINARY_FINAL'       => 'Qualifying',
-  '1ST_QUALIFYING_ROUND'    => 'Qualifying',
-  '2ND_QUALIFYING_ROUND'    => 'Qualifying',
-  '3RD_QUALIFYING_ROUND'    => 'Qualifying',
-  'PLAY_OFF_ROUND'          => 'Qualifying',
-  'ROUND_OF_16'             => 'Knockout',
-  'QUARTER_FINALS'          => 'Knockout',
-  'SEMI_FINALS'             => 'Knockout',
-  'FINAL'                   => 'Knockout',
-}
-
-STAGE_TO_ROUND = {
-  'PRELIMINARY_SEMI_FINALS' => 'Preliminary Semifinals',
-  'PRELIMINARY_FINAL'       => 'Preliminary Final',
-  '1ST_QUALIFYING_ROUND'    => 'Qual. Round 1',
-  '2ND_QUALIFYING_ROUND'    => 'Qual. Round 2',
-  '3RD_QUALIFYING_ROUND'    => 'Qual. Round 3',
-  'PLAY_OFF_ROUND'          => 'Playoff Round',
-  'ROUND_OF_16'             => 'Round of 16',
-  'QUARTER_FINALS'          => 'Quarterfinals',
-  'SEMI_FINALS'             => 'Semifinals',
-  'FINAL'                   => 'Final',
-}
-
-
-
-
-def self.convert_cl( league:, season: )
-  season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
-
-  data           = Webcache.read_json( Metal.competition_matches_url( LEAGUES[league.downcase], season.start_year ))
-  data_teams     = Webcache.read_json( Metal.competition_teams_url(   LEAGUES[league.downcase], season.start_year ))
-
-
-  ## build a (reverse) team lookup by name
-  puts "#{data_teams['teams'].size} teams"
-
-  teams_by_name = data_teams['teams'].reduce( {} ) do |h,rec|
-     h[ rec['name'] ] = rec
-     h
-  end
-
-  pp teams_by_name.keys
-
-
-
-mods = MODS[ league.downcase ] || {}
-
-
-recs = []
-
-teams = Hash.new( 0 )
-stat = Stat.new
-
-matches = data[ 'matches' ]
-matches.each do |m|
-  stat.update( m )
-
-  team1 = m['homeTeam']['name']
-  team2 = m['awayTeam']['name']
-
-
-  score = m['score']
-
-
-
-  if m['stage'] == 'GROUP_STAGE'
-    stage = 'Group'
-    round = "Matchday #{m['matchday']}"
-    if m['group'] =~ /Group ([A-Z])/
-      group = $1
-    else
-      puts "!! ERROR - no group name found for group >#{m['group']}<"
-      exit 1
-    end
-  else
-    stage = STAGE_TO_STAGE[ m['stage'] ]
-    if stage.nil?
-      puts "!! ERROR - no stage mapping found for stage >#{m['stage']}<"
-      exit 1
-    end
-    round = STAGE_TO_ROUND[ m['stage'] ]
-    if round.nil?
-      puts "!! ERROR - no round mapping found for stage >#{m['stage']}<"
-      exit 1
-    end
-    group = ''
-  end
-
-
-    teams[ team1 ] += 1
-    teams[ team2 ] += 1
-
-    ### mods - rename club names
-    unless mods.nil? || mods.empty?
-      team1 = mods[ team1 ]      if mods[ team1 ]
-      team2 = mods[ team2 ]      if mods[ team2 ]
-    end
-
-
-    ## e.g. "utcDate": "2020-05-09T00:00:00Z",
-    date_str = m['utcDate']
-    date = DateTime.strptime( date_str, '%Y-%m-%dT%H:%M:%SZ' )
-
-
-    comments = ''
-    ft       = ''
-    ht       = ''
-    et       = ''
-    pen      = ''
-
-    case m['status']
-    when 'SCHEDULED', 'IN_PLAY'
-      ft  = ''
-      ht  = ''
-      et  = ''
-      pen = ''
-    when 'AWARDED'
-      ## todo/fix: assert duration == "REGULAR"
-      ft = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
-      ft << ' (*)'
-      ht = ''
-      comments = 'awarded'
-    when 'FINISHED'
-      ## note: if extraTime present
-      ## than fullTime is extraTime score!!
-      ##   AND   fullTime - extraTime is fullTime score!!
-      ## double check in other season too??
-      ##   - checked in cl 2018/19
-
-      if score['extraTime']['homeTeam'] && score['extraTime']['awayTeam']
-        et = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
-        ft = "#{score['fullTime']['homeTeam']-score['extraTime']['homeTeam']}-#{score['fullTime']['awayTeam']-score['extraTime']['awayTeam']}"
-      else
-        ft  = "#{score['fullTime']['homeTeam']}-#{score['fullTime']['awayTeam']}"
-      end
-
-      ht  = "#{score['halfTime']['homeTeam']}-#{score['halfTime']['awayTeam']}"
-
-      pen = if score['penalties']['homeTeam'] && score['penalties']['awayTeam']
-              "#{score['penalties']['homeTeam']}-#{score['penalties']['awayTeam']}"
-            else
-              ''
-            end
-    else
-      puts "!! ERROR: unsupported match status >#{m['status']}< - sorry:"
-      pp m
-      exit 1
-    end
-
-
-    recs << [stage,
-             round,
-             group,
-             date.to_date.strftime( '%Y-%m-%d' ),
-             team1,
-             ft,
-             ht,
-             team2,
-             et,
-             pen,
-             comments
-            ]
-
-
-    print '%2s' % m['matchday']
-    print ' - '
-    print '%-24s' % team1
-    print '  '
-    print ft
-    print ' '
-    print "(#{ht})"    unless ht.empty?
-    print '  '
-    print '%-24s' % team2
-    print '  '
-    print comments
-    print ' | '
-    ## print date.to_date  ## strip time
-    print date.to_date.strftime( '%a %b %-d %Y' )
-    print ' -- '
-    print date
-    print "\n"
-
-end # each match
-
-
-## note: get season from first match
-##   assert - all other matches include the same season
-## e.g.
-# "season": {
-#  "id": 154,
-#  "startDate": "2018-08-03",
-#  "endDate": "2019-05-05",
-#  "currentMatchday": 46
-# }
-
-start_date = Date.strptime( matches[0]['season']['startDate'], '%Y-%m-%d' )
-end_date   = Date.strptime( matches[0]['season']['endDate'],   '%Y-%m-%d' )
-
-dates = "#{start_date.strftime('%b %-d')} - #{end_date.strftime('%b %-d')}"
-
-buf = ''
-buf << "#{season.key} (#{dates}) - "
-buf << "#{teams.keys.size} clubs, "
-buf << "#{stat[:all][:matches]} matches, "
-buf << "#{stat[:all][:goals]} goals"
-buf << "\n"
-
-puts buf
-
-
-
-# recs = recs.sort { |l,r| l[1] <=> r[1] }
-## reformat date / beautify e.g. Sat Aug 7 1993
-recs.each { |rec| rec[3] = Date.strptime( rec[3], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
-
-headers = [
-  'Stage',
-  'Round',
-  'Group',
-  'Date',
-  'Team 1',
-  'FT',
-  'HT',
-  'Team 2',
-  'ET',
-  'P',
-  'Comments'
-]
-
-## note: change season_key from 2019/20 to 2019-20  (for path/directory!!!!)
-Cache::CsvMatchWriter.write( "#{config.convert.out_dir}/#{season.to_path}/#{league.downcase}.csv",
-                             recs,
-                             headers: headers )
-
-
-teams.each do |name, count|
-  rec = teams_by_name[ name ]
-  print "  #{count}x  "
-  print name
-  if rec
-    print " | #{rec['shortName']} "   if name != rec['shortName']
-    print " › #{rec['area']['name']}"
-    print "  - #{rec['address']}"
-  else
-    print "!! ERROR - no team record found in teams.json for #{name}"
-    exit 1
-  end
-  print "\n"
-end
-
-pp stat
-end   # method convert_cl
-end #  module Footballdata
-
-

data/lib/football-sources/apis/mods.rb

@@ -1,20 +0,0 @@
-module Footballdata
-
-  #########
-  ## Mods
-  # e.g.
-  # Cardiff City FC | Cardiff  › Wales  - Cardiff City Stadium, Leckwith Road Cardiff CF11 8AZ
-  # AS Monaco FC | Monaco  › Monaco     - Avenue des Castellans Monaco 98000
-
-
-  MODS = {
-    'br.1' => {
-           'América FC' => 'América MG',   # in year 2018
-              },
-    'pt.1'  => {
-           'Vitória SC' => 'Vitória Guimarães',  ## avoid easy confusion with Vitória SC <=> Vitória FC
-           'Vitória FC' => 'Vitória Setúbal',
-             },
-  }
-
-end  # module Footballdata