worldfootball 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 170277c7714f9b75e93176eb5fff6242fb6efb85051bc8977f7f635dbebf0513
+  data.tar.gz: 5b15d132765c3ee2df4cbdd847b43ba6c7e7bd617ed55bb5fe4dfabc0becb0e0
+SHA512:
+  metadata.gz: c95b4b2becf545be2c208a207e8980d06369148d18b0657cddfb81470331c828ee8492649908ece372fb996b9d1a6dfc8eeaa45c54a3757eba7fb6d02e363bf0
+  data.tar.gz: ca127cb3f69c861dba48b049ed6da30b9ecde1f3935b30422d6184869b650b5cafc056be7fa37c7198bbd72ad65bdd262e56c82631c62629f4bbb4222992a9c9

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+### 0.1.1
+
+### 0.0.1 / 2024-07-04
+
+* Everything is new. First release.
+

data/Manifest.txt

@@ -0,0 +1,29 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+bin/wfb
+lib/worldfootball.rb
+lib/worldfootball/build.rb
+lib/worldfootball/convert.rb
+lib/worldfootball/convert_reports.rb
+lib/worldfootball/download.rb
+lib/worldfootball/generator.rb
+lib/worldfootball/leagues.rb
+lib/worldfootball/leagues/asia.rb
+lib/worldfootball/leagues/europe--british_isles.rb
+lib/worldfootball/leagues/europe--central.rb
+lib/worldfootball/leagues/europe--eastern.rb
+lib/worldfootball/leagues/europe--northern.rb
+lib/worldfootball/leagues/europe--southern.rb
+lib/worldfootball/leagues/europe--western.rb
+lib/worldfootball/leagues/europe.rb
+lib/worldfootball/leagues/north_america.rb
+lib/worldfootball/leagues/pacific.rb
+lib/worldfootball/leagues/south_america.rb
+lib/worldfootball/mods.rb
+lib/worldfootball/page.rb
+lib/worldfootball/page_report.rb
+lib/worldfootball/page_schedule.rb
+lib/worldfootball/vacuum.rb
+lib/worldfootball/version.rb

data/README.md

@@ -0,0 +1,27 @@
+# worldfootball  - get world football (leagues, cups & more) match data via the worldfootball.net/weltfussball.de pages
+ 
+
+* home  :: [github.com/sportdb/sport.db](https://github.com/sportdb/sport.db)
+* bugs  :: [github.com/sportdb/sport.db/issues](https://github.com/sportdb/sport.db/issues)
+* gem   :: [rubygems.org/gems/worldfootball](https://rubygems.org/gems/worldfootball)
+* rdoc  :: [rubydoc.info/gems/worldfootball](http://rubydoc.info/gems/worldfootball)
+
+
+
+## Usage
+
+
+To be done
+
+
+
+## License
+
+The `worldfootball` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+
+## Questions? Comments?
+
+Yes, you can. More than welcome.
+See [Help & Support »](https://github.com/openfootball/help)

data/Rakefile

@@ -0,0 +1,34 @@
+require 'hoe'
+require './lib/worldfootball/version.rb'
+
+Hoe.spec 'worldfootball' do
+
+  self.version = Worldfootball::VERSION
+
+  self.summary = "worldfootball - get world football (leagues, cups & more) match data via the worldfootball.net/weltfussball.de pages"
+  self.description = summary
+
+  self.urls = { home: 'https://github.com/sportdb/sport.db' }
+
+  self.author = 'Gerald Bauer'
+  self.email  = 'gerald.bauer@gmail.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.extra_deps = [
+    ## ['tzinfo'],
+    ['season-formats'], 
+    ['webget'],  
+    ['nokogiri'],
+    ['cocos'],  ## later pull in with sportsdb-writers
+  ]
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+   required_ruby_version: '>= 2.2.2'
+  }
+
+end

data/bin/wfb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfb
+
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else 
+                     './cache'
+                 end
+
+
+require 'optparse'
+
+
+module Worldfootball
+def self.main( args=ARGV )    
+
+opts = {}
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+##
+## check if git has a offline option?? (use same)
+##             check for other tools - why? why not?
+
+
+  parser.on( "--cache", "--cached", "--offline",
+               "use cached data in #{Webcache.root}" ) do |cached|
+    opts[:cached] = cached
+  end
+end
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+if ['league', 'leagues', 'ls'].include?( args[0] || 'leagues' )
+    keys = LEAGUES.keys
+     
+    pp keys
+    puts "   #{keys.size} league(s)"
+    
+    # puts
+    # puts " pages:"
+    # pp Worldfootball::PAGES
+
+    exit 0
+end
+
+
+
+league_code = args[0].downcase
+
+league =  find_league( league_code )  ## league info lookup
+ 
+season = Season(  args[1] || '2024/25' )
+
+pages = league.pages( season: season )
+
+pp pages
+puts "   #{pages.size} page(s)"
+
+
+## wrap single page record into array
+pages = pages.is_a?( Array ) ? pages : [pages]
+
+if opts[:cached]  
+     # do nothing
+else  ## download to cache
+    pages.each_with_index do |page_rec,i|
+        slug = page_rec[:slug]
+        puts "==> #{i+1}/#{pages.size} - #{league_code} @ #{slug}..."    
+        page = Metal.download_schedule( slug  )  
+    end  
+end
+
+
+pages.each_with_index do |page_rec,i|
+    slug = page_rec[:slug]
+
+    puts "==> #{i+1}/#{pages.size} - #{league_code} @ #{slug}..."
+    page = Page::Schedule.from_cache( slug )
+    matches = page.matches
+
+    puts "   #{matches.size} match(es)"
+end
+
+
+end   # def self.main
+end   # module Worldfootball
+    
+
+Worldfootball.main( ARGV )
+
+        
+puts "bye"

data/lib/worldfootball/build.rb

@@ -0,0 +1,245 @@
+
+module Worldfootball
+
+
+ROUND_TO_EN = {
+  '1. Runde'      => 'Round 1',
+  '2. Runde'      => 'Round 2',
+  '3. Runde'      => 'Round 3',
+  '4. Runde'      => 'Round 4',
+  'Achtelfinale'  => 'Round of 16',
+  'Viertelfinale' => 'Quarterfinals',
+  'Halbfinale'    => 'Semifinals',
+  'Finale'        => 'Final',
+}
+
+
+## todo/check:  english league cup/trophy has NO ET - also support - make more flexible!!!
+
+## build "standard" match records from "raw" table rows
+def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such - why? why not?
+   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+   raise ArgumentError, "league key as string expected"  unless league.is_a?(String)  ## note: do NOT pass in league struct! pass in key (string)
+
+   print "  #{rows.size} rows - build #{league} #{season}"
+   print " - #{stage}" unless stage.empty?
+   print "\n"
+
+
+   ## note: use only first part from key for lookup
+   ##    e.g. at.1  => at
+   ##         eng.1 => eng
+   ##     and so on
+   mods = MODS[ league.split('.')[0] ] || {}
+
+   score_errors = SCORE_ERRORS[ league ] || {}
+
+
+   i = 0
+   recs = []
+   rows.each do |row|
+     i += 1
+
+
+  if row[:round] =~ /Spieltag/
+    puts
+    print '[%03d] ' % (i+1)
+    print row[:round]
+
+    if m = row[:round].match( /([0-9]+)\. Spieltag/ )
+      ## todo/check: always use a string even if number (as a string eg. '1' etc.)
+      round = m[1]  ## note: keep as string (NOT number)
+      print " => #{round}"
+    else
+      puts "!! ERROR: cannot find matchday number"
+      exit 1
+    end
+    print "\n"
+  elsif row[:round] =~ /[1-9]\.[ ]Runde|
+                          Achtelfinale|
+                          Viertelfinale|
+                          Halbfinale|
+                          Finale
+                          /x
+    puts
+    print '[%03d] ' % (i+1)
+    print row[:round]
+
+
+    ## do NOT translate rounds (to english) - keep in german / deutsch (de)
+    if ['at.cup', 'at.1',    ## at.1 - incl. europa league playoff
+        'de.cup'].include?( league )
+      round = row[:round]
+    else
+      round = ROUND_TO_EN[ row[:round] ]
+      if round.nil?
+        puts "!! ERROR: no mapping for round to english (en) found >#{row[:round]}<:"
+        pp row
+        exit 1
+      end
+      print " => #{round}"
+    end
+    print "\n"
+  else
+    puts "!! ERROR: unknown round >#{row[:round]}< for league >#{league}<:"
+    pp row
+    exit 1
+  end
+
+
+    date_str  = row[:date]
+    time_str  = row[:time]
+    team1_str = row[:team1]
+    team2_str = row[:team2]
+    score_str = row[:score]
+
+    ## convert date from string e.g. 2019-25-10
+    date = Date.strptime( date_str, '%Y-%m-%d' )
+
+
+    ### check for score_error; first (step 1) lookup by date
+    score_error = score_errors[ date.strftime('%Y-%m-%d') ]
+    if score_error
+      if team1_str == score_error[0] &&
+         team2_str == score_error[1]
+         ## check if team names match too; if yes, apply fix/patch!!
+         if score_str != score_error[2][0]
+           puts "!! WARN - score fix changed? - expected #{score_error[2][0]}, got #{score_str} - fixing to #{score_error[2][1]}"
+           pp row
+         end
+         puts "FIX - applying score error fix - from #{score_error[2][0]} to => #{score_error[2][1]}"
+         score_str = score_error[2][1]
+      end
+    end
+
+
+    print '[%03d]    ' % (i+1)
+    print "%-10s | " % date_str
+    print "%-5s | "  % time_str
+    print "%-22s | " % team1_str
+    print "%-22s | " % team2_str
+    print score_str
+    print "\n"
+
+
+    ## check for 0:3 Wert.   - change Wert. to awd.  (awarded)
+    score_str = score_str.sub( /Wert\./i, 'awd.' )
+
+    ## clean team name (e.g. remove (old))
+    ##   and asciify (e.g. ’ to ' )
+    team1_str = norm_team( team1_str )
+    team2_str = norm_team( team2_str )
+
+    team1_str = mods[ team1_str ]   if mods[ team1_str ]
+    team2_str = mods[ team2_str ]   if mods[ team2_str ]
+
+
+
+
+    ht, ft, et, pen, comments = parse_score( score_str )
+
+
+
+    recs <<  [stage,
+              round,
+              date.strftime( '%Y-%m-%d' ),
+              time_str,
+              team1_str,
+              ft,
+              ht,
+              team2_str,
+              et,              # extra: incl. extra time
+              pen,             # extra: incl. penalties
+              comments]
+   end  # each row
+   recs
+end  # build
+
+
+
+def self.parse_score( score_str )
+  comments = String.new( '' )     ## check - rename to/use status or such - why? why not?
+
+  ## split score
+  ft  = ''
+  ht  = ''
+  et  = ''
+  pen = ''
+  if score_str == '---'   ## in the future (no score yet) - was -:-
+    ft = ''
+    ht = ''
+  elsif score_str == 'n.gesp.' ||   ## cancelled (british) / canceled (us)
+        score_str == 'ausg.'   ||   ## todo/check: change to some other status ????
+        score_str == 'annull.'      ## todo/check: change to some other status (see ie 2012) ????
+    ft = '(*)'
+    ht = ''
+    comments = 'cancelled'
+  elsif score_str == 'abgebr.'  ## abandoned  -- waiting for replay?
+    ft = '(*)'
+    ht = ''
+    comments = 'abandoned'
+  elsif score_str == 'verl.'   ## postponed
+    ft = ''
+    ht = ''
+    comments = 'postponed'
+  # 5-4 (0-0, 1-1, 2-2) i.E.
+  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*
+                      \(([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*,[ ]*
+                        ([0-9]+) [ ]*-[ ]* ([0-9]+)
+                          [ ]*,[ ]*
+                       ([0-9]+) [ ]*-[ ]* ([0-9]+)\)
+                          [ ]*
+                       i\.E\.
+                     /x
+    pen = "#{$1}-#{$2}"
+    ht  = "#{$3}-#{$4}"
+    ft  = "#{$5}-#{$6}"
+    et  = "#{$7}-#{$8}"
+  # 2-1 (1-0, 1-1) n.V
+  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
+                      [ ]*
+                    \(([0-9]+) [ ]*-[ ]* ([0-9]+)
+                       [ ]*,[ ]*
+                      ([0-9]+) [ ]*-[ ]* ([0-9]+)
+                      \)
+                       [ ]*
+                       n\.V\.
+                     /x
+    et  = "#{$1}-#{$2}"
+    ht  = "#{$3}-#{$4}"
+    ft  = "#{$5}-#{$6}"
+  elsif score_str =~ /([0-9]+)
+                          [ ]*-[ ]*
+                      ([0-9]+)
+                          [ ]*
+                      \(([0-9]+)
+                          [ ]*-[ ]*
+                        ([0-9]+)
+                      \)
+                     /x
+    ft = "#{$1}-#{$2}"
+    ht = "#{$3}-#{$4}"
+  elsif  score_str =~ /([0-9]+)
+                         [ ]*-[ ]*
+                       ([0-9]+)
+                         [ ]*
+                        ([a-z.]+)
+                       /x
+    ft = "#{$1}-#{$2} (*)"
+    ht = ''
+    comments = $3
+  elsif score_str =~ /^([0-9]+)-([0-9]+)$/
+     ft = "#{$1}-#{$2}"     ## e.g. see luxemburg and others
+     ht = ''
+  else
+     puts "!! ERROR - unsupported score format >#{score_str}< - sorry; maybe add a score error fix/patch"
+     exit 1
+  end
+
+  [ht, ft, et, pen, comments]
+end
+
+end # module Worldfootball

data/lib/worldfootball/convert.rb

@@ -0,0 +1,162 @@
+
+module Worldfootball
+
+
+#################
+# todo/fix -  use timezone instead of offset !!!
+#  e.g
+=begin
+   TIMEZONES = {
+  'eng.1' => 'Europe/London',  
+  'eng.2' => 'Europe/London',
+
+  'es.1'  => 'Europe/Madrid',
+
+  'de.1'  => 'Europe/Berlin',
+  'fr.1'  => 'Europe/Paris', 
+  'it.1'  => 'Europe/Rome',
+  'nl.1'  => 'Europe/Amsterdam',
+
+  'pt.1'  => 'Europe/Lisbon',   
+
+  ## todo/fix - pt.1
+  ##  one team in madeira!!! check for different timezone??
+  ##  CD Nacional da Madeira 
+
+  'br.1'  => 'America/Sao_Paulo',
+  ## todo/fix - brazil has 4 timezones
+  ##           really only two in use for clubs
+  ##             west and east (amazonas et al)
+  ##           for now use west for all - why? why not?
+}
+=end
+
+## todo - find "proper/classic" timezone ("winter time")
+
+##  Brasilia - Distrito Federal, Brasil  (GMT-3)  -- summer time?
+##  Ciudad de México, CDMX, México       (GMT-5)  -- summer time?
+##  Londres, Reino Unido (GMT+1)
+##   Madrid -- ?
+##   Lisboa -- ?
+##   Moskow -- ?
+##
+## todo/check - quick fix timezone offsets for leagues for now
+##   - find something better - why? why not?
+## note: assume time is in GMT+1
+OFFSETS = {
+  'eng.1' => -1,
+  'eng.2' => -1,
+  'eng.3' => -1,
+  'eng.4' => -1,
+  'eng.5' => -1,
+
+  'es.1'  => -1,
+  'es.2'  => -1,
+
+  'pt.1'  => -1,
+  'pt.2'  => -1,
+
+  'br.1'  => -5,
+  'mx.1'  => -7,
+}
+
+
+def self.convert( league:, season: )   
+  season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league = find_league( league )
+
+  pages = league.pages( season: season )
+
+  ## check: rename (optional) offset to time_offset or such?
+  offset = OFFSETS[ league ]
+
+
+  # note: assume stages if pages is an array (of hash table/records)
+  #         (and NOT a single hash table/record)
+  if pages.is_a?(Array)
+    recs = []
+    pages.each do |page_meta|
+      slug       = page_meta[:slug]
+      stage_name = page_meta[:stage]
+      ## todo/fix: report error/check if stage.name is nil!!!
+
+      print "  parsing #{slug}..."
+
+      # unless File.exist?( path )
+      #  puts "!! WARN - missing stage >#{stage_name}< source - >#{path}<"
+      #  next
+      # end
+
+      page = Page::Schedule.from_cache( slug )
+      print "  title=>#{page.title}<..."
+      print "\n"
+
+      rows = page.matches
+      stage_recs = build( rows, season: season, league: league.key, stage: stage_name )
+
+      pp stage_recs[0]   ## check first record
+      recs += stage_recs
+    end
+  else
+    page_meta = pages
+    slug = page_meta[:slug]
+
+    print "  parsing #{slug}..."
+
+    page = Page::Schedule.from_cache( slug )
+    print "  title=>#{page.title}<..."
+    print "\n"
+
+    rows = page.matches
+    recs = build( rows, season: season, league: league.key )
+
+    pp recs[0]   ## check first record
+  end
+
+  recs = recs.map { |rec| fix_date( rec, offset ) }    if offset
+
+##   note:  sort matches by date before saving/writing!!!!
+##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
+## note: assume date is third column!!! (stage/round/date/...)
+recs = recs.sort { |l,r| l[2] <=> r[2] }
+## reformat date / beautify e.g. Sat Aug 7 1993
+recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+
+   ## remove unused columns (e.g. stage, et, p, etc.)
+   recs, headers = vacuum( recs )
+
+   puts headers
+   pp recs[0]   ## check first record
+
+   out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
+
+   puts "write #{out_path}..."
+   write_csv( out_path, recs, headers: headers )
+end
+
+
+
+## helper to fix dates to use local timezone (and not utc/london time)
+def self.fix_date( row, offset )
+  return row    if row[3].nil? || row[3].empty?   ## note: time (column) required for fix
+
+  col = row[2]
+  if col =~ /^\d{4}-\d{2}-\d{2}$/
+    date_fmt = '%Y-%m-%d'   # e.g. 2002-08-17
+  else
+    puts "!!! ERROR - wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
+    ## todo/fix: add to errors/warns list - why? why not?
+    exit 1
+  end
+
+  date = DateTime.strptime( "#{row[2]} #{row[3]}", "#{date_fmt} %H:%M" )
+  ## NOTE - MUST be -7/24.0!!!! or such to work
+  date = date + (offset/24.0)
+
+  row[2] = date.strftime( date_fmt )  ## overwrite "old"
+  row[3] = date.strftime( '%H:%M' )
+  row   ## return row for possible pipelining - why? why not?
+end
+
+end # module Worldfootball