worldfootball 0.1.1 → 0.2.0

data/lib/worldfootball/build.rb

@@ -7,6 +7,11 @@ ROUND_TO_EN = {
   '2. Runde'      => 'Round 2',
   '3. Runde'      => 'Round 3',
   '4. Runde'      => 'Round 4',
+  '5. Runde'      => 'Round 5',
+  '6. Runde'      => 'Round 6',
+  '7. Runde'      => 'Round 7',
+  '8. Runde'      => 'Round 8',
+  '9. Runde'      => 'Round 9',
   'Achtelfinale'  => 'Round of 16',
   'Viertelfinale' => 'Quarterfinals',
   'Halbfinale'    => 'Semifinals',
@@ -20,13 +25,17 @@ ROUND_TO_EN = {
 def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such - why? why not?
    season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
 
-   raise ArgumentError, "league key as string expected"  unless league.is_a?(String)  ## note: do NOT pass in league struct! pass in key (string)
+   ## note: do NOT pass in league struct! pass in key (string)
+   raise ArgumentError, "league key as string expected"  unless league.is_a?(String)
 
-   print "  #{rows.size} rows - build #{league} #{season}"
+   print "  #{rows.size} row(s) - Worldfootball.build #{league} #{season}"
    print " - #{stage}" unless stage.empty?
    print "\n"
 
 
+   zone = find_zone!( league: league, season: season )
+
+
    ## note: use only first part from key for lookup
    ##    e.g. at.1  => at
    ##         eng.1 => eng
@@ -47,44 +56,49 @@ def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such
     print '[%03d] ' % (i+1)
     print row[:round]
 
-    if m = row[:round].match( /([0-9]+)\. Spieltag/ )
+    if (m = row[:round].match( /^(?<num>[0-9]+)\. Spieltag$/ ))
       ## todo/check: always use a string even if number (as a string eg. '1' etc.)
-      round = m[1]  ## note: keep as string (NOT number)
+      round = m[:num]  ## note: keep as string (NOT number)
       print " => #{round}"
     else
-      puts "!! ERROR: cannot find matchday number"
+      puts "!! ERROR: cannot find matchday number in >#{row[:round]}<:"
+      pp row
       exit 1
     end
     print "\n"
-  elsif row[:round] =~ /[1-9]\.[ ]Runde|
+
+  ## note - must start line e.g.
+  ##            do NOT match => Qual. 1. Runde  (1. Runde)!!!
+  elsif row[:round] =~ /^(
+                          [1-9]\.[ ]Runde|
                           Achtelfinale|
                           Viertelfinale|
                           Halbfinale|
                           Finale
-                          /x
+                         )$
+                        /x
     puts
     print '[%03d] ' % (i+1)
     print row[:round]
 
+    round = ROUND_TO_EN[ row[:round] ]
+    print " => #{round}"
+    print "\n"
 
-    ## do NOT translate rounds (to english) - keep in german / deutsch (de)
-    if ['at.cup', 'at.1',    ## at.1 - incl. europa league playoff
-        'de.cup'].include?( league )
-      round = row[:round]
-    else
-      round = ROUND_TO_EN[ row[:round] ]
-      if round.nil?
-        puts "!! ERROR: no mapping for round to english (en) found >#{row[:round]}<:"
-        pp row
-        exit 1
-      end
-      print " => #{round}"
+    if round.nil?
+      puts "!! ERROR: no mapping for round to english (en) found >#{row[:round]}<:"
+      pp row
+      exit 1
     end
-    print "\n"
   else
-    puts "!! ERROR: unknown round >#{row[:round]}< for league >#{league}<:"
+    puts
+    print '[%03d] ' % (i+1)
+    print row[:round]
+    print "\n"
+
+    puts "!! WARN: unknown round >#{row[:round]}< for league >#{league}<:"
     pp row
-    exit 1
+    round = row[:round]
   end
 
 
@@ -94,12 +108,10 @@ def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such
     team2_str = row[:team2]
     score_str = row[:score]
 
-    ## convert date from string e.g. 2019-25-10
-    date = Date.strptime( date_str, '%Y-%m-%d' )
 
 
     ### check for score_error; first (step 1) lookup by date
-    score_error = score_errors[ date.strftime('%Y-%m-%d') ]
+    score_error = score_errors[ date_str ]
     if score_error
       if team1_str == score_error[0] &&
          team2_str == score_error[1]
@@ -123,8 +135,6 @@ def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such
     print "\n"
 
 
-    ## check for 0:3 Wert.   - change Wert. to awd.  (awarded)
-    score_str = score_str.sub( /Wert\./i, 'awd.' )
 
     ## clean team name (e.g. remove (old))
     ##   and asciify (e.g. ’ to ' )
@@ -135,111 +145,48 @@ def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such
     team2_str = mods[ team2_str ]   if mods[ team2_str ]
 
 
+    ht, ft, et, pen, comments = parse_score( score_str )
+
 
+   ###################
+   ### calculate date & times
+   ## convert date from string e.g. 2019-25-10
+   ## date = Date.strptime( date_str, '%Y-%m-%d' )
 
-    ht, ft, et, pen, comments = parse_score( score_str )
+   if time_str.nil? || time_str.empty?
+       ## no time
+       ##   assume  00:00:00T
+       time_str     = ''
+       timezone     = ''
+       utc          = ''
+   else
+      ## note - assume central european (summer) time (cet/cest) - UTC+1 or UTC+2
+      cet = CET.strptime( "#{date_str} #{time_str}", '%Y-%m-%d %H:%M' )
 
+      utc = cet.getutc   ## convert to utc
+      local =  zone.to_local( utc )  # convert to local via utc
+      ## overwrite old with local
+      date_str = local.strftime( '%Y-%m-%d' )
+      time_str = local.strftime( '%H:%M' )
+      timezone = local.strftime( '%Z/%z' )
+      utc      = utc.strftime( '%Y-%m-%dT%H:%MZ' )
+   end
 
 
     recs <<  [stage,
               round,
-              date.strftime( '%Y-%m-%d' ),
+              date_str,
               time_str,
+              timezone,
               team1_str,
               ft,
               ht,
               team2_str,
               et,              # extra: incl. extra time
               pen,             # extra: incl. penalties
-              comments]
+              comments,
+              utc]
    end  # each row
    recs
 end  # build
-
-
-
-def self.parse_score( score_str )
-  comments = String.new( '' )     ## check - rename to/use status or such - why? why not?
-
-  ## split score
-  ft  = ''
-  ht  = ''
-  et  = ''
-  pen = ''
-  if score_str == '---'   ## in the future (no score yet) - was -:-
-    ft = ''
-    ht = ''
-  elsif score_str == 'n.gesp.' ||   ## cancelled (british) / canceled (us)
-        score_str == 'ausg.'   ||   ## todo/check: change to some other status ????
-        score_str == 'annull.'      ## todo/check: change to some other status (see ie 2012) ????
-    ft = '(*)'
-    ht = ''
-    comments = 'cancelled'
-  elsif score_str == 'abgebr.'  ## abandoned  -- waiting for replay?
-    ft = '(*)'
-    ht = ''
-    comments = 'abandoned'
-  elsif score_str == 'verl.'   ## postponed
-    ft = ''
-    ht = ''
-    comments = 'postponed'
-  # 5-4 (0-0, 1-1, 2-2) i.E.
-  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
-                          [ ]*
-                      \(([0-9]+) [ ]*-[ ]* ([0-9]+)
-                          [ ]*,[ ]*
-                        ([0-9]+) [ ]*-[ ]* ([0-9]+)
-                          [ ]*,[ ]*
-                       ([0-9]+) [ ]*-[ ]* ([0-9]+)\)
-                          [ ]*
-                       i\.E\.
-                     /x
-    pen = "#{$1}-#{$2}"
-    ht  = "#{$3}-#{$4}"
-    ft  = "#{$5}-#{$6}"
-    et  = "#{$7}-#{$8}"
-  # 2-1 (1-0, 1-1) n.V
-  elsif score_str =~ /([0-9]+) [ ]*-[ ]* ([0-9]+)
-                      [ ]*
-                    \(([0-9]+) [ ]*-[ ]* ([0-9]+)
-                       [ ]*,[ ]*
-                      ([0-9]+) [ ]*-[ ]* ([0-9]+)
-                      \)
-                       [ ]*
-                       n\.V\.
-                     /x
-    et  = "#{$1}-#{$2}"
-    ht  = "#{$3}-#{$4}"
-    ft  = "#{$5}-#{$6}"
-  elsif score_str =~ /([0-9]+)
-                          [ ]*-[ ]*
-                      ([0-9]+)
-                          [ ]*
-                      \(([0-9]+)
-                          [ ]*-[ ]*
-                        ([0-9]+)
-                      \)
-                     /x
-    ft = "#{$1}-#{$2}"
-    ht = "#{$3}-#{$4}"
-  elsif  score_str =~ /([0-9]+)
-                         [ ]*-[ ]*
-                       ([0-9]+)
-                         [ ]*
-                        ([a-z.]+)
-                       /x
-    ft = "#{$1}-#{$2} (*)"
-    ht = ''
-    comments = $3
-  elsif score_str =~ /^([0-9]+)-([0-9]+)$/
-     ft = "#{$1}-#{$2}"     ## e.g. see luxemburg and others
-     ht = ''
-  else
-     puts "!! ERROR - unsupported score format >#{score_str}< - sorry; maybe add a score error fix/patch"
-     exit 1
-  end
-
-  [ht, ft, et, pen, comments]
-end
-
 end # module Worldfootball

data/lib/worldfootball/cache.rb

@@ -0,0 +1,46 @@
+
+module Worldfootball
+
+
+def self.list_pages     ## todo/check - rename to/use list_cached_pages  
+    start_time = Time.now   ## todo: use Timer? t = Timer.start / stop / diff etc. - why? why not?
+
+    # pages = Dir.glob( './dl/at*' )
+    pages = Dir.glob( "#{Webcache.root}/www.weltfussball.de/alle_spiele/*.html" )
+    puts "  #{pages.size} page(s)"   #=> 576 pages
+    puts
+    
+       
+    leagues = {}
+    
+    pages.each do |path|
+       basename = File.basename( path, File.extname( path ) )
+       print "%-50s" % basename
+       print " => "
+    
+       page = Worldfootball.find_page( basename )
+       if page
+         league_key = page[:league]
+         season_key = page[:season]
+    
+         print "    "
+         print "%-12s"    % league_key
+         print "| %-10s"  % season_key
+         print "\n"
+    
+         seasons = leagues[league_key] ||= []
+         seasons << season_key   unless seasons.include?( season_key )
+       else
+         print "??"
+         print "\n"
+       end
+    end
+        
+    puts "  #{pages.size} page(s)"   #=> 576 pages
+    puts
+        
+    end_time = Time.now
+    diff_time = end_time - start_time
+    puts "convert_all: done in #{diff_time} sec(s)"
+end
+end # module Worldfootball

data/lib/worldfootball/config.rb

@@ -0,0 +1,85 @@
+module Worldfootball
+
+
+####
+# config for slug to local basename / directories
+#   e.g.
+#     aut-bundesliga-2023-2024  =>  austria/2023-24/1_bundesliga.txt
+
+## add (timezone) offset here too - why? why not?
+LEAGUE_SETUPS  = {
+  ## note - for now auto-generate  path via name (downcased)
+  ##         e.g. Belgium => /belgium
+
+  ## top five (europe)
+  'eng' => { code: 'eng', name: 'England'  },
+  'es'  =>  { code: 'esp', name: 'Spain' },
+  # 'fr'  =>  { code: 'fra', name: 'France' },
+  # 'de'  =>  { code: '???', name: 'Germany' },
+  'it'  =>  { code: 'ita', name: 'Italy' },
+
+
+  'be' =>  { code: 'bel', name: 'Belgium'   },
+  'at' =>  { code: 'aut', name: 'Austria'   },
+  'hu' =>  { code: 'hun', name: 'Hungary'   },
+
+  'tr' =>  { code: 'tur', name: 'Turkey' },
+  'nl' =>  { code: 'ned', name: 'Netherlands' },
+  'ch' =>  { code: 'sui',  name: 'Switzerland' },
+
+
+  'cz' =>  { code: 'cze', name: 'Czech Republic' },
+  'dk' =>  { code: 'den', name: 'Denmark' },
+  'fi' =>  { code: 'fin', name: 'Finland' },
+  'gr' =>  { code: 'gre', name: 'Greece' },
+
+  'ie' =>  { code: 'irl', name: 'Ireland' },
+  'sco' =>  { code: 'sco', name: 'Scotland' },
+
+  'lu' =>  { code: 'lux', name: 'Luxembourg' },
+  'pl' =>  { code: 'pol', name: 'Poland' },
+  'pt' =>  { code: 'por', name: 'Portugal' },
+  'ro' =>  { code: 'rou', name: 'Romania' },
+  'ru' =>  { code: 'rus', name: 'Russia' },
+  'se' =>  { code: 'swe', name: 'Sweden' },
+  'ua' =>  { code: 'ukr', name: 'Ukraine' },
+
+
+  'eg' =>  { code: 'egy', name: 'Egypt' },
+  'jp' =>  { code: 'jpn', name: 'Japan' },
+  'cn' =>  { code: 'chn', name: 'China' },
+
+  ## note - for now do NOT add United States to league name
+  ##     e.g. 1   - Major League Soccer
+  ##          2   - USL Championship
+  ##          cup - U.S. Open Cup
+  'us' =>  { code: 'usa', name: nil,  path: 'united-states' },
+
+  'mx' =>  { code: 'mex', name: 'Mexico'    },
+  'ar' =>  { code: 'arg', name: 'Argentina' },
+  'br' =>  { code: 'bra', name: 'Brazil' },
+
+   'uy' =>  { code: 'uru', name: 'Uruguay' },
+  'pe' =>  { code: 'per', name: 'Peru' },
+  'ec' =>  { code: 'ecu', name: 'Ecuador' },
+  'bo' =>  { code: 'bol', name: 'Bolivia' },
+  'cl' =>  { code: 'chi', name: 'Chile' },
+  'co' =>  { code: 'col', name: 'Colombia' },
+
+   'cr' =>  { code: 'crc', name: 'Costa Rica' },
+  'gt' =>  { code: 'gua', name: 'Guatemala' },
+  'hn' =>  { code: 'hon', name: 'Honduras' },
+  'sv' =>  { code: 'slv', name: 'El Salvador' },
+  'ni' =>  { code: 'nca', name: 'Nicaragua' },
+
+
+  ## int'l tournaments
+  'uefa.cl'      => { code: nil, name: 'UEFA',     path: 'europe' },
+  'uefa.el'      => { code: nil, name: 'UEFA',     path: 'europe' },
+  'concacaf.cl'  => { code: nil, name: nil,       path: 'north-america' },
+  'copa.l'       => { code: nil, name: nil,       path: 'south-america' },
+}
+
+
+
+end  # module Worldfootball

data/lib/worldfootball/convert.rb

@@ -2,84 +2,18 @@
 module Worldfootball
 
 
-#################
-# todo/fix -  use timezone instead of offset !!!
-#  e.g
-=begin
-   TIMEZONES = {
-  'eng.1' => 'Europe/London',  
-  'eng.2' => 'Europe/London',
-
-  'es.1'  => 'Europe/Madrid',
-
-  'de.1'  => 'Europe/Berlin',
-  'fr.1'  => 'Europe/Paris', 
-  'it.1'  => 'Europe/Rome',
-  'nl.1'  => 'Europe/Amsterdam',
-
-  'pt.1'  => 'Europe/Lisbon',   
-
-  ## todo/fix - pt.1
-  ##  one team in madeira!!! check for different timezone??
-  ##  CD Nacional da Madeira 
-
-  'br.1'  => 'America/Sao_Paulo',
-  ## todo/fix - brazil has 4 timezones
-  ##           really only two in use for clubs
-  ##             west and east (amazonas et al)
-  ##           for now use west for all - why? why not?
-}
-=end
-
-## todo - find "proper/classic" timezone ("winter time")
-
-##  Brasilia - Distrito Federal, Brasil  (GMT-3)  -- summer time?
-##  Ciudad de México, CDMX, México       (GMT-5)  -- summer time?
-##  Londres, Reino Unido (GMT+1)
-##   Madrid -- ?
-##   Lisboa -- ?
-##   Moskow -- ?
-##
-## todo/check - quick fix timezone offsets for leagues for now
-##   - find something better - why? why not?
-## note: assume time is in GMT+1
-OFFSETS = {
-  'eng.1' => -1,
-  'eng.2' => -1,
-  'eng.3' => -1,
-  'eng.4' => -1,
-  'eng.5' => -1,
-
-  'es.1'  => -1,
-  'es.2'  => -1,
-
-  'pt.1'  => -1,
-  'pt.2'  => -1,
-
-  'br.1'  => -5,
-  'mx.1'  => -7,
-}
-
-
-def self.convert( league:, season: )   
+def self.convert( league:, season: )
   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league = find_league( league )
+  league = find_league!( league )
+  pages  = league.pages!( season: season )
 
-  pages = league.pages( season: season )
 
-  ## check: rename (optional) offset to time_offset or such?
-  offset = OFFSETS[ league ]
-
-
-  # note: assume stages if pages is an array (of hash table/records)
-  #         (and NOT a single hash table/record)
-  if pages.is_a?(Array)
     recs = []
-    pages.each do |page_meta|
-      slug       = page_meta[:slug]
-      stage_name = page_meta[:stage]
-      ## todo/fix: report error/check if stage.name is nil!!!
+    pages.each do |slug, stage|
+      ## note: stage might be nil
+      ## todo/fix: report error/check if stage is nil!!!
+      stage ||= ''
 
       print "  parsing #{slug}..."
 
@@ -93,35 +27,24 @@ def self.convert( league:, season: )
       print "\n"
 
       rows = page.matches
-      stage_recs = build( rows, season: season, league: league.key, stage: stage_name )
+      stage_recs = build( rows,
+                          season: season,
+                          league: league.key,
+                          stage: stage )
 
       pp stage_recs[0]   ## check first record
       recs += stage_recs
     end
-  else
-    page_meta = pages
-    slug = page_meta[:slug]
-
-    print "  parsing #{slug}..."
-
-    page = Page::Schedule.from_cache( slug )
-    print "  title=>#{page.title}<..."
-    print "\n"
 
-    rows = page.matches
-    recs = build( rows, season: season, league: league.key )
-
-    pp recs[0]   ## check first record
-  end
-
-  recs = recs.map { |rec| fix_date( rec, offset ) }    if offset
 
 ##   note:  sort matches by date before saving/writing!!!!
 ##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
 ## note: assume date is third column!!! (stage/round/date/...)
 recs = recs.sort { |l,r| l[2] <=> r[2] }
 ## reformat date / beautify e.g. Sat Aug 7 1993
-recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+recs.each do |rec|
+         rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+       end
 
    ## remove unused columns (e.g. stage, et, p, etc.)
    recs, headers = vacuum( recs )
@@ -134,29 +57,5 @@ recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b
    puts "write #{out_path}..."
    write_csv( out_path, recs, headers: headers )
 end
-
-
-
-## helper to fix dates to use local timezone (and not utc/london time)
-def self.fix_date( row, offset )
-  return row    if row[3].nil? || row[3].empty?   ## note: time (column) required for fix
-
-  col = row[2]
-  if col =~ /^\d{4}-\d{2}-\d{2}$/
-    date_fmt = '%Y-%m-%d'   # e.g. 2002-08-17
-  else
-    puts "!!! ERROR - wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
-    ## todo/fix: add to errors/warns list - why? why not?
-    exit 1
-  end
-
-  date = DateTime.strptime( "#{row[2]} #{row[3]}", "#{date_fmt} %H:%M" )
-  ## NOTE - MUST be -7/24.0!!!! or such to work
-  date = date + (offset/24.0)
-
-  row[2] = date.strftime( date_fmt )  ## overwrite "old"
-  row[3] = date.strftime( '%H:%M' )
-  row   ## return row for possible pipelining - why? why not?
-end
-
 end # module Worldfootball
+

data/lib/worldfootball/download.rb

@@ -7,14 +7,9 @@ module Worldfootball
 def self.schedule( league:, season: )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league  = find_league( league )
-
-  pages =  league.pages( season: season )
-
-  ## if single (simple) page setup - wrap in array
-  pages = pages.is_a?(Array) ? pages : [pages]
-  pages.each do |page_meta|
-    Metal.download_schedule( page_meta[:slug] )
+  pages = find_league_pages!( league: league, season: season )
+  pages.each do |slug, _|
+    Metal.download_schedule( slug )
   end # each page
 end
 
@@ -22,14 +17,9 @@ end
 def self.reports( league:, season:, cache: true ) ## todo/check: rename to reports_for_schedule or such - why? why not?
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league  = find_league( league )
-
-  pages =  league.pages( season: season )
-
-  ## if single (simple) page setup - wrap in array
-  pages = pages.is_a?(Array) ? pages : [pages]
-  pages.each do |page_meta|
-    Metal.download_reports_for_schedule( page_meta[:slug], cache: cache )
+  pages = find_league_pages!( league: league, season: season )
+  pages.each do |slug, _|
+    Metal.download_reports_for_schedule( slug, cache: cache )
   end # each page
 end
 
@@ -41,7 +31,7 @@ end
 
 ## todo/check: put in Downloader namespace/class - why? why not?
 ##   or use Metal    - no "porcelain" downloaders / machinery
-class Metal 
+class Metal
 
   BASE_URL = 'https://www.weltfussball.de'
 
@@ -117,7 +107,7 @@ class Metal
     end
   end
 
-  
+
   def self.download_page( url )  ## get & record/save to cache
     response = Webget.page( url )  ## fetch (and cache) html page (via HTTP GET)