worldfootball 0.1.1 → 0.1.2

data/lib/worldfootball/cache.rb

@@ -0,0 +1,46 @@
+
+module Worldfootball
+
+
+def self.list_pages     ## todo/check - rename to/use list_cached_pages  
+    start_time = Time.now   ## todo: use Timer? t = Timer.start / stop / diff etc. - why? why not?
+
+    # pages = Dir.glob( './dl/at*' )
+    pages = Dir.glob( "#{Webcache.root}/www.weltfussball.de/alle_spiele/*.html" )
+    puts "  #{pages.size} page(s)"   #=> 576 pages
+    puts
+    
+       
+    leagues = {}
+    
+    pages.each do |path|
+       basename = File.basename( path, File.extname( path ) )
+       print "%-50s" % basename
+       print " => "
+    
+       page = Worldfootball.find_page( basename )
+       if page
+         league_key = page[:league]
+         season_key = page[:season]
+    
+         print "    "
+         print "%-12s"    % league_key
+         print "| %-10s"  % season_key
+         print "\n"
+    
+         seasons = leagues[league_key] ||= []
+         seasons << season_key   unless seasons.include?( season_key )
+       else
+         print "??"
+         print "\n"
+       end
+    end
+        
+    puts "  #{pages.size} page(s)"   #=> 576 pages
+    puts
+        
+    end_time = Time.now
+    diff_time = end_time - start_time
+    puts "convert_all: done in #{diff_time} sec(s)"
+end
+end # module Worldfootball

data/lib/worldfootball/config.rb

@@ -0,0 +1,220 @@
+module Worldfootball
+
+#################
+# todo/fix -  use timezone instead of offset !!!
+#  e.g
+=begin
+   TIMEZONES = {
+  'eng.1' => 'Europe/London',
+  'eng.2' => 'Europe/London',
+
+  'es.1'  => 'Europe/Madrid',
+
+  'de.1'  => 'Europe/Berlin',
+  'fr.1'  => 'Europe/Paris',
+  'it.1'  => 'Europe/Rome',
+  'nl.1'  => 'Europe/Amsterdam',
+
+  'pt.1'  => 'Europe/Lisbon',
+
+  ## todo/fix - pt.1
+  ##  one team in madeira!!! check for different timezone??
+  ##  CD Nacional da Madeira
+
+  'br.1'  => 'America/Sao_Paulo',
+  ## todo/fix - brazil has 4 timezones
+  ##           really only two in use for clubs
+  ##             west and east (amazonas et al)
+  ##           for now use west for all - why? why not?
+}
+=end
+
+## todo - find "proper/classic" timezone ("winter time")
+
+##  Brasilia - Distrito Federal, Brasil  (GMT-3)  -- summer time?
+##  Ciudad de México, CDMX, México       (GMT-5)  -- summer time?
+##  Londres, Reino Unido (GMT+1)
+##   Madrid -- ?
+##   Lisboa -- ?
+##   Moskow -- ?
+##
+## todo/check - quick fix timezone offsets for leagues for now
+##   - find something better - why? why not?
+## note: assume time is in GMT/UTC+1  - PLUS SUMMERTIME!!!
+##        todo/fix - consider summertime before conversion too!!!
+
+
+OFFSETS = {
+
+  ## fix - change to gmt/utc offset
+  ##        if offset == 1 (GMT/UTC+1)
+  ##           do NOTHING (default date/timezone of pages)
+
+  'eng' => 0,
+ # 'eng.1' => 0,
+ # 'eng.2' => 0,
+ # 'eng.3' => 0,
+ # 'eng.4' => 0,
+ # 'eng.5' => 0,
+
+  'ie'  =>  0,
+  'sco' =>  0,
+
+  'pt' => 0,
+ # 'pt.1'  => 0,
+ # 'pt.2'  => 0,
+
+  'fi' => 2, # +2
+  'gr' => 2, # +2
+  'ro' =>  2, # +2
+  'ua' =>  2, # +2
+
+  'ru' =>  3,   # +3
+  'tr'   => 3,  # +3 turkey time/moscow time
+
+
+  'us'      => -5,   # (gmt-5) new york
+
+  'mx'      => -6,
+ # 'mx.1'    => -6,
+ # 'mx.2'    => -6,
+ # 'mx.3'    => -6,
+ # 'mx.cup'  => -6,
+
+  'cr' => -6,    # gmt-6
+  'gt' => -6,    # gmt-6
+  'hn' => -6,    # gmt-6
+  'sv' => -6,    # gmt-6
+  'ni' => -6,    # gmt-6
+
+  'uy' =>  -3,     #   gmt-3
+  'pe' =>  -5,     #  gmt-5
+  'ec' =>  -5,     #  gmt-5
+  'co' =>  -5,     #  gmt-5
+  'bo' =>  -4,     #  gmt-4
+  'cl' =>  -4,     #  gmt-4
+
+  'br'  => -4,    # gmt-3  - change to -3?
+ # 'br.1'  => -4,
+  'ar'   => -4,   # gmt-3  - change to -3?
+ # 'ar.1'  => -4,
+
+
+  'eg' =>   3,  # +3  (gmt+3)
+  'jp' =>   9,  # +9  (gmt+9)
+  'cn' =>   7,  # +7  (gmt+7)
+
+
+   ## note - central european time (cet) - no need for date auto-fix
+   'at' => 1,
+   'de' => 1,
+   'ch' => 1,
+   'hu' => 1,
+    'cz' => 1,
+    'pl' => 1,
+    'nl' => 1,
+    'lu' => 1,
+    'be' => 1,
+    'dk' => 1,
+    'se' => 1,
+    'it' => 1,
+    'fr' => 1,
+    'es' => 1,
+    ##   see https://en.wikipedia.org/wiki/Time_in_Europe
+
+
+  ################
+  ## int'l tournaments
+  # 'uefa.cl'
+  # 'uefa.el'
+  'uefa.cl'   => 1,
+  'uefa.el'   => 1,
+
+  'concacaf.cl'  => -6,   ### use mx time
+  'copa.l'       => -4,    ### use brazil time
+}
+
+
+####
+# config for slug to local basename / directories
+#   e.g.
+#     aut-bundesliga-2023-2024  =>  austria/2023-24/1_bundesliga.txt
+
+
+## add (timezone) offset here too - why? why not?
+LEAGUE_SETUPS  = {
+  ## note - for now auto-generate  path via name (downcased)
+  ##         e.g. Belgium => /belgium
+
+  ## top five (europe)
+  'eng' => { code: 'eng', name: 'England'  },
+  'es'  =>  { code: 'esp', name: 'Spain' },
+  # 'fr'  =>  { code: 'fra', name: 'France' },
+  # 'de'  =>  { code: '???', name: 'Germany' },
+  'it'  =>  { code: 'ita', name: 'Italy' },
+
+
+  'be' =>  { code: 'bel', name: 'Belgium'   },
+  'at' =>  { code: 'aut', name: 'Austria'   },
+  'hu' =>  { code: 'hun', name: 'Hungary'   },
+
+  'tr' =>  { code: 'tur', name: 'Turkey' },
+  'nl' =>  { code: 'ned', name: 'Netherlands' },
+  'ch' =>  { code: 'sui',  name: 'Switzerland' },
+
+
+  'cz' =>  { code: 'cze', name: 'Czech Republic' },
+  'dk' =>  { code: 'den', name: 'Denmark' },
+  'fi' =>  { code: 'fin', name: 'Finland' },
+  'gr' =>  { code: 'gre', name: 'Greece' },
+
+  'ie' =>  { code: 'irl', name: 'Ireland' },
+  'sco' =>  { code: 'sco', name: 'Scotland' },
+
+  'lu' =>  { code: 'lux', name: 'Luxembourg' },
+  'pl' =>  { code: 'pol', name: 'Poland' },
+  'pt' =>  { code: 'por', name: 'Portugal' },
+  'ro' =>  { code: 'rou', name: 'Romania' },
+  'ru' =>  { code: 'rus', name: 'Russia' },
+  'se' =>  { code: 'swe', name: 'Sweden' },
+  'ua' =>  { code: 'ukr', name: 'Ukraine' },
+
+
+  'eg' =>  { code: 'egy', name: 'Egypt' },
+  'jp' =>  { code: 'jpn', name: 'Japan' },
+  'cn' =>  { code: 'chn', name: 'China' },
+
+  ## note - for now do NOT add United States to league name
+  ##     e.g. 1   - Major League Soccer
+  ##          2   - USL Championship
+  ##          cup - U.S. Open Cup
+  'us' =>  { code: 'usa', name: nil,  path: 'united-states' },
+
+  'mx' =>  { code: 'mex', name: 'Mexico'    },
+  'ar' =>  { code: 'arg', name: 'Argentina' },
+  'br' =>  { code: 'bra', name: 'Brazil' },
+
+   'uy' =>  { code: 'uru', name: 'Uruguay' },
+  'pe' =>  { code: 'per', name: 'Peru' },
+  'ec' =>  { code: 'ecu', name: 'Ecuador' },
+  'bo' =>  { code: 'bol', name: 'Bolivia' },
+  'cl' =>  { code: 'chi', name: 'Chile' },
+  'co' =>  { code: 'col', name: 'Colombia' },
+
+   'cr' =>  { code: 'crc', name: 'Costa Rica' },
+  'gt' =>  { code: 'gua', name: 'Guatemala' },
+  'hn' =>  { code: 'hon', name: 'Honduras' },
+  'sv' =>  { code: 'slv', name: 'El Salvador' },
+  'ni' =>  { code: 'nca', name: 'Nicaragua' },
+
+
+  ## int'l tournaments
+  'uefa.cl'      => { code: nil, name: 'UEFA',     path: 'europe' },
+  'uefa.el'      => { code: nil, name: 'UEFA',     path: 'europe' },
+  'concacaf.cl'  => { code: nil, name: nil,       path: 'north-america' },
+  'copa.l'       => { code: nil, name: nil,       path: 'south-america' },
+}
+
+
+
+end  # module Worldfootball

data/lib/worldfootball/convert.rb

@@ -2,84 +2,18 @@
 module Worldfootball
 
 
-#################
-# todo/fix -  use timezone instead of offset !!!
-#  e.g
-=begin
-   TIMEZONES = {
-  'eng.1' => 'Europe/London',  
-  'eng.2' => 'Europe/London',
-
-  'es.1'  => 'Europe/Madrid',
-
-  'de.1'  => 'Europe/Berlin',
-  'fr.1'  => 'Europe/Paris', 
-  'it.1'  => 'Europe/Rome',
-  'nl.1'  => 'Europe/Amsterdam',
-
-  'pt.1'  => 'Europe/Lisbon',   
-
-  ## todo/fix - pt.1
-  ##  one team in madeira!!! check for different timezone??
-  ##  CD Nacional da Madeira 
-
-  'br.1'  => 'America/Sao_Paulo',
-  ## todo/fix - brazil has 4 timezones
-  ##           really only two in use for clubs
-  ##             west and east (amazonas et al)
-  ##           for now use west for all - why? why not?
-}
-=end
-
-## todo - find "proper/classic" timezone ("winter time")
-
-##  Brasilia - Distrito Federal, Brasil  (GMT-3)  -- summer time?
-##  Ciudad de México, CDMX, México       (GMT-5)  -- summer time?
-##  Londres, Reino Unido (GMT+1)
-##   Madrid -- ?
-##   Lisboa -- ?
-##   Moskow -- ?
-##
-## todo/check - quick fix timezone offsets for leagues for now
-##   - find something better - why? why not?
-## note: assume time is in GMT+1
-OFFSETS = {
-  'eng.1' => -1,
-  'eng.2' => -1,
-  'eng.3' => -1,
-  'eng.4' => -1,
-  'eng.5' => -1,
-
-  'es.1'  => -1,
-  'es.2'  => -1,
-
-  'pt.1'  => -1,
-  'pt.2'  => -1,
-
-  'br.1'  => -5,
-  'mx.1'  => -7,
-}
-
-
-def self.convert( league:, season: )   
+def self.convert( league:, season: )
   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league = find_league( league )
+  league = find_league!( league )
+  pages  = league.pages!( season: season )
 
-  pages = league.pages( season: season )
 
-  ## check: rename (optional) offset to time_offset or such?
-  offset = OFFSETS[ league ]
-
-
-  # note: assume stages if pages is an array (of hash table/records)
-  #         (and NOT a single hash table/record)
-  if pages.is_a?(Array)
     recs = []
-    pages.each do |page_meta|
-      slug       = page_meta[:slug]
-      stage_name = page_meta[:stage]
-      ## todo/fix: report error/check if stage.name is nil!!!
+    pages.each do |slug, stage|
+      ## note: stage might be nil
+      ## todo/fix: report error/check if stage is nil!!!
+      stage ||= ''
 
       print "  parsing #{slug}..."
 
@@ -93,35 +27,27 @@ def self.convert( league:, season: )
       print "\n"
 
       rows = page.matches
-      stage_recs = build( rows, season: season, league: league.key, stage: stage_name )
+      stage_recs = build( rows,
+                          season: season,
+                          league: league.key,
+                          stage: stage )
 
       pp stage_recs[0]   ## check first record
       recs += stage_recs
     end
-  else
-    page_meta = pages
-    slug = page_meta[:slug]
-
-    print "  parsing #{slug}..."
-
-    page = Page::Schedule.from_cache( slug )
-    print "  title=>#{page.title}<..."
-    print "\n"
 
-    rows = page.matches
-    recs = build( rows, season: season, league: league.key )
 
-    pp recs[0]   ## check first record
-  end
+  recs = fix_dates( recs, league: league.key )
 
-  recs = recs.map { |rec| fix_date( rec, offset ) }    if offset
 
 ##   note:  sort matches by date before saving/writing!!!!
 ##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
 ## note: assume date is third column!!! (stage/round/date/...)
 recs = recs.sort { |l,r| l[2] <=> r[2] }
 ## reformat date / beautify e.g. Sat Aug 7 1993
-recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+recs.each do |rec|
+         rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+       end
 
    ## remove unused columns (e.g. stage, et, p, etc.)
    recs, headers = vacuum( recs )
@@ -136,10 +62,44 @@ recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b
 end
 
 
-
 ## helper to fix dates to use local timezone (and not utc/london time)
-def self.fix_date( row, offset )
-  return row    if row[3].nil? || row[3].empty?   ## note: time (column) required for fix
+def self.fix_dates( rows, league: )
+
+  ## check: rename (optional) offset to time_offset or such?
+  ##   note - retry with league_country (e.g. eng.1 => eng etc.)
+  offset = OFFSETS[ league ] ||
+           OFFSETS[ league.split('.')[0] ]
+
+  if offset.nil?
+      puts "!! ERROR - no timezone/offset configured for league >#{league}<:"
+      pp rows[0]  ## print first row too (for dates etc.)
+      exit 1
+  end
+
+
+   ## todo/check - rename offset to timezone
+   ##   or utc_offset or such - why? why not
+
+   ## note - assume central european time (cet) - GMT/UTC+1
+   ##           e.g. offset = 1 for cet (and 0 for gmt/london) etc.
+   diff_cet  =  offset-1
+
+   return rows   if diff_cet == 0   ## no need to convert if in cet
+
+   rows.map { |row| _fix_date( row, offset ) }
+end
+
+
+def self._fix_date( row, offset )
+  ## note: time (column) required for fix
+  return row    if row[3].nil? || row[3].empty?
+
+  ## note - assume central european time (cet) - GMT/UTC+1
+  diff_cet  =  offset-1
+
+  return row   if diff_cet == 0
+
+
 
   col = row[2]
   if col =~ /^\d{4}-\d{2}-\d{2}$/
@@ -152,7 +112,7 @@ def self.fix_date( row, offset )
 
   date = DateTime.strptime( "#{row[2]} #{row[3]}", "#{date_fmt} %H:%M" )
   ## NOTE - MUST be -7/24.0!!!! or such to work
-  date = date + (offset/24.0)
+  date = date + (diff_cet/24.0)
 
   row[2] = date.strftime( date_fmt )  ## overwrite "old"
   row[3] = date.strftime( '%H:%M' )

data/lib/worldfootball/download.rb

@@ -7,14 +7,9 @@ module Worldfootball
 def self.schedule( league:, season: )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league  = find_league( league )
-
-  pages =  league.pages( season: season )
-
-  ## if single (simple) page setup - wrap in array
-  pages = pages.is_a?(Array) ? pages : [pages]
-  pages.each do |page_meta|
-    Metal.download_schedule( page_meta[:slug] )
+  pages = find_league_pages!( league: league, season: season )
+  pages.each do |slug, _|
+    Metal.download_schedule( slug )
   end # each page
 end
 
@@ -22,14 +17,9 @@ end
 def self.reports( league:, season:, cache: true ) ## todo/check: rename to reports_for_schedule or such - why? why not?
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
-  league  = find_league( league )
-
-  pages =  league.pages( season: season )
-
-  ## if single (simple) page setup - wrap in array
-  pages = pages.is_a?(Array) ? pages : [pages]
-  pages.each do |page_meta|
-    Metal.download_reports_for_schedule( page_meta[:slug], cache: cache )
+  pages = find_league_pages!( league: league, season: season )
+  pages.each do |slug, _|
+    Metal.download_reports_for_schedule( slug, cache: cache )
   end # each page
 end
 
@@ -41,7 +31,7 @@ end
 
 ## todo/check: put in Downloader namespace/class - why? why not?
 ##   or use Metal    - no "porcelain" downloaders / machinery
-class Metal 
+class Metal
 
   BASE_URL = 'https://www.weltfussball.de'
 
@@ -117,7 +107,7 @@ class Metal
     end
   end
 
-  
+
   def self.download_page( url )  ## get & record/save to cache
     response = Webget.page( url )  ## fetch (and cache) html page (via HTTP GET)