worldfootball 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89d825112036b8740d446612f179f44d96d33fbba1874ab51d8f9e5dda1e01a9
-  data.tar.gz: 38d332b34ef4adb9017cbcee8434aa2f85d8a53a46e4556a267cb45443e17abd
+  metadata.gz: 53f1c44ded1d0dcec880df13b61a77265febcceacd88acb5ce883db39914cce9
+  data.tar.gz: b56eb75f1277026f77c29ee8f4d51788978955accbac5812659d1f3597e9747f
 SHA512:
-  metadata.gz: cf178997ed45d02f06e1039bab38d84979ab6df75422d73988c7dc2617d1a1cd4538f4ab5884ad4479faa130bd8203d8b58938f09fec661b937396cbc7364146
-  data.tar.gz: 23e76db98b915fe5f280653d4928cfb7f6c7e1c8adbc506b7ae87ad6ba99fcb30748968111bcb0b6e24c545ee89d42906abc662d07440f6aad87227aac8cdda2
+  metadata.gz: 501b137138a280f74d1cbd50c8f9ebc44ec8f1b7c7c9268005deb2d3a04b2d0046d05bf5a5ff7b3fb7c6f9f7b938423cdd83bb3b983d020bfb26057f2ed463c9
+  data.tar.gz: 84d716bcbffb32c11da5fd3a6b549a115ecbea0b4747182d84e3e7280cf3ff2de37ee57d37b20288d43f57374727fa907b256ed745b04d8b32b44ae17809d8ee

data/CHANGELOG.md

@@ -1,5 +1,4 @@
-### 0.3.0
-
+### 0.3.1
 ### 0.0.1 / 2024-07-04
 
 * Everything is new. First release.

data/Manifest.txt

@@ -3,6 +3,11 @@ Manifest.txt
 README.md
 Rakefile
 bin/wfb
+bin/wfbconf
+bin/wfbconv
+bin/wfbdump
+bin/wfbgen
+bin/wfbup
 config/leagues/africa.csv
 config/leagues/america.csv
 config/leagues/asia.csv

data/README.md

@@ -39,14 +39,56 @@ and so on.
 
 
 
-### More
+### More Command-Line Tools
 
+<!--
 Debugging tips & tricks. List all cached (offline local) match schedule pages:
 
 ```
 $ wfb cache
 ```
+-->
 
+wfbup -
+wfbconv -
+wfbgen -
+wfbconf -   
+wfbdump
+
+
+#### wfbup -  download leagues (if no league passed in, download all!)
+
+```
+$ wfbup        # download ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbup at.1   # download ALL seasons for league 
+```
+
+#### wfbconv - convert (to .csv) leagues (if no league passed in, converts all!)
+
+```
+$ wfbconv        # convert ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbconv at.1   # convert ALL seasons for league 
+```
+
+#### wfbgen - generate (.txt) leagues (if no league passed in, generate all!)
+
+```
+$ wfbgen        # generate ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbgen at.1   # generate ALL seasons for league 
+```
+
+
+#### wfbconf - check built-in config(uration) for league
+
+```
+$ wfbconf eng.1
+```
+
+#### wfbdump - dump (page) slug incl. matches, teams, rounds & more
+
+```
+$ wfbdump aut-bundesliga-2024-2025
+```
 
 
 

data/bin/wfb

@@ -4,8 +4,12 @@
 ##   ruby -I ./lib bin/wfb
 ##    or
 ##   ruby -I wfb/lib wfb/bin/wfb
+##    or
+##   ruby -I wfb/lib wfb/bin/wfb -f max.csv --cached
 
 
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
 require 'worldfootball'
 
 
@@ -25,7 +29,7 @@ Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
                                         end
 
 Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
-                                           puts "  setting convert out_dir to >/sports/cache.wfb.txt<"
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
                                            '/sports/cache.wfb.txt'
                                         else
                                            './tmp' ## use tmp in working dir
@@ -35,6 +39,7 @@ Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
 require 'optparse'
 
 
+
 module Worldfootball
 def self.main( args=ARGV )
 
@@ -130,15 +135,14 @@ end
 #  assume leagues
 
 datasets = if opts[:file]
-              read_datasets( opts[:file] )
+              read_leagueset( opts[:file] )
            else
-              parse_datasets_args( args )
+              parse_leagueset_args( args )
            end
 
 
 ## step 0 - validate and fill-up seasons etc.
-datasets.each do |dataset|
-  league_key, seasons = dataset
+datasets.each do |league_key, seasons|
 
   league =  find_league!( league_key )  ## league info lookup
 
@@ -150,8 +154,8 @@ datasets.each do |dataset|
   ##              might be 2024/25 or 2024 or
   #                 for world cup 2022 or such
   if seasons.empty?
-     seasons = [Season(league.seasons.keys[0])]
-     dataset[1] = seasons
+     season = Season(league.seasons.keys[0])
+     seasons << season
   end
 end
 
@@ -206,36 +210,13 @@ if opts[:convert]
   end
 end
 
+
 if opts[:generate]
 ## step 3 - generate
   datasets.each do |league_key, seasons|
     seasons.each do |season|
-
-          path = "#{config.convert.out_dir}/#{season.to_path}/#{league_key}.csv"
-          ## get matches
-          puts "  ---> reading matches in #{path} ..."
-          matches = SportDb::CsvMatchParser.read( path )
-          puts "     #{matches.size} matches"
-
-          ## build
-          txt = SportDb::TxtMatchWriter.build( matches )
-          puts txt
-
-          path =  if season >= Season( '2000' )
-                    "#{config.generate.out_dir}/#{season.to_path}/#{league_key}.txt"
-                  else
-                    decade = season.start_year - (season.start_year%10)
-                    ## use archive-style before 2000!!!
-                    "#{config.generate.out_dir}/archive/#{decade}s/#{season.to_path}/#{league_key}.txt"
-                  end
-
-          buf = String.new
-          ## note - use league key for league name for now!!
-          buf << "= #{league_key.upcase.gsub('.', ' ')} #{season.key}\n\n"
-          buf << txt
-
-          puts "   writing to >#{path}<..."
-          write_text( path, buf )
+         generate( league: league_key,
+                   season: season )
     end
   end
   ## for debugging dump page (slug) titles

data/bin/wfbconf

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbconf
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbconf
+
+####
+#  wfbconf - check built-in config(uration) for league
+#    e.g.     wfbconf eng.1
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+
+Worldfootball.debug = true
+
+##
+## download fresh copy?
+## Worldfootball::Metal.download_schedule( slug )
+
+key = args[0] || 'eng.1'
+
+league = Worldfootball::LEAGUES[key]
+seasons = league.seasons
+pp seasons
+
+puts "  #{key} - #{seasons.size} season(s)"
+
+puts
+latest = seasons.keys[0]
+puts " latest #{latest}: "
+pp seasons[latest]
+
+
+puts "bye"

data/bin/wfbconv

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbconv
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbconv
+
+###
+#  wfbconv -  convert (to .csv) all leagues  (if no league passed in, converts all!)
+# 
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+## convert (default) output directory
+Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
+                                           puts "  setting convert out_dir to >/sports/cache.wfb<"
+                                           '/sports/cache.wfb'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+opts = {
+    force:   false,   # a.k.a. overwrite
+ }
+
+
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+
+  parser.on( "--force",
+               "always overwrite (force) datafile - default is (#{opts[:force]})" ) do |force|
+    opts[:force] = true   # true|false
+  end
+end
+
+
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+keys =  if args.size == 0
+           Worldfootball::LEAGUES.keys
+        else
+           args
+        end
+
+
+
+keys.each_with_index do |key, i|
+  league = Worldfootball::LEAGUES[key]
+  seasons = league.seasons
+
+  puts "==> #{i+1}/#{keys.size} #{key} - #{seasons.size} seasons(s)..."
+
+  seasons.each_with_index do |season_rec,j|
+    season = season_rec[0]
+
+    next if key == 'nl.cup' && season == '1959/60'
+
+    puts "  #{j+1}/#{seasons.size} #{key} #{season}..."
+    Worldfootball.convert( league: key, season: season,
+                           overwrite: opts[:force] )
+  end
+end
+
+
+puts "bye"

data/bin/wfbdump

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbdump
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbdump
+
+
+#####
+#  wfbdump - dump matches, teams, rounds and more for (page) slug
+#    e.g.
+#    wfbdump caf-champions-league-2008
+#    wfbdump aut-bundesliga-2024-2025
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+
+Worldfootball.debug = true
+
+##
+## download fresh copy?
+## Worldfootball::Metal.download_schedule( slug )
+
+slug = args[0] || 'caf-champions-league-2008'
+
+page = Worldfootball::Page::Schedule.from_cache( slug )
+
+matches = page.matches
+teams   = page.teams
+rounds  = page.rounds
+
+puts "  #{matches.size} match(es), #{teams.size} team(s), #{rounds.size} round(s)"
+# pp matches
+
+puts
+puts "  #{teams.size} team(s)"
+pp teams
+
+puts
+puts "  #{rounds.size} round(s)"
+pp rounds
+
+
+puts "==> page props"
+print "title: "
+puts page.title
+puts
+print "keywords: "
+puts page.keywords
+puts
+print "url: "
+puts page.url
+
+
+puts "bye"

data/bin/wfbgen

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbgen
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbgen
+
+###############
+#  wfbgen  
+#  
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sportdb-writers/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+## convert (default) output directory
+Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
+                                           puts "  setting convert out_dir to >/sports/cache.wfb<"
+                                           '/sports/cache.wfb'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+opts = {
+    force:   false,   # a.k.a. overwrite
+ }
+
+
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+
+  parser.on( "--force",
+               "always overwrite (force) datafile - default is (#{opts[:force]})" ) do |force|
+    opts[:force] = true   # true|false
+  end
+end
+
+
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+keys =  if args.size == 0
+           Worldfootball::LEAGUES.keys
+        else
+           args
+        end
+
+
+
+keys.each_with_index do |key, i|
+  league = Worldfootball::LEAGUES[key]
+  seasons = league.seasons
+
+  puts "==> #{i+1}/#{keys.size} #{key} - #{seasons.size} seasons(s)..."
+
+  seasons.each_with_index do |season_rec,j|
+    season = season_rec[0]
+
+    next if key == 'nl.cup' && season == '1959/60'
+
+    puts "  #{j+1}/#{seasons.size} #{key} #{season}..."
+    Worldfootball.generate( league: key, season: season,
+                            overwrite: opts[:force] )
+  end
+end
+
+
+puts "bye"

data/bin/wfbup

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbup
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbup
+
+
+##
+##  add offset for restart!!!!
+##   e.g. si.1  - maybe add season later!!!
+##    or better add a expired option e.g. 24h or such !!!!
+## -  stopping at si.1 2017/18...
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+## convert (default) output directory
+Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
+                                           puts "  setting convert out_dir to >/sports/cache.wfb<"
+                                           '/sports/cache.wfb'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+opts = {
+    force:   false,   # a.k.a. overwrite
+ }
+
+
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+
+  parser.on( "--force",
+               "always overwrite (force) datafile - default is (#{opts[:force]})" ) do |force|
+    opts[:force] = true   # true|false
+  end
+end
+
+
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+keys =  if args.size == 0
+           Worldfootball::LEAGUES.keys
+        else
+           args
+        end
+
+
+
+keys.each_with_index do |key, i|
+  league = Worldfootball::LEAGUES[key]
+  seasons = league.seasons
+
+  puts "==> #{i+1}/#{keys.size} #{key} - #{seasons.size} seasons(s)..."
+
+  seasons.each_with_index do |season_rec,j|
+    season = season_rec[0]
+
+    puts "  #{j+1}/#{seasons.size} #{key} #{season}..."
+    Worldfootball.schedule( league: key, season: season,
+                            overwrite: opts[:force] )
+  end
+end
+
+
+puts "bye"

data/config/leagues/america.csv

@@ -23,12 +23,19 @@ ca.cup,   can-canadian-championship-2024
 #        - Finale
 #      /mex-primera-division-2018-2019-clausura-playoffs/
 mx.1,    mex-primera-division-2024-2025-apertura_2
-mx.2,    mex-liga-de-expansion-2024-2025-apertura
+
+##  merge  expansion and ascenso into one (mx.2) - possible? why? why not?
+
+## first expansion season ->  2020/21 Apertura
+## last ascenso season    ->  2019/20 Clausura
+mx.2.expansion,   mex-liga-de-expansion-2024-2025-apertura
+mx.2.ascenso,     mex-liga-de-ascenso-2019-2020-clausura
+
 mx.cup,  mex-copa-mx-2019-2020
 
 
-### fix add from / to seasons !!! for mx!!!!
-mx.3,    mex-premier-de-ascenso-2020-2021-grupo-b
+mx.3.a,  mex-lp-serie-a-2024-2025-apertura
+mx.3.b,  mex-lp-serie-b-2024-2025-apertura
 
 
 

data/config/leagues/europe.csv

@@ -33,7 +33,10 @@ hu.1,       hun-nb-i-2024-2025
 
 cz.1,       cze-1-fotbalova-liga-2024-2025
 cz.2,       cze-2-fotbalova-liga-2024-2025
-cz.3,       cze-3-fotbalova-liga-2024-2025-cfl-a
+
+## commented out for now (three leagues in one - split !!)
+## cz.3,       cze-3-fotbalova-liga-2024-2025-cfl-a
+
 
 sk.1,       svk-super-liga-2024-2025
 
@@ -151,7 +154,7 @@ rs.1,    srb-super-liga-2024-2025
 si.1,    svn-prvaliga-2024-2025
 
 ## todo/fix - change to kos.1 - why? why not?
-xk.1,     kos-superliga-2024-2025
+## xk.1,     kos-superliga-2024-2025
 kos.1,    kos-superliga-2024-2025
 
 

data/config/rounds.csv

@@ -23,6 +23,12 @@ key,  name1, name2
 *,  Qual. 2. Runde,  Qual. Round 2
 
 
+*, Entscheidungsspiel,  Decider
+*, Finalrunde,          Finals
+*, Ligaphase,           League phase
+
+
+
 
 ## es to en
 *,  Recalificación,  Reclassification

data/config/stages.csv

@@ -1,51 +1,38 @@
 key,  name1, name2,
 
-*,   Meisterschaft,    Playoffs - Championship
-*,   Abstieg,          Playoffs - Relegation
-*,   Relegation,       Playoffs - Relegation
+*,   Meisterschaft,    Championship
+*,   Abstieg,          Relegation
 
+## use Playoffs - Championship  -- why? why not?
+## use Playoffs - Relegation    -- why? why not?
 
-sco.1,  Championship,   Playoffs - Championship
-sco.1,  Relegation',    Playoffs - Relegation
 
-
-at.1,   Meistergruppe,          Playoffs - Championship
-at.1,   Qualifikationsgruppe,   Playoffs - Relegation
+at.1,   Meistergruppe,          Championship
+at.1,   Qualifikationsgruppe,   Relegation
 at.1,   Playoff,                Europa League Finals
 
 
-pl.1,   Playoffs,   Playoffs - Championship
-pl.2,   Abstieg,    Playoffs - Relegation
-
-
-sk.1,   Meisterschaft,   Playoffs - Championship
-sk.1,   Abstieg,         Playoffs - Relegation
+sk.1,   Meisterschaft,   Championship
+sk.1,   Abstieg,         Relegation
 sk.1,   Europa League,   Europa League Finals
 
 
-ro.1, Championship,    Playoffs - Championship
-ro.1, Relegation,      Playoffs - Relegation
-
-ru.1, Meisterschaft,   Playoffs - Championship
-ru.1, Relegation,      Playoffs - Relegation
-
-
-ua.1, Meisterschaft,    Playoffs - Championship
-ua.1, Abstieg,          Playoffs - Relegation
+ua.1, Meisterschaft,    Championship
+ua.1, Abstieg,          Relegation
 ua.1, Playoffs EL,      Europa League Finals
 
 
-fi.1,  Meisterschaft,   Playoffs - Championship
-fi.1,  Abstieg,         Playoffs - Challenger
+fi.1,  Meisterschaft,   Championship
+fi.1,  Abstieg,         Relegation
 fi.1,  Playoff EL,      Europa League Finals
 
-dk.1,  Meisterschaft,   Playoffs - Championship
-dk.1,  Abstieg,         Playoffs - Relegation
+dk.1,  Meisterschaft,   Championship
+dk.1,  Abstieg,         Relegation
 dk.1,  Europa League,   Europa League Finals
 
 
-gr.1,  Meisterschaft,    Playoffs - Championship
-gr.1,  Abstieg,          Playoffs - Relegation
+gr.1,  Meisterschaft,    Championship
+gr.1,  Abstieg,          Relegation
 gr.1,  Playoffs,         Playoffs
 gr.1,  Spiel um Platz 6, Match 6th Place
 
@@ -55,9 +42,6 @@ mx.1,  Apertura Playoffs,   Apertura - Liguilla
 mx.1,  Clausura Playoffs,   Clausura - Liguilla
 
 
-kr.1,   Meisterschaft,    Playoffs - Championship
-kr.1,   Abstieg,          Playoffs - Relegation
-
 
 nz.1,  Playoffs,         Playoff Finals
 

data/lib/worldfootball/build-parse_score.rb

@@ -31,13 +31,18 @@ def self.parse_score( score_str )
   et  = ''
   pen = ''
 
+##
+##  [085]    2021-10-21 | 22:00 | Metropolitanos FC      | LALA FC                | Aufg.
+##  !! ERROR - unsupported score format >Aufg.< - sorry; maybe add a score error fix/patch
+##  - handle with Aufg.
 
   if score_str == '---'   ## in the future (no score yet) - was -:-
     ft = ''
     ht = ''
   elsif score_str == 'n.gesp.' ||   ## cancelled (british) / canceled (us)
         score_str == 'ausg.'   ||   ## todo/check: change to some other status ????
-        score_str == 'annull.'      ## todo/check: change to some other status (see ie 2012) ????
+        score_str == 'annull.' ||   ## todo/check: change to some other status (see ie 2012) ????
+        score_str == 'Aufg.'
     ft = '(*)'
     ht = ''
     comments = 'cancelled'
@@ -172,13 +177,13 @@ def self.parse_score( score_str )
     puts "!! WARN - weird score n.V. only - >#{score_str}<"
   elsif score_str =~ /^([0-9]+) [ ]*-[ ]* ([0-9]+)
                           [ ]*
-                       i\.E\.
+                        (?: i\.E\. | n\.P\. )
                        $/x
     pen = "#{$1}-#{$2}"
     et  = ''
     ht  = ''
     ft  = ''
-    puts "!! WARN - weird score i.E. only - >#{score_str}<"
+    puts "!! WARN - weird score i.E. (n.P.) only - >#{score_str}<"
   else
      puts "!! ERROR - unsupported score format >#{score_str}< - sorry; maybe add a score error fix/patch"
      exit 1

data/lib/worldfootball/convert.rb

@@ -2,13 +2,23 @@
 module Worldfootball
 
 
-def self.convert( league:, season: )
+def self.convert( league:, season:,
+                  overwrite: true )
   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
 
   league = find_league!( league )
   pages  = league.pages!( season: season )
 
 
+  out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
+  if !overwrite && File.exist?( out_path )
+    ## skip generation
+    puts "  OK #{league.key} #{season}   (do NOT overwrite)"
+    return
+  end
+
+
+
     ## collect all teams
     teams_by_ref = {}
 
@@ -48,17 +58,19 @@ def self.convert( league:, season: )
           team_name  = norm_team( h[:name] )      ## note: norm team name!!!
           team_ref   = h[:ref]
 
+###
+##   quick fix for broken refs/links
+##    olympique-lyon => olympique-lyonnais
+       #   team_ref = 'olympique-lyonnais'   if team_ref == 'olympique-lyon'
+
           ## note: skip N.N.  (place holder team)
           ##        team_ref is nil etc.
           next if team_name == 'N.N.'
 
           team_stat = teams_by_ref[ team_ref ] ||= { count: 0,
-                                                     name:  team_name }
+                                                     names:  [] }
           team_stat[:count] += team_count
-          if team_name != team_stat[:name]
-              puts "!! ASSERT ERROR - team ref with differet names; expected #{team_stat[:name]} - got #{team_name}"
-              exit 1
-          end
+          team_stat[:names] << team_name   unless team_stat[:names].include?( team_name )
       end
 
 
@@ -73,9 +85,12 @@ def self.convert( league:, season: )
 
 
     clubs_intl  =  ['uefa.cl', 'uefa.el', 'uefa.conf',
-                    'copa.l',
+                    'uefa.cl.q', 'uefa.el.q', 'uefa.conf.q',
+                     'copa.l',
+                    'concacaf.cl',
                     'caf.cl',
-                    'afl'].include?(league.key) ? true : false
+                    'afl',
+                  ].include?(league.key) ? true : false
 
     ####
     #   auto-add (fifa) country code if int'l club tournament
@@ -83,7 +98,6 @@ def self.convert( league:, season: )
  ##
  ##   get country codes for team ref
        teams_by_ref.each do |team_slug, h|
-
           Metal.download_team( team_slug, cache: true )
           team_page = Page::Team.from_cache( team_slug )
           props = team_page.props
@@ -99,14 +113,20 @@ def self.convert( league:, season: )
 
        ## generate lookup by name
        teams_by_name = teams_by_ref.reduce( {} ) do |h, (slug,rec)|
-            h[ rec[:name]] = rec
-            h
+  ### todo/fix
+  ##    report warning if names size is > 1!!!!
+  ##
+             rec[:names].each do |name|
+                h[ name ] = rec
+              end
+              h
        end
 
+
     #####
     ## dump team refs
     puts "  #{teams_by_ref.size} team(s) by ref:"
-    pp teams_by_name
+    pp teams_by_ref
 
     ## quick hack
     ##  add country (fifa) codes to team names
@@ -129,10 +149,22 @@ def self.convert( league:, season: )
 ##   note:  sort matches by date before saving/writing!!!!
 ##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
 ## note: assume date is third column!!! (stage/round/date/...)
-recs = recs.sort { |l,r| l[2] <=> r[2] }
+
+### note - do NOT sort for now
+##    keep "original" page order - why? why not?
+## recs = recs.sort { |l,r| l[2] <=> r[2] }
+
+
 ## reformat date / beautify e.g. Sat Aug 7 1993
 recs.each do |rec|
-         rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+            if rec[2]
+              if rec[2] =~ /^\d{4}-\d{1,2}-\d{1,2}$/
+               rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+              else
+                ## report unknown date format warning
+                puts "WARN - unsupported date format (cannot parse?) >#{rec[2]}<"
+              end
+            end
        end
 
    ## remove unused columns (e.g. stage, et, p, etc.)
@@ -141,10 +173,13 @@ recs.each do |rec|
    puts headers
    pp recs[0]   ## check first record
 
-   out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
-
-   puts "write #{out_path}..."
+   puts "   writing to >#{out_path}< - #{recs.size} record(s)..."
    write_csv( out_path, recs, headers: headers )
+
+   ## add to tmp too for debugging
+   out_path2 = "#{config.convert.out_dir}/tmp/#{league.key}/#{season.to_path}.csv"
+   puts "   writing to >#{out_path2}< - #{recs.size} record(s)..."
+   write_csv( out_path2, recs, headers: headers )
 end
 end # module Worldfootball
 

data/lib/worldfootball/download.rb

@@ -4,12 +4,16 @@ module Worldfootball
 
 #################
 ##  porcelain "api"
-def self.schedule( league:, season: )
+def self.schedule( league:, season:, overwrite: true )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
   pages = find_league_pages!( league: league, season: season )
   pages.each do |slug, _|
-    Metal.download_schedule( slug )
+    if !overwrite && Webcache.cached?( Metal.schedule_url( slug ))
+      puts "  OK #{league} #{season}  - #{slug}   (do NOT overwrite)"
+    else
+      Metal.download_schedule( slug )
+    end
   end # each page
 end
 

data/lib/worldfootball/leagues.rb

@@ -61,9 +61,12 @@ class LeagueItem  # nested inside LeagueConfig
         season, stage = text.split( ' ', 2 )
 
 ## todo/fix: add a waring here and auto log to logs.txt!!!!
-        next if season == '2019-2021'
-        next if season == '1958/1960'
-        next if season == '1955/1958'
+        if ['2019-2021',
+            '1958/1960',
+            '1955/1958'  ].include?( season )
+           log( "!! WARN - seasons for league #{@key} incl. invalid season #{season} - slug #{slug}; skipping season" )
+           next   ## note - skip invalid season entry
+        end
 
         season = Season.parse( season )
 
@@ -106,6 +109,13 @@ class LeagueItem  # nested inside LeagueConfig
     recs = seasons[season.key]
     recs ?  recs.reverse : nil
   end
+
+  def log( msg )  ### append to log
+    File.open( './logs.txt', 'a:utf-8' ) do |f|
+      f.write( msg )
+      f.write( "\n" )
+    end
+  end
 end # class LeagueItem
 
 
@@ -122,6 +132,9 @@ def size()   @table.size; end
 end # class LeagueConfig
 
 
+
+
+
 LEAGUES = LeagueConfig.new
 ['africa',
  'america',

data/lib/worldfootball/mods.rb

@@ -10,11 +10,20 @@ module Worldfootball
 def self.norm_team( team )
    ## clean team name and asciify (e.g. ’->' )
    team = team.sub( '(old)', '' ).strip
-   team = team.gsub( '’', "'" )     ## e.g. Hawke’s Bay United FC
 
-   ## Criciúma - SC     =>     Criciúma - SC
-   ## Bahia - BA        =>     Bahia - BA
+   ## e.g. Hawke’s Bay United FC   or
+   ##         ASC Monts d`Or Chasselay   or
+   ##        VV Heerlen ´16  / EMM ´15 / Wormer SV´30 / Swift ´36 / etc.
+   team = team.gsub( /[’´`]/, "'" )
+
+
+   ## br
+   ## Criciúma - SC      =>     Criciúma SC
+   ## Bahia - BA         =>     Bahia BA
+   ##  cz
+   ## Baník Most - Souš  =>     Baník Most Souš
    ##  remove inline dash ( - ) with single space
+   ##  to log
    team = team.gsub( /[ ]+[-][ ]+/, ' ' )
 
 
@@ -24,6 +33,22 @@ def self.norm_team( team )
    ##   others too?  - move to mods instead of generic rule - why? why not?
    team = team.sub( /[ ]+\(A\)/, ' II' )
 
+##
+##  remove ()  - used/reserved for country code for now - why? why not?
+##  e.g. Lloyds FC (Sittingbourne)  => Lloyds FC Sittingbourne
+##       August 1st (Army Team)     => August 1st Army Team
+##
+## add warning - why? why not?
+   team = team.sub( /\(
+                        ([^)]+?)   ## eat-up all non-greed to next )
+                     \)/x, '\1' )
+
+##
+## strip special case
+##   MFK Frýdek-Místek, a.s.   =>  MFK Frýdek-Místek
+     team = team.sub( ', a.s.', '' )
+
+
    ################
    ## quick hack - norm(alize) all N.N. to N.N.
    ##   e.g.

data/lib/worldfootball/page_schedule.rb

@@ -12,6 +12,15 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
   end
 
 
+  PLACEHOLDERS = [
+     'N.N.',
+     'Verlierer HF 1',
+     'Verlierer HF 2',
+     'Sieger HF 1',
+     'Sieger HF 2',
+    ]
+  def placeholder?( str )  PLACEHOLDERS.include?( str ); end
+
 
   def matches
     @matches ||= begin
@@ -20,10 +29,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
  # <table class="standard_tabelle" cellpadding="3" cellspacing="1">
 
  ## note: use > for "strict" sibling (child without any in-betweens)
- table = doc.css( 'div.data > table.standard_tabelle' ).first    ## get table
+ tables = doc.css( 'div.data > table.standard_tabelle' )    ## get table
  # puts table.class.name  #=> Nokogiri::XML::Element
  # puts table.text
 
+ assert( tables.size==1, "expected one table.standard_tabelle; got #{tables.size}" )
+ table = tables.first
  assert( table, 'no table.standard_tabelle found in schedule page!!')
 
  trs   = table.css( 'tr' )
@@ -77,8 +88,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
     ths =  tr.css( 'th' )
     tds =  tr.css( 'td' )
 
-   if tr.text.strip =~ /Spieltag/ ||
-      tr.text.strip =~ /[1-9]\.[ ]Runde|
+    tr_text = squish( tr.text )
+
+   if tr_text =~ /Spieltag/ ||
+      tr_text =~ /[1-9]\.[ ]Runde|
                            Qual\.[ ][1-9]\.[ ]Runde|  # see EL or CL Quali
                            Qualifikation|     # see CA Championship
                            Sechzehntelfinale|   # see EL
@@ -110,25 +123,24 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      if debug?
        puts
        print '[%03d] ' % i
-       ## print squish( tr.text )
-       print "round >#{tr.text.strip}<"
+       print "round >#{tr_text}<"
        print "\n"
      end
 
-     last_round = tr.text.strip
+     last_round = tr_text
    elsif ths.count > 0 &&
          tds.count == 0
        ## check for round NOT yet configured!!!
-       puts "!! WARN: found unregistered round line >#{tr.text.strip}<"
-       log( "!! WARN: found unregistered round line >#{tr.text.strip}< in page #{title}" )
+       puts "!! WARN: found unregistered round line >#{tr_text}<"
+       log( "!! WARN: found unregistered round line >#{tr_text}< in page #{title}" )
 
-       last_round = tr.text.strip
+       last_round = tr_text
    else   ## assume table row (tr) is match line
 
      date_str  = squish( tds[0].text )
      time_str  = squish( tds[1].text )
 
-     date_str = last_date_str    if date_str.empty?
+     date_str = last_date_str    if date_str.empty? && last_date_str
 
      if debug?
        ## note: for debugging - print as we go along (parsing)
@@ -148,7 +160,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      else
        team1_str    = squish( tds[2].text )
        team1_ref    = nil
-       puts "!! WARN: no team1_ref for >#{team1_str}< found"
+       puts "!! WARN: no team1_ref for >#{team1_str}< found"  unless placeholder?( team1_str )
      end
 
      if debug?
@@ -170,9 +182,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      else
        team2_str    = squish( tds[4].text )
        team2_ref    = nil
-       puts "!! WARN: no team2_ref for >#{team2_str}< found"
+       puts "!! WARN: no team2_ref for >#{team2_str}< found"  unless placeholder?( team2_str )
      end
 
+
      if debug?
        ## note: for debugging - print as we go along (parsing)
        print "%-22s | " % team2_str
@@ -198,8 +211,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      ##   <img src="https://s.hs-data.com/bilder/shared/live/2.png" /></a>
      ## </td>
      img = tds[6].css( 'img' )[0]
+
+
+
      if img && img[:src].index( '/live/')
-       puts "!! WARN: live match badge, resetting score from #{score_str} to -:-"
+       ## puts "!! WARN: live match badge, resetting score from #{score_str} to -:-"
+       print " LIVE BADGE "
        score_str = '-:-'  # note: -:- gets replaced to ---
      end
 
@@ -217,8 +234,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      ## special case for '00.00.0000'
      ##   CANNOT parse
      ##   use empty date - why? why not?
+     ##   if start with 00.00. e.g. 00.00.1939
+
 
-     date     = if date_str == '00.00.0000'
+     date     = if date_str == '00.00.0000' ||
+                   date_str.start_with?( '00.00.' ) ||
+                   date_str.empty?
                   nil
                 else
                   Date.strptime( date_str, '%d.%m.%Y' )
@@ -237,7 +258,9 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
                report_ref: score_ref
              }
 
-     last_date_str = date_str
+     ## note - only update last date if date present
+     ##             might be empty (not available) in the beginning
+     last_date_str = date_str      if !date_str.empty?
    end
   end # each tr (table row)
 

data/lib/worldfootball/version.rb

@@ -2,7 +2,7 @@
 module Worldfootball
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 3
-  PATCH = 0
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/worldfootball.rb

@@ -41,6 +41,14 @@ module Worldfootball
    def self.debug=(value) @debug = value; end
    def self.debug?()      @debug ||= false; end  ## note: default is FALSE
 
+   def self.log( msg )  ### append to log
+      File.open( './logs.txt', 'a:utf-8' ) do |f|
+        f.write( msg )
+        f.write( "\n" )
+      end
+   end
+
+
 
 
 class Configuration
@@ -71,6 +79,60 @@ end   # module Worldfootball
 
 
 
+###
+# todo - move generate to generate file!!!
+module Worldfootball
+def self.generate( league:, season:,
+                   overwrite: true )
+   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+   league = find_league!( league )
+   pages  = league.pages!( season: season )
+
+
+   out_path = if season >= Season( '2000' )
+                "#{config.generate.out_dir}/#{season.to_path}/#{league.key}.txt"
+              else
+                decade = season.start_year - (season.start_year%10)
+                ## use archive-style before 2000!!!
+                "#{config.generate.out_dir}/archive/#{decade}s/#{season.to_path}/#{league.key}.txt"
+              end
+
+   ## check if output exists already
+   if !overwrite && File.exist?( out_path )
+     ## skip generation
+     puts "  OK #{league.key} #{season}   (do NOT overwrite)"
+     return
+   end
+
+
+   ## get matches
+   path = "#{config.convert.out_dir}/#{season.to_path}/#{league.key}.csv"
+   puts "  ---> reading matches in #{path} ..."
+   matches = SportDb::CsvMatchParser.read( path )
+   puts "     #{matches.size} matches"
+
+   ## build
+   txt = SportDb::TxtMatchWriter.build( matches )
+   puts txt
+
+
+   buf = String.new
+   ## note - use league key for league name for now!!
+   buf << "= #{league.key.upcase.gsub('.', ' ')} #{season.key}\n\n"
+   buf << txt
+
+   puts "   writing to >#{out_path}<..."
+   write_text( out_path, buf )
+
+   ## add to tmp too for debugging
+   out_path2 = "#{config.generate.out_dir}/tmp/#{league.key}/#{season.to_path}.txt"
+   puts "   writing to >#{out_path2}<..."
+   write_text( out_path2, buf )
+end
+end   # module Worldfootball
+
+
 
 
 puts Worldfootball.banner    ## say hello

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worldfootball
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-06 00:00:00.000000000 Z
+date: 2024-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: football-timezones
@@ -106,19 +106,24 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
 description: worldfootball - get world football (leagues, cups & more) match data
   via the worldfootball.net/weltfussball.de pages
 email: gerald.bauer@gmail.com
 executables:
 - wfb
+- wfbconf
+- wfbconv
+- wfbdump
+- wfbgen
+- wfbup
 extensions: []
 extra_rdoc_files:
 - CHANGELOG.md
@@ -130,6 +135,11 @@ files:
 - README.md
 - Rakefile
 - bin/wfb
+- bin/wfbconf
+- bin/wfbconv
+- bin/wfbdump
+- bin/wfbgen
+- bin/wfbup
 - config/leagues/africa.csv
 - config/leagues/america.csv
 - config/leagues/asia.csv
@@ -175,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: worldfootball - get world football (leagues, cups & more) match data via