worldfootball 0.2.6 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d824bdedb279bf91779c99445fcf5bd5a6df8acc342232624032256a32fbe18
-  data.tar.gz: 14eb224174efe18645412a88eaa332e01901247dd28ffa795cba8b902d3a1dd6
+  metadata.gz: 53f1c44ded1d0dcec880df13b61a77265febcceacd88acb5ce883db39914cce9
+  data.tar.gz: b56eb75f1277026f77c29ee8f4d51788978955accbac5812659d1f3597e9747f
 SHA512:
-  metadata.gz: b0e4885e80c6ff3fb13a00a654524ea4514fcac730438096e412cd76a5ba14c17fa563996afd81176f7538bffb6ac8bdc3c75de7af7ace2124269b7c72c0265b
-  data.tar.gz: feb02079e9c481c3b2e53d5c72030e5e9e687f94bf053c29134da613933c2f5f6b569339fb8980ab3c048e248c171c93ea23a4b4a350c5dfede8563990b5a298
+  metadata.gz: 501b137138a280f74d1cbd50c8f9ebc44ec8f1b7c7c9268005deb2d3a04b2d0046d05bf5a5ff7b3fb7c6f9f7b938423cdd83bb3b983d020bfb26057f2ed463c9
+  data.tar.gz: 84d716bcbffb32c11da5fd3a6b549a115ecbea0b4747182d84e3e7280cf3ff2de37ee57d37b20288d43f57374727fa907b256ed745b04d8b32b44ae17809d8ee

data/CHANGELOG.md

@@ -1,5 +1,4 @@
-### 0.2.6
-
+### 0.3.1
 ### 0.0.1 / 2024-07-04
 
 * Everything is new. First release.

data/Manifest.txt

@@ -3,12 +3,17 @@ Manifest.txt
 README.md
 Rakefile
 bin/wfb
-config/leagues_africa.csv
-config/leagues_america.csv
-config/leagues_asia.csv
-config/leagues_europe.csv
-config/leagues_middle_east.csv
-config/leagues_pacific.csv
+bin/wfbconf
+bin/wfbconv
+bin/wfbdump
+bin/wfbgen
+bin/wfbup
+config/leagues/africa.csv
+config/leagues/america.csv
+config/leagues/asia.csv
+config/leagues/europe.csv
+config/leagues/middle_east.csv
+config/leagues/pacific.csv
 config/rounds.csv
 config/stages.csv
 lib/worldfootball.rb

data/README.md

@@ -1,5 +1,5 @@
 # worldfootball  - get world football (leagues, cups & more) match data via the worldfootball.net/weltfussball.de pages
- 
+
 
 * home  :: [github.com/sportdb/sport.db](https://github.com/sportdb/sport.db)
 * bugs  :: [github.com/sportdb/sport.db/issues](https://github.com/sportdb/sport.db/issues)
@@ -39,14 +39,56 @@ and so on.
 
 
 
-### More
+### More Command-Line Tools
 
-Debugging tips. List all cached (offline local) match schedule pages:
+<!--
+Debugging tips & tricks. List all cached (offline local) match schedule pages:
 
 ```
 $ wfb cache
 ```
+-->
+
+wfbup -
+wfbconv -
+wfbgen -
+wfbconf -   
+wfbdump
+
+
+#### wfbup -  download leagues (if no league passed in, download all!)
+
+```
+$ wfbup        # download ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbup at.1   # download ALL seasons for league 
+```
+
+#### wfbconv - convert (to .csv) leagues (if no league passed in, converts all!)
+
+```
+$ wfbconv        # convert ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbconv at.1   # convert ALL seasons for league 
+```
+
+#### wfbgen - generate (.txt) leagues (if no league passed in, generate all!)
 
+```
+$ wfbgen        # generate ALL leagues and ALL seasons (uses all built-in configs)
+$ wfbgen at.1   # generate ALL seasons for league 
+```
+
+
+#### wfbconf - check built-in config(uration) for league
+
+```
+$ wfbconf eng.1
+```
+
+#### wfbdump - dump (page) slug incl. matches, teams, rounds & more
+
+```
+$ wfbdump aut-bundesliga-2024-2025
+```
 
 
 

data/Rakefile

@@ -22,6 +22,7 @@ Hoe.spec 'worldfootball' do
     ['fifa'],
     ['webget'],
     ['nokogiri'],
+    ['sportdb-writers'],
   ]
 
   self.licenses = ['Public Domain']

data/bin/wfb

@@ -2,7 +2,14 @@
 
 ## tip: to test run:
 ##   ruby -I ./lib bin/wfb
+##    or
+##   ruby -I wfb/lib wfb/bin/wfb
+##    or
+##   ruby -I wfb/lib wfb/bin/wfb -f max.csv --cached
 
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
 require 'worldfootball'
 
 
@@ -18,19 +25,30 @@ Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
                                            puts "  setting convert out_dir to >/sports/cache.wfb<"
                                            '/sports/cache.wfb'
                                         else
-                                           '.' ## use working dir
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
                                         end
 
+
 require 'optparse'
 
 
+
 module Worldfootball
 def self.main( args=ARGV )
 
 opts = {
-    cached:  false,
-    convert: true,
-    file:    nil,
+    debug:    false,
+    cached:   false,
+    convert:  true,
+    generate: true,
+    file:     nil,
  }
 
 
@@ -49,14 +67,22 @@ parser = OptionParser.new do |parser|
   end
 
   parser.on( "--[no-]convert",
-               "turn on/off conversion to .csv in #{Worldfootball.config.convert.out_dir} - default is (#{opts[:convert]})" ) do |convert|
+               "turn on/off conversion to (tabular) .csv format in #{Worldfootball.config.convert.out_dir} - default is (#{opts[:convert]})" ) do |convert|
     opts[:convert] = convert   # true|false
   end
 
+  parser.on( "--[no-]generate",
+               "turn on/off generation in football.txt format in #{Worldfootball.config.generate.out_dir} - default is (#{opts[:generate]})" ) do |generate|
+    opts[:generate] = generate   # true|false
+  end
+
+
   parser.on( "--print", "--pp",
-                 "pretty print cached data in #{Webcache.root}; no download & conversion") do |print|
-    opts[:cached]  = true
-    opts[:convert] = false
+                 "pretty print cached data in #{Webcache.root}; no download & conversion & generation") do |print|
+    opts[:debug]    = true
+    opts[:cached]   = true
+    opts[:convert]  = false
+    opts[:generate] = false
   end
 
   parser.on( "-f FILE", "--file FILE",
@@ -76,6 +102,14 @@ puts "ARGV:"
 p args
 
 
+## turn on debug output
+if opts[:debug]
+   Worldfootball.debug = true
+else
+  Worldfootball.debug = false
+end
+
+
 ### check special (built-in) commands first
 if ['cache'].include?( args[0] )
     Worldfootball.list_pages
@@ -101,15 +135,14 @@ end
 #  assume leagues
 
 datasets = if opts[:file]
-              read_datasets( opts[:file] )
+              read_leagueset( opts[:file] )
            else
-              parse_datasets_args( args )
+              parse_leagueset_args( args )
            end
 
 
 ## step 0 - validate and fill-up seasons etc.
-datasets.each do |dataset|
-  league_key, seasons = dataset
+datasets.each do |league_key, seasons|
 
   league =  find_league!( league_key )  ## league info lookup
 
@@ -121,12 +154,17 @@ datasets.each do |dataset|
   ##              might be 2024/25 or 2024 or
   #                 for world cup 2022 or such
   if seasons.empty?
-     seasons = [Season(league.seasons.keys[0])]
-     dataset[1] = seasons
+     season = Season(league.seasons.keys[0])
+     seasons << season
   end
 end
 
 
+###
+## collect league names & more
+extra = {}
+
+
 ## step 1 - download
 datasets.each do |league_key, seasons|
   league =  find_league!( league_key )  ## league info lookup
@@ -152,6 +190,10 @@ datasets.each do |league_key, seasons|
         matches = page.matches
 
         puts "   #{matches.size} match(es)"
+
+        league_extra = extra[ league.key ] ||= {}
+        season_extra = league_extra[ season.key] ||= { names: [] }
+        season_extra[:names]  <<  page.title
       end
   end # each seasons
 end # each league
@@ -169,6 +211,20 @@ if opts[:convert]
 end
 
 
+if opts[:generate]
+## step 3 - generate
+  datasets.each do |league_key, seasons|
+    seasons.each do |season|
+         generate( league: league_key,
+                   season: season )
+    end
+  end
+  ## for debugging dump page (slug) titles
+  pp extra
+end
+
+
+
 end   # def self.main
 end   # module Worldfootball
 

data/bin/wfbconf

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbconf
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbconf
+
+####
+#  wfbconf - check built-in config(uration) for league
+#    e.g.     wfbconf eng.1
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+
+Worldfootball.debug = true
+
+##
+## download fresh copy?
+## Worldfootball::Metal.download_schedule( slug )
+
+key = args[0] || 'eng.1'
+
+league = Worldfootball::LEAGUES[key]
+seasons = league.seasons
+pp seasons
+
+puts "  #{key} - #{seasons.size} season(s)"
+
+puts
+latest = seasons.keys[0]
+puts " latest #{latest}: "
+pp seasons[latest]
+
+
+puts "bye"

data/bin/wfbconv

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbconv
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbconv
+
+###
+#  wfbconv -  convert (to .csv) all leagues  (if no league passed in, converts all!)
+# 
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+## convert (default) output directory
+Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
+                                           puts "  setting convert out_dir to >/sports/cache.wfb<"
+                                           '/sports/cache.wfb'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+opts = {
+    force:   false,   # a.k.a. overwrite
+ }
+
+
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+
+  parser.on( "--force",
+               "always overwrite (force) datafile - default is (#{opts[:force]})" ) do |force|
+    opts[:force] = true   # true|false
+  end
+end
+
+
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+keys =  if args.size == 0
+           Worldfootball::LEAGUES.keys
+        else
+           args
+        end
+
+
+
+keys.each_with_index do |key, i|
+  league = Worldfootball::LEAGUES[key]
+  seasons = league.seasons
+
+  puts "==> #{i+1}/#{keys.size} #{key} - #{seasons.size} seasons(s)..."
+
+  seasons.each_with_index do |season_rec,j|
+    season = season_rec[0]
+
+    next if key == 'nl.cup' && season == '1959/60'
+
+    puts "  #{j+1}/#{seasons.size} #{key} #{season}..."
+    Worldfootball.convert( league: key, season: season,
+                           overwrite: opts[:force] )
+  end
+end
+
+
+puts "bye"

data/bin/wfbdump

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbdump
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbdump
+
+
+#####
+#  wfbdump - dump matches, teams, rounds and more for (page) slug
+#    e.g.
+#    wfbdump caf-champions-league-2008
+#    wfbdump aut-bundesliga-2024-2025
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+
+Worldfootball.debug = true
+
+##
+## download fresh copy?
+## Worldfootball::Metal.download_schedule( slug )
+
+slug = args[0] || 'caf-champions-league-2008'
+
+page = Worldfootball::Page::Schedule.from_cache( slug )
+
+matches = page.matches
+teams   = page.teams
+rounds  = page.rounds
+
+puts "  #{matches.size} match(es), #{teams.size} team(s), #{rounds.size} round(s)"
+# pp matches
+
+puts
+puts "  #{teams.size} team(s)"
+pp teams
+
+puts
+puts "  #{rounds.size} round(s)"
+pp rounds
+
+
+puts "==> page props"
+print "title: "
+puts page.title
+puts
+print "keywords: "
+puts page.keywords
+puts
+print "url: "
+puts page.url
+
+
+puts "bye"

data/bin/wfbgen

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/wfbgen
+##    or
+##   ruby -I wfb/lib wfb/bin/wfbgen
+
+###############
+#  wfbgen  
+#  
+
+
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/timezones/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sportdb-writers/lib' )
+$LOAD_PATH.unshift( '/sports/sportdb/sport.db/fifa/lib' )
+
+require 'worldfootball'
+
+
+Webcache.root =  if File.exist?( '/sports/cache' )
+                     puts "  setting web cache to >/sports/cache<"
+                     '/sports/cache'
+                 else
+                     './cache'
+                 end
+
+## convert (default) output directory
+Worldfootball.config.convert.out_dir = if File.exist?( '/sports/cache.wfb')
+                                           puts "  setting convert out_dir to >/sports/cache.wfb<"
+                                           '/sports/cache.wfb'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+Worldfootball.config.generate.out_dir = if File.exist?( '/sports/cache.wfb.txt')
+                                           puts "  setting generate out_dir to >/sports/cache.wfb.txt<"
+                                           '/sports/cache.wfb.txt'
+                                        else
+                                           './tmp' ## use tmp in working dir
+                                        end
+
+
+require 'optparse'
+
+##
+#  by default convert all with overwrite/force set to false
+
+Webget.config.sleep = 2
+
+args = ARGV
+
+opts = {
+    force:   false,   # a.k.a. overwrite
+ }
+
+
+parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+
+  parser.on( "--force",
+               "always overwrite (force) datafile - default is (#{opts[:force]})" ) do |force|
+    opts[:force] = true   # true|false
+  end
+end
+
+
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+keys =  if args.size == 0
+           Worldfootball::LEAGUES.keys
+        else
+           args
+        end
+
+
+
+keys.each_with_index do |key, i|
+  league = Worldfootball::LEAGUES[key]
+  seasons = league.seasons
+
+  puts "==> #{i+1}/#{keys.size} #{key} - #{seasons.size} seasons(s)..."
+
+  seasons.each_with_index do |season_rec,j|
+    season = season_rec[0]
+
+    next if key == 'nl.cup' && season == '1959/60'
+
+    puts "  #{j+1}/#{seasons.size} #{key} #{season}..."
+    Worldfootball.generate( league: key, season: season,
+                            overwrite: opts[:force] )
+  end
+end
+
+
+puts "bye"