worldfootball 0.3.0 → 0.3.2

data/lib/worldfootball/build.rb

@@ -73,11 +73,29 @@ def self.build( rows, season:, league:, stage: '' )   ## rename to fixup or such
 
     date_str  = row[:date]
     time_str  = row[:time]
+
+
     team1_str = row[:team1]
     team2_str = row[:team2]
-    score_str = row[:score]
+   
+    ## R. Betis / Jag. Białystok v Celje / Fiorentina
+    ## Legia Warszawa / Chelsea v Djurgårdens / Rapid Wien
+    team1_ref = row[:team1_ref]
+    team2_ref = row[:team2_ref]
+ 
+    ##  reset team name to N.N.  if team_ref is nil and name include " / " 
+    if team1_ref.nil? && team1_str.index( ' / ')
+      puts "!! WARN - change placeholder team #{team1_str} to N.N."
+      team1_str = 'N.N.'
+    end 
+    if team2_ref.nil? && team2_str.index( ' / ')
+      puts "!! WARN - change placeholder team #{team2_str} to N.N."
+      team2_str = 'N.N.'
+    end 
 
 
+    score_str = row[:score]
+
 
     ### check for score_error; first (step 1) lookup by date
     score_error = score_errors[ date_str ]

data/lib/worldfootball/convert.rb

@@ -2,13 +2,23 @@
 module Worldfootball
 
 
-def self.convert( league:, season: )
+def self.convert( league:, season:,
+                  overwrite: true )
   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
 
   league = find_league!( league )
   pages  = league.pages!( season: season )
 
 
+  out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
+  if !overwrite && File.exist?( out_path )
+    ## skip generation
+    puts "  OK #{league.key} #{season}   (do NOT overwrite)"
+    return
+  end
+
+
+
     ## collect all teams
     teams_by_ref = {}
 
@@ -43,22 +53,35 @@ def self.convert( league:, season: )
       ## e.g. {:count=>2, :name=>"AS Arta", :ref=>"as-arta"},
       ##      {:count=>4, :name=>"Dekedaha FC", :ref=>"dekedaha-fc"},
       ##        ...
+      ###   {:count=>2, :name=>"Arsenal / Real Madrid", :ref=>nil},
+      ##   {:count=>2, :name=>"PSG / Aston Villa", :ref=>nil},
+      ##   {:count=>2, :name=>"Barcelona / Dortmund", :ref=>nil},
+      ##   {:count=>2, :name=>"Bayern / Inter", :ref=>nil},
+      ##   {:count=>1, :name=>"Sieger HF 1", :ref=>nil},
+      ##  {:count=>1, :name=>"Sieger HF 2", :ref=>nil}]
+
+
       teams.each do |h|
           team_count = h[:count]
           team_name  = norm_team( h[:name] )      ## note: norm team name!!!
           team_ref   = h[:ref]
 
+###
+##   quick fix for broken refs/links
+##    olympique-lyon => olympique-lyonnais
+       #   team_ref = 'olympique-lyonnais'   if team_ref == 'olympique-lyon'
+
           ## note: skip N.N.  (place holder team)
           ##        team_ref is nil etc.
           next if team_name == 'N.N.'
-
+          ### warn if team_ref is nil and skip !!!
+          next if team_ref.nil?
+      
+          
           team_stat = teams_by_ref[ team_ref ] ||= { count: 0,
-                                                     name:  team_name }
+                                                     names:  [] }
           team_stat[:count] += team_count
-          if team_name != team_stat[:name]
-              puts "!! ASSERT ERROR - team ref with differet names; expected #{team_stat[:name]} - got #{team_name}"
-              exit 1
-          end
+          team_stat[:names] << team_name   unless team_stat[:names].include?( team_name )
       end
 
 
@@ -73,9 +96,12 @@ def self.convert( league:, season: )
 
 
     clubs_intl  =  ['uefa.cl', 'uefa.el', 'uefa.conf',
-                    'copa.l',
+                    'uefa.cl.q', 'uefa.el.q', 'uefa.conf.q',
+                     'copa.l', 'copa.s',
+                    'concacaf.cl',
                     'caf.cl',
-                    'afl'].include?(league.key) ? true : false
+                    'afl',
+                  ].include?(league.key) ? true : false
 
     ####
     #   auto-add (fifa) country code if int'l club tournament
@@ -83,7 +109,6 @@ def self.convert( league:, season: )
  ##
  ##   get country codes for team ref
        teams_by_ref.each do |team_slug, h|
-
           Metal.download_team( team_slug, cache: true )
           team_page = Page::Team.from_cache( team_slug )
           props = team_page.props
@@ -99,26 +124,36 @@ def self.convert( league:, season: )
 
        ## generate lookup by name
        teams_by_name = teams_by_ref.reduce( {} ) do |h, (slug,rec)|
-            h[ rec[:name]] = rec
-            h
+  ### todo/fix
+  ##    report warning if names size is > 1!!!!
+  ##
+             rec[:names].each do |name|
+                h[ name ] = rec
+              end
+              h
        end
 
+
     #####
     ## dump team refs
     puts "  #{teams_by_ref.size} team(s) by ref:"
-    pp teams_by_name
+    pp teams_by_ref
 
     ## quick hack
     ##  add country (fifa) codes to team names
+    ##
+    ##  note - some placeholder teams have no ref!! and, thus no entry with country code
         recs.each do |rec|
            team1_org  =  rec[5]
-           if team1_org != 'N.N.'   ## note - skip place holder; keep as-is
+           if team1_org != 'N.N.' &&  ## note - skip place holder; keep as-is
+              teams_by_name[team1_org]
              country_code = teams_by_name[team1_org][:code]
              rec[5]  = "#{team1_org} (#{country_code})"
            end
 
            team2_org = rec[8]
-           if team2_org != 'N.N.'   ## note - skip place holder; keep as-is
+           if team2_org != 'N.N.' &&   ## note - skip place holder; keep as-is
+              teams_by_name[team2_org]
              country_code = teams_by_name[team2_org][:code]
              rec[8]  = "#{team2_org} (#{country_code})"
            end
@@ -129,10 +164,22 @@ def self.convert( league:, season: )
 ##   note:  sort matches by date before saving/writing!!!!
 ##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
 ## note: assume date is third column!!! (stage/round/date/...)
-recs = recs.sort { |l,r| l[2] <=> r[2] }
+
+### note - do NOT sort for now
+##    keep "original" page order - why? why not?
+## recs = recs.sort { |l,r| l[2] <=> r[2] }
+
+
 ## reformat date / beautify e.g. Sat Aug 7 1993
 recs.each do |rec|
-         rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+            if rec[2]
+              if rec[2] =~ /^\d{4}-\d{1,2}-\d{1,2}$/
+               rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' )
+              else
+                ## report unknown date format warning
+                puts "WARN - unsupported date format (cannot parse?) >#{rec[2]}<"
+              end
+            end
        end
 
    ## remove unused columns (e.g. stage, et, p, etc.)
@@ -141,10 +188,13 @@ recs.each do |rec|
    puts headers
    pp recs[0]   ## check first record
 
-   out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
-
-   puts "write #{out_path}..."
+   puts "   writing to >#{out_path}< - #{recs.size} record(s)..."
    write_csv( out_path, recs, headers: headers )
+
+   ## add to tmp too for debugging
+   out_path2 = "#{config.convert.out_dir}/tmp/#{league.key}/#{season.to_path}.csv"
+   puts "   writing to >#{out_path2}< - #{recs.size} record(s)..."
+   write_csv( out_path2, recs, headers: headers )
 end
 end # module Worldfootball
 

data/lib/worldfootball/download.rb

@@ -4,12 +4,16 @@ module Worldfootball
 
 #################
 ##  porcelain "api"
-def self.schedule( league:, season: )
+def self.schedule( league:, season:, overwrite: true )
   season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
 
   pages = find_league_pages!( league: league, season: season )
   pages.each do |slug, _|
-    Metal.download_schedule( slug )
+    if !overwrite && Webcache.cached?( Metal.schedule_url( slug ))
+      puts "  OK #{league} #{season}  - #{slug}   (do NOT overwrite)"
+    else
+      Metal.download_schedule( slug )
+    end
   end # each page
 end
 

data/lib/worldfootball/leagues.rb

@@ -61,9 +61,12 @@ class LeagueItem  # nested inside LeagueConfig
         season, stage = text.split( ' ', 2 )
 
 ## todo/fix: add a waring here and auto log to logs.txt!!!!
-        next if season == '2019-2021'
-        next if season == '1958/1960'
-        next if season == '1955/1958'
+        if ['2019-2021',
+            '1958/1960',
+            '1955/1958'  ].include?( season )
+           log( "!! WARN - seasons for league #{@key} incl. invalid season #{season} - slug #{slug}; skipping season" )
+           next   ## note - skip invalid season entry
+        end
 
         season = Season.parse( season )
 
@@ -106,6 +109,13 @@ class LeagueItem  # nested inside LeagueConfig
     recs = seasons[season.key]
     recs ?  recs.reverse : nil
   end
+
+  def log( msg )  ### append to log
+    File.open( './logs.txt', 'a:utf-8' ) do |f|
+      f.write( msg )
+      f.write( "\n" )
+    end
+  end
 end # class LeagueItem
 
 
@@ -122,6 +132,9 @@ def size()   @table.size; end
 end # class LeagueConfig
 
 
+
+
+
 LEAGUES = LeagueConfig.new
 ['africa',
  'america',

data/lib/worldfootball/mods.rb

@@ -10,11 +10,20 @@ module Worldfootball
 def self.norm_team( team )
    ## clean team name and asciify (e.g. ’->' )
    team = team.sub( '(old)', '' ).strip
-   team = team.gsub( '’', "'" )     ## e.g. Hawke’s Bay United FC
 
-   ## Criciúma - SC     =>     Criciúma - SC
-   ## Bahia - BA        =>     Bahia - BA
+   ## e.g. Hawke’s Bay United FC   or
+   ##         ASC Monts d`Or Chasselay   or
+   ##        VV Heerlen ´16  / EMM ´15 / Wormer SV´30 / Swift ´36 / etc.
+   team = team.gsub( /[’´`]/, "'" )
+
+
+   ## br
+   ## Criciúma - SC      =>     Criciúma SC
+   ## Bahia - BA         =>     Bahia BA
+   ##  cz
+   ## Baník Most - Souš  =>     Baník Most Souš
    ##  remove inline dash ( - ) with single space
+   ##  to log
    team = team.gsub( /[ ]+[-][ ]+/, ' ' )
 
 
@@ -24,6 +33,22 @@ def self.norm_team( team )
    ##   others too?  - move to mods instead of generic rule - why? why not?
    team = team.sub( /[ ]+\(A\)/, ' II' )
 
+##
+##  remove ()  - used/reserved for country code for now - why? why not?
+##  e.g. Lloyds FC (Sittingbourne)  => Lloyds FC Sittingbourne
+##       August 1st (Army Team)     => August 1st Army Team
+##
+## add warning - why? why not?
+   team = team.sub( /\(
+                        ([^)]+?)   ## eat-up all non-greed to next )
+                     \)/x, '\1' )
+
+##
+## strip special case
+##   MFK Frýdek-Místek, a.s.   =>  MFK Frýdek-Místek
+     team = team.sub( ', a.s.', '' )
+
+
    ################
    ## quick hack - norm(alize) all N.N. to N.N.
    ##   e.g.

data/lib/worldfootball/page_schedule.rb

@@ -12,6 +12,15 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
   end
 
 
+  PLACEHOLDERS = [
+     'N.N.',
+     'Verlierer HF 1',
+     'Verlierer HF 2',
+     'Sieger HF 1',
+     'Sieger HF 2',
+    ]
+  def placeholder?( str )  PLACEHOLDERS.include?( str ); end
+
 
   def matches
     @matches ||= begin
@@ -20,10 +29,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
  # <table class="standard_tabelle" cellpadding="3" cellspacing="1">
 
  ## note: use > for "strict" sibling (child without any in-betweens)
- table = doc.css( 'div.data > table.standard_tabelle' ).first    ## get table
+ tables = doc.css( 'div.data > table.standard_tabelle' )    ## get table
  # puts table.class.name  #=> Nokogiri::XML::Element
  # puts table.text
 
+ assert( tables.size==1, "expected one table.standard_tabelle; got #{tables.size}" )
+ table = tables.first
  assert( table, 'no table.standard_tabelle found in schedule page!!')
 
  trs   = table.css( 'tr' )
@@ -77,8 +88,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
     ths =  tr.css( 'th' )
     tds =  tr.css( 'td' )
 
-   if tr.text.strip =~ /Spieltag/ ||
-      tr.text.strip =~ /[1-9]\.[ ]Runde|
+    tr_text = squish( tr.text )
+
+   if tr_text =~ /Spieltag/ ||
+      tr_text =~ /[1-9]\.[ ]Runde|
                            Qual\.[ ][1-9]\.[ ]Runde|  # see EL or CL Quali
                            Qualifikation|     # see CA Championship
                            Sechzehntelfinale|   # see EL
@@ -110,25 +123,24 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      if debug?
        puts
        print '[%03d] ' % i
-       ## print squish( tr.text )
-       print "round >#{tr.text.strip}<"
+       print "round >#{tr_text}<"
        print "\n"
      end
 
-     last_round = tr.text.strip
+     last_round = tr_text
    elsif ths.count > 0 &&
          tds.count == 0
        ## check for round NOT yet configured!!!
-       puts "!! WARN: found unregistered round line >#{tr.text.strip}<"
-       log( "!! WARN: found unregistered round line >#{tr.text.strip}< in page #{title}" )
+       puts "!! WARN: found unregistered round line >#{tr_text}<"
+       log( "!! WARN: found unregistered round line >#{tr_text}< in page #{title}" )
 
-       last_round = tr.text.strip
+       last_round = tr_text
    else   ## assume table row (tr) is match line
 
      date_str  = squish( tds[0].text )
      time_str  = squish( tds[1].text )
 
-     date_str = last_date_str    if date_str.empty?
+     date_str = last_date_str    if date_str.empty? && last_date_str
 
      if debug?
        ## note: for debugging - print as we go along (parsing)
@@ -148,7 +160,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      else
        team1_str    = squish( tds[2].text )
        team1_ref    = nil
-       puts "!! WARN: no team1_ref for >#{team1_str}< found"
+       puts "!! WARN: no team1_ref for >#{team1_str}< found"  unless placeholder?( team1_str )
      end
 
      if debug?
@@ -170,9 +182,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      else
        team2_str    = squish( tds[4].text )
        team2_ref    = nil
-       puts "!! WARN: no team2_ref for >#{team2_str}< found"
+       puts "!! WARN: no team2_ref for >#{team2_str}< found"  unless placeholder?( team2_str )
      end
 
+
      if debug?
        ## note: for debugging - print as we go along (parsing)
        print "%-22s | " % team2_str
@@ -198,8 +211,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      ##   <img src="https://s.hs-data.com/bilder/shared/live/2.png" /></a>
      ## </td>
      img = tds[6].css( 'img' )[0]
+
+
+
      if img && img[:src].index( '/live/')
-       puts "!! WARN: live match badge, resetting score from #{score_str} to -:-"
+       ## puts "!! WARN: live match badge, resetting score from #{score_str} to -:-"
+       print " LIVE BADGE "
        score_str = '-:-'  # note: -:- gets replaced to ---
      end
 
@@ -217,8 +234,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
      ## special case for '00.00.0000'
      ##   CANNOT parse
      ##   use empty date - why? why not?
+     ##   if start with 00.00. e.g. 00.00.1939
+
 
-     date     = if date_str == '00.00.0000'
+     date     = if date_str == '00.00.0000' ||
+                   date_str.start_with?( '00.00.' ) ||
+                   date_str.empty?
                   nil
                 else
                   Date.strptime( date_str, '%d.%m.%Y' )
@@ -237,7 +258,9 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
                report_ref: score_ref
              }
 
-     last_date_str = date_str
+     ## note - only update last date if date present
+     ##             might be empty (not available) in the beginning
+     last_date_str = date_str      if !date_str.empty?
    end
   end # each tr (table row)
 

data/lib/worldfootball/version.rb

@@ -2,7 +2,7 @@
 module Worldfootball
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 3
-  PATCH = 0
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/worldfootball.rb

@@ -41,6 +41,14 @@ module Worldfootball
    def self.debug=(value) @debug = value; end
    def self.debug?()      @debug ||= false; end  ## note: default is FALSE
 
+   def self.log( msg )  ### append to log
+      File.open( './logs.txt', 'a:utf-8' ) do |f|
+        f.write( msg )
+        f.write( "\n" )
+      end
+   end
+
+
 
 
 class Configuration
@@ -71,6 +79,60 @@ end   # module Worldfootball
 
 
 
+###
+# todo - move generate to generate file!!!
+module Worldfootball
+def self.generate( league:, season:,
+                   overwrite: true )
+   season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+   league = find_league!( league )
+   pages  = league.pages!( season: season )
+
+
+   out_path = if season >= Season( '2000' )
+                "#{config.generate.out_dir}/#{season.to_path}/#{league.key}.txt"
+              else
+                decade = season.start_year - (season.start_year%10)
+                ## use archive-style before 2000!!!
+                "#{config.generate.out_dir}/archive/#{decade}s/#{season.to_path}/#{league.key}.txt"
+              end
+
+   ## check if output exists already
+   if !overwrite && File.exist?( out_path )
+     ## skip generation
+     puts "  OK #{league.key} #{season}   (do NOT overwrite)"
+     return
+   end
+
+
+   ## get matches
+   path = "#{config.convert.out_dir}/#{season.to_path}/#{league.key}.csv"
+   puts "  ---> reading matches in #{path} ..."
+   matches = SportDb::CsvMatchParser.read( path )
+   puts "     #{matches.size} matches"
+
+   ## build
+   txt = SportDb::TxtMatchWriter.build( matches )
+   puts txt
+
+
+   buf = String.new
+   ## note - use league key for league name for now!!
+   buf << "= #{league.key.upcase.gsub('.', ' ')} #{season.key}\n\n"
+   buf << txt
+
+   puts "   writing to >#{out_path}<..."
+   write_text( out_path, buf )
+
+   ## add to tmp too for debugging
+   out_path2 = "#{config.generate.out_dir}/tmp/#{league.key}/#{season.to_path}.txt"
+   puts "   writing to >#{out_path2}<..."
+   write_text( out_path2, buf )
+end
+end   # module Worldfootball
+
+
 
 
 puts Worldfootball.banner    ## say hello

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worldfootball
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-06 00:00:00.000000000 Z
+date: 2025-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: football-timezones
@@ -106,19 +106,25 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
 description: worldfootball - get world football (leagues, cups & more) match data
   via the worldfootball.net/weltfussball.de pages
 email: gerald.bauer@gmail.com
 executables:
 - wfb
+- wfbconf
+- wfbconv
+- wfbdump
+- wfbgen
+- wfbsync
+- wfbup
 extensions: []
 extra_rdoc_files:
 - CHANGELOG.md
@@ -130,6 +136,12 @@ files:
 - README.md
 - Rakefile
 - bin/wfb
+- bin/wfbconf
+- bin/wfbconv
+- bin/wfbdump
+- bin/wfbgen
+- bin/wfbsync
+- bin/wfbup
 - config/leagues/africa.csv
 - config/leagues/america.csv
 - config/leagues/asia.csv
@@ -175,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: worldfootball - get world football (leagues, cups & more) match data via