worldfootball 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bc6759b77de7fe44eccf59f09fd7b6cc3ddc4576612f4fb208c38a59a02536e
-  data.tar.gz: 912fe7d04165b50bc3e50f5e5a1ea554d7fd788d4d019c2b719c8625dd222a3e
+  metadata.gz: 9d824bdedb279bf91779c99445fcf5bd5a6df8acc342232624032256a32fbe18
+  data.tar.gz: 14eb224174efe18645412a88eaa332e01901247dd28ffa795cba8b902d3a1dd6
 SHA512:
-  metadata.gz: 74bf6c7e7a8aeaeb18d9bd1b74764328301b3aa3c309ca814ea494be704958a4f086e7bf94b1b1ea1ad9f974b7be31eb6e0d2a0c611b6c35b85caca06d8f6e71
-  data.tar.gz: 7cadf69fe3acafc2f976e3f5c980c8411e8758f5426559d56860c791b473debdb4e1067ad071de6704e7da3ccc434c3af14454bed6ba32ac825c2b491467db08
+  metadata.gz: b0e4885e80c6ff3fb13a00a654524ea4514fcac730438096e412cd76a5ba14c17fa563996afd81176f7538bffb6ac8bdc3c75de7af7ace2124269b7c72c0265b
+  data.tar.gz: feb02079e9c481c3b2e53d5c72030e5e9e687f94bf053c29134da613933c2f5f6b569339fb8980ab3c048e248c171c93ea23a4b4a350c5dfede8563990b5a298

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-### 0.2.5
+### 0.2.6
 
 ### 0.0.1 / 2024-07-04
 

data/Rakefile

@@ -19,6 +19,7 @@ Hoe.spec 'worldfootball' do
 
   self.extra_deps = [
     ['football-timezones'],
+    ['fifa'],
     ['webget'],
     ['nokogiri'],
   ]

data/config/rounds.csv

@@ -56,3 +56,30 @@ caf.cl, Gruppe B,  Group
 caf.cl, Gruppe C,  Group
 caf.cl, Gruppe D,  Group
 
+
+uefa.cl, Gruppe A,  Group
+uefa.cl, Gruppe B,  Group
+uefa.cl, Gruppe C,  Group
+uefa.cl, Gruppe D,  Group
+uefa.cl, Gruppe E,  Group
+uefa.cl, Gruppe F,  Group
+uefa.cl, Gruppe G,  Group
+uefa.cl, Gruppe H,  Group
+
+uefa.el, Gruppe A,  Group
+uefa.el, Gruppe B,  Group
+uefa.el, Gruppe C,  Group
+uefa.el, Gruppe D,  Group
+uefa.el, Gruppe E,  Group
+uefa.el, Gruppe F,  Group
+uefa.el, Gruppe G,  Group
+uefa.el, Gruppe H,  Group
+
+uefa.conf, Gruppe A,  Group
+uefa.conf, Gruppe B,  Group
+uefa.conf, Gruppe C,  Group
+uefa.conf, Gruppe D,  Group
+uefa.conf, Gruppe E,  Group
+uefa.conf, Gruppe F,  Group
+uefa.conf, Gruppe G,  Group
+uefa.conf, Gruppe H,  Group

data/lib/worldfootball/convert.rb

@@ -9,6 +9,9 @@ def self.convert( league:, season: )
   pages  = league.pages!( season: season )
 
 
+    ## collect all teams
+    teams_by_ref = {}
+
     recs = []
     pages.each do |slug, stage|
       ## note: stage might be nil
@@ -35,6 +38,30 @@ def self.convert( league:, season: )
       print "\n"
 
       rows = page.matches
+
+      teams = page.teams
+      ## e.g. {:count=>2, :name=>"AS Arta", :ref=>"as-arta"},
+      ##      {:count=>4, :name=>"Dekedaha FC", :ref=>"dekedaha-fc"},
+      ##        ...
+      teams.each do |h|
+          team_count = h[:count]
+          team_name  = norm_team( h[:name] )      ## note: norm team name!!!
+          team_ref   = h[:ref]
+
+          ## note: skip N.N.  (place holder team)
+          ##        team_ref is nil etc.
+          next if team_name == 'N.N.'
+
+          team_stat = teams_by_ref[ team_ref ] ||= { count: 0,
+                                                     name:  team_name }
+          team_stat[:count] += team_count
+          if team_name != team_stat[:name]
+              puts "!! ASSERT ERROR - team ref with differet names; expected #{team_stat[:name]} - got #{team_name}"
+              exit 1
+          end
+      end
+
+
       stage_recs = build( rows,
                           season: season,
                           league: league.key,
@@ -45,6 +72,60 @@ def self.convert( league:, season: )
     end
 
 
+    clubs_intl  =  ['uefa.cl', 'uefa.el', 'uefa.conf',
+                    'copa.l',
+                    'caf.cl',
+                    'afl'].include?(league.key) ? true : false
+
+    ####
+    #   auto-add (fifa) country code if int'l club tournament
+    if clubs_intl
+ ##
+ ##   get country codes for team ref
+       teams_by_ref.each do |team_slug, h|
+
+          Metal.download_team( team_slug, cache: true )
+          team_page = Page::Team.from_cache( team_slug )
+          props = team_page.props
+          pp props
+          country_name = props[:country]
+          cty = Fifa.world.find_by_name( country_name )
+          if cty.nil?
+            puts "!! ERROR - no country found for #{country_name}"
+            exit 1
+          end
+          h[:code] = cty.code
+       end
+
+       ## generate lookup by name
+       teams_by_name = teams_by_ref.reduce( {} ) do |h, (slug,rec)|
+            h[ rec[:name]] = rec
+            h
+       end
+
+    #####
+    ## dump team refs
+    puts "  #{teams_by_ref.size} team(s) by ref:"
+    pp teams_by_name
+
+    ## quick hack
+    ##  add country (fifa) codes to team names
+        recs.each do |rec|
+           team1_org  =  rec[5]
+           if team1_org != 'N.N.'   ## note - skip place holder; keep as-is
+             country_code = teams_by_name[team1_org][:code]
+             rec[5]  = "#{team1_org} (#{country_code})"
+           end
+
+           team2_org = rec[8]
+           if team2_org != 'N.N.'   ## note - skip place holder; keep as-is
+             country_code = teams_by_name[team2_org][:code]
+             rec[8]  = "#{team2_org} (#{country_code})"
+           end
+        end
+    end
+
+
 ##   note:  sort matches by date before saving/writing!!!!
 ##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
 ## note: assume date is third column!!! (stage/round/date/...)

data/lib/worldfootball/leagues.rb

@@ -24,7 +24,10 @@ class LeagueItem  # nested inside LeagueConfig
        ### todo/fix:
        ##     use from cache if not older than 1 (or 5/10?) hour(s) or such
        ##           why? why not?
-       Worldfootball::Metal.download_schedule( @slug )
+       schedule_url = Worldfootball::Metal.schedule_url( @slug )
+       if Webcache.expired_in_1d?( schedule_url )
+         Worldfootball::Metal.download_schedule( @slug )
+       end
        page = Worldfootball::Page::Schedule.from_cache( @slug )
 
      ## pp page.seasons

data/lib/worldfootball/mods.rb

@@ -24,6 +24,12 @@ def self.norm_team( team )
    ##   others too?  - move to mods instead of generic rule - why? why not?
    team = team.sub( /[ ]+\(A\)/, ' II' )
 
+   ################
+   ## quick hack - norm(alize) all N.N. to N.N.
+   ##   e.g.
+   team = 'N.N.'    if ['Sieger HF 1',
+                        'Sieger HF 2'].include?( team )
+
    team
 end
 

data/lib/worldfootball/page.rb

@@ -85,9 +85,13 @@ GENERATED_RE = %r{
      end
    end
 
+
 ######################
 ##  helper methods
 
+def debug?()  Worldfootball.debug?; end
+
+
 def squish( str )
   str = str.strip
   str = str.gsub( "\u{00A0}", ' ' )  # Unicode Character 'NO-BREAK SPACE' (U+00A0)

data/lib/worldfootball/page_schedule.rb

@@ -51,7 +51,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
 ##          <td colspan="2"></td>
 ##        </tr>
   ##
-  #   <tr class="e2-parent" data-liga_id="530" data-gs_match_id="10259222" 
+  #   <tr class="e2-parent" data-liga_id="530" data-gs_match_id="10259222"
   #       style="display:none;">
   ##    <td colspan="2"></td>
   ##    <td colspan="3">
@@ -68,7 +68,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
 
    i += 1
 
-    ## puts "[debug] row #{i} >#{tr.text.strip}<" 
+    ## puts "[debug] row #{i} >#{tr.text.strip}<"
 
     ### note - assume for now match lines use tds
     ##            and round lines use ths (NOT tds)!!
@@ -76,7 +76,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
 
     ths =  tr.css( 'th' )
     tds =  tr.css( 'td' )
-    
+
    if tr.text.strip =~ /Spieltag/ ||
       tr.text.strip =~ /[1-9]\.[ ]Runde|
                            Qual\.[ ][1-9]\.[ ]Runde|  # see EL or CL Quali
@@ -93,28 +93,30 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
                            Spiele|                # see Serie A 1960-61 Relegation
                            3\.[ ]Platz|      # see bra-serie-a-2000-yellow-module-playoffs
                            Spiel[ ]um[ ]Platz[ ]3|  # see campeonato-2009-cuadrangulares-deportivo-cuenca-cs-emelec
-                           Relegation|     # see egy-premiership-2013-2014-abstiegsplayoff    
-                           Copa[ ]Libertadores|  # see ecu-campeonato-2012-segunda-etapa-playoffs 
+                           Relegation|     # see egy-premiership-2013-2014-abstiegsplayoff
+                           Copa[ ]Libertadores|  # see ecu-campeonato-2012-segunda-etapa-playoffs
                            Copa[ ]Sudamericana|  # see campeonato-2012-liguilla-final-playoffs-cs-emelec-ldu-quito
-                           Repechaje|  # see nca-liga-primera-2023-2024-clausura-playoffs 
-                           Final[ ]de[ ]Grupos|   # see hon-liga-nacional-2020-2021-clausura-playoffs  
+                           Repechaje|  # see nca-liga-primera-2023-2024-clausura-playoffs
+                           Final[ ]de[ ]Grupos|   # see hon-liga-nacional-2020-2021-clausura-playoffs
                            Gran[ ]Final|   # see liga-nacional-2020-2021-apertura-playoffs-finale-olimpia-motagua
                            Finalrunde|  # see hon-liga-nacional-2019-2020-apertura-pentagonal
-                           Zona[ ]A|    # see gua-liga-nacional-2020-2021-clausura      
-                           Zona[ ]B|    # see liga-nacional-2020-2021-clausura-zona-a-comunicaciones-deportivo-malacateco    
+                           Zona[ ]A|    # see gua-liga-nacional-2020-2021-clausura
+                           Zona[ ]B|    # see liga-nacional-2020-2021-clausura-zona-a-comunicaciones-deportivo-malacateco
                            Interzone|    # see liga-nacional-2020-2021-clausura-zona-b-achuapa-sanarate
                            Final[ ]Segunda[ ]Ronda|    # see crc-primera-division-2018-2019-apertura-playoffs
                            Quadrangular      # see crc-primera-division-2016-2017-verano-playoffs
                            /x
-                          
-     puts
-     print '[%03d] ' % i
-     ## print squish( tr.text )
-     print "round >#{tr.text.strip}<"
-     print "\n"
+
+     if debug?
+       puts
+       print '[%03d] ' % i
+       ## print squish( tr.text )
+       print "round >#{tr.text.strip}<"
+       print "\n"
+     end
 
      last_round = tr.text.strip
-   elsif ths.count > 0 && 
+   elsif ths.count > 0 &&
          tds.count == 0
        ## check for round NOT yet configured!!!
        puts "!! WARN: found unregistered round line >#{tr.text.strip}<"
@@ -128,11 +130,12 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
 
      date_str = last_date_str    if date_str.empty?
 
-     ## note: for debugging - print as we go along (parsing)
-     print '[%03d]    ' % i
-     print "%-10s | " % date_str
-     print "%-5s | " % time_str
-
+     if debug?
+       ## note: for debugging - print as we go along (parsing)
+       print '[%03d]    ' % i
+       print "%-10s | " % date_str
+       print "%-5s | " % time_str
+     end
 
      # was: team1_str = squish( tds[2].text )
 
@@ -148,8 +151,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
        puts "!! WARN: no team1_ref for >#{team1_str}< found"
      end
 
-     ## note: for debugging - print as we go along (parsing)
-     print "%-22s | " % team1_str
+     if debug?
+       ## note: for debugging - print as we go along (parsing)
+       print "%-22s | " % team1_str
+     end
 
      ##  <td> - </td>
      ## e.g. -
@@ -168,9 +173,10 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
        puts "!! WARN: no team2_ref for >#{team2_str}< found"
      end
 
-     ## note: for debugging - print as we go along (parsing)
-     print "%-22s | " % team2_str
-
+     if debug?
+       ## note: for debugging - print as we go along (parsing)
+       print "%-22s | " % team2_str
+     end
 
 
      ### was: score_str = squish( tds[5].text )
@@ -197,11 +203,11 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
        score_str = '-:-'  # note: -:- gets replaced to ---
      end
 
-
-     print "%-10s | " % score_str
-     print (score_ref ? score_ref : 'n/a')
-     print "\n"
-
+     if debug?
+       print "%-10s | " % score_str
+       print (score_ref ? score_ref : 'n/a')
+       print "\n"
+     end
 
      ## change  2:1 (1:1)  to 2-1 (1-1)
      score_str = score_str.gsub( ':', '-' )
@@ -214,7 +220,7 @@ class Schedule < Page  ## note: use nested class for now - why? why not?
 
      date     = if date_str == '00.00.0000'
                   nil
-                else 
+                else
                   Date.strptime( date_str, '%d.%m.%Y' )
                 end
 

data/lib/worldfootball/page_team.rb

@@ -10,7 +10,100 @@ class Team < Page  ## note: use nested class for now - why? why not?
     new( html )
   end
 
+  ##  rename to properties or attributes or ??
+  ##             or infobox or data  or ?? why? why not?
+  def props
+    sidebar = doc.css( 'div.sidebar' ).first
+    assert( sidebar, 'no div.sidebar found in team page!!' )
 
+    h2  =  sidebar.css( 'h2' ).first
+    assert( h2, 'no h2 inside div.sidebar found in team page!!')
+    name = h2.text
+
+    table = sidebar.css( 'table' ).first
+    assert( table, 'no table inside div.sidebar found in team page!!')
+
+    long_name = nil
+    country   = nil
+    ground    = nil
+    founded   = nil
+
+    trs = table.css( 'tr' )
+    trs.each_with_index do |tr,i|
+      text = squish(tr.text)
+      ## puts "[#{i+1}] >#{text}<"
+
+      ## assume long name first column if NOT starting w/ Land:
+      if i==0 && !text.start_with?( 'Land' )
+        long_name = text
+      end
+
+      if text.start_with?( 'Land:' )
+         country = text.sub( 'Land:', '' ).strip
+      end
+
+      if text.start_with?( 'gegründet:' )
+         founded = text.sub( 'gegründet:', '' ).strip
+      end
+
+      if text.start_with?( 'Stadion:' )
+        ground = text.sub( 'Stadion:', '' ).strip
+      end
+    end
+
+    data =  { name: name }
+    data[ :long_name] = long_name   if long_name
+    data[ :country ] = country   if country
+    data[ :founded ] = founded   if founded && founded != '0000'
+    data[ :ground] = ground   if ground && ground != '----------'
+
+    data
+  end
+
+=begin
+<div class="sidebar">
+
+<div class="box emblemwrapper">
+<div class="head">
+<h2>Al Ahly SC</h2>
+</div>
+<div class="data " align="center">
+<div class="emblem"><a href="/teams/al-ahly-sc/"><img src="https://s.hs-data.com/bilder/wappen/mittel/1480.gif?fallback=png" border="0" width="100" hspace="5" vspace="5" alt="Al Ahly SC" title="Al Ahly SC" /></a></div>
+<div class="emblem_background"><a href="/teams/al-ahly-sc/"><img src="https://s.hs-data.com/bilder/wappen/mittel/1480.gif?fallback=png" border="0" width="100" hspace="5" vspace="5" alt="Al Ahly SC" title="Al Ahly SC" /></a></div>
+</div>
+<div class="data">
+<table class="standard_tabelle yellow" cellpadding="3" cellspacing="0">
+<tr>
+<td colspan="2" align="center">Al Ahly Sporting Club</td>
+</tr>
+<tr>
+<td colspan="2">&nbsp;</td>
+</tr>
+<tr>
+<td align="right"><b>Land:</b></td>
+<td>
+<img src="https://s.hs-data.com/bilder/flaggen_neu/68.gif" width="18" height="12" hspace="5" title="Ägypten" align="absmiddle" />
+Ägypten </td>
+</tr>
+<tr>
+<td align="right"><b>gegründet:</b></td>
+<td>24.04.1907</td>
+</tr>
+<tr>
+<td align="right"><b>Stadion:</b></td>
+<td><a href="/spielorte/international-stadium-cairo/" title="International Stadium">International Stadium</a></td>
+</tr>
+<tr>
+<td align="right"><b>Homepage:</b></td>
+<td><a href="http://alahlyegypt.com/" target="_blank">alahlyegypt.com/</a></td>
+</tr>
+<tr>
+<td colspan="2" align="right"><b><a href="/teams/al-ahly-sc/1/" title="Weitere Infos zu Al Ahly SC">zum Profil &raquo;</a></b></td>
+</tr>
+</table>
+</div>
+</div>
+=end
 
 
 ######

data/lib/worldfootball/vacuum.rb

@@ -2,15 +2,15 @@ module Worldfootball
 
 
 MAX_HEADERS = [
-'Stage',
-'Round',
-'Date',
-'Time',
-'Timezone',
-'Team 1',
-'FT',
-'HT',
-'Team 2',
+'Stage',    # 0
+'Round',    # 1
+'Date',     # 2
+'Time',     # 3
+'Timezone',  #4
+'Team 1',    #5
+'FT',      #6
+'HT',      #7
+'Team 2',   #8
 'ET',
 'P',
 'Comments',    ## e.g. awarded, cancelled/canceled, etc.

data/lib/worldfootball/version.rb

@@ -2,7 +2,7 @@
 module Worldfootball
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 2
-  PATCH = 5
+  PATCH = 6
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/worldfootball.rb

@@ -1,5 +1,7 @@
 ## 3rd party (our own)
 require 'football/timezones'  ## note - pulls in season/formats, cocos & tzinfo
+require 'fifa'
+
 require 'webget'              ## incl. webget, webcache, webclient, etc.
 require 'nokogiri'
 
@@ -31,6 +33,13 @@ require_relative 'worldfootball/convert_reports'
 
 module Worldfootball
 
+   #########
+   ## add a global debug flag
+   def self.debug=(value) @debug = value; end
+   def self.debug?()      @debug ||= false; end  ## note: default is FALSE
+
+
+
 class Configuration
   #########
   ## nested configuration classes - use - why? why not?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worldfootball
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-03 00:00:00.000000000 Z
+date: 2024-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: football-timezones
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fifa
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webget
   requirement: !ruby/object:Gem::Requirement