worldfootball 0.1.1

data/lib/worldfootball/page_report.rb

@@ -0,0 +1,186 @@
+
+module Worldfootball
+  class Page
+
+class Report < Page  ## note: use nested class for now - why? why not?
+
+
+  def self.from_cache( slug )
+    url  = Metal.report_url( slug )
+    html = Webcache.read( url )
+    new( html )
+  end
+
+
+
+  def find_table_tore
+    # <table class="" cellpadding="3" cellspacing="1">
+    # <tr>
+    #   <td colspan="2" class="ueberschrift" align="center">Tore</td>
+    # </tr>
+
+   ## get table
+   ##   first table row is Tore
+    tables = doc.css( 'table.standard_tabelle' )
+    # puts "  found #{tables.size} table.standard_tabelle" # e.g. found 6 table.standard_tabelle
+    tables.each do |table|
+      trs = table.css( 'tr' )
+      ## puts "     found #{trs.size} trs"
+      tds = trs[0].css( 'td' )
+      ## puts "     found #{tds.size} tds"
+
+      if tds.size > 0 && tds[0].text == 'Tore'
+        return table
+      end
+    end
+
+    nil  ## nothing found; auto-report error -why? why not?
+  end
+
+  def goals
+    @goals ||= begin
+
+ # <div class="data">
+ # <table class="standard_tabelle" cellpadding="3" cellspacing="1">
+
+ # puts table.class.name  #=> Nokogiri::XML::Element
+ # puts table.text
+
+ table  = find_table_tore
+ ## pp table
+
+ trs   = table.css( 'tr' )
+ # puts trs.size
+
+
+
+ rows = []
+ last_score1 = 0
+ last_score2 = 0
+
+ trs.each_with_index do |tr,i|
+
+   next if i==0   # skip Tore headline row
+
+   break if i==1 && tr.text.strip == 'keine'  ## assume 0:0 - no goals
+
+# <tr>
+#  <td class="hell" align="center" width="20%">
+#   <b>0 : 1</b>
+#  </td>
+#  <td class="hell" style="padding-left: 50px;">
+#   <a href="/spieler_profil/luis-phelipe/" title="Luis Phelipe">Luis Phelipe</a> 34. / Rechtsschuss								&nbsp;(<a href="/spieler_profil/alexander-prass/" title="Alexander Prass">Alexander Prass</a>)
+# </td>
+# </tr>
+
+       tds = tr.css( 'td' )
+
+     score_str  = squish( tds[0].text )
+
+     player_str = squish( tds[1].text )
+
+     print '[%03d] ' % i
+     print score_str
+     print " | "
+     print player_str
+     print "\n"
+
+     score_str = score_str.gsub( ':', '-' )
+     score_str = score_str.gsub( ' ', '' )   ## remove all white space
+
+
+     ### todo/fix: use new Score.split helper here
+     ## score1, score2 = Score.split( score_str )
+     parts = score_str.split('-')
+     score1 = parts[0].to_i
+     score2 = parts[1].to_i
+
+     if last_score1+1 == score1 && last_score2 == score2
+       team = 1
+     elsif last_score2+1 == score2 && last_score1 == score1
+       team = 2
+     else
+       puts "!! ERROR - unexpected score advance (one goal at a time expected):"
+       puts "  #{last_score1}-#{last_score2}=> #{score1}-#{score2}"
+       exit 1
+     end
+
+
+     last_score1 = score1
+     last_score2 = score2
+
+
+
+     if player_str.index('/')
+       parts = player_str.split('/')
+       # pp parts
+       notes = parts[1].strip
+
+       if parts[0].strip =~ /^([^0-9]+)[ ]+([0-9]+)\.$/
+         player_name = $1
+         goal_minute = $2
+          # puts " >#{player_name}< | >#{goal_minute}<"
+       else
+         puts "!! ERROR - unknown goal format (in part i):"
+         puts player_str
+         pp parts
+         exit 1
+       end
+      else  # (simple line with no divider (/)
+        #  Andrés Andrade 88.  (Nicolas Meister)
+        if m = %r{^([^0-9]+)
+                  [ ]+
+                 ([0-9]+)\.
+                  (?:
+                    [ ]+
+                    (\([^)]+\))
+                  )?
+                $}x.match( player_str )
+            player_name = m[1]
+            goal_minute = m[2]
+            notes       = m[3] ? m[3] : ''
+        else
+          puts "!! ERROR - unknown goal format:"
+          puts player_str
+          exit 1
+        end
+      end
+
+
+     ## check for "flags" e.g. own goal or penalty
+     ##   if found - remove from notes (use its own flag)
+     owngoal = false
+     penalty = false
+
+    if notes.index( 'Eigentor' )
+       owngoal = true
+       notes = notes.sub('Eigentor', '' ).strip
+    elsif notes.index( 'Elfmeter' )
+      ## e.g.  Elfmeter  (Marco Hausjell)
+       penalty = true
+       notes = notes.sub('Elfmeter', '' ).strip
+    else
+      ## nothing - keep going
+    end
+
+     rec = { score:   score_str,
+             team:    team,     # 1 or 2
+             player:  player_name,
+             minute:  goal_minute
+             }
+      rec[:owngoal] = true   if owngoal
+      rec[:penalty] = true   if penalty
+      rec[:notes]   = notes  unless notes.empty?
+
+      rows << rec
+   end ## each tr
+      rows
+   end
+ end  # goals
+
+
+end # class Report
+
+
+  end # class Page
+end # module Worldfootball

data/lib/worldfootball/page_schedule.rb

@@ -0,0 +1,292 @@
+
+module Worldfootball
+  class Page
+
+class Schedule < Page  ## note: use nested class for now - why? why not?
+
+
+  def self.from_cache( slug )
+    url  = Metal.schedule_url( slug )
+    html = Webcache.read( url )
+    new( html )
+  end
+
+
+
+  def matches
+    @matches ||= begin
+
+ # <div class="data">
+ # <table class="standard_tabelle" cellpadding="3" cellspacing="1">
+
+ ## note: use > for "strict" sibling (child without any in-betweens)
+ table = doc.css( 'div.data > table.standard_tabelle' ).first    ## get table
+ # puts table.class.name  #=> Nokogiri::XML::Element
+ # puts table.text
+
+ trs   = table.css( 'tr' )
+ # puts trs.size
+ i = 0
+
+ last_date_str = nil
+ last_round    = nil
+
+ rows = []
+
+
+## ghost trs? what for? see for an example in bra
+##    check for style display:none - why? why not?
+##
+## <tr class="e2-parent" data-liga_id="88" data-gs_match_id="9062777"
+##                         style="display:none;">
+##          <td colspan="2"></td>
+##          <td colspan="3">
+##            <span class="e2" data-liga_id="88"  data-gs_match_id="9062777"></span>
+##          </td>
+##          <td colspan="2"></td>
+##        </tr>
+
+
+ trs.each do |tr|
+
+   if tr['style'] && tr['style'].index( 'display') &&
+                     tr['style'].index( 'none')
+     puts "!! WARN: skipping ghost line >#{tr.text.strip}<"
+     next
+   end
+
+
+   i += 1
+
+   if tr.text.strip =~ /Spieltag/ ||
+      tr.text.strip =~ /[1-9]\.[ ]Runde|
+                           Qual\.[ ][1-9]\.[ ]Runde|  # see EL or CL Quali
+                           Qualifikation|     # see CA Championship
+                           Sechzehntelfinale|   # see EL
+                           Achtelfinale|
+                           Viertelfinale|
+                           Halbfinale|
+                           Finale|
+                           Gruppe[ ][A-Z]|    # see CL
+                           Playoffs           # see EL Quali
+                           /x
+     puts
+     print '[%03d] ' % i
+     ## print squish( tr.text )
+     print "round >#{tr.text.strip}<"
+     print "\n"
+
+     last_round = tr.text.strip
+   else   ## assume table row (tr) is match line
+     tds = tr.css( 'td' )
+
+     date_str  = squish( tds[0].text )
+     time_str  = squish( tds[1].text )
+
+     date_str = last_date_str    if date_str.empty?
+
+     ## note: for debugging - print as we go along (parsing)
+     print '[%03d]    ' % i
+     print "%-10s | " % date_str
+     print "%-5s | " % time_str
+
+
+     # was: team1_str = squish( tds[2].text )
+
+     ## <td><a href="/teams/hibernian-fc/" title="Hibernian FC">Hibernian FC</a></td>
+     ##  todo/check: check if tooltip title always equals text - why? why not?
+     team1_anchor = tds[2].css( 'a' )[0]
+     if team1_anchor  # note: <a> might be optional (and team name only be plain text)
+       team1_str    = squish( team1_anchor.text )
+       team1_ref    = norm_team_ref( team1_anchor[:href] )
+     else
+       team1_str    = squish( tds[2].text )
+       team1_ref    = nil
+       puts "!! WARN: no team1_ref for >#{team1_str}< found"
+     end
+
+     ## note: for debugging - print as we go along (parsing)
+     print "%-22s | " % team1_str
+
+     ##  <td> - </td>
+     ## e.g. -
+     vs_str =    squish( tds[3].text )  ## use to assert column!!!
+     assert( vs_str == '-',  "- for vs. expected; got #{vs_str}")
+     ## was: team2_str = squish( tds[4].text )
+
+     ## <td><a href="/teams/st-johnstone-fc/" title="St. Johnstone FC">St. Johnstone FC</a></td>
+     team2_anchor = tds[4].css( 'a' )[0]
+     if team2_anchor
+       team2_str    = squish( team2_anchor.text )
+       team2_ref    = norm_team_ref( team2_anchor[:href] )
+     else
+       team2_str    = squish( tds[4].text )
+       team2_ref    = nil
+       puts "!! WARN: no team2_ref for >#{team2_str}< found"
+     end
+
+     ## note: for debugging - print as we go along (parsing)
+     print "%-22s | " % team2_str
+
+
+
+     ### was: score_str = squish( tds[5].text )
+     ## <a href="/spielbericht/premiership-2020-2021-hibernian-fc-st-johnstone-fc/" title="Spielschema Hibernian FC - St. Johnstone FC">-:-</a>
+
+     score_anchor = tds[5].css( 'a' )[0]
+     if score_anchor   ## note: score ref (match report) is optional!!!!
+       score_str    = squish( score_anchor.text )
+       score_ref    = norm_score_ref( score_anchor[:href] )
+     else
+       score_str    = squish( tds[5].text )
+       score_ref    = nil
+     end
+
+
+     ##  todo - find a better way to check for live match
+     ## check for live badge image
+     ## <td>
+     ##   <img src="https://s.hs-data.com/bilder/shared/live/2.png" /></a>
+     ## </td>
+     img = tds[6].css( 'img' )[0]
+     if img && img[:src].index( '/live/')
+       puts "!! WARN: live match badge, resetting score from #{score_str} to -:-"
+       score_str = '-:-'  # note: -:- gets replaced to ---
+     end
+
+
+     print "%-10s | " % score_str
+     print (score_ref ? score_ref : 'n/a')
+     print "\n"
+
+
+     ## change  2:1 (1:1)  to 2-1 (1-1)
+     score_str = score_str.gsub( ':', '-' )
+
+     ## convert date from 25.10.2019 to 2019-25-10
+     date     = Date.strptime( date_str, '%d.%m.%Y' )
+
+     ## note: keep structure flat for now
+     ##        (AND not nested e.g. team:{text:,ref:}) - why? why not?
+     rows << { round:      last_round,
+               date:       date.strftime( '%Y-%m-%d' ),
+               time:       time_str,
+               team1:      team1_str,
+               team1_ref:  team1_ref,
+               score:      score_str,
+               team2:      team2_str,
+               team2_ref:  team2_ref,
+               report_ref: score_ref
+             }
+
+     last_date_str = date_str
+   end
+  end # each tr (table row)
+
+    rows
+   end
+ end  # matches
+
+
+
+ def teams
+  @teams ||= begin
+     h = {}
+     matches.each do |match|
+       ## index by name/text for now NOT ref - why? why not?
+       [{text: match[:team1],
+         ref:  match[:team1_ref]},
+        {text: match[:team2],
+         ref:  match[:team2_ref]}].each do |team|
+         rec = h[ team[:text] ] ||= { count: 0,
+                                      name: team[ :text],
+                                      ref:  team[ :ref ] }
+         rec[ :count ] += 1
+         ## todo/check:  check/assert that name and ref are always equal - why? why not?
+       end
+     end
+
+     h.values
+  end
+ end
+
+ def rounds
+  @rounds ||= begin
+     h = {}
+     matches.each do |match|
+       rec = h[ match[:round] ] ||= { count: 0,
+                                      name: match[ :round] }
+        rec[ :count ] += 1
+     end
+
+     h.values
+  end
+ end
+
+
+ def seasons
+  # <select name="saison" ...
+  @seasons ||= begin
+     recs = []
+     season = doc.css( 'select[name="saison"]').first
+     options = season.css( 'option' )
+
+     options.each do |option|
+        recs << { text: squish( option.text ),
+                  ref:  norm_season_ref( option[:value] )
+                }
+     end
+     recs
+  end
+end
+
+
+######
+## helpers
+
+## todo/check - rename/use HREF and not REF - why? why not?
+REF_SCORE_RE = %r{^/spielbericht/
+                     ([a-z0-9_-]+)/$}x
+
+def norm_score_ref( str )
+  ## check ref format / path
+  if m=REF_SCORE_RE.match( str )
+    m[1]
+  else
+    puts "!! ERROR: unexpected score href format >#{str}<"
+    exit 1
+  end
+end
+
+
+REF_TEAM_RE  = %r{^/teams/
+                     ([a-z0-9_-]+)/$}x
+
+def norm_team_ref( str )
+  ## check ref format / path
+  if m=REF_TEAM_RE.match( str )
+    m[1]
+  else
+    puts "!! ERROR: unexpected team href format >#{str}<"
+    exit 1
+  end
+end
+
+
+REF_SEASON_RE = %r{^/alle_spiele/
+                        ([a-z0-9_-]+)/$}x
+
+def norm_season_ref( str )
+  ## check ref format / path
+  if m=REF_SEASON_RE.match( str )
+    m[1]
+  else
+    puts "!! ERROR: unexpected season href format >#{str}<"
+    exit 1
+  end
+end
+end # class Schedule
+
+
+  end # class Page
+end # module Worldfootball

data/lib/worldfootball/vacuum.rb

@@ -0,0 +1,66 @@
+module Worldfootball
+
+
+MAX_HEADERS = [
+'Stage',
+'Round',
+'Date',
+'Time',
+'Team 1',
+'FT',
+'HT',
+'Team 2',
+'ET',
+'P',
+'Comments']    ## e.g. awarded, cancelled/canceled, etc.
+
+MIN_HEADERS = [   ## always keep even if all empty
+'Date',
+'Team 1',
+'FT',
+'Team 2'
+]
+
+def self.vacuum( rows, headers: MAX_HEADERS, fixed_headers: MIN_HEADERS )
+  ## check for unused columns and strip/remove
+  counter = Array.new( MAX_HEADERS.size, 0 )
+  rows.each do |row|
+     row.each_with_index do |col, idx|
+       counter[idx] += 1  unless col.nil? || col.empty?
+     end
+  end
+
+  pp counter
+
+  ## check empty columns
+  headers       = []
+  indices       = []
+  empty_headers = []
+  empty_indices = []
+
+  counter.each_with_index do |num, idx|
+     header = MAX_HEADERS[ idx ]
+     if num > 0 || (num == 0 && fixed_headers.include?( header ))
+       headers << header
+       indices << idx
+     else
+       empty_headers << header
+       empty_indices << idx
+     end
+  end
+
+  if empty_indices.size > 0
+    rows = rows.map do |row|
+             row_vacuumed = []
+             row.each_with_index do |col, idx|
+               ## todo/fix: use values or such??
+               row_vacuumed << col   unless empty_indices.include?( idx )
+             end
+             row_vacuumed
+         end
+    end
+
+  [rows, headers]
+end
+
+end # module Worldfootball

data/lib/worldfootball/version.rb

@@ -0,0 +1,20 @@
+
+module Worldfootball
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 1
+  PATCH = 1
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "worldfootball/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}] in (#{root})"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
+  end
+end   # module Worldfootball
+

data/lib/worldfootball.rb

@@ -0,0 +1,66 @@
+## 3rd party (our own)
+require 'season/formats'   ## add season support
+require 'webget'           ## incl. webget, webcache, webclient, etc.
+
+require 'cocos'
+
+## 3rd party
+require 'nokogiri'
+
+
+
+###
+# our own code
+require_relative 'worldfootball/version'
+require_relative 'worldfootball/leagues'
+require_relative 'worldfootball/download'
+require_relative 'worldfootball/page'
+require_relative 'worldfootball/page_schedule'
+require_relative 'worldfootball/page_report'
+
+
+require_relative 'worldfootball/mods'
+require_relative 'worldfootball/vacuum'
+require_relative 'worldfootball/build'
+require_relative 'worldfootball/convert'
+require_relative 'worldfootball/convert_reports'
+
+
+require_relative 'worldfootball/generator'
+
+
+
+module Worldfootball
+
+class Configuration
+  #########
+  ## nested configuration classes - use - why? why not?
+  class Convert
+     def out_dir()       @out_dir || './o'; end
+     def out_dir=(value) @out_dir = value; end
+  end
+
+ def convert()  @convert ||= Convert.new; end
+end # class Configuration
+
+## lets you use
+##   Worldfootball.configure do |config|
+##      config.convert.out_dir = './o'
+##   end
+def self.configure() yield( config ); end
+def self.config()    @config ||= Configuration.new;  end
+
+end   # module Worldfootball
+
+
+
+
+### for processing tool
+##   (auto-)add sportdb/writer  (pulls in sportdb/catalogs and gitti)
+#  require 'sportdb/writers'
+
+
+
+
+puts Worldfootball.banner    ## say hello
+