worldfootball 0.1.1

data/lib/worldfootball/leagues/europe--southern.rb

@@ -0,0 +1,86 @@
+
+module Worldfootball
+
+LEAGUES_EUROPE.merge!({
+
+  # /ita-serie-a-2019-2020/
+  # /ita-serie-b-2020-2021/
+  'it.1' =>  { pages: 'ita-serie-a' },
+  'it.2' =>  { pages: 'ita-serie-b' },
+
+  # /por-primeira-liga-2019-2020/
+  # por-primeira-liga-2020-2021
+  # por-primeira-liga-2019-2020
+  # por-primeira-liga-2018-2019
+  # por-primeira-liga-2017-2018
+  # por-primeira-liga-2016-2017
+  # por-primeira-liga-2015-2016
+  # por-primeira-liga-2014-2015
+  # por-primeira-liga-2013-2014
+  # por-liga-zon-sagres-2012-2013
+  # por-liga-zon-sagres-2011-2012
+  # por-liga-sagres-2010-2011
+  # ...
+  # /por-segunda-liga-2019-2020/
+  # note: Sponsorship names for seasons
+  #  2002–2005: SuperLiga GalpEnergia
+  #  2005–2006: Liga betandwin.com
+  #  2006–2008: BWINLIGA
+  #  2008–2010: Liga Sagres
+  #  2010–2014: Liga ZON Sagres
+  #  2014–2020: Liga NOS
+  'pt.1' =>  {
+    pages: ['por-primeira-liga',
+            'por-liga-zon-sagres',
+            'por-liga-sagres'
+           ],
+    season: ->( season ) {
+            case season
+            when Season('2013/14')..Season('2020/21') then 1
+            when Season('2011/12')..Season('2012/13') then 2
+            when Season('2010/11')                    then 3
+            end
+    },
+   },
+  'pt.2' =>  { pages: 'por-segunda-liga' },
+
+  # /esp-primera-division-2019-2020/
+  'es.1'  => { pages: 'esp-primera-division' },
+  'es.2'  => { pages: 'esp-segunda-division' },
+
+  # /tur-sueperlig-2020-2021/
+  'tr.1'  => { pages: 'tur-sueperlig' },
+  'tr.2'  => { pages: 'tur-1-lig' },
+
+
+  # /gre-super-league-2020-2021/
+  'gr.1'  => {
+    pages: {
+      ## note: change from superleague to super-league !!!
+      'gre-super-league'                             => 'Regular Season',
+      'gre-super-league-{season}-meisterschaft'      => 'Playoffs - Championship',
+      'gre-super-league-{season}-abstieg'            => 'Playoffs - Relegation',
+      'gre-superleague'                              => 'Regular Season',
+      'gre-superleague-{end_year}-playoffs'          => 'Playoffs',
+      'gre-superleague-{end_year}-spiel-um-platz-6'  => 'Match 6th Place',
+    },
+    season: ->( season ) {
+            case season
+            when Season('2020/21') then [1]    ## just getting started
+            when Season('2019/20') then [4,2,3]
+            when Season('2017/18')..Season('2018/19') then 4
+            when Season('2013/14')..Season('2016/17') then [4,5]
+            when Season('2012/13')                    then [4,5,6]
+            when Season('2010/11')..Season('2011/12') then [4,5]
+            end
+    },
+  },
+
+
+  ## todo/check: add europe southeastern or balkans - why? why not?
+  # e.g. /cro-1-hnl-2020-2021/
+  'hr.1' => { pages: 'cro-1-hnl' },
+
+})
+
+end

data/lib/worldfootball/leagues/europe--western.rb

@@ -0,0 +1,38 @@
+
+module Worldfootball
+
+LEAGUES_EUROPE.merge!({
+  'fr.1'  => { pages: 'fra-ligue-1' },
+  'fr.2'  => { pages: 'fra-ligue-2' },
+
+  # e.g. /lux-nationaldivision-2020-2021/
+  'lu.1' => { pages: 'lux-nationaldivision' },
+
+  # e.g. /ned-eredivisie-2020-2021/
+  'nl.1' => { pages: 'ned-eredivisie' },
+  # Championship play-offs
+  # Europa League play-offs (Group A + Group B / Finals )
+
+  # e.g. /bel-eerste-klasse-a-2020-2021/
+  #      /bel-europa-league-playoffs-2018-2019-playoff/
+  #       - Halbfinale
+  #       - Finale
+  'be.1' => {
+    pages: {
+      'bel-eerste-klasse-a-{season}'                => 'Regular Season',
+      'bel-eerste-klasse-a-{season}-playoff-i'      => 'Playoffs - Championship',
+      'bel-europa-league-playoffs-{season}'         => 'Playoffs - Europa League',           ## note: missing groups (A & B)
+      'bel-europa-league-playoffs-{season}-playoff' => 'Playoffs - Europa League - Finals',
+    },
+    season: ->( season ) {
+      case season
+      when Season('2020/21') then [1]     # just getting started
+      when Season('2019/20') then [1]     # covid-19 - no championship & europa
+      when Season('2018/19') then [1,2,3,4]
+      end
+    }
+  },
+
+})
+
+end

data/lib/worldfootball/leagues/europe.rb

@@ -0,0 +1,13 @@
+
+module Worldfootball
+  LEAGUES_EUROPE = {}
+end
+
+
+require_relative 'europe--western'
+require_relative 'europe--british_isles'
+require_relative 'europe--northern'
+require_relative 'europe--central'
+require_relative 'europe--eastern'
+require_relative 'europe--southern'
+

data/lib/worldfootball/leagues/north_america.rb

@@ -0,0 +1,44 @@
+module Worldfootball
+
+LEAGUES_NORTH_AMERICA = {
+
+# todo/fix: adjust date/time by -6 or 7 hours!!!
+#   /can-canadian-championship-2020/
+#     - Qual. 1. Runde
+#     - Qual. 2. Runde
+#     - Qual. 3. Runde
+#   todo/fix: check for leagues - premier league? championship? soccer league?
+#  'ca.1' => { slug: 'can-canadian-championship' },
+
+
+
+# todo/fix: adjust date/time by -7 hours!!!
+##  e.g. 25.07.2020	02:30  => 24.07.2020 19.30
+#        11.01.2020	04:00  => 10.01.2020 21.00
+#
+# e.g. /mex-primera-division-2020-2021-apertura/
+#      /mex-primera-division-2019-2020-clausura/
+#      /mex-primera-division-2019-2020-apertura-playoffs/
+#        - Viertelfinale
+#        - Halbfinale
+#        - Finale
+#      /mex-primera-division-2018-2019-clausura-playoffs/
+'mx.1' => {
+  pages: {
+    'mex-primera-division-{season}-apertura'          => 'Apertura',            # 1
+    'mex-primera-division-{season}-apertura-playoffs' => 'Apertura - Liguilla', # 2
+    'mex-primera-division-{season}-clausura'          => 'Clausura',            # 3
+    'mex-primera-division-{season}-clausura-playoffs' => 'Clausura - Liguilla', # 4
+ },
+ season: ->( season ) {
+  case season
+  when Season('2020/21') then [1]        # just getting started
+  when Season('2019/20') then [1,2,3]    # covid-19 - no liguilla
+  when Season('2010/11')..Season('2018/19') then [1,2,3,4]
+  end
+ }
+},
+}
+
+end # module Worldfootball
+

data/lib/worldfootball/leagues/pacific.rb

@@ -0,0 +1,21 @@
+module Worldfootball
+
+LEAGUES_PACIFIC = {
+
+  # /nzl-nz-football-championship-2019-2020/
+  # /nzl-nz-football-championship-2018-2019-playoffs/
+  'nz.1' => {
+    pages: {
+      'nzl-nz-football-championship-{season}'          => 'Regular Season', # 1
+      'nzl-nz-football-championship-{season}-playoffs' => 'Playoff Finals', # 2
+     },
+     season: ->( season ) {
+      case season
+      when Season('2019/20') then [1]    ## covid-19 - no playoffs/finals
+      when Season('2018/19') then [1,2]
+      end
+     }
+  },
+}
+
+end # module Worldfootball

data/lib/worldfootball/leagues/south_america.rb

@@ -0,0 +1,11 @@
+module Worldfootball
+
+  LEAGUES_SOUTH_AMERICA = {
+
+    # todo/fix: adjust date/time by -6 or 7 hours!!!
+    # /bra-serie-a-2020/
+    'br.1' => { pages: 'bra-serie-a' },
+  }
+
+end # module Worldfootball
+

data/lib/worldfootball/leagues.rb

@@ -0,0 +1,200 @@
+
+
+require_relative 'leagues/europe'
+require_relative 'leagues/north_america'
+require_relative 'leagues/south_america'
+require_relative 'leagues/pacific'
+require_relative 'leagues/asia'
+
+
+module Worldfootball
+
+LEAGUES = [LEAGUES_EUROPE,
+           LEAGUES_NORTH_AMERICA,
+           LEAGUES_SOUTH_AMERICA,
+           LEAGUES_PACIFIC,
+           LEAGUES_ASIA].reduce({}) { |mem,h| mem.merge!( h ); mem }
+
+
+class League
+    def initialize( key, data )
+      @key  = key
+      ## @data = data
+
+      @pages       = data[:pages]
+      @season_proc = data[:season] || ->(season) { nil }
+    end
+
+    def key()   @key; end
+
+    def pages( season: )
+      ## note: return for no stages / simple case - just a string
+      ##   and for the stages case ALWAYS an array (even if it has only one page (with stage))
+
+      if @pages.is_a?( String )
+        # assume always "simple/regular" format w/o stages
+        slug = @pages
+        { slug: fill_slug( slug, season: season ) }
+      else
+         ## check for league format / stages
+         ##   return array (of strings) or nil (for no stages - "simple" format)
+         indices = @season_proc.call( season )
+         if indices.nil?
+           puts "!! ERROR - no configuration found for season >#{season}< for league >#{@key}< found; sorry"
+           exit 1
+         elsif indices.is_a?( Integer )  ## single number - single/regular format w/o stage
+          # note: starting with 0 (always use idx-1) !!!
+           slug = if @pages.is_a?( Array )
+                    @pages[indices-1]
+                  else ## assume hash (and key is page slug)
+                    @pages.keys[indices-1]
+                  end
+           { slug: fill_slug( slug, season: season ) }
+         else  ## assume regular case - array of integers
+           recs = []
+           indices.each do |idx|
+              slug = key = @pages.keys[idx-1]
+              recs << { slug:  fill_slug( slug, season: season ),
+                        stage: @pages[key] }  ## note: include mapping for page to stage name!!
+           end
+           recs
+        end
+      end
+    end # pages
+
+
+    ######
+    # helper method
+    def fill_slug( slug, season: )
+      ## note: fill-in/check for place holders too
+      slug = if slug.index( '{season}' )
+               slug.sub( '{season}', season.to_path( :long ) )  ## e.g. 2010-2011
+             elsif slug.index( '{end_year}' )
+               slug.sub( '{end_year}', season.end_year.to_s )   ## e.g. 2011
+             else
+               ## assume convenience fallback - append regular season
+               "#{slug}-#{season.to_path( :long )}"
+            end
+
+      puts "  slug=>#{slug}<"
+
+      slug
+    end
+  end # class League
+
+
+
+  def self.find_league( key )  ## league info lookup
+    data = LEAGUES[ key ]
+    if data.nil?
+      puts "!! ERROR - no league found for >#{key}<; add to leagues tables"
+      exit 1
+    end
+    League.new( key, data )   ## use a convenience wrapper for now
+  end
+
+
+
+### "reverse" lookup by page - returns league AND season
+##  note: "blind" season template para - might be season or start_year etc.
+##   e.g.  {season} or {start_year} becomes {}
+
+PAGE_VAR_RE = /{
+                [^}]+
+               }/x
+
+
+def self.norm_slug( slug )
+    ## assume convenience fallback - append regular season
+    slug.index( '{' ) ? slug : "#{slug}-{season}"
+end
+
+PAGES ||=
+ LEAGUES.reduce( {} ) do |pages, (key, data)|
+                                     if data[:pages].is_a?( String )
+                                       slug = data[:pages]
+                                       slug = Worldfootball.norm_slug( slug )
+                                       pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug }
+                                     elsif data[:pages].is_a?( Array )
+                                       data[:pages].each do |slug|
+                                        slug = Worldfootball.norm_slug( slug )
+                                         pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug }
+                                       end
+                                        ## elsif data[:pages].nil?
+                                        ## todo/fix: missing pages!!!
+                                     else ## assume hash
+                                      ## add stage to pages too - why? why not?
+                                       data[:pages].each do |slug, stage|
+                                         slug = Worldfootball.norm_slug( slug )
+                                         pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug, stage: stage }
+                                       end
+                                     end
+                                     pages
+                                    end
+
+# e.g. 2000 or 2000-2001
+SEASON_RE = /[0-9]{4}
+              (?:
+                -[0-9]{4}
+              )?
+            /x
+
+
+  def self.find_page!( slug )
+    page = find_page( slug )
+    if page.nil?
+      puts "!! ERROR: no mapping for page >#{slug}< found; sorry"
+
+      season_str = nil
+      norm = slug.sub( SEASON_RE ) do |match|  ## replace season with var placeholder {}
+                season_str = match   ## keep reference to season str
+                '{}'  ## replace with {}
+              end
+
+      puts "   season:      >#{season_str}<"
+      puts "   slug (norm): >#{norm}<"
+      puts
+      ## pp PAGES
+      exit 1
+    end
+    page
+  end
+
+
+
+  def self.find_page( slug )
+    ## return league key and season
+    season_str = nil
+    norm = slug.sub( SEASON_RE ) do |match|  ## replace season with var placeholder {}
+              season_str = match   ## keep reference to season str
+              '{}'  ## replace with {}
+            end
+
+    if season_str.nil?
+      puts "!! ERROR: no season found in page slug >#{slug}<; sorry"
+      exit 1
+    end
+
+    rec = PAGES[ norm ]
+    return nil  if rec.nil?
+
+
+    league_key = rec[:league]
+    slug_tmpl  = rec[:slug]
+    season = if slug_tmpl.index( '{start_year}' )
+               ## todo/check - season_str must be year (e.g. 2020 or such and NOT 2020-2021)
+               Season( "#{season_str.to_i}-#{season_str.to_i+1}" )
+             elsif slug_tmpl.index( '{end_year}' )
+               ## todo/check - season_str must be year (e.g. 2020 or such and NOT 2020-2021)
+               Season( "#{season_str.to_i-1}-#{season_str.to_i}" )
+             else  ## assume "regular" seasson - pass through as is
+               Season( season_str )
+             end
+
+    ## return hash table / record
+    { league: league_key,
+      season: season.key }
+  end
+
+
+end # module Worldfootball

data/lib/worldfootball/mods.rb

@@ -0,0 +1,72 @@
+#### todo/check: move MODS and SCORE_ERRORS out-of-lib
+##               and into config or such - why? why not?
+
+
+module Worldfootball
+
+
+######
+# "global" helpers
+def self.norm_team( team )
+   ## clean team name and asciify (e.g. ’->' )
+   team = team.sub( '(old)', '' ).strip
+   team = team.gsub( '’', "'" )     ## e.g. Hawke’s Bay United FC
+   team
+end
+
+
+
+MODS = {
+ 'at' => {
+    ## AT 1
+    'SC Magna Wiener Neustadt' => 'SC Wiener Neustadt', # in 2010/11
+    'KSV Superfund'            => 'Kapfenberger SV',    # in 2010/11
+    'Kapfenberger SV 1919'     => 'Kapfenberger SV',    # in 2011/12
+    'FC Trenkwalder Admira'    => 'FC Admira Wacker',    # in 2011/12
+    ## AT 2
+    'Austria Wien (A)'         => 'Young Violets',  # in 2019/20
+    'FC Wacker Innsbruck (A)'  => 'FC Wacker Innsbruck II',   # in 2018/19
+    ## AT CUP
+    'Rapid Wien (A)'           => 'Rapid Wien II',  # in 2011/12
+    'Sturm Graz (A)'           => 'Sturm Graz II',
+    'Kapfenberger SV 1919 (A)' => 'Kapfenberger SV II',
+    'SV Grödig (A)'            => 'SV Grödig II',
+    'RB Salzburg (A)'          => 'RB Salzburg II',
+    'SR WGFM Donaufeld'        => 'SR Donaufeld Wien',
+    'FC Trenkwalder Admira (A)' => 'FC Admira Wacker II',
+    ## AT 3.O (Regionalliga Ost)
+    'FC Admira Wacker (A)'     => 'FC Admira Wacker II',  # in 2020/21
+  },
+ 'nz' => {
+    ## NZ 1
+   'Wellington Phoenix (R)' => 'Wellington Phoenix Reserves',
+  },
+}
+
+
+
+## fix/patch known score format errors in at/de cups
+##   new convention
+##   for a fix require league, date, and team1 & team2 for now!!!!
+##    - do NOT use some "generic" fix / patch!!!!
+##
+## old de/at patches/fixes:
+##  '0-1 (0-0, 0-0, 0-0) n.V.' => '0-1 (0-0, 0-0) n.V.',       # too long
+##  '2-1 (1-1, 1-1, 1-0) n.V.' => '2-1 (1-1, 1-1) n.V.',
+##  '4-2 (0-0, 0-0) i.E.'      => '4-2 (0-0, 0-0, 0-0) i.E.',  # too short
+
+
+SCORE_ERRORS = {
+  'ro.1' => {
+    ## 2013/14
+    '2013-07-29' => [ 'FC Brașov', 'Săgeata Năvodari', ['1-1 (0-0, 0-1)', '1-1 (0-0)']],
+  },
+  'gr.1' => {
+    ## 2010/11
+    '2010-11-24' => [ 'Ergotelis',    'Olympiakos Piräus', ['0-2 (0-0, 0-0, 0-0)', '0-2 (0-0)']],
+    '2010-11-28' => [ 'Panserraikos', 'Aris Saloniki',     ['1-0 (1-0, 0-0, 0-0)', '1-0 (1-0)']],
+  }
+}
+
+
+end # module Worldfootball

data/lib/worldfootball/page.rb

@@ -0,0 +1,106 @@
+
+module Worldfootball
+  class Page
+
+  def self.from_file( path )
+    html = File.open( path, 'r:utf-8' ) {|f| f.read }
+    new( html )
+  end
+
+  def initialize( html )
+    @html = html
+  end
+
+  def doc
+    ## note: if we use a fragment and NOT a document - no access to page head (and meta elements and such)
+    @doc ||= Nokogiri::HTML( @html )
+  end
+
+  def title
+   # <title>Bundesliga 2010/2011 &raquo; Spielplan</title>
+     @title ||= doc.css( 'title' ).first
+     @title.text  ## get element's text content
+  end
+
+  def keywords
+     # <meta name="keywords"
+     #  content="Bundesliga, 2010/2011, Spielplan, KSV Superfund, SC Magna Wiener Neustadt, SV Ried, FC Wacker Innsbruck, Austria Wien, Sturm Graz, SV Mattersburg, LASK Linz, Rapid Wien, RB Salzburg" />
+     @keywords ||= doc.css( 'meta[name="keywords"]' ).first
+     @keywords[:content]  ## get content attribute
+     ## or      doc.xpath( '//meta[@name="keywords"]' ).first
+     ## pp keywords
+     # puts "  #{keywords[:content]}"
+
+     # keywords = doc.at( 'meta[@name="Keywords"]' )
+     # pp keywords
+     ## check for
+  end
+
+  # <meta property="og:url"
+  #       content="//www.weltfussball.de/alle_spiele/aut-bundesliga-2010-2011/" />
+  def url
+    @url ||= doc.css( 'meta[property="og:url"]' ).first
+    @url[:content]
+  end
+
+
+
+##  <!-- [generated 2020-06-30 22:30:19] -->
+##  <!-- [generated 2020-06-30 22:30:19] -->
+GENERATED_RE = %r{
+  <!--
+    [ ]+
+    \[generated
+        [ ]+
+      (?<date>\d+-\d+-\d+)
+        [ ]+
+      (?<time>\d+:\d+:\d+)
+    \]
+    [ ]+
+    -->
+   }x
+
+
+   def generated
+      @generated ||= begin
+        m=GENERATED_RE.match( @html )
+        if m
+         DateTime.strptime( "#{m[:date]} #{m[:time]}", '%Y-%m-%d %H:%M:%S')
+        else
+         puts "!! WARN - no generated timestamp found in page"
+         nil
+        end
+      end
+   end
+
+   ### convenience helper / formatter
+   def generated_in_days_ago
+     if generated
+      diff_in_days = Date.today.jd - generated.jd
+      "#{diff_in_days}d"
+     else
+      '?'
+     end
+   end
+
+######################
+##  helper methods
+
+def squish( str )
+  str = str.strip
+  str = str.gsub( "\u{00A0}", ' ' )  # Unicode Character 'NO-BREAK SPACE' (U+00A0)
+  str = str.gsub( /[ \t\n]+/, ' ' )  ## fold whitespace to one max.
+  str
+end
+
+def assert( cond, msg )
+  if cond
+    # do nothing
+  else
+    puts "!!! assert failed (in parse page) - #{msg}"
+    exit 1
+  end
+end
+
+end # class Page
+end # module Worldfootball