football-sources 0.0.1

data/lib/football-sources/worldfootball/config.rb

@@ -0,0 +1,16 @@
+module Worldfootball
+
+### add some more config options / settings
+class Configuration
+   #########
+   ## nested configuration classes - use - why? why not?
+   class Convert
+      def out_dir()       @out_dir || './o'; end
+      def out_dir=(value) @out_dir = value; end
+   end
+
+  def convert()  @convert ||= Convert.new; end
+end # class Configuration
+
+
+end   # module Worldfootball

data/lib/football-sources/worldfootball/convert.rb

@@ -0,0 +1,100 @@
+
+module Worldfootball
+
+
+
+def self.convert( league:, season:, offset: nil )  ## check: rename (optional) offset to time_offset or such?
+  season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league = find_league( league )
+
+  pages = league.pages( season: season )
+
+  # note: assume stages if pages is an array (of hash table/records)
+  #         (and NOT a single hash table/record)
+  if pages.is_a?(Array)
+    recs = []
+    pages.each do |page_meta|
+      slug       = page_meta[:slug]
+      stage_name = page_meta[:stage]
+      ## todo/fix: report error/check if stage.name is nil!!!
+
+      print "  parsing #{slug}..."
+
+      # unless File.exist?( path )
+      #  puts "!! WARN - missing stage >#{stage_name}< source - >#{path}<"
+      #  next
+      # end
+
+      page = Page::Schedule.from_cache( slug )
+      print "  title=>#{page.title}<..."
+      print "\n"
+
+      rows = page.matches
+      stage_recs = build( rows, season: season, league: league.key, stage: stage_name )
+
+      pp stage_recs[0]   ## check first record
+      recs += stage_recs
+    end
+  else
+    page_meta = pages
+    slug = page_meta[:slug]
+
+    print "  parsing #{slug}..."
+
+    page = Page::Schedule.from_cache( slug )
+    print "  title=>#{page.title}<..."
+    print "\n"
+
+    rows = page.matches
+    recs = build( rows, season: season, league: league.key )
+
+    pp recs[0]   ## check first record
+  end
+
+  recs = recs.map { |rec| fix_date( rec, offset ) }    if offset
+
+##   note:  sort matches by date before saving/writing!!!!
+##     note: for now assume date in string in 1999-11-30 format (allows sort by "simple" a-z)
+## note: assume date is third column!!! (stage/round/date/...)
+recs = recs.sort { |l,r| l[2] <=> r[2] }
+## reformat date / beautify e.g. Sat Aug 7 1993
+recs.each { |rec| rec[2] = Date.strptime( rec[2], '%Y-%m-%d' ).strftime( '%a %b %-d %Y' ) }
+
+   ## remove unused columns (e.g. stage, et, p, etc.)
+   recs, headers = vacuum( recs )
+
+   puts headers
+   pp recs[0]   ## check first record
+
+   out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}.csv"
+
+   puts "write #{out_path}..."
+   Cache::CsvMatchWriter.write( out_path, recs, headers: headers )
+end
+
+
+
+## helper to fix dates to use local timezone (and not utc/london time)
+def self.fix_date( row, offset )
+  return row    if row[3].nil? || row[3].empty?   ## note: time (column) required for fix
+
+  col = row[2]
+  if col =~ /^\d{4}-\d{2}-\d{2}$/
+    date_fmt = '%Y-%m-%d'   # e.g. 2002-08-17
+  else
+    puts "!!! ERROR - wrong (unknown) date format >>#{col}<<; cannot continue; fix it; sorry"
+    ## todo/fix: add to errors/warns list - why? why not?
+    exit 1
+  end
+
+  date = DateTime.strptime( "#{row[2]} #{row[3]}", "#{date_fmt} %H:%M" )
+  ## NOTE - MUST be -7/24.0!!!! or such to work
+  date = date + (offset/24.0)
+
+  row[2] = date.strftime( date_fmt )  ## overwrite "old"
+  row[3] = date.strftime( '%H:%M' )
+  row   ## return row for possible pipelining - why? why not?
+end
+
+end # module Worldfootball

data/lib/football-sources/worldfootball/convert_reports.rb

@@ -0,0 +1,107 @@
+module Worldfootball
+
+
+def self.convert_reports( league:, season: )
+  season = Season( season )  ## cast (ensure) season class (NOT string, integer, etc.)
+
+  league = find_league( league )
+
+   ## note: use only first part from key for lookup
+   ##    e.g. at.1  => at
+   ##         eng.1 => eng
+   ##     and so on
+   mods = MODS[ league.key.split('.')[0] ] || {}
+
+
+
+  pages = league.pages( season: season )
+
+  recs = []
+
+  ## if single (simple) page setup - wrap in array
+  pages = pages.is_a?(Array) ? pages : [pages]
+  pages.each do |page_meta|  # note: use page_info for now (or page_rec or page_meta or such)
+
+    page = Page::Schedule.from_cache( page_meta[:slug] )
+    print "  page title=>#{page.title}<..."
+    print "\n"
+
+    matches = page.matches
+
+    puts "matches - #{matches.size} rows:"
+    pp matches[0]
+
+    puts "#{page.generated_in_days_ago}  - #{page.generated}"
+
+
+    matches.each_with_index do |match,i|
+
+      report_ref = match[:report_ref]
+      if report_ref.nil?
+        puts "!! WARN: no match report ref found for match:"
+        pp match
+        next
+      end
+
+      puts "reading #{i+1}/#{matches.size} - #{report_ref}..."
+      report = Page::Report.from_cache( report_ref )
+
+      puts
+      puts report.title
+      puts report.generated
+
+      rows = report.goals
+      puts "goals - #{rows.size} records"
+      ## pp rows
+
+
+      if rows.size > 0
+        ## add goals
+        date = Date.strptime( match[:date], '%Y-%m-%d')
+
+        team1 = match[:team1]
+        team2 = match[:team2]
+
+        ## clean team name (e.g. remove (old))
+        ##   and asciify (e.g. ’ to ' )
+        team1 = norm_team( team1 )
+        team2 = norm_team( team2 )
+
+        team1 = mods[ team1 ]   if mods[ team1 ]
+        team2 = mods[ team2 ]   if mods[ team2 ]
+
+        match_id = "#{team1} - #{team2} | #{date.strftime('%b %-d %Y')}"
+
+
+        rows.each do |row|
+          extra = if row[:owngoal]
+                   '(og)'  ## or use OG or O.G.- why? why not?
+                  elsif row[:penalty]
+                   '(pen)' ## or use P or PEN - why? why not?
+                  else
+                    ''
+                  end
+
+          rec = [match_id,
+                row[:score],
+                "#{row[:minute]}'",
+                extra,
+                row[:player],
+                row[:notes]]
+          recs << rec
+        end
+      end
+     end #  each match
+    end # each page
+
+  ## pp recs
+
+  out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}~goals.csv"
+
+  headers  = ['Match', 'Score', 'Minute', 'Extra', 'Player', 'Notes']
+
+  puts "write #{out_path}..."
+  Cache::CsvMatchWriter.write( out_path, recs, headers: headers )
+end
+end  # module Worldfootballl
+

data/lib/football-sources/worldfootball/download.rb

@@ -0,0 +1,61 @@
+
+
+module Worldfootball
+
+##
+## note/fix!!!!
+##   do NOT allow redirects for now - report error!!!
+##   does NOT return 404 page not found errors; always redirects (301) to home page
+##    on missing pages:
+##      301 Moved Permanently location=https://www.weltfussball.de/
+##      301 Moved Permanently location=https://www.weltfussball.de/
+
+
+
+
+# url = "https://www.weltfussball.de/alle_spiele/eng-league-one-#{season}/"
+# url = "https://www.weltfussball.de/alle_spiele/eng-league-two-#{season}/"
+#        https://www.weltfussball.de/alle_spiele/eng-national-league-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/eng-fa-cup-2018-2019/
+#        https://www.weltfussball.de/alle_spiele/eng-league-cup-2019-2020/
+
+#        https://www.weltfussball.de/alle_spiele/fra-ligue-2-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/ita-serie-b-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/rus-premier-liga-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/rus-1-division-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/tur-sueperlig-2019-2020/
+#        https://www.weltfussball.de/alle_spiele/tur-1-lig-2019-2020/
+
+
+
+  def self.schedule( league:, season: )
+    season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+    league  = find_league( league )
+
+    pages =  league.pages( season: season )
+
+    ## if single (simple) page setup - wrap in array
+    pages = pages.is_a?(Array) ? pages : [pages]
+    pages.each do |page_meta|
+      Metal.schedule( page_meta[:slug] )
+    end # each page
+  end
+
+
+  def self.schedule_reports( league:, season:, cache: true ) ## todo/check: rename to reports_for_schedule or such - why? why not?
+    season = Season( season )   ## cast (ensure) season class (NOT string, integer, etc.)
+
+    league  = find_league( league )
+
+    pages =  league.pages( season: season )
+
+    ## if single (simple) page setup - wrap in array
+    pages = pages.is_a?(Array) ? pages : [pages]
+    pages.each do |page_meta|
+      Metal.schedule_reports( page_meta[:slug], cache: cache )
+    end # each page
+  end
+
+
+end # module Worldfootball

data/lib/football-sources/worldfootball/leagues.rb

@@ -0,0 +1,200 @@
+
+
+require_relative 'leagues/europe'
+require_relative 'leagues/north_america'
+require_relative 'leagues/south_america'
+require_relative 'leagues/pacific'
+require_relative 'leagues/asia'
+
+
+module Worldfootball
+
+LEAGUES = [LEAGUES_EUROPE,
+           LEAGUES_NORTH_AMERICA,
+           LEAGUES_SOUTH_AMERICA,
+           LEAGUES_PACIFIC,
+           LEAGUES_ASIA].reduce({}) { |mem,h| mem.merge!( h ); mem }
+
+
+class League
+    def initialize( key, data )
+      @key  = key
+      ## @data = data
+
+      @pages       = data[:pages]
+      @season_proc = data[:season] || ->(season) { nil }
+    end
+
+    def key()   @key; end
+
+    def pages( season: )
+      ## note: return for no stages / simple case - just a string
+      ##   and for the stages case ALWAYS an array (even if it has only one page (with stage))
+
+      if @pages.is_a?( String )
+        # assume always "simple/regular" format w/o stages
+        slug = @pages
+        { slug: fill_slug( slug, season: season ) }
+      else
+         ## check for league format / stages
+         ##   return array (of strings) or nil (for no stages - "simple" format)
+         indices = @season_proc.call( season )
+         if indices.nil?
+           puts "!! ERROR - no configuration found for season >#{season}< for league >#{@key}< found; sorry"
+           exit 1
+         elsif indices.is_a?( Integer )  ## single number - single/regular format w/o stage
+          # note: starting with 0 (always use idx-1) !!!
+           slug = if @pages.is_a?( Array )
+                    @pages[indices-1]
+                  else ## assume hash (and key is page slug)
+                    @pages.keys[indices-1]
+                  end
+           { slug: fill_slug( slug, season: season ) }
+         else  ## assume regular case - array of integers
+           recs = []
+           indices.each do |idx|
+              slug = key = @pages.keys[idx-1]
+              recs << { slug:  fill_slug( slug, season: season ),
+                        stage: @pages[key] }  ## note: include mapping for page to stage name!!
+           end
+           recs
+        end
+      end
+    end # pages
+
+
+    ######
+    # helper method
+    def fill_slug( slug, season: )
+      ## note: fill-in/check for place holders too
+      slug = if slug.index( '{season}' )
+               slug.sub( '{season}', season.to_path( :long ) )  ## e.g. 2010-2011
+             elsif slug.index( '{end_year}' )
+               slug.sub( '{end_year}', season.end_year.to_s )   ## e.g. 2011
+             else
+               ## assume convenience fallback - append regular season
+               "#{slug}-#{season.to_path( :long )}"
+            end
+
+      puts "  slug=>#{slug}<"
+
+      slug
+    end
+  end # class League
+
+
+
+  def self.find_league( key )  ## league info lookup
+    data = LEAGUES[ key ]
+    if data.nil?
+      puts "!! ERROR - no league found for >#{key}<; add to leagues tables"
+      exit 1
+    end
+    League.new( key, data )   ## use a convenience wrapper for now
+  end
+
+
+
+### "reverse" lookup by page - returns league AND season
+##  note: "blind" season template para - might be season or start_year etc.
+##   e.g.  {season} or {start_year} becomes {}
+
+PAGE_VAR_RE = /{
+                [^}]+
+               }/x
+
+
+def self.norm_slug( slug )
+    ## assume convenience fallback - append regular season
+    slug.index( '{' ) ? slug : "#{slug}-{season}"
+end
+
+PAGES ||=
+ LEAGUES.reduce( {} ) do |pages, (key, data)|
+                                     if data[:pages].is_a?( String )
+                                       slug = data[:pages]
+                                       slug = Worldfootball.norm_slug( slug )
+                                       pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug }
+                                     elsif data[:pages].is_a?( Array )
+                                       data[:pages].each do |slug|
+                                        slug = Worldfootball.norm_slug( slug )
+                                         pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug }
+                                       end
+                                        ## elsif data[:pages].nil?
+                                        ## todo/fix: missing pages!!!
+                                     else ## assume hash
+                                      ## add stage to pages too - why? why not?
+                                       data[:pages].each do |slug, stage|
+                                         slug = Worldfootball.norm_slug( slug )
+                                         pages[ slug.sub( PAGE_VAR_RE, '{}') ] = { league: key, slug: slug, stage: stage }
+                                       end
+                                     end
+                                     pages
+                                    end
+
+# e.g. 2000 or 2000-2001
+SEASON_RE = /[0-9]{4}
+              (?:
+                -[0-9]{4}
+              )?
+            /x
+
+
+  def self.find_page!( slug )
+    page = find_page( slug )
+    if page.nil?
+      puts "!! ERROR: no mapping for page >#{slug}< found; sorry"
+
+      season_str = nil
+      norm = slug.sub( SEASON_RE ) do |match|  ## replace season with var placeholder {}
+                season_str = match   ## keep reference to season str
+                '{}'  ## replace with {}
+              end
+
+      puts "   season:      >#{season_str}<"
+      puts "   slug (norm): >#{norm}<"
+      puts
+      ## pp PAGES
+      exit 1
+    end
+    page
+  end
+
+
+
+  def self.find_page( slug )
+    ## return league key and season
+    season_str = nil
+    norm = slug.sub( SEASON_RE ) do |match|  ## replace season with var placeholder {}
+              season_str = match   ## keep reference to season str
+              '{}'  ## replace with {}
+            end
+
+    if season_str.nil?
+      puts "!! ERROR: no season found in page slug >#{slug}<; sorry"
+      exit 1
+    end
+
+    rec = PAGES[ norm ]
+    return nil  if rec.nil?
+
+
+    league_key = rec[:league]
+    slug_tmpl  = rec[:slug]
+    season = if slug_tmpl.index( '{start_year}' )
+               ## todo/check - season_str must be year (e.g. 2020 or such and NOT 2020-2021)
+               Season( "#{season_str.to_i}-#{season_str.to_i+1}" )
+             elsif slug_tmpl.index( '{end_year}' )
+               ## todo/check - season_str must be year (e.g. 2020 or such and NOT 2020-2021)
+               Season( "#{season_str.to_i-1}-#{season_str.to_i}" )
+             else  ## assume "regular" seasson - pass through as is
+               Season( season_str )
+             end
+
+    ## return hash table / record
+    { league: league_key,
+      season: season.key }
+  end
+
+
+end # module Worldfootball