football-sources 0.0.1

data/lib/football-sources/worldfootball/leagues/europe--western.rb

@@ -0,0 +1,38 @@
+
+module Worldfootball
+
+LEAGUES_EUROPE.merge!({
+  'fr.1'  => { pages: 'fra-ligue-1' },
+  'fr.2'  => { pages: 'fra-ligue-2' },
+
+  # e.g. /lux-nationaldivision-2020-2021/
+  'lu.1' => { pages: 'lux-nationaldivision' },
+
+  # e.g. /ned-eredivisie-2020-2021/
+  'nl.1' => { pages: 'ned-eredivisie' },
+  # Championship play-offs
+  # Europa League play-offs (Group A + Group B / Finals )
+
+  # e.g. /bel-eerste-klasse-a-2020-2021/
+  #      /bel-europa-league-playoffs-2018-2019-playoff/
+  #       - Halbfinale
+  #       - Finale
+  'be.1' => {
+    pages: {
+      'bel-eerste-klasse-a-{season}'                => 'Regular Season',
+      'bel-eerste-klasse-a-{season}-playoff-i'      => 'Playoffs - Championship',
+      'bel-europa-league-playoffs-{season}'         => 'Playoffs - Europa League',           ## note: missing groups (A & B)
+      'bel-europa-league-playoffs-{season}-playoff' => 'Playoffs - Europa League - Finals',
+    },
+    season: ->( season ) {
+      case season
+      when Season('2020/21') then [1]     # just getting started
+      when Season('2019/20') then [1]     # covid-19 - no championship & europa
+      when Season('2018/19') then [1,2,3,4]
+      end
+    }
+  },
+
+})
+
+end

data/lib/football-sources/worldfootball/leagues/europe.rb

@@ -0,0 +1,13 @@
+
+module Worldfootball
+  LEAGUES_EUROPE = {}
+end
+
+
+require_relative 'europe--western'
+require_relative 'europe--british_isles'
+require_relative 'europe--northern'
+require_relative 'europe--central'
+require_relative 'europe--eastern'
+require_relative 'europe--southern'
+

data/lib/football-sources/worldfootball/leagues/north_america.rb

@@ -0,0 +1,44 @@
+module Worldfootball
+
+LEAGUES_NORTH_AMERICA = {
+
+# todo/fix: adjust date/time by -6 or 7 hours!!!
+#   /can-canadian-championship-2020/
+#     - Qual. 1. Runde
+#     - Qual. 2. Runde
+#     - Qual. 3. Runde
+#   todo/fix: check for leagues - premier league? championship? soccer league?
+#  'ca.1' => { slug: 'can-canadian-championship' },
+
+
+
+# todo/fix: adjust date/time by -7 hours!!!
+##  e.g. 25.07.2020	02:30  => 24.07.2020 19.30
+#        11.01.2020	04:00  => 10.01.2020 21.00
+#
+# e.g. /mex-primera-division-2020-2021-apertura/
+#      /mex-primera-division-2019-2020-clausura/
+#      /mex-primera-division-2019-2020-apertura-playoffs/
+#        - Viertelfinale
+#        - Halbfinale
+#        - Finale
+#      /mex-primera-division-2018-2019-clausura-playoffs/
+'mx.1' => {
+  pages: {
+    'mex-primera-division-{season}-apertura'          => 'Apertura',            # 1
+    'mex-primera-division-{season}-apertura-playoffs' => 'Apertura - Liguilla', # 2
+    'mex-primera-division-{season}-clausura'          => 'Clausura',            # 3
+    'mex-primera-division-{season}-clausura-playoffs' => 'Clausura - Liguilla', # 4
+ },
+ season: ->( season ) {
+  case season
+  when Season('2020/21') then [1]        # just getting started
+  when Season('2019/20') then [1,2,3]    # covid-19 - no liguilla
+  when Season('2010/11')..Season('2018/19') then [1,2,3,4]
+  end
+ }
+},
+}
+
+end # module Worldfootball
+

data/lib/football-sources/worldfootball/leagues/pacific.rb

@@ -0,0 +1,21 @@
+module Worldfootball
+
+LEAGUES_PACIFIC = {
+
+  # /nzl-nz-football-championship-2019-2020/
+  # /nzl-nz-football-championship-2018-2019-playoffs/
+  'nz.1' => {
+    pages: {
+      'nzl-nz-football-championship-{season}'          => 'Regular Season', # 1
+      'nzl-nz-football-championship-{season}-playoffs' => 'Playoff Finals', # 2
+     },
+     season: ->( season ) {
+      case season
+      when Season('2019/20') then [1]    ## covid-19 - no playoffs/finals
+      when Season('2018/19') then [1,2]
+      end
+     }
+  },
+}
+
+end # module Worldfootball

data/lib/football-sources/worldfootball/leagues/south_america.rb

@@ -0,0 +1,11 @@
+module Worldfootball
+
+  LEAGUES_SOUTH_AMERICA = {
+
+    # todo/fix: adjust date/time by -6 or 7 hours!!!
+    # /bra-serie-a-2020/
+    'br.1' => { pages: 'bra-serie-a' },
+  }
+
+end # module Worldfootball
+

data/lib/football-sources/worldfootball/mods.rb

@@ -0,0 +1,72 @@
+#### todo/check: move MODS and SCORE_ERRORS out-of-lib
+##               and into config or such - why? why not?
+
+
+module Worldfootball
+
+
+######
+# "global" helpers
+def self.norm_team( team )
+   ## clean team name and asciify (e.g. ’->' )
+   team = team.sub( '(old)', '' ).strip
+   team = team.gsub( '’', "'" )     ## e.g. Hawke’s Bay United FC
+   team
+end
+
+
+
+MODS = {
+ 'at' => {
+    ## AT 1
+    'SC Magna Wiener Neustadt' => 'SC Wiener Neustadt', # in 2010/11
+    'KSV Superfund'            => 'Kapfenberger SV',    # in 2010/11
+    'Kapfenberger SV 1919'     => 'Kapfenberger SV',    # in 2011/12
+    'FC Trenkwalder Admira'    => 'FC Admira Wacker',    # in 2011/12
+    ## AT 2
+    'Austria Wien (A)'         => 'Young Violets',  # in 2019/20
+    'FC Wacker Innsbruck (A)'  => 'FC Wacker Innsbruck II',   # in 2018/19
+    ## AT CUP
+    'Rapid Wien (A)'           => 'Rapid Wien II',  # in 2011/12
+    'Sturm Graz (A)'           => 'Sturm Graz II',
+    'Kapfenberger SV 1919 (A)' => 'Kapfenberger SV II',
+    'SV Grödig (A)'            => 'SV Grödig II',
+    'RB Salzburg (A)'          => 'RB Salzburg II',
+    'SR WGFM Donaufeld'        => 'SR Donaufeld Wien',
+    'FC Trenkwalder Admira (A)' => 'FC Admira Wacker II',
+    ## AT 3.O (Regionalliga Ost)
+    'FC Admira Wacker (A)'     => 'FC Admira Wacker II',  # in 2020/21
+  },
+ 'nz' => {
+    ## NZ 1
+   'Wellington Phoenix (R)' => 'Wellington Phoenix Reserves',
+  },
+}
+
+
+
+## fix/patch known score format errors in at/de cups
+##   new convention
+##   for a fix require league, date, and team1 & team2 for now!!!!
+##    - do NOT use some "generic" fix / patch!!!!
+##
+## old de/at patches/fixes:
+##  '0-1 (0-0, 0-0, 0-0) n.V.' => '0-1 (0-0, 0-0) n.V.',       # too long
+##  '2-1 (1-1, 1-1, 1-0) n.V.' => '2-1 (1-1, 1-1) n.V.',
+##  '4-2 (0-0, 0-0) i.E.'      => '4-2 (0-0, 0-0, 0-0) i.E.',  # too short
+
+
+SCORE_ERRORS = {
+  'ro.1' => {
+    ## 2013/14
+    '2013-07-29' => [ 'FC Brașov', 'Săgeata Năvodari', ['1-1 (0-0, 0-1)', '1-1 (0-0)']],
+  },
+  'gr.1' => {
+    ## 2010/11
+    '2010-11-24' => [ 'Ergotelis',    'Olympiakos Piräus', ['0-2 (0-0, 0-0, 0-0)', '0-2 (0-0)']],
+    '2010-11-28' => [ 'Panserraikos', 'Aris Saloniki',     ['1-0 (1-0, 0-0, 0-0)', '1-0 (1-0)']],
+  }
+}
+
+
+end # module Worldfootball

data/lib/football-sources/worldfootball/tool.rb

@@ -0,0 +1,100 @@
+
+
+module Worldfootball
+################################
+# add more helpers
+#  move upstream for (re)use - why? why not?
+
+## todo/check: what to do: if league is both included and excluded?
+##   include forces include? or exclude has the last word? - why? why not?
+##  Excludes match before includes,
+##   meaning that something that has been excluded cannot be included again
+
+## todo - find "proper/classic" timezone ("winter time")
+
+##  Brasilia - Distrito Federal, Brasil  (GMT-3)  -- summer time?
+##  Ciudad de México, CDMX, México       (GMT-5)  -- summer time?
+##  Londres, Reino Unido (GMT+1)
+##   Madrid -- ?
+##   Lisboa -- ?
+##   Moskow -- ?
+##
+## todo/check - quick fix timezone offsets for leagues for now
+##   - find something better - why? why not?
+## note: assume time is in GMT+1
+OFFSETS = {
+  'eng.1' => -1,
+  'eng.2' => -1,
+  'eng.3' => -1,
+  'eng.4' => -1,
+  'eng.5' => -1,
+
+  'es.1'  => -1,
+  'es.2'  => -1,
+
+  'pt.1'  => -1,
+  'pt.2'  => -1,
+
+  'br.1'  => -5,
+  'mx.1'  => -7,
+}
+
+
+
+
+class Tool
+  def initialize( includes: nil,
+                  excludes: nil )
+    ## quick fix: move/handle empty array upstream!!!!
+    includes = nil   if includes.is_a?(Array) && includes.empty?
+
+    @includes = includes
+    @excludes = excludes
+  end
+
+
+
+  def download( datasets )
+    datasets.each do |dataset|
+      league   = dataset[0]
+      seasons  = dataset[1]
+
+      with_filter( league ) do
+        seasons.each do |season|
+          puts "downloading #{league} #{season}..."
+
+          Worldfootball.schedule( league: league,
+                                  season: season )
+        end
+      end
+    end
+  end
+
+  def convert( datasets )
+    datasets.each do |dataset|
+      league   = dataset[0]
+      seasons  = dataset[1]
+
+      with_filter( league ) do
+        seasons.each do |season|
+           Worldfootball.convert( league: league,
+                                  season: season,
+                                  offset: OFFSETS[ league ] )
+        end
+      end
+    end
+  end
+
+
+#########################################
+# more helpers
+  def with_filter( league, &blk )
+    return  if @excludes && @excludes.find { |q| league.start_with?( q.downcase ) }
+    return  if @includes && @includes.find { |q| league.start_with?( q.downcase ) }.nil?
+    blk.call()
+  end
+
+end # class Tool
+end # module Worldfootball
+
+

data/lib/football-sources/worldfootball/vacuum.rb

@@ -0,0 +1,66 @@
+module Worldfootball
+
+
+MAX_HEADERS = [
+'Stage',
+'Round',
+'Date',
+'Time',
+'Team 1',
+'FT',
+'HT',
+'Team 2',
+'ET',
+'P',
+'Comments']    ## e.g. awarded, cancelled/canceled, etc.
+
+MIN_HEADERS = [   ## always keep even if all empty
+'Date',
+'Team 1',
+'FT',
+'Team 2'
+]
+
+def self.vacuum( rows, headers: MAX_HEADERS, fixed_headers: MIN_HEADERS )
+  ## check for unused columns and strip/remove
+  counter = Array.new( MAX_HEADERS.size, 0 )
+  rows.each do |row|
+     row.each_with_index do |col, idx|
+       counter[idx] += 1  unless col.nil? || col.empty?
+     end
+  end
+
+  pp counter
+
+  ## check empty columns
+  headers       = []
+  indices       = []
+  empty_headers = []
+  empty_indices = []
+
+  counter.each_with_index do |num, idx|
+     header = MAX_HEADERS[ idx ]
+     if num > 0 || (num == 0 && fixed_headers.include?( header ))
+       headers << header
+       indices << idx
+     else
+       empty_headers << header
+       empty_indices << idx
+     end
+  end
+
+  if empty_indices.size > 0
+    rows = rows.map do |row|
+             row_vacuumed = []
+             row.each_with_index do |col, idx|
+               ## todo/fix: use values or such??
+               row_vacuumed << col   unless empty_indices.include?( idx )
+             end
+             row_vacuumed
+         end
+    end
+
+  [rows, headers]
+end
+
+end # module Worldfootball

data/lib/football/sources.rb

@@ -0,0 +1,6 @@
+# note: allow require 'football/sources' too
+#        (in addition to require 'football-sources')
+
+require_relative '../football-sources'
+
+

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift( '../webget/lib')
+
+require 'minitest/autorun'
+
+
+## our own code
+require 'football/sources'     ## or require 'football-sources'
+

data/test/test_version.rb

@@ -0,0 +1,16 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_version.rb
+
+
+require 'helper'
+
+class TestVersion < MiniTest::Test
+
+  def test_version
+    pp FootballSources::VERSION
+    pp FootballSources.banner
+    pp FootballSources.root
+  end
+
+end # class TestVersion

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: football-sources
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-11-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webget-football
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: sportdb-catalogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+description: football-sources - get football data via web pages or web api (json)
+  calls
+email: opensport@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/football-sources.rb
+- lib/football-sources/apis.rb
+- lib/football-sources/apis/config.rb
+- lib/football-sources/apis/convert.rb
+- lib/football-sources/apis/convert_cl.rb
+- lib/football-sources/apis/download.rb
+- lib/football-sources/apis/stat.rb
+- lib/football-sources/version.rb
+- lib/football-sources/worldfootball.rb
+- lib/football-sources/worldfootball/build.rb
+- lib/football-sources/worldfootball/config.rb
+- lib/football-sources/worldfootball/convert.rb
+- lib/football-sources/worldfootball/convert_reports.rb
+- lib/football-sources/worldfootball/download.rb
+- lib/football-sources/worldfootball/leagues.rb
+- lib/football-sources/worldfootball/leagues/asia.rb
+- lib/football-sources/worldfootball/leagues/europe--british_isles.rb
+- lib/football-sources/worldfootball/leagues/europe--central.rb
+- lib/football-sources/worldfootball/leagues/europe--eastern.rb
+- lib/football-sources/worldfootball/leagues/europe--northern.rb
+- lib/football-sources/worldfootball/leagues/europe--southern.rb
+- lib/football-sources/worldfootball/leagues/europe--western.rb
+- lib/football-sources/worldfootball/leagues/europe.rb
+- lib/football-sources/worldfootball/leagues/north_america.rb
+- lib/football-sources/worldfootball/leagues/pacific.rb
+- lib/football-sources/worldfootball/leagues/south_america.rb
+- lib/football-sources/worldfootball/mods.rb
+- lib/football-sources/worldfootball/tool.rb
+- lib/football-sources/worldfootball/vacuum.rb
+- lib/football/sources.rb
+- test/helper.rb
+- test/test_version.rb
+homepage: https://github.com/sportdb/sport.db
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: football-sources - get football data via web pages or web api (json) calls
+test_files: []