footballdata-12xpert 0.0.1

data/lib/footballdata-12xpert/download.rb

@@ -0,0 +1,92 @@
+module Footballdata12xpert
+
+
+def self.download( *country_keys, start: nil )   ## download all datasets (all leagues, all seasons)
+  ## note: always downcase and symbolize keys (thus, allow strings too for now)
+  country_keys = country_keys.map {|key| key.downcase.to_sym }
+
+  SOURCES_I.each do |country_key, country_sources|
+    if country_keys.empty? || country_keys.include?( country_key )
+      download_season_by_season( country_sources, start: start )
+    else
+      ## skipping country
+    end
+  end
+
+  SOURCES_II.each do |country_key, country_basename|
+    if country_keys.empty? || country_keys.include?( country_key )
+      download_all_seasons( country_basename )
+    else
+      ## skipping country
+    end
+  end
+end  ## method download
+
+
+
+###
+# private helpers / machinery
+
+BASE_URL = 'http://www.football-data.co.uk'
+
+def self.season_by_season_url( basename, season )
+  # build short format e.g. 2008/09 becomes 0809
+  #                         2019/20 becomes 1920 etc.
+  season_path = "%02d%02d" % [season.start_year % 100, season.end_year % 100]
+  # note: add "magic" mmz4281/ directory to base_url
+  "#{BASE_URL}/mmz4281/#{season_path}/#{basename}.csv"
+end
+
+def self.all_seasons_url( basename )
+  # note: add new/ directory  to base_url
+  "#{BASE_URL}/new/#{basename}.csv"
+end
+
+
+
+def self.download_season_by_season( sources, start: nil )   ## format i - one datafile per season
+  start = Season.parse( start )   if start  ## convert to season obj
+
+  sources.each do |rec|
+    season     = Season.parse( rec[0] )   ## note: dirname is season_key e.g. 2011-12 etc.
+    basenames  = rec[1]
+
+    if start && season < start
+      puts "skipping #{season} before #{start}"
+      next
+    end
+
+    basenames.each do |basename|
+      url = season_by_season_url( basename, season )
+      get( url )
+    end
+  end
+end
+
+def self.download_all_seasons( basename )   ## format ii - all-seasons-in-one-datafile
+  url = all_seasons_url( basename )
+  get( url )
+end
+
+
+
+#############
+# helpers
+def self.get( url )
+  ## [debug] GET=http://www.football-data.co.uk/mmz4281/0405/SC0.csv
+  ##    Encoding::UndefinedConversionError: "\xA0" from ASCII-8BIT to UTF-8
+  ##     note:  0xA0 (160) is NBSP (non-breaking space) in Windows-1252
+
+  ## note: assume windows encoding (for football-data.uk)
+  ##   use "Windows-1252" for input and convert to utf-8
+  ##
+  ##    see https://www.justinweiss.com/articles/3-steps-to-fix-encoding-problems-in-ruby/
+  ##    see https://en.wikipedia.org/wiki/Windows-1252
+
+  response = Webget.dataset( url, encoding: 'Windows-1252' )
+
+  ## note: exit on get / fetch error - do NOT continue for now - why? why not?
+  exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
+end
+
+end # module Footballdata12xpert

data/lib/footballdata-12xpert/version.rb

@@ -0,0 +1,22 @@
+## todo: change to FootballDb::Module::Footballdata12xpert or such - why? why not?
+
+module Footballdata12xpert
+
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
+  PATCH = 1
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "footballdata-12xpert/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
+  end
+
+end # module Footballdata12xpert

data/lib/footballdata/12xpert.rb

@@ -0,0 +1,6 @@
+# note: allow require 'footballdata/12xpert' too
+#        (in addition to require 'footballdata-12xpert')
+
+require_relative '../footballdata-12xpert'
+
+

data/test/helper.rb

@@ -0,0 +1,10 @@
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+
+require 'minitest/autorun'
+
+
+## our own code
+
+require 'footballdata/12xpert'

data/test/test_download.rb

@@ -0,0 +1,27 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_download.rb
+
+
+require 'helper'
+
+class TestDownload < MiniTest::Test
+
+  def test_sources
+    pp Footballdata12xpert::SOURCES_I
+    pp Footballdata12xpert::SOURCES_II
+  end
+
+  def test_download_at
+    Footballdata12xpert.download( 'at' )
+  end
+
+  def test_download_eng
+    Footballdata12xpert.download( 'eng', start: '2020/21' )
+  end
+
+  def xxx_test_download_all   ## note: test disabled for now
+    Footballdata12xpert.download
+  end
+
+end # class TestDownload

data/test/test_import.rb

@@ -0,0 +1,33 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_import.rb
+
+
+require 'helper'
+
+class TestImport < MiniTest::Test
+
+  def setup
+    ## SportDb::Import.config.clubs_dir = '../../../openfootball/clubs'
+
+    SportDb.connect( adapter:  'sqlite3',
+                     database: ':memory:' )
+
+    ## build database schema / tables
+    SportDb.create_all
+  end
+
+
+  def test_import_at
+    Footballdata.import( :at, dir: './dl'  )
+  end
+
+  def test_import_eng
+    Footballdata.import( :eng, dir: './dl')
+  end
+
+  def test_import_all
+     Footballdata.import( dir: './dl' )
+  end
+
+end # class TestImport

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: footballdata-12xpert
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webget
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: footballdb-clubs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2020.9.15
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2020.9.15
+- !ruby/object:Gem::Dependency
+  name: sportdb-importers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+description: footballdata-12xpert - download, convert & import 22+ top football leagues
+  from 25 seasons back to 1993/94 from Joseph Buchdahl (12Xpert)'s Football Data website
+  (football-data.co.uk) up and running since 2001 (and updated twice a week)
+email: opensport@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/footballdata-12xpert.rb
+- lib/footballdata-12xpert/config.rb
+- lib/footballdata-12xpert/convert.rb
+- lib/footballdata-12xpert/download.rb
+- lib/footballdata-12xpert/version.rb
+- lib/footballdata/12xpert.rb
+- test/helper.rb
+- test/test_download.rb
+- test/test_import.rb
+homepage: https://github.com/sportdb/sport.db.sources
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: footballdata-12xpert - download, convert & import 22+ top football leagues
+  from 25 seasons back to 1993/94 from Joseph Buchdahl (12Xpert)'s Football Data website
+  (football-data.co.uk) up and running since 2001 (and updated twice a week)
+test_files: []