footballdata-12xpert 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,92 @@
1
+ module Footballdata12xpert
2
+
3
+
4
+ def self.download( *country_keys, start: nil ) ## download all datasets (all leagues, all seasons)
5
+ ## note: always downcase and symbolize keys (thus, allow strings too for now)
6
+ country_keys = country_keys.map {|key| key.downcase.to_sym }
7
+
8
+ SOURCES_I.each do |country_key, country_sources|
9
+ if country_keys.empty? || country_keys.include?( country_key )
10
+ download_season_by_season( country_sources, start: start )
11
+ else
12
+ ## skipping country
13
+ end
14
+ end
15
+
16
+ SOURCES_II.each do |country_key, country_basename|
17
+ if country_keys.empty? || country_keys.include?( country_key )
18
+ download_all_seasons( country_basename )
19
+ else
20
+ ## skipping country
21
+ end
22
+ end
23
+ end ## method download
24
+
25
+
26
+
27
+ ###
28
+ # private helpers / machinery
29
+
30
+ BASE_URL = 'http://www.football-data.co.uk'
31
+
32
+ def self.season_by_season_url( basename, season )
33
+ # build short format e.g. 2008/09 becomes 0809
34
+ # 2019/20 becomes 1920 etc.
35
+ season_path = "%02d%02d" % [season.start_year % 100, season.end_year % 100]
36
+ # note: add "magic" mmz4281/ directory to base_url
37
+ "#{BASE_URL}/mmz4281/#{season_path}/#{basename}.csv"
38
+ end
39
+
40
+ def self.all_seasons_url( basename )
41
+ # note: add new/ directory to base_url
42
+ "#{BASE_URL}/new/#{basename}.csv"
43
+ end
44
+
45
+
46
+
47
+ def self.download_season_by_season( sources, start: nil ) ## format i - one datafile per season
48
+ start = Season.parse( start ) if start ## convert to season obj
49
+
50
+ sources.each do |rec|
51
+ season = Season.parse( rec[0] ) ## note: dirname is season_key e.g. 2011-12 etc.
52
+ basenames = rec[1]
53
+
54
+ if start && season < start
55
+ puts "skipping #{season} before #{start}"
56
+ next
57
+ end
58
+
59
+ basenames.each do |basename|
60
+ url = season_by_season_url( basename, season )
61
+ get( url )
62
+ end
63
+ end
64
+ end
65
+
66
+ def self.download_all_seasons( basename ) ## format ii - all-seasons-in-one-datafile
67
+ url = all_seasons_url( basename )
68
+ get( url )
69
+ end
70
+
71
+
72
+
73
+ #############
74
+ # helpers
75
+ def self.get( url )
76
+ ## [debug] GET=http://www.football-data.co.uk/mmz4281/0405/SC0.csv
77
+ ## Encoding::UndefinedConversionError: "\xA0" from ASCII-8BIT to UTF-8
78
+ ## note: 0xA0 (160) is NBSP (non-breaking space) in Windows-1252
79
+
80
+ ## note: assume windows encoding (for football-data.uk)
81
+ ## use "Windows-1252" for input and convert to utf-8
82
+ ##
83
+ ## see https://www.justinweiss.com/articles/3-steps-to-fix-encoding-problems-in-ruby/
84
+ ## see https://en.wikipedia.org/wiki/Windows-1252
85
+
86
+ response = Webget.dataset( url, encoding: 'Windows-1252' )
87
+
88
+ ## note: exit on get / fetch error - do NOT continue for now - why? why not?
89
+ exit 1 if response.status.nok? ## e.g. HTTP status code != 200
90
+ end
91
+
92
+ end # module Footballdata12xpert
@@ -0,0 +1,22 @@
1
+ ## todo: change to FootballDb::Module::Footballdata12xpert or such - why? why not?
2
+
3
+ module Footballdata12xpert
4
+
5
+ MAJOR = 0 ## todo: namespace inside version or something - why? why not??
6
+ MINOR = 0
7
+ PATCH = 1
8
+ VERSION = [MAJOR,MINOR,PATCH].join('.')
9
+
10
+ def self.version
11
+ VERSION
12
+ end
13
+
14
+ def self.banner
15
+ "footballdata-12xpert/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
16
+ end
17
+
18
+ def self.root
19
+ File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
20
+ end
21
+
22
+ end # module Footballdata12xpert
@@ -0,0 +1,6 @@
1
+ # note: allow require 'footballdata/12xpert' too
2
+ # (in addition to require 'footballdata-12xpert')
3
+
4
+ require_relative '../footballdata-12xpert'
5
+
6
+
@@ -0,0 +1,10 @@
1
+ ## $:.unshift(File.dirname(__FILE__))
2
+
3
+ ## minitest setup
4
+
5
+ require 'minitest/autorun'
6
+
7
+
8
+ ## our own code
9
+
10
+ require 'footballdata/12xpert'
@@ -0,0 +1,27 @@
1
+ ###
2
+ # to run use
3
+ # ruby -I ./lib -I ./test test/test_download.rb
4
+
5
+
6
+ require 'helper'
7
+
8
+ class TestDownload < MiniTest::Test
9
+
10
+ def test_sources
11
+ pp Footballdata12xpert::SOURCES_I
12
+ pp Footballdata12xpert::SOURCES_II
13
+ end
14
+
15
+ def test_download_at
16
+ Footballdata12xpert.download( 'at' )
17
+ end
18
+
19
+ def test_download_eng
20
+ Footballdata12xpert.download( 'eng', start: '2020/21' )
21
+ end
22
+
23
+ def xxx_test_download_all ## note: test disabled for now
24
+ Footballdata12xpert.download
25
+ end
26
+
27
+ end # class TestDownload
@@ -0,0 +1,33 @@
1
+ ###
2
+ # to run use
3
+ # ruby -I ./lib -I ./test test/test_import.rb
4
+
5
+
6
+ require 'helper'
7
+
8
+ class TestImport < MiniTest::Test
9
+
10
+ def setup
11
+ ## SportDb::Import.config.clubs_dir = '../../../openfootball/clubs'
12
+
13
+ SportDb.connect( adapter: 'sqlite3',
14
+ database: ':memory:' )
15
+
16
+ ## build database schema / tables
17
+ SportDb.create_all
18
+ end
19
+
20
+
21
+ def test_import_at
22
+ Footballdata.import( :at, dir: './dl' )
23
+ end
24
+
25
+ def test_import_eng
26
+ Footballdata.import( :eng, dir: './dl')
27
+ end
28
+
29
+ def test_import_all
30
+ Footballdata.import( dir: './dl' )
31
+ end
32
+
33
+ end # class TestImport
metadata ADDED
@@ -0,0 +1,141 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: footballdata-12xpert
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Gerald Bauer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-11-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: webget
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 0.2.1
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 0.2.1
27
+ - !ruby/object:Gem::Dependency
28
+ name: footballdb-clubs
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 2020.9.15
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 2020.9.15
41
+ - !ruby/object:Gem::Dependency
42
+ name: sportdb-importers
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 1.1.1
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 1.1.1
55
+ - !ruby/object:Gem::Dependency
56
+ name: rdoc
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '4.0'
62
+ - - "<"
63
+ - !ruby/object:Gem::Version
64
+ version: '7'
65
+ type: :development
66
+ prerelease: false
67
+ version_requirements: !ruby/object:Gem::Requirement
68
+ requirements:
69
+ - - ">="
70
+ - !ruby/object:Gem::Version
71
+ version: '4.0'
72
+ - - "<"
73
+ - !ruby/object:Gem::Version
74
+ version: '7'
75
+ - !ruby/object:Gem::Dependency
76
+ name: hoe
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '3.22'
82
+ type: :development
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '3.22'
89
+ description: footballdata-12xpert - download, convert & import 22+ top football leagues
90
+ from 25 seasons back to 1993/94 from Joseph Buchdahl (12Xpert)'s Football Data website
91
+ (football-data.co.uk) up and running since 2001 (and updated twice a week)
92
+ email: opensport@googlegroups.com
93
+ executables: []
94
+ extensions: []
95
+ extra_rdoc_files:
96
+ - CHANGELOG.md
97
+ - Manifest.txt
98
+ - README.md
99
+ files:
100
+ - CHANGELOG.md
101
+ - Manifest.txt
102
+ - README.md
103
+ - Rakefile
104
+ - lib/footballdata-12xpert.rb
105
+ - lib/footballdata-12xpert/config.rb
106
+ - lib/footballdata-12xpert/convert.rb
107
+ - lib/footballdata-12xpert/download.rb
108
+ - lib/footballdata-12xpert/version.rb
109
+ - lib/footballdata/12xpert.rb
110
+ - test/helper.rb
111
+ - test/test_download.rb
112
+ - test/test_import.rb
113
+ homepage: https://github.com/sportdb/sport.db.sources
114
+ licenses:
115
+ - Public Domain
116
+ metadata: {}
117
+ post_install_message:
118
+ rdoc_options:
119
+ - "--main"
120
+ - README.md
121
+ require_paths:
122
+ - lib
123
+ required_ruby_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: 2.2.2
128
+ required_rubygems_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ requirements: []
134
+ rubyforge_project:
135
+ rubygems_version: 2.5.2
136
+ signing_key:
137
+ specification_version: 4
138
+ summary: footballdata-12xpert - download, convert & import 22+ top football leagues
139
+ from 25 seasons back to 1993/94 from Joseph Buchdahl (12Xpert)'s Football Data website
140
+ (football-data.co.uk) up and running since 2001 (and updated twice a week)
141
+ test_files: []