football-sources 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +6 -4
- data/Manifest.txt +2 -20
- data/README.md +30 -169
- data/Rakefile +36 -31
- data/bin/fbgen +131 -0
- data/lib/football/sources.rb +6 -6
- data/lib/football-sources/process.rb +68 -0
- data/lib/football-sources/version.rb +19 -19
- data/lib/football-sources.rb +21 -57
- metadata +51 -40
- data/lib/football-sources/apis/config.rb +0 -17
- data/lib/football-sources/apis/convert.rb +0 -239
- data/lib/football-sources/apis/convert_cl.rb +0 -267
- data/lib/football-sources/apis/mods.rb +0 -20
- data/lib/football-sources/apis/stat.rb +0 -59
- data/lib/football-sources/apis.rb +0 -10
- data/lib/football-sources/fbref/build.rb +0 -96
- data/lib/football-sources/fbref/config.rb +0 -16
- data/lib/football-sources/fbref/convert.rb +0 -95
- data/lib/football-sources/fbref.rb +0 -4
- data/lib/football-sources/worldfootball/build.rb +0 -245
- data/lib/football-sources/worldfootball/config.rb +0 -16
- data/lib/football-sources/worldfootball/convert.rb +0 -100
- data/lib/football-sources/worldfootball/convert_reports.rb +0 -107
- data/lib/football-sources/worldfootball/jobs.rb +0 -76
- data/lib/football-sources/worldfootball/mods.rb +0 -72
- data/lib/football-sources/worldfootball/vacuum.rb +0 -66
- data/lib/football-sources/worldfootball.rb +0 -19
- data/test/helper.rb +0 -8
- data/test/test_version.rb +0 -16
@@ -1,107 +0,0 @@
|
|
1
|
-
module Worldfootball
|
2
|
-
|
3
|
-
|
4
|
-
def self.convert_reports( league:, season: )
|
5
|
-
season = Season( season ) ## cast (ensure) season class (NOT string, integer, etc.)
|
6
|
-
|
7
|
-
league = find_league( league )
|
8
|
-
|
9
|
-
## note: use only first part from key for lookup
|
10
|
-
## e.g. at.1 => at
|
11
|
-
## eng.1 => eng
|
12
|
-
## and so on
|
13
|
-
mods = MODS[ league.key.split('.')[0] ] || {}
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
pages = league.pages( season: season )
|
18
|
-
|
19
|
-
recs = []
|
20
|
-
|
21
|
-
## if single (simple) page setup - wrap in array
|
22
|
-
pages = pages.is_a?(Array) ? pages : [pages]
|
23
|
-
pages.each do |page_meta| # note: use page_info for now (or page_rec or page_meta or such)
|
24
|
-
|
25
|
-
page = Page::Schedule.from_cache( page_meta[:slug] )
|
26
|
-
print " page title=>#{page.title}<..."
|
27
|
-
print "\n"
|
28
|
-
|
29
|
-
matches = page.matches
|
30
|
-
|
31
|
-
puts "matches - #{matches.size} rows:"
|
32
|
-
pp matches[0]
|
33
|
-
|
34
|
-
puts "#{page.generated_in_days_ago} - #{page.generated}"
|
35
|
-
|
36
|
-
|
37
|
-
matches.each_with_index do |match,i|
|
38
|
-
|
39
|
-
report_ref = match[:report_ref]
|
40
|
-
if report_ref.nil?
|
41
|
-
puts "!! WARN: no match report ref found for match:"
|
42
|
-
pp match
|
43
|
-
next
|
44
|
-
end
|
45
|
-
|
46
|
-
puts "reading #{i+1}/#{matches.size} - #{report_ref}..."
|
47
|
-
report = Page::Report.from_cache( report_ref )
|
48
|
-
|
49
|
-
puts
|
50
|
-
puts report.title
|
51
|
-
puts report.generated
|
52
|
-
|
53
|
-
rows = report.goals
|
54
|
-
puts "goals - #{rows.size} records"
|
55
|
-
## pp rows
|
56
|
-
|
57
|
-
|
58
|
-
if rows.size > 0
|
59
|
-
## add goals
|
60
|
-
date = Date.strptime( match[:date], '%Y-%m-%d')
|
61
|
-
|
62
|
-
team1 = match[:team1]
|
63
|
-
team2 = match[:team2]
|
64
|
-
|
65
|
-
## clean team name (e.g. remove (old))
|
66
|
-
## and asciify (e.g. ’ to ' )
|
67
|
-
team1 = norm_team( team1 )
|
68
|
-
team2 = norm_team( team2 )
|
69
|
-
|
70
|
-
team1 = mods[ team1 ] if mods[ team1 ]
|
71
|
-
team2 = mods[ team2 ] if mods[ team2 ]
|
72
|
-
|
73
|
-
match_id = "#{team1} - #{team2} | #{date.strftime('%b %-d %Y')}"
|
74
|
-
|
75
|
-
|
76
|
-
rows.each do |row|
|
77
|
-
extra = if row[:owngoal]
|
78
|
-
'(og)' ## or use OG or O.G.- why? why not?
|
79
|
-
elsif row[:penalty]
|
80
|
-
'(pen)' ## or use P or PEN - why? why not?
|
81
|
-
else
|
82
|
-
''
|
83
|
-
end
|
84
|
-
|
85
|
-
rec = [match_id,
|
86
|
-
row[:score],
|
87
|
-
"#{row[:minute]}'",
|
88
|
-
extra,
|
89
|
-
row[:player],
|
90
|
-
row[:notes]]
|
91
|
-
recs << rec
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end # each match
|
95
|
-
end # each page
|
96
|
-
|
97
|
-
## pp recs
|
98
|
-
|
99
|
-
out_path = "#{config.convert.out_dir}/#{season.path}/#{league.key}~goals.csv"
|
100
|
-
|
101
|
-
headers = ['Match', 'Score', 'Minute', 'Extra', 'Player', 'Notes']
|
102
|
-
|
103
|
-
puts "write #{out_path}..."
|
104
|
-
Cache::CsvMatchWriter.write( out_path, recs, headers: headers )
|
105
|
-
end
|
106
|
-
end # module Worldfootballl
|
107
|
-
|
@@ -1,76 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
module Worldfootball
|
4
|
-
################################
|
5
|
-
# add more helpers
|
6
|
-
# move upstream for (re)use - why? why not?
|
7
|
-
|
8
|
-
## todo/check: what to do: if league is both included and excluded?
|
9
|
-
## include forces include? or exclude has the last word? - why? why not?
|
10
|
-
## Excludes match before includes,
|
11
|
-
## meaning that something that has been excluded cannot be included again
|
12
|
-
|
13
|
-
## todo - find "proper/classic" timezone ("winter time")
|
14
|
-
|
15
|
-
## Brasilia - Distrito Federal, Brasil (GMT-3) -- summer time?
|
16
|
-
## Ciudad de México, CDMX, México (GMT-5) -- summer time?
|
17
|
-
## Londres, Reino Unido (GMT+1)
|
18
|
-
## Madrid -- ?
|
19
|
-
## Lisboa -- ?
|
20
|
-
## Moskow -- ?
|
21
|
-
##
|
22
|
-
## todo/check - quick fix timezone offsets for leagues for now
|
23
|
-
## - find something better - why? why not?
|
24
|
-
## note: assume time is in GMT+1
|
25
|
-
OFFSETS = {
|
26
|
-
'eng.1' => -1,
|
27
|
-
'eng.2' => -1,
|
28
|
-
'eng.3' => -1,
|
29
|
-
'eng.4' => -1,
|
30
|
-
'eng.5' => -1,
|
31
|
-
|
32
|
-
'es.1' => -1,
|
33
|
-
'es.2' => -1,
|
34
|
-
|
35
|
-
'pt.1' => -1,
|
36
|
-
'pt.2' => -1,
|
37
|
-
|
38
|
-
'br.1' => -5,
|
39
|
-
'mx.1' => -7,
|
40
|
-
}
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
class Job ## todo/check: use a module (NOT a class) - why? why not?
|
45
|
-
def self.download( datasets )
|
46
|
-
datasets.each_with_index do |dataset,i|
|
47
|
-
league = dataset[0]
|
48
|
-
seasons = dataset[1]
|
49
|
-
|
50
|
-
puts "downloading [#{i+1}/#{datasets.size}] #{league}..."
|
51
|
-
seasons.each_with_index do |season,j|
|
52
|
-
puts " season [#{j+1}/#{season.size}] #{league} #{season}..."
|
53
|
-
Worldfootball.schedule( league: league,
|
54
|
-
season: season )
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def self.convert( datasets )
|
60
|
-
datasets.each_with_index do |dataset,i|
|
61
|
-
league = dataset[0]
|
62
|
-
seasons = dataset[1]
|
63
|
-
|
64
|
-
puts "converting [#{i+1}/#{datasets.size}] #{league}..."
|
65
|
-
seasons.each_with_index do |season,j|
|
66
|
-
puts " season [#{j+1}/#{season.size}] #{league} #{season}..."
|
67
|
-
Worldfootball.convert( league: league,
|
68
|
-
season: season,
|
69
|
-
offset: OFFSETS[ league ] )
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end # class Job
|
74
|
-
|
75
|
-
end # module Worldfootball
|
76
|
-
|
@@ -1,72 +0,0 @@
|
|
1
|
-
#### todo/check: move MODS and SCORE_ERRORS out-of-lib
|
2
|
-
## and into config or such - why? why not?
|
3
|
-
|
4
|
-
|
5
|
-
module Worldfootball
|
6
|
-
|
7
|
-
|
8
|
-
######
|
9
|
-
# "global" helpers
|
10
|
-
def self.norm_team( team )
|
11
|
-
## clean team name and asciify (e.g. ’->' )
|
12
|
-
team = team.sub( '(old)', '' ).strip
|
13
|
-
team = team.gsub( '’', "'" ) ## e.g. Hawke’s Bay United FC
|
14
|
-
team
|
15
|
-
end
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
MODS = {
|
20
|
-
'at' => {
|
21
|
-
## AT 1
|
22
|
-
'SC Magna Wiener Neustadt' => 'SC Wiener Neustadt', # in 2010/11
|
23
|
-
'KSV Superfund' => 'Kapfenberger SV', # in 2010/11
|
24
|
-
'Kapfenberger SV 1919' => 'Kapfenberger SV', # in 2011/12
|
25
|
-
'FC Trenkwalder Admira' => 'FC Admira Wacker', # in 2011/12
|
26
|
-
## AT 2
|
27
|
-
'Austria Wien (A)' => 'Young Violets', # in 2019/20
|
28
|
-
'FC Wacker Innsbruck (A)' => 'FC Wacker Innsbruck II', # in 2018/19
|
29
|
-
## AT CUP
|
30
|
-
'Rapid Wien (A)' => 'Rapid Wien II', # in 2011/12
|
31
|
-
'Sturm Graz (A)' => 'Sturm Graz II',
|
32
|
-
'Kapfenberger SV 1919 (A)' => 'Kapfenberger SV II',
|
33
|
-
'SV Grödig (A)' => 'SV Grödig II',
|
34
|
-
'RB Salzburg (A)' => 'RB Salzburg II',
|
35
|
-
'SR WGFM Donaufeld' => 'SR Donaufeld Wien',
|
36
|
-
'FC Trenkwalder Admira (A)' => 'FC Admira Wacker II',
|
37
|
-
## AT 3.O (Regionalliga Ost)
|
38
|
-
'FC Admira Wacker (A)' => 'FC Admira Wacker II', # in 2020/21
|
39
|
-
},
|
40
|
-
'nz' => {
|
41
|
-
## NZ 1
|
42
|
-
'Wellington Phoenix (R)' => 'Wellington Phoenix Reserves',
|
43
|
-
},
|
44
|
-
}
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
## fix/patch known score format errors in at/de cups
|
49
|
-
## new convention
|
50
|
-
## for a fix require league, date, and team1 & team2 for now!!!!
|
51
|
-
## - do NOT use some "generic" fix / patch!!!!
|
52
|
-
##
|
53
|
-
## old de/at patches/fixes:
|
54
|
-
## '0-1 (0-0, 0-0, 0-0) n.V.' => '0-1 (0-0, 0-0) n.V.', # too long
|
55
|
-
## '2-1 (1-1, 1-1, 1-0) n.V.' => '2-1 (1-1, 1-1) n.V.',
|
56
|
-
## '4-2 (0-0, 0-0) i.E.' => '4-2 (0-0, 0-0, 0-0) i.E.', # too short
|
57
|
-
|
58
|
-
|
59
|
-
SCORE_ERRORS = {
|
60
|
-
'ro.1' => {
|
61
|
-
## 2013/14
|
62
|
-
'2013-07-29' => [ 'FC Brașov', 'Săgeata Năvodari', ['1-1 (0-0, 0-1)', '1-1 (0-0)']],
|
63
|
-
},
|
64
|
-
'gr.1' => {
|
65
|
-
## 2010/11
|
66
|
-
'2010-11-24' => [ 'Ergotelis', 'Olympiakos Piräus', ['0-2 (0-0, 0-0, 0-0)', '0-2 (0-0)']],
|
67
|
-
'2010-11-28' => [ 'Panserraikos', 'Aris Saloniki', ['1-0 (1-0, 0-0, 0-0)', '1-0 (1-0)']],
|
68
|
-
}
|
69
|
-
}
|
70
|
-
|
71
|
-
|
72
|
-
end # module Worldfootball
|
@@ -1,66 +0,0 @@
|
|
1
|
-
module Worldfootball
|
2
|
-
|
3
|
-
|
4
|
-
MAX_HEADERS = [
|
5
|
-
'Stage',
|
6
|
-
'Round',
|
7
|
-
'Date',
|
8
|
-
'Time',
|
9
|
-
'Team 1',
|
10
|
-
'FT',
|
11
|
-
'HT',
|
12
|
-
'Team 2',
|
13
|
-
'ET',
|
14
|
-
'P',
|
15
|
-
'Comments'] ## e.g. awarded, cancelled/canceled, etc.
|
16
|
-
|
17
|
-
MIN_HEADERS = [ ## always keep even if all empty
|
18
|
-
'Date',
|
19
|
-
'Team 1',
|
20
|
-
'FT',
|
21
|
-
'Team 2'
|
22
|
-
]
|
23
|
-
|
24
|
-
def self.vacuum( rows, headers: MAX_HEADERS, fixed_headers: MIN_HEADERS )
|
25
|
-
## check for unused columns and strip/remove
|
26
|
-
counter = Array.new( MAX_HEADERS.size, 0 )
|
27
|
-
rows.each do |row|
|
28
|
-
row.each_with_index do |col, idx|
|
29
|
-
counter[idx] += 1 unless col.nil? || col.empty?
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
pp counter
|
34
|
-
|
35
|
-
## check empty columns
|
36
|
-
headers = []
|
37
|
-
indices = []
|
38
|
-
empty_headers = []
|
39
|
-
empty_indices = []
|
40
|
-
|
41
|
-
counter.each_with_index do |num, idx|
|
42
|
-
header = MAX_HEADERS[ idx ]
|
43
|
-
if num > 0 || (num == 0 && fixed_headers.include?( header ))
|
44
|
-
headers << header
|
45
|
-
indices << idx
|
46
|
-
else
|
47
|
-
empty_headers << header
|
48
|
-
empty_indices << idx
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
if empty_indices.size > 0
|
53
|
-
rows = rows.map do |row|
|
54
|
-
row_vacuumed = []
|
55
|
-
row.each_with_index do |col, idx|
|
56
|
-
## todo/fix: use values or such??
|
57
|
-
row_vacuumed << col unless empty_indices.include?( idx )
|
58
|
-
end
|
59
|
-
row_vacuumed
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
[rows, headers]
|
64
|
-
end
|
65
|
-
|
66
|
-
end # module Worldfootball
|
@@ -1,19 +0,0 @@
|
|
1
|
-
###########################
|
2
|
-
# note: split code in two parts
|
3
|
-
# metal - "bare" basics - no ref to sportdb
|
4
|
-
# and rest / convert with sportdb references / goodies
|
5
|
-
|
6
|
-
|
7
|
-
## our own code
|
8
|
-
require_relative 'worldfootball/config'
|
9
|
-
require_relative 'worldfootball/mods'
|
10
|
-
require_relative 'worldfootball/vacuum'
|
11
|
-
require_relative 'worldfootball/build'
|
12
|
-
require_relative 'worldfootball/convert'
|
13
|
-
require_relative 'worldfootball/convert_reports'
|
14
|
-
|
15
|
-
|
16
|
-
require_relative 'worldfootball/jobs'
|
17
|
-
|
18
|
-
|
19
|
-
|
data/test/helper.rb
DELETED
data/test/test_version.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
###
|
2
|
-
# to run use
|
3
|
-
# ruby -I ./lib -I ./test test/test_version.rb
|
4
|
-
|
5
|
-
|
6
|
-
require 'helper'
|
7
|
-
|
8
|
-
class TestVersion < MiniTest::Test
|
9
|
-
|
10
|
-
def test_version
|
11
|
-
pp FootballSources::VERSION
|
12
|
-
pp FootballSources.banner
|
13
|
-
pp FootballSources.root
|
14
|
-
end
|
15
|
-
|
16
|
-
end # class TestVersion
|