sportdb-formats 2.0.2 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -1
- data/Manifest.txt +2 -20
- data/Rakefile +2 -7
- data/bin/fbchk +173 -0
- data/lib/sportdb/formats/quick_match_linter.rb +201 -0
- data/lib/sportdb/formats/version.rb +2 -2
- data/lib/sportdb/formats.rb +10 -269
- metadata +11 -85
- data/bin/fbx +0 -146
- data/lib/sportdb/formats/country/country_reader.rb +0 -142
- data/lib/sportdb/formats/csv/goal.rb +0 -192
- data/lib/sportdb/formats/csv/goal_parser_csv.rb +0 -28
- data/lib/sportdb/formats/csv/match_parser_csv.rb +0 -490
- data/lib/sportdb/formats/csv/match_status_parser.rb +0 -90
- data/lib/sportdb/formats/datafile.rb +0 -59
- data/lib/sportdb/formats/event/event_reader.rb +0 -119
- data/lib/sportdb/formats/ground/ground_reader.rb +0 -289
- data/lib/sportdb/formats/league/league_outline_reader.rb +0 -176
- data/lib/sportdb/formats/league/league_reader.rb +0 -152
- data/lib/sportdb/formats/match/conf_parser.rb +0 -132
- data/lib/sportdb/formats/match/match_parser.rb +0 -735
- data/lib/sportdb/formats/search/sport.rb +0 -372
- data/lib/sportdb/formats/search/structs.rb +0 -116
- data/lib/sportdb/formats/search/world.rb +0 -157
- data/lib/sportdb/formats/team/club_reader.rb +0 -318
- data/lib/sportdb/formats/team/club_reader_history.rb +0 -203
- data/lib/sportdb/formats/team/club_reader_props.rb +0 -90
- data/lib/sportdb/formats/team/wiki_reader.rb +0 -108
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2504c391e84f0f27c476d3cd355dedcaf594c1d8d11f0769590b8be904b0a8b0
|
4
|
+
data.tar.gz: a16b1d8b7f7ee50efd9b88d50beafd66bce70c8c2d632969208f0b8b2a48d777
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb5e968e9c148c1e11434ec33a7c8909fff8bcd45ee8d39eafb6005a678cfee766390178d2d70d68bc0b164c1e0a691759dc663eb962c902032c206ea778d0c5
|
7
|
+
data.tar.gz: 58ff1d22a5af74b3831afb119a47011c572ef23b87c1e5b6e7399d75d68c6f5286919d8402dd0503f87a9c2da1f6fc81ad9524e9c10e3e18c2e4207ecbe92ac3
|
data/CHANGELOG.md
CHANGED
data/Manifest.txt
CHANGED
@@ -2,28 +2,10 @@ CHANGELOG.md
|
|
2
2
|
Manifest.txt
|
3
3
|
README.md
|
4
4
|
Rakefile
|
5
|
-
bin/
|
5
|
+
bin/fbchk
|
6
6
|
lib/sportdb/formats.rb
|
7
|
-
lib/sportdb/formats/country/country_reader.rb
|
8
|
-
lib/sportdb/formats/csv/goal.rb
|
9
|
-
lib/sportdb/formats/csv/goal_parser_csv.rb
|
10
|
-
lib/sportdb/formats/csv/match_parser_csv.rb
|
11
|
-
lib/sportdb/formats/csv/match_status_parser.rb
|
12
|
-
lib/sportdb/formats/datafile.rb
|
13
7
|
lib/sportdb/formats/datafile_package.rb
|
14
|
-
lib/sportdb/formats/event/event_reader.rb
|
15
|
-
lib/sportdb/formats/ground/ground_reader.rb
|
16
|
-
lib/sportdb/formats/league/league_outline_reader.rb
|
17
|
-
lib/sportdb/formats/league/league_reader.rb
|
18
|
-
lib/sportdb/formats/match/conf_parser.rb
|
19
|
-
lib/sportdb/formats/match/match_parser.rb
|
20
8
|
lib/sportdb/formats/package.rb
|
21
|
-
lib/sportdb/formats/
|
22
|
-
lib/sportdb/formats/search/structs.rb
|
23
|
-
lib/sportdb/formats/search/world.rb
|
9
|
+
lib/sportdb/formats/quick_match_linter.rb
|
24
10
|
lib/sportdb/formats/team/club_index_history.rb
|
25
|
-
lib/sportdb/formats/team/club_reader.rb
|
26
|
-
lib/sportdb/formats/team/club_reader_history.rb
|
27
|
-
lib/sportdb/formats/team/club_reader_props.rb
|
28
|
-
lib/sportdb/formats/team/wiki_reader.rb
|
29
11
|
lib/sportdb/formats/version.rb
|
data/Rakefile
CHANGED
@@ -20,13 +20,8 @@ Hoe.spec 'sportdb-formats' do
|
|
20
20
|
self.licenses = ['Public Domain']
|
21
21
|
|
22
22
|
self.extra_deps = [
|
23
|
-
['sportdb-
|
24
|
-
['sportdb-
|
25
|
-
['sportdb-catalogs', '>= 1.2.2'],
|
26
|
-
['date-formats', '>= 1.0.2'],
|
27
|
-
['cocos', '>= 0.4.0'],
|
28
|
-
['logutils', '>= 0.6.1'],
|
29
|
-
|
23
|
+
['sportdb-search', '>= 0.0.1'],
|
24
|
+
['sportdb-quick', '>= 0.2.0'],
|
30
25
|
['rubyzip', '>= 2.3.2' ],
|
31
26
|
]
|
32
27
|
|
data/bin/fbchk
ADDED
@@ -0,0 +1,173 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
## tip: to test run:
|
4
|
+
## ruby -I ./lib bin/fbchk
|
5
|
+
|
6
|
+
## our own code
|
7
|
+
require 'sportdb/formats'
|
8
|
+
|
9
|
+
|
10
|
+
|
11
|
+
require 'optparse'
|
12
|
+
|
13
|
+
|
14
|
+
|
15
|
+
## local hack
|
16
|
+
## if exists up-to-date catalog db (use local version NOT built-in)
|
17
|
+
catalog_path = '../catalog/catalog.db'
|
18
|
+
if File.exist?( catalog_path )
|
19
|
+
SportDb::Import.config.catalog_path = catalog_path
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
args = ARGV
|
24
|
+
opts = { debug: false,
|
25
|
+
file: nil,
|
26
|
+
teams: true, ## check/lint teams (name errros etc.)
|
27
|
+
}
|
28
|
+
|
29
|
+
parser = OptionParser.new do |parser|
|
30
|
+
parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
31
|
+
|
32
|
+
##
|
33
|
+
## check if git has a offline option?? (use same)
|
34
|
+
## check for other tools - why? why not?
|
35
|
+
# parser.on( "-q", "--quiet",
|
36
|
+
# "less debug output/messages - default is (#{!opts[:debug]})" ) do |debug|
|
37
|
+
# opts[:debug] = false
|
38
|
+
# end
|
39
|
+
parser.on( "--verbose", "--debug",
|
40
|
+
"turn on verbose / debug output (default: #{opts[:debug]})" ) do |debug|
|
41
|
+
opts[:debug] = true
|
42
|
+
end
|
43
|
+
|
44
|
+
parser.on( "-f FILE", "--file FILE",
|
45
|
+
"read datafiles (pathspecs) via .csv file") do |file|
|
46
|
+
opts[:file] = file
|
47
|
+
end
|
48
|
+
|
49
|
+
parser.on( "--[no]-teams",
|
50
|
+
"turn on/off team name checks (default: #{opts[:teams]})") do |teams|
|
51
|
+
opts[:teams] = teams
|
52
|
+
end
|
53
|
+
end
|
54
|
+
parser.parse!( args )
|
55
|
+
|
56
|
+
|
57
|
+
puts "OPTS:"
|
58
|
+
p opts
|
59
|
+
puts "ARGV:"
|
60
|
+
p args
|
61
|
+
|
62
|
+
|
63
|
+
## todo/check - use packs or projects or such
|
64
|
+
## instead of specs - why? why not?
|
65
|
+
specs = []
|
66
|
+
if opts[:file]
|
67
|
+
recs = read_csv( opts[:file] )
|
68
|
+
pp recs
|
69
|
+
## note - make pathspecs relative to passed in file arg!!!
|
70
|
+
basedir = File.dirname( opts[:file] )
|
71
|
+
recs.each do |rec|
|
72
|
+
paths = SportDb::Parser::Opts.find( rec['path'], dir: basedir )
|
73
|
+
specs << [paths, rec]
|
74
|
+
end
|
75
|
+
else
|
76
|
+
paths = if args.empty?
|
77
|
+
[]
|
78
|
+
else
|
79
|
+
## check for directories
|
80
|
+
## and auto-expand
|
81
|
+
SportDb::Parser::Opts.expand_args( args )
|
82
|
+
end
|
83
|
+
specs << [paths, {}]
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
if opts[:debug]
|
88
|
+
SportDb::QuickMatchLinter.debug = true
|
89
|
+
SportDb::QuickMatchReader.debug = true
|
90
|
+
SportDb::MatchParser.debug = true
|
91
|
+
else
|
92
|
+
SportDb::QuickMatchLinter.debug = false
|
93
|
+
SportDb::QuickMatchReader.debug = false
|
94
|
+
SportDb::MatchParser.debug = false
|
95
|
+
LogUtils::Logger.root.level = :info
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
specs.each_with_index do |(paths, rec),i|
|
100
|
+
errors = []
|
101
|
+
paths.each_with_index do |path,j|
|
102
|
+
puts "==> [#{j+1}/#{paths.size}] reading >#{path}<..."
|
103
|
+
quick = SportDb::QuickMatchLinter.new( read_text( path ),
|
104
|
+
check_teams: opts[:teams] )
|
105
|
+
matches = quick.parse
|
106
|
+
|
107
|
+
|
108
|
+
if quick.errors?
|
109
|
+
puts "!! #{quick.errors.size} error(s):"
|
110
|
+
pp quick.errors
|
111
|
+
|
112
|
+
quick.errors.each do |err|
|
113
|
+
errors << [ path, *err ] # note: use splat (*) to add extra values (starting with msg)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
puts " #{matches.size} match(es)"
|
117
|
+
end
|
118
|
+
|
119
|
+
if errors.size > 0
|
120
|
+
puts
|
121
|
+
puts "!! #{errors.size} PARSE ERRORS in #{paths.size} datafile(s)"
|
122
|
+
pp errors
|
123
|
+
else
|
124
|
+
puts
|
125
|
+
puts " OK - no parse errors in #{paths.size} datafile(s)"
|
126
|
+
end
|
127
|
+
|
128
|
+
## add errors to rec via rec['errors'] to allow
|
129
|
+
## for further processing/reporting
|
130
|
+
rec['errors'] = errors
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
|
135
|
+
###
|
136
|
+
## generate a report if --file option used
|
137
|
+
if opts[:file]
|
138
|
+
|
139
|
+
buf = String.new
|
140
|
+
|
141
|
+
buf << "# fbchk summary report - #{specs.size} dataset(s)\n\n"
|
142
|
+
|
143
|
+
specs.each_with_index do |(paths, rec),i|
|
144
|
+
errors = rec['errors']
|
145
|
+
|
146
|
+
if errors.size > 0
|
147
|
+
buf << "!! #{errors.size} ERROR(S) "
|
148
|
+
else
|
149
|
+
buf << " OK "
|
150
|
+
end
|
151
|
+
buf << "%-20s" % rec['path']
|
152
|
+
buf << " - #{paths.size} datafile(s)"
|
153
|
+
buf << "\n"
|
154
|
+
|
155
|
+
if errors.size > 0
|
156
|
+
buf << errors.pretty_inspect
|
157
|
+
buf << "\n"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
puts
|
162
|
+
puts "SUMMARY:"
|
163
|
+
puts buf
|
164
|
+
|
165
|
+
basedir = File.dirname( opts[:file] )
|
166
|
+
basename = File.basename( opts[:file], File.extname( opts[:file] ))
|
167
|
+
outpath = "#{basedir}/fbcheck.#{basename}.txt"
|
168
|
+
write_text( outpath, buf )
|
169
|
+
end
|
170
|
+
|
171
|
+
|
172
|
+
puts "bye"
|
173
|
+
|
@@ -0,0 +1,201 @@
|
|
1
|
+
#####
|
2
|
+
## quick match linter for datafiles with league outlines
|
3
|
+
|
4
|
+
|
5
|
+
module SportDb
|
6
|
+
class QuickMatchLinter
|
7
|
+
|
8
|
+
def self.debug=(value) @@debug = value; end
|
9
|
+
def self.debug?() @@debug ||= false; end ## note: default is FALSE
|
10
|
+
def debug?() self.class.debug?; end
|
11
|
+
|
12
|
+
|
13
|
+
|
14
|
+
def self.read( path ) ## use - rename to read_file or from_file etc. - why? why not?
|
15
|
+
txt = File.open( path, 'r:utf-8' ) {|f| f.read }
|
16
|
+
parse( txt )
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.parse( txt )
|
20
|
+
new( txt ).parse
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
include Logging
|
25
|
+
|
26
|
+
def initialize( txt,
|
27
|
+
check_teams: true )
|
28
|
+
@errors = []
|
29
|
+
@txt = txt
|
30
|
+
@check_teams = check_teams
|
31
|
+
end
|
32
|
+
|
33
|
+
def check_teams?() @check_teams; end
|
34
|
+
|
35
|
+
|
36
|
+
|
37
|
+
attr_reader :errors
|
38
|
+
def errors?() @errors.size > 0; end
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
CLUB_NAME_RE = %r{^
|
43
|
+
(?<name>[^()]+?) ## non-greedy
|
44
|
+
(?:
|
45
|
+
\s+
|
46
|
+
\(
|
47
|
+
(?<code>[A-Z][A-Za-z]{2,3}) ## optional (country) code; support single code e.g. (A) - why? why not?
|
48
|
+
\)
|
49
|
+
)?
|
50
|
+
$}x ## note - allow (URU) and (Uru) - why? why not
|
51
|
+
|
52
|
+
|
53
|
+
def parse
|
54
|
+
## note: every (new) read call - resets errors list to empty
|
55
|
+
@errors = []
|
56
|
+
|
57
|
+
data = {} # return data hash with leagues
|
58
|
+
# and seasons
|
59
|
+
# for now merge stage into matches
|
60
|
+
|
61
|
+
secs = QuickLeagueOutlineReader.parse( @txt )
|
62
|
+
pp secs if debug?
|
63
|
+
|
64
|
+
secs.each do |sec| ## sec(tion)s
|
65
|
+
season = Season.parse( sec[:season] ) ## convert (str) to season obj!!!
|
66
|
+
league = sec[:league]
|
67
|
+
stage = sec[:stage]
|
68
|
+
lines = sec[:lines]
|
69
|
+
|
70
|
+
start = if season.year?
|
71
|
+
Date.new( season.start_year, 1, 1 )
|
72
|
+
else
|
73
|
+
Date.new( season.start_year, 7, 1 )
|
74
|
+
end
|
75
|
+
|
76
|
+
###
|
77
|
+
## db check - check league
|
78
|
+
## fix/fix - add season to match_by !!!!!!!
|
79
|
+
## note - changed to match (incl. code) from match_by(name:) only!!!
|
80
|
+
recs = Import::League.match( league )
|
81
|
+
league_rec = nil
|
82
|
+
if recs.size == 1
|
83
|
+
league_rec = recs[0]
|
84
|
+
puts " OK #{league} => #{league_rec.name}"
|
85
|
+
elsif recs.size == 0
|
86
|
+
msg = "NAME ERROR - no league match found for >#{league}<"
|
87
|
+
@errors << [msg]
|
88
|
+
puts "!! #{msg}"
|
89
|
+
else
|
90
|
+
msg = "NAME ERROR - ambigous; too many league matches (#{recs.size}) found for >#{league}<"
|
91
|
+
@errors << [msg]
|
92
|
+
puts "!! #{msg}"
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
parser = MatchParser.new( lines,
|
97
|
+
start ) ## note: keep season start_at date for now (no need for more specific stage date need for now)
|
98
|
+
|
99
|
+
auto_conf_teams, matches, rounds, groups = parser.parse
|
100
|
+
|
101
|
+
## auto-add "upstream" errors from parser
|
102
|
+
@errors += parser.errors if parser.errors?
|
103
|
+
|
104
|
+
|
105
|
+
if debug?
|
106
|
+
puts ">>> #{auto_conf_teams.size} teams:"
|
107
|
+
pp auto_conf_teams
|
108
|
+
puts ">>> #{matches.size} matches:"
|
109
|
+
## pp matches
|
110
|
+
puts ">>> #{rounds.size} rounds:"
|
111
|
+
pp rounds
|
112
|
+
puts ">>> #{groups.size} groups:"
|
113
|
+
pp groups
|
114
|
+
end
|
115
|
+
|
116
|
+
###
|
117
|
+
## db check - check teams
|
118
|
+
## only clubs for now
|
119
|
+
## fix add better support for champs etc
|
120
|
+
## and national teams!!!
|
121
|
+
if check_teams?
|
122
|
+
auto_conf_teams.each do |team|
|
123
|
+
recs = if league_rec && !league_rec.intl?
|
124
|
+
Import::Club.match_by( name: team, league: league_rec )
|
125
|
+
else
|
126
|
+
###
|
127
|
+
## get country code from name
|
128
|
+
## e.g. Liverpool FC (ENG) or
|
129
|
+
## Liverpool FC (URU) etc.
|
130
|
+
|
131
|
+
## check for country code
|
132
|
+
if m=CLUB_NAME_RE.match( team )
|
133
|
+
if m[:code]
|
134
|
+
Import::Club.match_by( name: m[:name],
|
135
|
+
country: m[:code] )
|
136
|
+
else
|
137
|
+
msg = "PARSE ERROR - country code missing for club name in int'l tournament; may not be unique >#{team}<"
|
138
|
+
@errors << [msg]
|
139
|
+
Import::Club.match_by( name: team )
|
140
|
+
end
|
141
|
+
else
|
142
|
+
msg = "PARSE ERROR - invalid club name; cannot match with CLUB_NAME_RE >#{team}<"
|
143
|
+
@errors << [msg]
|
144
|
+
[]
|
145
|
+
end
|
146
|
+
end
|
147
|
+
team_rec = nil
|
148
|
+
if recs.size == 1
|
149
|
+
team_rec = recs[0]
|
150
|
+
puts " OK #{team} => #{team_rec.name}, #{team_rec.country.name}"
|
151
|
+
elsif recs.size == 0
|
152
|
+
msg = "NAME ERROR - no team match found for >#{team}<"
|
153
|
+
@errors << [msg]
|
154
|
+
puts "!! #{msg}"
|
155
|
+
else
|
156
|
+
msg = "NAME ERROR - ambigous; too many team matches (#{recs.size}) found for >#{team}<"
|
157
|
+
@errors << [msg]
|
158
|
+
puts "!! #{msg}"
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
|
164
|
+
## note: pass along stage (if present): stage - optional from heading!!!!
|
165
|
+
if stage
|
166
|
+
matches.each do |match|
|
167
|
+
match = match.update( stage: stage )
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
data[ league ] ||= {}
|
172
|
+
data[ league ][ season.key ] ||= []
|
173
|
+
|
174
|
+
data[ league ][ season.key ] += matches
|
175
|
+
## note - skip teams, rounds, and groups for now
|
176
|
+
end
|
177
|
+
|
178
|
+
## check - only one league and one season
|
179
|
+
## allowed in quick style
|
180
|
+
|
181
|
+
|
182
|
+
leagues = data.keys
|
183
|
+
if leagues.size != 1
|
184
|
+
puts "!! (QUICK) PARSE ERROR - expected one league only; got #{leagues.size}:"
|
185
|
+
pp leagues
|
186
|
+
exit 1
|
187
|
+
end
|
188
|
+
|
189
|
+
seasons = data[ leagues[0] ].keys
|
190
|
+
if seasons.size != 1
|
191
|
+
puts "!! (QUICK) PARSE ERROR - expected one #{leagues[0]} season only; got #{seasons.size}:"
|
192
|
+
pp seasons
|
193
|
+
exit 1
|
194
|
+
end
|
195
|
+
|
196
|
+
data[ leagues[0] ][ seasons[0] ]
|
197
|
+
end # method parse
|
198
|
+
|
199
|
+
end # class QuickMatchLinter
|
200
|
+
end # module SportDb
|
201
|
+
|