datafile 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{HISTORY.md → CHANGELOG.md} +0 -0
- data/Manifest.txt +23 -30
- data/Rakefile +2 -3
- data/data/football.txt +20 -11
- data/lib/datafile.rb +29 -19
- data/lib/datafile/builder.rb +1 -24
- data/lib/datafile/datafile.rb +25 -144
- data/lib/datafile/{datasets/dataset.rb → dataset.rb} +17 -25
- data/lib/datafile/version.rb +7 -4
- data/lib/datafile/workers/file/dataset.rb +20 -57
- data/lib/datafile/workers/file/registry.rb +18 -18
- data/lib/datafile/workers/file/worker.rb +4 -24
- data/lib/datafile/workers/zip/dataset.rb +19 -8
- data/lib/datafile/workers/zip/worker.rb +6 -22
- data/test/test_builder.rb +9 -9
- data/test/test_file_dataset_registry.rb +2 -3
- data/test/test_file_worker.rb +4 -5
- data/test/test_football_dataset.rb +9 -9
- metadata +10 -37
- data/.gemtest +0 -0
- data/lib/datafile/builder2.rb +0 -90
- data/lib/datafile/workers/dataset.rb +0 -40
- data/lib/datafile/workers/zip/beer.rb +0 -18
- data/lib/datafile/workers/zip/football.rb +0 -18
- data/lib/datafile/workers/zip/world.rb +0 -18
- data/test/datafile2/at.rb +0 -51
- data/test/test_builder2.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e02f51529c40c3573f5bf51670b8e704e518306e
|
4
|
+
data.tar.gz: 309d29e81297fcf2a63b95a1c26f72003aa842b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77bf2afdca97946e52e99bb6f7b2ba96e714b5890a3be11df6f4a3a3b19e43281e30ee78002125928039535b6813c4ec362e1bd8ecf184fa0935e6487d7fa0e4
|
7
|
+
data.tar.gz: 41d975fe70d41f75e46031dd7ce006629375b9703dd9449199e6ee3fd3e6461b71db33618ac57720347a5c9fd99889f42caddf3c354bf72ea16aa05cad1a8180
|
data/{HISTORY.md → CHANGELOG.md}
RENAMED
File without changes
|
data/Manifest.txt
CHANGED
@@ -1,30 +1,23 @@
|
|
1
|
-
|
2
|
-
Manifest.txt
|
3
|
-
README.md
|
4
|
-
Rakefile
|
5
|
-
data/football.txt
|
6
|
-
lib/datafile.rb
|
7
|
-
lib/datafile/builder.rb
|
8
|
-
lib/datafile/
|
9
|
-
lib/datafile/
|
10
|
-
lib/datafile/
|
11
|
-
lib/datafile/
|
12
|
-
lib/datafile/workers/
|
13
|
-
lib/datafile/workers/file/
|
14
|
-
lib/datafile/workers/
|
15
|
-
lib/datafile/workers/
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
test/
|
22
|
-
test/
|
23
|
-
test/
|
24
|
-
test/datafile2/at.rb
|
25
|
-
test/helper.rb
|
26
|
-
test/test_builder.rb
|
27
|
-
test/test_builder2.rb
|
28
|
-
test/test_file_dataset_registry.rb
|
29
|
-
test/test_file_worker.rb
|
30
|
-
test/test_football_dataset.rb
|
1
|
+
CHANGELOG.md
|
2
|
+
Manifest.txt
|
3
|
+
README.md
|
4
|
+
Rakefile
|
5
|
+
data/football.txt
|
6
|
+
lib/datafile.rb
|
7
|
+
lib/datafile/builder.rb
|
8
|
+
lib/datafile/datafile.rb
|
9
|
+
lib/datafile/dataset.rb
|
10
|
+
lib/datafile/version.rb
|
11
|
+
lib/datafile/workers/file/dataset.rb
|
12
|
+
lib/datafile/workers/file/registry.rb
|
13
|
+
lib/datafile/workers/file/worker.rb
|
14
|
+
lib/datafile/workers/zip/dataset.rb
|
15
|
+
lib/datafile/workers/zip/worker.rb
|
16
|
+
test/datafile/eurocup.rb
|
17
|
+
test/datafile/stadiums.rb
|
18
|
+
test/datafile/world.rb
|
19
|
+
test/helper.rb
|
20
|
+
test/test_builder.rb
|
21
|
+
test/test_file_dataset_registry.rb
|
22
|
+
test/test_file_worker.rb
|
23
|
+
test/test_football_dataset.rb
|
data/Rakefile
CHANGED
@@ -15,17 +15,16 @@ Hoe.spec 'datafile' do
|
|
15
15
|
|
16
16
|
# switch extension to .markdown for gihub formatting
|
17
17
|
self.readme_file = 'README.md'
|
18
|
-
self.history_file = '
|
18
|
+
self.history_file = 'CHANGELOG.md'
|
19
19
|
|
20
20
|
self.extra_deps = [
|
21
21
|
['logutils'],
|
22
|
-
['textutils'],
|
23
22
|
['fetcher'],
|
24
23
|
]
|
25
24
|
|
26
25
|
self.licenses = ['Public Domain']
|
27
26
|
|
28
27
|
self.spec_extras = {
|
29
|
-
|
28
|
+
required_ruby_version: '>= 2.2.2'
|
30
29
|
}
|
31
30
|
end
|
data/data/football.txt
CHANGED
@@ -24,22 +24,31 @@ openfootball/confed-cup
|
|
24
24
|
# -- Clubs
|
25
25
|
|
26
26
|
openfootball/clubs
|
27
|
-
|
28
|
-
|
29
|
-
openfootball/
|
30
|
-
openfootball/
|
31
|
-
openfootball/
|
32
|
-
openfootball/
|
33
|
-
openfootball/
|
34
|
-
openfootball/
|
27
|
+
openfootball/world
|
28
|
+
|
29
|
+
openfootball/england
|
30
|
+
openfootball/austria
|
31
|
+
openfootball/deutschland
|
32
|
+
openfootball/espana
|
33
|
+
openfootball/italy
|
34
|
+
openfootball/france
|
35
35
|
openfootball/europe-champions-league
|
36
36
|
|
37
|
-
openfootball/
|
38
|
-
openfootball/mx-mexico
|
37
|
+
openfootball/mexico
|
39
38
|
openfootball/north-america-champions-league
|
40
39
|
|
41
|
-
openfootball/
|
40
|
+
openfootball/brazil
|
42
41
|
openfootball/copa-sudamericana
|
43
42
|
openfootball/copa-libertadores
|
44
43
|
|
45
44
|
openfootball/club-world-cup
|
45
|
+
|
46
|
+
|
47
|
+
|
48
|
+
############
|
49
|
+
# in csv format
|
50
|
+
|
51
|
+
footballcsv/england
|
52
|
+
footballcsv/deutschland
|
53
|
+
|
54
|
+
## todo/fix: add some more ...
|
data/lib/datafile.rb
CHANGED
@@ -1,39 +1,49 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require 'logutils'
|
4
|
+
require 'fetcher' ## add (HTTP/S) download support (of blobs/binaries/zips)
|
4
5
|
|
5
6
|
|
6
7
|
# our own code
|
7
|
-
|
8
8
|
require 'datafile/version' # let it always go first
|
9
9
|
|
10
|
-
require 'datafile/datasets/dataset'
|
11
10
|
|
12
|
-
|
11
|
+
module Datafile
|
12
|
+
|
13
|
+
class Error < StandardError
|
14
|
+
end
|
15
|
+
|
16
|
+
####
|
17
|
+
# todo/check:
|
18
|
+
# rename to DatasetNotFound or similar??
|
19
|
+
# use "common" error class - why? why not?
|
20
|
+
class DatasetNotFoundError < Error
|
21
|
+
attr_reader :message
|
22
|
+
|
23
|
+
def initialize( message )
|
24
|
+
@message = message
|
25
|
+
end
|
26
|
+
|
27
|
+
def to_s
|
28
|
+
"datset not found => #{@message}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end # module Datafile
|
32
|
+
|
33
|
+
|
34
|
+
require 'datafile/dataset'
|
35
|
+
require 'datafile/builder'
|
36
|
+
|
13
37
|
|
14
|
-
require 'datafile/workers/file/dataset'
|
15
38
|
require 'datafile/workers/file/registry'
|
39
|
+
require 'datafile/workers/file/dataset'
|
16
40
|
require 'datafile/workers/file/worker'
|
17
41
|
|
18
42
|
require 'datafile/workers/zip/dataset'
|
19
|
-
require 'datafile/workers/zip/beer'
|
20
|
-
require 'datafile/workers/zip/football'
|
21
|
-
require 'datafile/workers/zip/world'
|
22
43
|
require 'datafile/workers/zip/worker'
|
23
44
|
|
24
45
|
require 'datafile/datafile'
|
25
|
-
require 'datafile/builder'
|
26
|
-
require 'datafile/builder2'
|
27
|
-
|
28
|
-
|
29
|
-
module Datafile
|
30
|
-
def self.data_path
|
31
|
-
"#{root}/data"
|
32
|
-
end
|
33
|
-
end # module Datafile
|
34
|
-
|
35
46
|
|
36
47
|
|
37
48
|
# say hello
|
38
|
-
puts Datafile.banner if defined?($
|
39
|
-
|
49
|
+
puts Datafile.banner if defined?($RUBYCOCO_DEBUG) && $RUBYCOCO_DEBUG
|
data/lib/datafile/builder.rb
CHANGED
@@ -5,7 +5,7 @@ module Datafile
|
|
5
5
|
class Builder ## "simple" builder (one file, one datafile)
|
6
6
|
|
7
7
|
def self.load_file( path )
|
8
|
-
code = File.
|
8
|
+
code = File.open( path, 'r:utf-8' ).read
|
9
9
|
self.load( code )
|
10
10
|
end
|
11
11
|
|
@@ -18,36 +18,14 @@ class Builder ## "simple" builder (one file, one datafile)
|
|
18
18
|
|
19
19
|
include LogUtils::Logging
|
20
20
|
|
21
|
-
## add models shortcuts/constants if exist
|
22
|
-
## todo/check: is there a better way ?
|
23
|
-
## e.g. just include in inline section/block - for example?
|
24
|
-
## check if already included? and check on load to include?
|
25
|
-
## models might not yet be required
|
26
|
-
|
27
|
-
if defined?( WorldDb ) && defined?( WorldDb::Models )
|
28
|
-
include WorldDb::Models
|
29
|
-
else
|
30
|
-
puts "*** sorry; can't include WorldDb::Models; not yet defined (required)"
|
31
|
-
end
|
32
|
-
## todo/check: also include FootballDb::Models and BeerDb::Models ???
|
33
|
-
|
34
|
-
|
35
21
|
def initialize
|
36
22
|
@datafile = Datafile.new
|
37
23
|
end
|
38
24
|
|
39
25
|
attr_reader :datafile
|
40
26
|
|
41
|
-
## "special" datasets
|
42
|
-
|
43
|
-
def inline( &block )
|
44
|
-
logger.info( "[builder] add inline script-block" )
|
45
|
-
@datafile.inlines << Inline.new( block )
|
46
|
-
end
|
47
|
-
|
48
27
|
|
49
28
|
## "classic" standard datasets
|
50
|
-
|
51
29
|
def beer( name, opts={} )
|
52
30
|
logger.info( "[builder] add beer-dataset '#{name}'" )
|
53
31
|
@datafile.datasets << BeerDataset.new( name, opts )
|
@@ -65,4 +43,3 @@ class Builder ## "simple" builder (one file, one datafile)
|
|
65
43
|
|
66
44
|
end # class Builder
|
67
45
|
end # module Datafile
|
68
|
-
|
data/lib/datafile/datafile.rb
CHANGED
@@ -2,52 +2,12 @@
|
|
2
2
|
|
3
3
|
module Datafile
|
4
4
|
|
5
|
-
class Script
|
6
|
-
include LogUtils::Logging
|
7
|
-
|
8
|
-
def initialize( proc )
|
9
|
-
@proc = proc
|
10
|
-
end
|
11
|
-
|
12
|
-
def call
|
13
|
-
logger.info( "[script] calling calc block" )
|
14
|
-
@proc.call
|
15
|
-
end
|
16
|
-
|
17
|
-
def dump
|
18
|
-
puts " script: #{@proc.inspect}"
|
19
|
-
end
|
20
|
-
end ## class Script
|
21
|
-
|
22
|
-
|
23
|
-
### todo/check: use Script for Inline too?? - why, why not???
|
24
|
-
### - use setup/pre/before and post/after or something??
|
25
|
-
## - note: for now always is pre/before
|
26
|
-
|
27
|
-
class Inline
|
28
|
-
include LogUtils::Logging
|
29
|
-
|
30
|
-
def initialize( proc )
|
31
|
-
@proc = proc
|
32
|
-
end
|
33
|
-
|
34
|
-
def call
|
35
|
-
logger.info( "[inline] calling script block" )
|
36
|
-
@proc.call
|
37
|
-
end
|
38
|
-
|
39
|
-
def dump
|
40
|
-
puts " script: #{@proc.inspect}"
|
41
|
-
end
|
42
|
-
end ## class Inline
|
43
|
-
|
44
|
-
|
45
5
|
|
46
6
|
class Datafile
|
47
7
|
|
48
8
|
## convenience method - use like Datafile.load_file()
|
49
9
|
def self.load_file( path='./Datafile' )
|
50
|
-
code = File.
|
10
|
+
code = File.open( path, 'r:utf-8' ).read
|
51
11
|
self.load( code )
|
52
12
|
end
|
53
13
|
|
@@ -68,25 +28,34 @@ class Datafile
|
|
68
28
|
|
69
29
|
include LogUtils::Logging
|
70
30
|
|
31
|
+
|
71
32
|
def initialize( opts={} )
|
72
33
|
@opts = opts
|
73
34
|
@datasets = []
|
74
|
-
|
75
|
-
@
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
35
|
+
|
36
|
+
@worker = if opts[:file]
|
37
|
+
FileWorker.new( self )
|
38
|
+
else ## default to zip worker for now
|
39
|
+
ZipWorker.new( self )
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :datasets
|
44
|
+
|
45
|
+
|
46
|
+
def worker=( value ) # lets you change worker - find a better way - how, why, why not??
|
47
|
+
@worker = if value.is_a?( Class ) ## let's you pass in FileWorker or ZipWorker etc.
|
48
|
+
value.new( self )
|
49
|
+
elsif value.to_sym == :file
|
50
|
+
FileWorker.new( self )
|
51
|
+
elsif value.to_sym == :zip
|
52
|
+
ZipWorker.new( self )
|
53
|
+
else
|
54
|
+
value
|
55
|
+
end
|
88
56
|
end
|
89
57
|
|
58
|
+
|
90
59
|
def guess_file_or_zip_worker ## change/rename to configure_file_or_zip_worker - why? why not??
|
91
60
|
## if opts file or zip exists do NOT change (assume set manually)
|
92
61
|
return if @opts[:file] || @opts[:zip]
|
@@ -99,20 +68,10 @@ class Datafile
|
|
99
68
|
end
|
100
69
|
|
101
70
|
|
102
|
-
attr_reader :datasets
|
103
|
-
attr_reader :scripts ## calc(ulation) scripts (calc blocks)
|
104
|
-
attr_reader :inlines ## inline script blocks -- use before?? run before datasets
|
105
|
-
attr_reader :name
|
106
|
-
attr_reader :deps ## dep(endencies)
|
107
|
-
|
108
|
-
attr_accessor :worker # lets you change worker - find a better way - how, why, why not??
|
109
|
-
|
110
|
-
|
111
71
|
def run
|
112
72
|
logger.info( "[datafile] begin - run" )
|
113
73
|
download # step 1 - download zips for datasets
|
114
74
|
read # step 2 - read in datasets from zips - note: includes running inlines
|
115
|
-
calc # step 3 - run calc(ulations) scripts
|
116
75
|
logger.info( "[datafile] end - run" )
|
117
76
|
end
|
118
77
|
|
@@ -120,7 +79,7 @@ class Datafile
|
|
120
79
|
def download
|
121
80
|
logger.info( "[datafile] dowload" )
|
122
81
|
@worker.download
|
123
|
-
## check: use @worker.download( @datasets) - why, why not?? link worker w/ datafile - why, why not??
|
82
|
+
## check: use @worker.download( @datasets) - why, why not?? link worker w/ datafile - why, why not??
|
124
83
|
end
|
125
84
|
|
126
85
|
def read
|
@@ -128,89 +87,11 @@ class Datafile
|
|
128
87
|
@worker.read
|
129
88
|
end
|
130
89
|
|
131
|
-
def calc
|
132
|
-
logger.info( "[datafile] calc" )
|
133
|
-
@worker.calc
|
134
|
-
end
|
135
|
-
|
136
90
|
def dump
|
137
91
|
## for debugging dump datasets (note: will/might also check if zip exits)
|
138
92
|
logger.info( "[datafile] dump datasets (for debugging)" )
|
139
93
|
@worker.dump
|
140
94
|
end
|
141
95
|
|
142
|
-
|
143
|
-
=begin
|
144
|
-
def download_world ## only dl world datasets (skip all others)
|
145
|
-
logger.info( "[datafile] dowload world datasets" )
|
146
|
-
@datasets.each do |dataset|
|
147
|
-
if dataset.kind_of? WorldDataset
|
148
|
-
dataset.download()
|
149
|
-
else
|
150
|
-
# skip all others
|
151
|
-
end
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
def download_beer ## only dl beer datasets (skip all others)
|
156
|
-
logger.info( "[datafile] dowload beer datasets" )
|
157
|
-
@datasets.each do |dataset|
|
158
|
-
if dataset.kind_of? BeerDataset
|
159
|
-
dataset.download()
|
160
|
-
else
|
161
|
-
# skip all others
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
def download_football ## only dl football datasets (skip all others)
|
167
|
-
logger.info( "[datafile] dowload football datasets" )
|
168
|
-
@datasets.each do |dataset|
|
169
|
-
if dataset.kind_of? FootballDataset
|
170
|
-
dataset.download()
|
171
|
-
else
|
172
|
-
# skip all others
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
176
|
-
=end
|
177
|
-
|
178
|
-
|
179
|
-
=begin
|
180
|
-
def read_world
|
181
|
-
logger.info( "[datafile] read world datasets" )
|
182
|
-
@datasets.each do |dataset|
|
183
|
-
if dataset.kind_of?( WorldDataset )
|
184
|
-
dataset.read()
|
185
|
-
else
|
186
|
-
# skip all others
|
187
|
-
end
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
def read_beer
|
192
|
-
logger.info( "[datafile] read beer datasets" )
|
193
|
-
@datasets.each do |dataset|
|
194
|
-
if dataset.kind_of?( BeerDataset )
|
195
|
-
dataset.read()
|
196
|
-
else
|
197
|
-
# skip all others
|
198
|
-
end
|
199
|
-
end
|
200
|
-
end
|
201
|
-
|
202
|
-
def read_football
|
203
|
-
logger.info( "[datafile] read football datasets" )
|
204
|
-
@datasets.each do |dataset|
|
205
|
-
if dataset.kind_of?( FootballDataset )
|
206
|
-
dataset.read()
|
207
|
-
else
|
208
|
-
# skip all others
|
209
|
-
end
|
210
|
-
end
|
211
|
-
end
|
212
|
-
=end
|
213
|
-
|
214
96
|
end # class Datafile
|
215
97
|
end # module Datafile
|
216
|
-
|