spout 0.8.0.beta1 → 0.8.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -9
- data/README.md +30 -13
- data/lib/spout/actions.rb +9 -6
- data/lib/spout/commands/coverage.rb +75 -0
- data/lib/spout/commands/graphs.rb +94 -146
- data/lib/spout/commands/images.rb +276 -0
- data/lib/spout/helpers/chart_types.rb +17 -6
- data/lib/spout/helpers/subject_loader.rb +71 -0
- data/lib/spout/models/coverage_result.rb +65 -0
- data/lib/spout/tasks/engine.rake +7 -139
- data/lib/spout/templates/gitignore +1 -1
- data/lib/spout/version.rb +1 -1
- metadata +6 -3
- data/lib/spout/commands/json_charts_and_tables.rb +0 -202
@@ -0,0 +1,276 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'rubygems'
|
4
|
+
require 'json'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
require 'spout/helpers/subject_loader'
|
8
|
+
require 'spout/helpers/chart_types'
|
9
|
+
|
10
|
+
module Spout
|
11
|
+
module Commands
|
12
|
+
class Images
|
13
|
+
|
14
|
+
def initialize(types, variable_ids, sizes, standard_version)
|
15
|
+
@variable_files = Dir.glob('variables/**/*.json')
|
16
|
+
@standard_version = standard_version
|
17
|
+
|
18
|
+
@valid_ids = variable_ids
|
19
|
+
|
20
|
+
@number_of_rows = nil
|
21
|
+
|
22
|
+
spout_config = YAML.load_file('.spout.yml')
|
23
|
+
|
24
|
+
@visit = ''
|
25
|
+
|
26
|
+
if spout_config.kind_of?(Hash)
|
27
|
+
@visit = spout_config['visit'].to_s.strip
|
28
|
+
end
|
29
|
+
|
30
|
+
t = Time.now
|
31
|
+
FileUtils.mkpath "graphs/#{@standard_version}"
|
32
|
+
|
33
|
+
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
|
34
|
+
|
35
|
+
@subject_loader.load_subjects_from_csvs!
|
36
|
+
@subjects = @subject_loader.subjects
|
37
|
+
|
38
|
+
compute_images
|
39
|
+
puts "Took #{Time.now - t} seconds."
|
40
|
+
end
|
41
|
+
|
42
|
+
def compute_images
|
43
|
+
|
44
|
+
options_folder = "images/#{@standard_version}"
|
45
|
+
FileUtils.mkpath( options_folder )
|
46
|
+
tmp_options_file = File.join( options_folder, 'options.json' )
|
47
|
+
|
48
|
+
sizes = []
|
49
|
+
|
50
|
+
variable_files_count = @variable_files.count
|
51
|
+
@variable_files.each_with_index do |variable_file, file_index|
|
52
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
53
|
+
next unless json
|
54
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
|
55
|
+
next unless ["numeric", "integer", "choices"].include?(json["type"])
|
56
|
+
variable_name = json['id'].to_s.downcase
|
57
|
+
next unless Spout::Models::Subject.method_defined?(variable_name)
|
58
|
+
|
59
|
+
puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
|
60
|
+
|
61
|
+
filtered_subjects = @subjects.select{ |s| s.send(@visit) != nil }
|
62
|
+
|
63
|
+
File.open(tmp_options_file, "w") do |outfile|
|
64
|
+
chart_json = Spout::Helpers::ChartTypes::chart_histogram(@visit, filtered_subjects, json, variable_name)
|
65
|
+
outfile.puts <<-eos
|
66
|
+
{
|
67
|
+
"credits": {
|
68
|
+
"enabled": false
|
69
|
+
},
|
70
|
+
"chart": {
|
71
|
+
"type": "column"
|
72
|
+
},
|
73
|
+
"title": {
|
74
|
+
"text": ""
|
75
|
+
},
|
76
|
+
"xAxis": {
|
77
|
+
"categories": #{chart_json[:categories].to_json}
|
78
|
+
},
|
79
|
+
"yAxis": {
|
80
|
+
"title": {
|
81
|
+
"text": #{chart_json[:units].to_json}
|
82
|
+
}
|
83
|
+
},
|
84
|
+
"plotOptions": {
|
85
|
+
"column": {
|
86
|
+
"pointPadding": 0.2,
|
87
|
+
"borderWidth": 0,
|
88
|
+
"stacking": #{chart_json[:stacking].to_json}
|
89
|
+
}
|
90
|
+
},
|
91
|
+
"series": #{chart_json[:series].to_json}
|
92
|
+
}
|
93
|
+
eos
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
run_phantom_js("#{json['id']}-lg.png", 600, tmp_options_file) if sizes.size == 0 or sizes.include?('lg')
|
98
|
+
run_phantom_js("#{json['id']}.png", 75, tmp_options_file) if sizes.size == 0 or sizes.include?('sm')
|
99
|
+
end
|
100
|
+
File.delete(tmp_options_file) if File.exists?(tmp_options_file)
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
# def initialize(types, variable_ids, sizes, standard_version)
|
105
|
+
# @standard_version = standard_version
|
106
|
+
# total_index_count = Dir.glob("variables/**/*.json").count
|
107
|
+
|
108
|
+
# last_completed = 0
|
109
|
+
|
110
|
+
# options_folder = "images/#{@standard_version}"
|
111
|
+
# FileUtils.mkpath( options_folder )
|
112
|
+
# tmp_options_file = File.join( options_folder, 'options.json' )
|
113
|
+
|
114
|
+
# Dir.glob("csvs/#{standard_version}/*.csv").each do |csv_file|
|
115
|
+
# puts "Working on: #{csv_file}"
|
116
|
+
# t = Time.now
|
117
|
+
# csv_table = CSV.table(csv_file, encoding: 'iso-8859-1').by_col!
|
118
|
+
# puts "Loaded #{csv_file} in #{Time.now - t} seconds."
|
119
|
+
|
120
|
+
# total_header_count = csv_table.headers.count
|
121
|
+
# csv_table.headers.each_with_index do |header, index|
|
122
|
+
# puts "Column #{ index + 1 } of #{ total_header_count } for #{header} in #{csv_file}"
|
123
|
+
# if variable_file = Dir.glob("variables/**/#{header.downcase}.json", File::FNM_CASEFOLD).first
|
124
|
+
# json = JSON.parse(File.read(variable_file)) rescue json = nil
|
125
|
+
# next unless json
|
126
|
+
# next unless ["choices", "numeric", "integer"].include?(json["type"])
|
127
|
+
# next unless types.size == 0 or types.include?(json['type'])
|
128
|
+
# next unless variable_ids.size == 0 or variable_ids.include?(json['id'].to_s.downcase)
|
129
|
+
|
130
|
+
# basename = File.basename(variable_file).gsub(/\.json$/, '').downcase
|
131
|
+
# col_data = csv_table[header]
|
132
|
+
|
133
|
+
# case json["type"] when "choices"
|
134
|
+
# domain_file = Dir.glob("domains/**/#{json['domain']}.json").first
|
135
|
+
# domain_json = JSON.parse(File.read(domain_file)) rescue domain_json = nil
|
136
|
+
# next unless domain_json
|
137
|
+
|
138
|
+
# create_pie_chart_options_file(col_data, tmp_options_file, domain_json)
|
139
|
+
# when 'numeric', 'integer'
|
140
|
+
# create_line_chart_options_file(col_data, tmp_options_file, json["units"])
|
141
|
+
# else
|
142
|
+
# next
|
143
|
+
# end
|
144
|
+
|
145
|
+
# run_phantom_js("#{basename}-lg.png", 600, tmp_options_file) if sizes.size == 0 or sizes.include?('lg')
|
146
|
+
# run_phantom_js("#{basename}.png", 75, tmp_options_file) if sizes.size == 0 or sizes.include?('sm')
|
147
|
+
# end
|
148
|
+
# end
|
149
|
+
# end
|
150
|
+
# File.delete(tmp_options_file) if File.exists?(tmp_options_file)
|
151
|
+
# end
|
152
|
+
|
153
|
+
# def graph_values(col_data)
|
154
|
+
# categories = []
|
155
|
+
|
156
|
+
# col_data = col_data.select{|v| !['', 'null'].include?(v.to_s.strip.downcase)}.collect(&:to_f)
|
157
|
+
|
158
|
+
# all_integers = false
|
159
|
+
# all_integers = (col_data.count{|i| i.denominator != 1} == 0)
|
160
|
+
|
161
|
+
# minimum = col_data.min || 0
|
162
|
+
# maximum = col_data.max || 100
|
163
|
+
|
164
|
+
# default_max_buckets = 30
|
165
|
+
# max_buckets = all_integers ? [maximum - minimum + 1, default_max_buckets].min : default_max_buckets
|
166
|
+
# bucket_size = (maximum - minimum + 1).to_f / max_buckets
|
167
|
+
|
168
|
+
# (0..(max_buckets-1)).each do |bucket|
|
169
|
+
# val_min = (bucket_size * bucket) + minimum
|
170
|
+
# val_max = bucket_size * (bucket + 1) + minimum
|
171
|
+
# # Greater or equal to val_min, less than val_max
|
172
|
+
# # categories << "'#{val_min} to #{val_max}'"
|
173
|
+
# categories << "#{all_integers || (maximum - minimum) > (default_max_buckets / 2) ? val_min.round : "%0.02f" % val_min}"
|
174
|
+
# end
|
175
|
+
|
176
|
+
# new_values = []
|
177
|
+
# (0..max_buckets-1).each do |bucket|
|
178
|
+
# val_min = (bucket_size * bucket) + minimum
|
179
|
+
# val_max = bucket_size * (bucket + 1) + minimum
|
180
|
+
# # Greater or equal to val_min, less than val_max
|
181
|
+
# new_values << col_data.count{|i| i >= val_min and i < val_max}
|
182
|
+
# end
|
183
|
+
|
184
|
+
# values = []
|
185
|
+
|
186
|
+
# values << { name: '', data: new_values, showInLegend: false }
|
187
|
+
|
188
|
+
# [ values, categories ]
|
189
|
+
# end
|
190
|
+
|
191
|
+
|
192
|
+
# def create_pie_chart_options_file(values, options_file, domain_json)
|
193
|
+
|
194
|
+
# values.select!{|v| !['', 'null'].include?(v.to_s.strip.downcase) }
|
195
|
+
# counts = values.group_by{|a| a}.collect{|k,v| [(domain_json.select{|h| h['value'] == k.to_s}.first['display_name'] rescue (k.to_s == '' ? 'NULL' : k)), v.count]}
|
196
|
+
|
197
|
+
# total_count = counts.collect(&:last).inject(&:+)
|
198
|
+
|
199
|
+
# data = counts.collect{|value, count| [value, (count * 100.0 / total_count)]}
|
200
|
+
|
201
|
+
# File.open(options_file, "w") do |outfile|
|
202
|
+
# outfile.puts <<-eos
|
203
|
+
# {
|
204
|
+
# "title": {
|
205
|
+
# "text": ""
|
206
|
+
# },
|
207
|
+
|
208
|
+
# "credits": {
|
209
|
+
# "enabled": false,
|
210
|
+
# },
|
211
|
+
# "series": [{
|
212
|
+
# "type": "pie",
|
213
|
+
# "name": "",
|
214
|
+
# "data": #{data.to_json}
|
215
|
+
# }]
|
216
|
+
# }
|
217
|
+
# eos
|
218
|
+
# end
|
219
|
+
# end
|
220
|
+
|
221
|
+
|
222
|
+
# def create_line_chart_options_file(values, options_file, units)
|
223
|
+
# ( series, categories ) = graph_values(values)
|
224
|
+
|
225
|
+
# File.open(options_file, "w") do |outfile|
|
226
|
+
# outfile.puts <<-eos
|
227
|
+
# {
|
228
|
+
# "chart": {
|
229
|
+
# "type": "areaspline"
|
230
|
+
# },
|
231
|
+
# "title": {
|
232
|
+
# "text": ""
|
233
|
+
# },
|
234
|
+
# "credits": {
|
235
|
+
# "enabled": false,
|
236
|
+
# },
|
237
|
+
# "xAxis": {
|
238
|
+
# "categories": #{categories.to_json},
|
239
|
+
# "labels": {
|
240
|
+
# "step": #{(categories.size.to_f / 12).ceil}
|
241
|
+
# },
|
242
|
+
# "title": {
|
243
|
+
# "text": #{units.to_json}
|
244
|
+
# }
|
245
|
+
# },
|
246
|
+
# "yAxis": {
|
247
|
+
# "maxPadding": 0,
|
248
|
+
# "minPadding": 0,
|
249
|
+
# "title": {
|
250
|
+
# "text": "Count"
|
251
|
+
# }
|
252
|
+
# },
|
253
|
+
# "series": #{series.to_json}
|
254
|
+
# }
|
255
|
+
# eos
|
256
|
+
# end
|
257
|
+
# end
|
258
|
+
|
259
|
+
def run_phantom_js(png_name, width, tmp_options_file)
|
260
|
+
graph_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
|
261
|
+
directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
|
262
|
+
|
263
|
+
open_command = if RUBY_PLATFORM.match(/mingw/) != nil
|
264
|
+
'phantomjs.exe'
|
265
|
+
else
|
266
|
+
'phantomjs'
|
267
|
+
end
|
268
|
+
|
269
|
+
phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
|
270
|
+
# puts phantomjs_command
|
271
|
+
`#{phantomjs_command}`
|
272
|
+
end
|
273
|
+
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'spout/helpers/array_statistics'
|
1
2
|
require 'spout/helpers/table_formatting'
|
2
3
|
|
3
4
|
module Spout
|
@@ -40,6 +41,16 @@ module Spout
|
|
40
41
|
get_json(json['domain'], 'domain')
|
41
42
|
end
|
42
43
|
|
44
|
+
def self.domain_array(variable_name)
|
45
|
+
variable_file = Dir.glob("variables/**/#{variable_name}.json").first
|
46
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
47
|
+
if json
|
48
|
+
domain_json = get_domain(json)
|
49
|
+
domain_json ? domain_json.collect{|option_hash| [option_hash['display_name'], option_hash['value']]} : []
|
50
|
+
else
|
51
|
+
[]
|
52
|
+
end
|
53
|
+
end
|
43
54
|
|
44
55
|
def self.chart_arbitrary_choices_by_quartile(chart_type, subjects, json, method)
|
45
56
|
# CHART TYPE IS THE QUARTILE VARIABLE
|
@@ -188,7 +199,7 @@ module Spout
|
|
188
199
|
|
189
200
|
def self.chart_arbitrary_choices(chart_type, subjects, json, method)
|
190
201
|
return unless chart_variable_json = get_variable(chart_type)
|
191
|
-
return unless chart_variable_domain =
|
202
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
192
203
|
return unless domain_json = get_domain(json)
|
193
204
|
|
194
205
|
|
@@ -213,7 +224,7 @@ module Spout
|
|
213
224
|
|
214
225
|
def self.chart_arbitrary(chart_type, subjects, json, method, visits)
|
215
226
|
return unless chart_variable_json = get_variable(chart_type)
|
216
|
-
return unless chart_variable_domain =
|
227
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
217
228
|
return chart_arbitrary_by_quartile(chart_type, subjects, json, method, visits) if ['numeric', 'integer'].include?(chart_variable_json['type'])
|
218
229
|
|
219
230
|
return chart_arbitrary_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
|
@@ -248,7 +259,7 @@ module Spout
|
|
248
259
|
|
249
260
|
def self.table_arbitrary(chart_type, subjects, json, method, subtitle = nil)
|
250
261
|
return unless chart_variable_json = get_variable(chart_type)
|
251
|
-
return unless chart_variable_domain =
|
262
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
252
263
|
return table_arbitrary_by_quartile(chart_type, subjects, json, method, subtitle) if ['numeric', 'integer'].include?(chart_variable_json['type'])
|
253
264
|
return table_arbitrary_choices(chart_type, subjects, json, method, subtitle) if json['type'] == 'choices'
|
254
265
|
|
@@ -284,7 +295,7 @@ module Spout
|
|
284
295
|
|
285
296
|
def self.table_arbitrary_choices(chart_type, subjects, json, method, subtitle)
|
286
297
|
return unless chart_variable_json = get_variable(chart_type)
|
287
|
-
return unless chart_variable_domain =
|
298
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
288
299
|
return unless domain_json = get_domain(json)
|
289
300
|
|
290
301
|
headers = [
|
@@ -330,7 +341,7 @@ module Spout
|
|
330
341
|
def self.chart_histogram_choices(chart_type, subjects, json, method)
|
331
342
|
return unless domain_json = get_domain(json)
|
332
343
|
return unless chart_variable_json = get_variable(chart_type)
|
333
|
-
return unless chart_variable_domain =
|
344
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
334
345
|
|
335
346
|
|
336
347
|
title = "#{json['display_name']}"
|
@@ -360,7 +371,7 @@ module Spout
|
|
360
371
|
def self.chart_histogram(chart_type, subjects, json, method)
|
361
372
|
return chart_histogram_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
|
362
373
|
return unless chart_variable_json = get_variable(chart_type)
|
363
|
-
return unless chart_variable_domain =
|
374
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
364
375
|
|
365
376
|
title = "#{json['display_name']}"
|
366
377
|
subtitle = "By Visit"
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'spout/models/subject'
|
2
|
+
|
3
|
+
|
4
|
+
module Spout
|
5
|
+
module Helpers
|
6
|
+
class SubjectLoader
|
7
|
+
attr_accessor :subjects
|
8
|
+
|
9
|
+
def initialize(variable_files, valid_ids, standard_version, number_of_rows, visit)
|
10
|
+
@subjects = []
|
11
|
+
@variable_files = variable_files
|
12
|
+
@valid_ids = valid_ids
|
13
|
+
@standard_version = standard_version
|
14
|
+
@number_of_rows = number_of_rows
|
15
|
+
@visit = visit
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_subjects_from_csvs!
|
19
|
+
load_subjects_from_csvs_part_one!
|
20
|
+
load_subjects_from_csvs_part_two!
|
21
|
+
end
|
22
|
+
|
23
|
+
def load_subjects_from_csvs_part_one!
|
24
|
+
@subjects = []
|
25
|
+
|
26
|
+
csv_files = Dir.glob("csvs/#{@standard_version}/*.csv")
|
27
|
+
csv_files.each_with_index do |csv_file, index|
|
28
|
+
count = 0
|
29
|
+
puts "Parsing: #{csv_file}"
|
30
|
+
CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true, header_converters: lambda { |h| h.to_s.downcase } ) do |line|
|
31
|
+
|
32
|
+
row = line.to_hash
|
33
|
+
count += 1
|
34
|
+
puts "Line: #{count}" if (count % 1000 == 0)
|
35
|
+
@subjects << Spout::Models::Subject.create do |t|
|
36
|
+
t._visit = row[@visit]
|
37
|
+
|
38
|
+
row.each do |key,value|
|
39
|
+
unless t.respond_to?(key)
|
40
|
+
t.class.send(:define_method, "#{key}") { instance_variable_get("@#{key}") }
|
41
|
+
t.class.send(:define_method, "#{key}=") { |value| instance_variable_set("@#{key}", value) }
|
42
|
+
end
|
43
|
+
|
44
|
+
unless value == nil
|
45
|
+
t.send("#{key}=", value)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
# puts "Memory Used: " + (`ps -o rss -p #{$$}`.strip.split.last.to_i / 1024).to_s + " MB" if count % 1000 == 0
|
50
|
+
# break if count >= 1000
|
51
|
+
break if @number_of_rows != nil and count >= @number_of_rows
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def load_subjects_from_csvs_part_two!
|
57
|
+
@variable_files.each do |variable_file|
|
58
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
59
|
+
next unless json
|
60
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
|
61
|
+
next unless ["numeric", "integer"].include?(json["type"])
|
62
|
+
method = json['id'].to_s.downcase
|
63
|
+
next unless Spout::Models::Subject.method_defined?(method)
|
64
|
+
|
65
|
+
@subjects.each{ |s| s.send(method) != nil ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
|
66
|
+
end
|
67
|
+
@subjects
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spout/tests/variable_type_validation'
|
2
|
+
|
3
|
+
module Spout
|
4
|
+
module Models
|
5
|
+
class CoverageResult
|
6
|
+
attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
|
7
|
+
|
8
|
+
def initialize(csv, column, csv_values)
|
9
|
+
load_json(column)
|
10
|
+
load_valid_values
|
11
|
+
|
12
|
+
@csv_values = csv_values
|
13
|
+
@values_test = check_values
|
14
|
+
@variable_type_test = check_variable_type
|
15
|
+
@domain_test = check_domain_specified
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_json(column)
|
19
|
+
file = Dir.glob("variables/**/#{column}.json").first
|
20
|
+
@file_name_test = (file != nil)
|
21
|
+
@json = JSON.parse(File.read(file)) rescue @json = {}
|
22
|
+
@json_id_test = (@json['id'].to_s.downcase == column)
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_valid_values
|
26
|
+
valid_values = []
|
27
|
+
if @json['type'] == 'choices'
|
28
|
+
file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
29
|
+
if json = JSON.parse(File.read(file)) rescue false
|
30
|
+
valid_values = json.collect{|hash| hash['value']}
|
31
|
+
end
|
32
|
+
end
|
33
|
+
@valid_values = valid_values
|
34
|
+
end
|
35
|
+
|
36
|
+
def number_of_errors
|
37
|
+
@file_name_test && @json_id_test && @values_test && @variable_type_test && @domain_test ? 0 : 1
|
38
|
+
end
|
39
|
+
|
40
|
+
def check_values
|
41
|
+
@json['type'] != 'choices' || (@valid_values | @csv_values.compact).size == @valid_values.size
|
42
|
+
end
|
43
|
+
|
44
|
+
def check_variable_type
|
45
|
+
Spout::Tests::VariableTypeValidation::VALID_VARIABLE_TYPES.include?(@json['type'])
|
46
|
+
end
|
47
|
+
|
48
|
+
def check_domain_specified
|
49
|
+
if @json['type'] != 'choices'
|
50
|
+
true
|
51
|
+
else
|
52
|
+
domain_file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
53
|
+
if domain_json = JSON.parse(File.read(domain_file)) rescue false
|
54
|
+
return domain_json.kind_of?(Array)
|
55
|
+
end
|
56
|
+
false
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def errored?
|
61
|
+
error == true
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/spout/tasks/engine.rake
CHANGED
@@ -46,160 +46,28 @@ namespace :spout do
|
|
46
46
|
|
47
47
|
desc 'Match CSV dataset with JSON repository'
|
48
48
|
task :coverage do
|
49
|
-
require 'spout/
|
50
|
-
|
51
|
-
choice_variables = []
|
52
|
-
|
53
|
-
Dir.glob("variables/**/*.json").each do |file|
|
54
|
-
if json = JSON.parse(File.read(file)) rescue false
|
55
|
-
choice_variables << json['id'] if json['type'] == 'choices'
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
all_column_headers = []
|
60
|
-
value_hash = {}
|
61
|
-
csv_names = []
|
62
|
-
|
63
|
-
Dir.glob("csvs/*.csv").each do |csv_file|
|
64
|
-
csv_name = csv_file.split('/').last.to_s
|
65
|
-
csv_names << csv_name
|
66
|
-
puts "\nParsing: #{csv_name}"
|
67
|
-
|
68
|
-
column_headers = []
|
69
|
-
row_count = 0
|
70
|
-
|
71
|
-
CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
|
72
|
-
row = line.to_hash
|
73
|
-
column_headers = row.collect{|key, val| [csv_name, key.to_s.downcase]} if row_count == 0
|
74
|
-
|
75
|
-
print "." if row_count % 100 == 0
|
76
|
-
|
77
|
-
choice_variables.each do |column_name|
|
78
|
-
value_hash[column_name] ||= []
|
79
|
-
value_hash[column_name] = value_hash[column_name] | [row[column_name]] if row[column_name]
|
80
|
-
end
|
81
|
-
|
82
|
-
row_count += 1
|
83
|
-
end
|
84
|
-
|
85
|
-
print "done\n"
|
86
|
-
|
87
|
-
all_column_headers += column_headers
|
88
|
-
end
|
89
|
-
|
90
|
-
@matching_results = []
|
91
|
-
|
92
|
-
all_column_headers.each do |csv, column|
|
93
|
-
scr = SpoutCoverageResult.new(csv, column, value_hash[column])
|
94
|
-
@matching_results << [ csv, column, scr ]
|
95
|
-
end
|
96
|
-
|
97
|
-
@matching_results.sort!{|a,b| [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]}
|
98
|
-
|
99
|
-
@coverage_results = []
|
100
|
-
|
101
|
-
csv_names.each do |csv_name|
|
102
|
-
total_column_count = @matching_results.select{|mr| mr[0] == csv_name}.count
|
103
|
-
mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2].number_of_errors == 0}.count
|
104
|
-
@coverage_results << [ csv_name, total_column_count, mapped_column_count ]
|
105
|
-
end
|
106
|
-
|
107
|
-
coverage_folder = File.join(Dir.pwd, 'coverage')
|
108
|
-
FileUtils.mkpath coverage_folder
|
109
|
-
coverage_file = File.join(coverage_folder, 'index.html')
|
110
|
-
|
111
|
-
print "\nGenerating: index.html\n\n"
|
112
|
-
|
113
|
-
File.open(coverage_file, 'w+') do |file|
|
114
|
-
erb_location = File.join( File.dirname(__FILE__), '../views/index.html.erb' )
|
115
|
-
file.puts ERB.new(File.read(erb_location)).result(binding)
|
116
|
-
end
|
117
|
-
|
118
|
-
open_command = 'open' if RUBY_PLATFORM.match(/darwin/) != nil
|
119
|
-
open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
|
120
|
-
|
121
|
-
system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
|
122
|
-
puts "#{coverage_file}\n\n"
|
49
|
+
require 'spout/commands/coverage'
|
50
|
+
Spout::Commands::Coverage.new(standard_version)
|
123
51
|
end
|
124
52
|
|
125
53
|
desc 'Match CSV dataset with JSON repository'
|
126
|
-
task :
|
127
|
-
require 'spout/commands/
|
54
|
+
task :images do
|
55
|
+
require 'spout/commands/images'
|
128
56
|
types = ENV['types'].to_s.split(',').collect{|t| t.to_s.downcase}
|
129
57
|
variable_ids = ENV['variable_ids'].to_s.split(',').collect{|vid| vid.to_s.downcase}
|
130
58
|
sizes = ENV['sizes'].to_s.split(',').collect{|s| s.to_s.downcase}
|
131
|
-
Spout::Commands::
|
59
|
+
Spout::Commands::Images.new(types, variable_ids, sizes, standard_version)
|
132
60
|
end
|
133
61
|
|
134
62
|
desc 'Generate JSON charts and tables'
|
135
63
|
task :json do
|
136
|
-
require 'spout/commands/
|
64
|
+
require 'spout/commands/graphs'
|
137
65
|
variables = ENV['variables'].to_s.split(',').collect{|s| s.to_s.downcase}
|
138
|
-
Spout::Commands::
|
66
|
+
Spout::Commands::Graphs.new(variables, standard_version)
|
139
67
|
end
|
140
68
|
|
141
69
|
end
|
142
70
|
|
143
|
-
class SpoutCoverageResult
|
144
|
-
attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
|
145
|
-
|
146
|
-
def initialize(csv, column, csv_values)
|
147
|
-
load_json(column)
|
148
|
-
load_valid_values
|
149
|
-
|
150
|
-
@csv_values = csv_values
|
151
|
-
@values_test = check_values
|
152
|
-
@variable_type_test = check_variable_type
|
153
|
-
@domain_test = check_domain_specified
|
154
|
-
end
|
155
|
-
|
156
|
-
def load_json(column)
|
157
|
-
file = Dir.glob("variables/**/#{column}.json").first
|
158
|
-
@file_name_test = (file != nil)
|
159
|
-
@json = JSON.parse(File.read(file)) rescue @json = {}
|
160
|
-
@json_id_test = (@json['id'].to_s.downcase == column)
|
161
|
-
end
|
162
|
-
|
163
|
-
def load_valid_values
|
164
|
-
valid_values = []
|
165
|
-
if @json['type'] == 'choices'
|
166
|
-
file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
167
|
-
if json = JSON.parse(File.read(file)) rescue false
|
168
|
-
valid_values = json.collect{|hash| hash['value']}
|
169
|
-
end
|
170
|
-
end
|
171
|
-
@valid_values = valid_values
|
172
|
-
end
|
173
|
-
|
174
|
-
def number_of_errors
|
175
|
-
@file_name_test && @json_id_test && @values_test && @variable_type_test && @domain_test ? 0 : 1
|
176
|
-
end
|
177
|
-
|
178
|
-
def check_values
|
179
|
-
@json['type'] != 'choices' || (@valid_values | @csv_values.compact).size == @valid_values.size
|
180
|
-
end
|
181
|
-
|
182
|
-
def check_variable_type
|
183
|
-
Spout::Tests::VariableTypeValidation::VALID_VARIABLE_TYPES.include?(@json['type'])
|
184
|
-
end
|
185
|
-
|
186
|
-
def check_domain_specified
|
187
|
-
if @json['type'] != 'choices'
|
188
|
-
true
|
189
|
-
else
|
190
|
-
domain_file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
191
|
-
if domain_json = JSON.parse(File.read(domain_file)) rescue false
|
192
|
-
return domain_json.kind_of?(Array)
|
193
|
-
end
|
194
|
-
false
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
def errored?
|
199
|
-
error == true
|
200
|
-
end
|
201
|
-
end
|
202
|
-
|
203
71
|
def number_with_delimiter(number, delimiter = ",")
|
204
72
|
number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
|
205
73
|
end
|