spout 0.8.0.beta1 → 0.8.0.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -9
- data/README.md +30 -13
- data/lib/spout/actions.rb +9 -6
- data/lib/spout/commands/coverage.rb +75 -0
- data/lib/spout/commands/graphs.rb +94 -146
- data/lib/spout/commands/images.rb +276 -0
- data/lib/spout/helpers/chart_types.rb +17 -6
- data/lib/spout/helpers/subject_loader.rb +71 -0
- data/lib/spout/models/coverage_result.rb +65 -0
- data/lib/spout/tasks/engine.rake +7 -139
- data/lib/spout/templates/gitignore +1 -1
- data/lib/spout/version.rb +1 -1
- metadata +6 -3
- data/lib/spout/commands/json_charts_and_tables.rb +0 -202
@@ -0,0 +1,276 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'rubygems'
|
4
|
+
require 'json'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
require 'spout/helpers/subject_loader'
|
8
|
+
require 'spout/helpers/chart_types'
|
9
|
+
|
10
|
+
module Spout
|
11
|
+
module Commands
|
12
|
+
class Images
|
13
|
+
|
14
|
+
def initialize(types, variable_ids, sizes, standard_version)
|
15
|
+
@variable_files = Dir.glob('variables/**/*.json')
|
16
|
+
@standard_version = standard_version
|
17
|
+
|
18
|
+
@valid_ids = variable_ids
|
19
|
+
|
20
|
+
@number_of_rows = nil
|
21
|
+
|
22
|
+
spout_config = YAML.load_file('.spout.yml')
|
23
|
+
|
24
|
+
@visit = ''
|
25
|
+
|
26
|
+
if spout_config.kind_of?(Hash)
|
27
|
+
@visit = spout_config['visit'].to_s.strip
|
28
|
+
end
|
29
|
+
|
30
|
+
t = Time.now
|
31
|
+
FileUtils.mkpath "graphs/#{@standard_version}"
|
32
|
+
|
33
|
+
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
|
34
|
+
|
35
|
+
@subject_loader.load_subjects_from_csvs!
|
36
|
+
@subjects = @subject_loader.subjects
|
37
|
+
|
38
|
+
compute_images
|
39
|
+
puts "Took #{Time.now - t} seconds."
|
40
|
+
end
|
41
|
+
|
42
|
+
def compute_images
|
43
|
+
|
44
|
+
options_folder = "images/#{@standard_version}"
|
45
|
+
FileUtils.mkpath( options_folder )
|
46
|
+
tmp_options_file = File.join( options_folder, 'options.json' )
|
47
|
+
|
48
|
+
sizes = []
|
49
|
+
|
50
|
+
variable_files_count = @variable_files.count
|
51
|
+
@variable_files.each_with_index do |variable_file, file_index|
|
52
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
53
|
+
next unless json
|
54
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
|
55
|
+
next unless ["numeric", "integer", "choices"].include?(json["type"])
|
56
|
+
variable_name = json['id'].to_s.downcase
|
57
|
+
next unless Spout::Models::Subject.method_defined?(variable_name)
|
58
|
+
|
59
|
+
puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
|
60
|
+
|
61
|
+
filtered_subjects = @subjects.select{ |s| s.send(@visit) != nil }
|
62
|
+
|
63
|
+
File.open(tmp_options_file, "w") do |outfile|
|
64
|
+
chart_json = Spout::Helpers::ChartTypes::chart_histogram(@visit, filtered_subjects, json, variable_name)
|
65
|
+
outfile.puts <<-eos
|
66
|
+
{
|
67
|
+
"credits": {
|
68
|
+
"enabled": false
|
69
|
+
},
|
70
|
+
"chart": {
|
71
|
+
"type": "column"
|
72
|
+
},
|
73
|
+
"title": {
|
74
|
+
"text": ""
|
75
|
+
},
|
76
|
+
"xAxis": {
|
77
|
+
"categories": #{chart_json[:categories].to_json}
|
78
|
+
},
|
79
|
+
"yAxis": {
|
80
|
+
"title": {
|
81
|
+
"text": #{chart_json[:units].to_json}
|
82
|
+
}
|
83
|
+
},
|
84
|
+
"plotOptions": {
|
85
|
+
"column": {
|
86
|
+
"pointPadding": 0.2,
|
87
|
+
"borderWidth": 0,
|
88
|
+
"stacking": #{chart_json[:stacking].to_json}
|
89
|
+
}
|
90
|
+
},
|
91
|
+
"series": #{chart_json[:series].to_json}
|
92
|
+
}
|
93
|
+
eos
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
run_phantom_js("#{json['id']}-lg.png", 600, tmp_options_file) if sizes.size == 0 or sizes.include?('lg')
|
98
|
+
run_phantom_js("#{json['id']}.png", 75, tmp_options_file) if sizes.size == 0 or sizes.include?('sm')
|
99
|
+
end
|
100
|
+
File.delete(tmp_options_file) if File.exists?(tmp_options_file)
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
# def initialize(types, variable_ids, sizes, standard_version)
|
105
|
+
# @standard_version = standard_version
|
106
|
+
# total_index_count = Dir.glob("variables/**/*.json").count
|
107
|
+
|
108
|
+
# last_completed = 0
|
109
|
+
|
110
|
+
# options_folder = "images/#{@standard_version}"
|
111
|
+
# FileUtils.mkpath( options_folder )
|
112
|
+
# tmp_options_file = File.join( options_folder, 'options.json' )
|
113
|
+
|
114
|
+
# Dir.glob("csvs/#{standard_version}/*.csv").each do |csv_file|
|
115
|
+
# puts "Working on: #{csv_file}"
|
116
|
+
# t = Time.now
|
117
|
+
# csv_table = CSV.table(csv_file, encoding: 'iso-8859-1').by_col!
|
118
|
+
# puts "Loaded #{csv_file} in #{Time.now - t} seconds."
|
119
|
+
|
120
|
+
# total_header_count = csv_table.headers.count
|
121
|
+
# csv_table.headers.each_with_index do |header, index|
|
122
|
+
# puts "Column #{ index + 1 } of #{ total_header_count } for #{header} in #{csv_file}"
|
123
|
+
# if variable_file = Dir.glob("variables/**/#{header.downcase}.json", File::FNM_CASEFOLD).first
|
124
|
+
# json = JSON.parse(File.read(variable_file)) rescue json = nil
|
125
|
+
# next unless json
|
126
|
+
# next unless ["choices", "numeric", "integer"].include?(json["type"])
|
127
|
+
# next unless types.size == 0 or types.include?(json['type'])
|
128
|
+
# next unless variable_ids.size == 0 or variable_ids.include?(json['id'].to_s.downcase)
|
129
|
+
|
130
|
+
# basename = File.basename(variable_file).gsub(/\.json$/, '').downcase
|
131
|
+
# col_data = csv_table[header]
|
132
|
+
|
133
|
+
# case json["type"] when "choices"
|
134
|
+
# domain_file = Dir.glob("domains/**/#{json['domain']}.json").first
|
135
|
+
# domain_json = JSON.parse(File.read(domain_file)) rescue domain_json = nil
|
136
|
+
# next unless domain_json
|
137
|
+
|
138
|
+
# create_pie_chart_options_file(col_data, tmp_options_file, domain_json)
|
139
|
+
# when 'numeric', 'integer'
|
140
|
+
# create_line_chart_options_file(col_data, tmp_options_file, json["units"])
|
141
|
+
# else
|
142
|
+
# next
|
143
|
+
# end
|
144
|
+
|
145
|
+
# run_phantom_js("#{basename}-lg.png", 600, tmp_options_file) if sizes.size == 0 or sizes.include?('lg')
|
146
|
+
# run_phantom_js("#{basename}.png", 75, tmp_options_file) if sizes.size == 0 or sizes.include?('sm')
|
147
|
+
# end
|
148
|
+
# end
|
149
|
+
# end
|
150
|
+
# File.delete(tmp_options_file) if File.exists?(tmp_options_file)
|
151
|
+
# end
|
152
|
+
|
153
|
+
# def graph_values(col_data)
|
154
|
+
# categories = []
|
155
|
+
|
156
|
+
# col_data = col_data.select{|v| !['', 'null'].include?(v.to_s.strip.downcase)}.collect(&:to_f)
|
157
|
+
|
158
|
+
# all_integers = false
|
159
|
+
# all_integers = (col_data.count{|i| i.denominator != 1} == 0)
|
160
|
+
|
161
|
+
# minimum = col_data.min || 0
|
162
|
+
# maximum = col_data.max || 100
|
163
|
+
|
164
|
+
# default_max_buckets = 30
|
165
|
+
# max_buckets = all_integers ? [maximum - minimum + 1, default_max_buckets].min : default_max_buckets
|
166
|
+
# bucket_size = (maximum - minimum + 1).to_f / max_buckets
|
167
|
+
|
168
|
+
# (0..(max_buckets-1)).each do |bucket|
|
169
|
+
# val_min = (bucket_size * bucket) + minimum
|
170
|
+
# val_max = bucket_size * (bucket + 1) + minimum
|
171
|
+
# # Greater or equal to val_min, less than val_max
|
172
|
+
# # categories << "'#{val_min} to #{val_max}'"
|
173
|
+
# categories << "#{all_integers || (maximum - minimum) > (default_max_buckets / 2) ? val_min.round : "%0.02f" % val_min}"
|
174
|
+
# end
|
175
|
+
|
176
|
+
# new_values = []
|
177
|
+
# (0..max_buckets-1).each do |bucket|
|
178
|
+
# val_min = (bucket_size * bucket) + minimum
|
179
|
+
# val_max = bucket_size * (bucket + 1) + minimum
|
180
|
+
# # Greater or equal to val_min, less than val_max
|
181
|
+
# new_values << col_data.count{|i| i >= val_min and i < val_max}
|
182
|
+
# end
|
183
|
+
|
184
|
+
# values = []
|
185
|
+
|
186
|
+
# values << { name: '', data: new_values, showInLegend: false }
|
187
|
+
|
188
|
+
# [ values, categories ]
|
189
|
+
# end
|
190
|
+
|
191
|
+
|
192
|
+
# def create_pie_chart_options_file(values, options_file, domain_json)
|
193
|
+
|
194
|
+
# values.select!{|v| !['', 'null'].include?(v.to_s.strip.downcase) }
|
195
|
+
# counts = values.group_by{|a| a}.collect{|k,v| [(domain_json.select{|h| h['value'] == k.to_s}.first['display_name'] rescue (k.to_s == '' ? 'NULL' : k)), v.count]}
|
196
|
+
|
197
|
+
# total_count = counts.collect(&:last).inject(&:+)
|
198
|
+
|
199
|
+
# data = counts.collect{|value, count| [value, (count * 100.0 / total_count)]}
|
200
|
+
|
201
|
+
# File.open(options_file, "w") do |outfile|
|
202
|
+
# outfile.puts <<-eos
|
203
|
+
# {
|
204
|
+
# "title": {
|
205
|
+
# "text": ""
|
206
|
+
# },
|
207
|
+
|
208
|
+
# "credits": {
|
209
|
+
# "enabled": false,
|
210
|
+
# },
|
211
|
+
# "series": [{
|
212
|
+
# "type": "pie",
|
213
|
+
# "name": "",
|
214
|
+
# "data": #{data.to_json}
|
215
|
+
# }]
|
216
|
+
# }
|
217
|
+
# eos
|
218
|
+
# end
|
219
|
+
# end
|
220
|
+
|
221
|
+
|
222
|
+
# def create_line_chart_options_file(values, options_file, units)
|
223
|
+
# ( series, categories ) = graph_values(values)
|
224
|
+
|
225
|
+
# File.open(options_file, "w") do |outfile|
|
226
|
+
# outfile.puts <<-eos
|
227
|
+
# {
|
228
|
+
# "chart": {
|
229
|
+
# "type": "areaspline"
|
230
|
+
# },
|
231
|
+
# "title": {
|
232
|
+
# "text": ""
|
233
|
+
# },
|
234
|
+
# "credits": {
|
235
|
+
# "enabled": false,
|
236
|
+
# },
|
237
|
+
# "xAxis": {
|
238
|
+
# "categories": #{categories.to_json},
|
239
|
+
# "labels": {
|
240
|
+
# "step": #{(categories.size.to_f / 12).ceil}
|
241
|
+
# },
|
242
|
+
# "title": {
|
243
|
+
# "text": #{units.to_json}
|
244
|
+
# }
|
245
|
+
# },
|
246
|
+
# "yAxis": {
|
247
|
+
# "maxPadding": 0,
|
248
|
+
# "minPadding": 0,
|
249
|
+
# "title": {
|
250
|
+
# "text": "Count"
|
251
|
+
# }
|
252
|
+
# },
|
253
|
+
# "series": #{series.to_json}
|
254
|
+
# }
|
255
|
+
# eos
|
256
|
+
# end
|
257
|
+
# end
|
258
|
+
|
259
|
+
def run_phantom_js(png_name, width, tmp_options_file)
|
260
|
+
graph_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
|
261
|
+
directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
|
262
|
+
|
263
|
+
open_command = if RUBY_PLATFORM.match(/mingw/) != nil
|
264
|
+
'phantomjs.exe'
|
265
|
+
else
|
266
|
+
'phantomjs'
|
267
|
+
end
|
268
|
+
|
269
|
+
phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
|
270
|
+
# puts phantomjs_command
|
271
|
+
`#{phantomjs_command}`
|
272
|
+
end
|
273
|
+
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'spout/helpers/array_statistics'
|
1
2
|
require 'spout/helpers/table_formatting'
|
2
3
|
|
3
4
|
module Spout
|
@@ -40,6 +41,16 @@ module Spout
|
|
40
41
|
get_json(json['domain'], 'domain')
|
41
42
|
end
|
42
43
|
|
44
|
+
def self.domain_array(variable_name)
|
45
|
+
variable_file = Dir.glob("variables/**/#{variable_name}.json").first
|
46
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
47
|
+
if json
|
48
|
+
domain_json = get_domain(json)
|
49
|
+
domain_json ? domain_json.collect{|option_hash| [option_hash['display_name'], option_hash['value']]} : []
|
50
|
+
else
|
51
|
+
[]
|
52
|
+
end
|
53
|
+
end
|
43
54
|
|
44
55
|
def self.chart_arbitrary_choices_by_quartile(chart_type, subjects, json, method)
|
45
56
|
# CHART TYPE IS THE QUARTILE VARIABLE
|
@@ -188,7 +199,7 @@ module Spout
|
|
188
199
|
|
189
200
|
def self.chart_arbitrary_choices(chart_type, subjects, json, method)
|
190
201
|
return unless chart_variable_json = get_variable(chart_type)
|
191
|
-
return unless chart_variable_domain =
|
202
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
192
203
|
return unless domain_json = get_domain(json)
|
193
204
|
|
194
205
|
|
@@ -213,7 +224,7 @@ module Spout
|
|
213
224
|
|
214
225
|
def self.chart_arbitrary(chart_type, subjects, json, method, visits)
|
215
226
|
return unless chart_variable_json = get_variable(chart_type)
|
216
|
-
return unless chart_variable_domain =
|
227
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
217
228
|
return chart_arbitrary_by_quartile(chart_type, subjects, json, method, visits) if ['numeric', 'integer'].include?(chart_variable_json['type'])
|
218
229
|
|
219
230
|
return chart_arbitrary_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
|
@@ -248,7 +259,7 @@ module Spout
|
|
248
259
|
|
249
260
|
def self.table_arbitrary(chart_type, subjects, json, method, subtitle = nil)
|
250
261
|
return unless chart_variable_json = get_variable(chart_type)
|
251
|
-
return unless chart_variable_domain =
|
262
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
252
263
|
return table_arbitrary_by_quartile(chart_type, subjects, json, method, subtitle) if ['numeric', 'integer'].include?(chart_variable_json['type'])
|
253
264
|
return table_arbitrary_choices(chart_type, subjects, json, method, subtitle) if json['type'] == 'choices'
|
254
265
|
|
@@ -284,7 +295,7 @@ module Spout
|
|
284
295
|
|
285
296
|
def self.table_arbitrary_choices(chart_type, subjects, json, method, subtitle)
|
286
297
|
return unless chart_variable_json = get_variable(chart_type)
|
287
|
-
return unless chart_variable_domain =
|
298
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
288
299
|
return unless domain_json = get_domain(json)
|
289
300
|
|
290
301
|
headers = [
|
@@ -330,7 +341,7 @@ module Spout
|
|
330
341
|
def self.chart_histogram_choices(chart_type, subjects, json, method)
|
331
342
|
return unless domain_json = get_domain(json)
|
332
343
|
return unless chart_variable_json = get_variable(chart_type)
|
333
|
-
return unless chart_variable_domain =
|
344
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
334
345
|
|
335
346
|
|
336
347
|
title = "#{json['display_name']}"
|
@@ -360,7 +371,7 @@ module Spout
|
|
360
371
|
def self.chart_histogram(chart_type, subjects, json, method)
|
361
372
|
return chart_histogram_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
|
362
373
|
return unless chart_variable_json = get_variable(chart_type)
|
363
|
-
return unless chart_variable_domain =
|
374
|
+
return unless chart_variable_domain = domain_array(chart_type)
|
364
375
|
|
365
376
|
title = "#{json['display_name']}"
|
366
377
|
subtitle = "By Visit"
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'spout/models/subject'
|
2
|
+
|
3
|
+
|
4
|
+
module Spout
|
5
|
+
module Helpers
|
6
|
+
class SubjectLoader
|
7
|
+
attr_accessor :subjects
|
8
|
+
|
9
|
+
def initialize(variable_files, valid_ids, standard_version, number_of_rows, visit)
|
10
|
+
@subjects = []
|
11
|
+
@variable_files = variable_files
|
12
|
+
@valid_ids = valid_ids
|
13
|
+
@standard_version = standard_version
|
14
|
+
@number_of_rows = number_of_rows
|
15
|
+
@visit = visit
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_subjects_from_csvs!
|
19
|
+
load_subjects_from_csvs_part_one!
|
20
|
+
load_subjects_from_csvs_part_two!
|
21
|
+
end
|
22
|
+
|
23
|
+
def load_subjects_from_csvs_part_one!
|
24
|
+
@subjects = []
|
25
|
+
|
26
|
+
csv_files = Dir.glob("csvs/#{@standard_version}/*.csv")
|
27
|
+
csv_files.each_with_index do |csv_file, index|
|
28
|
+
count = 0
|
29
|
+
puts "Parsing: #{csv_file}"
|
30
|
+
CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true, header_converters: lambda { |h| h.to_s.downcase } ) do |line|
|
31
|
+
|
32
|
+
row = line.to_hash
|
33
|
+
count += 1
|
34
|
+
puts "Line: #{count}" if (count % 1000 == 0)
|
35
|
+
@subjects << Spout::Models::Subject.create do |t|
|
36
|
+
t._visit = row[@visit]
|
37
|
+
|
38
|
+
row.each do |key,value|
|
39
|
+
unless t.respond_to?(key)
|
40
|
+
t.class.send(:define_method, "#{key}") { instance_variable_get("@#{key}") }
|
41
|
+
t.class.send(:define_method, "#{key}=") { |value| instance_variable_set("@#{key}", value) }
|
42
|
+
end
|
43
|
+
|
44
|
+
unless value == nil
|
45
|
+
t.send("#{key}=", value)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
# puts "Memory Used: " + (`ps -o rss -p #{$$}`.strip.split.last.to_i / 1024).to_s + " MB" if count % 1000 == 0
|
50
|
+
# break if count >= 1000
|
51
|
+
break if @number_of_rows != nil and count >= @number_of_rows
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def load_subjects_from_csvs_part_two!
|
57
|
+
@variable_files.each do |variable_file|
|
58
|
+
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
59
|
+
next unless json
|
60
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
|
61
|
+
next unless ["numeric", "integer"].include?(json["type"])
|
62
|
+
method = json['id'].to_s.downcase
|
63
|
+
next unless Spout::Models::Subject.method_defined?(method)
|
64
|
+
|
65
|
+
@subjects.each{ |s| s.send(method) != nil ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
|
66
|
+
end
|
67
|
+
@subjects
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spout/tests/variable_type_validation'
|
2
|
+
|
3
|
+
module Spout
|
4
|
+
module Models
|
5
|
+
class CoverageResult
|
6
|
+
attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
|
7
|
+
|
8
|
+
def initialize(csv, column, csv_values)
|
9
|
+
load_json(column)
|
10
|
+
load_valid_values
|
11
|
+
|
12
|
+
@csv_values = csv_values
|
13
|
+
@values_test = check_values
|
14
|
+
@variable_type_test = check_variable_type
|
15
|
+
@domain_test = check_domain_specified
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_json(column)
|
19
|
+
file = Dir.glob("variables/**/#{column}.json").first
|
20
|
+
@file_name_test = (file != nil)
|
21
|
+
@json = JSON.parse(File.read(file)) rescue @json = {}
|
22
|
+
@json_id_test = (@json['id'].to_s.downcase == column)
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_valid_values
|
26
|
+
valid_values = []
|
27
|
+
if @json['type'] == 'choices'
|
28
|
+
file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
29
|
+
if json = JSON.parse(File.read(file)) rescue false
|
30
|
+
valid_values = json.collect{|hash| hash['value']}
|
31
|
+
end
|
32
|
+
end
|
33
|
+
@valid_values = valid_values
|
34
|
+
end
|
35
|
+
|
36
|
+
def number_of_errors
|
37
|
+
@file_name_test && @json_id_test && @values_test && @variable_type_test && @domain_test ? 0 : 1
|
38
|
+
end
|
39
|
+
|
40
|
+
def check_values
|
41
|
+
@json['type'] != 'choices' || (@valid_values | @csv_values.compact).size == @valid_values.size
|
42
|
+
end
|
43
|
+
|
44
|
+
def check_variable_type
|
45
|
+
Spout::Tests::VariableTypeValidation::VALID_VARIABLE_TYPES.include?(@json['type'])
|
46
|
+
end
|
47
|
+
|
48
|
+
def check_domain_specified
|
49
|
+
if @json['type'] != 'choices'
|
50
|
+
true
|
51
|
+
else
|
52
|
+
domain_file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
53
|
+
if domain_json = JSON.parse(File.read(domain_file)) rescue false
|
54
|
+
return domain_json.kind_of?(Array)
|
55
|
+
end
|
56
|
+
false
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def errored?
|
61
|
+
error == true
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/spout/tasks/engine.rake
CHANGED
@@ -46,160 +46,28 @@ namespace :spout do
|
|
46
46
|
|
47
47
|
desc 'Match CSV dataset with JSON repository'
|
48
48
|
task :coverage do
|
49
|
-
require 'spout/
|
50
|
-
|
51
|
-
choice_variables = []
|
52
|
-
|
53
|
-
Dir.glob("variables/**/*.json").each do |file|
|
54
|
-
if json = JSON.parse(File.read(file)) rescue false
|
55
|
-
choice_variables << json['id'] if json['type'] == 'choices'
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
all_column_headers = []
|
60
|
-
value_hash = {}
|
61
|
-
csv_names = []
|
62
|
-
|
63
|
-
Dir.glob("csvs/*.csv").each do |csv_file|
|
64
|
-
csv_name = csv_file.split('/').last.to_s
|
65
|
-
csv_names << csv_name
|
66
|
-
puts "\nParsing: #{csv_name}"
|
67
|
-
|
68
|
-
column_headers = []
|
69
|
-
row_count = 0
|
70
|
-
|
71
|
-
CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
|
72
|
-
row = line.to_hash
|
73
|
-
column_headers = row.collect{|key, val| [csv_name, key.to_s.downcase]} if row_count == 0
|
74
|
-
|
75
|
-
print "." if row_count % 100 == 0
|
76
|
-
|
77
|
-
choice_variables.each do |column_name|
|
78
|
-
value_hash[column_name] ||= []
|
79
|
-
value_hash[column_name] = value_hash[column_name] | [row[column_name]] if row[column_name]
|
80
|
-
end
|
81
|
-
|
82
|
-
row_count += 1
|
83
|
-
end
|
84
|
-
|
85
|
-
print "done\n"
|
86
|
-
|
87
|
-
all_column_headers += column_headers
|
88
|
-
end
|
89
|
-
|
90
|
-
@matching_results = []
|
91
|
-
|
92
|
-
all_column_headers.each do |csv, column|
|
93
|
-
scr = SpoutCoverageResult.new(csv, column, value_hash[column])
|
94
|
-
@matching_results << [ csv, column, scr ]
|
95
|
-
end
|
96
|
-
|
97
|
-
@matching_results.sort!{|a,b| [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]}
|
98
|
-
|
99
|
-
@coverage_results = []
|
100
|
-
|
101
|
-
csv_names.each do |csv_name|
|
102
|
-
total_column_count = @matching_results.select{|mr| mr[0] == csv_name}.count
|
103
|
-
mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2].number_of_errors == 0}.count
|
104
|
-
@coverage_results << [ csv_name, total_column_count, mapped_column_count ]
|
105
|
-
end
|
106
|
-
|
107
|
-
coverage_folder = File.join(Dir.pwd, 'coverage')
|
108
|
-
FileUtils.mkpath coverage_folder
|
109
|
-
coverage_file = File.join(coverage_folder, 'index.html')
|
110
|
-
|
111
|
-
print "\nGenerating: index.html\n\n"
|
112
|
-
|
113
|
-
File.open(coverage_file, 'w+') do |file|
|
114
|
-
erb_location = File.join( File.dirname(__FILE__), '../views/index.html.erb' )
|
115
|
-
file.puts ERB.new(File.read(erb_location)).result(binding)
|
116
|
-
end
|
117
|
-
|
118
|
-
open_command = 'open' if RUBY_PLATFORM.match(/darwin/) != nil
|
119
|
-
open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
|
120
|
-
|
121
|
-
system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
|
122
|
-
puts "#{coverage_file}\n\n"
|
49
|
+
require 'spout/commands/coverage'
|
50
|
+
Spout::Commands::Coverage.new(standard_version)
|
123
51
|
end
|
124
52
|
|
125
53
|
desc 'Match CSV dataset with JSON repository'
|
126
|
-
task :
|
127
|
-
require 'spout/commands/
|
54
|
+
task :images do
|
55
|
+
require 'spout/commands/images'
|
128
56
|
types = ENV['types'].to_s.split(',').collect{|t| t.to_s.downcase}
|
129
57
|
variable_ids = ENV['variable_ids'].to_s.split(',').collect{|vid| vid.to_s.downcase}
|
130
58
|
sizes = ENV['sizes'].to_s.split(',').collect{|s| s.to_s.downcase}
|
131
|
-
Spout::Commands::
|
59
|
+
Spout::Commands::Images.new(types, variable_ids, sizes, standard_version)
|
132
60
|
end
|
133
61
|
|
134
62
|
desc 'Generate JSON charts and tables'
|
135
63
|
task :json do
|
136
|
-
require 'spout/commands/
|
64
|
+
require 'spout/commands/graphs'
|
137
65
|
variables = ENV['variables'].to_s.split(',').collect{|s| s.to_s.downcase}
|
138
|
-
Spout::Commands::
|
66
|
+
Spout::Commands::Graphs.new(variables, standard_version)
|
139
67
|
end
|
140
68
|
|
141
69
|
end
|
142
70
|
|
143
|
-
class SpoutCoverageResult
|
144
|
-
attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
|
145
|
-
|
146
|
-
def initialize(csv, column, csv_values)
|
147
|
-
load_json(column)
|
148
|
-
load_valid_values
|
149
|
-
|
150
|
-
@csv_values = csv_values
|
151
|
-
@values_test = check_values
|
152
|
-
@variable_type_test = check_variable_type
|
153
|
-
@domain_test = check_domain_specified
|
154
|
-
end
|
155
|
-
|
156
|
-
def load_json(column)
|
157
|
-
file = Dir.glob("variables/**/#{column}.json").first
|
158
|
-
@file_name_test = (file != nil)
|
159
|
-
@json = JSON.parse(File.read(file)) rescue @json = {}
|
160
|
-
@json_id_test = (@json['id'].to_s.downcase == column)
|
161
|
-
end
|
162
|
-
|
163
|
-
def load_valid_values
|
164
|
-
valid_values = []
|
165
|
-
if @json['type'] == 'choices'
|
166
|
-
file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
167
|
-
if json = JSON.parse(File.read(file)) rescue false
|
168
|
-
valid_values = json.collect{|hash| hash['value']}
|
169
|
-
end
|
170
|
-
end
|
171
|
-
@valid_values = valid_values
|
172
|
-
end
|
173
|
-
|
174
|
-
def number_of_errors
|
175
|
-
@file_name_test && @json_id_test && @values_test && @variable_type_test && @domain_test ? 0 : 1
|
176
|
-
end
|
177
|
-
|
178
|
-
def check_values
|
179
|
-
@json['type'] != 'choices' || (@valid_values | @csv_values.compact).size == @valid_values.size
|
180
|
-
end
|
181
|
-
|
182
|
-
def check_variable_type
|
183
|
-
Spout::Tests::VariableTypeValidation::VALID_VARIABLE_TYPES.include?(@json['type'])
|
184
|
-
end
|
185
|
-
|
186
|
-
def check_domain_specified
|
187
|
-
if @json['type'] != 'choices'
|
188
|
-
true
|
189
|
-
else
|
190
|
-
domain_file = Dir.glob("domains/**/#{@json['domain']}.json").first
|
191
|
-
if domain_json = JSON.parse(File.read(domain_file)) rescue false
|
192
|
-
return domain_json.kind_of?(Array)
|
193
|
-
end
|
194
|
-
false
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
def errored?
|
199
|
-
error == true
|
200
|
-
end
|
201
|
-
end
|
202
|
-
|
203
71
|
def number_with_delimiter(number, delimiter = ",")
|
204
72
|
number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
|
205
73
|
end
|