spout 0.10.0.beta7 → 0.10.0.beta8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/spout/commands/deploy.rb +61 -11
- data/lib/spout/commands/exporter.rb +3 -3
- data/lib/spout/commands/graphs.rb +31 -5
- data/lib/spout/commands/images.rb +44 -9
- data/lib/spout/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 97f132b532821e57396f65c1752060dac1a33c14
|
4
|
+
data.tar.gz: f87fc93a7f9a60c34e81f602c47a770bbd29eba9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7c9ed942121864d02d02bdcdac9115e8b04093e368f31dfd9a4e37541726f5ff803654a93b6d62c872c314efb40f4757bf759f6d1098281dc91f090bbd201229
|
7
|
+
data.tar.gz: 741c7cfd6b2f365aff937f8d2508a04aaaef81fad297cca5773f3c2a82e1759cda0aba69a68d6e72a6d6547a39151f8915d64ac7e0d688ee5f605dd5a39fe216
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'colorize'
|
2
2
|
require 'net/http'
|
3
|
+
require 'io/console'
|
3
4
|
|
4
5
|
require 'spout/helpers/config_reader'
|
5
6
|
require 'spout/helpers/quietly'
|
@@ -48,13 +49,20 @@ module Spout
|
|
48
49
|
def initialize(argv, version)
|
49
50
|
@environment = argv[1].to_s
|
50
51
|
@version = version
|
51
|
-
@skip_checks = (argv.delete('--skip-checks') != nil)
|
52
|
+
@skip_checks = (argv.delete('--skip-checks') != nil or argv.delete('--no-checks') != nil)
|
52
53
|
|
53
|
-
@skip_graphs = (argv.delete('--skip-graphs') != nil)
|
54
|
-
@skip_images = (argv.delete('--skip-images') != nil)
|
55
|
-
@
|
54
|
+
@skip_graphs = (argv.delete('--skip-graphs') != nil or argv.delete('--no-graphs') != nil)
|
55
|
+
@skip_images = (argv.delete('--skip-images') != nil or argv.delete('--no-images') != nil)
|
56
|
+
@clean = (argv.delete('--clean') != nil or argv.delete('--no-resume'))
|
57
|
+
@skip_server_updates = (argv.delete('--skip-server-updates') != nil or argv.delete('--no-server-updates') != nil)
|
56
58
|
|
57
|
-
|
59
|
+
@token = argv.select{|a| /^--token=/ =~ a}.collect{|a| a.gsub(/^--token=/, '')}.first
|
60
|
+
|
61
|
+
begin
|
62
|
+
run_all
|
63
|
+
rescue Interrupt, IRB::Abort
|
64
|
+
puts "\nINTERRUPTED".colorize(:red)
|
65
|
+
end
|
58
66
|
end
|
59
67
|
|
60
68
|
def run_all
|
@@ -66,6 +74,7 @@ module Spout
|
|
66
74
|
graph_generation unless @skip_graphs
|
67
75
|
image_generation unless @skip_images
|
68
76
|
dataset_uploads
|
77
|
+
data_dictionary_uploads
|
69
78
|
trigger_server_updates unless @skip_server_updates
|
70
79
|
rescue DeployError
|
71
80
|
end
|
@@ -170,7 +179,18 @@ module Spout
|
|
170
179
|
def user_authorization
|
171
180
|
puts " Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
|
172
181
|
print " Enter your token: "
|
173
|
-
@token =
|
182
|
+
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ''
|
183
|
+
|
184
|
+
response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
|
185
|
+
|
186
|
+
if response.kind_of?(Hash) and response['editor']
|
187
|
+
puts "AUTHORIZED".colorize(:green)
|
188
|
+
else
|
189
|
+
puts "UNAUTHORIZED".colorize(:red)
|
190
|
+
puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
|
191
|
+
raise DeployError
|
192
|
+
end
|
193
|
+
|
174
194
|
# failure ''
|
175
195
|
# puts "PASS".colorize(:green)
|
176
196
|
end
|
@@ -178,24 +198,31 @@ module Spout
|
|
178
198
|
def graph_generation
|
179
199
|
# failure ''
|
180
200
|
require 'spout/commands/graphs'
|
181
|
-
|
201
|
+
argv = []
|
202
|
+
argv << "--clean" if @clean
|
203
|
+
Spout::Commands::Graphs.new(argv, @version, true, @url, @slug, @token)
|
182
204
|
puts "\r Graph Generation: " + "DONE ".colorize(:green)
|
183
205
|
end
|
184
206
|
|
185
207
|
def image_generation
|
186
208
|
# failure ''
|
187
209
|
require 'spout/commands/images'
|
188
|
-
|
210
|
+
argv = []
|
211
|
+
argv << "--clean" if @clean
|
212
|
+
Spout::Commands::Images.new(argv, [], [], @version, [], true, @url, @slug, @token)
|
189
213
|
puts "\r Image Generation: " + "DONE ".colorize(:green)
|
190
214
|
end
|
191
215
|
|
192
216
|
def dataset_uploads
|
193
217
|
available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort : [])
|
194
|
-
|
195
218
|
semantic = Spout::Helpers::Semantic.new(@version, available_folders)
|
196
|
-
|
197
219
|
csv_directory = semantic.selected_folder
|
198
220
|
|
221
|
+
if @version != csv_directory
|
222
|
+
puts "\r Dataset Uploads: " + "SKIPPED - #{csv_directory} CSV dataset already on server".colorize(:blue)
|
223
|
+
return
|
224
|
+
end
|
225
|
+
|
199
226
|
csv_files = Dir.glob("csvs/#{csv_directory}/*.csv")
|
200
227
|
|
201
228
|
csv_files.each_with_index do |csv_file, index|
|
@@ -205,13 +232,36 @@ module Spout
|
|
205
232
|
puts "\r Dataset Uploads: " + "DONE ".colorize(:green)
|
206
233
|
end
|
207
234
|
|
235
|
+
def data_dictionary_uploads
|
236
|
+
print " Dictionary Uploads:"
|
237
|
+
|
238
|
+
require 'spout/commands/exporter'
|
239
|
+
Spout::Commands::Exporter.new(@version, ['--quiet'])
|
240
|
+
|
241
|
+
csv_files = Dir.glob("dd/#{@version}/*.csv")
|
242
|
+
csv_files.each_with_index do |csv_file, index|
|
243
|
+
print "\r Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
|
244
|
+
response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_dataset_csv.json", csv_file, @version, @token)
|
245
|
+
end
|
246
|
+
puts "\r Dictionary Uploads: " + "DONE ".colorize(:green)
|
247
|
+
end
|
248
|
+
|
208
249
|
def trigger_server_updates
|
209
250
|
print "Launch Server Scripts: "
|
210
251
|
response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/refresh_dictionary.json?version=#{@version}")
|
211
|
-
if response and response['refresh'] == 'success'
|
252
|
+
if response.kind_of?(Hash) and response['refresh'] == 'success'
|
212
253
|
puts "DONE".colorize(:green)
|
254
|
+
elsif response.kind_of?(Hash) and response['refresh'] == 'notagfound'
|
255
|
+
puts "FAIL".colorize(:red)
|
256
|
+
puts "#{INDENT}Tag not found in repository, resolve using: " + "git push --tags".colorize(:white)
|
257
|
+
raise DeployError
|
258
|
+
elsif response.kind_of?(Hash) and response['refresh'] == 'gitrepodoesnotexist'
|
259
|
+
puts "FAIL".colorize(:red)
|
260
|
+
puts "#{INDENT}Dataset data dictionary git repository has not been cloned on the server. Contact server admin.".colorize(:white)
|
261
|
+
raise DeployError
|
213
262
|
else
|
214
263
|
puts "FAIL".colorize(:red)
|
264
|
+
raise DeployError
|
215
265
|
end
|
216
266
|
end
|
217
267
|
|
@@ -9,7 +9,7 @@ module Spout
|
|
9
9
|
module Commands
|
10
10
|
class Exporter
|
11
11
|
def initialize(standard_version, argv)
|
12
|
-
@
|
12
|
+
@quiet = (argv.delete('--quiet') != nil)
|
13
13
|
@standard_version = standard_version
|
14
14
|
@config = Spout::Helpers::ConfigReader.new
|
15
15
|
expanded_export!
|
@@ -19,7 +19,7 @@ module Spout
|
|
19
19
|
|
20
20
|
def expanded_export!
|
21
21
|
folder = "dd/#{@standard_version}"
|
22
|
-
puts " create".colorize( :green ) + " #{folder}"
|
22
|
+
puts " create".colorize( :green ) + " #{folder}" unless @quiet
|
23
23
|
FileUtils.mkpath folder
|
24
24
|
|
25
25
|
generic_export(folder, 'variables', %w(id display_name description type units domain labels calculation))
|
@@ -29,7 +29,7 @@ module Spout
|
|
29
29
|
|
30
30
|
def generic_export(folder, type, keys, include_domain_name = false)
|
31
31
|
export_file = export_file_name(type)
|
32
|
-
puts " export".colorize( :blue ) + " #{folder}/#{export_file}"
|
32
|
+
puts " export".colorize( :blue ) + " #{folder}/#{export_file}" unless @quiet
|
33
33
|
CSV.open("#{folder}/#{export_file}", "wb") do |csv|
|
34
34
|
if include_domain_name
|
35
35
|
csv << ['folder', 'domain_id'] + keys
|
@@ -20,6 +20,10 @@ module Spout
|
|
20
20
|
@slug = slug
|
21
21
|
@token = token
|
22
22
|
|
23
|
+
argv = variables
|
24
|
+
|
25
|
+
@clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
|
26
|
+
|
23
27
|
@config = Spout::Helpers::ConfigReader.new
|
24
28
|
|
25
29
|
if Spout::Helpers::ChartTypes::get_json(@config.visit, 'variable') == nil
|
@@ -52,17 +56,33 @@ module Spout
|
|
52
56
|
@variable_files = Dir.glob('variables/**/*.json')
|
53
57
|
|
54
58
|
t = Time.now
|
55
|
-
|
59
|
+
@graphs_folder = File.join("graphs", @standard_version)
|
60
|
+
FileUtils.mkpath @graphs_folder
|
56
61
|
|
57
62
|
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
|
58
63
|
|
59
64
|
@subject_loader.load_subjects_from_csvs!
|
60
65
|
@subjects = @subject_loader.subjects
|
66
|
+
|
67
|
+
load_current_progress
|
68
|
+
|
61
69
|
compute_tables_and_charts
|
62
70
|
|
63
71
|
puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
|
64
72
|
end
|
65
73
|
|
74
|
+
def load_current_progress
|
75
|
+
@progress_file = File.join(@graphs_folder, ".progress.json")
|
76
|
+
@progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
|
77
|
+
@progress = {} if @clean
|
78
|
+
end
|
79
|
+
|
80
|
+
def save_current_progress
|
81
|
+
File.open(@progress_file,"w") do |f|
|
82
|
+
f.write(@progress.to_json)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
66
86
|
def compute_tables_and_charts
|
67
87
|
variable_files_count = @variable_files.count
|
68
88
|
@variable_files.each_with_index do |variable_file, file_index|
|
@@ -79,7 +99,8 @@ module Spout
|
|
79
99
|
puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
|
80
100
|
end
|
81
101
|
|
82
|
-
|
102
|
+
@progress[variable_name] ||= {}
|
103
|
+
next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
|
83
104
|
|
84
105
|
stats = {
|
85
106
|
charts: {},
|
@@ -109,13 +130,18 @@ module Spout
|
|
109
130
|
end
|
110
131
|
end
|
111
132
|
|
112
|
-
chart_json_file = File.join(
|
133
|
+
chart_json_file = File.join(@graphs_folder, "#{json['id']}.json")
|
113
134
|
File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
|
114
135
|
|
115
|
-
|
116
|
-
|
136
|
+
@progress[variable_name]['generated'] = true
|
137
|
+
|
138
|
+
if @deploy_mode and not @progress[variable_name]['uploaded'] == true
|
139
|
+
response = send_to_server(chart_json_file)
|
140
|
+
@progress[variable_name]['uploaded'] = (response.kind_of?(Hash) and response['upload'] == 'success')
|
117
141
|
end
|
118
142
|
|
143
|
+
save_current_progress
|
144
|
+
|
119
145
|
end
|
120
146
|
end
|
121
147
|
|
@@ -24,6 +24,7 @@ module Spout
|
|
24
24
|
@variable_files = Dir.glob('variables/**/*.json')
|
25
25
|
@standard_version = standard_version
|
26
26
|
@pretend = (argv.delete('--pretend') != nil)
|
27
|
+
@clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
|
27
28
|
@sizes = sizes
|
28
29
|
@types = types
|
29
30
|
|
@@ -34,17 +35,32 @@ module Spout
|
|
34
35
|
@config = Spout::Helpers::ConfigReader.new
|
35
36
|
|
36
37
|
t = Time.now
|
37
|
-
|
38
|
+
@images_folder = File.join("images", @standard_version)
|
39
|
+
FileUtils.mkpath @images_folder
|
38
40
|
|
39
41
|
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
|
40
42
|
|
41
43
|
@subject_loader.load_subjects_from_csvs!
|
42
44
|
@subjects = @subject_loader.subjects
|
43
45
|
|
46
|
+
load_current_progress
|
47
|
+
|
44
48
|
compute_images
|
45
49
|
puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
|
46
50
|
end
|
47
51
|
|
52
|
+
def load_current_progress
|
53
|
+
@progress_file = File.join(@images_folder, ".progress.json")
|
54
|
+
@progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
|
55
|
+
@progress = {} if @clean
|
56
|
+
end
|
57
|
+
|
58
|
+
def save_current_progress
|
59
|
+
File.open(@progress_file,"w") do |f|
|
60
|
+
f.write(@progress.to_json)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
48
64
|
def compute_images
|
49
65
|
|
50
66
|
options_folder = "images/#{@standard_version}"
|
@@ -67,6 +83,10 @@ module Spout
|
|
67
83
|
puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
|
68
84
|
end
|
69
85
|
|
86
|
+
@progress[variable_name] ||= {}
|
87
|
+
|
88
|
+
next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
|
89
|
+
|
70
90
|
filtered_subjects = @subjects.select{ |s| s.send(@config.visit) != nil }
|
71
91
|
|
72
92
|
chart_json = Spout::Helpers::ChartTypes::chart_histogram(@config.visit, filtered_subjects, json, variable_name)
|
@@ -103,16 +123,21 @@ module Spout
|
|
103
123
|
}
|
104
124
|
eos
|
105
125
|
end
|
106
|
-
run_phantom_js("#{
|
107
|
-
run_phantom_js("#{
|
126
|
+
run_phantom_js(variable_name, "#{variable_name}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
|
127
|
+
run_phantom_js(variable_name, "#{variable_name}.png", 75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
|
128
|
+
@progress[variable_name]['uploaded'] = (@deploy_mode and @progress[variable_name]['failed'] != true)
|
129
|
+
save_current_progress
|
108
130
|
end
|
109
131
|
|
110
132
|
end
|
111
133
|
File.delete(tmp_options_file) if File.exist?(tmp_options_file)
|
112
134
|
end
|
113
135
|
|
114
|
-
def run_phantom_js(png_name, width, tmp_options_file)
|
115
|
-
|
136
|
+
def run_phantom_js(variable_name, png_name, width, tmp_options_file)
|
137
|
+
@progress[variable_name]['generated'] ||= []
|
138
|
+
@progress[variable_name]['uploaded_files'] ||= []
|
139
|
+
|
140
|
+
image_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
|
116
141
|
directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
|
117
142
|
|
118
143
|
open_command = if RUBY_PLATFORM.match(/mingw/) != nil
|
@@ -121,19 +146,29 @@ module Spout
|
|
121
146
|
'phantomjs'
|
122
147
|
end
|
123
148
|
|
124
|
-
phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{
|
149
|
+
phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{image_path} -scale 2.5 -width #{width} -constr Chart"
|
125
150
|
|
126
151
|
if @pretend
|
127
152
|
puts phantomjs_command
|
128
153
|
else
|
129
|
-
|
154
|
+
if not @progress[variable_name]['generated'].include?(png_name) or not File.exist?(png_name) or (File.exist?(png_name) and File.size(png_name) == 0)
|
155
|
+
`#{phantomjs_command}`
|
156
|
+
@progress[variable_name]['generated'] << png_name
|
157
|
+
end
|
130
158
|
|
131
|
-
|
159
|
+
if @deploy_mode and not @progress[variable_name]['uploaded_files'].include?(png_name)
|
160
|
+
response = send_to_server(image_path)
|
161
|
+
if response.kind_of?(Hash) and response['upload'] == 'success'
|
162
|
+
@progress[variable_name]['uploaded_files'] << png_name
|
163
|
+
else
|
164
|
+
@progress[variable_name]['upload_failed'] = true
|
165
|
+
end
|
166
|
+
end
|
132
167
|
end
|
133
168
|
end
|
134
169
|
|
135
170
|
def send_to_server(file)
|
136
|
-
|
171
|
+
Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
|
137
172
|
end
|
138
173
|
|
139
174
|
end
|
data/lib/spout/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spout
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.0.
|
4
|
+
version: 0.10.0.beta8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Remo Mueller
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-10-
|
11
|
+
date: 2014-10-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|