spout 0.10.0.beta7 → 0.10.0.beta8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d4418b6291f10a58bcf649b785114423a462aca8
4
- data.tar.gz: 16a62a4c3357173af4808bd5a5ab9d767f692296
3
+ metadata.gz: 97f132b532821e57396f65c1752060dac1a33c14
4
+ data.tar.gz: f87fc93a7f9a60c34e81f602c47a770bbd29eba9
5
5
  SHA512:
6
- metadata.gz: fdaeae598fccf2e5d29be1d491efc4951d02e56eaeb2917fb91a944b8eaea24d5fd4780bebb281116d42e8271fc63798694a53a1bdc3a69d0a926e7535a684de
7
- data.tar.gz: 38b272dfe518b4e230b6e4eec679b00838259c0fc3b3d0e1b3f5af39a56325ce702c1af3542a885e7c9e71c066ed1a8c38a72fc30a7ae1be6fc593bb07c9f576
6
+ metadata.gz: 7c9ed942121864d02d02bdcdac9115e8b04093e368f31dfd9a4e37541726f5ff803654a93b6d62c872c314efb40f4757bf759f6d1098281dc91f090bbd201229
7
+ data.tar.gz: 741c7cfd6b2f365aff937f8d2508a04aaaef81fad297cca5773f3c2a82e1759cda0aba69a68d6e72a6d6547a39151f8915d64ac7e0d688ee5f605dd5a39fe216
@@ -1,5 +1,6 @@
1
1
  require 'colorize'
2
2
  require 'net/http'
3
+ require 'io/console'
3
4
 
4
5
  require 'spout/helpers/config_reader'
5
6
  require 'spout/helpers/quietly'
@@ -48,13 +49,20 @@ module Spout
48
49
  def initialize(argv, version)
49
50
  @environment = argv[1].to_s
50
51
  @version = version
51
- @skip_checks = (argv.delete('--skip-checks') != nil)
52
+ @skip_checks = (argv.delete('--skip-checks') != nil or argv.delete('--no-checks') != nil)
52
53
 
53
- @skip_graphs = (argv.delete('--skip-graphs') != nil)
54
- @skip_images = (argv.delete('--skip-images') != nil)
55
- @skip_server_updates = (argv.delete('--skip-server-updates') != nil)
54
+ @skip_graphs = (argv.delete('--skip-graphs') != nil or argv.delete('--no-graphs') != nil)
55
+ @skip_images = (argv.delete('--skip-images') != nil or argv.delete('--no-images') != nil)
56
+ @clean = (argv.delete('--clean') != nil or argv.delete('--no-resume'))
57
+ @skip_server_updates = (argv.delete('--skip-server-updates') != nil or argv.delete('--no-server-updates') != nil)
56
58
 
57
- run_all
59
+ @token = argv.select{|a| /^--token=/ =~ a}.collect{|a| a.gsub(/^--token=/, '')}.first
60
+
61
+ begin
62
+ run_all
63
+ rescue Interrupt, IRB::Abort
64
+ puts "\nINTERRUPTED".colorize(:red)
65
+ end
58
66
  end
59
67
 
60
68
  def run_all
@@ -66,6 +74,7 @@ module Spout
66
74
  graph_generation unless @skip_graphs
67
75
  image_generation unless @skip_images
68
76
  dataset_uploads
77
+ data_dictionary_uploads
69
78
  trigger_server_updates unless @skip_server_updates
70
79
  rescue DeployError
71
80
  end
@@ -170,7 +179,18 @@ module Spout
170
179
  def user_authorization
171
180
  puts " Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
172
181
  print " Enter your token: "
173
- @token = STDIN.gets.chomp
182
+ @token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ''
183
+
184
+ response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
185
+
186
+ if response.kind_of?(Hash) and response['editor']
187
+ puts "AUTHORIZED".colorize(:green)
188
+ else
189
+ puts "UNAUTHORIZED".colorize(:red)
190
+ puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
191
+ raise DeployError
192
+ end
193
+
174
194
  # failure ''
175
195
  # puts "PASS".colorize(:green)
176
196
  end
@@ -178,24 +198,31 @@ module Spout
178
198
  def graph_generation
179
199
  # failure ''
180
200
  require 'spout/commands/graphs'
181
- Spout::Commands::Graphs.new([], @version, true, @url, @slug, @token)
201
+ argv = []
202
+ argv << "--clean" if @clean
203
+ Spout::Commands::Graphs.new(argv, @version, true, @url, @slug, @token)
182
204
  puts "\r Graph Generation: " + "DONE ".colorize(:green)
183
205
  end
184
206
 
185
207
  def image_generation
186
208
  # failure ''
187
209
  require 'spout/commands/images'
188
- Spout::Commands::Images.new([], [], [], @version, [], true, @url, @slug, @token)
210
+ argv = []
211
+ argv << "--clean" if @clean
212
+ Spout::Commands::Images.new(argv, [], [], @version, [], true, @url, @slug, @token)
189
213
  puts "\r Image Generation: " + "DONE ".colorize(:green)
190
214
  end
191
215
 
192
216
  def dataset_uploads
193
217
  available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort : [])
194
-
195
218
  semantic = Spout::Helpers::Semantic.new(@version, available_folders)
196
-
197
219
  csv_directory = semantic.selected_folder
198
220
 
221
+ if @version != csv_directory
222
+ puts "\r Dataset Uploads: " + "SKIPPED - #{csv_directory} CSV dataset already on server".colorize(:blue)
223
+ return
224
+ end
225
+
199
226
  csv_files = Dir.glob("csvs/#{csv_directory}/*.csv")
200
227
 
201
228
  csv_files.each_with_index do |csv_file, index|
@@ -205,13 +232,36 @@ module Spout
205
232
  puts "\r Dataset Uploads: " + "DONE ".colorize(:green)
206
233
  end
207
234
 
235
+ def data_dictionary_uploads
236
+ print " Dictionary Uploads:"
237
+
238
+ require 'spout/commands/exporter'
239
+ Spout::Commands::Exporter.new(@version, ['--quiet'])
240
+
241
+ csv_files = Dir.glob("dd/#{@version}/*.csv")
242
+ csv_files.each_with_index do |csv_file, index|
243
+ print "\r Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
244
+ response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_dataset_csv.json", csv_file, @version, @token)
245
+ end
246
+ puts "\r Dictionary Uploads: " + "DONE ".colorize(:green)
247
+ end
248
+
208
249
  def trigger_server_updates
209
250
  print "Launch Server Scripts: "
210
251
  response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/refresh_dictionary.json?version=#{@version}")
211
- if response and response['refresh'] == 'success'
252
+ if response.kind_of?(Hash) and response['refresh'] == 'success'
212
253
  puts "DONE".colorize(:green)
254
+ elsif response.kind_of?(Hash) and response['refresh'] == 'notagfound'
255
+ puts "FAIL".colorize(:red)
256
+ puts "#{INDENT}Tag not found in repository, resolve using: " + "git push --tags".colorize(:white)
257
+ raise DeployError
258
+ elsif response.kind_of?(Hash) and response['refresh'] == 'gitrepodoesnotexist'
259
+ puts "FAIL".colorize(:red)
260
+ puts "#{INDENT}Dataset data dictionary git repository has not been cloned on the server. Contact server admin.".colorize(:white)
261
+ raise DeployError
213
262
  else
214
263
  puts "FAIL".colorize(:red)
264
+ raise DeployError
215
265
  end
216
266
  end
217
267
 
@@ -9,7 +9,7 @@ module Spout
9
9
  module Commands
10
10
  class Exporter
11
11
  def initialize(standard_version, argv)
12
- @csv_file = argv[1].to_s
12
+ @quiet = (argv.delete('--quiet') != nil)
13
13
  @standard_version = standard_version
14
14
  @config = Spout::Helpers::ConfigReader.new
15
15
  expanded_export!
@@ -19,7 +19,7 @@ module Spout
19
19
 
20
20
  def expanded_export!
21
21
  folder = "dd/#{@standard_version}"
22
- puts " create".colorize( :green ) + " #{folder}"
22
+ puts " create".colorize( :green ) + " #{folder}" unless @quiet
23
23
  FileUtils.mkpath folder
24
24
 
25
25
  generic_export(folder, 'variables', %w(id display_name description type units domain labels calculation))
@@ -29,7 +29,7 @@ module Spout
29
29
 
30
30
  def generic_export(folder, type, keys, include_domain_name = false)
31
31
  export_file = export_file_name(type)
32
- puts " export".colorize( :blue ) + " #{folder}/#{export_file}"
32
+ puts " export".colorize( :blue ) + " #{folder}/#{export_file}" unless @quiet
33
33
  CSV.open("#{folder}/#{export_file}", "wb") do |csv|
34
34
  if include_domain_name
35
35
  csv << ['folder', 'domain_id'] + keys
@@ -20,6 +20,10 @@ module Spout
20
20
  @slug = slug
21
21
  @token = token
22
22
 
23
+ argv = variables
24
+
25
+ @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
26
+
23
27
  @config = Spout::Helpers::ConfigReader.new
24
28
 
25
29
  if Spout::Helpers::ChartTypes::get_json(@config.visit, 'variable') == nil
@@ -52,17 +56,33 @@ module Spout
52
56
  @variable_files = Dir.glob('variables/**/*.json')
53
57
 
54
58
  t = Time.now
55
- FileUtils.mkpath "graphs/#{@standard_version}"
59
+ @graphs_folder = File.join("graphs", @standard_version)
60
+ FileUtils.mkpath @graphs_folder
56
61
 
57
62
  @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
58
63
 
59
64
  @subject_loader.load_subjects_from_csvs!
60
65
  @subjects = @subject_loader.subjects
66
+
67
+ load_current_progress
68
+
61
69
  compute_tables_and_charts
62
70
 
63
71
  puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
64
72
  end
65
73
 
74
+ def load_current_progress
75
+ @progress_file = File.join(@graphs_folder, ".progress.json")
76
+ @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
77
+ @progress = {} if @clean
78
+ end
79
+
80
+ def save_current_progress
81
+ File.open(@progress_file,"w") do |f|
82
+ f.write(@progress.to_json)
83
+ end
84
+ end
85
+
66
86
  def compute_tables_and_charts
67
87
  variable_files_count = @variable_files.count
68
88
  @variable_files.each_with_index do |variable_file, file_index|
@@ -79,7 +99,8 @@ module Spout
79
99
  puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
80
100
  end
81
101
 
82
-
102
+ @progress[variable_name] ||= {}
103
+ next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
83
104
 
84
105
  stats = {
85
106
  charts: {},
@@ -109,13 +130,18 @@ module Spout
109
130
  end
110
131
  end
111
132
 
112
- chart_json_file = File.join('graphs', @standard_version, "#{json['id']}.json")
133
+ chart_json_file = File.join(@graphs_folder, "#{json['id']}.json")
113
134
  File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
114
135
 
115
- if @deploy_mode
116
- send_to_server(chart_json_file)
136
+ @progress[variable_name]['generated'] = true
137
+
138
+ if @deploy_mode and not @progress[variable_name]['uploaded'] == true
139
+ response = send_to_server(chart_json_file)
140
+ @progress[variable_name]['uploaded'] = (response.kind_of?(Hash) and response['upload'] == 'success')
117
141
  end
118
142
 
143
+ save_current_progress
144
+
119
145
  end
120
146
  end
121
147
 
@@ -24,6 +24,7 @@ module Spout
24
24
  @variable_files = Dir.glob('variables/**/*.json')
25
25
  @standard_version = standard_version
26
26
  @pretend = (argv.delete('--pretend') != nil)
27
+ @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
27
28
  @sizes = sizes
28
29
  @types = types
29
30
 
@@ -34,17 +35,32 @@ module Spout
34
35
  @config = Spout::Helpers::ConfigReader.new
35
36
 
36
37
  t = Time.now
37
- FileUtils.mkpath "graphs/#{@standard_version}"
38
+ @images_folder = File.join("images", @standard_version)
39
+ FileUtils.mkpath @images_folder
38
40
 
39
41
  @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
40
42
 
41
43
  @subject_loader.load_subjects_from_csvs!
42
44
  @subjects = @subject_loader.subjects
43
45
 
46
+ load_current_progress
47
+
44
48
  compute_images
45
49
  puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
46
50
  end
47
51
 
52
+ def load_current_progress
53
+ @progress_file = File.join(@images_folder, ".progress.json")
54
+ @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
55
+ @progress = {} if @clean
56
+ end
57
+
58
+ def save_current_progress
59
+ File.open(@progress_file,"w") do |f|
60
+ f.write(@progress.to_json)
61
+ end
62
+ end
63
+
48
64
  def compute_images
49
65
 
50
66
  options_folder = "images/#{@standard_version}"
@@ -67,6 +83,10 @@ module Spout
67
83
  puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
68
84
  end
69
85
 
86
+ @progress[variable_name] ||= {}
87
+
88
+ next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
89
+
70
90
  filtered_subjects = @subjects.select{ |s| s.send(@config.visit) != nil }
71
91
 
72
92
  chart_json = Spout::Helpers::ChartTypes::chart_histogram(@config.visit, filtered_subjects, json, variable_name)
@@ -103,16 +123,21 @@ module Spout
103
123
  }
104
124
  eos
105
125
  end
106
- run_phantom_js("#{json['id']}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
107
- run_phantom_js("#{json['id']}.png", 75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
126
+ run_phantom_js(variable_name, "#{variable_name}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
127
+ run_phantom_js(variable_name, "#{variable_name}.png", 75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
128
+ @progress[variable_name]['uploaded'] = (@deploy_mode and @progress[variable_name]['failed'] != true)
129
+ save_current_progress
108
130
  end
109
131
 
110
132
  end
111
133
  File.delete(tmp_options_file) if File.exist?(tmp_options_file)
112
134
  end
113
135
 
114
- def run_phantom_js(png_name, width, tmp_options_file)
115
- graph_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
136
+ def run_phantom_js(variable_name, png_name, width, tmp_options_file)
137
+ @progress[variable_name]['generated'] ||= []
138
+ @progress[variable_name]['uploaded_files'] ||= []
139
+
140
+ image_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
116
141
  directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
117
142
 
118
143
  open_command = if RUBY_PLATFORM.match(/mingw/) != nil
@@ -121,19 +146,29 @@ module Spout
121
146
  'phantomjs'
122
147
  end
123
148
 
124
- phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
149
+ phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{image_path} -scale 2.5 -width #{width} -constr Chart"
125
150
 
126
151
  if @pretend
127
152
  puts phantomjs_command
128
153
  else
129
- `#{phantomjs_command}`
154
+ if not @progress[variable_name]['generated'].include?(png_name) or not File.exist?(png_name) or (File.exist?(png_name) and File.size(png_name) == 0)
155
+ `#{phantomjs_command}`
156
+ @progress[variable_name]['generated'] << png_name
157
+ end
130
158
 
131
- send_to_server(graph_path) if @deploy_mode
159
+ if @deploy_mode and not @progress[variable_name]['uploaded_files'].include?(png_name)
160
+ response = send_to_server(image_path)
161
+ if response.kind_of?(Hash) and response['upload'] == 'success'
162
+ @progress[variable_name]['uploaded_files'] << png_name
163
+ else
164
+ @progress[variable_name]['upload_failed'] = true
165
+ end
166
+ end
132
167
  end
133
168
  end
134
169
 
135
170
  def send_to_server(file)
136
- response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
171
+ Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
137
172
  end
138
173
 
139
174
  end
data/lib/spout/version.rb CHANGED
@@ -3,7 +3,7 @@ module Spout
3
3
  MAJOR = 0
4
4
  MINOR = 10
5
5
  TINY = 0
6
- BUILD = "beta7" # nil, "pre", "rc", "rc2"
6
+ BUILD = "beta8" # nil, "pre", "rc", "rc2"
7
7
 
8
8
  STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
9
9
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: spout
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.0.beta7
4
+ version: 0.10.0.beta8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Remo Mueller
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-10-17 00:00:00.000000000 Z
11
+ date: 2014-10-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake