spout 0.10.2 → 0.11.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -0
  3. data/README.md +3 -30
  4. data/lib/spout/commands/coverage.rb +2 -1
  5. data/lib/spout/commands/deploy.rb +82 -77
  6. data/lib/spout/commands/exporter.rb +2 -3
  7. data/lib/spout/commands/graphs.rb +68 -67
  8. data/lib/spout/commands/help.rb +155 -0
  9. data/lib/spout/helpers/array_statistics.rb +36 -30
  10. data/lib/spout/helpers/chart_types.rb +2 -2
  11. data/lib/spout/helpers/config_reader.rb +5 -5
  12. data/lib/spout/helpers/json_request.rb +1 -2
  13. data/lib/spout/helpers/json_request_generic.rb +87 -0
  14. data/lib/spout/helpers/quietly.rb +2 -4
  15. data/lib/spout/helpers/semantic.rb +7 -11
  16. data/lib/spout/helpers/send_file.rb +23 -25
  17. data/lib/spout/helpers/subject_loader.rb +41 -32
  18. data/lib/spout/helpers/table_formatting.rb +7 -6
  19. data/lib/spout/models/bucket.rb +5 -4
  20. data/lib/spout/models/coverage_result.rb +1 -1
  21. data/lib/spout/models/dictionary.rb +3 -1
  22. data/lib/spout/models/domain.rb +7 -6
  23. data/lib/spout/models/empty.rb +17 -0
  24. data/lib/spout/models/form.rb +8 -5
  25. data/lib/spout/models/graphables/default.rb +41 -18
  26. data/lib/spout/models/graphables/histogram.rb +6 -7
  27. data/lib/spout/models/graphables.rb +3 -5
  28. data/lib/spout/models/option.rb +6 -2
  29. data/lib/spout/models/outlier_result.rb +3 -3
  30. data/lib/spout/models/record.rb +21 -3
  31. data/lib/spout/models/subject.rb +4 -7
  32. data/lib/spout/models/tables/choices_vs_choices.rb +29 -17
  33. data/lib/spout/models/tables/choices_vs_numeric.rb +19 -12
  34. data/lib/spout/models/tables/default.rb +19 -32
  35. data/lib/spout/models/tables/numeric_vs_choices.rb +9 -13
  36. data/lib/spout/models/tables/numeric_vs_numeric.rb +9 -11
  37. data/lib/spout/models/tables.rb +4 -6
  38. data/lib/spout/models/variable.rb +51 -13
  39. data/lib/spout/tasks/engine.rake +1 -1
  40. data/lib/spout/templates/ruby-version +1 -1
  41. data/lib/spout/templates/travis.yml +1 -1
  42. data/lib/spout/tests/domain_format.rb +2 -2
  43. data/lib/spout/tests/domain_name_format.rb +15 -0
  44. data/lib/spout/tests/form_name_format.rb +14 -0
  45. data/lib/spout/tests/variable_name_format.rb +14 -0
  46. data/lib/spout/tests.rb +18 -13
  47. data/lib/spout/version.rb +3 -3
  48. data/lib/spout/views/index.html.erb +2 -2
  49. data/lib/spout/views/outliers.html.erb +1 -1
  50. data/lib/spout.rb +13 -58
  51. data/spout.gemspec +14 -15
  52. metadata +25 -25
  53. data/lib/spout/commands/images.rb +0 -199
  54. data/lib/spout/support/javascripts/data.js +0 -17
  55. data/lib/spout/support/javascripts/highcharts-convert.js +0 -583
  56. data/lib/spout/support/javascripts/highcharts-more.js +0 -50
  57. data/lib/spout/support/javascripts/highstock.js +0 -353
  58. data/lib/spout/support/javascripts/jquery.1.9.1.min.js +0 -5
@@ -1,199 +0,0 @@
1
- require 'csv'
2
- require 'fileutils'
3
- require 'rubygems'
4
- require 'json'
5
- require 'yaml'
6
-
7
- require 'spout/models/variable'
8
- require 'spout/models/graphables'
9
- require 'spout/helpers/subject_loader'
10
- require 'spout/helpers/chart_types'
11
- require 'spout/helpers/config_reader'
12
- require 'spout/helpers/send_file'
13
-
14
- module Spout
15
- module Commands
16
- class Images
17
-
18
- def initialize(types, variable_ids, sizes, standard_version, argv, deploy_mode = false, url = '', slug = '', token = '', webserver_name = '', subjects = nil)
19
- @deploy_mode = deploy_mode
20
- @url = url
21
- @standard_version = standard_version
22
- @slug = slug
23
- @token = token
24
- @webserver_name = webserver_name
25
-
26
-
27
- @dictionary_root = Dir.pwd
28
- @variable_files = Dir.glob(File.join(@dictionary_root, 'variables', '**', '*.json'))
29
- @standard_version = standard_version
30
- @pretend = (argv.delete('--pretend') != nil)
31
- @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
32
- @sizes = sizes
33
- @types = types
34
-
35
- @valid_ids = variable_ids
36
-
37
- @number_of_rows = nil
38
-
39
- @config = Spout::Helpers::ConfigReader.new
40
-
41
- t = Time.now
42
- @images_folder = File.join("images", @standard_version)
43
- FileUtils.mkpath @images_folder
44
-
45
- @subjects = if subjects
46
- subjects
47
- else
48
- @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
49
- @subject_loader.load_subjects_from_csvs!
50
- @subjects = @subject_loader.subjects
51
- end
52
-
53
- load_current_progress
54
-
55
- compute_images
56
- puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
57
- end
58
-
59
- def load_current_progress
60
- @progress_file = File.join(@images_folder, ".progress.json")
61
- @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
62
- @progress = {} if !@progress.kind_of?(Hash) or @clean or @progress['SPOUT_VERSION'] != Spout::VERSION::STRING
63
- @progress['SPOUT_VERSION'] = Spout::VERSION::STRING
64
- end
65
-
66
- def save_current_progress
67
- File.open(@progress_file,"w") do |f|
68
- f.write(JSON.pretty_generate(@progress) + "\n")
69
- end
70
- end
71
-
72
- def compute_images
73
- begin
74
- iterate_through_variables
75
- ensure
76
- save_current_progress
77
- end
78
- end
79
-
80
- def iterate_through_variables
81
-
82
- options_folder = "images/#{@standard_version}"
83
- FileUtils.mkpath( options_folder )
84
- tmp_options_file = File.join( options_folder, 'options.json' )
85
-
86
- chart_variable = Spout::Models::Variable.find_by_id(@config.visit)
87
- variable_files_count = @variable_files.count
88
-
89
- @variable_files.each_with_index do |variable_file, file_index|
90
- variable = Spout::Models::Variable.new(variable_file, @dictionary_root)
91
-
92
- next unless variable.errors.size == 0
93
-
94
- next unless @valid_ids.include?(variable.id) or @valid_ids.size == 0
95
- next unless @types.include?(variable.type) or @types.size == 0
96
- next unless ["numeric", "integer", "choices"].include?(variable.type)
97
- next unless Spout::Models::Subject.method_defined?(variable.id)
98
-
99
- if @deploy_mode
100
- print "\r Image Generation: " + "#{"% 3d" % ((file_index+1)*100/variable_files_count)}% Uploaded".colorize(:white)
101
- else
102
- puts "#{file_index+1} of #{variable_files_count}: #{variable.folder}#{variable.id}"
103
- end
104
-
105
- @progress[variable.id] ||= {}
106
- @progress[variable.id]['uploaded'] ||= []
107
-
108
- next if (not @deploy_mode and @progress[variable.id]['generated'] == true) or (@deploy_mode and @progress[variable.id]['uploaded'].include?(@webserver_name))
109
-
110
- filtered_subjects = @subjects.select{ |s| s.send(@config.visit) != nil }
111
-
112
- graph = Spout::Models::Graphables.for(variable, chart_variable, nil, filtered_subjects)
113
-
114
- if graph.valid?
115
- File.open(tmp_options_file, "w") do |outfile|
116
- outfile.puts <<-eos
117
- {
118
- "credits": {
119
- "enabled": false
120
- },
121
- "chart": {
122
- "type": "column"
123
- },
124
- "title": {
125
- "text": ""
126
- },
127
- "xAxis": {
128
- "categories": #{graph.categories.to_json}
129
- },
130
- "yAxis": {
131
- "title": {
132
- "text": #{graph.units.to_json}
133
- }
134
- },
135
- "plotOptions": {
136
- "column": {
137
- "pointPadding": 0.2,
138
- "borderWidth": 0,
139
- "stacking": #{graph.stacking.to_json}
140
- }
141
- },
142
- "series": #{graph.series.to_json}
143
- }
144
- eos
145
- end
146
- run_phantom_js(variable, "#{variable.id}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
147
- run_phantom_js(variable, "#{variable.id}.png", 75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
148
-
149
- @progress[variable.id]['uploaded'] << @webserver_name if @deploy_mode and @progress[variable.id]['upload_failed'] != true
150
- @progress[variable.id].delete('uploaded_files')
151
- @progress[variable.id].delete('upload_failed')
152
- end
153
-
154
- end
155
- File.delete(tmp_options_file) if File.exist?(tmp_options_file)
156
- end
157
-
158
- def run_phantom_js(variable, png_name, width, tmp_options_file)
159
- @progress[variable.id]['generated'] ||= []
160
- @progress[variable.id]['uploaded_files'] ||= []
161
-
162
- image_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
163
- directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
164
-
165
- open_command = if RUBY_PLATFORM.match(/mingw/) != nil
166
- 'phantomjs.exe'
167
- else
168
- 'phantomjs'
169
- end
170
-
171
- phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{image_path} -scale 2.5 -width #{width} -constr Chart"
172
-
173
- if @pretend
174
- puts phantomjs_command
175
- else
176
- if not @progress[variable.id]['generated'].include?(png_name) or not File.exist?(png_name) or (File.exist?(png_name) and File.size(png_name) == 0)
177
- `#{phantomjs_command}`
178
- @progress[variable.id]['generated'] << png_name
179
- end
180
-
181
- if @deploy_mode and not @progress[variable.id]['uploaded_files'].include?(png_name)
182
- response = send_to_server(image_path)
183
- if response.kind_of?(Hash) and response['upload'] == 'success'
184
- @progress[variable.id]['uploaded_files'] << png_name
185
- else
186
- puts "\nUPLOAD FAILED: ".colorize(:red) + File.basename(png_name)
187
- @progress[variable.id]['upload_failed'] = true
188
- end
189
- end
190
- end
191
- end
192
-
193
- def send_to_server(file)
194
- Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
195
- end
196
-
197
- end
198
- end
199
- end
@@ -1,17 +0,0 @@
1
- /*
2
- Data plugin for Highcharts
3
-
4
- (c) 2012-2013 Torstein Hønsi
5
- Last revision 2013-06-07
6
-
7
- License: www.highcharts.com/license
8
- */
9
- (function(h){var k=h.each,m=function(b,a){this.init(b,a)};h.extend(m.prototype,{init:function(b,a){this.options=b;this.chartOptions=a;this.columns=b.columns||this.rowsToColumns(b.rows)||[];this.columns.length?this.dataFound():(this.parseCSV(),this.parseTable(),this.parseGoogleSpreadsheet())},getColumnDistribution:function(){var b=this.chartOptions,a=b&&b.chart&&b.chart.type,c=[];k(b&&b.series||[],function(b){c.push((h.seriesTypes[b.type||a||"line"].prototype.pointArrayMap||[0]).length)});this.valueCount=
10
- {global:(h.seriesTypes[a||"line"].prototype.pointArrayMap||[0]).length,individual:c}},dataFound:function(){this.parseTypes();this.findHeaderRow();this.parsed();this.complete()},parseCSV:function(){var b=this,a=this.options,c=a.csv,d=this.columns,f=a.startRow||0,i=a.endRow||Number.MAX_VALUE,j=a.startColumn||0,e=a.endColumn||Number.MAX_VALUE,g=0;c&&(c=c.replace(/\r\n/g,"\n").replace(/\r/g,"\n").split(a.lineDelimiter||"\n"),k(c,function(c,h){var n=b.trim(c),p=n.indexOf("#")===0;h>=f&&h<=i&&!p&&n!==""&&
11
- (n=c.split(a.itemDelimiter||","),k(n,function(b,a){a>=j&&a<=e&&(d[a-j]||(d[a-j]=[]),d[a-j][g]=b)}),g+=1)}),this.dataFound())},parseTable:function(){var b=this.options,a=b.table,c=this.columns,d=b.startRow||0,f=b.endRow||Number.MAX_VALUE,i=b.startColumn||0,j=b.endColumn||Number.MAX_VALUE,e;a&&(typeof a==="string"&&(a=document.getElementById(a)),k(a.getElementsByTagName("tr"),function(a,b){e=0;b>=d&&b<=f&&k(a.childNodes,function(a){if((a.tagName==="TD"||a.tagName==="TH")&&e>=i&&e<=j)c[e]||(c[e]=[]),
12
- c[e][b-d]=a.innerHTML,e+=1})}),this.dataFound())},parseGoogleSpreadsheet:function(){var b=this,a=this.options,c=a.googleSpreadsheetKey,d=this.columns,f=a.startRow||0,i=a.endRow||Number.MAX_VALUE,j=a.startColumn||0,e=a.endColumn||Number.MAX_VALUE,g,h;c&&jQuery.getJSON("https://spreadsheets.google.com/feeds/cells/"+c+"/"+(a.googleSpreadsheetWorksheet||"od6")+"/public/values?alt=json-in-script&callback=?",function(a){var a=a.feed.entry,c,k=a.length,m=0,o=0,l;for(l=0;l<k;l++)c=a[l],m=Math.max(m,c.gs$cell.col),
13
- o=Math.max(o,c.gs$cell.row);for(l=0;l<m;l++)if(l>=j&&l<=e)d[l-j]=[],d[l-j].length=Math.min(o,i-f);for(l=0;l<k;l++)if(c=a[l],g=c.gs$cell.row-1,h=c.gs$cell.col-1,h>=j&&h<=e&&g>=f&&g<=i)d[h-j][g-f]=c.content.$t;b.dataFound()})},findHeaderRow:function(){k(this.columns,function(){});this.headerRow=0},trim:function(b){return typeof b==="string"?b.replace(/^\s+|\s+$/g,""):b},parseTypes:function(){for(var b=this.columns,a=b.length,c,d,f,i;a--;)for(c=b[a].length;c--;)d=b[a][c],f=parseFloat(d),i=this.trim(d),
14
- i==f?(b[a][c]=f,f>31536E6?b[a].isDatetime=!0:b[a].isNumeric=!0):(d=this.parseDate(d),a===0&&typeof d==="number"&&!isNaN(d)?(b[a][c]=d,b[a].isDatetime=!0):b[a][c]=i===""?null:i)},dateFormats:{"YYYY-mm-dd":{regex:"^([0-9]{4})-([0-9]{2})-([0-9]{2})$",parser:function(b){return Date.UTC(+b[1],b[2]-1,+b[3])}}},parseDate:function(b){var a=this.options.parseDate,c,d,f;a&&(c=a(b));if(typeof b==="string")for(d in this.dateFormats)a=this.dateFormats[d],(f=b.match(a.regex))&&(c=a.parser(f));return c},rowsToColumns:function(b){var a,
15
- c,d,f,i;if(b){i=[];c=b.length;for(a=0;a<c;a++){f=b[a].length;for(d=0;d<f;d++)i[d]||(i[d]=[]),i[d][a]=b[a][d]}}return i},parsed:function(){this.options.parsed&&this.options.parsed.call(this,this.columns)},complete:function(){var b=this.columns,a,c,d=this.options,f,i,j,e,g,k;if(d.complete){this.getColumnDistribution();b.length>1&&(a=b.shift(),this.headerRow===0&&a.shift(),a.isDatetime?c="datetime":a.isNumeric||(c="category"));for(e=0;e<b.length;e++)if(this.headerRow===0)b[e].name=b[e].shift();i=[];
16
- for(e=0,k=0;e<b.length;k++){f=h.pick(this.valueCount.individual[k],this.valueCount.global);j=[];for(g=0;g<b[e].length;g++)j[g]=[a[g],b[e][g]!==void 0?b[e][g]:null],f>1&&j[g].push(b[e+1][g]!==void 0?b[e+1][g]:null),f>2&&j[g].push(b[e+2][g]!==void 0?b[e+2][g]:null),f>3&&j[g].push(b[e+3][g]!==void 0?b[e+3][g]:null),f>4&&j[g].push(b[e+4][g]!==void 0?b[e+4][g]:null);i[k]={name:b[e].name,data:j};e+=f}d.complete({xAxis:{type:c},series:i})}}});h.Data=m;h.data=function(b,a){return new m(b,a)};h.wrap(h.Chart.prototype,
17
- "init",function(b,a,c){var d=this;a&&a.data?h.data(h.extend(a.data,{complete:function(f){a.series&&k(a.series,function(b,c){a.series[c]=h.merge(b,f.series[c])});a=h.merge(f,a);b.call(d,a,c)}}),a):b.call(d,a,c)})})(Highcharts);