spout 0.12.0.beta2 → 0.12.0.rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +35 -3
- data/lib/spout/commands/coverage.rb +21 -30
- data/lib/spout/commands/deploy.rb +14 -18
- data/lib/spout/commands/exporter.rb +1 -1
- data/lib/spout/commands/graphs.rb +4 -4
- data/lib/spout/commands/help.rb +17 -1
- data/lib/spout/commands/importer.rb +41 -3
- data/lib/spout/commands/project_generator.rb +26 -10
- data/lib/spout/commands/update.rb +37 -0
- data/lib/spout/helpers/json_request.rb +68 -21
- data/lib/spout/helpers/subject_loader.rb +1 -1
- data/lib/spout/models/coverage_result.rb +11 -4
- data/lib/spout/templates/CHANGELOG.md.erb +3 -0
- data/lib/spout/templates/Gemfile +2 -0
- data/lib/spout/templates/README.md.erb +46 -0
- data/lib/spout/templates/Rakefile +2 -0
- data/lib/spout/templates/VERSION +1 -0
- data/lib/spout/templates/gitignore +1 -2
- data/lib/spout/templates/spout.yml.erb +3 -0
- data/lib/spout/templates/test/dictionary_test.rb +3 -2
- data/lib/spout/tests/domain_existence_validation.rb +9 -10
- data/lib/spout/tests/domain_format.rb +8 -8
- data/lib/spout/version.rb +1 -1
- data/lib/spout/views/index.html.erb +18 -15
- data/lib/spout.rb +13 -6
- metadata +7 -4
- data/lib/spout/helpers/json_request_generic.rb +0 -89
- data/lib/spout/templates/spout.yml +0 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b6b577bc1b70562b8fd31c019ebfaf216fbb903c
|
4
|
+
data.tar.gz: 84a57bcf18809bcbe7241e969247ae45023fc583
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4498b53635947fb0810c6e3165b58048cf0855130cb6b3a75e47c24ba075d6509cbe10fe2ddcf7c2df01248a73d74f45e79b0847d4a35629c1f399c5ca4b17b0
|
7
|
+
data.tar.gz: c81170c50d3cf60e70f931a8eb75fa09dee569705ec687e513b09fdf7d67cbb418c8e5e5c7b9373d3e5fd7649b9f2feb92a114540e0987d8a18df767d47f7c0c
|
data/CHANGELOG.md
CHANGED
@@ -3,11 +3,17 @@
|
|
3
3
|
### Enhancements
|
4
4
|
- **General Changes**
|
5
5
|
- Spout now provides a warning and skips columns in CSVs with blank headers
|
6
|
+
- `spout new` command now generates `CHANGELOG.md`, `README.md`, and `VERSION`
|
7
|
+
placeholder files
|
8
|
+
- Check for the latest version available using `spout update`
|
9
|
+
- Integer and numeric variables can now reference a domain for missing values
|
6
10
|
- **Exporter Changes**
|
7
11
|
- The export command now exports the variable forms attribute
|
12
|
+
- Dictionary exports now are in `exports` instead of `dd` folder
|
8
13
|
- **Importer Changes**
|
9
14
|
- The import command now reads in the `commonly_used` column
|
10
15
|
- The import command now imports the variable forms attribute
|
16
|
+
- CSVs of forms can now be imported using `spout import <forms.csv> --forms`
|
11
17
|
- **Gem Changes**
|
12
18
|
- Updated to Ruby 2.4.1
|
13
19
|
- Updated to bundler 1.13
|
@@ -17,6 +23,9 @@
|
|
17
23
|
### Refactoring
|
18
24
|
- General code cleanup based on Rubocop recommendations
|
19
25
|
|
26
|
+
### Bug Fixes
|
27
|
+
- The `test_units` example method now works if `nil` is defined in `VALID_UNITS`
|
28
|
+
|
20
29
|
## 0.11.1 (February 4, 2016)
|
21
30
|
|
22
31
|
### Bug Fix
|
data/README.md
CHANGED
@@ -105,6 +105,29 @@ Other columns that are imported include:
|
|
105
105
|
`folder`: The name of the folder path where the domain resides.
|
106
106
|
|
107
107
|
|
108
|
+
#### Importing forms from an existing CSV file
|
109
|
+
|
110
|
+
```
|
111
|
+
spout import data_dictionary_domains.csv --forms
|
112
|
+
```
|
113
|
+
|
114
|
+
The CSV should contain at minimal three column headers:
|
115
|
+
|
116
|
+
`folder`: This can be blank, however it is used to place forms into a folder
|
117
|
+
hiearchy. The folder column can contain forward slashes `/` to place a form
|
118
|
+
into a subfolder. An example may be, `id`: `family_history`,
|
119
|
+
`folder`: `Demographics/BaselineVisit` would create a file
|
120
|
+
`forms/Demographics/BaselineVisit/family_history.json`
|
121
|
+
|
122
|
+
`id`: The reference name of the form.
|
123
|
+
|
124
|
+
`display_name`: The name of the form.
|
125
|
+
|
126
|
+
Other columns that are imported include:
|
127
|
+
|
128
|
+
`code_book`: The file name of the document or PDF, including the file extension.
|
129
|
+
|
130
|
+
|
108
131
|
### Test your repository
|
109
132
|
|
110
133
|
If you created your data dictionary repository using `spout new`, you can go
|
@@ -166,9 +189,10 @@ class DictionaryTest < Minitest::Test
|
|
166
189
|
|
167
190
|
@variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
|
168
191
|
define_method("test_units: #{variable.path}") do
|
169
|
-
|
170
|
-
|
171
|
-
|
192
|
+
message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
|
193
|
+
" Valid types: " +
|
194
|
+
VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(', ')
|
195
|
+
assert VALID_UNITS.include?(variable.units), message
|
172
196
|
end
|
173
197
|
end
|
174
198
|
end
|
@@ -388,3 +412,11 @@ The following steps are run:
|
|
388
412
|
- `README.md` and `KNOWNISSUES.md` are uploaded
|
389
413
|
- **Server-Side Updates**
|
390
414
|
- Server refreshes dataset folder to reflect new dataset and data dictionaries
|
415
|
+
|
416
|
+
### Check if you are using the latest version of Spout
|
417
|
+
|
418
|
+
You can check if a newer version of Spout is available by typing:
|
419
|
+
|
420
|
+
```
|
421
|
+
spout update
|
422
|
+
```
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'yaml'
|
4
3
|
require 'erb'
|
5
4
|
require 'fileutils'
|
5
|
+
require 'yaml'
|
6
6
|
|
7
7
|
require 'spout/helpers/subject_loader'
|
8
8
|
require 'spout/models/coverage_result'
|
@@ -12,67 +12,58 @@ require 'spout/helpers/array_statistics'
|
|
12
12
|
|
13
13
|
module Spout
|
14
14
|
module Commands
|
15
|
+
# Generate a coverage report for the data dictionary.
|
15
16
|
class Coverage
|
16
17
|
include Spout::Helpers::NumberHelper
|
17
18
|
|
18
19
|
def initialize(standard_version, argv)
|
19
20
|
@standard_version = standard_version
|
20
|
-
@console =
|
21
|
-
|
22
|
-
@variable_files = Dir.glob("variables/**/*.json")
|
21
|
+
@console = !argv.delete('--console').nil?
|
22
|
+
@variable_files = Dir.glob('variables/**/*.json')
|
23
23
|
@valid_ids = []
|
24
24
|
@number_of_rows = nil
|
25
|
-
|
26
25
|
@config = Spout::Helpers::ConfigReader.new
|
27
|
-
|
28
|
-
|
26
|
+
@subject_loader = Spout::Helpers::SubjectLoader.new(
|
27
|
+
@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit
|
28
|
+
)
|
29
29
|
@subject_loader.load_subjects_from_csvs_part_one! # Not Part Two which is essentially cleaning the data
|
30
30
|
@subjects = @subject_loader.subjects
|
31
|
-
|
32
31
|
run_coverage_report!
|
33
32
|
end
|
34
33
|
|
35
34
|
def run_coverage_report!
|
36
35
|
puts "Generating: index.html\n\n"
|
37
|
-
|
38
36
|
@matching_results = []
|
39
|
-
|
40
37
|
@subject_loader.all_methods.each do |method, csv_files|
|
41
38
|
scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).compact_empty.uniq)
|
42
|
-
@matching_results << [
|
39
|
+
@matching_results << [csv_files, method, scr]
|
43
40
|
end
|
44
|
-
|
45
|
-
variable_ids = Dir.glob("variables/**/*.json").collect{ |file| file.gsub(/^(.*)\/|\.json$/, '').downcase }
|
41
|
+
variable_ids = Dir.glob('variables/**/*.json').collect { |file| file.gsub(%r{^(.*)/|\.json$}, '').downcase }
|
46
42
|
@extra_variable_ids = (variable_ids - @subject_loader.all_methods.keys).sort
|
47
|
-
|
48
43
|
@subject_loader.load_variable_domains!
|
49
|
-
domain_ids = Dir.glob(
|
44
|
+
domain_ids = Dir.glob('domains/**/*.json').collect { |file| file.gsub(%r{^(.*)/|\.json$}, '').downcase }
|
50
45
|
@extra_domain_ids = (domain_ids - @subject_loader.all_domains).sort
|
51
|
-
|
52
|
-
|
53
|
-
|
46
|
+
@matching_results.sort! do |a, b|
|
47
|
+
[b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]
|
48
|
+
end
|
54
49
|
@coverage_results = []
|
55
|
-
|
56
50
|
@subject_loader.csv_files.each do |csv_file|
|
57
|
-
total_column_count = @matching_results.select{|mr| mr[0].include?(csv_file)}.count
|
58
|
-
mapped_column_count = @matching_results
|
59
|
-
|
51
|
+
total_column_count = @matching_results.select { |mr| mr[0].include?(csv_file) }.count
|
52
|
+
mapped_column_count = @matching_results
|
53
|
+
.select { |mr| mr[0].include?(csv_file) && mr[2].number_of_errors.zero? }.count
|
54
|
+
@coverage_results << [csv_file, total_column_count, mapped_column_count]
|
60
55
|
end
|
61
|
-
|
62
56
|
coverage_folder = File.join(Dir.pwd, 'coverage')
|
63
57
|
FileUtils.mkpath coverage_folder
|
64
58
|
coverage_file = File.join(coverage_folder, 'index.html')
|
65
|
-
|
66
59
|
File.open(coverage_file, 'w+') do |file|
|
67
|
-
erb_location = File.join(
|
60
|
+
erb_location = File.join(File.dirname(__FILE__), '../views/index.html.erb')
|
68
61
|
file.puts ERB.new(File.read(erb_location)).result(binding)
|
69
62
|
end
|
70
|
-
|
71
63
|
unless @console
|
72
|
-
open_command = 'open'
|
73
|
-
open_command = 'start'
|
74
|
-
|
75
|
-
system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
|
64
|
+
open_command = 'open' unless RUBY_PLATFORM.match(/darwin/).nil?
|
65
|
+
open_command = 'start' unless RUBY_PLATFORM.match(/mingw/).nil?
|
66
|
+
system "#{open_command} #{coverage_file}" if %w(start open).include?(open_command)
|
76
67
|
end
|
77
68
|
puts "#{coverage_file}\n\n"
|
78
69
|
end
|
@@ -10,7 +10,6 @@ require 'spout/helpers/quietly'
|
|
10
10
|
require 'spout/helpers/send_file'
|
11
11
|
require 'spout/helpers/semantic'
|
12
12
|
require 'spout/helpers/json_request'
|
13
|
-
require 'spout/helpers/json_request_generic'
|
14
13
|
|
15
14
|
# - **User Authorization**
|
16
15
|
# - User authenticates via token, the user must be a dataset editor
|
@@ -37,6 +36,7 @@ end
|
|
37
36
|
|
38
37
|
module Spout
|
39
38
|
module Commands
|
39
|
+
# Deploys a data dictionary and associated dataset to the server.
|
40
40
|
class Deploy
|
41
41
|
include Spout::Helpers::Quietly
|
42
42
|
|
@@ -211,22 +211,17 @@ module Spout
|
|
211
211
|
end
|
212
212
|
|
213
213
|
def user_authorization
|
214
|
-
puts
|
215
|
-
print
|
214
|
+
puts ' Get your token here: ' + "#{@url}/token".colorize(:blue).on_white.underline
|
215
|
+
print ' Enter your token: '
|
216
216
|
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ''
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
if response.is_a?(Hash) and response['editor']
|
217
|
+
(json, _status) = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
|
218
|
+
if json.is_a?(Hash) && json['editor']
|
221
219
|
puts 'AUTHORIZED'.colorize(:green)
|
222
220
|
else
|
223
221
|
puts 'UNAUTHORIZED'.colorize(:red)
|
224
222
|
puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
|
225
|
-
|
223
|
+
raise DeployError
|
226
224
|
end
|
227
|
-
|
228
|
-
# failure ''
|
229
|
-
# puts 'PASS'.colorize(:green)
|
230
225
|
end
|
231
226
|
|
232
227
|
def upload_variables
|
@@ -328,12 +323,12 @@ module Spout
|
|
328
323
|
|
329
324
|
print 'Launch Server Scripts: '
|
330
325
|
params = { auth_token: @token, dataset: @slug, version: @version, folders: @created_folders.compact.uniq }
|
331
|
-
(
|
332
|
-
if
|
326
|
+
(json, _status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/dictionary/refresh.json", params)
|
327
|
+
if json.is_a?(Hash) && json['refresh'] == 'success'
|
333
328
|
puts 'DONE'.colorize(:green)
|
334
329
|
else
|
335
330
|
puts 'FAIL'.colorize(:red)
|
336
|
-
|
331
|
+
raise DeployError
|
337
332
|
end
|
338
333
|
end
|
339
334
|
|
@@ -342,11 +337,12 @@ module Spout
|
|
342
337
|
puts ' Set Default Version: ' + 'SKIP'.colorize(:blue)
|
343
338
|
return
|
344
339
|
end
|
345
|
-
|
346
340
|
print ' Set Default Version: '
|
347
341
|
params = { auth_token: @token, dataset: @slug, version: @version }
|
348
|
-
(
|
349
|
-
|
342
|
+
(json, _status) = Spout::Helpers::JsonRequest.post(
|
343
|
+
"#{@url}/api/v1/dictionary/update_default_version.json", params
|
344
|
+
)
|
345
|
+
if json.is_a?(Hash) && json['version_update'] == 'success'
|
350
346
|
puts @version.to_s.colorize(:green)
|
351
347
|
else
|
352
348
|
failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) + ' for ' + @slug.to_s.colorize(:white) + ' dataset.')
|
@@ -356,7 +352,7 @@ module Spout
|
|
356
352
|
def failure(message)
|
357
353
|
puts 'FAIL'.colorize(:red)
|
358
354
|
puts message
|
359
|
-
|
355
|
+
raise DeployError
|
360
356
|
end
|
361
357
|
|
362
358
|
def upload_file(file, folder)
|
@@ -14,7 +14,7 @@ require 'spout/models/graphables'
|
|
14
14
|
require 'spout/models/tables'
|
15
15
|
require 'spout/helpers/config_reader'
|
16
16
|
require 'spout/helpers/send_file'
|
17
|
-
require 'spout/helpers/
|
17
|
+
require 'spout/helpers/json_request'
|
18
18
|
require 'spout/version'
|
19
19
|
|
20
20
|
module Spout
|
@@ -176,12 +176,12 @@ module Spout
|
|
176
176
|
domain: (variable.domain ? variable.domain.deploy_params : nil),
|
177
177
|
forms: variable.forms.collect(&:deploy_params) }
|
178
178
|
params[:variable][:spout_stats] = stats.to_json
|
179
|
-
(
|
180
|
-
if
|
179
|
+
(json, status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/variables/create_or_update.json", params)
|
180
|
+
if json.is_a?(Hash) && status.is_a?(Net::HTTPSuccess)
|
181
181
|
@progress[variable.id]['uploaded'] << @webserver_name
|
182
182
|
else
|
183
183
|
puts "\nUPLOAD FAILED: ".colorize(:red) + variable.id
|
184
|
-
puts "- Error: #{
|
184
|
+
puts "- Error: #{json.inspect}"
|
185
185
|
end
|
186
186
|
end
|
187
187
|
end
|
data/lib/spout/commands/help.rb
CHANGED
@@ -29,6 +29,7 @@ The most common spout commands are:
|
|
29
29
|
in `<project_name>/graphs/<version>/`
|
30
30
|
[d]eploy NAME Push dataset and data dictionary to a
|
31
31
|
webserver specified in `.spout.yml`
|
32
|
+
[u]pdate Update the Spout gem
|
32
33
|
[v]ersion Returns the version of Spout
|
33
34
|
|
34
35
|
Commands can be referenced by the first letter:
|
@@ -42,7 +43,9 @@ EOT
|
|
42
43
|
|
43
44
|
def new_project
|
44
45
|
puts <<-EOT
|
45
|
-
Usage: spout new <project_name>
|
46
|
+
Usage: spout new <project_name> [--skip-gemfile]
|
47
|
+
|
48
|
+
Use `--skip-gemfile` to skip installing gems after project creation.
|
46
49
|
|
47
50
|
More information here:
|
48
51
|
|
@@ -55,6 +58,8 @@ EOT
|
|
55
58
|
puts <<-EOT
|
56
59
|
Usage: spout version
|
57
60
|
|
61
|
+
Returns version of spout.
|
62
|
+
|
58
63
|
EOT
|
59
64
|
end
|
60
65
|
|
@@ -62,6 +67,15 @@ EOT
|
|
62
67
|
puts <<-EOT
|
63
68
|
Usage: spout test
|
64
69
|
|
70
|
+
EOT
|
71
|
+
end
|
72
|
+
|
73
|
+
def update
|
74
|
+
puts <<-EOT
|
75
|
+
Usage: spout update
|
76
|
+
|
77
|
+
Checks if a newer version of Spout is available.
|
78
|
+
|
65
79
|
EOT
|
66
80
|
end
|
67
81
|
|
@@ -71,10 +85,12 @@ Usage: spout import <csv_file>
|
|
71
85
|
|
72
86
|
Optional Flags:
|
73
87
|
--domains Specify to import CSV of domains
|
88
|
+
--forms Specify to import CSV of forms
|
74
89
|
|
75
90
|
More information:
|
76
91
|
https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file
|
77
92
|
https://github.com/sleepepi/spout#importing-domains-from-an-existing-csv-file
|
93
|
+
https://github.com/sleepepi/spout#importing-forms-from-an-existing-csv-file
|
78
94
|
EOT
|
79
95
|
end
|
80
96
|
|
@@ -10,6 +10,7 @@ module Spout
|
|
10
10
|
class Importer
|
11
11
|
def initialize(argv)
|
12
12
|
use_domains = !argv.delete('--domains').nil?
|
13
|
+
use_forms = !argv.delete('--forms').nil?
|
13
14
|
@csv_file = argv[1].to_s
|
14
15
|
unless File.exist?(@csv_file)
|
15
16
|
puts csv_usage
|
@@ -17,6 +18,8 @@ module Spout
|
|
17
18
|
end
|
18
19
|
if use_domains
|
19
20
|
import_domains
|
21
|
+
elsif use_forms
|
22
|
+
import_forms
|
20
23
|
else
|
21
24
|
import_variables
|
22
25
|
end
|
@@ -62,7 +65,7 @@ EOT
|
|
62
65
|
hash['forms'] = forms unless forms.empty?
|
63
66
|
hash['other'] = row unless row.empty?
|
64
67
|
|
65
|
-
file_name = File.join(folder, id
|
68
|
+
file_name = File.join(folder, "#{id}.json")
|
66
69
|
File.open(file_name, 'w') do |file|
|
67
70
|
file.write(JSON.pretty_generate(hash) + "\n")
|
68
71
|
end
|
@@ -117,11 +120,45 @@ EOT
|
|
117
120
|
end
|
118
121
|
end
|
119
122
|
|
123
|
+
def import_forms
|
124
|
+
CSV.parse(File.open(@csv_file, 'r:iso-8859-1:utf-8', &:read), headers: true) do |line|
|
125
|
+
row = line.to_hash
|
126
|
+
unless row.keys.include?('id')
|
127
|
+
puts "\nMissing column header `".colorize(:red) +
|
128
|
+
'id'.colorize(:light_cyan) +
|
129
|
+
'` in data dictionary.'.colorize(:red) +
|
130
|
+
additional_csv_info
|
131
|
+
exit(1)
|
132
|
+
end
|
133
|
+
unless row.keys.include?('display_name')
|
134
|
+
puts "\nMissing column header `".colorize(:red) +
|
135
|
+
'display_name'.colorize(:light_cyan) +
|
136
|
+
'` in data dictionary.'.colorize(:red) +
|
137
|
+
additional_csv_info
|
138
|
+
exit(1)
|
139
|
+
end
|
140
|
+
next if row['id'] == ''
|
141
|
+
folder = File.join('forms', row.delete('folder').to_s)
|
142
|
+
FileUtils.mkpath folder
|
143
|
+
hash = {}
|
144
|
+
id = row.delete('id').to_s.downcase
|
145
|
+
hash['id'] = id
|
146
|
+
hash['display_name'] = tenderize(row.delete('display_name').to_s)
|
147
|
+
hash['code_book'] = row.delete('code_book').to_s
|
148
|
+
hash['other'] = row unless row.empty?
|
149
|
+
file_name = File.join(folder, "#{id}.json")
|
150
|
+
File.open(file_name, 'w') do |file|
|
151
|
+
file.write(JSON.pretty_generate(hash) + "\n")
|
152
|
+
end
|
153
|
+
puts ' create'.colorize(:green) + " #{file_name}"
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
120
157
|
# Converts ALL-CAPS display names to title case
|
121
158
|
# Ex: BODY MASS INDEX changes to Body Mass Index
|
122
159
|
# Ex: Patient ID stays the same as Patient ID
|
123
160
|
def tenderize(text)
|
124
|
-
if
|
161
|
+
if /[a-z]/ =~ text
|
125
162
|
text
|
126
163
|
else
|
127
164
|
text.downcase.gsub(/\b\w/) { $&.upcase }
|
@@ -131,7 +168,8 @@ EOT
|
|
131
168
|
private
|
132
169
|
|
133
170
|
def additional_csv_info
|
134
|
-
"\n\nFor additional information on specifying CSV column headers before import see:\n\n " +
|
171
|
+
"\n\nFor additional information on specifying CSV column headers before import see:\n\n " +
|
172
|
+
"https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize(:light_cyan) + "\n\n"
|
135
173
|
end
|
136
174
|
end
|
137
175
|
end
|
@@ -1,12 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'colorize'
|
4
|
+
require 'date'
|
5
|
+
require 'erb'
|
4
6
|
require 'fileutils'
|
5
7
|
|
6
8
|
TEMPLATES_DIRECTORY = File.expand_path('../../templates', __FILE__)
|
7
9
|
|
8
10
|
module Spout
|
9
11
|
module Commands
|
12
|
+
# Generates folder and file structure for a new spout data dictionary.
|
10
13
|
class ProjectGenerator
|
11
14
|
def initialize(argv)
|
12
15
|
generate_folder_structure!(argv)
|
@@ -14,7 +17,8 @@ module Spout
|
|
14
17
|
|
15
18
|
def generate_folder_structure!(argv)
|
16
19
|
skip_gemfile = !argv.delete('--skip-gemfile').nil?
|
17
|
-
@
|
20
|
+
@project_name = argv[1].to_s.strip
|
21
|
+
@full_path = File.join(@project_name)
|
18
22
|
usage = <<-EOT
|
19
23
|
|
20
24
|
Usage: spout new FOLDER
|
@@ -30,9 +34,12 @@ EOT
|
|
30
34
|
copy_file 'gitignore', '.gitignore'
|
31
35
|
copy_file 'ruby-version', '.ruby-version'
|
32
36
|
copy_file 'travis.yml', '.travis.yml'
|
33
|
-
|
37
|
+
evaluate_file 'spout.yml.erb', '.spout.yml'
|
38
|
+
evaluate_file 'CHANGELOG.md.erb', 'CHANGELOG.md'
|
34
39
|
copy_file 'Gemfile'
|
35
40
|
copy_file 'Rakefile'
|
41
|
+
evaluate_file 'README.md.erb', 'README.md'
|
42
|
+
copy_file 'VERSION'
|
36
43
|
directory 'domains'
|
37
44
|
copy_file 'keep', 'domains/.keep'
|
38
45
|
directory 'variables'
|
@@ -42,11 +49,10 @@ EOT
|
|
42
49
|
directory 'test'
|
43
50
|
copy_file 'test/dictionary_test.rb'
|
44
51
|
copy_file 'test/test_helper.rb'
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
end
|
52
|
+
return if skip_gemfile
|
53
|
+
puts ' run'.colorize(:green) + ' bundle install'.colorize(:light_cyan)
|
54
|
+
Dir.chdir(@full_path)
|
55
|
+
system 'bundle install'
|
50
56
|
end
|
51
57
|
|
52
58
|
private
|
@@ -55,16 +61,26 @@ EOT
|
|
55
61
|
file_name = template_file if file_name == ''
|
56
62
|
file_path = File.join(@full_path, file_name)
|
57
63
|
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
58
|
-
puts
|
64
|
+
puts ' create'.colorize(:green) + " #{file_name}"
|
59
65
|
FileUtils.copy(template_file_path, file_path)
|
60
66
|
end
|
61
67
|
|
68
|
+
def evaluate_file(template_file, file_name)
|
69
|
+
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
70
|
+
template = ERB.new(File.read(template_file_path))
|
71
|
+
file_path = File.join(@full_path, file_name)
|
72
|
+
file_out = File.new(file_path, 'w')
|
73
|
+
file_out.syswrite(template.result(binding))
|
74
|
+
puts ' create'.colorize(:green) + " #{file_name}"
|
75
|
+
ensure
|
76
|
+
file_out.close if file_out
|
77
|
+
end
|
78
|
+
|
62
79
|
def directory(directory_name)
|
63
80
|
directory_path = File.join(@full_path, directory_name)
|
64
|
-
puts
|
81
|
+
puts ' create'.colorize(:green) + " #{directory_name}"
|
65
82
|
FileUtils.mkpath(directory_path)
|
66
83
|
end
|
67
|
-
|
68
84
|
end
|
69
85
|
end
|
70
86
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'colorize'
|
4
|
+
require 'spout/helpers/json_request'
|
5
|
+
|
6
|
+
module Spout
|
7
|
+
module Commands
|
8
|
+
# Command to check if there is an updated version of the gem available.
|
9
|
+
class Update
|
10
|
+
class << self
|
11
|
+
def start(*args)
|
12
|
+
new(*args).start
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(argv)
|
17
|
+
end
|
18
|
+
|
19
|
+
def start
|
20
|
+
(json, _status) = Spout::Helpers::JsonRequest.get('https://rubygems.org/api/v1/gems/spout.json')
|
21
|
+
if json
|
22
|
+
if json['version'] == Spout::VERSION::STRING
|
23
|
+
puts 'The spout gem is ' + 'up-to-date'.colorize(:green) + '!'
|
24
|
+
else
|
25
|
+
puts
|
26
|
+
puts "A newer version (v#{json['version']}) is available! Type the following command to update:"
|
27
|
+
puts
|
28
|
+
puts ' gem install spout --no-document'.colorize(:white)
|
29
|
+
puts
|
30
|
+
end
|
31
|
+
else
|
32
|
+
puts 'Unable to connect to RubyGems.org. Please try again later.'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -3,42 +3,89 @@
|
|
3
3
|
require 'openssl'
|
4
4
|
require 'net/http'
|
5
5
|
require 'json'
|
6
|
+
require 'cgi'
|
6
7
|
|
7
|
-
# TODO: Deprecated, use JsonRequestGeneric instead
|
8
8
|
module Spout
|
9
9
|
module Helpers
|
10
|
+
# Generates JSON web requests for GET, POST, and PATCH.
|
10
11
|
class JsonRequest
|
11
12
|
class << self
|
12
|
-
def get(*args)
|
13
|
-
new(*args).get
|
13
|
+
def get(url, *args)
|
14
|
+
new(url, *args).get
|
15
|
+
end
|
16
|
+
|
17
|
+
def post(url, *args)
|
18
|
+
new(url, *args).post
|
19
|
+
end
|
20
|
+
|
21
|
+
def patch(url, *args)
|
22
|
+
new(url, *args).patch
|
14
23
|
end
|
15
24
|
end
|
16
25
|
|
17
26
|
attr_reader :url
|
18
27
|
|
19
|
-
def initialize(url)
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
rescue
|
28
|
+
def initialize(url, args = {})
|
29
|
+
@params = nested_hash_to_params(args)
|
30
|
+
@url = URI.parse(url)
|
31
|
+
|
32
|
+
@http = Net::HTTP.new(@url.host, @url.port)
|
33
|
+
if @url.scheme == 'https'
|
34
|
+
@http.use_ssl = true
|
35
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
28
36
|
end
|
37
|
+
rescue
|
38
|
+
@error = "Invalid URL: #{url.inspect}"
|
39
|
+
puts @error.colorize(:red)
|
29
40
|
end
|
30
41
|
|
31
42
|
def get
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
43
|
+
return unless @error.nil?
|
44
|
+
full_path = @url.path
|
45
|
+
query = ([@url.query] + @params).flatten.compact.join('&')
|
46
|
+
full_path += "?#{query}" if query.to_s != ''
|
47
|
+
response = @http.start do |http|
|
48
|
+
http.get(full_path)
|
49
|
+
end
|
50
|
+
[JSON.parse(response.body), response]
|
51
|
+
rescue => e
|
52
|
+
puts "GET Error: #{e}".colorize(:red)
|
53
|
+
end
|
54
|
+
|
55
|
+
def post
|
56
|
+
return unless @error.nil?
|
57
|
+
response = @http.start do |http|
|
58
|
+
http.post(@url.path, @params.flatten.compact.join('&'))
|
59
|
+
end
|
60
|
+
[JSON.parse(response.body), response]
|
61
|
+
rescue => e
|
62
|
+
puts "POST ERROR: #{e}".colorize(:red)
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def patch
|
67
|
+
@params << '_method=patch'
|
68
|
+
post
|
69
|
+
end
|
70
|
+
|
71
|
+
def nested_hash_to_params(args)
|
72
|
+
args.collect do |key, value|
|
73
|
+
key_value_to_string(key, value, nil)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def key_value_to_string(key, value, scope = nil)
|
78
|
+
current_scope = (scope ? "#{scope}[#{key}]" : key)
|
79
|
+
if value.is_a? Hash
|
80
|
+
value.collect do |k,v|
|
81
|
+
key_value_to_string(k, v, current_scope)
|
82
|
+
end.join('&')
|
83
|
+
elsif value.is_a? Array
|
84
|
+
value.collect do |v|
|
85
|
+
key_value_to_string('', v, current_scope)
|
38
86
|
end
|
39
|
-
|
40
|
-
|
41
|
-
nil
|
87
|
+
else
|
88
|
+
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
42
89
|
end
|
43
90
|
end
|
44
91
|
end
|
@@ -126,7 +126,7 @@ module Spout
|
|
126
126
|
@variable_files.each do |variable_file|
|
127
127
|
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
128
128
|
next unless json
|
129
|
-
next unless ['choices'
|
129
|
+
next unless json['type'] == 'choices' || json['domain'].to_s.downcase.strip != ''
|
130
130
|
domain = json['domain'].to_s.downcase
|
131
131
|
@all_domains << domain
|
132
132
|
end
|
@@ -4,8 +4,11 @@ require 'spout/tests/variable_type_validation'
|
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Models
|
7
|
+
# Contains the coverage of a specific variable.
|
7
8
|
class CoverageResult
|
8
|
-
attr_accessor :error, :error_message, :file_name_test, :json_id_test,
|
9
|
+
attr_accessor :error, :error_message, :file_name_test, :json_id_test,
|
10
|
+
:values_test, :valid_values, :csv_values,
|
11
|
+
:variable_type_test, :json, :domain_test
|
9
12
|
|
10
13
|
def initialize(column, csv_values)
|
11
14
|
load_json(column)
|
@@ -26,10 +29,10 @@ module Spout
|
|
26
29
|
|
27
30
|
def load_valid_values
|
28
31
|
valid_values = []
|
29
|
-
if @json['type'] == 'choices'
|
32
|
+
if @json['type'] == 'choices' || domain_name != ''
|
30
33
|
file = Dir.glob("domains/**/#{@json['domain'].to_s.downcase}.json", File::FNM_CASEFOLD).first
|
31
34
|
if json = JSON.parse(File.read(file)) rescue false
|
32
|
-
valid_values = json.collect{|hash| hash['value']}
|
35
|
+
valid_values = json.collect { |hash| hash['value'] }
|
33
36
|
end
|
34
37
|
end
|
35
38
|
@valid_values = valid_values
|
@@ -48,7 +51,7 @@ module Spout
|
|
48
51
|
end
|
49
52
|
|
50
53
|
def check_domain_specified
|
51
|
-
if @json['type'] != 'choices'
|
54
|
+
if @json['type'] != 'choices' && domain_name == ''
|
52
55
|
true
|
53
56
|
else
|
54
57
|
domain_file = Dir.glob("domains/**/#{@json['domain'].to_s.downcase}.json", File::FNM_CASEFOLD).first
|
@@ -62,6 +65,10 @@ module Spout
|
|
62
65
|
def errored?
|
63
66
|
error == true
|
64
67
|
end
|
68
|
+
|
69
|
+
def domain_name
|
70
|
+
@json['domain'].to_s.downcase.strip
|
71
|
+
end
|
65
72
|
end
|
66
73
|
end
|
67
74
|
end
|
data/lib/spout/templates/Gemfile
CHANGED
@@ -0,0 +1,46 @@
|
|
1
|
+
<%= @project_name.capitalize %> Data Dictionary
|
2
|
+
======================
|
3
|
+
|
4
|
+
[](https://travis-ci.org/<REPOSITORY>/<%= @project_name.downcase %>-data-dictionary)
|
5
|
+
|
6
|
+
### Exports
|
7
|
+
|
8
|
+
The <%= @project_name.downcase %> data dictionary can be exported to CSV by typing:
|
9
|
+
|
10
|
+
```
|
11
|
+
spout export
|
12
|
+
```
|
13
|
+
|
14
|
+
The `spout export` command will generate CSV files that describe the data
|
15
|
+
dictionary.
|
16
|
+
|
17
|
+
|
18
|
+
### Testing
|
19
|
+
|
20
|
+
The <%= @project_name.downcase %> data dictionary is tested using the
|
21
|
+
[Spout Gem](https://github.com/sleepepi/spout).
|
22
|
+
|
23
|
+
Data dictionary tests can be run by typing:
|
24
|
+
|
25
|
+
```
|
26
|
+
spout test
|
27
|
+
```
|
28
|
+
|
29
|
+
|
30
|
+
### Releases
|
31
|
+
|
32
|
+
The data dictionary is tagged at various time points using
|
33
|
+
[Git tags](http://git-scm.com/book/en/Git-Basics-Tagging). The tags are used to
|
34
|
+
reference a series of CSV files that correspond to the data dictionary itself.
|
35
|
+
|
36
|
+
For example, CSV files of the underlying data that have been tagged as `v0.1.0`
|
37
|
+
will be compatible with the Data Dictionary `~> 0.1.0`,
|
38
|
+
(including `0.1.1`, `0.1.2`, `0.1.3`). However if the data dictionary contains
|
39
|
+
changes to the underlying dataset, then the minor version number is bumped, and
|
40
|
+
the patch level is reset to zero. If, for example, the CSV dataset changed to
|
41
|
+
`v0.2.0`, then it would be compatible with `0.2.0`, `0.2.1`, `0.2.2`, etc. The
|
42
|
+
approach for changing version numbers uses a variation on
|
43
|
+
[Semantic Versioning](http://semver.org).
|
44
|
+
|
45
|
+
A full list of changes for each version can be viewed in the
|
46
|
+
[CHANGELOG](https://github.com/<REPOSITORY>/<%= @project_name.downcase %>-data-dictionary/blob/master/CHANGELOG.md).
|
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -15,8 +15,9 @@ class DictionaryTest < Minitest::Test
|
|
15
15
|
# VALID_UNITS = ['minutes', 'hours'] # Add your own valid units to this array
|
16
16
|
# @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
|
17
17
|
# define_method("test_units: #{variable.path}") do
|
18
|
-
# message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n
|
19
|
-
#
|
18
|
+
# message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
|
19
|
+
# " Valid types: " +
|
20
|
+
# VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(', ')
|
20
21
|
# assert VALID_UNITS.include?(variable.units), message
|
21
22
|
# end
|
22
23
|
# end
|
@@ -2,32 +2,31 @@
|
|
2
2
|
|
3
3
|
module Spout
|
4
4
|
module Tests
|
5
|
+
# If a variable references a domain, then the domain should exist and be
|
6
|
+
# defined.
|
5
7
|
module DomainExistenceValidation
|
6
|
-
|
7
8
|
def assert_domain_existence(item)
|
8
|
-
domain_names = Dir.glob(
|
9
|
-
|
9
|
+
domain_names = Dir.glob('domains/**/*.json').collect do |file|
|
10
|
+
file.split('/').last.to_s.downcase.split('.json').first
|
11
|
+
end
|
10
12
|
result = begin
|
11
|
-
domain_name = JSON.parse(File.read(item))[
|
13
|
+
domain_name = JSON.parse(File.read(item))['domain']
|
12
14
|
domain_names.include?(domain_name)
|
13
15
|
rescue JSON::ParserError
|
14
16
|
domain_name = ''
|
15
17
|
false
|
16
18
|
end
|
17
|
-
|
18
19
|
message = "The domain #{domain_name} referenced by #{item} does not exist."
|
19
|
-
|
20
20
|
assert result, message
|
21
21
|
end
|
22
22
|
|
23
|
-
Dir.glob(
|
24
|
-
if (not [nil, ''].include?(JSON.parse(File.read(file))[
|
25
|
-
define_method("test_domain_exists: "
|
23
|
+
Dir.glob('variables/**/*.json').each do |file|
|
24
|
+
if (not [nil, ''].include?(JSON.parse(File.read(file))['domain']) rescue false)
|
25
|
+
define_method("test_domain_exists: #{file}") do
|
26
26
|
assert_domain_existence file
|
27
27
|
end
|
28
28
|
end
|
29
29
|
end
|
30
|
-
|
31
30
|
end
|
32
31
|
end
|
33
32
|
end
|
@@ -3,30 +3,30 @@
|
|
3
3
|
module Spout
|
4
4
|
module Tests
|
5
5
|
module DomainFormat
|
6
|
-
|
6
|
+
# Verifies the format of a domain.
|
7
7
|
def assert_domain_format(item)
|
8
8
|
result = begin
|
9
9
|
json = JSON.parse(File.read(item))
|
10
10
|
if json.is_a?(Array)
|
11
|
-
json.empty?
|
11
|
+
json.empty? || json.select { |o| !o.is_a?(Hash) }.empty?
|
12
12
|
else
|
13
13
|
false
|
14
14
|
end
|
15
15
|
rescue JSON::ParserError
|
16
16
|
false
|
17
17
|
end
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
message = \
|
19
|
+
"Must be an array of choice hashes. Ex:\n[\n {\n \"value\": "\
|
20
|
+
" \"1\",\n \"display_name\": \"Option 1\",\n \"description\""\
|
21
|
+
": \"...\"\n },\n { ... },\n ...\n]"
|
21
22
|
assert result, message
|
22
23
|
end
|
23
24
|
|
24
|
-
Dir.glob(
|
25
|
-
define_method("test_domain_format: "
|
25
|
+
Dir.glob('domains/**/*.json').each do |file|
|
26
|
+
define_method("test_domain_format: #{file}") do
|
26
27
|
assert_domain_format file
|
27
28
|
end
|
28
29
|
end
|
29
|
-
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
data/lib/spout/version.rb
CHANGED
@@ -102,7 +102,7 @@ tfoot td {
|
|
102
102
|
<tr>
|
103
103
|
<td class="text-muted">Variables Not Found in Any CSV</td>
|
104
104
|
<td></td>
|
105
|
-
<td><code class="default"><%= number_with_delimiter(
|
105
|
+
<td><code class="default"><%= number_with_delimiter(@extra_variable_ids.size) %></code></td>
|
106
106
|
<td></td>
|
107
107
|
<td></td>
|
108
108
|
</tr>
|
@@ -111,7 +111,7 @@ tfoot td {
|
|
111
111
|
<tr>
|
112
112
|
<td class="text-muted">Domains Not Referenced by Any Variable</td>
|
113
113
|
<td></td>
|
114
|
-
<td><code class="default"><%= number_with_delimiter(
|
114
|
+
<td><code class="default"><%= number_with_delimiter(@extra_domain_ids.size) %></code></td>
|
115
115
|
<td></td>
|
116
116
|
<td></td>
|
117
117
|
</tr>
|
@@ -126,9 +126,9 @@ tfoot td {
|
|
126
126
|
<span class="text-muted">---</span>
|
127
127
|
<% end %>
|
128
128
|
</td>
|
129
|
-
<td><code class="default"><%= number_with_delimiter(
|
130
|
-
<td><code class="success"><%= number_with_delimiter(
|
131
|
-
<td><code><%= number_with_delimiter(
|
129
|
+
<td><code class="default"><%= number_with_delimiter(total_column_count) %></code></td>
|
130
|
+
<td><code class="success"><%= number_with_delimiter(mapped_column_count) %></code></td>
|
131
|
+
<td><code><%= number_with_delimiter(total_column_count - mapped_column_count) %></code></td>
|
132
132
|
</tr>
|
133
133
|
<% end %>
|
134
134
|
</tbody>
|
@@ -177,7 +177,7 @@ tfoot td {
|
|
177
177
|
<% matched = @matching_results.count{|csv_files, column, scr| scr.file_name_test} %>
|
178
178
|
<% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
|
179
179
|
<% missing_percent = 100 - matched_percent %>
|
180
|
-
<%= number_with_delimiter(
|
180
|
+
<%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
|
181
181
|
<div class="progress progress-striped">
|
182
182
|
<div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
|
183
183
|
<div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
|
@@ -187,7 +187,7 @@ tfoot td {
|
|
187
187
|
<% matched = @matching_results.count{|csv_files, column, scr| scr.json_id_test} %>
|
188
188
|
<% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
|
189
189
|
<% missing_percent = 100 - matched_percent %>
|
190
|
-
<%= number_with_delimiter(
|
190
|
+
<%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
|
191
191
|
<div class="progress progress-striped">
|
192
192
|
<div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
|
193
193
|
<div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
|
@@ -197,7 +197,7 @@ tfoot td {
|
|
197
197
|
<% matched = @matching_results.count{|csv_files, column, scr| scr.variable_type_test} %>
|
198
198
|
<% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
|
199
199
|
<% missing_percent = 100 - matched_percent %>
|
200
|
-
<%= number_with_delimiter(
|
200
|
+
<%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
|
201
201
|
<div class="progress progress-striped">
|
202
202
|
<div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
|
203
203
|
<div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
|
@@ -208,7 +208,7 @@ tfoot td {
|
|
208
208
|
<% total_count = @matching_results.count{|csv_files, column, scr| scr.json['type'] == 'choices'} %>
|
209
209
|
<% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
|
210
210
|
<% missing_percent = 100 - matched_percent %>
|
211
|
-
<%= number_with_delimiter(
|
211
|
+
<%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
|
212
212
|
<div class="progress progress-striped">
|
213
213
|
<div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
|
214
214
|
<div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
|
@@ -262,8 +262,8 @@ tfoot td {
|
|
262
262
|
<% end %>
|
263
263
|
</td>
|
264
264
|
<td>
|
265
|
-
<% if scr.json['type'] == 'choices' && scr.file_name_test %>
|
266
|
-
<% if scr.domain_test
|
265
|
+
<% if (scr.json['type'] == 'choices' || scr.json['domain'].to_s.downcase.strip != '') && scr.file_name_test %>
|
266
|
+
<% if scr.domain_test || scr.json['domain'].to_s.strip == '' %>
|
267
267
|
<code class="<%= 'success' if scr.domain_test %>">"domain": <%= scr.json['domain'].inspect %></code>
|
268
268
|
<% else %>
|
269
269
|
<span class="text-danger"><code><%= scr.json['domain'] %>.json</code> missing</span>
|
@@ -271,15 +271,18 @@ tfoot td {
|
|
271
271
|
<% end %>
|
272
272
|
</td>
|
273
273
|
<td style="white-space:nowrap">
|
274
|
-
<% if scr.json['type'] == 'choices' %>
|
275
|
-
<%
|
276
|
-
<%
|
274
|
+
<% if scr.json['type'] == 'choices' || scr.json['domain'].to_s.downcase.strip != '' %>
|
275
|
+
<% unused_domain_values = scr.valid_values - scr.csv_values %>
|
276
|
+
<% unused_values = (scr.valid_values | scr.csv_values) - (scr.valid_values & scr.csv_values) %>
|
277
|
+
<% if scr.values_test && unused_values.empty? %>
|
277
278
|
<div class="text-success" style="text-align:center"><span class="glyphicon glyphicon-ok"></span></div>
|
278
279
|
<% else %>
|
279
280
|
<% (scr.valid_values + scr.csv_values.compact.sort).uniq.each do |value| %>
|
281
|
+
<% value_as_number = format('%g', value) rescue value_as_number = nil %>
|
282
|
+
<% next if !scr.valid_values.include?(value) && %w(numeric integer).include?(scr.json['type']) && !value_as_number.nil? %>
|
280
283
|
<% class_type = '' %>
|
281
284
|
<% class_type = 'success' if scr.valid_values.include?(value) %>
|
282
|
-
<% class_type = 'default' if
|
285
|
+
<% class_type = 'default' if unused_domain_values.include?(value) %>
|
283
286
|
<code class="<%= class_type %>"><%= value %></code>
|
284
287
|
<% end %>
|
285
288
|
<% end %>
|
data/lib/spout.rb
CHANGED
@@ -5,17 +5,19 @@ require 'spout/version'
|
|
5
5
|
require 'spout/models/dictionary'
|
6
6
|
|
7
7
|
Spout::COMMANDS = {
|
8
|
-
'n' => :new_project,
|
9
|
-
'v' => :version,
|
10
|
-
't' => :test,
|
11
|
-
'i' => :importer,
|
12
|
-
'e' => :exporter,
|
13
8
|
'c' => :coverage_report,
|
9
|
+
'd' => :deploy,
|
10
|
+
'e' => :exporter,
|
14
11
|
'g' => :generate_charts_and_tables,
|
12
|
+
'i' => :importer,
|
13
|
+
'n' => :new_project,
|
15
14
|
'o' => :outliers_report,
|
16
|
-
'
|
15
|
+
't' => :test,
|
16
|
+
'u' => :update,
|
17
|
+
'v' => :version
|
17
18
|
}
|
18
19
|
|
20
|
+
# Launch spout commands from command line.
|
19
21
|
module Spout
|
20
22
|
def self.launch(argv)
|
21
23
|
send((Spout::COMMANDS[argv.first.to_s.scan(/\w/).first] || :help), argv)
|
@@ -70,6 +72,11 @@ module Spout
|
|
70
72
|
# Spout::Commands::TestRunner.new(argv)
|
71
73
|
end
|
72
74
|
|
75
|
+
def self.update(argv)
|
76
|
+
require 'spout/commands/update'
|
77
|
+
Spout::Commands::Update.start(argv)
|
78
|
+
end
|
79
|
+
|
73
80
|
def self.version(_argv)
|
74
81
|
puts "Spout #{Spout::VERSION::STRING}"
|
75
82
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spout
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.0.
|
4
|
+
version: 0.12.0.rc
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Remo Mueller
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03-
|
11
|
+
date: 2017-03-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -117,13 +117,13 @@ files:
|
|
117
117
|
- lib/spout/commands/importer.rb
|
118
118
|
- lib/spout/commands/outliers.rb
|
119
119
|
- lib/spout/commands/project_generator.rb
|
120
|
+
- lib/spout/commands/update.rb
|
120
121
|
- lib/spout/helpers/array_statistics.rb
|
121
122
|
- lib/spout/helpers/chart_types.rb
|
122
123
|
- lib/spout/helpers/config_reader.rb
|
123
124
|
- lib/spout/helpers/iterators.rb
|
124
125
|
- lib/spout/helpers/json_loader.rb
|
125
126
|
- lib/spout/helpers/json_request.rb
|
126
|
-
- lib/spout/helpers/json_request_generic.rb
|
127
127
|
- lib/spout/helpers/number_helper.rb
|
128
128
|
- lib/spout/helpers/quietly.rb
|
129
129
|
- lib/spout/helpers/semantic.rb
|
@@ -156,12 +156,15 @@ files:
|
|
156
156
|
- lib/spout/models/variable.rb
|
157
157
|
- lib/spout/tasks.rb
|
158
158
|
- lib/spout/tasks/engine.rake
|
159
|
+
- lib/spout/templates/CHANGELOG.md.erb
|
159
160
|
- lib/spout/templates/Gemfile
|
161
|
+
- lib/spout/templates/README.md.erb
|
160
162
|
- lib/spout/templates/Rakefile
|
163
|
+
- lib/spout/templates/VERSION
|
161
164
|
- lib/spout/templates/gitignore
|
162
165
|
- lib/spout/templates/keep
|
163
166
|
- lib/spout/templates/ruby-version
|
164
|
-
- lib/spout/templates/spout.yml
|
167
|
+
- lib/spout/templates/spout.yml.erb
|
165
168
|
- lib/spout/templates/test/dictionary_test.rb
|
166
169
|
- lib/spout/templates/test/test_helper.rb
|
167
170
|
- lib/spout/templates/travis.yml
|
@@ -1,89 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'openssl'
|
4
|
-
require 'net/http'
|
5
|
-
require 'json'
|
6
|
-
require 'cgi'
|
7
|
-
|
8
|
-
module Spout
|
9
|
-
module Helpers
|
10
|
-
class JsonRequestGeneric
|
11
|
-
class << self
|
12
|
-
def get(url, *args)
|
13
|
-
new(url, *args).get
|
14
|
-
end
|
15
|
-
|
16
|
-
def post(url, *args)
|
17
|
-
new(url, *args).post
|
18
|
-
end
|
19
|
-
|
20
|
-
def patch(url, *args)
|
21
|
-
new(url, *args).patch
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
attr_reader :url
|
26
|
-
|
27
|
-
def initialize(url, args = {})
|
28
|
-
@params = nested_hash_to_params(args)
|
29
|
-
@url = URI.parse(url)
|
30
|
-
|
31
|
-
@http = Net::HTTP.new(@url.host, @url.port)
|
32
|
-
if @url.scheme == 'https'
|
33
|
-
@http.use_ssl = true
|
34
|
-
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
35
|
-
end
|
36
|
-
rescue => e
|
37
|
-
puts "Error sending JsonRequestGeneric: #{e}".colorize(:red)
|
38
|
-
end
|
39
|
-
|
40
|
-
def get
|
41
|
-
full_path = @url.path
|
42
|
-
query = ([@url.query] + @params).flatten.compact.join('&')
|
43
|
-
full_path += "?#{query}" if query.to_s != ''
|
44
|
-
response = @http.start do |http|
|
45
|
-
http.get(full_path)
|
46
|
-
end
|
47
|
-
[JSON.parse(response.body), response]
|
48
|
-
rescue => e
|
49
|
-
puts "GET Error: #{e}".colorize(:red)
|
50
|
-
end
|
51
|
-
|
52
|
-
def post
|
53
|
-
response = @http.start do |http|
|
54
|
-
http.post(@url.path, @params.flatten.compact.join('&'))
|
55
|
-
end
|
56
|
-
[JSON.parse(response.body), response]
|
57
|
-
rescue => e
|
58
|
-
puts "POST ERROR: #{e}".colorize(:red)
|
59
|
-
nil
|
60
|
-
end
|
61
|
-
|
62
|
-
def patch
|
63
|
-
@params << '_method=patch'
|
64
|
-
post
|
65
|
-
end
|
66
|
-
|
67
|
-
def nested_hash_to_params(args)
|
68
|
-
args.collect do |key, value|
|
69
|
-
key_value_to_string(key, value, nil)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def key_value_to_string(key, value, scope = nil)
|
74
|
-
current_scope = (scope ? "#{scope}[#{key}]" : key)
|
75
|
-
if value.is_a? Hash
|
76
|
-
value.collect do |k,v|
|
77
|
-
key_value_to_string(k, v, current_scope)
|
78
|
-
end.join('&')
|
79
|
-
elsif value.is_a? Array
|
80
|
-
value.collect do |v|
|
81
|
-
key_value_to_string('', v, current_scope)
|
82
|
-
end
|
83
|
-
else
|
84
|
-
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|