spout 0.12.1 → 0.13.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +5 -5
- data/Rakefile +4 -4
- data/bin/spout +1 -1
- data/lib/spout/commands/coverage.rb +18 -18
- data/lib/spout/commands/deploy.rb +89 -89
- data/lib/spout/commands/exporter.rb +16 -16
- data/lib/spout/commands/graphs.rb +37 -37
- data/lib/spout/commands/help.rb +1 -1
- data/lib/spout/commands/importer.rb +59 -57
- data/lib/spout/commands/outliers.rb +17 -17
- data/lib/spout/commands/project_generator.rb +25 -25
- data/lib/spout/commands/update.rb +38 -38
- data/lib/spout/helpers/array_statistics.rb +7 -7
- data/lib/spout/helpers/chart_types.rb +2 -2
- data/lib/spout/helpers/config_reader.rb +21 -20
- data/lib/spout/helpers/framework.rb +11 -11
- data/lib/spout/helpers/iterators.rb +1 -3
- data/lib/spout/helpers/json_loader.rb +2 -4
- data/lib/spout/helpers/json_request.rb +11 -11
- data/lib/spout/helpers/number_helper.rb +1 -1
- data/lib/spout/helpers/quietly.rb +1 -1
- data/lib/spout/helpers/semantic.rb +1 -1
- data/lib/spout/helpers/send_file.rb +14 -14
- data/lib/spout/helpers/subject_loader.rb +24 -24
- data/lib/spout/helpers/table_formatting.rb +12 -12
- data/lib/spout/models/coverage_result.rb +8 -8
- data/lib/spout/models/dictionary.rb +10 -10
- data/lib/spout/models/domain.rb +7 -7
- data/lib/spout/models/empty.rb +1 -1
- data/lib/spout/models/form.rb +6 -6
- data/lib/spout/models/graphables/choices_vs_choices.rb +3 -5
- data/lib/spout/models/graphables/choices_vs_numeric.rb +4 -6
- data/lib/spout/models/graphables/default.rb +6 -6
- data/lib/spout/models/graphables/histogram.rb +4 -4
- data/lib/spout/models/graphables/numeric_vs_choices.rb +1 -1
- data/lib/spout/models/graphables/numeric_vs_numeric.rb +6 -6
- data/lib/spout/models/graphables.rb +14 -14
- data/lib/spout/models/outlier_result.rb +16 -18
- data/lib/spout/models/record.rb +3 -3
- data/lib/spout/models/tables/choices_vs_choices.rb +11 -11
- data/lib/spout/models/tables/choices_vs_numeric.rb +8 -8
- data/lib/spout/models/tables/default.rb +5 -5
- data/lib/spout/models/tables/numeric_vs_choices.rb +7 -7
- data/lib/spout/models/tables/numeric_vs_numeric.rb +7 -7
- data/lib/spout/models/tables.rb +11 -11
- data/lib/spout/models/variable.rb +14 -14
- data/lib/spout/tasks/engine.rake +4 -4
- data/lib/spout/templates/CHANGELOG.md.erb +1 -1
- data/lib/spout/templates/Gemfile +2 -2
- data/lib/spout/templates/Rakefile +1 -1
- data/lib/spout/templates/test/dictionary_test.rb +3 -3
- data/lib/spout/templates/test/test_helper.rb +1 -1
- data/lib/spout/tests/domain_existence_validation.rb +6 -6
- data/lib/spout/tests/domain_format.rb +1 -1
- data/lib/spout/tests/domain_name_format.rb +3 -3
- data/lib/spout/tests/domain_name_uniqueness.rb +2 -2
- data/lib/spout/tests/domain_specified.rb +1 -3
- data/lib/spout/tests/form_existence_validation.rb +2 -4
- data/lib/spout/tests/form_name_format.rb +3 -3
- data/lib/spout/tests/form_name_match.rb +1 -1
- data/lib/spout/tests/json_helper.rb +1 -1
- data/lib/spout/tests/json_validation.rb +0 -2
- data/lib/spout/tests/variable_name_format.rb +3 -3
- data/lib/spout/tests/variable_name_match.rb +1 -3
- data/lib/spout/tests/variable_name_uniqueness.rb +2 -2
- data/lib/spout/tests/variable_type_validation.rb +2 -2
- data/lib/spout/tests.rb +30 -30
- data/lib/spout/version.rb +4 -4
- data/lib/spout/views/index.html.erb +15 -15
- data/lib/spout/views/outliers.html.erb +8 -8
- data/lib/spout.rb +26 -26
- data/spout.gemspec +22 -22
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 415a16b52a348d3d3c7361b6ea226e7d7aa10edc
|
4
|
+
data.tar.gz: '08bbf0c3477b4689a0cc4dbacc52750ec53a7eb2'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 97b50714ade02df37afd4fac776bbd5d4515db7e628b5acee6639d1f711c025cf6ed6c5e6bd2b9f0c0826061487adc85c44d44683520556bd3d12e43ac2a7d6f
|
7
|
+
data.tar.gz: 26a33f2ffdeb525e047a10efc87324d8a935a5bb47f60780ffff55237f56f39dbc70b4530abd871fc29851a2171ff38d35ac64c7c24c6bd7c038af4b7efdec2d
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -16,7 +16,7 @@ Spout has been used extensively to curate and clean datasets available on the
|
|
16
16
|
|
17
17
|
Add this line to your application's Gemfile:
|
18
18
|
|
19
|
-
gem
|
19
|
+
gem "spout"
|
20
20
|
|
21
21
|
And then execute:
|
22
22
|
|
@@ -143,7 +143,7 @@ tests, or just a subset of Spout tests.
|
|
143
143
|
`test/dictionary_test.rb`
|
144
144
|
|
145
145
|
```ruby
|
146
|
-
require
|
146
|
+
require "spout/tests"
|
147
147
|
|
148
148
|
class DictionaryTest < Minitest::Test
|
149
149
|
# This line includes all default Spout Dictionary tests
|
@@ -152,7 +152,7 @@ end
|
|
152
152
|
```
|
153
153
|
|
154
154
|
```ruby
|
155
|
-
require
|
155
|
+
require "spout/tests"
|
156
156
|
|
157
157
|
class DictionaryTest < Minitest::Test
|
158
158
|
# You can include only certain Spout tests by including them individually
|
@@ -185,13 +185,13 @@ class DictionaryTest < Minitest::Test
|
|
185
185
|
# that can be used to write custom tests.
|
186
186
|
include Spout::Helpers::Iterators
|
187
187
|
|
188
|
-
VALID_UNITS = [
|
188
|
+
VALID_UNITS = ["minutes", "hours"]
|
189
189
|
|
190
190
|
@variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
|
191
191
|
define_method("test_units: #{variable.path}") do
|
192
192
|
message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
|
193
193
|
" Valid types: " +
|
194
|
-
VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(
|
194
|
+
VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(", ")
|
195
195
|
assert VALID_UNITS.include?(variable.units), message
|
196
196
|
end
|
197
197
|
end
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rake/testtask"
|
4
4
|
|
5
5
|
Rake::TestTask.new(:test) do |t|
|
6
|
-
t.libs <<
|
7
|
-
t.libs <<
|
8
|
-
t.pattern =
|
6
|
+
t.libs << "lib"
|
7
|
+
t.libs << "test"
|
8
|
+
t.pattern = "test/**/*_test.rb"
|
9
9
|
t.verbose = false
|
10
10
|
end
|
11
11
|
|
data/bin/spout
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "erb"
|
4
|
+
require "fileutils"
|
5
|
+
require "yaml"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
7
|
+
require "spout/helpers/subject_loader"
|
8
|
+
require "spout/models/coverage_result"
|
9
|
+
require "spout/helpers/number_helper"
|
10
|
+
require "spout/helpers/config_reader"
|
11
|
+
require "spout/helpers/array_statistics"
|
12
12
|
|
13
13
|
module Spout
|
14
14
|
module Commands
|
@@ -18,8 +18,8 @@ module Spout
|
|
18
18
|
|
19
19
|
def initialize(standard_version, argv)
|
20
20
|
@standard_version = standard_version
|
21
|
-
@console = !argv.delete(
|
22
|
-
@variable_files = Dir.glob(
|
21
|
+
@console = !argv.delete("--console").nil?
|
22
|
+
@variable_files = Dir.glob("variables/**/*.json")
|
23
23
|
@valid_ids = []
|
24
24
|
@number_of_rows = nil
|
25
25
|
@config = Spout::Helpers::ConfigReader.new
|
@@ -38,10 +38,10 @@ module Spout
|
|
38
38
|
scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).compact_empty.uniq)
|
39
39
|
@matching_results << [csv_files, method, scr]
|
40
40
|
end
|
41
|
-
variable_ids = Dir.glob(
|
41
|
+
variable_ids = Dir.glob("variables/**/*.json").collect { |file| file.gsub(%r{^(.*)/|\.json$}, "").downcase }
|
42
42
|
@extra_variable_ids = (variable_ids - @subject_loader.all_methods.keys).sort
|
43
43
|
@subject_loader.load_variable_domains!
|
44
|
-
domain_ids = Dir.glob(
|
44
|
+
domain_ids = Dir.glob("domains/**/*.json").collect { |file| file.gsub(%r{^(.*)/|\.json$}, "").downcase }
|
45
45
|
@extra_domain_ids = (domain_ids - @subject_loader.all_domains).sort
|
46
46
|
@matching_results.sort! do |a, b|
|
47
47
|
[b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]
|
@@ -53,16 +53,16 @@ module Spout
|
|
53
53
|
.select { |mr| mr[0].include?(csv_file) && mr[2].number_of_errors.zero? }.count
|
54
54
|
@coverage_results << [csv_file, total_column_count, mapped_column_count]
|
55
55
|
end
|
56
|
-
coverage_folder = File.join(Dir.pwd,
|
56
|
+
coverage_folder = File.join(Dir.pwd, "coverage")
|
57
57
|
FileUtils.mkpath coverage_folder
|
58
|
-
coverage_file = File.join(coverage_folder,
|
59
|
-
File.open(coverage_file,
|
60
|
-
erb_location = File.join(File.dirname(__FILE__),
|
58
|
+
coverage_file = File.join(coverage_folder, "index.html")
|
59
|
+
File.open(coverage_file, "w+") do |file|
|
60
|
+
erb_location = File.join(File.dirname(__FILE__), "../views/index.html.erb")
|
61
61
|
file.puts ERB.new(File.read(erb_location)).result(binding)
|
62
62
|
end
|
63
63
|
unless @console
|
64
|
-
open_command =
|
65
|
-
open_command =
|
64
|
+
open_command = "open" unless RUBY_PLATFORM.match(/darwin/).nil?
|
65
|
+
open_command = "start" unless RUBY_PLATFORM.match(/mingw/).nil?
|
66
66
|
system "#{open_command} #{coverage_file}" if %w(start open).include?(open_command)
|
67
67
|
end
|
68
68
|
puts "#{coverage_file}\n\n"
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "net/http"
|
5
|
+
require "io/console"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
7
|
+
require "spout/helpers/subject_loader"
|
8
|
+
require "spout/helpers/config_reader"
|
9
|
+
require "spout/helpers/quietly"
|
10
|
+
require "spout/helpers/send_file"
|
11
|
+
require "spout/helpers/semantic"
|
12
|
+
require "spout/helpers/json_request"
|
13
13
|
|
14
14
|
# - **User Authorization**
|
15
15
|
# - User authenticates via token, the user must be a dataset editor
|
@@ -41,34 +41,34 @@ module Spout
|
|
41
41
|
include Spout::Helpers::Quietly
|
42
42
|
|
43
43
|
INDENT_LENGTH = 23
|
44
|
-
INDENT =
|
44
|
+
INDENT = " " * INDENT_LENGTH
|
45
45
|
|
46
46
|
attr_accessor :token, :version, :slug, :url, :config, :environment, :webserver_name, :subjects
|
47
47
|
|
48
48
|
def initialize(argv, version)
|
49
|
-
argv.shift # Remove
|
49
|
+
argv.shift # Remove "download" command from argv list
|
50
50
|
@environment = argv.shift
|
51
51
|
@version = version
|
52
|
-
@skip_checks = !(argv.delete(
|
52
|
+
@skip_checks = !(argv.delete("--skip-checks").nil? && argv.delete("--no-checks").nil?)
|
53
53
|
|
54
|
-
@skip_tests = !(argv.delete(
|
55
|
-
@skip_coverage = !(argv.delete(
|
54
|
+
@skip_tests = !(argv.delete("--skip-tests").nil? && argv.delete("--no-tests").nil?)
|
55
|
+
@skip_coverage = !(argv.delete("--skip-coverage").nil? && argv.delete("--no-coverage").nil?)
|
56
56
|
|
57
|
-
@skip_variables = !(argv.delete(
|
58
|
-
@skip_dataset = !(argv.delete(
|
59
|
-
@skip_dictionary = !(argv.delete(
|
60
|
-
@skip_documentation = !(argv.delete(
|
61
|
-
@clean = !(argv.delete(
|
62
|
-
@skip_server_scripts = !(argv.delete(
|
63
|
-
@archive_only = !(argv.delete(
|
57
|
+
@skip_variables = !(argv.delete("--skip-variables").nil? && argv.delete("--no-variables").nil?)
|
58
|
+
@skip_dataset = !(argv.delete("--skip-dataset").nil? && argv.delete("--no-dataset").nil?)
|
59
|
+
@skip_dictionary = !(argv.delete("--skip-dictionary").nil? && argv.delete("--no-dictionary").nil?)
|
60
|
+
@skip_documentation = !(argv.delete("--skip-documentation").nil? && argv.delete("--no-documentation").nil?)
|
61
|
+
@clean = !(argv.delete("--no-resume").nil? && argv.delete("--clean").nil?)
|
62
|
+
@skip_server_scripts = !(argv.delete("--skip-server-scripts").nil? && argv.delete("--no-server-scripts").nil?)
|
63
|
+
@archive_only = !(argv.delete("--archive-only").nil?)
|
64
64
|
|
65
65
|
token_arg = argv.find { |arg| /^--token=/ =~ arg }
|
66
66
|
argv.delete(token_arg)
|
67
|
-
@token = token_arg.gsub(/^--token=/,
|
67
|
+
@token = token_arg.gsub(/^--token=/, "") if token_arg
|
68
68
|
|
69
69
|
rows_arg = argv.find { |arg| /^--rows=(\d*)/ =~ arg }
|
70
70
|
argv.delete(rows_arg)
|
71
|
-
@number_of_rows = rows_arg.gsub(/--rows=/,
|
71
|
+
@number_of_rows = rows_arg.gsub(/--rows=/, "").to_i if rows_arg
|
72
72
|
|
73
73
|
@argv = argv
|
74
74
|
|
@@ -98,12 +98,12 @@ module Spout
|
|
98
98
|
end
|
99
99
|
|
100
100
|
def config_file_load
|
101
|
-
print
|
101
|
+
print " `.spout.yml` Check: "
|
102
102
|
@config = Spout::Helpers::ConfigReader.new
|
103
103
|
|
104
104
|
@slug = @config.slug
|
105
105
|
|
106
|
-
if @slug ==
|
106
|
+
if @slug == ""
|
107
107
|
message = "#{INDENT}Please specify a dataset slug in your `.spout.yml` file!".colorize(:red) + " Ex:\n---\nslug: mydataset\n".colorize(:orange)
|
108
108
|
failure(message)
|
109
109
|
end
|
@@ -113,7 +113,7 @@ module Spout
|
|
113
113
|
failure(message)
|
114
114
|
end
|
115
115
|
|
116
|
-
matching_webservers = @config.webservers.select { |wh| /^#{@environment}/i =~ wh[
|
116
|
+
matching_webservers = @config.webservers.select { |wh| /^#{@environment}/i =~ wh["name"].to_s.downcase }
|
117
117
|
if matching_webservers.count == 0
|
118
118
|
message = "#{INDENT}0 webservers match '#{@environment}'.".colorize(:red) + " The following webservers exist in your `.spout.yml` file:\n" + "#{INDENT}#{@config.webservers.collect{|wh| wh['name'].to_s.downcase}.join(', ')}".colorize(:white)
|
119
119
|
failure(message)
|
@@ -122,17 +122,17 @@ module Spout
|
|
122
122
|
failure(message)
|
123
123
|
end
|
124
124
|
|
125
|
-
@webserver_name = matching_webservers.first[
|
126
|
-
@url = URI.parse(matching_webservers.first[
|
125
|
+
@webserver_name = matching_webservers.first["name"].to_s.strip rescue @webserver_name = ""
|
126
|
+
@url = URI.parse(matching_webservers.first["url"].to_s.strip) rescue @url = nil
|
127
127
|
|
128
|
-
if @url.to_s ==
|
128
|
+
if @url.to_s == ""
|
129
129
|
message = "#{INDENT}Invalid URL format for #{matching_webservers.first['name'].to_s.strip.downcase} webserver: ".colorize(:red) + "'#{matching_webservers.first['url'].to_s.strip}'".colorize(:white)
|
130
130
|
failure(message)
|
131
131
|
end
|
132
132
|
|
133
|
-
puts
|
134
|
-
puts
|
135
|
-
puts
|
133
|
+
puts "PASS".colorize(:green)
|
134
|
+
puts " Target Server: " + "#{@url}".colorize(:white)
|
135
|
+
puts " Target Dataset: " + "#{@slug}".colorize(:white)
|
136
136
|
end
|
137
137
|
|
138
138
|
# - **Version Check**
|
@@ -141,7 +141,7 @@ module Spout
|
|
141
141
|
# - "v#{VERSION}" matches HEAD git tag annotation
|
142
142
|
def version_check
|
143
143
|
if @skip_checks
|
144
|
-
puts
|
144
|
+
puts " Version Check: " + "SKIP".colorize(:blue)
|
145
145
|
return
|
146
146
|
end
|
147
147
|
|
@@ -149,19 +149,19 @@ module Spout
|
|
149
149
|
`git status --porcelain`
|
150
150
|
end
|
151
151
|
|
152
|
-
print
|
153
|
-
if stdout.to_s.strip ==
|
154
|
-
puts
|
152
|
+
print " Git Status Check: "
|
153
|
+
if stdout.to_s.strip == ""
|
154
|
+
puts "PASS".colorize(:green) + " " + "nothing to commit, working directory clean".colorize(:white)
|
155
155
|
else
|
156
|
-
message = "#{INDENT}working directory contains uncomitted changes\n#{INDENT}use `".colorize(:red) +
|
156
|
+
message = "#{INDENT}working directory contains uncomitted changes\n#{INDENT}use `".colorize(:red) + "--skip-checks".colorize(:white) + "` to ignore this step".colorize(:red)
|
157
157
|
failure message
|
158
158
|
end
|
159
159
|
|
160
|
-
changelog = File.open(
|
160
|
+
changelog = File.open("CHANGELOG.md", &:readline).strip rescue changelog = ""
|
161
161
|
if changelog.match(/^## #{@version.split('.')[0..2].join('.')}/)
|
162
162
|
puts " CHANGELOG.md: " + "PASS".colorize(:green) + " " + changelog.colorize(:white)
|
163
163
|
else
|
164
|
-
print
|
164
|
+
print " CHANGELOG.md: "
|
165
165
|
message = "#{INDENT}Expected: ".colorize(:red) + "## #{@version}".colorize(:white) +
|
166
166
|
"\n#{INDENT} Actual: ".colorize(:red) + changelog.colorize(:white)
|
167
167
|
failure message
|
@@ -171,30 +171,30 @@ module Spout
|
|
171
171
|
`git describe --exact-match HEAD --tags`
|
172
172
|
end
|
173
173
|
|
174
|
-
print
|
174
|
+
print " Version Check: "
|
175
175
|
tag = stdout.to_s.strip
|
176
176
|
if "v#{@version}" != tag
|
177
|
-
message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) +
|
177
|
+
message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) + " does not match git tag on HEAD commit ".colorize(:red) + "'#{tag}'".colorize(:white)
|
178
178
|
failure message
|
179
179
|
else
|
180
|
-
puts
|
180
|
+
puts "PASS".colorize(:green) + " VERSION " + "'v#{@version}'".colorize(:white) + " matches git tag " + "'#{tag}'".colorize(:white)
|
181
181
|
end
|
182
182
|
end
|
183
183
|
|
184
184
|
def test_check
|
185
185
|
if @skip_tests
|
186
|
-
puts
|
186
|
+
puts " Spout Tests: " + "SKIP".colorize(:blue)
|
187
187
|
return
|
188
188
|
end
|
189
189
|
|
190
|
-
print
|
190
|
+
print " Spout Tests: "
|
191
191
|
|
192
192
|
stdout = quietly do
|
193
193
|
`spout t`
|
194
194
|
end
|
195
195
|
|
196
196
|
if stdout.match(/[^\d]0 failures, 0 errors,/)
|
197
|
-
puts
|
197
|
+
puts "PASS".colorize(:green)
|
198
198
|
else
|
199
199
|
message = "#{INDENT}spout t".colorize(:white) + " had errors or failures".colorize(:red) + "\n#{INDENT}Please fix all errors and failures and then run spout deploy again."
|
200
200
|
failure message
|
@@ -203,22 +203,22 @@ module Spout
|
|
203
203
|
|
204
204
|
def coverage_check
|
205
205
|
if @skip_coverage
|
206
|
-
puts
|
206
|
+
puts " Dataset Coverage: " + "SKIP".colorize(:blue)
|
207
207
|
return
|
208
208
|
end
|
209
209
|
|
210
|
-
puts
|
210
|
+
puts " Dataset Coverage: " + "NOT IMPLEMENTED".colorize(:yellow)
|
211
211
|
end
|
212
212
|
|
213
213
|
def user_authorization
|
214
|
-
puts
|
215
|
-
print
|
216
|
-
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip ==
|
214
|
+
puts " Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
|
215
|
+
print " Enter your token: "
|
216
|
+
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ""
|
217
217
|
(json, _status) = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
|
218
|
-
if json.is_a?(Hash) && json[
|
219
|
-
puts
|
218
|
+
if json.is_a?(Hash) && json["editor"]
|
219
|
+
puts "AUTHORIZED".colorize(:green)
|
220
220
|
else
|
221
|
-
puts
|
221
|
+
puts "UNAUTHORIZED".colorize(:red)
|
222
222
|
puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
|
223
223
|
raise DeployError
|
224
224
|
end
|
@@ -226,7 +226,7 @@ module Spout
|
|
226
226
|
|
227
227
|
def upload_variables
|
228
228
|
if @skip_variables
|
229
|
-
puts
|
229
|
+
puts " Upload Variables: " + "SKIP".colorize(:blue)
|
230
230
|
return
|
231
231
|
end
|
232
232
|
load_subjects_from_csvs
|
@@ -235,122 +235,122 @@ module Spout
|
|
235
235
|
|
236
236
|
def load_subjects_from_csvs
|
237
237
|
@dictionary_root = Dir.pwd
|
238
|
-
@variable_files = Dir.glob(File.join(@dictionary_root,
|
238
|
+
@variable_files = Dir.glob(File.join(@dictionary_root, "variables", "**", "*.json"))
|
239
239
|
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, @number_of_rows, @config.visit)
|
240
240
|
@subject_loader.load_subjects_from_csvs!
|
241
241
|
@subjects = @subject_loader.subjects
|
242
242
|
end
|
243
243
|
|
244
244
|
def graph_generation
|
245
|
-
# failure
|
246
|
-
require
|
247
|
-
@argv <<
|
245
|
+
# failure ""
|
246
|
+
require "spout/commands/graphs"
|
247
|
+
@argv << "--clean" if @clean
|
248
248
|
Spout::Commands::Graphs.new(@argv, @version, true, @url, @slug, @token, @webserver_name, @subjects)
|
249
|
-
puts "\r Upload Variables: " +
|
249
|
+
puts "\r Upload Variables: " + "DONE ".colorize(:green)
|
250
250
|
end
|
251
251
|
|
252
252
|
def dataset_uploads
|
253
253
|
if @skip_dataset
|
254
|
-
puts
|
254
|
+
puts " Dataset Uploads: " + "SKIP".colorize(:blue)
|
255
255
|
return
|
256
256
|
end
|
257
257
|
|
258
|
-
available_folders = (Dir.exist?(
|
258
|
+
available_folders = (Dir.exist?("csvs") ? Dir.entries("csvs").select { |e| File.directory? File.join("csvs", e) }.reject { |e| [".", ".."].include?(e) }.sort : [])
|
259
259
|
semantic = Spout::Helpers::Semantic.new(@version, available_folders)
|
260
260
|
csv_directory = semantic.selected_folder
|
261
261
|
csv_files = Dir.glob("csvs/#{csv_directory}/**/*.csv")
|
262
262
|
|
263
263
|
csv_files.each_with_index do |csv_file, index|
|
264
264
|
print "\r Dataset Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
|
265
|
-
folder = csv_file.gsub(%r{^csvs/#{csv_directory}},
|
266
|
-
folder = folder.gsub(%r{/$},
|
265
|
+
folder = csv_file.gsub(%r{^csvs/#{csv_directory}}, "").gsub(/#{File.basename(csv_file)}$/, "")
|
266
|
+
folder = folder.gsub(%r{/$}, "")
|
267
267
|
@created_folders << "datasets#{folder}"
|
268
|
-
@created_folders <<
|
268
|
+
@created_folders << "datasets/archive"
|
269
269
|
@created_folders << "datasets/archive/#{@version}#{folder}"
|
270
270
|
upload_file(csv_file, "datasets#{folder}") unless @archive_only
|
271
271
|
upload_file(csv_file, "datasets/archive/#{@version}#{folder}")
|
272
272
|
end
|
273
|
-
puts "\r Dataset Uploads: " +
|
273
|
+
puts "\r Dataset Uploads: " + "DONE ".colorize(:green)
|
274
274
|
end
|
275
275
|
|
276
276
|
def data_dictionary_uploads
|
277
277
|
if @skip_dictionary
|
278
|
-
puts
|
278
|
+
puts " Dictionary Uploads: " + "SKIP".colorize(:blue)
|
279
279
|
return
|
280
280
|
end
|
281
281
|
|
282
|
-
print
|
282
|
+
print " Dictionary Uploads:"
|
283
283
|
|
284
|
-
require
|
285
|
-
Spout::Commands::Exporter.new(@version, [
|
284
|
+
require "spout/commands/exporter"
|
285
|
+
Spout::Commands::Exporter.new(@version, ["--quiet"])
|
286
286
|
|
287
287
|
csv_files = Dir.glob("exports/#{@version}/*.csv")
|
288
288
|
csv_files.each_with_index do |csv_file, index|
|
289
289
|
print "\r Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
|
290
|
-
@created_folders <<
|
291
|
-
@created_folders <<
|
290
|
+
@created_folders << "datasets"
|
291
|
+
@created_folders << "datasets/archive"
|
292
292
|
@created_folders << "datasets/archive/#{@version}"
|
293
|
-
upload_file(csv_file,
|
293
|
+
upload_file(csv_file, "datasets") unless @archive_only
|
294
294
|
upload_file(csv_file, "datasets/archive/#{@version}")
|
295
295
|
end
|
296
|
-
puts "\r Dictionary Uploads: " +
|
296
|
+
puts "\r Dictionary Uploads: " + "DONE ".colorize(:green)
|
297
297
|
end
|
298
298
|
|
299
299
|
def markdown_uploads
|
300
300
|
if @skip_documentation
|
301
|
-
puts
|
301
|
+
puts "Documentation Uploads: " + "SKIP".colorize(:blue)
|
302
302
|
return
|
303
303
|
end
|
304
304
|
|
305
|
-
print
|
305
|
+
print "Documentation Uploads:"
|
306
306
|
markdown_files = Dir.glob(%w(CHANGELOG.md KNOWNISSUES.md))
|
307
307
|
markdown_files.each_with_index do |markdown_file, index|
|
308
308
|
print "\rDocumentation Uploads: " + "#{index + 1} of #{markdown_files.count}".colorize(:green)
|
309
|
-
@created_folders <<
|
310
|
-
@created_folders <<
|
309
|
+
@created_folders << "datasets"
|
310
|
+
@created_folders << "datasets/archive"
|
311
311
|
@created_folders << "datasets/archive/#{@version}"
|
312
|
-
upload_file(markdown_file,
|
312
|
+
upload_file(markdown_file, "datasets") unless @archive_only
|
313
313
|
upload_file(markdown_file, "datasets/archive/#{@version}")
|
314
314
|
end
|
315
|
-
puts "\rDocumentation Uploads: " +
|
315
|
+
puts "\rDocumentation Uploads: " + "DONE ".colorize(:green)
|
316
316
|
end
|
317
317
|
|
318
318
|
def trigger_server_updates
|
319
319
|
if @skip_server_scripts
|
320
|
-
puts
|
320
|
+
puts "Launch Server Scripts: " + "SKIP".colorize(:blue)
|
321
321
|
return
|
322
322
|
end
|
323
323
|
|
324
|
-
print
|
324
|
+
print "Launch Server Scripts: "
|
325
325
|
params = { auth_token: @token, dataset: @slug, version: @version, folders: @created_folders.compact.uniq }
|
326
326
|
(json, _status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/dictionary/refresh.json", params)
|
327
|
-
if json.is_a?(Hash) && json[
|
328
|
-
puts
|
327
|
+
if json.is_a?(Hash) && json["refresh"] == "success"
|
328
|
+
puts "DONE".colorize(:green)
|
329
329
|
else
|
330
|
-
puts
|
330
|
+
puts "FAIL".colorize(:red)
|
331
331
|
raise DeployError
|
332
332
|
end
|
333
333
|
end
|
334
334
|
|
335
335
|
def set_default_dataset_version
|
336
336
|
if @archive_only
|
337
|
-
puts
|
337
|
+
puts " Set Default Version: " + "SKIP".colorize(:blue)
|
338
338
|
return
|
339
339
|
end
|
340
|
-
print
|
340
|
+
print " Set Default Version: "
|
341
341
|
params = { auth_token: @token, dataset: @slug, version: @version }
|
342
342
|
(json, _status) = Spout::Helpers::JsonRequest.post(
|
343
343
|
"#{@url}/api/v1/dictionary/update_default_version.json", params
|
344
344
|
)
|
345
|
-
if json.is_a?(Hash) && json[
|
345
|
+
if json.is_a?(Hash) && json["version_update"] == "success"
|
346
346
|
puts @version.to_s.colorize(:green)
|
347
347
|
else
|
348
|
-
failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) +
|
348
|
+
failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) + " for " + @slug.to_s.colorize(:white) + " dataset.")
|
349
349
|
end
|
350
350
|
end
|
351
351
|
|
352
352
|
def failure(message)
|
353
|
-
puts
|
353
|
+
puts "FAIL".colorize(:red)
|
354
354
|
puts message
|
355
355
|
raise DeployError
|
356
356
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "csv"
|
4
|
+
require "json"
|
5
|
+
require "fileutils"
|
6
|
+
require "colorize"
|
7
7
|
|
8
|
-
require
|
8
|
+
require "spout/helpers/config_reader"
|
9
9
|
|
10
10
|
module Spout
|
11
11
|
module Commands
|
12
12
|
# Exports the JSON data dictionary to a CSV format.
|
13
13
|
class Exporter
|
14
14
|
def initialize(standard_version, argv)
|
15
|
-
@quiet = !argv.delete(
|
15
|
+
@quiet = !argv.delete("--quiet").nil?
|
16
16
|
@standard_version = standard_version
|
17
17
|
@config = Spout::Helpers::ConfigReader.new
|
18
18
|
expanded_export!
|
@@ -22,24 +22,24 @@ module Spout
|
|
22
22
|
|
23
23
|
def expanded_export!
|
24
24
|
folder = "exports/#{@standard_version}"
|
25
|
-
puts
|
25
|
+
puts " create".colorize(:green) + " #{folder}" unless @quiet
|
26
26
|
FileUtils.mkpath folder
|
27
27
|
generic_export(
|
28
28
|
folder,
|
29
|
-
|
29
|
+
"variables",
|
30
30
|
%w(
|
31
31
|
id display_name description type units domain labels calculation
|
32
32
|
commonly_used forms
|
33
33
|
)
|
34
34
|
)
|
35
|
-
generic_export(folder,
|
36
|
-
generic_export(folder,
|
35
|
+
generic_export(folder, "domains", %w(value display_name description), true)
|
36
|
+
generic_export(folder, "forms", %w(id display_name code_book))
|
37
37
|
end
|
38
38
|
|
39
39
|
def generic_export(folder, type, keys, include_domain_name = false)
|
40
40
|
export_file = export_file_name(type)
|
41
|
-
puts
|
42
|
-
CSV.open("#{folder}/#{export_file}",
|
41
|
+
puts " export".colorize(:blue) + " #{folder}/#{export_file}" unless @quiet
|
42
|
+
CSV.open("#{folder}/#{export_file}", "wb") do |csv|
|
43
43
|
csv << if include_domain_name
|
44
44
|
%w(folder domain_id) + keys
|
45
45
|
else
|
@@ -56,7 +56,7 @@ module Spout
|
|
56
56
|
end
|
57
57
|
else
|
58
58
|
csv << [relative_folder] + keys.collect do |key|
|
59
|
-
json[key].is_a?(Array) ? json[key].join(
|
59
|
+
json[key].is_a?(Array) ? json[key].join(";") : json[key].to_s
|
60
60
|
end
|
61
61
|
end
|
62
62
|
end
|
@@ -65,7 +65,7 @@ module Spout
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def export_file_name(type)
|
68
|
-
if @config.slug ==
|
68
|
+
if @config.slug == ""
|
69
69
|
"#{type}.csv"
|
70
70
|
else
|
71
71
|
"#{@config.slug}-data-dictionary-#{@standard_version}-#{type}.csv"
|
@@ -73,11 +73,11 @@ module Spout
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def generic_folder_path(file, type)
|
76
|
-
file.gsub(/#{type}\//,
|
76
|
+
file.gsub(/#{type}\//, "").split("/")[0..-2].join("/")
|
77
77
|
end
|
78
78
|
|
79
79
|
def extract_domain_name(file)
|
80
|
-
file.gsub(/domains\//,
|
80
|
+
file.gsub(/domains\//, "").split("/").last.to_s.gsub(/.json/, "")
|
81
81
|
end
|
82
82
|
end
|
83
83
|
end
|