spout 0.12.1 → 0.13.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +5 -5
- data/Rakefile +4 -4
- data/bin/spout +1 -1
- data/lib/spout/commands/coverage.rb +18 -18
- data/lib/spout/commands/deploy.rb +89 -89
- data/lib/spout/commands/exporter.rb +16 -16
- data/lib/spout/commands/graphs.rb +37 -37
- data/lib/spout/commands/help.rb +1 -1
- data/lib/spout/commands/importer.rb +59 -57
- data/lib/spout/commands/outliers.rb +17 -17
- data/lib/spout/commands/project_generator.rb +25 -25
- data/lib/spout/commands/update.rb +38 -38
- data/lib/spout/helpers/array_statistics.rb +7 -7
- data/lib/spout/helpers/chart_types.rb +2 -2
- data/lib/spout/helpers/config_reader.rb +21 -20
- data/lib/spout/helpers/framework.rb +11 -11
- data/lib/spout/helpers/iterators.rb +1 -3
- data/lib/spout/helpers/json_loader.rb +2 -4
- data/lib/spout/helpers/json_request.rb +11 -11
- data/lib/spout/helpers/number_helper.rb +1 -1
- data/lib/spout/helpers/quietly.rb +1 -1
- data/lib/spout/helpers/semantic.rb +1 -1
- data/lib/spout/helpers/send_file.rb +14 -14
- data/lib/spout/helpers/subject_loader.rb +24 -24
- data/lib/spout/helpers/table_formatting.rb +12 -12
- data/lib/spout/models/coverage_result.rb +8 -8
- data/lib/spout/models/dictionary.rb +10 -10
- data/lib/spout/models/domain.rb +7 -7
- data/lib/spout/models/empty.rb +1 -1
- data/lib/spout/models/form.rb +6 -6
- data/lib/spout/models/graphables/choices_vs_choices.rb +3 -5
- data/lib/spout/models/graphables/choices_vs_numeric.rb +4 -6
- data/lib/spout/models/graphables/default.rb +6 -6
- data/lib/spout/models/graphables/histogram.rb +4 -4
- data/lib/spout/models/graphables/numeric_vs_choices.rb +1 -1
- data/lib/spout/models/graphables/numeric_vs_numeric.rb +6 -6
- data/lib/spout/models/graphables.rb +14 -14
- data/lib/spout/models/outlier_result.rb +16 -18
- data/lib/spout/models/record.rb +3 -3
- data/lib/spout/models/tables/choices_vs_choices.rb +11 -11
- data/lib/spout/models/tables/choices_vs_numeric.rb +8 -8
- data/lib/spout/models/tables/default.rb +5 -5
- data/lib/spout/models/tables/numeric_vs_choices.rb +7 -7
- data/lib/spout/models/tables/numeric_vs_numeric.rb +7 -7
- data/lib/spout/models/tables.rb +11 -11
- data/lib/spout/models/variable.rb +14 -14
- data/lib/spout/tasks/engine.rake +4 -4
- data/lib/spout/templates/CHANGELOG.md.erb +1 -1
- data/lib/spout/templates/Gemfile +2 -2
- data/lib/spout/templates/Rakefile +1 -1
- data/lib/spout/templates/test/dictionary_test.rb +3 -3
- data/lib/spout/templates/test/test_helper.rb +1 -1
- data/lib/spout/tests/domain_existence_validation.rb +6 -6
- data/lib/spout/tests/domain_format.rb +1 -1
- data/lib/spout/tests/domain_name_format.rb +3 -3
- data/lib/spout/tests/domain_name_uniqueness.rb +2 -2
- data/lib/spout/tests/domain_specified.rb +1 -3
- data/lib/spout/tests/form_existence_validation.rb +2 -4
- data/lib/spout/tests/form_name_format.rb +3 -3
- data/lib/spout/tests/form_name_match.rb +1 -1
- data/lib/spout/tests/json_helper.rb +1 -1
- data/lib/spout/tests/json_validation.rb +0 -2
- data/lib/spout/tests/variable_name_format.rb +3 -3
- data/lib/spout/tests/variable_name_match.rb +1 -3
- data/lib/spout/tests/variable_name_uniqueness.rb +2 -2
- data/lib/spout/tests/variable_type_validation.rb +2 -2
- data/lib/spout/tests.rb +30 -30
- data/lib/spout/version.rb +4 -4
- data/lib/spout/views/index.html.erb +15 -15
- data/lib/spout/views/outliers.html.erb +8 -8
- data/lib/spout.rb +26 -26
- data/spout.gemspec +22 -22
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 415a16b52a348d3d3c7361b6ea226e7d7aa10edc
|
4
|
+
data.tar.gz: '08bbf0c3477b4689a0cc4dbacc52750ec53a7eb2'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 97b50714ade02df37afd4fac776bbd5d4515db7e628b5acee6639d1f711c025cf6ed6c5e6bd2b9f0c0826061487adc85c44d44683520556bd3d12e43ac2a7d6f
|
7
|
+
data.tar.gz: 26a33f2ffdeb525e047a10efc87324d8a935a5bb47f60780ffff55237f56f39dbc70b4530abd871fc29851a2171ff38d35ac64c7c24c6bd7c038af4b7efdec2d
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -16,7 +16,7 @@ Spout has been used extensively to curate and clean datasets available on the
|
|
16
16
|
|
17
17
|
Add this line to your application's Gemfile:
|
18
18
|
|
19
|
-
gem
|
19
|
+
gem "spout"
|
20
20
|
|
21
21
|
And then execute:
|
22
22
|
|
@@ -143,7 +143,7 @@ tests, or just a subset of Spout tests.
|
|
143
143
|
`test/dictionary_test.rb`
|
144
144
|
|
145
145
|
```ruby
|
146
|
-
require
|
146
|
+
require "spout/tests"
|
147
147
|
|
148
148
|
class DictionaryTest < Minitest::Test
|
149
149
|
# This line includes all default Spout Dictionary tests
|
@@ -152,7 +152,7 @@ end
|
|
152
152
|
```
|
153
153
|
|
154
154
|
```ruby
|
155
|
-
require
|
155
|
+
require "spout/tests"
|
156
156
|
|
157
157
|
class DictionaryTest < Minitest::Test
|
158
158
|
# You can include only certain Spout tests by including them individually
|
@@ -185,13 +185,13 @@ class DictionaryTest < Minitest::Test
|
|
185
185
|
# that can be used to write custom tests.
|
186
186
|
include Spout::Helpers::Iterators
|
187
187
|
|
188
|
-
VALID_UNITS = [
|
188
|
+
VALID_UNITS = ["minutes", "hours"]
|
189
189
|
|
190
190
|
@variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
|
191
191
|
define_method("test_units: #{variable.path}") do
|
192
192
|
message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
|
193
193
|
" Valid types: " +
|
194
|
-
VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(
|
194
|
+
VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(", ")
|
195
195
|
assert VALID_UNITS.include?(variable.units), message
|
196
196
|
end
|
197
197
|
end
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rake/testtask"
|
4
4
|
|
5
5
|
Rake::TestTask.new(:test) do |t|
|
6
|
-
t.libs <<
|
7
|
-
t.libs <<
|
8
|
-
t.pattern =
|
6
|
+
t.libs << "lib"
|
7
|
+
t.libs << "test"
|
8
|
+
t.pattern = "test/**/*_test.rb"
|
9
9
|
t.verbose = false
|
10
10
|
end
|
11
11
|
|
data/bin/spout
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "erb"
|
4
|
+
require "fileutils"
|
5
|
+
require "yaml"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
7
|
+
require "spout/helpers/subject_loader"
|
8
|
+
require "spout/models/coverage_result"
|
9
|
+
require "spout/helpers/number_helper"
|
10
|
+
require "spout/helpers/config_reader"
|
11
|
+
require "spout/helpers/array_statistics"
|
12
12
|
|
13
13
|
module Spout
|
14
14
|
module Commands
|
@@ -18,8 +18,8 @@ module Spout
|
|
18
18
|
|
19
19
|
def initialize(standard_version, argv)
|
20
20
|
@standard_version = standard_version
|
21
|
-
@console = !argv.delete(
|
22
|
-
@variable_files = Dir.glob(
|
21
|
+
@console = !argv.delete("--console").nil?
|
22
|
+
@variable_files = Dir.glob("variables/**/*.json")
|
23
23
|
@valid_ids = []
|
24
24
|
@number_of_rows = nil
|
25
25
|
@config = Spout::Helpers::ConfigReader.new
|
@@ -38,10 +38,10 @@ module Spout
|
|
38
38
|
scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).compact_empty.uniq)
|
39
39
|
@matching_results << [csv_files, method, scr]
|
40
40
|
end
|
41
|
-
variable_ids = Dir.glob(
|
41
|
+
variable_ids = Dir.glob("variables/**/*.json").collect { |file| file.gsub(%r{^(.*)/|\.json$}, "").downcase }
|
42
42
|
@extra_variable_ids = (variable_ids - @subject_loader.all_methods.keys).sort
|
43
43
|
@subject_loader.load_variable_domains!
|
44
|
-
domain_ids = Dir.glob(
|
44
|
+
domain_ids = Dir.glob("domains/**/*.json").collect { |file| file.gsub(%r{^(.*)/|\.json$}, "").downcase }
|
45
45
|
@extra_domain_ids = (domain_ids - @subject_loader.all_domains).sort
|
46
46
|
@matching_results.sort! do |a, b|
|
47
47
|
[b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]
|
@@ -53,16 +53,16 @@ module Spout
|
|
53
53
|
.select { |mr| mr[0].include?(csv_file) && mr[2].number_of_errors.zero? }.count
|
54
54
|
@coverage_results << [csv_file, total_column_count, mapped_column_count]
|
55
55
|
end
|
56
|
-
coverage_folder = File.join(Dir.pwd,
|
56
|
+
coverage_folder = File.join(Dir.pwd, "coverage")
|
57
57
|
FileUtils.mkpath coverage_folder
|
58
|
-
coverage_file = File.join(coverage_folder,
|
59
|
-
File.open(coverage_file,
|
60
|
-
erb_location = File.join(File.dirname(__FILE__),
|
58
|
+
coverage_file = File.join(coverage_folder, "index.html")
|
59
|
+
File.open(coverage_file, "w+") do |file|
|
60
|
+
erb_location = File.join(File.dirname(__FILE__), "../views/index.html.erb")
|
61
61
|
file.puts ERB.new(File.read(erb_location)).result(binding)
|
62
62
|
end
|
63
63
|
unless @console
|
64
|
-
open_command =
|
65
|
-
open_command =
|
64
|
+
open_command = "open" unless RUBY_PLATFORM.match(/darwin/).nil?
|
65
|
+
open_command = "start" unless RUBY_PLATFORM.match(/mingw/).nil?
|
66
66
|
system "#{open_command} #{coverage_file}" if %w(start open).include?(open_command)
|
67
67
|
end
|
68
68
|
puts "#{coverage_file}\n\n"
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "net/http"
|
5
|
+
require "io/console"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
7
|
+
require "spout/helpers/subject_loader"
|
8
|
+
require "spout/helpers/config_reader"
|
9
|
+
require "spout/helpers/quietly"
|
10
|
+
require "spout/helpers/send_file"
|
11
|
+
require "spout/helpers/semantic"
|
12
|
+
require "spout/helpers/json_request"
|
13
13
|
|
14
14
|
# - **User Authorization**
|
15
15
|
# - User authenticates via token, the user must be a dataset editor
|
@@ -41,34 +41,34 @@ module Spout
|
|
41
41
|
include Spout::Helpers::Quietly
|
42
42
|
|
43
43
|
INDENT_LENGTH = 23
|
44
|
-
INDENT =
|
44
|
+
INDENT = " " * INDENT_LENGTH
|
45
45
|
|
46
46
|
attr_accessor :token, :version, :slug, :url, :config, :environment, :webserver_name, :subjects
|
47
47
|
|
48
48
|
def initialize(argv, version)
|
49
|
-
argv.shift # Remove
|
49
|
+
argv.shift # Remove "download" command from argv list
|
50
50
|
@environment = argv.shift
|
51
51
|
@version = version
|
52
|
-
@skip_checks = !(argv.delete(
|
52
|
+
@skip_checks = !(argv.delete("--skip-checks").nil? && argv.delete("--no-checks").nil?)
|
53
53
|
|
54
|
-
@skip_tests = !(argv.delete(
|
55
|
-
@skip_coverage = !(argv.delete(
|
54
|
+
@skip_tests = !(argv.delete("--skip-tests").nil? && argv.delete("--no-tests").nil?)
|
55
|
+
@skip_coverage = !(argv.delete("--skip-coverage").nil? && argv.delete("--no-coverage").nil?)
|
56
56
|
|
57
|
-
@skip_variables = !(argv.delete(
|
58
|
-
@skip_dataset = !(argv.delete(
|
59
|
-
@skip_dictionary = !(argv.delete(
|
60
|
-
@skip_documentation = !(argv.delete(
|
61
|
-
@clean = !(argv.delete(
|
62
|
-
@skip_server_scripts = !(argv.delete(
|
63
|
-
@archive_only = !(argv.delete(
|
57
|
+
@skip_variables = !(argv.delete("--skip-variables").nil? && argv.delete("--no-variables").nil?)
|
58
|
+
@skip_dataset = !(argv.delete("--skip-dataset").nil? && argv.delete("--no-dataset").nil?)
|
59
|
+
@skip_dictionary = !(argv.delete("--skip-dictionary").nil? && argv.delete("--no-dictionary").nil?)
|
60
|
+
@skip_documentation = !(argv.delete("--skip-documentation").nil? && argv.delete("--no-documentation").nil?)
|
61
|
+
@clean = !(argv.delete("--no-resume").nil? && argv.delete("--clean").nil?)
|
62
|
+
@skip_server_scripts = !(argv.delete("--skip-server-scripts").nil? && argv.delete("--no-server-scripts").nil?)
|
63
|
+
@archive_only = !(argv.delete("--archive-only").nil?)
|
64
64
|
|
65
65
|
token_arg = argv.find { |arg| /^--token=/ =~ arg }
|
66
66
|
argv.delete(token_arg)
|
67
|
-
@token = token_arg.gsub(/^--token=/,
|
67
|
+
@token = token_arg.gsub(/^--token=/, "") if token_arg
|
68
68
|
|
69
69
|
rows_arg = argv.find { |arg| /^--rows=(\d*)/ =~ arg }
|
70
70
|
argv.delete(rows_arg)
|
71
|
-
@number_of_rows = rows_arg.gsub(/--rows=/,
|
71
|
+
@number_of_rows = rows_arg.gsub(/--rows=/, "").to_i if rows_arg
|
72
72
|
|
73
73
|
@argv = argv
|
74
74
|
|
@@ -98,12 +98,12 @@ module Spout
|
|
98
98
|
end
|
99
99
|
|
100
100
|
def config_file_load
|
101
|
-
print
|
101
|
+
print " `.spout.yml` Check: "
|
102
102
|
@config = Spout::Helpers::ConfigReader.new
|
103
103
|
|
104
104
|
@slug = @config.slug
|
105
105
|
|
106
|
-
if @slug ==
|
106
|
+
if @slug == ""
|
107
107
|
message = "#{INDENT}Please specify a dataset slug in your `.spout.yml` file!".colorize(:red) + " Ex:\n---\nslug: mydataset\n".colorize(:orange)
|
108
108
|
failure(message)
|
109
109
|
end
|
@@ -113,7 +113,7 @@ module Spout
|
|
113
113
|
failure(message)
|
114
114
|
end
|
115
115
|
|
116
|
-
matching_webservers = @config.webservers.select { |wh| /^#{@environment}/i =~ wh[
|
116
|
+
matching_webservers = @config.webservers.select { |wh| /^#{@environment}/i =~ wh["name"].to_s.downcase }
|
117
117
|
if matching_webservers.count == 0
|
118
118
|
message = "#{INDENT}0 webservers match '#{@environment}'.".colorize(:red) + " The following webservers exist in your `.spout.yml` file:\n" + "#{INDENT}#{@config.webservers.collect{|wh| wh['name'].to_s.downcase}.join(', ')}".colorize(:white)
|
119
119
|
failure(message)
|
@@ -122,17 +122,17 @@ module Spout
|
|
122
122
|
failure(message)
|
123
123
|
end
|
124
124
|
|
125
|
-
@webserver_name = matching_webservers.first[
|
126
|
-
@url = URI.parse(matching_webservers.first[
|
125
|
+
@webserver_name = matching_webservers.first["name"].to_s.strip rescue @webserver_name = ""
|
126
|
+
@url = URI.parse(matching_webservers.first["url"].to_s.strip) rescue @url = nil
|
127
127
|
|
128
|
-
if @url.to_s ==
|
128
|
+
if @url.to_s == ""
|
129
129
|
message = "#{INDENT}Invalid URL format for #{matching_webservers.first['name'].to_s.strip.downcase} webserver: ".colorize(:red) + "'#{matching_webservers.first['url'].to_s.strip}'".colorize(:white)
|
130
130
|
failure(message)
|
131
131
|
end
|
132
132
|
|
133
|
-
puts
|
134
|
-
puts
|
135
|
-
puts
|
133
|
+
puts "PASS".colorize(:green)
|
134
|
+
puts " Target Server: " + "#{@url}".colorize(:white)
|
135
|
+
puts " Target Dataset: " + "#{@slug}".colorize(:white)
|
136
136
|
end
|
137
137
|
|
138
138
|
# - **Version Check**
|
@@ -141,7 +141,7 @@ module Spout
|
|
141
141
|
# - "v#{VERSION}" matches HEAD git tag annotation
|
142
142
|
def version_check
|
143
143
|
if @skip_checks
|
144
|
-
puts
|
144
|
+
puts " Version Check: " + "SKIP".colorize(:blue)
|
145
145
|
return
|
146
146
|
end
|
147
147
|
|
@@ -149,19 +149,19 @@ module Spout
|
|
149
149
|
`git status --porcelain`
|
150
150
|
end
|
151
151
|
|
152
|
-
print
|
153
|
-
if stdout.to_s.strip ==
|
154
|
-
puts
|
152
|
+
print " Git Status Check: "
|
153
|
+
if stdout.to_s.strip == ""
|
154
|
+
puts "PASS".colorize(:green) + " " + "nothing to commit, working directory clean".colorize(:white)
|
155
155
|
else
|
156
|
-
message = "#{INDENT}working directory contains uncomitted changes\n#{INDENT}use `".colorize(:red) +
|
156
|
+
message = "#{INDENT}working directory contains uncomitted changes\n#{INDENT}use `".colorize(:red) + "--skip-checks".colorize(:white) + "` to ignore this step".colorize(:red)
|
157
157
|
failure message
|
158
158
|
end
|
159
159
|
|
160
|
-
changelog = File.open(
|
160
|
+
changelog = File.open("CHANGELOG.md", &:readline).strip rescue changelog = ""
|
161
161
|
if changelog.match(/^## #{@version.split('.')[0..2].join('.')}/)
|
162
162
|
puts " CHANGELOG.md: " + "PASS".colorize(:green) + " " + changelog.colorize(:white)
|
163
163
|
else
|
164
|
-
print
|
164
|
+
print " CHANGELOG.md: "
|
165
165
|
message = "#{INDENT}Expected: ".colorize(:red) + "## #{@version}".colorize(:white) +
|
166
166
|
"\n#{INDENT} Actual: ".colorize(:red) + changelog.colorize(:white)
|
167
167
|
failure message
|
@@ -171,30 +171,30 @@ module Spout
|
|
171
171
|
`git describe --exact-match HEAD --tags`
|
172
172
|
end
|
173
173
|
|
174
|
-
print
|
174
|
+
print " Version Check: "
|
175
175
|
tag = stdout.to_s.strip
|
176
176
|
if "v#{@version}" != tag
|
177
|
-
message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) +
|
177
|
+
message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) + " does not match git tag on HEAD commit ".colorize(:red) + "'#{tag}'".colorize(:white)
|
178
178
|
failure message
|
179
179
|
else
|
180
|
-
puts
|
180
|
+
puts "PASS".colorize(:green) + " VERSION " + "'v#{@version}'".colorize(:white) + " matches git tag " + "'#{tag}'".colorize(:white)
|
181
181
|
end
|
182
182
|
end
|
183
183
|
|
184
184
|
def test_check
|
185
185
|
if @skip_tests
|
186
|
-
puts
|
186
|
+
puts " Spout Tests: " + "SKIP".colorize(:blue)
|
187
187
|
return
|
188
188
|
end
|
189
189
|
|
190
|
-
print
|
190
|
+
print " Spout Tests: "
|
191
191
|
|
192
192
|
stdout = quietly do
|
193
193
|
`spout t`
|
194
194
|
end
|
195
195
|
|
196
196
|
if stdout.match(/[^\d]0 failures, 0 errors,/)
|
197
|
-
puts
|
197
|
+
puts "PASS".colorize(:green)
|
198
198
|
else
|
199
199
|
message = "#{INDENT}spout t".colorize(:white) + " had errors or failures".colorize(:red) + "\n#{INDENT}Please fix all errors and failures and then run spout deploy again."
|
200
200
|
failure message
|
@@ -203,22 +203,22 @@ module Spout
|
|
203
203
|
|
204
204
|
def coverage_check
|
205
205
|
if @skip_coverage
|
206
|
-
puts
|
206
|
+
puts " Dataset Coverage: " + "SKIP".colorize(:blue)
|
207
207
|
return
|
208
208
|
end
|
209
209
|
|
210
|
-
puts
|
210
|
+
puts " Dataset Coverage: " + "NOT IMPLEMENTED".colorize(:yellow)
|
211
211
|
end
|
212
212
|
|
213
213
|
def user_authorization
|
214
|
-
puts
|
215
|
-
print
|
216
|
-
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip ==
|
214
|
+
puts " Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
|
215
|
+
print " Enter your token: "
|
216
|
+
@token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ""
|
217
217
|
(json, _status) = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
|
218
|
-
if json.is_a?(Hash) && json[
|
219
|
-
puts
|
218
|
+
if json.is_a?(Hash) && json["editor"]
|
219
|
+
puts "AUTHORIZED".colorize(:green)
|
220
220
|
else
|
221
|
-
puts
|
221
|
+
puts "UNAUTHORIZED".colorize(:red)
|
222
222
|
puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
|
223
223
|
raise DeployError
|
224
224
|
end
|
@@ -226,7 +226,7 @@ module Spout
|
|
226
226
|
|
227
227
|
def upload_variables
|
228
228
|
if @skip_variables
|
229
|
-
puts
|
229
|
+
puts " Upload Variables: " + "SKIP".colorize(:blue)
|
230
230
|
return
|
231
231
|
end
|
232
232
|
load_subjects_from_csvs
|
@@ -235,122 +235,122 @@ module Spout
|
|
235
235
|
|
236
236
|
def load_subjects_from_csvs
|
237
237
|
@dictionary_root = Dir.pwd
|
238
|
-
@variable_files = Dir.glob(File.join(@dictionary_root,
|
238
|
+
@variable_files = Dir.glob(File.join(@dictionary_root, "variables", "**", "*.json"))
|
239
239
|
@subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, @number_of_rows, @config.visit)
|
240
240
|
@subject_loader.load_subjects_from_csvs!
|
241
241
|
@subjects = @subject_loader.subjects
|
242
242
|
end
|
243
243
|
|
244
244
|
def graph_generation
|
245
|
-
# failure
|
246
|
-
require
|
247
|
-
@argv <<
|
245
|
+
# failure ""
|
246
|
+
require "spout/commands/graphs"
|
247
|
+
@argv << "--clean" if @clean
|
248
248
|
Spout::Commands::Graphs.new(@argv, @version, true, @url, @slug, @token, @webserver_name, @subjects)
|
249
|
-
puts "\r Upload Variables: " +
|
249
|
+
puts "\r Upload Variables: " + "DONE ".colorize(:green)
|
250
250
|
end
|
251
251
|
|
252
252
|
def dataset_uploads
|
253
253
|
if @skip_dataset
|
254
|
-
puts
|
254
|
+
puts " Dataset Uploads: " + "SKIP".colorize(:blue)
|
255
255
|
return
|
256
256
|
end
|
257
257
|
|
258
|
-
available_folders = (Dir.exist?(
|
258
|
+
available_folders = (Dir.exist?("csvs") ? Dir.entries("csvs").select { |e| File.directory? File.join("csvs", e) }.reject { |e| [".", ".."].include?(e) }.sort : [])
|
259
259
|
semantic = Spout::Helpers::Semantic.new(@version, available_folders)
|
260
260
|
csv_directory = semantic.selected_folder
|
261
261
|
csv_files = Dir.glob("csvs/#{csv_directory}/**/*.csv")
|
262
262
|
|
263
263
|
csv_files.each_with_index do |csv_file, index|
|
264
264
|
print "\r Dataset Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
|
265
|
-
folder = csv_file.gsub(%r{^csvs/#{csv_directory}},
|
266
|
-
folder = folder.gsub(%r{/$},
|
265
|
+
folder = csv_file.gsub(%r{^csvs/#{csv_directory}}, "").gsub(/#{File.basename(csv_file)}$/, "")
|
266
|
+
folder = folder.gsub(%r{/$}, "")
|
267
267
|
@created_folders << "datasets#{folder}"
|
268
|
-
@created_folders <<
|
268
|
+
@created_folders << "datasets/archive"
|
269
269
|
@created_folders << "datasets/archive/#{@version}#{folder}"
|
270
270
|
upload_file(csv_file, "datasets#{folder}") unless @archive_only
|
271
271
|
upload_file(csv_file, "datasets/archive/#{@version}#{folder}")
|
272
272
|
end
|
273
|
-
puts "\r Dataset Uploads: " +
|
273
|
+
puts "\r Dataset Uploads: " + "DONE ".colorize(:green)
|
274
274
|
end
|
275
275
|
|
276
276
|
def data_dictionary_uploads
|
277
277
|
if @skip_dictionary
|
278
|
-
puts
|
278
|
+
puts " Dictionary Uploads: " + "SKIP".colorize(:blue)
|
279
279
|
return
|
280
280
|
end
|
281
281
|
|
282
|
-
print
|
282
|
+
print " Dictionary Uploads:"
|
283
283
|
|
284
|
-
require
|
285
|
-
Spout::Commands::Exporter.new(@version, [
|
284
|
+
require "spout/commands/exporter"
|
285
|
+
Spout::Commands::Exporter.new(@version, ["--quiet"])
|
286
286
|
|
287
287
|
csv_files = Dir.glob("exports/#{@version}/*.csv")
|
288
288
|
csv_files.each_with_index do |csv_file, index|
|
289
289
|
print "\r Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
|
290
|
-
@created_folders <<
|
291
|
-
@created_folders <<
|
290
|
+
@created_folders << "datasets"
|
291
|
+
@created_folders << "datasets/archive"
|
292
292
|
@created_folders << "datasets/archive/#{@version}"
|
293
|
-
upload_file(csv_file,
|
293
|
+
upload_file(csv_file, "datasets") unless @archive_only
|
294
294
|
upload_file(csv_file, "datasets/archive/#{@version}")
|
295
295
|
end
|
296
|
-
puts "\r Dictionary Uploads: " +
|
296
|
+
puts "\r Dictionary Uploads: " + "DONE ".colorize(:green)
|
297
297
|
end
|
298
298
|
|
299
299
|
def markdown_uploads
|
300
300
|
if @skip_documentation
|
301
|
-
puts
|
301
|
+
puts "Documentation Uploads: " + "SKIP".colorize(:blue)
|
302
302
|
return
|
303
303
|
end
|
304
304
|
|
305
|
-
print
|
305
|
+
print "Documentation Uploads:"
|
306
306
|
markdown_files = Dir.glob(%w(CHANGELOG.md KNOWNISSUES.md))
|
307
307
|
markdown_files.each_with_index do |markdown_file, index|
|
308
308
|
print "\rDocumentation Uploads: " + "#{index + 1} of #{markdown_files.count}".colorize(:green)
|
309
|
-
@created_folders <<
|
310
|
-
@created_folders <<
|
309
|
+
@created_folders << "datasets"
|
310
|
+
@created_folders << "datasets/archive"
|
311
311
|
@created_folders << "datasets/archive/#{@version}"
|
312
|
-
upload_file(markdown_file,
|
312
|
+
upload_file(markdown_file, "datasets") unless @archive_only
|
313
313
|
upload_file(markdown_file, "datasets/archive/#{@version}")
|
314
314
|
end
|
315
|
-
puts "\rDocumentation Uploads: " +
|
315
|
+
puts "\rDocumentation Uploads: " + "DONE ".colorize(:green)
|
316
316
|
end
|
317
317
|
|
318
318
|
def trigger_server_updates
|
319
319
|
if @skip_server_scripts
|
320
|
-
puts
|
320
|
+
puts "Launch Server Scripts: " + "SKIP".colorize(:blue)
|
321
321
|
return
|
322
322
|
end
|
323
323
|
|
324
|
-
print
|
324
|
+
print "Launch Server Scripts: "
|
325
325
|
params = { auth_token: @token, dataset: @slug, version: @version, folders: @created_folders.compact.uniq }
|
326
326
|
(json, _status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/dictionary/refresh.json", params)
|
327
|
-
if json.is_a?(Hash) && json[
|
328
|
-
puts
|
327
|
+
if json.is_a?(Hash) && json["refresh"] == "success"
|
328
|
+
puts "DONE".colorize(:green)
|
329
329
|
else
|
330
|
-
puts
|
330
|
+
puts "FAIL".colorize(:red)
|
331
331
|
raise DeployError
|
332
332
|
end
|
333
333
|
end
|
334
334
|
|
335
335
|
def set_default_dataset_version
|
336
336
|
if @archive_only
|
337
|
-
puts
|
337
|
+
puts " Set Default Version: " + "SKIP".colorize(:blue)
|
338
338
|
return
|
339
339
|
end
|
340
|
-
print
|
340
|
+
print " Set Default Version: "
|
341
341
|
params = { auth_token: @token, dataset: @slug, version: @version }
|
342
342
|
(json, _status) = Spout::Helpers::JsonRequest.post(
|
343
343
|
"#{@url}/api/v1/dictionary/update_default_version.json", params
|
344
344
|
)
|
345
|
-
if json.is_a?(Hash) && json[
|
345
|
+
if json.is_a?(Hash) && json["version_update"] == "success"
|
346
346
|
puts @version.to_s.colorize(:green)
|
347
347
|
else
|
348
|
-
failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) +
|
348
|
+
failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) + " for " + @slug.to_s.colorize(:white) + " dataset.")
|
349
349
|
end
|
350
350
|
end
|
351
351
|
|
352
352
|
def failure(message)
|
353
|
-
puts
|
353
|
+
puts "FAIL".colorize(:red)
|
354
354
|
puts message
|
355
355
|
raise DeployError
|
356
356
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "csv"
|
4
|
+
require "json"
|
5
|
+
require "fileutils"
|
6
|
+
require "colorize"
|
7
7
|
|
8
|
-
require
|
8
|
+
require "spout/helpers/config_reader"
|
9
9
|
|
10
10
|
module Spout
|
11
11
|
module Commands
|
12
12
|
# Exports the JSON data dictionary to a CSV format.
|
13
13
|
class Exporter
|
14
14
|
def initialize(standard_version, argv)
|
15
|
-
@quiet = !argv.delete(
|
15
|
+
@quiet = !argv.delete("--quiet").nil?
|
16
16
|
@standard_version = standard_version
|
17
17
|
@config = Spout::Helpers::ConfigReader.new
|
18
18
|
expanded_export!
|
@@ -22,24 +22,24 @@ module Spout
|
|
22
22
|
|
23
23
|
def expanded_export!
|
24
24
|
folder = "exports/#{@standard_version}"
|
25
|
-
puts
|
25
|
+
puts " create".colorize(:green) + " #{folder}" unless @quiet
|
26
26
|
FileUtils.mkpath folder
|
27
27
|
generic_export(
|
28
28
|
folder,
|
29
|
-
|
29
|
+
"variables",
|
30
30
|
%w(
|
31
31
|
id display_name description type units domain labels calculation
|
32
32
|
commonly_used forms
|
33
33
|
)
|
34
34
|
)
|
35
|
-
generic_export(folder,
|
36
|
-
generic_export(folder,
|
35
|
+
generic_export(folder, "domains", %w(value display_name description), true)
|
36
|
+
generic_export(folder, "forms", %w(id display_name code_book))
|
37
37
|
end
|
38
38
|
|
39
39
|
def generic_export(folder, type, keys, include_domain_name = false)
|
40
40
|
export_file = export_file_name(type)
|
41
|
-
puts
|
42
|
-
CSV.open("#{folder}/#{export_file}",
|
41
|
+
puts " export".colorize(:blue) + " #{folder}/#{export_file}" unless @quiet
|
42
|
+
CSV.open("#{folder}/#{export_file}", "wb") do |csv|
|
43
43
|
csv << if include_domain_name
|
44
44
|
%w(folder domain_id) + keys
|
45
45
|
else
|
@@ -56,7 +56,7 @@ module Spout
|
|
56
56
|
end
|
57
57
|
else
|
58
58
|
csv << [relative_folder] + keys.collect do |key|
|
59
|
-
json[key].is_a?(Array) ? json[key].join(
|
59
|
+
json[key].is_a?(Array) ? json[key].join(";") : json[key].to_s
|
60
60
|
end
|
61
61
|
end
|
62
62
|
end
|
@@ -65,7 +65,7 @@ module Spout
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def export_file_name(type)
|
68
|
-
if @config.slug ==
|
68
|
+
if @config.slug == ""
|
69
69
|
"#{type}.csv"
|
70
70
|
else
|
71
71
|
"#{@config.slug}-data-dictionary-#{@standard_version}-#{type}.csv"
|
@@ -73,11 +73,11 @@ module Spout
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def generic_folder_path(file, type)
|
76
|
-
file.gsub(/#{type}\//,
|
76
|
+
file.gsub(/#{type}\//, "").split("/")[0..-2].join("/")
|
77
77
|
end
|
78
78
|
|
79
79
|
def extract_domain_name(file)
|
80
|
-
file.gsub(/domains\//,
|
80
|
+
file.gsub(/domains\//, "").split("/").last.to_s.gsub(/.json/, "")
|
81
81
|
end
|
82
82
|
end
|
83
83
|
end
|