spout 0.12.1 → 0.13.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +5 -5
- data/Rakefile +4 -4
- data/bin/spout +1 -1
- data/lib/spout/commands/coverage.rb +18 -18
- data/lib/spout/commands/deploy.rb +89 -89
- data/lib/spout/commands/exporter.rb +16 -16
- data/lib/spout/commands/graphs.rb +37 -37
- data/lib/spout/commands/help.rb +1 -1
- data/lib/spout/commands/importer.rb +59 -57
- data/lib/spout/commands/outliers.rb +17 -17
- data/lib/spout/commands/project_generator.rb +25 -25
- data/lib/spout/commands/update.rb +38 -38
- data/lib/spout/helpers/array_statistics.rb +7 -7
- data/lib/spout/helpers/chart_types.rb +2 -2
- data/lib/spout/helpers/config_reader.rb +21 -20
- data/lib/spout/helpers/framework.rb +11 -11
- data/lib/spout/helpers/iterators.rb +1 -3
- data/lib/spout/helpers/json_loader.rb +2 -4
- data/lib/spout/helpers/json_request.rb +11 -11
- data/lib/spout/helpers/number_helper.rb +1 -1
- data/lib/spout/helpers/quietly.rb +1 -1
- data/lib/spout/helpers/semantic.rb +1 -1
- data/lib/spout/helpers/send_file.rb +14 -14
- data/lib/spout/helpers/subject_loader.rb +24 -24
- data/lib/spout/helpers/table_formatting.rb +12 -12
- data/lib/spout/models/coverage_result.rb +8 -8
- data/lib/spout/models/dictionary.rb +10 -10
- data/lib/spout/models/domain.rb +7 -7
- data/lib/spout/models/empty.rb +1 -1
- data/lib/spout/models/form.rb +6 -6
- data/lib/spout/models/graphables/choices_vs_choices.rb +3 -5
- data/lib/spout/models/graphables/choices_vs_numeric.rb +4 -6
- data/lib/spout/models/graphables/default.rb +6 -6
- data/lib/spout/models/graphables/histogram.rb +4 -4
- data/lib/spout/models/graphables/numeric_vs_choices.rb +1 -1
- data/lib/spout/models/graphables/numeric_vs_numeric.rb +6 -6
- data/lib/spout/models/graphables.rb +14 -14
- data/lib/spout/models/outlier_result.rb +16 -18
- data/lib/spout/models/record.rb +3 -3
- data/lib/spout/models/tables/choices_vs_choices.rb +11 -11
- data/lib/spout/models/tables/choices_vs_numeric.rb +8 -8
- data/lib/spout/models/tables/default.rb +5 -5
- data/lib/spout/models/tables/numeric_vs_choices.rb +7 -7
- data/lib/spout/models/tables/numeric_vs_numeric.rb +7 -7
- data/lib/spout/models/tables.rb +11 -11
- data/lib/spout/models/variable.rb +14 -14
- data/lib/spout/tasks/engine.rake +4 -4
- data/lib/spout/templates/CHANGELOG.md.erb +1 -1
- data/lib/spout/templates/Gemfile +2 -2
- data/lib/spout/templates/Rakefile +1 -1
- data/lib/spout/templates/test/dictionary_test.rb +3 -3
- data/lib/spout/templates/test/test_helper.rb +1 -1
- data/lib/spout/tests/domain_existence_validation.rb +6 -6
- data/lib/spout/tests/domain_format.rb +1 -1
- data/lib/spout/tests/domain_name_format.rb +3 -3
- data/lib/spout/tests/domain_name_uniqueness.rb +2 -2
- data/lib/spout/tests/domain_specified.rb +1 -3
- data/lib/spout/tests/form_existence_validation.rb +2 -4
- data/lib/spout/tests/form_name_format.rb +3 -3
- data/lib/spout/tests/form_name_match.rb +1 -1
- data/lib/spout/tests/json_helper.rb +1 -1
- data/lib/spout/tests/json_validation.rb +0 -2
- data/lib/spout/tests/variable_name_format.rb +3 -3
- data/lib/spout/tests/variable_name_match.rb +1 -3
- data/lib/spout/tests/variable_name_uniqueness.rb +2 -2
- data/lib/spout/tests/variable_type_validation.rb +2 -2
- data/lib/spout/tests.rb +30 -30
- data/lib/spout/version.rb +4 -4
- data/lib/spout/views/index.html.erb +15 -15
- data/lib/spout/views/outliers.html.erb +8 -8
- data/lib/spout.rb +26 -26
- data/spout.gemspec +22 -22
- metadata +5 -5
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "spout/helpers/json_request"
|
5
|
+
require "spout/helpers/framework"
|
6
6
|
|
7
7
|
module Spout
|
8
8
|
module Commands
|
@@ -17,27 +17,27 @@ module Spout
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def initialize(argv)
|
20
|
-
@full_path = File.join(
|
20
|
+
@full_path = File.join(".")
|
21
21
|
end
|
22
22
|
|
23
23
|
def start
|
24
|
-
(json, _status) = Spout::Helpers::JsonRequest.get(
|
24
|
+
(json, _status) = Spout::Helpers::JsonRequest.get("https://rubygems.org/api/v1/gems/spout.json")
|
25
25
|
if json
|
26
|
-
if json[
|
27
|
-
puts
|
28
|
-
check_framework if File.exist?(
|
26
|
+
if json["version"] == Spout::VERSION::STRING
|
27
|
+
puts "The spout gem is " + "up-to-date".colorize(:green) + "!"
|
28
|
+
check_framework if File.exist?("Gemfile")
|
29
29
|
else
|
30
30
|
puts "A newer version (v#{json['version']}) is available!\n\n"
|
31
|
-
if File.exist?(
|
32
|
-
puts
|
33
|
-
puts " gem
|
31
|
+
if File.exist?("Gemfile")
|
32
|
+
puts "Add the following to your Gemfile and run " + "bundle update".colorize(:green) + ".\n\n"
|
33
|
+
puts " gem \"spout\", \"~> #{json['version']}\"\n".colorize(:white)
|
34
34
|
else
|
35
35
|
puts "Type the following command to update:\n\n"
|
36
|
-
puts
|
36
|
+
puts " gem install spout --no-document".colorize(:white) + "\n\n"
|
37
37
|
end
|
38
38
|
end
|
39
39
|
else
|
40
|
-
puts
|
40
|
+
puts "Unable to connect to RubyGems.org. Please try again later."
|
41
41
|
end
|
42
42
|
end
|
43
43
|
|
@@ -50,62 +50,62 @@ module Spout
|
|
50
50
|
end
|
51
51
|
|
52
52
|
def check_gitignore_file
|
53
|
-
if File.exist?(
|
54
|
-
lines = IO.readlines(
|
55
|
-
addables = [
|
56
|
-
removables = [
|
53
|
+
if File.exist?(".gitignore")
|
54
|
+
lines = IO.readlines(".gitignore").collect(&:strip)
|
55
|
+
addables = ["/coverage", "/csvs", "/exports", "/graphs"]
|
56
|
+
removables = ["/dd", "/images"]
|
57
57
|
unless ((removables & lines) | (addables - lines)).empty?
|
58
|
-
puts
|
59
|
-
puts
|
58
|
+
puts "File: " + ".gitignore".colorize(:white)
|
59
|
+
puts "----------------"
|
60
60
|
(removables & lines).each do |removable|
|
61
|
-
puts
|
61
|
+
puts "REMOVE LINE ".colorize(:red) + removable.colorize(:white)
|
62
62
|
end
|
63
63
|
(addables - lines).each do |addable|
|
64
|
-
puts
|
64
|
+
puts " ADD LINE ".colorize(:green) + addable.colorize(:white)
|
65
65
|
end
|
66
66
|
puts
|
67
67
|
end
|
68
68
|
else
|
69
|
-
copy_file
|
69
|
+
copy_file "gitignore", ".gitignore"
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
73
|
def check_ruby_version
|
74
|
-
if File.exist?(
|
75
|
-
lines = IO.readlines(
|
76
|
-
template_lines = IO.readlines(File.expand_path(
|
74
|
+
if File.exist?(".ruby-version")
|
75
|
+
lines = IO.readlines(".ruby-version").collect(&:strip)
|
76
|
+
template_lines = IO.readlines(File.expand_path("../../templates/ruby-version", __FILE__)).collect(&:strip)
|
77
77
|
if template_lines.first != lines.first
|
78
|
-
puts
|
79
|
-
puts
|
80
|
-
print
|
81
|
-
print
|
78
|
+
puts "File: " + ".ruby-version".colorize(:white)
|
79
|
+
puts "-------------------"
|
80
|
+
print "Update Ruby from " + lines.first.to_s.colorize(:red)
|
81
|
+
print " to " + template_lines.first.to_s.colorize(:green)
|
82
82
|
puts "\n\n"
|
83
83
|
end
|
84
84
|
else
|
85
|
-
copy_file
|
85
|
+
copy_file "ruby-version", ".ruby-version"
|
86
86
|
end
|
87
87
|
end
|
88
88
|
|
89
89
|
def check_file_presence
|
90
90
|
@project_name = File.basename(Dir.pwd)
|
91
|
-
evaluate_file
|
92
|
-
evaluate_file
|
93
|
-
copy_file
|
91
|
+
evaluate_file "CHANGELOG.md.erb", "CHANGELOG.md" unless File.exist?("CHANGELOG.md")
|
92
|
+
evaluate_file "README.md.erb", "README.md" unless File.exist?("README.md")
|
93
|
+
copy_file "VERSION" unless File.exist?("VERSION")
|
94
94
|
end
|
95
95
|
|
96
96
|
def check_folder_presence
|
97
97
|
folders = %w(domains forms variables).reject { |f| Dir.exist?(f) }
|
98
98
|
folders.each do |folder|
|
99
99
|
directory folder
|
100
|
-
copy_file
|
100
|
+
copy_file "keep", "#{folder}/.keep"
|
101
101
|
end
|
102
102
|
end
|
103
103
|
|
104
104
|
def check_test_folder
|
105
|
-
return if Dir.exist?(
|
106
|
-
directory
|
107
|
-
copy_file
|
108
|
-
copy_file
|
105
|
+
return if Dir.exist?("test")
|
106
|
+
directory "test"
|
107
|
+
copy_file "test/dictionary_test.rb"
|
108
|
+
copy_file "test/test_helper.rb"
|
109
109
|
end
|
110
110
|
end
|
111
111
|
end
|
@@ -116,13 +116,13 @@ module Spout
|
|
116
116
|
module Helpers
|
117
117
|
class ArrayStatistics
|
118
118
|
def self.calculations
|
119
|
-
[[
|
120
|
-
[
|
121
|
-
[
|
122
|
-
[
|
123
|
-
[
|
124
|
-
[
|
125
|
-
[
|
119
|
+
[["N", :n, :count],
|
120
|
+
["Mean", :mean, :decimal],
|
121
|
+
["StdDev", :standard_deviation, :decimal, "± %s"],
|
122
|
+
["Median", :median, :decimal],
|
123
|
+
["Min", :compact_min, :decimal],
|
124
|
+
["Max", :compact_max, :decimal],
|
125
|
+
["Unknown", :unknown, :count]]
|
126
126
|
end
|
127
127
|
end
|
128
128
|
end
|
@@ -1,45 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "yaml"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
|
+
# Loads the .spout.yml configuration file.
|
7
8
|
class ConfigReader
|
8
|
-
|
9
9
|
attr_reader :slug, :visit, :charts, :webservers
|
10
10
|
|
11
11
|
def initialize
|
12
|
-
@slug =
|
13
|
-
@visit =
|
12
|
+
@slug = ""
|
13
|
+
@visit = ""
|
14
14
|
@charts = []
|
15
15
|
@webservers = []
|
16
16
|
parse_yaml_file
|
17
17
|
end
|
18
18
|
|
19
19
|
def parse_yaml_file
|
20
|
-
spout_config = YAML.load_file(
|
20
|
+
spout_config = YAML.load_file(".spout.yml")
|
21
21
|
|
22
22
|
if spout_config.is_a?(Hash)
|
23
|
-
@slug = spout_config[
|
24
|
-
@visit = spout_config[
|
25
|
-
|
26
|
-
@charts =
|
27
|
-
spout_config[
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
23
|
+
@slug = spout_config["slug"].to_s.strip
|
24
|
+
@visit = spout_config["visit"].to_s.strip
|
25
|
+
|
26
|
+
@charts = \
|
27
|
+
if spout_config["charts"].is_a?(Array)
|
28
|
+
spout_config["charts"].select { |c| c.is_a?(Hash) }
|
29
|
+
else
|
30
|
+
[]
|
31
|
+
end
|
32
|
+
|
33
|
+
@webservers = \
|
34
|
+
if spout_config["webservers"].is_a?(Array)
|
35
|
+
spout_config["webservers"].select { |c| c.is_a?(Hash) }
|
36
|
+
else
|
37
|
+
[]
|
38
|
+
end
|
37
39
|
else
|
38
40
|
puts "The YAML file needs to be in the following format:"
|
39
41
|
puts "---\nvisit: visit_variable_name\ncharts:\n- chart: age_variable_name\n title: Age\n- chart: gender_variable_name\n title: Gender\n- chart: race_variable_name\n title: Race\n"
|
40
42
|
end
|
41
43
|
end
|
42
|
-
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
@@ -1,21 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "date"
|
5
|
+
require "erb"
|
6
|
+
require "fileutils"
|
7
7
|
|
8
|
-
TEMPLATES_DIRECTORY = File.expand_path(
|
8
|
+
TEMPLATES_DIRECTORY = File.expand_path("../../templates", __FILE__)
|
9
9
|
|
10
10
|
module Spout
|
11
11
|
module Helpers
|
12
12
|
# Helpers to generate and update Spout dictionary framework.
|
13
13
|
module Framework
|
14
|
-
def copy_file(template_file, file_name =
|
15
|
-
file_name = template_file if file_name ==
|
14
|
+
def copy_file(template_file, file_name = "")
|
15
|
+
file_name = template_file if file_name == ""
|
16
16
|
file_path = File.join(@full_path, file_name)
|
17
17
|
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
18
|
-
puts
|
18
|
+
puts " create".colorize(:green) + " #{file_name}"
|
19
19
|
FileUtils.copy(template_file_path, file_path)
|
20
20
|
end
|
21
21
|
|
@@ -23,16 +23,16 @@ module Spout
|
|
23
23
|
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
24
24
|
template = ERB.new(File.read(template_file_path))
|
25
25
|
file_path = File.join(@full_path, file_name)
|
26
|
-
file_out = File.new(file_path,
|
26
|
+
file_out = File.new(file_path, "w")
|
27
27
|
file_out.syswrite(template.result(binding))
|
28
|
-
puts
|
28
|
+
puts " create".colorize(:green) + " #{file_name}"
|
29
29
|
ensure
|
30
30
|
file_out.close if file_out
|
31
31
|
end
|
32
32
|
|
33
33
|
def directory(directory_name)
|
34
34
|
directory_path = File.join(@full_path, directory_name)
|
35
|
-
puts
|
35
|
+
puts " create".colorize(:green) + " #{directory_name}"
|
36
36
|
FileUtils.mkpath(directory_path)
|
37
37
|
end
|
38
38
|
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "spout/models/dictionary"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
7
|
module Iterators
|
8
|
-
|
9
8
|
def self.included(c)
|
10
9
|
class << c; attr_accessor :dictionary, :variables, :domains, :forms; end
|
11
10
|
c.instance_variable_set(:@dictionary, Spout::Models::Dictionary.new(Dir.pwd).load_all!)
|
@@ -13,7 +12,6 @@ module Spout
|
|
13
12
|
c.instance_variable_set(:@domains, c.instance_variable_get(:@dictionary).domains)
|
14
13
|
c.instance_variable_set(:@forms, c.instance_variable_get(:@dictionary).forms)
|
15
14
|
end
|
16
|
-
|
17
15
|
end
|
18
16
|
end
|
19
17
|
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "json"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
7
|
class JsonLoader
|
8
|
-
|
9
8
|
def self.get_json(file_name, file_type)
|
10
9
|
file = Dir.glob("#{file_type.to_s.downcase}s/**/#{file_name.to_s.downcase}.json", File::FNM_CASEFOLD).first
|
11
10
|
json = JSON.parse(File.read(file)) rescue json = nil
|
@@ -13,9 +12,8 @@ module Spout
|
|
13
12
|
end
|
14
13
|
|
15
14
|
def self.get_variable(variable_name)
|
16
|
-
get_json(variable_name,
|
15
|
+
get_json(variable_name, "variable")
|
17
16
|
end
|
18
|
-
|
19
17
|
end
|
20
18
|
end
|
21
19
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
|
+
require "cgi"
|
7
7
|
|
8
8
|
module Spout
|
9
9
|
module Helpers
|
@@ -30,7 +30,7 @@ module Spout
|
|
30
30
|
@url = URI.parse(url)
|
31
31
|
|
32
32
|
@http = Net::HTTP.new(@url.host, @url.port)
|
33
|
-
if @url.scheme ==
|
33
|
+
if @url.scheme == "https"
|
34
34
|
@http.use_ssl = true
|
35
35
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
36
36
|
end
|
@@ -42,8 +42,8 @@ module Spout
|
|
42
42
|
def get
|
43
43
|
return unless @error.nil?
|
44
44
|
full_path = @url.path
|
45
|
-
query = ([@url.query] + @params).flatten.compact.join(
|
46
|
-
full_path += "?#{query}" if query.to_s !=
|
45
|
+
query = ([@url.query] + @params).flatten.compact.join("&")
|
46
|
+
full_path += "?#{query}" if query.to_s != ""
|
47
47
|
response = @http.start do |http|
|
48
48
|
http.get(full_path)
|
49
49
|
end
|
@@ -55,7 +55,7 @@ module Spout
|
|
55
55
|
def post
|
56
56
|
return unless @error.nil?
|
57
57
|
response = @http.start do |http|
|
58
|
-
http.post(@url.path, @params.flatten.compact.join(
|
58
|
+
http.post(@url.path, @params.flatten.compact.join("&"))
|
59
59
|
end
|
60
60
|
[JSON.parse(response.body), response]
|
61
61
|
rescue => e
|
@@ -64,7 +64,7 @@ module Spout
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def patch
|
67
|
-
@params <<
|
67
|
+
@params << "_method=patch"
|
68
68
|
post
|
69
69
|
end
|
70
70
|
|
@@ -79,10 +79,10 @@ module Spout
|
|
79
79
|
if value.is_a? Hash
|
80
80
|
value.collect do |k,v|
|
81
81
|
key_value_to_string(k, v, current_scope)
|
82
|
-
end.join(
|
82
|
+
end.join("&")
|
83
83
|
elsif value.is_a? Array
|
84
84
|
value.collect do |v|
|
85
|
-
key_value_to_string(
|
85
|
+
key_value_to_string("", v, current_scope)
|
86
86
|
end
|
87
87
|
else
|
88
88
|
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
@@ -7,7 +7,7 @@ module Spout
|
|
7
7
|
# From Rails: http://apidock.com/rails/v3.2.13/Kernel/silence_stream
|
8
8
|
def silence_stream(stream)
|
9
9
|
old_stream = stream.dup
|
10
|
-
stream.reopen(/mswin|mingw/ =~ RbConfig::CONFIG[
|
10
|
+
stream.reopen(/mswin|mingw/ =~ RbConfig::CONFIG["host_os"] ? "NUL:" : "/dev/null")
|
11
11
|
stream.sync = true
|
12
12
|
yield
|
13
13
|
ensure
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
6
|
|
7
7
|
module Spout
|
8
8
|
module Helpers
|
@@ -17,13 +17,13 @@ module Spout
|
|
17
17
|
|
18
18
|
def initialize(url, filename, version, token, slug, folder)
|
19
19
|
@params = {}
|
20
|
-
@params[
|
21
|
-
@params[
|
22
|
-
@params[
|
23
|
-
@params[
|
20
|
+
@params["version"] = version
|
21
|
+
@params["auth_token"] = token if token
|
22
|
+
@params["dataset"] = slug if slug
|
23
|
+
@params["folder"] = folder if folder
|
24
24
|
begin
|
25
|
-
file = File.open(filename,
|
26
|
-
@params[
|
25
|
+
file = File.open(filename, "rb")
|
26
|
+
@params["file"] = file
|
27
27
|
|
28
28
|
mp = Multipart::MultipartPost.new
|
29
29
|
@query, @headers = mp.prepare_query(@params)
|
@@ -35,7 +35,7 @@ module Spout
|
|
35
35
|
@url = URI.parse(url)
|
36
36
|
|
37
37
|
@http = Net::HTTP.new(@url.host, @url.port)
|
38
|
-
if @url.scheme ==
|
38
|
+
if @url.scheme == "https"
|
39
39
|
@http.use_ssl = true
|
40
40
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
41
41
|
end
|
@@ -77,14 +77,14 @@ module Multipart
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def to_multipart
|
80
|
-
mime_type =
|
80
|
+
mime_type = "application/octet-stream"
|
81
81
|
"Content-Disposition: form-data; name=\"#{k}\"; filename=\"#{filename}\"\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-Type: #{mime_type}\r\n\r\n" + content + "\r\n"
|
82
82
|
end
|
83
83
|
end
|
84
84
|
|
85
85
|
class MultipartPost
|
86
|
-
BOUNDARY =
|
87
|
-
HEADER = {
|
86
|
+
BOUNDARY = "a#41-93r1-^Õ-rule0000"
|
87
|
+
HEADER = { "Content-type" => "multipart/form-data, boundary=#{BOUNDARY} " }
|
88
88
|
|
89
89
|
def prepare_query(params)
|
90
90
|
fp = []
|
@@ -95,7 +95,7 @@ module Multipart
|
|
95
95
|
fp.push(Param.new(k, v))
|
96
96
|
end
|
97
97
|
end
|
98
|
-
query = fp.collect { |p| "--#{BOUNDARY}\r\n" + p.to_multipart }.join(
|
98
|
+
query = fp.collect { |p| "--#{BOUNDARY}\r\n" + p.to_multipart }.join("") + "--#{BOUNDARY}--"
|
99
99
|
return query, HEADER
|
100
100
|
end
|
101
101
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "csv"
|
5
|
+
require "json"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
7
|
+
require "spout/models/subject"
|
8
|
+
require "spout/helpers/semantic"
|
9
|
+
require "spout/models/empty"
|
10
10
|
|
11
11
|
module Spout
|
12
12
|
module Helpers
|
@@ -24,7 +24,7 @@ module Spout
|
|
24
24
|
@all_methods = {}
|
25
25
|
@all_domains = []
|
26
26
|
@csv_files = []
|
27
|
-
@csv_directory =
|
27
|
+
@csv_directory = ""
|
28
28
|
end
|
29
29
|
|
30
30
|
def load_subjects_from_csvs!
|
@@ -35,31 +35,31 @@ module Spout
|
|
35
35
|
def load_subjects_from_csvs_part_one!
|
36
36
|
@subjects = []
|
37
37
|
|
38
|
-
available_folders = (Dir.exist?(
|
38
|
+
available_folders = (Dir.exist?("csvs") ? Dir.entries("csvs").select { |e| File.directory? File.join("csvs", e) }.reject { |e| [".", ".."].include?(e) }.sort : [])
|
39
39
|
|
40
40
|
@semantic = Spout::Helpers::Semantic.new(@standard_version, available_folders)
|
41
41
|
|
42
42
|
@csv_directory = @semantic.selected_folder
|
43
43
|
|
44
|
-
csv_root = File.join(
|
44
|
+
csv_root = File.join("csvs", @csv_directory)
|
45
45
|
@csv_files = Dir.glob("#{csv_root}/**/*.csv").sort
|
46
46
|
|
47
47
|
if @csv_directory != @standard_version
|
48
|
-
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) +
|
48
|
+
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + " for dictionary version " + @standard_version.to_s.colorize(:green) + "\n"
|
49
49
|
else
|
50
50
|
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + "\n"
|
51
51
|
end
|
52
52
|
|
53
53
|
last_folder = nil
|
54
54
|
@csv_files.each do |csv_file|
|
55
|
-
relative_path = csv_file.gsub(%r{^#{csv_root}},
|
55
|
+
relative_path = csv_file.gsub(%r{^#{csv_root}}, "")
|
56
56
|
current_file = File.basename(relative_path)
|
57
|
-
current_folder = relative_path.gsub(/#{current_file}$/,
|
57
|
+
current_folder = relative_path.gsub(/#{current_file}$/, "")
|
58
58
|
count = 1 # Includes counting the header row
|
59
|
-
puts " #{current_folder}".colorize(:white) if current_folder.to_s !=
|
59
|
+
puts " #{current_folder}".colorize(:white) if current_folder.to_s != "" && current_folder != last_folder
|
60
60
|
print " #{current_file}"
|
61
61
|
last_folder = current_folder
|
62
|
-
CSV.parse(File.open(csv_file,
|
62
|
+
CSV.parse(File.open(csv_file, "r:iso-8859-1:utf-8", &:read), headers: true, header_converters: lambda { |h| h.to_s.downcase }) do |line|
|
63
63
|
row = line.to_hash
|
64
64
|
count += 1
|
65
65
|
print "\r #{current_file} " + "##{count}".colorize(:yellow) if (count % 10 == 0)
|
@@ -69,7 +69,7 @@ module Spout
|
|
69
69
|
|
70
70
|
row.each_with_index do |(key, value), index|
|
71
71
|
method = key.to_s.downcase.strip
|
72
|
-
if method ==
|
72
|
+
if method == ""
|
73
73
|
puts "\nSkipping column #{index + 1} due to blank header.".colorize(:red) if count == 2
|
74
74
|
next
|
75
75
|
end
|
@@ -99,20 +99,20 @@ module Spout
|
|
99
99
|
|
100
100
|
def load_subjects_from_csvs_part_two!
|
101
101
|
variable_count = @variable_files.count
|
102
|
-
print
|
102
|
+
print "Converting numeric values to floats"
|
103
103
|
@variable_files.each_with_index do |variable_file, index|
|
104
104
|
print "\rConverting numeric values to floats:#{'% 3d' % ((index + 1) * 100 / variable_count)}%"
|
105
105
|
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
106
106
|
next unless json
|
107
|
-
next unless @valid_ids.include?(json[
|
108
|
-
next unless %w(numeric integer).include?(json[
|
109
|
-
method = json[
|
107
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) || @valid_ids.size == 0
|
108
|
+
next unless %w(numeric integer).include?(json["type"])
|
109
|
+
method = json["id"].to_s.downcase
|
110
110
|
next unless Spout::Models::Subject.method_defined?(method)
|
111
111
|
|
112
112
|
domain_json = get_domain(json)
|
113
113
|
# Make all domain options nil for numerics/integers
|
114
114
|
if domain_json
|
115
|
-
domain_values = domain_json.collect { |option_hash| option_hash[
|
115
|
+
domain_values = domain_json.collect { |option_hash| option_hash["value"] }
|
116
116
|
@subjects.each { |s| domain_values.include?(s.send(method)) ? s.send("#{method}=", nil) : nil }
|
117
117
|
end
|
118
118
|
|
@@ -126,8 +126,8 @@ module Spout
|
|
126
126
|
@variable_files.each do |variable_file|
|
127
127
|
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
128
128
|
next unless json
|
129
|
-
next unless json[
|
130
|
-
domain = json[
|
129
|
+
next unless json["type"] == "choices" || json["domain"].to_s.downcase.strip != ""
|
130
|
+
domain = json["domain"].to_s.downcase
|
131
131
|
@all_domains << domain
|
132
132
|
end
|
133
133
|
@all_domains = @all_domains.compact.uniq.sort
|
@@ -141,11 +141,11 @@ module Spout
|
|
141
141
|
end
|
142
142
|
|
143
143
|
def get_variable(variable_name)
|
144
|
-
get_json(variable_name,
|
144
|
+
get_json(variable_name, "variable")
|
145
145
|
end
|
146
146
|
|
147
147
|
def get_domain(json)
|
148
|
-
get_json(json[
|
148
|
+
get_json(json["domain"], "domain")
|
149
149
|
end
|
150
150
|
end
|
151
151
|
end
|