spout 0.12.1 → 0.13.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +5 -5
- data/Rakefile +4 -4
- data/bin/spout +1 -1
- data/lib/spout/commands/coverage.rb +18 -18
- data/lib/spout/commands/deploy.rb +89 -89
- data/lib/spout/commands/exporter.rb +16 -16
- data/lib/spout/commands/graphs.rb +37 -37
- data/lib/spout/commands/help.rb +1 -1
- data/lib/spout/commands/importer.rb +59 -57
- data/lib/spout/commands/outliers.rb +17 -17
- data/lib/spout/commands/project_generator.rb +25 -25
- data/lib/spout/commands/update.rb +38 -38
- data/lib/spout/helpers/array_statistics.rb +7 -7
- data/lib/spout/helpers/chart_types.rb +2 -2
- data/lib/spout/helpers/config_reader.rb +21 -20
- data/lib/spout/helpers/framework.rb +11 -11
- data/lib/spout/helpers/iterators.rb +1 -3
- data/lib/spout/helpers/json_loader.rb +2 -4
- data/lib/spout/helpers/json_request.rb +11 -11
- data/lib/spout/helpers/number_helper.rb +1 -1
- data/lib/spout/helpers/quietly.rb +1 -1
- data/lib/spout/helpers/semantic.rb +1 -1
- data/lib/spout/helpers/send_file.rb +14 -14
- data/lib/spout/helpers/subject_loader.rb +24 -24
- data/lib/spout/helpers/table_formatting.rb +12 -12
- data/lib/spout/models/coverage_result.rb +8 -8
- data/lib/spout/models/dictionary.rb +10 -10
- data/lib/spout/models/domain.rb +7 -7
- data/lib/spout/models/empty.rb +1 -1
- data/lib/spout/models/form.rb +6 -6
- data/lib/spout/models/graphables/choices_vs_choices.rb +3 -5
- data/lib/spout/models/graphables/choices_vs_numeric.rb +4 -6
- data/lib/spout/models/graphables/default.rb +6 -6
- data/lib/spout/models/graphables/histogram.rb +4 -4
- data/lib/spout/models/graphables/numeric_vs_choices.rb +1 -1
- data/lib/spout/models/graphables/numeric_vs_numeric.rb +6 -6
- data/lib/spout/models/graphables.rb +14 -14
- data/lib/spout/models/outlier_result.rb +16 -18
- data/lib/spout/models/record.rb +3 -3
- data/lib/spout/models/tables/choices_vs_choices.rb +11 -11
- data/lib/spout/models/tables/choices_vs_numeric.rb +8 -8
- data/lib/spout/models/tables/default.rb +5 -5
- data/lib/spout/models/tables/numeric_vs_choices.rb +7 -7
- data/lib/spout/models/tables/numeric_vs_numeric.rb +7 -7
- data/lib/spout/models/tables.rb +11 -11
- data/lib/spout/models/variable.rb +14 -14
- data/lib/spout/tasks/engine.rake +4 -4
- data/lib/spout/templates/CHANGELOG.md.erb +1 -1
- data/lib/spout/templates/Gemfile +2 -2
- data/lib/spout/templates/Rakefile +1 -1
- data/lib/spout/templates/test/dictionary_test.rb +3 -3
- data/lib/spout/templates/test/test_helper.rb +1 -1
- data/lib/spout/tests/domain_existence_validation.rb +6 -6
- data/lib/spout/tests/domain_format.rb +1 -1
- data/lib/spout/tests/domain_name_format.rb +3 -3
- data/lib/spout/tests/domain_name_uniqueness.rb +2 -2
- data/lib/spout/tests/domain_specified.rb +1 -3
- data/lib/spout/tests/form_existence_validation.rb +2 -4
- data/lib/spout/tests/form_name_format.rb +3 -3
- data/lib/spout/tests/form_name_match.rb +1 -1
- data/lib/spout/tests/json_helper.rb +1 -1
- data/lib/spout/tests/json_validation.rb +0 -2
- data/lib/spout/tests/variable_name_format.rb +3 -3
- data/lib/spout/tests/variable_name_match.rb +1 -3
- data/lib/spout/tests/variable_name_uniqueness.rb +2 -2
- data/lib/spout/tests/variable_type_validation.rb +2 -2
- data/lib/spout/tests.rb +30 -30
- data/lib/spout/version.rb +4 -4
- data/lib/spout/views/index.html.erb +15 -15
- data/lib/spout/views/outliers.html.erb +8 -8
- data/lib/spout.rb +26 -26
- data/spout.gemspec +22 -22
- metadata +5 -5
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "spout/helpers/json_request"
|
5
|
+
require "spout/helpers/framework"
|
6
6
|
|
7
7
|
module Spout
|
8
8
|
module Commands
|
@@ -17,27 +17,27 @@ module Spout
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def initialize(argv)
|
20
|
-
@full_path = File.join(
|
20
|
+
@full_path = File.join(".")
|
21
21
|
end
|
22
22
|
|
23
23
|
def start
|
24
|
-
(json, _status) = Spout::Helpers::JsonRequest.get(
|
24
|
+
(json, _status) = Spout::Helpers::JsonRequest.get("https://rubygems.org/api/v1/gems/spout.json")
|
25
25
|
if json
|
26
|
-
if json[
|
27
|
-
puts
|
28
|
-
check_framework if File.exist?(
|
26
|
+
if json["version"] == Spout::VERSION::STRING
|
27
|
+
puts "The spout gem is " + "up-to-date".colorize(:green) + "!"
|
28
|
+
check_framework if File.exist?("Gemfile")
|
29
29
|
else
|
30
30
|
puts "A newer version (v#{json['version']}) is available!\n\n"
|
31
|
-
if File.exist?(
|
32
|
-
puts
|
33
|
-
puts " gem
|
31
|
+
if File.exist?("Gemfile")
|
32
|
+
puts "Add the following to your Gemfile and run " + "bundle update".colorize(:green) + ".\n\n"
|
33
|
+
puts " gem \"spout\", \"~> #{json['version']}\"\n".colorize(:white)
|
34
34
|
else
|
35
35
|
puts "Type the following command to update:\n\n"
|
36
|
-
puts
|
36
|
+
puts " gem install spout --no-document".colorize(:white) + "\n\n"
|
37
37
|
end
|
38
38
|
end
|
39
39
|
else
|
40
|
-
puts
|
40
|
+
puts "Unable to connect to RubyGems.org. Please try again later."
|
41
41
|
end
|
42
42
|
end
|
43
43
|
|
@@ -50,62 +50,62 @@ module Spout
|
|
50
50
|
end
|
51
51
|
|
52
52
|
def check_gitignore_file
|
53
|
-
if File.exist?(
|
54
|
-
lines = IO.readlines(
|
55
|
-
addables = [
|
56
|
-
removables = [
|
53
|
+
if File.exist?(".gitignore")
|
54
|
+
lines = IO.readlines(".gitignore").collect(&:strip)
|
55
|
+
addables = ["/coverage", "/csvs", "/exports", "/graphs"]
|
56
|
+
removables = ["/dd", "/images"]
|
57
57
|
unless ((removables & lines) | (addables - lines)).empty?
|
58
|
-
puts
|
59
|
-
puts
|
58
|
+
puts "File: " + ".gitignore".colorize(:white)
|
59
|
+
puts "----------------"
|
60
60
|
(removables & lines).each do |removable|
|
61
|
-
puts
|
61
|
+
puts "REMOVE LINE ".colorize(:red) + removable.colorize(:white)
|
62
62
|
end
|
63
63
|
(addables - lines).each do |addable|
|
64
|
-
puts
|
64
|
+
puts " ADD LINE ".colorize(:green) + addable.colorize(:white)
|
65
65
|
end
|
66
66
|
puts
|
67
67
|
end
|
68
68
|
else
|
69
|
-
copy_file
|
69
|
+
copy_file "gitignore", ".gitignore"
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
73
|
def check_ruby_version
|
74
|
-
if File.exist?(
|
75
|
-
lines = IO.readlines(
|
76
|
-
template_lines = IO.readlines(File.expand_path(
|
74
|
+
if File.exist?(".ruby-version")
|
75
|
+
lines = IO.readlines(".ruby-version").collect(&:strip)
|
76
|
+
template_lines = IO.readlines(File.expand_path("../../templates/ruby-version", __FILE__)).collect(&:strip)
|
77
77
|
if template_lines.first != lines.first
|
78
|
-
puts
|
79
|
-
puts
|
80
|
-
print
|
81
|
-
print
|
78
|
+
puts "File: " + ".ruby-version".colorize(:white)
|
79
|
+
puts "-------------------"
|
80
|
+
print "Update Ruby from " + lines.first.to_s.colorize(:red)
|
81
|
+
print " to " + template_lines.first.to_s.colorize(:green)
|
82
82
|
puts "\n\n"
|
83
83
|
end
|
84
84
|
else
|
85
|
-
copy_file
|
85
|
+
copy_file "ruby-version", ".ruby-version"
|
86
86
|
end
|
87
87
|
end
|
88
88
|
|
89
89
|
def check_file_presence
|
90
90
|
@project_name = File.basename(Dir.pwd)
|
91
|
-
evaluate_file
|
92
|
-
evaluate_file
|
93
|
-
copy_file
|
91
|
+
evaluate_file "CHANGELOG.md.erb", "CHANGELOG.md" unless File.exist?("CHANGELOG.md")
|
92
|
+
evaluate_file "README.md.erb", "README.md" unless File.exist?("README.md")
|
93
|
+
copy_file "VERSION" unless File.exist?("VERSION")
|
94
94
|
end
|
95
95
|
|
96
96
|
def check_folder_presence
|
97
97
|
folders = %w(domains forms variables).reject { |f| Dir.exist?(f) }
|
98
98
|
folders.each do |folder|
|
99
99
|
directory folder
|
100
|
-
copy_file
|
100
|
+
copy_file "keep", "#{folder}/.keep"
|
101
101
|
end
|
102
102
|
end
|
103
103
|
|
104
104
|
def check_test_folder
|
105
|
-
return if Dir.exist?(
|
106
|
-
directory
|
107
|
-
copy_file
|
108
|
-
copy_file
|
105
|
+
return if Dir.exist?("test")
|
106
|
+
directory "test"
|
107
|
+
copy_file "test/dictionary_test.rb"
|
108
|
+
copy_file "test/test_helper.rb"
|
109
109
|
end
|
110
110
|
end
|
111
111
|
end
|
@@ -116,13 +116,13 @@ module Spout
|
|
116
116
|
module Helpers
|
117
117
|
class ArrayStatistics
|
118
118
|
def self.calculations
|
119
|
-
[[
|
120
|
-
[
|
121
|
-
[
|
122
|
-
[
|
123
|
-
[
|
124
|
-
[
|
125
|
-
[
|
119
|
+
[["N", :n, :count],
|
120
|
+
["Mean", :mean, :decimal],
|
121
|
+
["StdDev", :standard_deviation, :decimal, "± %s"],
|
122
|
+
["Median", :median, :decimal],
|
123
|
+
["Min", :compact_min, :decimal],
|
124
|
+
["Max", :compact_max, :decimal],
|
125
|
+
["Unknown", :unknown, :count]]
|
126
126
|
end
|
127
127
|
end
|
128
128
|
end
|
@@ -1,45 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "yaml"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
|
+
# Loads the .spout.yml configuration file.
|
7
8
|
class ConfigReader
|
8
|
-
|
9
9
|
attr_reader :slug, :visit, :charts, :webservers
|
10
10
|
|
11
11
|
def initialize
|
12
|
-
@slug =
|
13
|
-
@visit =
|
12
|
+
@slug = ""
|
13
|
+
@visit = ""
|
14
14
|
@charts = []
|
15
15
|
@webservers = []
|
16
16
|
parse_yaml_file
|
17
17
|
end
|
18
18
|
|
19
19
|
def parse_yaml_file
|
20
|
-
spout_config = YAML.load_file(
|
20
|
+
spout_config = YAML.load_file(".spout.yml")
|
21
21
|
|
22
22
|
if spout_config.is_a?(Hash)
|
23
|
-
@slug = spout_config[
|
24
|
-
@visit = spout_config[
|
25
|
-
|
26
|
-
@charts =
|
27
|
-
spout_config[
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
23
|
+
@slug = spout_config["slug"].to_s.strip
|
24
|
+
@visit = spout_config["visit"].to_s.strip
|
25
|
+
|
26
|
+
@charts = \
|
27
|
+
if spout_config["charts"].is_a?(Array)
|
28
|
+
spout_config["charts"].select { |c| c.is_a?(Hash) }
|
29
|
+
else
|
30
|
+
[]
|
31
|
+
end
|
32
|
+
|
33
|
+
@webservers = \
|
34
|
+
if spout_config["webservers"].is_a?(Array)
|
35
|
+
spout_config["webservers"].select { |c| c.is_a?(Hash) }
|
36
|
+
else
|
37
|
+
[]
|
38
|
+
end
|
37
39
|
else
|
38
40
|
puts "The YAML file needs to be in the following format:"
|
39
41
|
puts "---\nvisit: visit_variable_name\ncharts:\n- chart: age_variable_name\n title: Age\n- chart: gender_variable_name\n title: Gender\n- chart: race_variable_name\n title: Race\n"
|
40
42
|
end
|
41
43
|
end
|
42
|
-
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
@@ -1,21 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "date"
|
5
|
+
require "erb"
|
6
|
+
require "fileutils"
|
7
7
|
|
8
|
-
TEMPLATES_DIRECTORY = File.expand_path(
|
8
|
+
TEMPLATES_DIRECTORY = File.expand_path("../../templates", __FILE__)
|
9
9
|
|
10
10
|
module Spout
|
11
11
|
module Helpers
|
12
12
|
# Helpers to generate and update Spout dictionary framework.
|
13
13
|
module Framework
|
14
|
-
def copy_file(template_file, file_name =
|
15
|
-
file_name = template_file if file_name ==
|
14
|
+
def copy_file(template_file, file_name = "")
|
15
|
+
file_name = template_file if file_name == ""
|
16
16
|
file_path = File.join(@full_path, file_name)
|
17
17
|
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
18
|
-
puts
|
18
|
+
puts " create".colorize(:green) + " #{file_name}"
|
19
19
|
FileUtils.copy(template_file_path, file_path)
|
20
20
|
end
|
21
21
|
|
@@ -23,16 +23,16 @@ module Spout
|
|
23
23
|
template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
|
24
24
|
template = ERB.new(File.read(template_file_path))
|
25
25
|
file_path = File.join(@full_path, file_name)
|
26
|
-
file_out = File.new(file_path,
|
26
|
+
file_out = File.new(file_path, "w")
|
27
27
|
file_out.syswrite(template.result(binding))
|
28
|
-
puts
|
28
|
+
puts " create".colorize(:green) + " #{file_name}"
|
29
29
|
ensure
|
30
30
|
file_out.close if file_out
|
31
31
|
end
|
32
32
|
|
33
33
|
def directory(directory_name)
|
34
34
|
directory_path = File.join(@full_path, directory_name)
|
35
|
-
puts
|
35
|
+
puts " create".colorize(:green) + " #{directory_name}"
|
36
36
|
FileUtils.mkpath(directory_path)
|
37
37
|
end
|
38
38
|
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "spout/models/dictionary"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
7
|
module Iterators
|
8
|
-
|
9
8
|
def self.included(c)
|
10
9
|
class << c; attr_accessor :dictionary, :variables, :domains, :forms; end
|
11
10
|
c.instance_variable_set(:@dictionary, Spout::Models::Dictionary.new(Dir.pwd).load_all!)
|
@@ -13,7 +12,6 @@ module Spout
|
|
13
12
|
c.instance_variable_set(:@domains, c.instance_variable_get(:@dictionary).domains)
|
14
13
|
c.instance_variable_set(:@forms, c.instance_variable_get(:@dictionary).forms)
|
15
14
|
end
|
16
|
-
|
17
15
|
end
|
18
16
|
end
|
19
17
|
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "json"
|
4
4
|
|
5
5
|
module Spout
|
6
6
|
module Helpers
|
7
7
|
class JsonLoader
|
8
|
-
|
9
8
|
def self.get_json(file_name, file_type)
|
10
9
|
file = Dir.glob("#{file_type.to_s.downcase}s/**/#{file_name.to_s.downcase}.json", File::FNM_CASEFOLD).first
|
11
10
|
json = JSON.parse(File.read(file)) rescue json = nil
|
@@ -13,9 +12,8 @@ module Spout
|
|
13
12
|
end
|
14
13
|
|
15
14
|
def self.get_variable(variable_name)
|
16
|
-
get_json(variable_name,
|
15
|
+
get_json(variable_name, "variable")
|
17
16
|
end
|
18
|
-
|
19
17
|
end
|
20
18
|
end
|
21
19
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
|
+
require "cgi"
|
7
7
|
|
8
8
|
module Spout
|
9
9
|
module Helpers
|
@@ -30,7 +30,7 @@ module Spout
|
|
30
30
|
@url = URI.parse(url)
|
31
31
|
|
32
32
|
@http = Net::HTTP.new(@url.host, @url.port)
|
33
|
-
if @url.scheme ==
|
33
|
+
if @url.scheme == "https"
|
34
34
|
@http.use_ssl = true
|
35
35
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
36
36
|
end
|
@@ -42,8 +42,8 @@ module Spout
|
|
42
42
|
def get
|
43
43
|
return unless @error.nil?
|
44
44
|
full_path = @url.path
|
45
|
-
query = ([@url.query] + @params).flatten.compact.join(
|
46
|
-
full_path += "?#{query}" if query.to_s !=
|
45
|
+
query = ([@url.query] + @params).flatten.compact.join("&")
|
46
|
+
full_path += "?#{query}" if query.to_s != ""
|
47
47
|
response = @http.start do |http|
|
48
48
|
http.get(full_path)
|
49
49
|
end
|
@@ -55,7 +55,7 @@ module Spout
|
|
55
55
|
def post
|
56
56
|
return unless @error.nil?
|
57
57
|
response = @http.start do |http|
|
58
|
-
http.post(@url.path, @params.flatten.compact.join(
|
58
|
+
http.post(@url.path, @params.flatten.compact.join("&"))
|
59
59
|
end
|
60
60
|
[JSON.parse(response.body), response]
|
61
61
|
rescue => e
|
@@ -64,7 +64,7 @@ module Spout
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def patch
|
67
|
-
@params <<
|
67
|
+
@params << "_method=patch"
|
68
68
|
post
|
69
69
|
end
|
70
70
|
|
@@ -79,10 +79,10 @@ module Spout
|
|
79
79
|
if value.is_a? Hash
|
80
80
|
value.collect do |k,v|
|
81
81
|
key_value_to_string(k, v, current_scope)
|
82
|
-
end.join(
|
82
|
+
end.join("&")
|
83
83
|
elsif value.is_a? Array
|
84
84
|
value.collect do |v|
|
85
|
-
key_value_to_string(
|
85
|
+
key_value_to_string("", v, current_scope)
|
86
86
|
end
|
87
87
|
else
|
88
88
|
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
@@ -7,7 +7,7 @@ module Spout
|
|
7
7
|
# From Rails: http://apidock.com/rails/v3.2.13/Kernel/silence_stream
|
8
8
|
def silence_stream(stream)
|
9
9
|
old_stream = stream.dup
|
10
|
-
stream.reopen(/mswin|mingw/ =~ RbConfig::CONFIG[
|
10
|
+
stream.reopen(/mswin|mingw/ =~ RbConfig::CONFIG["host_os"] ? "NUL:" : "/dev/null")
|
11
11
|
stream.sync = true
|
12
12
|
yield
|
13
13
|
ensure
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
6
|
|
7
7
|
module Spout
|
8
8
|
module Helpers
|
@@ -17,13 +17,13 @@ module Spout
|
|
17
17
|
|
18
18
|
def initialize(url, filename, version, token, slug, folder)
|
19
19
|
@params = {}
|
20
|
-
@params[
|
21
|
-
@params[
|
22
|
-
@params[
|
23
|
-
@params[
|
20
|
+
@params["version"] = version
|
21
|
+
@params["auth_token"] = token if token
|
22
|
+
@params["dataset"] = slug if slug
|
23
|
+
@params["folder"] = folder if folder
|
24
24
|
begin
|
25
|
-
file = File.open(filename,
|
26
|
-
@params[
|
25
|
+
file = File.open(filename, "rb")
|
26
|
+
@params["file"] = file
|
27
27
|
|
28
28
|
mp = Multipart::MultipartPost.new
|
29
29
|
@query, @headers = mp.prepare_query(@params)
|
@@ -35,7 +35,7 @@ module Spout
|
|
35
35
|
@url = URI.parse(url)
|
36
36
|
|
37
37
|
@http = Net::HTTP.new(@url.host, @url.port)
|
38
|
-
if @url.scheme ==
|
38
|
+
if @url.scheme == "https"
|
39
39
|
@http.use_ssl = true
|
40
40
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
41
41
|
end
|
@@ -77,14 +77,14 @@ module Multipart
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def to_multipart
|
80
|
-
mime_type =
|
80
|
+
mime_type = "application/octet-stream"
|
81
81
|
"Content-Disposition: form-data; name=\"#{k}\"; filename=\"#{filename}\"\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-Type: #{mime_type}\r\n\r\n" + content + "\r\n"
|
82
82
|
end
|
83
83
|
end
|
84
84
|
|
85
85
|
class MultipartPost
|
86
|
-
BOUNDARY =
|
87
|
-
HEADER = {
|
86
|
+
BOUNDARY = "a#41-93r1-^Õ-rule0000"
|
87
|
+
HEADER = { "Content-type" => "multipart/form-data, boundary=#{BOUNDARY} " }
|
88
88
|
|
89
89
|
def prepare_query(params)
|
90
90
|
fp = []
|
@@ -95,7 +95,7 @@ module Multipart
|
|
95
95
|
fp.push(Param.new(k, v))
|
96
96
|
end
|
97
97
|
end
|
98
|
-
query = fp.collect { |p| "--#{BOUNDARY}\r\n" + p.to_multipart }.join(
|
98
|
+
query = fp.collect { |p| "--#{BOUNDARY}\r\n" + p.to_multipart }.join("") + "--#{BOUNDARY}--"
|
99
99
|
return query, HEADER
|
100
100
|
end
|
101
101
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "colorize"
|
4
|
+
require "csv"
|
5
|
+
require "json"
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
7
|
+
require "spout/models/subject"
|
8
|
+
require "spout/helpers/semantic"
|
9
|
+
require "spout/models/empty"
|
10
10
|
|
11
11
|
module Spout
|
12
12
|
module Helpers
|
@@ -24,7 +24,7 @@ module Spout
|
|
24
24
|
@all_methods = {}
|
25
25
|
@all_domains = []
|
26
26
|
@csv_files = []
|
27
|
-
@csv_directory =
|
27
|
+
@csv_directory = ""
|
28
28
|
end
|
29
29
|
|
30
30
|
def load_subjects_from_csvs!
|
@@ -35,31 +35,31 @@ module Spout
|
|
35
35
|
def load_subjects_from_csvs_part_one!
|
36
36
|
@subjects = []
|
37
37
|
|
38
|
-
available_folders = (Dir.exist?(
|
38
|
+
available_folders = (Dir.exist?("csvs") ? Dir.entries("csvs").select { |e| File.directory? File.join("csvs", e) }.reject { |e| [".", ".."].include?(e) }.sort : [])
|
39
39
|
|
40
40
|
@semantic = Spout::Helpers::Semantic.new(@standard_version, available_folders)
|
41
41
|
|
42
42
|
@csv_directory = @semantic.selected_folder
|
43
43
|
|
44
|
-
csv_root = File.join(
|
44
|
+
csv_root = File.join("csvs", @csv_directory)
|
45
45
|
@csv_files = Dir.glob("#{csv_root}/**/*.csv").sort
|
46
46
|
|
47
47
|
if @csv_directory != @standard_version
|
48
|
-
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) +
|
48
|
+
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + " for dictionary version " + @standard_version.to_s.colorize(:green) + "\n"
|
49
49
|
else
|
50
50
|
puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + "\n"
|
51
51
|
end
|
52
52
|
|
53
53
|
last_folder = nil
|
54
54
|
@csv_files.each do |csv_file|
|
55
|
-
relative_path = csv_file.gsub(%r{^#{csv_root}},
|
55
|
+
relative_path = csv_file.gsub(%r{^#{csv_root}}, "")
|
56
56
|
current_file = File.basename(relative_path)
|
57
|
-
current_folder = relative_path.gsub(/#{current_file}$/,
|
57
|
+
current_folder = relative_path.gsub(/#{current_file}$/, "")
|
58
58
|
count = 1 # Includes counting the header row
|
59
|
-
puts " #{current_folder}".colorize(:white) if current_folder.to_s !=
|
59
|
+
puts " #{current_folder}".colorize(:white) if current_folder.to_s != "" && current_folder != last_folder
|
60
60
|
print " #{current_file}"
|
61
61
|
last_folder = current_folder
|
62
|
-
CSV.parse(File.open(csv_file,
|
62
|
+
CSV.parse(File.open(csv_file, "r:iso-8859-1:utf-8", &:read), headers: true, header_converters: lambda { |h| h.to_s.downcase }) do |line|
|
63
63
|
row = line.to_hash
|
64
64
|
count += 1
|
65
65
|
print "\r #{current_file} " + "##{count}".colorize(:yellow) if (count % 10 == 0)
|
@@ -69,7 +69,7 @@ module Spout
|
|
69
69
|
|
70
70
|
row.each_with_index do |(key, value), index|
|
71
71
|
method = key.to_s.downcase.strip
|
72
|
-
if method ==
|
72
|
+
if method == ""
|
73
73
|
puts "\nSkipping column #{index + 1} due to blank header.".colorize(:red) if count == 2
|
74
74
|
next
|
75
75
|
end
|
@@ -99,20 +99,20 @@ module Spout
|
|
99
99
|
|
100
100
|
def load_subjects_from_csvs_part_two!
|
101
101
|
variable_count = @variable_files.count
|
102
|
-
print
|
102
|
+
print "Converting numeric values to floats"
|
103
103
|
@variable_files.each_with_index do |variable_file, index|
|
104
104
|
print "\rConverting numeric values to floats:#{'% 3d' % ((index + 1) * 100 / variable_count)}%"
|
105
105
|
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
106
106
|
next unless json
|
107
|
-
next unless @valid_ids.include?(json[
|
108
|
-
next unless %w(numeric integer).include?(json[
|
109
|
-
method = json[
|
107
|
+
next unless @valid_ids.include?(json["id"].to_s.downcase) || @valid_ids.size == 0
|
108
|
+
next unless %w(numeric integer).include?(json["type"])
|
109
|
+
method = json["id"].to_s.downcase
|
110
110
|
next unless Spout::Models::Subject.method_defined?(method)
|
111
111
|
|
112
112
|
domain_json = get_domain(json)
|
113
113
|
# Make all domain options nil for numerics/integers
|
114
114
|
if domain_json
|
115
|
-
domain_values = domain_json.collect { |option_hash| option_hash[
|
115
|
+
domain_values = domain_json.collect { |option_hash| option_hash["value"] }
|
116
116
|
@subjects.each { |s| domain_values.include?(s.send(method)) ? s.send("#{method}=", nil) : nil }
|
117
117
|
end
|
118
118
|
|
@@ -126,8 +126,8 @@ module Spout
|
|
126
126
|
@variable_files.each do |variable_file|
|
127
127
|
json = JSON.parse(File.read(variable_file)) rescue json = nil
|
128
128
|
next unless json
|
129
|
-
next unless json[
|
130
|
-
domain = json[
|
129
|
+
next unless json["type"] == "choices" || json["domain"].to_s.downcase.strip != ""
|
130
|
+
domain = json["domain"].to_s.downcase
|
131
131
|
@all_domains << domain
|
132
132
|
end
|
133
133
|
@all_domains = @all_domains.compact.uniq.sort
|
@@ -141,11 +141,11 @@ module Spout
|
|
141
141
|
end
|
142
142
|
|
143
143
|
def get_variable(variable_name)
|
144
|
-
get_json(variable_name,
|
144
|
+
get_json(variable_name, "variable")
|
145
145
|
end
|
146
146
|
|
147
147
|
def get_domain(json)
|
148
|
-
get_json(json[
|
148
|
+
get_json(json["domain"], "domain")
|
149
149
|
end
|
150
150
|
end
|
151
151
|
end
|