spout 0.12.0.beta2 → 0.12.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8d706e8d746c0ae4c64014bbcf38f47d51e388a
-  data.tar.gz: b2a49b2f165531f6a96ad677cb6a1d9f4045a283
+  metadata.gz: b6b577bc1b70562b8fd31c019ebfaf216fbb903c
+  data.tar.gz: 84a57bcf18809bcbe7241e969247ae45023fc583
 SHA512:
-  metadata.gz: 9b4fa147fc6914b594afd25acc72f5de98c929dbc2631489fd8267c1a3ff3e432a16292a28ce2f5565d011879263a3cfa265b004c1af9deebcf6525d6fa5411d
-  data.tar.gz: 9f3dfd7ee30b971b2d3152c486dac7bb87b9d3811a6d7fd30267bfdf9285ec4330cfece6956e217dffb64827fa7f902f9f43aec3aed4fd4ea25622b670965017
+  metadata.gz: 4498b53635947fb0810c6e3165b58048cf0855130cb6b3a75e47c24ba075d6509cbe10fe2ddcf7c2df01248a73d74f45e79b0847d4a35629c1f399c5ca4b17b0
+  data.tar.gz: c81170c50d3cf60e70f931a8eb75fa09dee569705ec687e513b09fdf7d67cbb418c8e5e5c7b9373d3e5fd7649b9f2feb92a114540e0987d8a18df767d47f7c0c

data/CHANGELOG.md

@@ -3,11 +3,17 @@
 ### Enhancements
 - **General Changes**
   - Spout now provides a warning and skips columns in CSVs with blank headers
+  - `spout new` command now generates `CHANGELOG.md`, `README.md`, and `VERSION`
+    placeholder files
+  - Check for the latest version available using `spout update`
+  - Integer and numeric variables can now reference a domain for missing values
 - **Exporter Changes**
   - The export command now exports the variable forms attribute
+  - Dictionary exports now are in `exports` instead of `dd` folder
 - **Importer Changes**
   - The import command now reads in the `commonly_used` column
   - The import command now imports the variable forms attribute
+  - CSVs of forms can now be imported using `spout import <forms.csv> --forms`
 - **Gem Changes**
   - Updated to Ruby 2.4.1
   - Updated to bundler 1.13
@@ -17,6 +23,9 @@
 ### Refactoring
 - General code cleanup based on Rubocop recommendations
 
+### Bug Fixes
+- The `test_units` example method now works if `nil` is defined in `VALID_UNITS`
+
 ## 0.11.1 (February 4, 2016)
 
 ### Bug Fix

data/README.md

@@ -105,6 +105,29 @@ Other columns that are imported include:
 `folder`: The name of the folder path where the domain resides.
 
 
+#### Importing forms from an existing CSV file
+
+```
+spout import data_dictionary_domains.csv --forms
+```
+
+The CSV should contain at minimal three column headers:
+
+`folder`: This can be blank, however it is used to place forms into a folder
+hiearchy. The folder column can contain forward slashes `/` to place a form
+into a subfolder. An example may be, `id`: `family_history`,
+`folder`: `Demographics/BaselineVisit` would create a file
+`forms/Demographics/BaselineVisit/family_history.json`
+
+`id`: The reference name of the form.
+
+`display_name`: The name of the form.
+
+Other columns that are imported include:
+
+`code_book`: The file name of the document or PDF, including the file extension.
+
+
 ### Test your repository
 
 If you created your data dictionary repository using `spout new`, you can go
@@ -166,9 +189,10 @@ class DictionaryTest < Minitest::Test
 
   @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
     define_method("test_units: #{variable.path}") do
-      puts variable.class
-      # => Spout::Models::Variable
-      assert VALID_UNITS.include?(variable.units)
+      message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
+                "             Valid types: " +
+                VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(', ')
+      assert VALID_UNITS.include?(variable.units), message
     end
   end
 end
@@ -388,3 +412,11 @@ The following steps are run:
   - `README.md` and `KNOWNISSUES.md` are uploaded
 - **Server-Side Updates**
   - Server refreshes dataset folder to reflect new dataset and data dictionaries
+
+### Check if you are using the latest version of Spout
+
+You can check if a newer version of Spout is available by typing:
+
+```
+spout update
+```

data/lib/spout/commands/coverage.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'yaml'
 require 'erb'
 require 'fileutils'
+require 'yaml'
 
 require 'spout/helpers/subject_loader'
 require 'spout/models/coverage_result'
@@ -12,67 +12,58 @@ require 'spout/helpers/array_statistics'
 
 module Spout
   module Commands
+    # Generate a coverage report for the data dictionary.
     class Coverage
       include Spout::Helpers::NumberHelper
 
       def initialize(standard_version, argv)
         @standard_version = standard_version
-        @console = (argv.delete('--console') != nil)
-
-        @variable_files = Dir.glob("variables/**/*.json")
+        @console = !argv.delete('--console').nil?
+        @variable_files = Dir.glob('variables/**/*.json')
         @valid_ids = []
         @number_of_rows = nil
-
         @config = Spout::Helpers::ConfigReader.new
-
-        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
+        @subject_loader = Spout::Helpers::SubjectLoader.new(
+          @variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit
+        )
         @subject_loader.load_subjects_from_csvs_part_one! # Not Part Two which is essentially cleaning the data
         @subjects = @subject_loader.subjects
-
         run_coverage_report!
       end
 
       def run_coverage_report!
         puts "Generating: index.html\n\n"
-
         @matching_results = []
-
         @subject_loader.all_methods.each do |method, csv_files|
           scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).compact_empty.uniq)
-          @matching_results << [ csv_files, method, scr ]
+          @matching_results << [csv_files, method, scr]
         end
-
-        variable_ids = Dir.glob("variables/**/*.json").collect{ |file| file.gsub(/^(.*)\/|\.json$/, '').downcase }
+        variable_ids = Dir.glob('variables/**/*.json').collect { |file| file.gsub(%r{^(.*)/|\.json$}, '').downcase }
         @extra_variable_ids = (variable_ids - @subject_loader.all_methods.keys).sort
-
         @subject_loader.load_variable_domains!
-        domain_ids = Dir.glob("domains/**/*.json").collect{ |file| file.gsub(/^(.*)\/|\.json$/, '').downcase }
+        domain_ids = Dir.glob('domains/**/*.json').collect { |file| file.gsub(%r{^(.*)/|\.json$}, '').downcase }
         @extra_domain_ids = (domain_ids - @subject_loader.all_domains).sort
-
-        @matching_results.sort!{|a,b| [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]}
-
+        @matching_results.sort! do |a, b|
+          [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]
+        end
         @coverage_results = []
-
         @subject_loader.csv_files.each do |csv_file|
-          total_column_count = @matching_results.select{|mr| mr[0].include?(csv_file)}.count
-          mapped_column_count = @matching_results.select{|mr| mr[0].include?(csv_file) and mr[2].number_of_errors == 0}.count
-          @coverage_results << [ csv_file, total_column_count, mapped_column_count ]
+          total_column_count = @matching_results.select { |mr| mr[0].include?(csv_file) }.count
+          mapped_column_count = @matching_results
+                                .select { |mr| mr[0].include?(csv_file) && mr[2].number_of_errors.zero? }.count
+          @coverage_results << [csv_file, total_column_count, mapped_column_count]
         end
-
         coverage_folder = File.join(Dir.pwd, 'coverage')
         FileUtils.mkpath coverage_folder
         coverage_file = File.join(coverage_folder, 'index.html')
-
         File.open(coverage_file, 'w+') do |file|
-          erb_location = File.join( File.dirname(__FILE__), '../views/index.html.erb' )
+          erb_location = File.join(File.dirname(__FILE__), '../views/index.html.erb')
           file.puts ERB.new(File.read(erb_location)).result(binding)
         end
-
         unless @console
-          open_command = 'open'  if RUBY_PLATFORM.match(/darwin/) != nil
-          open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
-
-          system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
+          open_command = 'open' unless RUBY_PLATFORM.match(/darwin/).nil?
+          open_command = 'start' unless RUBY_PLATFORM.match(/mingw/).nil?
+          system "#{open_command} #{coverage_file}" if %w(start open).include?(open_command)
         end
         puts "#{coverage_file}\n\n"
       end

data/lib/spout/commands/deploy.rb

@@ -10,7 +10,6 @@ require 'spout/helpers/quietly'
 require 'spout/helpers/send_file'
 require 'spout/helpers/semantic'
 require 'spout/helpers/json_request'
-require 'spout/helpers/json_request_generic'
 
 # - **User Authorization**
 #   - User authenticates via token, the user must be a dataset editor
@@ -37,6 +36,7 @@ end
 
 module Spout
   module Commands
+    # Deploys a data dictionary and associated dataset to the server.
     class Deploy
       include Spout::Helpers::Quietly
 
@@ -211,22 +211,17 @@ module Spout
       end
 
       def user_authorization
-        puts  "  Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
-        print "     Enter your token: "
+        puts  '  Get your token here: ' + "#{@url}/token".colorize(:blue).on_white.underline
+        print '     Enter your token: '
         @token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ''
-
-        response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
-
-        if response.is_a?(Hash) and response['editor']
+        (json, _status) = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
+        if json.is_a?(Hash) && json['editor']
           puts 'AUTHORIZED'.colorize(:green)
         else
           puts 'UNAUTHORIZED'.colorize(:red)
           puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
-          fail DeployError
+          raise DeployError
         end
-
-        # failure ''
-        # puts 'PASS'.colorize(:green)
       end
 
       def upload_variables
@@ -328,12 +323,12 @@ module Spout
 
         print 'Launch Server Scripts: '
         params = { auth_token: @token, dataset: @slug, version: @version, folders: @created_folders.compact.uniq }
-        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/dictionary/refresh.json", params)
-        if response.is_a?(Hash) && response['refresh'] == 'success'
+        (json, _status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/dictionary/refresh.json", params)
+        if json.is_a?(Hash) && json['refresh'] == 'success'
           puts 'DONE'.colorize(:green)
         else
           puts 'FAIL'.colorize(:red)
-          fail DeployError
+          raise DeployError
         end
       end
 
@@ -342,11 +337,12 @@ module Spout
           puts '  Set Default Version: ' + 'SKIP'.colorize(:blue)
           return
         end
-
         print '  Set Default Version: '
         params = { auth_token: @token, dataset: @slug, version: @version }
-        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/dictionary/update_default_version.json", params)
-        if response.is_a?(Hash) && response['version_update'] == 'success'
+        (json, _status) = Spout::Helpers::JsonRequest.post(
+          "#{@url}/api/v1/dictionary/update_default_version.json", params
+        )
+        if json.is_a?(Hash) && json['version_update'] == 'success'
           puts @version.to_s.colorize(:green)
         else
           failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) + ' for ' + @slug.to_s.colorize(:white) + ' dataset.')
@@ -356,7 +352,7 @@ module Spout
       def failure(message)
         puts 'FAIL'.colorize(:red)
         puts message
-        fail DeployError
+        raise DeployError
       end
 
       def upload_file(file, folder)

data/lib/spout/commands/exporter.rb

@@ -21,7 +21,7 @@ module Spout
       private
 
       def expanded_export!
-        folder = "dd/#{@standard_version}"
+        folder = "exports/#{@standard_version}"
         puts '      create'.colorize(:green) + "  #{folder}" unless @quiet
         FileUtils.mkpath folder
         generic_export(

data/lib/spout/commands/graphs.rb

@@ -14,7 +14,7 @@ require 'spout/models/graphables'
 require 'spout/models/tables'
 require 'spout/helpers/config_reader'
 require 'spout/helpers/send_file'
-require 'spout/helpers/json_request_generic'
+require 'spout/helpers/json_request'
 require 'spout/version'
 
 module Spout
@@ -176,12 +176,12 @@ module Spout
                    domain: (variable.domain ? variable.domain.deploy_params : nil),
                    forms: variable.forms.collect(&:deploy_params) }
         params[:variable][:spout_stats] = stats.to_json
-        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/variables/create_or_update.json", params)
-        if response.is_a?(Hash) && status.is_a?(Net::HTTPSuccess)
+        (json, status) = Spout::Helpers::JsonRequest.post("#{@url}/api/v1/variables/create_or_update.json", params)
+        if json.is_a?(Hash) && status.is_a?(Net::HTTPSuccess)
           @progress[variable.id]['uploaded'] << @webserver_name
         else
           puts "\nUPLOAD FAILED: ".colorize(:red) + variable.id
-          puts "- Error: #{response.inspect}"
+          puts "- Error: #{json.inspect}"
         end
       end
     end

data/lib/spout/commands/help.rb

@@ -29,6 +29,7 @@ The most common spout commands are:
                     in `<project_name>/graphs/<version>/`
   [d]eploy NAME     Push dataset and data dictionary to a
                     webserver specified in `.spout.yml`
+  [u]pdate          Update the Spout gem
   [v]ersion         Returns the version of Spout
 
 Commands can be referenced by the first letter:
@@ -42,7 +43,9 @@ EOT
 
       def new_project
         puts <<-EOT
-Usage: spout new <project_name>
+Usage: spout new <project_name> [--skip-gemfile]
+
+Use `--skip-gemfile` to skip installing gems after project creation.
 
 More information here:
 
@@ -55,6 +58,8 @@ EOT
         puts <<-EOT
 Usage: spout version
 
+Returns version of spout.
+
 EOT
       end
 
@@ -62,6 +67,15 @@ EOT
         puts <<-EOT
 Usage: spout test
 
+EOT
+      end
+
+      def update
+        puts <<-EOT
+Usage: spout update
+
+Checks if a newer version of Spout is available.
+
 EOT
       end
 
@@ -71,10 +85,12 @@ Usage: spout import <csv_file>
 
 Optional Flags:
   --domains    Specify to import CSV of domains
+  --forms      Specify to import CSV of forms
 
 More information:
 https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file
 https://github.com/sleepepi/spout#importing-domains-from-an-existing-csv-file
+https://github.com/sleepepi/spout#importing-forms-from-an-existing-csv-file
 EOT
       end
 

data/lib/spout/commands/importer.rb

@@ -10,6 +10,7 @@ module Spout
     class Importer
       def initialize(argv)
         use_domains = !argv.delete('--domains').nil?
+        use_forms = !argv.delete('--forms').nil?
         @csv_file = argv[1].to_s
         unless File.exist?(@csv_file)
           puts csv_usage
@@ -17,6 +18,8 @@ module Spout
         end
         if use_domains
           import_domains
+        elsif use_forms
+          import_forms
         else
           import_variables
         end
@@ -62,7 +65,7 @@ EOT
           hash['forms'] = forms unless forms.empty?
           hash['other'] = row unless row.empty?
 
-          file_name = File.join(folder, id + '.json')
+          file_name = File.join(folder, "#{id}.json")
           File.open(file_name, 'w') do |file|
             file.write(JSON.pretty_generate(hash) + "\n")
           end
@@ -117,11 +120,45 @@ EOT
         end
       end
 
+      def import_forms
+        CSV.parse(File.open(@csv_file, 'r:iso-8859-1:utf-8', &:read), headers: true) do |line|
+          row = line.to_hash
+          unless row.keys.include?('id')
+            puts "\nMissing column header `".colorize(:red) +
+                 'id'.colorize(:light_cyan) +
+                 '` in data dictionary.'.colorize(:red) +
+                 additional_csv_info
+            exit(1)
+          end
+          unless row.keys.include?('display_name')
+            puts "\nMissing column header `".colorize(:red) +
+                 'display_name'.colorize(:light_cyan) +
+                 '` in data dictionary.'.colorize(:red) +
+                 additional_csv_info
+            exit(1)
+          end
+          next if row['id'] == ''
+          folder = File.join('forms', row.delete('folder').to_s)
+          FileUtils.mkpath folder
+          hash = {}
+          id = row.delete('id').to_s.downcase
+          hash['id'] = id
+          hash['display_name'] = tenderize(row.delete('display_name').to_s)
+          hash['code_book'] = row.delete('code_book').to_s
+          hash['other'] = row unless row.empty?
+          file_name = File.join(folder, "#{id}.json")
+          File.open(file_name, 'w') do |file|
+            file.write(JSON.pretty_generate(hash) + "\n")
+          end
+          puts '      create'.colorize(:green) + "  #{file_name}"
+        end
+      end
+
       # Converts ALL-CAPS display names to title case
       # Ex: BODY MASS INDEX changes to Body Mass Index
       # Ex: Patient ID stays the same as Patient ID
       def tenderize(text)
-        if text.match(/[a-z]/)
+        if /[a-z]/ =~ text
           text
         else
           text.downcase.gsub(/\b\w/) { $&.upcase }
@@ -131,7 +168,8 @@ EOT
       private
 
       def additional_csv_info
-        "\n\nFor additional information on specifying CSV column headers before import see:\n\n    " + "https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize( :light_cyan ) + "\n\n"
+        "\n\nFor additional information on specifying CSV column headers before import see:\n\n    " +
+          "https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize(:light_cyan) + "\n\n"
       end
     end
   end

data/lib/spout/commands/project_generator.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 require 'colorize'
+require 'date'
+require 'erb'
 require 'fileutils'
 
 TEMPLATES_DIRECTORY = File.expand_path('../../templates', __FILE__)
 
 module Spout
   module Commands
+    # Generates folder and file structure for a new spout data dictionary.
     class ProjectGenerator
       def initialize(argv)
         generate_folder_structure!(argv)
@@ -14,7 +17,8 @@ module Spout
 
       def generate_folder_structure!(argv)
         skip_gemfile = !argv.delete('--skip-gemfile').nil?
-        @full_path = File.join(argv[1].to_s.strip)
+        @project_name = argv[1].to_s.strip
+        @full_path = File.join(@project_name)
         usage = <<-EOT
 
 Usage: spout new FOLDER
@@ -30,9 +34,12 @@ EOT
         copy_file 'gitignore', '.gitignore'
         copy_file 'ruby-version', '.ruby-version'
         copy_file 'travis.yml', '.travis.yml'
-        copy_file 'spout.yml', '.spout.yml'
+        evaluate_file 'spout.yml.erb', '.spout.yml'
+        evaluate_file 'CHANGELOG.md.erb', 'CHANGELOG.md'
         copy_file 'Gemfile'
         copy_file 'Rakefile'
+        evaluate_file 'README.md.erb', 'README.md'
+        copy_file 'VERSION'
         directory 'domains'
         copy_file 'keep', 'domains/.keep'
         directory 'variables'
@@ -42,11 +49,10 @@ EOT
         directory 'test'
         copy_file 'test/dictionary_test.rb'
         copy_file 'test/test_helper.rb'
-        unless skip_gemfile
-          puts "         run".colorize( :green ) + "  bundle install".colorize( :light_cyan )
-          Dir.chdir(@full_path)
-          system "bundle install"
-        end
+        return if skip_gemfile
+        puts '         run'.colorize(:green) + '  bundle install'.colorize(:light_cyan)
+        Dir.chdir(@full_path)
+        system 'bundle install'
       end
 
       private
@@ -55,16 +61,26 @@ EOT
         file_name = template_file if file_name == ''
         file_path = File.join(@full_path, file_name)
         template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
-        puts "      create".colorize( :green ) + "  #{file_name}"
+        puts '      create'.colorize(:green) + "  #{file_name}"
         FileUtils.copy(template_file_path, file_path)
       end
 
+      def evaluate_file(template_file, file_name)
+        template_file_path = File.join(TEMPLATES_DIRECTORY, template_file)
+        template = ERB.new(File.read(template_file_path))
+        file_path = File.join(@full_path, file_name)
+        file_out = File.new(file_path, 'w')
+        file_out.syswrite(template.result(binding))
+        puts '      create'.colorize(:green) + "  #{file_name}"
+      ensure
+        file_out.close if file_out
+      end
+
       def directory(directory_name)
         directory_path = File.join(@full_path, directory_name)
-        puts "      create".colorize( :green ) + "  #{directory_name}"
+        puts '      create'.colorize(:green) + "  #{directory_name}"
         FileUtils.mkpath(directory_path)
       end
-
     end
   end
 end

data/lib/spout/commands/update.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'colorize'
+require 'spout/helpers/json_request'
+
+module Spout
+  module Commands
+    # Command to check if there is an updated version of the gem available.
+    class Update
+      class << self
+        def start(*args)
+          new(*args).start
+        end
+      end
+
+      def initialize(argv)
+      end
+
+      def start
+        (json, _status) = Spout::Helpers::JsonRequest.get('https://rubygems.org/api/v1/gems/spout.json')
+        if json
+          if json['version'] == Spout::VERSION::STRING
+            puts 'The spout gem is ' + 'up-to-date'.colorize(:green) + '!'
+          else
+            puts
+            puts "A newer version (v#{json['version']}) is available! Type the following command to update:"
+            puts
+            puts '  gem install spout --no-document'.colorize(:white)
+            puts
+          end
+        else
+          puts 'Unable to connect to RubyGems.org. Please try again later.'
+        end
+      end
+    end
+  end
+end

data/lib/spout/helpers/json_request.rb

@@ -3,42 +3,89 @@
 require 'openssl'
 require 'net/http'
 require 'json'
+require 'cgi'
 
-# TODO: Deprecated, use JsonRequestGeneric instead
 module Spout
   module Helpers
+    # Generates JSON web requests for GET, POST, and PATCH.
     class JsonRequest
       class << self
-        def get(*args)
-          new(*args).get
+        def get(url, *args)
+          new(url, *args).get
+        end
+
+        def post(url, *args)
+          new(url, *args).post
+        end
+
+        def patch(url, *args)
+          new(url, *args).patch
         end
       end
 
       attr_reader :url
 
-      def initialize(url)
-        begin
-          @url = URI.parse(url)
-          @http = Net::HTTP.new(@url.host, @url.port)
-          if @url.scheme == 'https'
-            @http.use_ssl = true
-            @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          end
-        rescue
+      def initialize(url, args = {})
+        @params = nested_hash_to_params(args)
+        @url = URI.parse(url)
+
+        @http = Net::HTTP.new(@url.host, @url.port)
+        if @url.scheme == 'https'
+          @http.use_ssl = true
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
+      rescue
+        @error = "Invalid URL: #{url.inspect}"
+        puts @error.colorize(:red)
       end
 
       def get
-        begin
-          full_path = @url.path
-          full_path += "?#{@url.query}" if @url.query
-          req = Net::HTTP::Get.new(full_path)
-          response = @http.start do |http|
-            http.request(req)
+        return unless @error.nil?
+        full_path = @url.path
+        query = ([@url.query] + @params).flatten.compact.join('&')
+        full_path += "?#{query}" if query.to_s != ''
+        response = @http.start do |http|
+          http.get(full_path)
+        end
+        [JSON.parse(response.body), response]
+      rescue => e
+        puts "GET Error: #{e}".colorize(:red)
+      end
+
+      def post
+        return unless @error.nil?
+        response = @http.start do |http|
+          http.post(@url.path, @params.flatten.compact.join('&'))
+        end
+        [JSON.parse(response.body), response]
+      rescue => e
+        puts "POST ERROR: #{e}".colorize(:red)
+        nil
+      end
+
+      def patch
+        @params << '_method=patch'
+        post
+      end
+
+      def nested_hash_to_params(args)
+        args.collect do |key, value|
+          key_value_to_string(key, value, nil)
+        end
+      end
+
+      def key_value_to_string(key, value, scope = nil)
+        current_scope = (scope ? "#{scope}[#{key}]" : key)
+        if value.is_a? Hash
+          value.collect do |k,v|
+            key_value_to_string(k, v, current_scope)
+          end.join('&')
+        elsif value.is_a? Array
+          value.collect do |v|
+            key_value_to_string('', v, current_scope)
           end
-          JSON.parse(response.body)
-        rescue
-          nil
+        else
+          "#{current_scope}=#{CGI.escape(value.to_s)}"
         end
       end
     end

data/lib/spout/helpers/subject_loader.rb

@@ -126,7 +126,7 @@ module Spout
         @variable_files.each do |variable_file|
           json = JSON.parse(File.read(variable_file)) rescue json = nil
           next unless json
-          next unless ['choices'].include?(json['type'])
+          next unless json['type'] == 'choices' || json['domain'].to_s.downcase.strip != ''
           domain = json['domain'].to_s.downcase
           @all_domains << domain
         end

data/lib/spout/models/coverage_result.rb

@@ -4,8 +4,11 @@ require 'spout/tests/variable_type_validation'
 
 module Spout
   module Models
+    # Contains the coverage of a specific variable.
     class CoverageResult
-      attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
+      attr_accessor :error, :error_message, :file_name_test, :json_id_test,
+                    :values_test, :valid_values, :csv_values,
+                    :variable_type_test, :json, :domain_test
 
       def initialize(column, csv_values)
         load_json(column)
@@ -26,10 +29,10 @@ module Spout
 
       def load_valid_values
         valid_values = []
-        if @json['type'] == 'choices'
+        if @json['type'] == 'choices' || domain_name != ''
           file = Dir.glob("domains/**/#{@json['domain'].to_s.downcase}.json", File::FNM_CASEFOLD).first
           if json = JSON.parse(File.read(file)) rescue false
-            valid_values = json.collect{|hash| hash['value']}
+            valid_values = json.collect { |hash| hash['value'] }
           end
         end
         @valid_values = valid_values
@@ -48,7 +51,7 @@ module Spout
       end
 
       def check_domain_specified
-        if @json['type'] != 'choices'
+        if @json['type'] != 'choices' && domain_name == ''
           true
         else
           domain_file = Dir.glob("domains/**/#{@json['domain'].to_s.downcase}.json", File::FNM_CASEFOLD).first
@@ -62,6 +65,10 @@ module Spout
       def errored?
         error == true
       end
+
+      def domain_name
+        @json['domain'].to_s.downcase.strip
+      end
     end
   end
 end

data/lib/spout/templates/CHANGELOG.md.erb

@@ -0,0 +1,3 @@
+## 0.1.0 (<%= Date.today.strftime('%B %-d, %Y') %>)
+
+- Created the <%= @project_name.downcase %> data dictionary repository using Spout v<%= Spout::VERSION::STRING %>

data/lib/spout/templates/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gem 'spout'

data/lib/spout/templates/README.md.erb

@@ -0,0 +1,46 @@
+<%= @project_name.capitalize %> Data Dictionary
+======================
+
+[![Build Status](https://travis-ci.org/<REPOSITORY>/<%= @project_name.downcase %>-data-dictionary.svg?branch=master)](https://travis-ci.org/<REPOSITORY>/<%= @project_name.downcase %>-data-dictionary)
+
+### Exports
+
+The <%= @project_name.downcase %> data dictionary can be exported to CSV by typing:
+
+```
+spout export
+```
+
+The `spout export` command will generate CSV files that describe the data
+dictionary.
+
+
+### Testing
+
+The <%= @project_name.downcase %> data dictionary is tested using the
+[Spout Gem](https://github.com/sleepepi/spout).
+
+Data dictionary tests can be run by typing:
+
+```
+spout test
+```
+
+
+### Releases
+
+The data dictionary is tagged at various time points using
+[Git tags](http://git-scm.com/book/en/Git-Basics-Tagging). The tags are used to
+reference a series of CSV files that correspond to the data dictionary itself.
+
+For example, CSV files of the underlying data that have been tagged as `v0.1.0`
+will be compatible with the Data Dictionary `~> 0.1.0`,
+(including `0.1.1`, `0.1.2`, `0.1.3`). However if the data dictionary contains
+changes to the underlying dataset, then the minor version number is bumped, and
+the patch level is reset to zero. If, for example, the CSV dataset changed to
+`v0.2.0`, then it would be compatible with `0.2.0`, `0.2.1`, `0.2.2`, etc. The
+approach for changing version numbers uses a variation on
+[Semantic Versioning](http://semver.org).
+
+A full list of changes for each version can be viewed in the
+[CHANGELOG](https://github.com/<REPOSITORY>/<%= @project_name.downcase %>-data-dictionary/blob/master/CHANGELOG.md).

data/lib/spout/templates/Rakefile

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'spout/tasks'

data/lib/spout/templates/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/spout/templates/gitignore

@@ -5,8 +5,7 @@
 /Gemfile.lock
 
 # Data Dictionary folders
-/dd
 /coverage
 /csvs
+/exports
 /graphs
-/images

data/lib/spout/templates/spout.yml.erb

@@ -0,0 +1,3 @@
+slug: <%= @project_name.downcase %>
+visit: visit
+charts: []

data/lib/spout/templates/test/dictionary_test.rb

@@ -15,8 +15,9 @@ class DictionaryTest < Minitest::Test
   # VALID_UNITS = ['minutes', 'hours'] # Add your own valid units to this array
   # @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
   #   define_method("test_units: #{variable.path}") do
-  #     message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n             Valid types: " +
-  #               VALID_UNITS.sort.collect { |u| u.inspect.colorize(:white) }.join(', ')
+  #     message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n" +
+  #               "             Valid types: " +
+  #               VALID_UNITS.sort_by(&:to_s).collect { |u| u.inspect.colorize(:white) }.join(', ')
   #     assert VALID_UNITS.include?(variable.units), message
   #   end
   # end

data/lib/spout/tests/domain_existence_validation.rb

@@ -2,32 +2,31 @@
 
 module Spout
   module Tests
+    # If a variable references a domain, then the domain should exist and be
+    # defined.
     module DomainExistenceValidation
-
       def assert_domain_existence(item)
-        domain_names = Dir.glob("domains/**/*.json").collect{|file| file.split('/').last.to_s.downcase.split('.json').first}
-
+        domain_names = Dir.glob('domains/**/*.json').collect do |file|
+          file.split('/').last.to_s.downcase.split('.json').first
+        end
         result = begin
-          domain_name = JSON.parse(File.read(item))["domain"]
+          domain_name = JSON.parse(File.read(item))['domain']
           domain_names.include?(domain_name)
         rescue JSON::ParserError
           domain_name = ''
           false
         end
-
         message = "The domain #{domain_name} referenced by #{item} does not exist."
-
         assert result, message
       end
 
-      Dir.glob("variables/**/*.json").each do |file|
-        if (not [nil, ''].include?(JSON.parse(File.read(file))["domain"]) rescue false)
-          define_method("test_domain_exists: "+file) do
+      Dir.glob('variables/**/*.json').each do |file|
+        if (not [nil, ''].include?(JSON.parse(File.read(file))['domain']) rescue false)
+          define_method("test_domain_exists: #{file}") do
             assert_domain_existence file
           end
         end
       end
-
     end
   end
 end

data/lib/spout/tests/domain_format.rb

@@ -3,30 +3,30 @@
 module Spout
   module Tests
     module DomainFormat
-
+      # Verifies the format of a domain.
       def assert_domain_format(item)
         result = begin
           json = JSON.parse(File.read(item))
           if json.is_a?(Array)
-            json.empty? or json.select{|o| not o.is_a?(Hash)}.size == 0
+            json.empty? || json.select { |o| !o.is_a?(Hash) }.empty?
           else
             false
           end
         rescue JSON::ParserError
           false
         end
-
-        message = "Must be an array of choice hashes. Ex:\n[\n  {\n    \"value\":        \"1\",\n    \"display_name\": \"Option 1\",\n    \"description\":  \"...\"\n  },\n  { ... },\n  ...\n]"
-
+        message = \
+          "Must be an array of choice hashes. Ex:\n[\n  {\n    \"value\":     "\
+          "   \"1\",\n    \"display_name\": \"Option 1\",\n    \"description\""\
+          ":  \"...\"\n  },\n  { ... },\n  ...\n]"
         assert result, message
       end
 
-      Dir.glob("domains/**/*.json").each do |file|
-        define_method("test_domain_format: "+file) do
+      Dir.glob('domains/**/*.json').each do |file|
+        define_method("test_domain_format: #{file}") do
           assert_domain_format file
         end
       end
-
     end
   end
 end

data/lib/spout/version.rb

@@ -5,7 +5,7 @@ module Spout
     MAJOR = 0
     MINOR = 12
     TINY = 0
-    BUILD = 'beta2' # 'pre', 'rc', 'rc2', nil
+    BUILD = 'rc' # 'pre', 'rc', 'rc2', nil
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.').freeze
   end

data/lib/spout/views/index.html.erb

@@ -102,7 +102,7 @@ tfoot td {
               <tr>
                 <td class="text-muted">Variables Not Found in Any CSV</td>
                 <td></td>
-                <td><code class="default"><%= number_with_delimiter( @extra_variable_ids.size ) %></code></td>
+                <td><code class="default"><%= number_with_delimiter(@extra_variable_ids.size) %></code></td>
                 <td></td>
                 <td></td>
               </tr>
@@ -111,7 +111,7 @@ tfoot td {
               <tr>
                 <td class="text-muted">Domains Not Referenced by Any Variable</td>
                 <td></td>
-                <td><code class="default"><%= number_with_delimiter( @extra_domain_ids.size ) %></code></td>
+                <td><code class="default"><%= number_with_delimiter(@extra_domain_ids.size) %></code></td>
                 <td></td>
                 <td></td>
               </tr>
@@ -126,9 +126,9 @@ tfoot td {
                     <span class="text-muted">---</span>
                   <% end %>
                 </td>
-                <td><code class="default"><%= number_with_delimiter( total_column_count ) %></code></td>
-                <td><code class="success"><%= number_with_delimiter( mapped_column_count ) %></code></td>
-                <td><code><%= number_with_delimiter( total_column_count - mapped_column_count ) %></code></td>
+                <td><code class="default"><%= number_with_delimiter(total_column_count) %></code></td>
+                <td><code class="success"><%= number_with_delimiter(mapped_column_count) %></code></td>
+                <td><code><%= number_with_delimiter(total_column_count - mapped_column_count) %></code></td>
               </tr>
             <% end %>
           </tbody>
@@ -177,7 +177,7 @@ tfoot td {
                 <% matched = @matching_results.count{|csv_files, column, scr| scr.file_name_test} %>
                 <% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
                 <% missing_percent = 100 - matched_percent %>
-                <%= number_with_delimiter( matched ) %> of <%= number_with_delimiter( total_count ) %>
+                <%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
                 <div class="progress progress-striped">
                   <div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
                   <div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
@@ -187,7 +187,7 @@ tfoot td {
                 <% matched = @matching_results.count{|csv_files, column, scr| scr.json_id_test} %>
                 <% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
                 <% missing_percent = 100 - matched_percent %>
-                <%= number_with_delimiter( matched ) %> of <%= number_with_delimiter( total_count ) %>
+                <%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
                 <div class="progress progress-striped">
                   <div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
                   <div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
@@ -197,7 +197,7 @@ tfoot td {
                 <% matched = @matching_results.count{|csv_files, column, scr| scr.variable_type_test} %>
                 <% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
                 <% missing_percent = 100 - matched_percent %>
-                <%= number_with_delimiter( matched ) %> of <%= number_with_delimiter( total_count ) %>
+                <%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
                 <div class="progress progress-striped">
                   <div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
                   <div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
@@ -208,7 +208,7 @@ tfoot td {
                 <% total_count = @matching_results.count{|csv_files, column, scr| scr.json['type'] == 'choices'} %>
                 <% matched_percent = (total_count == 0 ? 0 : (matched * 100.0 / total_count).floor) %>
                 <% missing_percent = 100 - matched_percent %>
-                <%= number_with_delimiter( matched ) %> of <%= number_with_delimiter( total_count ) %>
+                <%= number_with_delimiter(matched) %> of <%= number_with_delimiter(total_count) %>
                 <div class="progress progress-striped">
                   <div class="progress-bar progress-bar-success" style="width: <%= matched_percent %>%"></div>
                   <div class="progress-bar progress-bar-danger" style="width: <%= missing_percent %>%"></div>
@@ -262,8 +262,8 @@ tfoot td {
                   <% end %>
                 </td>
                 <td>
-                  <% if scr.json['type'] == 'choices' && scr.file_name_test %>
-                    <% if scr.domain_test or scr.json['domain'].to_s.strip == '' %>
+                  <% if (scr.json['type'] == 'choices' || scr.json['domain'].to_s.downcase.strip != '') && scr.file_name_test %>
+                    <% if scr.domain_test || scr.json['domain'].to_s.strip == '' %>
                       <code class="<%= 'success' if scr.domain_test %>">"domain": <%= scr.json['domain'].inspect %></code>
                     <% else %>
                       <span class="text-danger"><code><%= scr.json['domain'] %>.json</code> missing</span>
@@ -271,15 +271,18 @@ tfoot td {
                   <% end %>
                 </td>
                 <td style="white-space:nowrap">
-                  <% if scr.json['type'] == 'choices' %>
-                    <% unused_values = scr.valid_values - scr.csv_values %>
-                    <% if scr.values_test && unused_values.size == 0 %>
+                  <% if scr.json['type'] == 'choices' || scr.json['domain'].to_s.downcase.strip != '' %>
+                    <% unused_domain_values = scr.valid_values - scr.csv_values %>
+                    <% unused_values = (scr.valid_values | scr.csv_values) - (scr.valid_values & scr.csv_values) %>
+                    <% if scr.values_test && unused_values.empty? %>
                       <div class="text-success" style="text-align:center"><span class="glyphicon glyphicon-ok"></span></div>
                     <% else %>
                       <% (scr.valid_values + scr.csv_values.compact.sort).uniq.each do |value| %>
+                        <% value_as_number = format('%g', value) rescue value_as_number = nil %>
+                        <% next if !scr.valid_values.include?(value) && %w(numeric integer).include?(scr.json['type']) && !value_as_number.nil? %>
                         <% class_type = '' %>
                         <% class_type = 'success' if scr.valid_values.include?(value) %>
-                        <% class_type = 'default' if unused_values.include?(value)  %>
+                        <% class_type = 'default' if unused_domain_values.include?(value)  %>
                         <code class="<%= class_type %>"><%= value %></code>
                       <% end %>
                     <% end %>

data/lib/spout.rb

@@ -5,17 +5,19 @@ require 'spout/version'
 require 'spout/models/dictionary'
 
 Spout::COMMANDS = {
-  'n' => :new_project,
-  'v' => :version,
-  't' => :test,
-  'i' => :importer,
-  'e' => :exporter,
   'c' => :coverage_report,
+  'd' => :deploy,
+  'e' => :exporter,
   'g' => :generate_charts_and_tables,
+  'i' => :importer,
+  'n' => :new_project,
   'o' => :outliers_report,
-  'd' => :deploy
+  't' => :test,
+  'u' => :update,
+  'v' => :version
 }
 
+# Launch spout commands from command line.
 module Spout
   def self.launch(argv)
     send((Spout::COMMANDS[argv.first.to_s.scan(/\w/).first] || :help), argv)
@@ -70,6 +72,11 @@ module Spout
     # Spout::Commands::TestRunner.new(argv)
   end
 
+  def self.update(argv)
+    require 'spout/commands/update'
+    Spout::Commands::Update.start(argv)
+  end
+
   def self.version(_argv)
     puts "Spout #{Spout::VERSION::STRING}"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.12.0.beta2
+  version: 0.12.0.rc
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-24 00:00:00.000000000 Z
+date: 2017-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -117,13 +117,13 @@ files:
 - lib/spout/commands/importer.rb
 - lib/spout/commands/outliers.rb
 - lib/spout/commands/project_generator.rb
+- lib/spout/commands/update.rb
 - lib/spout/helpers/array_statistics.rb
 - lib/spout/helpers/chart_types.rb
 - lib/spout/helpers/config_reader.rb
 - lib/spout/helpers/iterators.rb
 - lib/spout/helpers/json_loader.rb
 - lib/spout/helpers/json_request.rb
-- lib/spout/helpers/json_request_generic.rb
 - lib/spout/helpers/number_helper.rb
 - lib/spout/helpers/quietly.rb
 - lib/spout/helpers/semantic.rb
@@ -156,12 +156,15 @@ files:
 - lib/spout/models/variable.rb
 - lib/spout/tasks.rb
 - lib/spout/tasks/engine.rake
+- lib/spout/templates/CHANGELOG.md.erb
 - lib/spout/templates/Gemfile
+- lib/spout/templates/README.md.erb
 - lib/spout/templates/Rakefile
+- lib/spout/templates/VERSION
 - lib/spout/templates/gitignore
 - lib/spout/templates/keep
 - lib/spout/templates/ruby-version
-- lib/spout/templates/spout.yml
+- lib/spout/templates/spout.yml.erb
 - lib/spout/templates/test/dictionary_test.rb
 - lib/spout/templates/test/test_helper.rb
 - lib/spout/templates/travis.yml

data/lib/spout/helpers/json_request_generic.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-require 'openssl'
-require 'net/http'
-require 'json'
-require 'cgi'
-
-module Spout
-  module Helpers
-    class JsonRequestGeneric
-      class << self
-        def get(url, *args)
-          new(url, *args).get
-        end
-
-        def post(url, *args)
-          new(url, *args).post
-        end
-
-        def patch(url, *args)
-          new(url, *args).patch
-        end
-      end
-
-      attr_reader :url
-
-      def initialize(url, args = {})
-        @params = nested_hash_to_params(args)
-        @url = URI.parse(url)
-
-        @http = Net::HTTP.new(@url.host, @url.port)
-        if @url.scheme == 'https'
-          @http.use_ssl = true
-          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        end
-      rescue => e
-        puts "Error sending JsonRequestGeneric: #{e}".colorize(:red)
-      end
-
-      def get
-        full_path = @url.path
-        query = ([@url.query] + @params).flatten.compact.join('&')
-        full_path += "?#{query}" if query.to_s != ''
-        response = @http.start do |http|
-          http.get(full_path)
-        end
-        [JSON.parse(response.body), response]
-      rescue => e
-        puts "GET Error: #{e}".colorize(:red)
-      end
-
-      def post
-        response = @http.start do |http|
-          http.post(@url.path, @params.flatten.compact.join('&'))
-        end
-        [JSON.parse(response.body), response]
-      rescue => e
-        puts "POST ERROR: #{e}".colorize(:red)
-        nil
-      end
-
-      def patch
-        @params << '_method=patch'
-        post
-      end
-
-      def nested_hash_to_params(args)
-        args.collect do |key, value|
-          key_value_to_string(key, value, nil)
-        end
-      end
-
-      def key_value_to_string(key, value, scope = nil)
-        current_scope = (scope ? "#{scope}[#{key}]" : key)
-        if value.is_a? Hash
-          value.collect do |k,v|
-            key_value_to_string(k, v, current_scope)
-          end.join('&')
-        elsif value.is_a? Array
-          value.collect do |v|
-            key_value_to_string('', v, current_scope)
-          end
-        else
-          "#{current_scope}=#{CGI.escape(value.to_s)}"
-        end
-      end
-    end
-  end
-end

data/lib/spout/templates/spout.yml

@@ -1,3 +0,0 @@
-# slug: myrepo
-visit: visit
-charts: []