spout 0.10.2 → 0.11.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4dd489db288edeabd09bffefabbd2a7f0ed2ffe
-  data.tar.gz: 38acb6688a14106600a09042b78cea92291fe6e9
+  metadata.gz: 35179a9011f963b875618eeb8fd07854c58f9cce
+  data.tar.gz: 321e836a1a80d07092e0bbe5d921f712db8dcb18
 SHA512:
-  metadata.gz: af80eb4517f784fc425566e2b62887c1c43688153ff68c9f435337e2d6ff24afe3bf6f8956877b006ee6972f87c5852472aa888b5c70d6bccc5d17328f98b88b
-  data.tar.gz: 11ac5a8dd09a24bfa2313b4df3c9489aad31045bd9ab07008b7ff23da5c075f0336005d286b925e872715eb4364a29ef58d92ec3a93dd53e514723c017ef17c5
+  metadata.gz: d811aca192cc49aa05a8fc5dd5c4f1523f977b0fc2eb0e56c6f6ed5cd7f8295d73b69ccd93382183d3199c84986dff828c1faa611a39038b2b2bcb32ef14405f
+  data.tar.gz: 9ba01d7cb5f51f977b9433a8186f69241585bbba0eb2beb238c7794fe3c1eaa548da3450d527692747a950786ca02f609dad7e90058bc539a28ec371fbe4a521

data/CHANGELOG.md

@@ -1,3 +1,39 @@
+## 0.11.0
+
+### Deprecations
+- **Image Command**
+  - Spout variable graphs are no longer required as images by the NSRR webserver and have been removed
+
+### Enhancements
+- **Deploy Command**
+  - The deploy command can now accept flags and arguments that modify and scope graphs and images
+    - ex: `spout deploy WEBSERVER var1 var2 --rows=100 --clean`
+    - This pushes a clean version of graphs and images based on the first 100 rows in the CSVs for var1 and var2
+  - Spout now provides detailed information when deploying variables, forms, and domains
+  - CSVs can now be grouped into subfolders within their version folders, ex:
+    - `<project_name>/csvs/<VERSION>/dataset-main.csv`
+    - `<project_name>/csvs/<VERSION>/subfolder/dataset-extra-a.csv`
+    - `<project_name>/csvs/<VERSION>/subfolder/dataset-extra-b.csv`
+  - `CHANGELOG.md` and `KNOWNISSUES.md` are now uploaded alongside datasets during a deploy
+- **Graph Command**
+  - Tables no longer include unknown values for variables that don't exist across other CSVs with the same subject and visit
+  - Tables still track unknown values if the CSV contains a column for the variable, but the cell for subject is blank
+  - Histograms of variables that only contain a small range (less than 12) of integers now use discrete buckets of size 1 instead of creating fractional buckets
+  - Numbers less than one now respect the precision of the number when computing means and related statistics
+- **Export Command**
+  - Commonly Used attribute is now included in data dictionary exports
+- **Tests Added**
+  - New tests were added to check the format of variable names, form names, and domain names
+- **Help Command**
+  - The help command now allows users to learn more about individual commands
+    - ex: `spout help deploy`
+- **Gem Changes**
+  - Updated to Ruby 2.3.0
+  - Bundler is now a required dependency
+
+### Refactoring
+- Started refactoring code base to better adhere to RuboCop style recommendations
+
 ## 0.10.2 (December 29, 2014)
 
 ### Bug Fix

data/README.md

@@ -212,31 +212,6 @@ You can optionally provide a version string
 spout export [1.0.0]
 ```
 
-### Generate images for data in your dataset
-
-Spout lets you generate images for each variable defined in your dataset. Make sure to run `spout coverage` first to validate that your data dictionary and dataset match.
-
-This command will take some time, and requires [PhantomJS](http://phantomjs.org/) to be installed on your system.
-
-```
-spout pngs
-```
-
-The following flags can be passed to the `spout pngs` command:
-
-- `spout p --type-numeric`
-- `spout p --type-integer`
-- `spout p --type-choices`
-- `spout p --size-lg`
-- `spout p --size-sm`
-- `spout p --type-numeric --size-sm`
-
-For specific variables the following can be used:
-  - `spout p <variable_id>`
-
-Generated images are placed in: `./images/`
-
-
 ### Generate charts and tables for data in your dataset
 
 ```
@@ -274,7 +249,7 @@ spout g ahi bmi
 You can also specify a limit to the amount of rows to read in from the CSV files by specifying the `-rows` flag
 
 ```
-spout g -rows=10 ahi
+spout g --rows=10 ahi
 ```
 
 This will generate a graph for ahi for the first 10 rows of each dataset CSV.
@@ -369,13 +344,11 @@ The following steps are run:
 - **Graph Generation**
   - `spout g` is run
   - Graphs are pushed to server
-- **Image Generation**
-  - `spout p` is run
-  - `optipng` is run on image then uploaded to server
-  - Images are pushed to server
 - **Dataset Uploads**
   - Dataset CSV data dictionary is generated (variables, domains, forms)
   - Dataset and data dictionary CSVs uploaded to files section of dataset
+- **Documentation Uploads**
+  - `README.md` and `KNOWNISSUES.md` are uploaded
 - **Server-Side Updates**
   - Server checks out branch of specified tag
   - Server runs `load_data_dictionary!` for specified dataset slug

data/lib/spout/commands/coverage.rb

@@ -6,6 +6,7 @@ require 'spout/helpers/subject_loader'
 require 'spout/models/coverage_result'
 require 'spout/helpers/number_helper'
 require 'spout/helpers/config_reader'
+require 'spout/helpers/array_statistics'
 
 module Spout
   module Commands
@@ -35,7 +36,7 @@ module Spout
         @matching_results = []
 
         @subject_loader.all_methods.each do |method, csv_files|
-          scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).uniq)
+          scr = Spout::Models::CoverageResult.new(method, @subjects.collect(&method.to_sym).compact_empty.uniq)
           @matching_results << [ csv_files, method, scr ]
         end
 

data/lib/spout/commands/deploy.rb

@@ -21,10 +21,6 @@ require 'spout/helpers/json_request'
 # - **Graph Generation**
 #   - `spout g` is run
 #   - Graphs are pushed to server
-# - **Image Generation**
-#   - `spout p` is run
-#   - `optipng` is run on image then uploaded to server
-#   - Images are pushed to server
 # - **Dataset Uploads**
 #   - Dataset CSV data dictionary is generated (variables, domains, forms)
 #   - Dataset and data dictionary CSVs uploaded to files section of dataset
@@ -39,25 +35,38 @@ end
 module Spout
   module Commands
     class Deploy
-
       include Spout::Helpers::Quietly
 
       INDENT_LENGTH = 23
-      INDENT = " "*INDENT_LENGTH
+      INDENT = ' ' * INDENT_LENGTH
 
       attr_accessor :token, :version, :slug, :url, :config, :environment, :webserver_name, :subjects
 
       def initialize(argv, version)
-        @environment = argv[1].to_s
+        argv.shift # Remove 'download' command from argv list
+        @environment = argv.shift
         @version = version
-        @skip_checks = (argv.delete('--skip-checks') != nil or argv.delete('--no-checks') != nil)
+        @skip_checks = !(argv.delete('--skip-checks').nil? && argv.delete('--no-checks').nil?)
+
+        @skip_graphs = !(argv.delete('--skip-graphs').nil? && argv.delete('--no-graphs').nil?)
+        @skip_csvs = !(argv.delete('--skip-csvs').nil? && argv.delete('--no-csvs').nil?)
+        @clean = !(argv.delete('--no-resume').nil? && argv.delete('--clean').nil?)
+        @skip_server_updates = !(argv.delete('--skip-server-updates').nil? && argv.delete('--no-server-updates').nil?)
+
+        token_arg = argv.find { |arg| /^--token=/ =~ arg }
+        argv.delete(token_arg)
+        @token = token_arg.gsub(/^--token=/, '') if token_arg
+
+        rows_arg = argv.find { |arg| /^--rows=(\d*)/ =~ arg }
+        argv.delete(rows_arg)
+        @number_of_rows = rows_arg.gsub(/--rows=/, '').to_i if rows_arg
 
-        @skip_graphs = (argv.delete('--skip-graphs') != nil or argv.delete('--no-graphs') != nil)
-        @skip_images = (argv.delete('--skip-images') != nil or argv.delete('--no-images') != nil)
-        @clean = (argv.delete('--clean') != nil or argv.delete('--no-resume'))
-        @skip_server_updates = (argv.delete('--skip-server-updates') != nil or argv.delete('--no-server-updates') != nil)
+        # puts "token_arg: #{token_arg}".colorize(:red)
+        # puts "@token: #{@token}".colorize(:red)
+        # puts "rows_arg: #{rows_arg}".colorize(:red)
+        # puts "@number_of_rows: #{@number_of_rows}".colorize(:red)
 
-        @token = argv.select{|a| /^--token=/ =~ a}.collect{|a| a.gsub(/^--token=/, '')}.first
+        @argv = argv
 
         begin
           run_all
@@ -67,35 +76,31 @@ module Spout
       end
 
       def run_all
-        begin
-          config_file_load
-          version_check unless @skip_checks
-          test_check unless @skip_checks
-          user_authorization
-
-          load_subjects_from_csvs unless @skip_graphs and @skip_images
-
-
-          graph_generation unless @skip_graphs
-          image_generation unless @skip_images
-          dataset_uploads
-          data_dictionary_uploads
-          trigger_server_updates unless @skip_server_updates
-        rescue DeployError
-        end
+        config_file_load
+        version_check unless @skip_checks
+        test_check unless @skip_checks
+        user_authorization
+        load_subjects_from_csvs unless @skip_graphs
+        graph_generation unless @skip_graphs
+        dataset_uploads unless @skip_csvs
+        data_dictionary_uploads unless @skip_csvs
+        markdown_uploads unless @skip_csvs
+        trigger_server_updates unless @skip_server_updates
+      rescue DeployError
+        # Nothing on Deploy Error
       end
 
       def load_subjects_from_csvs
         @dictionary_root = Dir.pwd
         @variable_files = Dir.glob(File.join(@dictionary_root, 'variables', '**', '*.json'))
 
-        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, nil, @config.visit)
+        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, @number_of_rows, @config.visit)
         @subject_loader.load_subjects_from_csvs!
         @subjects = @subject_loader.subjects
       end
 
       def config_file_load
-        print "   `.spout.yml` Check: "
+        print '   `.spout.yml` Check: '
         @config = Spout::Helpers::ConfigReader.new
 
         @slug = @config.slug
@@ -110,7 +115,7 @@ module Spout
           failure(message)
         end
 
-        matching_webservers = @config.webservers.select{|wh| /^#{@environment}/i =~ wh['name'].to_s.downcase}
+        matching_webservers = @config.webservers.select { |wh| /^#{@environment}/i =~ wh['name'].to_s.downcase }
         if matching_webservers.count == 0
           message = "#{INDENT}0 webservers match '#{@environment}'.".colorize(:red) + " The following webservers exist in your `.spout.yml` file:\n" + "#{INDENT}#{@config.webservers.collect{|wh| wh['name'].to_s.downcase}.join(', ')}".colorize(:white)
           failure(message)
@@ -127,9 +132,9 @@ module Spout
           failure(message)
         end
 
-        puts "PASS".colorize(:green)
-        puts "        Target Server: " + "#{@url}".colorize(:white)
-        puts "       Target Dataset: " + "#{@slug}".colorize(:white)
+        puts 'PASS'.colorize(:green)
+        puts '        Target Server: ' + "#{@url}".colorize(:white)
+        puts '       Target Dataset: ' + "#{@slug}".colorize(:white)
       end
 
       # - **Version Check**
@@ -141,11 +146,11 @@ module Spout
           `git status --porcelain`
         end
 
-        print "     Git Status Check: "
+        print '     Git Status Check: '
         if stdout.to_s.strip == ''
-          puts "PASS".colorize(:green) + " " + "nothing to commit, working directory clean".colorize(:white)
+          puts 'PASS'.colorize(:green) + ' ' + 'nothing to commit, working directory clean'.colorize(:white)
         else
-          message = "#{INDENT}working directory contains uncomitted changes".colorize(:red)
+          message = "#{INDENT}working directory contains uncomitted changes\n#{INDENT}use `".colorize(:red) + '--skip-checks'.colorize(:white) + '` to ignore this step'.colorize(:red)
           failure message
         end
 
@@ -153,7 +158,7 @@ module Spout
         if changelog.match(/^## #{@version.split('.')[0..2].join('.')}/)
           puts "         CHANGELOG.md: " + "PASS".colorize(:green) + " " + changelog.colorize(:white)
         else
-          print "         CHANGELOG.md: "
+          print '         CHANGELOG.md: '
           message = "#{INDENT}Expected: ".colorize(:red) + "## #{@version}".colorize(:white) +
                   "\n#{INDENT}  Actual: ".colorize(:red) + changelog.colorize(:white)
           failure message
@@ -163,15 +168,14 @@ module Spout
           `git describe --exact-match HEAD --tags`
         end
 
-        print "        Version Check: "
+        print '        Version Check: '
         tag = stdout.to_s.strip
         if "v#{@version}" != tag
-          message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) + " does not match git tag on HEAD commit ".colorize(:red) + "'#{tag}'".colorize(:white)
+          message = "#{INDENT}Version specified in `VERSION` file ".colorize(:red) + "'v#{@version}'".colorize(:white) + ' does not match git tag on HEAD commit '.colorize(:red) + "'#{tag}'".colorize(:white)
           failure message
         else
-          puts   "PASS".colorize(:green) + " VERSION " + "'v#{@version}'".colorize(:white) + " matches git tag " + "'#{tag}'".colorize(:white)
+          puts 'PASS'.colorize(:green) + ' VERSION ' + "'v#{@version}'".colorize(:white) + ' matches git tag ' + "'#{tag}'".colorize(:white)
         end
-
       end
 
       def test_check
@@ -182,7 +186,7 @@ module Spout
         end
 
         if stdout.match(/[^\d]0 failures, 0 errors,/)
-          puts "PASS".colorize(:green)
+          puts 'PASS'.colorize(:green)
         else
           message = "#{INDENT}spout t".colorize(:white) + " had errors or failures".colorize(:red) + "\n#{INDENT}Please fix all errors and failures and then run spout deploy again."
           failure message
@@ -198,38 +202,28 @@ module Spout
 
         response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
 
-        if response.kind_of?(Hash) and response['editor']
-          puts "AUTHORIZED".colorize(:green)
+        if response.is_a?(Hash) and response['editor']
+          puts 'AUTHORIZED'.colorize(:green)
         else
-          puts "UNAUTHORIZED".colorize(:red)
+          puts 'UNAUTHORIZED'.colorize(:red)
           puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
           raise DeployError
         end
 
         # failure ''
-        # puts "PASS".colorize(:green)
+        # puts 'PASS'.colorize(:green)
       end
 
       def graph_generation
         # failure ''
         require 'spout/commands/graphs'
-        argv = []
-        argv << "--clean" if @clean
-        Spout::Commands::Graphs.new(argv, @version, true, @url, @slug, @token, @webserver_name, @subjects)
-        puts "\r     Graph Generation: " + "DONE          ".colorize(:green)
-      end
-
-      def image_generation
-        # failure ''
-        require 'spout/commands/images'
-        argv = []
-        argv << "--clean" if @clean
-        Spout::Commands::Images.new([], [], [], @version, argv, true, @url, @slug, @token, @webserver_name, @subjects)
-        puts "\r     Image Generation: " + "DONE          ".colorize(:green)
+        @argv << '--clean' if @clean
+        Spout::Commands::Graphs.new(@argv, @version, true, @url, @slug, @token, @webserver_name, @subjects)
+        puts "\r     Graph Generation: " + 'DONE          '.colorize(:green)
       end
 
       def dataset_uploads
-        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort : [])
+        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select { |e| File.directory? File.join('csvs', e) }.reject { |e| ['.', '..'].include?(e) }.sort : [])
         semantic = Spout::Helpers::Semantic.new(@version, available_folders)
         csv_directory = semantic.selected_folder
 
@@ -238,17 +232,18 @@ module Spout
           return
         end
 
-        csv_files = Dir.glob("csvs/#{csv_directory}/*.csv")
+        csv_files = Dir.glob("csvs/#{csv_directory}/**/*.csv")
 
         csv_files.each_with_index do |csv_file, index|
           print "\r      Dataset Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
-          response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_dataset_csv.json", csv_file, @version, @token)
+          folder = csv_file.gsub(/^csvs/, '').gsub(/#{File.basename(csv_file)}$/, '')
+          response = Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", csv_file, @version, @token, @slug, folder)
         end
         puts "\r      Dataset Uploads: " + "DONE          ".colorize(:green)
       end
 
       def data_dictionary_uploads
-        print   "   Dictionary Uploads:"
+        print   '   Dictionary Uploads:'
 
         require 'spout/commands/exporter'
         Spout::Commands::Exporter.new(@version, ['--quiet'])
@@ -256,32 +251,42 @@ module Spout
         csv_files = Dir.glob("dd/#{@version}/*.csv")
         csv_files.each_with_index do |csv_file, index|
           print "\r   Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
-          response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_dataset_csv.json", csv_file, @version, @token)
+          response = Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", csv_file, @version, @token, @slug, nil)
+        end
+        puts "\r   Dictionary Uploads: " + 'DONE          '.colorize(:green)
+      end
+
+      def markdown_uploads
+        print   'Documentation Uploads:'
+        markdown_files = Dir.glob(%w(CHANGELOG.md KNOWNISSUES.md))
+        markdown_files.each_with_index do |markdown_file, index|
+          print "\rDocumentation Uploads: " + "#{index + 1} of #{markdown_files.count}".colorize(:green)
+          Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", markdown_file, @version, @token, @slug, nil)
         end
-        puts "\r   Dictionary Uploads: " + "DONE          ".colorize(:green)
+        puts "\rDocumentation Uploads: " + 'DONE          '.colorize(:green)
       end
 
       def trigger_server_updates
-        print "Launch Server Scripts: "
+        print 'Launch Server Scripts: '
         response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/refresh_dictionary.json?version=#{@version}")
-        if response.kind_of?(Hash) and response['refresh'] == 'success'
-          puts "DONE".colorize(:green)
-        elsif response.kind_of?(Hash) and response['refresh'] == 'notagfound'
-          puts "FAIL".colorize(:red)
-          puts "#{INDENT}Tag not found in repository, resolve using: " + "git push --tags".colorize(:white)
+        if response.is_a?(Hash) && response['refresh'] == 'success'
+          puts 'DONE'.colorize(:green)
+        elsif response.is_a?(Hash) && response['refresh'] == 'notagfound'
+          puts 'FAIL'.colorize(:red)
+          puts "#{INDENT}Tag not found in repository, resolve using: " + 'git push --tags'.colorize(:white)
           raise DeployError
-        elsif response.kind_of?(Hash) and response['refresh'] == 'gitrepodoesnotexist'
-          puts "FAIL".colorize(:red)
+        elsif response.is_a?(Hash) && response['refresh'] == 'gitrepodoesnotexist'
+          puts 'FAIL'.colorize(:red)
           puts "#{INDENT}Dataset data dictionary git repository has not been cloned on the server. Contact server admin.".colorize(:white)
           raise DeployError
         else
-          puts "FAIL".colorize(:red)
+          puts 'FAIL'.colorize(:red)
           raise DeployError
         end
       end
 
       def failure(message)
-        puts "FAIL".colorize(:red)
+        puts 'FAIL'.colorize(:red)
         puts message
         raise DeployError
       end

data/lib/spout/commands/exporter.rb

@@ -22,7 +22,7 @@ module Spout
         puts "      create".colorize( :green ) + "  #{folder}" unless @quiet
         FileUtils.mkpath folder
 
-        generic_export(folder, 'variables', %w(id display_name description type units domain labels calculation))
+        generic_export(folder, 'variables', %w(id display_name description type units domain labels calculation commonly_used))
         generic_export(folder, 'domains', %w(value display_name description), true)
         generic_export(folder, 'forms', %w(id display_name code_book))
       end
@@ -45,7 +45,7 @@ module Spout
                   csv << [relative_folder, domain_name] + keys.collect{|key| hash[key]}
                 end
               else
-                csv << [relative_folder] + keys.collect{|key| json[key].kind_of?(Array) ? json[key].join(';') : json[key].to_s}
+                csv << [relative_folder] + keys.collect{|key| json[key].is_a?(Array) ? json[key].join(';') : json[key].to_s}
               end
             end
           end
@@ -67,7 +67,6 @@ module Spout
       def extract_domain_name(file)
         file.gsub(/domains\//, '').split('/').last.to_s.gsub(/.json/, '')
       end
-
     end
   end
 end