spout 0.10.2 → 0.11.0.beta1

data/lib/spout/commands/graphs.rb

@@ -12,28 +12,26 @@ require 'spout/models/graphables'
 require 'spout/models/tables'
 require 'spout/helpers/config_reader'
 require 'spout/helpers/send_file'
+require 'spout/helpers/json_request_generic'
 require 'spout/version'
 
 module Spout
   module Commands
     class Graphs
-      def initialize(variables, standard_version, deploy_mode = false, url = '', slug = '', token = '', webserver_name = '', subjects = nil)
+      def initialize(argv, standard_version, deploy_mode = false, url = '', slug = '', token = '', webserver_name = '', subjects = nil)
         @deploy_mode = deploy_mode
         @url = url
         @standard_version = standard_version
         @slug = slug
         @token = token
         @webserver_name = webserver_name
-
-        argv = variables
-
-        @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
+        @clean = !(argv.delete('--no-resume').nil? && argv.delete('--clean').nil?)
 
         @config = Spout::Helpers::ConfigReader.new
 
         @stratification_variable = Spout::Models::Variable.find_by_id @config.visit
 
-        if @stratification_variable == nil
+        if @stratification_variable.nil?
           if @config.visit == ''
             puts "The visit variable in .spout.yml can't be blank."
           else
@@ -42,56 +40,51 @@ module Spout
           return self
         end
 
-        missing_variables = @config.charts.select{|c| Spout::Models::Variable.find_by_id(c['chart']) == nil}
+        missing_variables = @config.charts.select { |c| Spout::Models::Variable.find_by_id(c['chart']).nil? }
         if missing_variables.count > 0
           puts "Could not find the following chart variable#{'s' unless missing_variables.size == 1}: #{missing_variables.join(', ')}"
           return self
         end
 
-        argv_string = variables.join(',')
-        @number_of_rows = nil
+        rows_arg = argv.find { |arg| /^--rows=(\d*)/ =~ arg }
+        argv.delete(rows_arg)
+        @number_of_rows = rows_arg.gsub(/--rows=/, '').to_i if rows_arg
 
-        if match_data = argv_string.match(/-rows=(\d*)/)
-          @number_of_rows = match_data[1].to_i
-          argv_string.gsub!(match_data[0], '')
-        end
+        @valid_ids = argv.collect { |s| s.to_s.downcase }.compact.reject { |s| s == '' }
 
-        @valid_ids = argv_string.split(',').compact.reject{|s| s == ''}
-
-        @chart_variables = @config.charts.unshift( { "chart" => @config.visit, "title" => 'Histogram' } )
+        @chart_variables = @config.charts.unshift('chart' => @config.visit, 'title' => 'Histogram')
 
         @dictionary_root = Dir.pwd
         @variable_files = Dir.glob(File.join(@dictionary_root, 'variables', '**', '*.json'))
 
         t = Time.now
-        @graphs_folder = File.join("graphs", @standard_version)
+        @graphs_folder = File.join('graphs', @standard_version)
         FileUtils.mkpath @graphs_folder
 
-
         @subjects = if subjects
-          subjects
-        else
-          @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
-          @subject_loader.load_subjects_from_csvs!
-          @subjects = @subject_loader.subjects
-        end
+                      subjects
+                    else
+                      @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
+                      @subject_loader.load_subjects_from_csvs!
+                      @subjects = @subject_loader.subjects
+                    end
 
         load_current_progress
 
         compute_tables_and_charts
 
-        puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
+        puts "Took #{Time.now - t} seconds." if @subjects.size > 0 && !@deploy_mode
       end
 
       def load_current_progress
-        @progress_file = File.join(@graphs_folder, ".progress.json")
+        @progress_file = File.join(@graphs_folder, '.progress.json')
         @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
-        @progress = {} if !@progress.kind_of?(Hash) or @clean or @progress['SPOUT_VERSION'] != Spout::VERSION::STRING
+        @progress = {} if !@progress.is_a?(Hash) || @clean || @progress['SPOUT_VERSION'] != Spout::VERSION::STRING
         @progress['SPOUT_VERSION'] = Spout::VERSION::STRING
       end
 
       def save_current_progress
-        File.open(@progress_file,"w") do |f|
+        File.open(@progress_file, 'w') do |f|
           f.write(JSON.pretty_generate(@progress) + "\n")
         end
       end
@@ -104,30 +97,25 @@ module Spout
         end
       end
 
-      def send_to_server(chart_json_file)
-        response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", chart_json_file, @standard_version, @token)
-      end
-
-
       def iterate_through_variables
         variable_files_count = @variable_files.count
         @variable_files.each_with_index do |variable_file, file_index|
           variable = Spout::Models::Variable.new(variable_file, @dictionary_root)
 
           next unless variable.errors.size == 0
-          next unless @valid_ids.include?(variable.id) or @valid_ids.size == 0
-          next unless ["numeric", "integer", "choices"].include?(variable.type)
+          next unless @valid_ids.include?(variable.id) || @valid_ids.size == 0
+          next unless %w(numeric integer choices).include?(variable.type)
           next unless Spout::Models::Subject.method_defined?(variable.id)
 
           if @deploy_mode
             print "\r     Graph Generation: " + "#{"% 3d" % ((file_index+1)*100/variable_files_count)}% Uploaded".colorize(:white)
           else
-            puts "#{file_index+1} of #{variable_files_count}: #{variable.folder}#{variable.id}"
+            puts "#{file_index + 1} of #{variable_files_count}: #{variable.folder}#{variable.id}"
           end
 
           @progress[variable.id] ||= {}
           @progress[variable.id]['uploaded'] ||= []
-          next if (not @deploy_mode and @progress[variable.id]['generated'] == true) or (@deploy_mode and @progress[variable.id]['uploaded'].include?(@webserver_name))
+          next if (!@deploy_mode && @progress[variable.id]['generated'] == true) || (@deploy_mode && @progress[variable.id]['uploaded'].include?(@webserver_name))
 
           stats = {
             charts: {},
@@ -135,51 +123,64 @@ module Spout
           }
 
           @chart_variables.each do |chart_type_hash|
-            chart_type = chart_type_hash["chart"]
-            chart_title = chart_type_hash["title"].downcase.gsub(' ', '-')
+            chart_type = chart_type_hash['chart']
+            chart_title = chart_type_hash['title'].downcase.gsub(' ', '-')
             chart_variable = Spout::Models::Variable.find_by_id(chart_type)
 
+            filtered_subjects = @subjects.reject { |s| s.send(chart_type).nil? || s.send(variable.id).nil? }
+
+            next if filtered_subjects.collect(&variable.id.to_sym).compact_empty.count == 0
             if chart_type == @config.visit
-              filtered_subjects = @subjects.select{ |s| s.send(chart_type) != nil }
-              if filtered_subjects.count > 0
-                graph = Spout::Models::Graphables.for(variable, chart_variable, nil, filtered_subjects)
-                stats[:charts][chart_title] = graph.to_hash
-                table = Spout::Models::Tables.for(variable, chart_variable, filtered_subjects, nil)
-                stats[:tables][chart_title] = table.to_hash
-              end
+              graph = Spout::Models::Graphables.for(variable, chart_variable, nil, filtered_subjects)
+              stats[:charts][chart_title] = graph.to_hash
+              table = Spout::Models::Tables.for(variable, chart_variable, filtered_subjects, nil, totals: false)
+              stats[:tables][chart_title] = table.to_hash
             else
-              filtered_subjects = @subjects.select{ |s| s.send(chart_type) != nil }
-              if filtered_subjects.collect(&variable.id.to_sym).compact.count > 0
-                graph = Spout::Models::Graphables.for(variable, chart_variable, @stratification_variable, filtered_subjects)
-                stats[:charts][chart_title] = graph.to_hash
-                stats[:tables][chart_title] = @stratification_variable.domain.options.collect do |option|
-                  visit_subjects = filtered_subjects.select{ |s| s._visit == option.value }
-                  unknown_subjects = visit_subjects.select{ |s| s.send(variable.id) == nil }
-                  table = Spout::Models::Tables.for(variable, chart_variable, visit_subjects, option.display_name)
-                  (visit_subjects.count > 0 && visit_subjects.count != unknown_subjects.count) ? table.to_hash : nil
-                end.compact
-              end
+              graph = Spout::Models::Graphables.for(variable, chart_variable, @stratification_variable, filtered_subjects)
+              stats[:charts][chart_title] = graph.to_hash
+              stats[:tables][chart_title] = @stratification_variable.domain.options.collect do |option|
+                visit_subjects = filtered_subjects.select { |s| s._visit == option.value }
+                Spout::Models::Tables.for(variable, chart_variable, visit_subjects, option.display_name).to_hash
+              end.compact
             end
           end
 
           chart_json_file = File.join(@graphs_folder, "#{variable.id}.json")
-          File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
-
+          File.open(chart_json_file, 'w') { |file| file.write(JSON.pretty_generate(stats) + "\n") }
           @progress[variable.id]['generated'] = true
 
-          if @deploy_mode and not @progress[variable.id]['uploaded'].include?(@webserver_name)
-            response = send_to_server(chart_json_file)
-            if response.kind_of?(Hash) and response['upload'] == 'success'
-              @progress[variable.id]['uploaded'] << @webserver_name
-            else
-              puts "\nUPLOAD FAILED: ".colorize(:red) + File.basename(chart_json_file)
+          if @deploy_mode && !@progress[variable.id]['uploaded'].include?(@webserver_name)
+            values = @subjects.collect(&variable.id.to_sym).compact_empty
+            variable.n = values.n
+            variable.unknown = values.unknown
+            variable.total = values.count
+            if %w(numeric integer).include?(variable.type)
+              variable.mean = values.mean
+              variable.stddev = values.standard_deviation
+              variable.median = values.median
+              variable.min = values.min
+              variable.max = values.max
             end
+            send_variable_params_to_server(variable, stats)
           end
         end
-
       end
 
-
+      def send_variable_params_to_server(variable, stats)
+        params = { auth_token: @token, version: @standard_version,
+                   dataset: @slug, variable: variable.deploy_params,
+                   domain: (variable.domain ? variable.domain.deploy_params : nil),
+                   forms: variable.forms.collect(&:deploy_params) }
+        params[:variable][:spout_stats] = stats.to_json
+        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/variables/create_or_update.json", params)
+        if response.is_a?(Hash) && status.is_a?(Net::HTTPSuccess)
+          # puts "response: #{response}".colorize(:blue)
+          @progress[variable.id]['uploaded'] << @webserver_name
+        else
+          puts "\nUPLOAD FAILED: ".colorize(:red) + variable.id
+          puts "- Error: #{response.inspect}"
+        end
+      end
     end
   end
 end

data/lib/spout/commands/help.rb

@@ -0,0 +1,155 @@
+require 'colorize'
+
+module Spout
+  module Commands
+    class Help
+      def initialize(argv)
+        send((Spout::COMMANDS[argv[1].to_s.scan(/\w/).first] || :help))
+      end
+
+      def help
+        puts <<-EOT
+Usage: spout COMMAND [ARGS]
+
+The most common spout commands are:
+  [n]ew             Create a new Spout dictionary.
+                    `spout new <project_name>` creates a new
+                    data dictionary in `./<project_name>`
+  [t]est            Run tests and show failing tests
+  [i]mport          Import a CSV file into the JSON dictionary
+  [e]xport [1.0.0]  Export the JSON dictionary to CSV format
+  [c]overage        Coverage report, requires dataset CSVs
+                    in `<project_name>/csvs/<version>`
+  [o]utliers        Outlier report, requires dataset CSVs
+                    in `<project_name>/csvs/<version>`
+  [g]raphs          Generates JSON graphs for each variable
+                    in a dataset and places them
+                    in `<project_name>/graphs/<version>/`
+  [d]eploy NAME     Push dataset and data dictionary to a
+                    webserver specified in `.spout.yml`
+  [v]ersion         Returns the version of Spout
+
+Commands can be referenced by the first letter:
+  Ex: `spout t`, for test
+
+You can also get more in depth help by typing:
+  Ex: `spout help deploy`, to list all deploy flags
+
+EOT
+      end
+
+      def new_project
+        puts <<-EOT
+Usage: spout new <project_name>
+
+More information here:
+
+https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file
+
+EOT
+      end
+
+      def version
+        puts <<-EOT
+Usage: spout version
+
+EOT
+      end
+
+      def test
+        puts <<-EOT
+Usage: spout test
+
+EOT
+      end
+
+      def importer
+        puts <<-EOT
+Usage: spout import <csv_file>
+
+Optional Flags:
+  --domains    Specify to import CSV of domains
+
+More information:
+https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file
+https://github.com/sleepepi/spout#importing-domains-from-an-existing-csv-file
+EOT
+      end
+
+      def exporter
+        puts <<-EOT
+Usage: spout export
+
+Exports data dictionary to CSV format.
+
+More information here:
+
+https://github.com/sleepepi/spout#create-a-csv-data-dictionary-from-your-json-repository
+
+EOT
+      end
+
+      def coverage_report
+        puts <<-EOT
+Usage: spout coverage
+
+Generates `coverage/index.html` that can be viewed in browser.
+
+EOT
+      end
+
+      def generate_charts_and_tables
+        puts <<-EOT
+Usage: spout graphs
+
+Optional Flags:
+  --clean       Regenerate all graphs (default is to resume
+                where command last left off)
+  --rows=N      Limit the number of rows read from CSVs to a
+                maximum of N rows
+  <variable>    Only generate graphs for the specified variable(s)
+                Ex: spout graphs age gender
+
+EOT
+      end
+
+      def outliers_report
+        puts <<-EOT
+Usage: spout outliers
+
+Generates `coverage/outliers.html` that can be viewed in browser.
+
+More information here:
+
+https://github.com/sleepepi/spout#identify-outliers-in-your-dataset
+
+EOT
+      end
+
+      def deploy
+        puts <<-EOT
+Usage: spout deploy NAME
+
+NAME is the name of the webserver listed in `.spout.yml` file
+Optional Flags:
+  --clean         Regenerate all variables (default is to resume
+                  where command last left off)
+  --rows=N        Limit the number of rows read from CSVs to a
+                  maximum of N rows
+  <variable>      Only deploy specified variable(s)
+                  Ex: spout deploy production age gender
+  --skip-checks   Skips Spout checks
+  --skip-graphs   Skip generation of variable graphs
+  --skip-csvs     Skip upload of Data Dictionary and Dataset CSVs
+  --token=TOKEN   Provide token via command-line for automated
+                  processes
+
+More information here:
+
+https://github.com/sleepepi/spout#deploy-your-data-dictionary-to-a-staging-or-production-webserver
+
+EOT
+      end
+    end
+  end
+end

data/lib/spout/helpers/array_statistics.rb

@@ -1,41 +1,46 @@
+# Extensions to the Array class to calculate quartiles, outliers, and statistics
 class Array
+  def compact_empty
+    compact.reject { |a| a.is_a?(Spout::Models::Empty) }
+  end
+
   def n
-    self.compact.count
+    compact_empty.count
   end
 
   def mean
-    array = self.compact
+    array = compact_empty
     return nil if array.size == 0
     array.inject(:+).to_f / array.size
   end
 
   def sample_variance
-    array = self.compact
+    array = compact_empty
     m = array.mean
-    sum = array.inject(0){|accum, i| accum +(i-m)**2 }
+    sum = array.inject(0) { |a, e| a + (e - m)**2 }
     sum / (array.length - 1).to_f
   end
 
   def standard_deviation
-    array = self.compact
+    array = compact_empty
     return nil if array.size < 2
-    return Math.sqrt(array.sample_variance)
+    Math.sqrt(array.sample_variance)
   end
 
   def median
-    array = self.compact.sort
+    array = compact_empty.sort
     return nil if array.size == 0
     len = array.size
-    len % 2 == 1 ? array[len/2] : (array[len/2 - 1] + array[len/2]).to_f / 2
+    len.odd? ? array[len / 2] : (array[len / 2 - 1] + array[len / 2]).to_f / 2
   end
 
   def unknown
-    self.select{|s| s == nil}.count
+    count { |a| a.is_a?(Spout::Models::Empty) }
   end
 
   def quartile_sizes
-    quartile_size = self.count / 4
-    quartile_fraction = self.count % 4
+    quartile_size = count / 4
+    quartile_fraction = count % 4
 
     quartile_sizes = [quartile_size] * 4
     (0..quartile_fraction - 1).to_a.each do |index|
@@ -46,75 +51,76 @@ class Array
   end
 
   def quartile_one
-    self[0..(self.quartile_sizes[0] - 1)]
+    self[0..(quartile_sizes[0] - 1)]
   end
 
   def quartile_two
-    sizes = self.quartile_sizes
+    sizes = quartile_sizes
     start = sizes[0]
     stop = start + sizes[1] - 1
     self[start..stop]
   end
 
   def quartile_three
-    sizes = self.quartile_sizes
+    sizes = quartile_sizes
     start = sizes[0] + sizes[1]
     stop = start + sizes[2] - 1
     self[start..stop]
   end
 
   def quartile_four
-    sizes = self.quartile_sizes
+    sizes = quartile_sizes
     start = sizes[0] + sizes[1] + sizes[2]
     stop = start + sizes[3] - 1
     self[start..stop]
   end
 
   def compact_min
-    self.compact.min
+    compact_empty.min
   end
 
   def compact_max
-    self.compact.max
+    compact_empty.max
   end
 
   def outliers
-    array = self.compact.sort.select{|v| v.kind_of?(Numeric)}
+    array = compact_empty.sort.select { |v| v.is_a?(Numeric) }
     q1 = (array.quartile_one + array.quartile_two).median
     q3 = (array.quartile_three + array.quartile_four).median
-    return [] if q1 == nil or q3 == nil
+    return [] if q1.nil? || q3.nil?
     iq_range = q3 - q1
     inner_fence_lower = q1 - iq_range * 1.5
     inner_fence_upper = q3 + iq_range * 1.5
-    outer_fence_lower = q1 - iq_range * 3
-    outer_fence_upper = q3 + iq_range * 3
-    array.select{ |v| v > inner_fence_upper or v < inner_fence_lower }
+    array.select { |v| v > inner_fence_upper || v < inner_fence_lower }
   end
 
   def major_outliers
-    array = self.compact.sort.select{|v| v.kind_of?(Numeric)}
+    array = compact_empty.sort.select { |v| v.is_a?(Numeric) }
     q1 = (array.quartile_one + array.quartile_two).median
     q3 = (array.quartile_three + array.quartile_four).median
-    return [] if q1 == nil or q3 == nil
+    return [] if q1.nil? || q3.nil?
     iq_range = q3 - q1
-    inner_fence_lower = q1 - iq_range * 1.5
-    inner_fence_upper = q3 + iq_range * 1.5
     outer_fence_lower = q1 - iq_range * 3
     outer_fence_upper = q3 + iq_range * 3
-    array.select{ |v| v > outer_fence_upper or v < outer_fence_lower }
+    array.select { |v| v > outer_fence_upper || v < outer_fence_lower }
   end
 
   def minor_outliers
-    self.outliers - self.major_outliers
+    outliers - major_outliers
   end
-
 end
 
 module Spout
   module Helpers
     class ArrayStatistics
       def self.calculations
-        [["N", :n, :count], ["Mean", :mean, :decimal], ["StdDev", :standard_deviation, :decimal, "± %s"], ["Median", :median, :decimal], ["Min", :compact_min, :decimal], ["Max", :compact_max, :decimal], ["Unknown", :unknown, :count]]
+        [['N', :n, :count],
+         ['Mean', :mean, :decimal],
+         ['StdDev', :standard_deviation, :decimal, '± %s'],
+         ['Median', :median, :decimal],
+         ['Min', :compact_min, :decimal],
+         ['Max', :compact_max, :decimal],
+         ['Unknown', :unknown, :count]]
       end
     end
   end

data/lib/spout/helpers/chart_types.rb

@@ -5,7 +5,7 @@ module Spout
   module Helpers
     class ChartTypes
       def self.get_bucket(buckets, value)
-        return nil if buckets.size == 0 or not value.kind_of?(Numeric)
+        return nil if buckets.size == 0 or not value.is_a?(Numeric)
         buckets.each do |b|
           return "#{b[0]} to #{b[1]}" if value >= b[0] and value <= b[1]
         end
@@ -17,7 +17,7 @@ module Spout
       end
 
       def self.continuous_buckets(values)
-        values.select!{|v| v.kind_of? Numeric}
+        values.select!{|v| v.is_a? Numeric}
         return [] if values.count == 0
         minimum_bucket = values.min
         maximum_bucket = values.max

data/lib/spout/helpers/config_reader.rb

@@ -17,18 +17,18 @@ module Spout
       def parse_yaml_file
         spout_config = YAML.load_file('.spout.yml')
 
-        if spout_config.kind_of?(Hash)
+        if spout_config.is_a?(Hash)
           @slug = spout_config['slug'].to_s.strip
           @visit = spout_config['visit'].to_s.strip
 
-          @charts = if spout_config['charts'].kind_of?(Array)
-            spout_config['charts'].select{|c| c.kind_of?(Hash)}
+          @charts = if spout_config['charts'].is_a?(Array)
+            spout_config['charts'].select{|c| c.is_a?(Hash)}
           else
             []
           end
 
-          @webservers = if spout_config['webservers'].kind_of?(Array)
-            spout_config['webservers'].select{|c| c.kind_of?(Hash)}
+          @webservers = if spout_config['webservers'].is_a?(Array)
+            spout_config['webservers'].select{|c| c.is_a?(Hash)}
           else
             []
           end

data/lib/spout/helpers/json_request.rb

@@ -2,6 +2,7 @@ require 'openssl'
 require 'net/http'
 require 'json'
 
+# TODO: Deprecated, use JsonRequestGeneric instead
 module Spout
   module Helpers
     class JsonRequest
@@ -41,5 +42,3 @@ module Spout
     end
   end
 end
-
-