spout 0.10.0.beta8 → 0.10.0.beta9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97f132b532821e57396f65c1752060dac1a33c14
-  data.tar.gz: f87fc93a7f9a60c34e81f602c47a770bbd29eba9
+  metadata.gz: 1510bf903f7dada97ac4f63538881fe5cddb0825
+  data.tar.gz: 8678041008439bd0eaea5e163eb6d4fc6337469b
 SHA512:
-  metadata.gz: 7c9ed942121864d02d02bdcdac9115e8b04093e368f31dfd9a4e37541726f5ff803654a93b6d62c872c314efb40f4757bf759f6d1098281dc91f090bbd201229
-  data.tar.gz: 741c7cfd6b2f365aff937f8d2508a04aaaef81fad297cca5773f3c2a82e1759cda0aba69a68d6e72a6d6547a39151f8915d64ac7e0d688ee5f605dd5a39fe216
+  metadata.gz: 390ca815b08364e3d3f8e068dd1104d784bdc48eb85bd2da9fc6e3d44562aec5ac75e22524d0580c1f0dba8ffec341021e463ac9b57d627ac7ae6b8e85d846f4
+  data.tar.gz: fa7bf7eb06e667c972354bd7b81f4d32e7f0c37de0a7115c06f484361943a7a9b737020b9434c4b7d132403035aa9c9f1caf89fccdfd3cf87e8b0357163fe935

data/CHANGELOG.md

@@ -9,6 +9,9 @@
     - Display names that use mixed case are unaffected
   - `spout import --domains` now marks options as `"missing": true` if the option value starts with a dot `.` or a dash `-`
     - Display names for domain options are also changed to title case if they are all caps
+- **Graph Command**
+  - Graphs and tables no longer display missing codes that do not exist in the dataset for variables of type `choices`
+  - Graphs and tables now remove domain values for variables of type `integer` or `numeric`
 - **Testing Changes**
   - Tests now include checks to assure that variable display_name fields don't exceed 255 length requirement
     - `include Spout::Tests::VariableDisplayNameLength`

data/lib/spout/commands/deploy.rb

@@ -60,7 +60,7 @@ module Spout
 
         begin
           run_all
-        rescue Interrupt, IRB::Abort
+        rescue Interrupt
           puts "\nINTERRUPTED".colorize(:red)
         end
       end
@@ -209,7 +209,7 @@ module Spout
         require 'spout/commands/images'
         argv = []
         argv << "--clean" if @clean
-        Spout::Commands::Images.new(argv, [], [], @version, [], true, @url, @slug, @token)
+        Spout::Commands::Images.new([], [], [], @version, argv, true, @url, @slug, @token)
         puts "\r     Image Generation: " + "DONE          ".colorize(:green)
       end
 

data/lib/spout/commands/graphs.rb

@@ -9,6 +9,7 @@ require 'spout/helpers/subject_loader'
 require 'spout/helpers/chart_types'
 require 'spout/helpers/config_reader'
 require 'spout/helpers/send_file'
+require 'spout/version'
 
 module Spout
   module Commands
@@ -74,7 +75,8 @@ module Spout
       def load_current_progress
         @progress_file = File.join(@graphs_folder, ".progress.json")
         @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
-        @progress = {} if @clean
+        @progress = {} if !@progress.kind_of?(Hash) or @clean or @progress['SPOUT_VERSION'] != Spout::VERSION::STRING
+        @progress['SPOUT_VERSION'] = Spout::VERSION::STRING
       end
 
       def save_current_progress
@@ -137,7 +139,12 @@ module Spout
 
           if @deploy_mode and not @progress[variable_name]['uploaded'] == true
             response = send_to_server(chart_json_file)
-            @progress[variable_name]['uploaded'] = (response.kind_of?(Hash) and response['upload'] == 'success')
+            if response.kind_of?(Hash) and response['upload'] == 'success'
+              @progress[variable_name]['uploaded'] = true
+            else
+              puts "\nUPLOAD FAILED: ".colorize(:red) + File.basename(chart_json_file)
+              @progress[variable_name]['uploaded'] = false
+            end
           end
 
           save_current_progress

data/lib/spout/commands/images.rb

@@ -52,7 +52,8 @@ module Spout
       def load_current_progress
         @progress_file = File.join(@images_folder, ".progress.json")
         @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
-        @progress = {} if @clean
+        @progress = {} if !@progress.kind_of?(Hash) or @clean or @progress['SPOUT_VERSION'] != Spout::VERSION::STRING
+        @progress['SPOUT_VERSION'] = Spout::VERSION::STRING
       end
 
       def save_current_progress
@@ -125,7 +126,9 @@ module Spout
             end
             run_phantom_js(variable_name, "#{variable_name}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
             run_phantom_js(variable_name, "#{variable_name}.png",     75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
-            @progress[variable_name]['uploaded'] = (@deploy_mode and @progress[variable_name]['failed'] != true)
+
+            @progress[variable_name]['uploaded'] = (@deploy_mode and @progress[variable_name]['upload_failed'] != true)
+
             save_current_progress
           end
 
@@ -161,6 +164,7 @@ module Spout
             if response.kind_of?(Hash) and response['upload'] == 'success'
               @progress[variable_name]['uploaded_files'] << png_name
             else
+              puts "\nUPLOAD FAILED: ".colorize(:red) + File.basename(png_name)
               @progress[variable_name]['upload_failed'] = true
             end
           end

data/lib/spout/helpers/chart_types.rb

@@ -72,6 +72,9 @@ module Spout
 
         filtered_subjects = subjects.select{ |s| s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
 
+        all_subject_values = filtered_subjects.collect(&method.to_sym).compact.sort
+        domain_json = remove_unused_missing_codes_from_domain(domain_json, all_subject_values.uniq)
+
         categories = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
           bucket = filtered_subjects.send(quartile).collect(&chart_type.to_sym)
           "#{bucket.min} to #{bucket.max}"
@@ -168,6 +171,8 @@ module Spout
         return unless domain_json = get_domain(json)
 
         filtered_subjects = subjects.select{ |s| s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
+        all_subject_values = filtered_subjects.collect(&method.to_sym).compact.sort
+        domain_json = remove_unused_missing_codes_from_domain(domain_json, all_subject_values.uniq)
 
         categories = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
           bucket = filtered_subjects.send(quartile).collect(&chart_type.to_sym)
@@ -216,6 +221,8 @@ module Spout
         units = 'percent'
         series = []
 
+        all_subject_values = subjects.collect(&method.to_sym).compact.sort
+        domain_json = remove_unused_missing_codes_from_domain(domain_json, all_subject_values.uniq)
 
         domain_json.each do |option_hash|
           domain_values = subjects.select{ |s| s.send(method) == option_hash['value'] }
@@ -311,6 +318,9 @@ module Spout
 
         filtered_subjects = subjects.select{ |s| s.send(chart_type) != nil }
 
+        all_subject_values = filtered_subjects.collect(&method.to_sym).compact.sort
+        domain_json = remove_unused_missing_codes_from_domain(domain_json, all_subject_values.uniq)
+
         rows = domain_json.collect do |option_hash|
           row_subjects = filtered_subjects.select{ |s| s.send(method) == option_hash['value'] }
           row_cells = chart_variable_domain.collect do |display_name, value|
@@ -358,6 +368,9 @@ module Spout
 
         all_subject_values = subjects.collect(&method.to_sym).compact.sort
         return nil if all_subject_values.count == 0
+
+        domain_json = remove_unused_missing_codes_from_domain(domain_json, all_subject_values.uniq) if domain_json
+
         categories = pull_categories(json, method, all_subject_values, domain_json)
 
         buckets = continuous_buckets(all_subject_values)
@@ -398,6 +411,11 @@ module Spout
         end
         data
       end
+
+      def self.remove_unused_missing_codes_from_domain(domain_json, unique_subject_values)
+        domain_json.select{|option_hash| option_hash['missing'] != true or (option_hash['missing'] == true and unique_subject_values.include?(option_hash['value']))}
+      end
+
     end
   end
 end

data/lib/spout/helpers/subject_loader.rb

@@ -40,12 +40,12 @@ module Spout
 
         @csv_files = Dir.glob("csvs/#{@csv_directory}/*.csv")
         @csv_files.each_with_index do |csv_file, index|
-          count = 0
+          count = 1 # Includes counting the header row
           print "\nParsing #{csv_file}"
           CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true, header_converters: lambda { |h| h.to_s.downcase } ) do |line|
             row = line.to_hash
             count += 1
-            print "\rParsing #{csv_file} - line ##{count}" if (count % 10 == 0)
+            print "\rParsing #{csv_file} - Row ##{count}" if (count % 10 == 0)
             @subjects << Spout::Models::Subject.create do |t|
               t._visit = row[@visit]
 
@@ -66,8 +66,9 @@ module Spout
               end
             end
             # puts "Memory Used: " + (`ps -o rss -p #{$$}`.strip.split.last.to_i / 1024).to_s + " MB" if count % 1000 == 0
-            break if @number_of_rows != nil and count >= @number_of_rows
+            break if @number_of_rows != nil and count - 1 >= @number_of_rows
           end
+          print "\rParsing #{csv_file} - Row ##{count}"
           puts "\n"
         end
 
@@ -80,7 +81,10 @@ module Spout
       end
 
       def load_subjects_from_csvs_part_two!
-        @variable_files.each do |variable_file|
+        variable_count = @variable_files.count
+        print "Converting numeric values to floats"
+        @variable_files.each_with_index do |variable_file, index|
+          print "\rConverting numeric values to floats:#{"% 3d" % ((index+1)*100/variable_count)}%"
           json = JSON.parse(File.read(variable_file)) rescue json = nil
           next unless json
           next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
@@ -88,8 +92,16 @@ module Spout
           method  = json['id'].to_s.downcase
           next unless Spout::Models::Subject.method_defined?(method)
 
+          domain_json = get_domain(json)
+          # Make all domain options nil for numerics/integers
+          if domain_json
+            domain_values = domain_json.collect{|option_hash| option_hash['value']}
+            @subjects.each{ |s| domain_values.include?(s.send(method)) ? s.send("#{method}=", nil) : nil }
+          end
+
           @subjects.each{ |s| s.send(method) != nil ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
         end
+        puts "\n"
         @subjects
       end
 
@@ -103,6 +115,22 @@ module Spout
         end
         @all_domains = @all_domains.compact.uniq.sort
       end
+
+      def get_json(file_name, file_type)
+        file = Dir.glob("#{file_type.to_s.downcase}s/**/#{file_name.to_s.downcase}.json", File::FNM_CASEFOLD).first
+        json = JSON.parse(File.read(file)) rescue json = nil
+        json
+      end
+
+      def get_variable(variable_name)
+        get_json(variable_name, 'variable')
+      end
+
+      def get_domain(json)
+        get_json(json['domain'], 'domain')
+      end
+
+
     end
   end
 end

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 10
     TINY = 0
-    BUILD = "beta8" # nil, "pre", "rc", "rc2"
+    BUILD = "beta9" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.10.0.beta8
+  version: 0.10.0.beta9
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2014-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake