spout 0.10.2 → 0.11.0.beta1

data/lib/spout/helpers/json_request_generic.rb

@@ -0,0 +1,87 @@
+require 'openssl'
+require 'net/http'
+require 'json'
+require 'cgi'
+
+module Spout
+  module Helpers
+    class JsonRequestGeneric
+      class << self
+        def get(url, *args)
+          new(url, *args).get
+        end
+
+        def post(url, *args)
+          new(url, *args).post
+        end
+
+        def patch(url, *args)
+          new(url, *args).patch
+        end
+      end
+
+      attr_reader :url
+
+      def initialize(url, args = {})
+        @params = nested_hash_to_params(args)
+        @url = URI.parse(url)
+
+        @http = Net::HTTP.new(@url.host, @url.port)
+        if @url.scheme == 'https'
+          @http.use_ssl = true
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+      rescue => e
+        puts "Error sending JsonRequestGeneric: #{e}".colorize(:red)
+      end
+
+      def get
+        full_path = @url.path
+        query = ([@url.query] + @params).flatten.compact.join('&')
+        full_path += "?#{query}" if query.to_s != ''
+        response = @http.start do |http|
+          http.get(full_path)
+        end
+        [JSON.parse(response.body), response]
+      rescue => e
+        puts "GET Error: #{e}".colorize(:red)
+      end
+
+      def post
+        response = @http.start do |http|
+          http.post(@url.path, @params.flatten.compact.join('&'))
+        end
+        [JSON.parse(response.body), response]
+      rescue => e
+        puts "POST ERROR: #{e}".colorize(:red)
+        nil
+      end
+
+      def patch
+        @params << '_method=patch'
+        post
+      end
+
+      def nested_hash_to_params(args)
+        args.collect do |key, value|
+          key_value_to_string(key, value, nil)
+        end
+      end
+
+      def key_value_to_string(key, value, scope = nil)
+        current_scope = (scope ? "#{scope}[#{key}]" : key)
+        if value.is_a? Hash
+          value.collect do |k,v|
+            key_value_to_string(k, v, current_scope)
+          end.join('&')
+        elsif value.is_a? Array
+          value.collect do |v|
+            key_value_to_string('', v, current_scope)
+          end
+        else
+          "#{current_scope}=#{CGI.escape(value.to_s)}"
+        end
+      end
+    end
+  end
+end

data/lib/spout/helpers/quietly.rb

@@ -1,12 +1,11 @@
-
 module Spout
   module Helpers
+    # Silences output for tests
     module Quietly
-
       # From Rails: http://apidock.com/rails/v3.2.13/Kernel/silence_stream
       def silence_stream(stream)
         old_stream = stream.dup
-        stream.reopen(RbConfig::CONFIG['host_os'] =~ /mswin|mingw/ ? 'NUL:' : '/dev/null')
+        stream.reopen(/mswin|mingw/ =~ RbConfig::CONFIG['host_os'] ? 'NUL:' : '/dev/null')
         stream.sync = true
         yield
       ensure
@@ -21,7 +20,6 @@ module Spout
           end
         end
       end
-
     end
   end
 end

data/lib/spout/helpers/semantic.rb

@@ -1,9 +1,7 @@
-  # def dataset_folders
-  #   Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort
-  # end
 module Spout
   module Helpers
-
+    # Helps to sort semantically versioned numbers to match versions that are
+    # close to each other.
     class Version
       attr_accessor :string
       attr_reader :major, :minor, :tiny, :build
@@ -26,7 +24,7 @@ module Spout
       end
 
       def build_number
-        (@build == nil ? 1 : 0)
+        (@build.nil? ? 1 : 0)
       end
 
       def rank
@@ -34,21 +32,21 @@ module Spout
       end
     end
 
+    # Finds compatible versions
     class Semantic
-
       attr_accessor :data_dictionary_version
 
       def initialize(version, version_strings)
         @data_dictionary_version = Spout::Helpers::Version.new(version)
-        @versions = version_strings.collect{ |vs| Spout::Helpers::Version.new(vs) }.sort_by(&:rank)
+        @versions = version_strings.collect { |vs| Spout::Helpers::Version.new(vs) }.sort_by(&:rank)
       end
 
       def valid_versions
-        @versions.select{ |v| v.major == major and v.minor == minor }
+        @versions.select { |v| v.major == major && v.minor == minor }
       end
 
       def selected_folder
-        if valid_versions.size == 0 or valid_versions.collect(&:string).include?(version)
+        if valid_versions.size == 0 || valid_versions.collect(&:string).include?(version)
           version
         else
           valid_versions.collect(&:string).last
@@ -74,8 +72,6 @@ module Spout
       def build
         @data_dictionary_version.build
       end
-
     end
-
   end
 end

data/lib/spout/helpers/send_file.rb

@@ -13,15 +13,15 @@ module Spout
 
       attr_reader :url
 
-      def initialize(url, filename, version, token, type = nil)
-
+      def initialize(url, filename, version, token, slug, folder)
         @params = {}
-        @params["version"] = version
-        @params["auth_token"] = token if token
-        @params["type"] = type if type
+        @params['version'] = version
+        @params['auth_token'] = token if token
+        @params['dataset'] = slug if slug
+        @params['folder'] = folder if folder
         begin
-          file = File.open(filename, "rb")
-          @params["file"] = file
+          file = File.open(filename, 'rb')
+          @params['file'] = file
 
           mp = Multipart::MultipartPost.new
           @query, @headers = mp.prepare_query(@params)
@@ -42,36 +42,33 @@ module Spout
       end
 
       def post
-        begin
-          response = @http.start do |http|
-            http.post(@url.path, @query, @headers)
-          end
-          JSON.parse(response.body)
-        rescue
-          nil
+        response = @http.start do |http|
+          http.post(@url.path, @query, @headers)
         end
+        JSON.parse(response.body)
+      rescue
+        nil
       end
     end
   end
 end
 
-
 module Multipart
   class Param
     attr_accessor :k, :v
-    def initialize( k, v )
+    def initialize(k, v)
       @k = k
       @v = v
     end
 
     def to_multipart
-      return "Content-Disposition: form-data; name=\"#{k}\"\r\n\r\n#{v}\r\n"
+      "Content-Disposition: form-data; name=\"#{k}\"\r\n\r\n#{v}\r\n"
     end
   end
 
   class FileParam
     attr_accessor :k, :filename, :content
-    def initialize( k, filename, content )
+    def initialize(k, filename, content)
       @k = k
       @filename = filename
       @content = content
@@ -79,23 +76,24 @@ module Multipart
 
     def to_multipart
       mime_type = 'application/octet-stream'
-      return "Content-Disposition: form-data; name=\"#{k}\"; filename=\"#{filename}\"\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-Type: #{mime_type}\r\n\r\n" + content + "\r\n"
+      "Content-Disposition: form-data; name=\"#{k}\"; filename=\"#{filename}\"\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-Type: #{mime_type}\r\n\r\n" + content + "\r\n"
     end
   end
+
   class MultipartPost
     BOUNDARY = 'a#41-93r1-^&#213-rule0000'
-    HEADER = {"Content-type" => "multipart/form-data, boundary=" + BOUNDARY + " "}
+    HEADER = { 'Content-type' => "multipart/form-data, boundary=#{BOUNDARY} " }
 
-    def prepare_query (params)
+    def prepare_query(params)
       fp = []
-      params.each {|k,v|
+      params.each do |k, v|
         if v.respond_to?(:read)
           fp.push(FileParam.new(k, v.path, v.read))
         else
-          fp.push(Param.new(k,v))
+          fp.push(Param.new(k, v))
         end
-      }
-      query = fp.collect {|p| "--" + BOUNDARY + "\r\n" + p.to_multipart }.join("") + "--" + BOUNDARY + "--"
+      end
+      query = fp.collect { |p| "--#{BOUNDARY}\r\n" + p.to_multipart }.join('') + "--#{BOUNDARY}--"
       return query, HEADER
     end
   end

data/lib/spout/helpers/subject_loader.rb

@@ -4,6 +4,7 @@ require 'json'
 
 require 'spout/models/subject'
 require 'spout/helpers/semantic'
+require 'spout/models/empty'
 
 module Spout
   module Helpers
@@ -32,74 +33,84 @@ module Spout
       def load_subjects_from_csvs_part_one!
         @subjects = []
 
-        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort : [])
+        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select { |e| File.directory? File.join('csvs', e) }.reject { |e| ['.', '..'].include?(e) }.sort : [])
 
         @semantic = Spout::Helpers::Semantic.new(@standard_version, available_folders)
 
         @csv_directory = @semantic.selected_folder
 
-        @csv_files = Dir.glob("csvs/#{@csv_directory}/*.csv")
-        @csv_files.each_with_index do |csv_file, index|
+        csv_root = File.join('csvs', @csv_directory)
+        @csv_files = Dir.glob("#{csv_root}/**/*.csv").sort
+
+        if @csv_directory != @standard_version
+          puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + ' for dictionary version ' + @standard_version.to_s.colorize(:green) + "\n"
+        else
+          puts "\n#{@csv_files.size == 0 ? 'No CSVs found' : 'Parsing files' } in " + "#{csv_root}".colorize(:white) + "\n"
+        end
+
+        last_folder = nil
+        @csv_files.each do |csv_file|
+          relative_path = csv_file.gsub(%r{^#{csv_root}}, '')
+          current_file = File.basename(relative_path)
+          current_folder = relative_path.gsub(/#{current_file}$/, '')
           count = 1 # Includes counting the header row
-          print "\nParsing #{csv_file}"
-          CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true, header_converters: lambda { |h| h.to_s.downcase } ) do |line|
+          puts "  #{current_folder}".colorize(:white) if current_folder.to_s != '' && current_folder != last_folder
+          print "    #{current_file}"
+          last_folder = current_folder
+          CSV.parse(File.open(csv_file, 'r:iso-8859-1:utf-8'){ |f| f.read }, headers: true, header_converters: lambda { |h| h.to_s.downcase }) do |line|
             row = line.to_hash
             count += 1
-            print "\rParsing #{csv_file} - Row ##{count}" if (count % 10 == 0)
+            print "\r    #{current_file} " + "##{count}".colorize(:yellow) if (count % 10 == 0)
             @subjects << Spout::Models::Subject.create do |t|
               t._visit = row[@visit]
+              t._csv = File.basename(csv_file)
 
-              row.each do |key,value|
+              row.each do |key, value|
                 method = key.to_s.downcase
-
+                next unless @valid_ids.include?(method) || @valid_ids.size == 0
                 unless t.respond_to?(method)
                   t.class.send(:define_method, "#{method}") { instance_variable_get("@#{method}") }
-                  t.class.send(:define_method, "#{method}=") { |value| instance_variable_set("@#{method}", value) }
+                  t.class.send(:define_method, "#{method}=") { |v| instance_variable_set("@#{method}", v) }
                 end
-
                 @all_methods[method] ||= []
                 @all_methods[method] = @all_methods[method] | [csv_file]
-
-                unless value == nil
+                if value.nil?
+                  t.send("#{method}=", Spout::Models::Empty.new)
+                else
                   t.send("#{method}=", value)
                 end
               end
             end
+
             # puts "Memory Used: " + (`ps -o rss -p #{$$}`.strip.split.last.to_i / 1024).to_s + " MB" if count % 1000 == 0
-            break if @number_of_rows != nil and count - 1 >= @number_of_rows
+            break if !@number_of_rows.nil? && count - 1 >= @number_of_rows
           end
-          print "\rParsing #{csv_file} - Row ##{count}"
-          puts "\n"
-        end
 
-        if @csv_directory != @standard_version
-          puts "#{@csv_files.size == 0 ? 'No CSVs found' : 'Using dataset' } in " + "csvs/#{@csv_directory}/".colorize( :green ) + " for dictionary version " + @standard_version.to_s.colorize( :green ) + "\n\n"
-        else
-          puts "#{@csv_files.size == 0 ? 'No CSVs found' : 'Using dataset' } in " + "csvs/#{@standard_version}/".colorize( :green ) + "\n\n"
+          print "\r    #{current_file} " + "##{count}".colorize(:green)
+          puts "\n"
         end
-
       end
 
       def load_subjects_from_csvs_part_two!
         variable_count = @variable_files.count
-        print "Converting numeric values to floats"
+        print 'Converting numeric values to floats'
         @variable_files.each_with_index do |variable_file, index|
-          print "\rConverting numeric values to floats:#{"% 3d" % ((index+1)*100/variable_count)}%"
+          print "\rConverting numeric values to floats:#{'% 3d' % ((index + 1) * 100 / variable_count)}%"
           json = JSON.parse(File.read(variable_file)) rescue json = nil
           next unless json
-          next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
-          next unless ["numeric", "integer"].include?(json["type"])
-          method  = json['id'].to_s.downcase
+          next unless @valid_ids.include?(json['id'].to_s.downcase) || @valid_ids.size == 0
+          next unless %w(numeric integer).include?(json['type'])
+          method = json['id'].to_s.downcase
           next unless Spout::Models::Subject.method_defined?(method)
 
           domain_json = get_domain(json)
           # Make all domain options nil for numerics/integers
           if domain_json
-            domain_values = domain_json.collect{|option_hash| option_hash['value']}
-            @subjects.each{ |s| domain_values.include?(s.send(method)) ? s.send("#{method}=", nil) : nil }
+            domain_values = domain_json.collect { |option_hash| option_hash['value'] }
+            @subjects.each { |s| domain_values.include?(s.send(method)) ? s.send("#{method}=", nil) : nil }
           end
 
-          @subjects.each{ |s| s.send(method) != nil ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
+          @subjects.each { |s| !s.send(method).nil? ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
         end
         puts "\n"
         @subjects
@@ -109,7 +120,7 @@ module Spout
         @variable_files.each do |variable_file|
           json = JSON.parse(File.read(variable_file)) rescue json = nil
           next unless json
-          next unless ["choices"].include?(json["type"])
+          next unless ['choices'].include?(json['type'])
           domain = json['domain'].to_s.downcase
           @all_domains << domain
         end
@@ -129,8 +140,6 @@ module Spout
       def get_domain(json)
         get_json(json['domain'], 'domain')
       end
-
-
     end
   end
 end

data/lib/spout/helpers/table_formatting.rb

@@ -1,18 +1,17 @@
 module Spout
   module Helpers
     class TableFormatting
-
       # def initialize(number)
       #   @number = number
       # end
 
-      def self.number_with_delimiter(number, delimiter = ",")
+      def self.number_with_delimiter(number, delimiter = ',')
         number.to_s.reverse.scan(/(?:\d*\.)?\d{1,3}-?/).join(',').reverse
       end
 
       # type:  :count    or   :decimal
       def self.format_number(number, type, format = nil)
-        if number == nil
+        if number.nil?
           format_nil(number)
         elsif type == :count
           format_count(number)
@@ -31,10 +30,9 @@ module Spout
       #     1000          ->         '1,000'
       # Input (Numeric)   -> Output (String)
       def self.format_count(number)
-        (number == 0 || number == nil) ? '-' : number_with_delimiter(number)
+        (number == 0 || number.nil?) ? '-' : number_with_delimiter(number)
       end
 
-
       # decimal:
       #        0          ->           '0.0'
       #       10          ->          '10.0'
@@ -43,7 +41,10 @@ module Spout
       # 12412423.42252525 ->  '12,412,423.4'
       # Input (Numeric)   -> Output (String)
       def self.format_decimal(number, format)
-        number = self.number_with_delimiter(number.round(1))
+        precision = 1
+        precision = -Math.log10(number.abs).floor if number.abs < 1.0 && number != 0
+
+        number = number_with_delimiter(number.round(precision))
         number = format % number if format
         number
       end

data/lib/spout/models/bucket.rb

@@ -1,22 +1,23 @@
 module Spout
   module Models
+    # Defines a continuous or discrete bucket for tables and graphs
     class Bucket
-
       attr_accessor :start, :stop
 
-      def initialize(start, stop)
+      def initialize(start, stop, discrete: false)
         @start = start
         @stop = stop
+        @discrete = discrete
       end
 
       def in_bucket?(value)
-        value >= @start and value <= @stop
+        value >= @start && value <= @stop
       end
 
       def display_name
+        return "#{@start}" if @discrete
         "#{@start} to #{@stop}"
       end
-
     end
   end
 end

data/lib/spout/models/coverage_result.rb

@@ -51,7 +51,7 @@ module Spout
         else
           domain_file = Dir.glob("domains/**/#{@json['domain'].to_s.downcase}.json", File::FNM_CASEFOLD).first
           if domain_json = JSON.parse(File.read(domain_file)) rescue false
-            return domain_json.kind_of?(Array)
+            return domain_json.is_a?(Array)
           end
           false
         end

data/lib/spout/models/dictionary.rb

@@ -4,6 +4,8 @@ require 'spout/models/form'
 
 module Spout
   module Models
+    # Creates a structure that contains a dictionaries variables, domains, and
+    # forms
     class Dictionary
       attr_accessor :variables, :domains, :forms
       attr_accessor :app_path
@@ -44,7 +46,7 @@ module Spout
       private
 
       def json_files(type)
-        Dir.glob(File.join(@app_path, type, "**", "*.json"))
+        Dir.glob(File.join(@app_path, type, '**', '*.json'))
       end
 
       def load_type!(method)

data/lib/spout/models/domain.rb

@@ -5,9 +5,7 @@ require 'spout/models/option'
 
 module Spout
   module Models
-
     class Domain < Spout::Models::Record
-
       attr_accessor :id, :folder, :options
       attr_reader :errors
 
@@ -26,11 +24,11 @@ module Spout
             nil
           end
         rescue => e
-          @errors << "Parsing error found in #{@id}.json: #{e.message}" if file_name != nil
+          @errors << "Parsing error found in #{@id}.json: #{e.message}" unless file_name.nil?
           nil
         end
 
-        if json and json.kind_of? Array
+        if json.is_a? Array
           @id = file_name.to_s.gsub(/^(.*)\/|\.json$/, '').downcase
           @options = (json || []).collect do |option|
             Spout::Models::Option.new(option)
@@ -38,10 +36,13 @@ module Spout
         elsif json
           @errors << "Domain must be a valid array in the following format: [\n  {\n    \"value\": \"1\",\n    \"display_name\": \"First Choice\",\n    \"description\": \"First Description\"\n  },\n  {\n    \"value\": \"2\",\n    \"display_name\": \"Second Choice\",\n    \"description\": \"Second Description\"\n  }\n]"
         end
-
       end
 
-
+      def deploy_params
+        { name: id, folder: folder.to_s.gsub(%r{/$}, ''),
+          options: options.collect(&:deploy_params),
+          spout_version: Spout::VERSION::STRING }
+      end
     end
   end
 end

data/lib/spout/models/empty.rb

@@ -0,0 +1,17 @@
+module Spout
+  module Models
+    # Used for empty values, these values exist in that the column is defined
+    # in the CSV, however the cell is blank. This is to differentiate this
+    # value from nil, where the subject row exists, but the column for the
+    # is not specified.
+    class Empty
+      def to_f
+        self
+      end
+
+      def to_s
+        'Empty'
+      end
+    end
+  end
+end

data/lib/spout/models/form.rb

@@ -9,8 +9,7 @@ require 'spout/models/record'
 module Spout
   module Models
     class Form < Spout::Models::Record
-
-      attr_accessor :id, :display_name, :code_book
+      attr_accessor :id, :folder, :display_name, :code_book
       attr_accessor :errors
 
       def initialize(file_name, dictionary_root)
@@ -26,8 +25,8 @@ module Spout
           nil
         end
 
-        if json and json.kind_of? Hash
-          %w( display_name code_book ).each do |method|
+        if json.is_a? Hash
+          %w(display_name code_book).each do |method|
             instance_variable_set("@#{method}", json[method])
           end
 
@@ -35,9 +34,13 @@ module Spout
         elsif json
           @errors << "Form must be a valid hash in the following format: {\n\"id\": \"FORM_ID\",\n  \"display_name\": \"FORM DISPLAY NAME\",\n  \"code_book\": \"FORMPDF.pdf\"\n}"
         end
-
       end
 
+      def deploy_params
+        { name: id, folder: folder.to_s.gsub(%r{/$}, ''),
+          display_name: display_name, code_book: code_book,
+          spout_version: Spout::VERSION::STRING }
+      end
     end
   end
 end