spout 0.7.0 → 0.8.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 700e0115db7d7231bd257fe56a4d7f4bff17ce01
-  data.tar.gz: e82a19b250ef2d068efa93d6acb9dc215e18c09b
+  metadata.gz: 55751ba1e111b174e9501fb52b1dd9a484b9ee1c
+  data.tar.gz: ca81a991a37e425a4166dd28e6aeffa183378b1c
 SHA512:
-  metadata.gz: a65d3873dcf0ab0c85673ac04761f97ba838e6a3415b02b5fa86d8024a2415b412267b50d1f818df02062dd7be48ae9d7a6c04a538af6fe17b1397c45670898f
-  data.tar.gz: 96843da45ebe630f2c4ba2eee3aa276f5c7735cc2fe731d082b09ff1ad209941123475de16e5b1d7e76d779d13483dc2857c68ef8c9a491347ec8e3e726c239d
+  metadata.gz: 6fec0a99eeb2b26af88652c7b6b79b3d5de8b750593aac2196275e1a1d365dda0814f47343ac312adaf9bb41bd29065a5d6dee8092de8927bdd41136d6ffbca0
+  data.tar.gz: 1d80934213ba8633506850aeabc142764f578d6fa727ffe7ac5ac76cc07422b4e0759d53e7200c5c686075077d3a37cd1101d7b650294d147db5a2f38f1e43eb

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.8.0
+
+### Enhancements
+- Added `spout json` command that generates JSON charts and tables of each variable in a dataset
+  - This command requires a .spout.yml file to be specified to identify the following variables:
+    - `visit`: This variable is used to separate subject encounters in a histogram
+    - `charts`: Array of choices, numeric, or integer variables for charts
+- **Gem Changes**
+  - Updated to colorize 0.7.2
+- Use of Ruby 2.1.2 is now recommended
+
 ## 0.7.0 (April 16, 2014)
 
 ### Enhancements

data/README.md

@@ -170,6 +170,33 @@ For specific variables the following can be used:
 
 Generated graphs are placed in: `./graphs/`
 
+
+### Generate charts and tables for data in your dataset
+
+```
+spout json
+```
+
+This command generates JSON charts and tables of each variable in a dataset
+
+Requires a Spout YAML configuration file, `.spout.yml`, in the root of the data dictionary that defines the variables used to create the charts:
+
+- `visit`: This variable is used to separate subject encounters in a histogram
+- `charts`: Array of choices, numeric, or integer variables for charts
+
+Example `.spout.yml` file:
+
+```yml
+visit: visitnumber
+charts:
+  - age
+  - gender
+  - race
+```
+
+This will generate charts and tables for each variable in the dataset plotted against the variables listed under `charts`.
+
+
 ### Export to the Hybrid Data Dictionary format from your JSON repository
 
 Exporting to a format compatible with [Hybrid](https://github.com/sleepepi/hybrid) is also available.

data/lib/spout/actions.rb

@@ -23,6 +23,8 @@ module Spout
         coverage_report(argv)
       when 'graphs', '-graphs', '--graphs', 'g', '-g'
         generate_graphs(argv.last(argv.size - 1))
+      when 'json', 'j'
+        generate_charts_and_tables(argv.last(argv.size - 1))
       else
         help
       end
@@ -140,6 +142,10 @@ EOT
         system "bundle exec rake spout:graphs #{params_string}"
       end
 
+      def generate_charts_and_tables(variables)
+        system "bundle exec rake spout:json variables=#{variables.join(',')}"
+      end
+
     private
 
       def copy_file(template_file, file_name = '')

data/lib/spout/commands/json_charts_and_tables.rb

@@ -0,0 +1,202 @@
+require 'csv'
+require 'fileutils'
+require 'rubygems'
+require 'json'
+require 'yaml'
+
+
+require 'spout/models/subject'
+require 'spout/helpers/array_statistics'
+require 'spout/helpers/chart_types'
+
+
+module Spout
+  module Commands
+    class JsonChartsAndTables
+      def initialize(variables)
+        spout_config = YAML.load_file('.spout.yml')
+
+        _visit = ''
+
+        if spout_config.kind_of?(Hash)
+          _visit = spout_config['visit'].to_s.strip
+
+          chart_variables = if spout_config['charts'].kind_of?(Array)
+            spout_config['charts'].collect{|c| c.to_s.strip}.select{|c| c != ''}
+          else
+            []
+          end
+        else
+          puts "The YAML file needs to be in the following format:"
+          puts "histogram: visitnumber  # VISIT_VARIABLE\ncharts:\n  - age_s1\n  - gender\n  - race\n"
+          exit
+        end
+
+        if Spout::Helpers::ChartTypes::get_json(_visit, 'variable') == nil
+          if _visit == ''
+            puts "The visit variable in .spout.yml can't be blank."
+          else
+            puts "Could not find the following visit variable: #{_visit}"
+          end
+          exit
+        end
+        missing_variables = chart_variables.select{|c| Spout::Helpers::ChartTypes::get_json(c, 'variable') == nil}
+        if missing_variables.count > 0
+          puts "Could not find the following chart variable#{'s' unless missing_variables.size == 1}: #{missing_variables.join(', ')}"
+          exit
+        end
+
+        argv_string = variables.join(',')
+        number_of_rows = nil
+
+        if match_data = argv_string.match(/-rows=(\d*)/)
+          number_of_rows = match_data[1].to_i
+          argv_string.gsub!(match_data[0], '')
+        end
+
+        valid_ids = argv_string.split(',').compact.reject{|s| s == ''}
+
+        @visit = _visit
+
+        chart_lookup = { _visit => "Histogram" }
+
+        chart_variables.each do |chart_variable|
+          json = Spout::Helpers::ChartTypes::get_json(chart_variable, 'variable')
+          chart_lookup[chart_variable] = json['display_name']
+        end
+
+
+
+        t = Time.now
+
+
+        version = standard_version
+
+        subjects = []
+
+        FileUtils.mkpath "charts/#{version}"
+
+        csv_files = Dir.glob("csvs/#{version}/*.csv")
+
+        csv_files.each_with_index do |csv_file, index|
+          count = 0
+          puts "Parsing: #{csv_file}"
+          CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true, header_converters: lambda { |h| h.to_s.downcase } ) do |line|
+
+            row = line.to_hash
+            count += 1
+            puts "Line: #{count}" if (count % 1000 == 0)
+            subjects << Spout::Models::Subject.create do |t|
+
+              t._visit = row[_visit] #.to_s.strip
+
+              row.each do |key,value|
+                unless t.respond_to?(key)
+                  t.class.send(:define_method, "#{key}") { instance_variable_get("@#{key}") }
+                  t.class.send(:define_method, "#{key}=") { |value| instance_variable_set("@#{key}", value) }
+                end
+
+                unless value == nil
+                  t.send("#{key}=", value)
+                end
+              end
+            end
+            # puts "Memory Used: " + (`ps -o rss -p #{$$}`.strip.split.last.to_i / 1024).to_s + " MB" if count % 1000 == 0
+            # break if count >= 1000
+            break if number_of_rows != nil and count >= number_of_rows
+          end
+        end
+
+        variable_files = Dir.glob('variables/**/*.json')
+        variable_files_count = variable_files.count
+
+        variable_files.each do |variable_file|
+          json = JSON.parse(File.read(variable_file)) rescue json = nil
+          next unless json
+          next unless valid_ids.include?(json["id"].to_s.downcase) or valid_ids.size == 0
+          next unless ["numeric", "integer"].include?(json["type"])
+          method  = json['id'].to_s.downcase
+          next unless Spout::Models::Subject.method_defined?(method)
+
+          subjects.each{ |s| s.send(method) != nil ? s.send("#{method}=", s.send("#{method}").to_f) : nil }
+        end
+
+        variable_files.each_with_index do |variable_file, file_index|
+          json = JSON.parse(File.read(variable_file)) rescue json = nil
+          next unless json
+          next unless valid_ids.include?(json["id"].to_s.downcase) or valid_ids.size == 0
+          next unless ["numeric", "integer", "choices"].include?(json["type"])
+          variable_name  = json['id'].to_s.downcase
+          next unless Spout::Models::Subject.method_defined?(variable_name)
+
+          puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
+
+
+          stats = {
+            charts: {},
+            tables: {}
+          }
+
+          chart_types = case json['type'] when 'integer', 'numeric', 'choices'
+            chart_lookup.keys
+          else
+            []
+          end
+
+          chart_types.each do |chart_type|
+            if chart_type == _visit
+              filtered_subjects = subjects.select{ |s| s.send(chart_type) != nil }  # and s.send(variable_name) != nil
+              if filtered_subjects.count > 0
+                stats[:charts][chart_lookup[chart_type].downcase] = Spout::Helpers::ChartTypes::chart_histogram(chart_type, filtered_subjects, json, variable_name)
+                stats[:tables][chart_lookup[chart_type].downcase] = Spout::Helpers::ChartTypes::table_arbitrary(chart_type, filtered_subjects, json, variable_name)
+              end
+            else
+              filtered_subjects = subjects.select{ |s| s.send(chart_type) != nil } # and s.send(variable_name) != nil
+              if filtered_subjects.count > 0
+                stats[:charts][chart_lookup[chart_type].downcase] = Spout::Helpers::ChartTypes::chart_arbitrary(chart_type, filtered_subjects, json, variable_name, visits)
+                stats[:tables][chart_lookup[chart_type].downcase] = visits.collect do |visit_display_name, visit_value|
+                  visit_subjects = filtered_subjects.select{ |s| s._visit == visit_value }
+                  unknown_subjects = visit_subjects.select{ |s| s.send(variable_name) == nil }
+                  (visit_subjects.count > 0 && visit_subjects.count != unknown_subjects.count) ? Spout::Helpers::ChartTypes::table_arbitrary(chart_type, visit_subjects, json, variable_name, visit_display_name) : nil
+                end.compact
+              end
+            end
+          end
+
+          chart_json_file = File.join('charts', version, "#{json['id']}.json")
+          File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
+
+        end
+
+        puts "Took #{Time.now - t} seconds."
+
+
+      end
+
+      # [["Visit 1", "1"], ["Visit 2", "2"], ["CVD Outcomes", "3"]]
+      def visits
+        @visits ||= begin
+          Spout::Commands::JsonChartsAndTables::domain_array(@visit)
+        end
+      end
+
+      # This is directly from Spout
+      def self.standard_version
+        version = File.open('VERSION', &:readline).strip rescue ''
+        version == '' ? '1.0.0' : version
+      end
+
+      def self.domain_array(variable_name)
+        variable_file = Dir.glob("variables/**/#{variable_name}.json").first
+        json = JSON.parse(File.read(variable_file)) rescue json = nil
+        if json
+          domain_json = Spout::Helpers::ChartTypes::get_domain(json)
+          domain_json ? domain_json.collect{|option_hash| [option_hash['display_name'], option_hash['value']]} : []
+        else
+          []
+        end
+      end
+
+    end
+  end
+end

data/lib/spout/helpers/array_statistics.rb

@@ -0,0 +1,93 @@
+# array_statistics.rb
+
+class Array
+  def n
+    self.compact.count
+  end
+
+  def mean
+    array = self.compact
+    return nil if array.size == 0
+    array.inject(:+).to_f / array.size
+  end
+
+  def sample_variance
+    array = self.compact
+    m = array.mean
+    sum = array.inject(0){|accum, i| accum +(i-m)**2 }
+    sum / (array.length - 1).to_f
+  end
+
+  def standard_deviation
+    array = self.compact
+    return nil if array.size < 2
+    return Math.sqrt(array.sample_variance)
+  end
+
+  def median
+    array = self.compact.sort
+    return nil if array.size == 0
+    len = array.size
+    len % 2 == 1 ? array[len/2] : (array[len/2 - 1] + array[len/2]).to_f / 2
+  end
+
+  def unknown
+    self.select{|s| s == nil}.count
+  end
+
+  def quartile_sizes
+    quartile_size = self.count / 4
+    quartile_fraction = self.count % 4
+
+    quartile_sizes = [quartile_size] * 4
+    (0..quartile_fraction - 1).to_a.each do |index|
+      quartile_sizes[index] += 1
+    end
+
+    quartile_sizes
+  end
+
+  def quartile_one
+    self[0..(self.quartile_sizes[0] - 1)]
+  end
+
+  def quartile_two
+    sizes = self.quartile_sizes
+    start = sizes[0]
+    stop = start + sizes[1] - 1
+    self[start..stop]
+  end
+
+  def quartile_three
+    sizes = self.quartile_sizes
+    start = sizes[0] + sizes[1]
+    stop = start + sizes[2] - 1
+    self[start..stop]
+  end
+
+  def quartile_four
+    sizes = self.quartile_sizes
+    start = sizes[0] + sizes[1] + sizes[2]
+    stop = start + sizes[3] - 1
+    self[start..stop]
+  end
+
+  def compact_min
+    self.compact.min
+  end
+
+  def compact_max
+    self.compact.max
+  end
+
+end
+
+module Spout
+  module Helpers
+    class ArrayStatistics
+      def self.calculations
+        [["N", :n, :count], ["Mean", :mean, :decimal], ["StdDev", :standard_deviation, :decimal, "± %s"], ["Median", :median, :decimal], ["Min", :compact_min, :decimal], ["Max", :compact_max, :decimal], ["Unknown", :unknown, :count]]
+      end
+    end
+  end
+end

data/lib/spout/helpers/chart_types.rb

@@ -0,0 +1,395 @@
+require 'spout/helpers/table_formatting'
+
+module Spout
+  module Helpers
+    class ChartTypes
+      def self.get_bucket(buckets, value)
+        buckets.each do |b|
+          return "#{b[0].round(1)} to #{b[1].round(1)}" if value >= b[0] and value <= b[1]
+        end
+        nil
+      end
+
+      def self.continuous_buckets(values)
+        return [] if values.count == 0
+        minimum_bucket = values.min
+        maximum_bucket = values.max
+
+        max_buckets = [(maximum_bucket - minimum_bucket), 12].min
+        bucket_size = (maximum_bucket - minimum_bucket) / max_buckets
+        buckets = []
+        (0..(max_buckets-1)).to_a.each do |index|
+          start = minimum_bucket + index * bucket_size
+          stop = start + bucket_size
+          buckets << [start,stop]
+        end
+        buckets
+      end
+
+      def self.get_json(file_name, file_type)
+        file = Dir.glob("#{file_type}s/**/#{file_name}.json").first
+        json = JSON.parse(File.read(file)) rescue json = nil
+        json
+      end
+
+      def self.get_variable(variable_name)
+        get_json(variable_name, 'variable')
+      end
+
+      def self.get_domain(json)
+        get_json(json['domain'], 'domain')
+      end
+
+
+      def self.chart_arbitrary_choices_by_quartile(chart_type, subjects, json, method)
+        # CHART TYPE IS THE QUARTILE VARIABLE
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless domain_json = get_domain(json)
+
+        title = "#{json['display_name']} by #{chart_variable_json['display_name']}"
+        subtitle = "By Visit"
+        # categories = ["Quartile One", "Quartile Two", "Quartile Three", "Quartile Four"]
+        units = 'percent'
+        series = []
+
+        filtered_subjects = subjects.select{ |s| s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
+
+        categories = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+          bucket = filtered_subjects.send(quartile).collect(&chart_type.to_sym)
+          "#{bucket.min} to #{bucket.max}"
+        end
+
+        domain_json.each do |option_hash|
+          data = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+            filtered_subjects.send(quartile).select{ |s| s.send(method) == option_hash['value'] }.count
+          end
+
+          series << { name: option_hash['display_name'], data: data } unless filtered_subjects.size == 0
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series, stacking: 'percent' }
+      end
+
+      def self.chart_arbitrary_by_quartile(chart_type, subjects, json, method, visits)
+        # CHART TYPE IS THE QUARTILE VARIABLE
+        return unless chart_variable_json = get_variable(chart_type)
+        return chart_arbitrary_choices_by_quartile(chart_type, subjects, json, method) if json['type'] == 'choices'
+
+        title = "#{json['display_name']} by #{chart_variable_json['display_name']}"
+        subtitle = "By Visit"
+        categories = ["Quartile One", "Quartile Two", "Quartile Three", "Quartile Four"]
+        units = json["units"]
+        series = []
+
+        series = []
+
+        visits.each do |visit_display_name, visit_value|
+          data = []
+          filtered_subjects = subjects.select{ |s| s._visit == visit_value and s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
+
+          [:quartile_one, :quartile_two, :quartile_three, :quartile_four].each do |quartile|
+            array = filtered_subjects.send(quartile).collect(&method.to_sym)
+            data << {       y: (array.mean.round(1) rescue 0.0),
+                         stddev: ("%0.1f" % array.standard_deviation rescue ''),
+                         median: ("%0.1f" % array.median rescue ''),
+                            min: ("%0.1f" % array.min rescue ''),
+                            max: ("%0.1f" % array.max rescue ''),
+                              n: array.n }
+          end
+
+          series << { name: visit_display_name, data: data } unless filtered_subjects.size == 0
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series }
+      end
+
+      def self.table_arbitrary_by_quartile(chart_type, subjects, json, method, subtitle = nil)
+        return table_arbitrary_choices_by_quartile(chart_type, subjects, json, method, subtitle) if json['type'] == 'choices'
+        # CHART TYPE IS THE QUARTILE VARIABLE
+        return unless chart_variable_json = get_variable(chart_type)
+
+
+        headers = [
+          [""] + Spout::Helpers::ArrayStatistics::calculations.collect{|calculation_label, calculation_method| calculation_label} + ["Total"]
+        ]
+
+        filtered_subjects = subjects.select{ |s| s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
+
+        rows = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+          bucket = filtered_subjects.send(quartile)
+          row_subjects = bucket.collect(&method.to_sym)
+          data = Spout::Helpers::ArrayStatistics::calculations.collect do |calculation_label, calculation_method, calculation_type, calculation_format|
+            TableFormatting::format_number(row_subjects.send(calculation_method), calculation_type, calculation_format)
+          end
+
+          row_name = if row_subjects.size == 0
+            quartile.to_s.capitalize.gsub('_one', ' One').gsub('_two', ' Two').gsub('_three', ' Three').gsub('_four', ' Four')
+          else
+            "#{bucket.collect(&chart_type.to_sym).min} to #{bucket.collect(&chart_type.to_sym).max} #{chart_variable_json['units']}"
+          end
+
+          [row_name] + data + [{ text: TableFormatting::format_number(row_subjects.count, :count), style: 'font-weight:bold'}]
+        end
+
+        total_values = Spout::Helpers::ArrayStatistics::calculations.collect do |calculation_label, calculation_method, calculation_type, calculation_format|
+          total_count = filtered_subjects.collect(&method.to_sym).send(calculation_method)
+          { text: TableFormatting::format_number(total_count, calculation_type, calculation_format), style: "font-weight:bold" }
+        end
+
+        footers = [
+          [{ text: "Total", style: "font-weight:bold" }] + total_values + [{ text: TableFormatting::format_number(filtered_subjects.count, :count), style: 'font-weight:bold'}]
+        ]
+
+        { title: "#{chart_variable_json['display_name']} vs #{json['display_name']}", subtitle: subtitle, headers: headers, footers: footers, rows: rows }
+
+      end
+
+      def self.table_arbitrary_choices_by_quartile(chart_type, subjects, json, method, subtitle)
+        # CHART TYPE IS THE QUARTILE VARIABLE
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless domain_json = get_domain(json)
+
+        filtered_subjects = subjects.select{ |s| s.send(method) != nil and s.send(chart_type) != nil }.sort_by(&chart_type.to_sym)
+
+        categories = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+          bucket = filtered_subjects.send(quartile).collect(&chart_type.to_sym)
+          "#{bucket.min} to #{bucket.max} #{chart_variable_json['units']}"
+        end
+
+        headers = [
+          [""] + categories + ["Total"]
+        ]
+
+        rows = []
+
+        rows = domain_json.collect do |option_hash|
+          row_subjects = filtered_subjects.select{ |s| s.send(method) == option_hash['value'] }
+
+          data = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+            bucket = filtered_subjects.send(quartile).select{ |s| s.send(method) == option_hash['value'] }
+            TableFormatting::format_number(bucket.count, :count)
+          end
+
+          [option_hash['display_name']] + data + [{ text: TableFormatting::format_number(row_subjects.count, :count), style: 'font-weight:bold'}]
+        end
+
+
+        total_values = [:quartile_one, :quartile_two, :quartile_three, :quartile_four].collect do |quartile|
+          { text: TableFormatting::format_number(filtered_subjects.send(quartile).count, :count), style: "font-weight:bold" }
+        end
+
+        footers = [
+          [{ text: "Total", style: "font-weight:bold" }] + total_values + [{ text: TableFormatting::format_number(filtered_subjects.count, :count), style: 'font-weight:bold'}]
+        ]
+
+        { title: "#{json['display_name']} vs #{chart_variable_json['display_name']}", subtitle: subtitle, headers: headers, footers: footers, rows: rows }
+      end
+
+
+      def self.chart_arbitrary_choices(chart_type, subjects, json, method)
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+        return unless domain_json = get_domain(json)
+
+
+        title = "#{json['display_name']} by #{chart_variable_json['display_name']}"
+        subtitle = "By Visit"
+        categories = chart_variable_domain.collect{|a| a[0]}
+        units = 'percent'
+        series = []
+
+
+        domain_json.each do |option_hash|
+          domain_values = subjects.select{ |s| s.send(method) == option_hash['value'] }
+
+          data = chart_variable_domain.collect do |display_name, value|
+            domain_values.select{ |s| s.send(chart_type) == value }.count
+          end
+          series << { name: option_hash['display_name'], data: data }
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series, stacking: 'percent' }
+      end
+
+      def self.chart_arbitrary(chart_type, subjects, json, method, visits)
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+        return chart_arbitrary_by_quartile(chart_type, subjects, json, method, visits) if ['numeric', 'integer'].include?(chart_variable_json['type'])
+
+        return chart_arbitrary_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
+
+        title = "#{json['display_name']} by #{chart_variable_json['display_name']}"
+        subtitle = "By Visit"
+        categories = []
+        units = json["units"]
+        series = []
+
+        data = []
+
+        visits.each do |visit_display_name, visit_value|
+          visit_subjects = subjects.select{ |s| s._visit == visit_value and s.send(method) != nil }
+          if visit_subjects.count > 0
+            chart_variable_domain.each_with_index do |(display_name, value), index|
+              values = visit_subjects.select{|s| s.send(chart_type) == value }.collect(&method.to_sym)
+              data[index] ||= []
+              data[index] << (values.mean.round(2) rescue 0.0)
+            end
+            categories << visit_display_name
+          end
+        end
+
+        chart_variable_domain.each_with_index do |(display_name, value), index|
+          series << { name: display_name, data: data[index] }
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series }
+      end
+
+
+      def self.table_arbitrary(chart_type, subjects, json, method, subtitle = nil)
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+        return table_arbitrary_by_quartile(chart_type, subjects, json, method, subtitle) if ['numeric', 'integer'].include?(chart_variable_json['type'])
+        return table_arbitrary_choices(chart_type, subjects, json, method, subtitle) if json['type'] == 'choices'
+
+        headers = [
+          [""] + Spout::Helpers::ArrayStatistics::calculations.collect{|calculation_label, calculation_method| calculation_label} + ["Total"]
+        ]
+
+        filtered_subjects = subjects.select{ |s| s.send(chart_type) != nil }
+
+        rows = chart_variable_domain.collect do |display_name, value|
+          row_subjects = filtered_subjects.select{ |s| s.send(chart_type) == value }
+
+          row_cells = Spout::Helpers::ArrayStatistics::calculations.collect do |calculation_label, calculation_method, calculation_type, calculation_format|
+            count = row_subjects.collect(&method.to_sym).send(calculation_method)
+            (count == 0 && calculation_method == :count) ? { text: '-', class: 'text-muted' } : TableFormatting::format_number(count, calculation_type, calculation_format)
+          end
+
+          [display_name] + row_cells + [{ text: TableFormatting::format_number(row_subjects.count, :count), style: 'font-weight:bold'}]
+        end
+
+        total_values = Spout::Helpers::ArrayStatistics::calculations.collect do |calculation_label, calculation_method, calculation_type, calculation_format|
+          total_count = filtered_subjects.collect(&method.to_sym).send(calculation_method)
+          { text: TableFormatting::format_number(total_count, calculation_type, calculation_format), style: "font-weight:bold" }
+        end
+
+        footers = [
+          [{ text: "Total", style: "font-weight:bold" }] + total_values + [{ text: TableFormatting::format_number(filtered_subjects.count, :count), style: 'font-weight:bold'}]
+        ]
+
+        { title: "#{chart_variable_json['display_name']} vs #{json['display_name']}", subtitle: subtitle, headers: headers, footers: footers, rows: rows }
+
+      end
+
+      def self.table_arbitrary_choices(chart_type, subjects, json, method, subtitle)
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+        return unless domain_json = get_domain(json)
+
+        headers = [
+          [""] + chart_variable_domain.collect{|display_name, value| display_name} + ["Total"]
+        ]
+
+        filtered_subjects = subjects.select{ |s| s.send(chart_type) != nil }
+
+        rows = domain_json.collect do |option_hash|
+          row_subjects = filtered_subjects.select{ |s| s.send(method) == option_hash['value'] }
+          row_cells = chart_variable_domain.collect do |display_name, value|
+            count = row_subjects.select{ |s| s.send(chart_type) == value }.count
+            count > 0 ? TableFormatting::format_number(count, :count) : { text: '-', class: 'text-muted' }
+          end
+
+          total = row_subjects.count
+
+          [option_hash['display_name']] + row_cells + [total == 0 ? { text: '-', class: 'text-muted' } : { text: TableFormatting::format_number(total, :count), style: 'font-weight:bold'}]
+        end
+
+        if filtered_subjects.select{|s| s.send(method) == nil }.count > 0
+          unknown_values = chart_variable_domain.collect do |display_name, value|
+            { text: TableFormatting::format_number(filtered_subjects.select{ |s| s.send(chart_type) == value and s.send(method) == nil }.count, :count), class: 'text-muted' }
+          end
+          rows << [{ text: 'Unknown', class: 'text-muted'}] + unknown_values + [ { text: filtered_subjects.select{|s| s.send(method) == nil}.count.to_s, style: 'font-weight:bold', class: 'text-muted' } ]
+        end
+
+
+
+        total_values = chart_variable_domain.collect do |display_name, value|
+          total_count = filtered_subjects.select{|s| s.send(chart_type) == value }.count
+          { text: (total_count == 0 ? "-" : TableFormatting::format_number(total_count, :count)), style: "font-weight:bold" }
+        end
+
+        footers = [
+          [{ text: "Total", style: "font-weight:bold" }] + total_values + [{ text: TableFormatting::format_number(filtered_subjects.count, :count), style: 'font-weight:bold'}]
+        ]
+
+        { title: "#{json['display_name']} vs #{chart_variable_json['display_name']}", subtitle: subtitle, headers: headers, footers: footers, rows: rows }
+      end
+
+
+      def self.chart_histogram_choices(chart_type, subjects, json, method)
+        return unless domain_json = get_domain(json)
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+
+
+        title = "#{json['display_name']}"
+        subtitle = "By Visit"
+
+        categories = domain_json.collect{|option_hash| option_hash['display_name']}
+
+        units = "Subjects"
+        series = []
+
+        chart_variable_domain.each do |display_name, value|
+          visit_subjects = subjects.select{ |s| s.send(chart_type) == value and s.send(method) != nil }.collect(&method.to_sym)
+          next unless visit_subjects.size > 0
+
+          data = []
+
+          domain_json.each do |option_hash|
+            data << visit_subjects.select{ |v| v == option_hash['value'] }.count
+          end
+
+          series << { name: display_name, data: data }
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series }
+      end
+
+      def self.chart_histogram(chart_type, subjects, json, method)
+        return chart_histogram_choices(chart_type, subjects, json, method) if json['type'] == 'choices'
+        return unless chart_variable_json = get_variable(chart_type)
+        return unless chart_variable_domain = Spout::Commands::JsonChartsAndTables::domain_array(chart_type)
+
+        title = "#{json['display_name']}"
+        subtitle = "By Visit"
+        categories = []
+        units = "Subjects"
+        series = []
+
+        all_subject_values = subjects.collect(&method.to_sym).compact.sort
+        return nil if all_subject_values.count == 0
+        buckets = continuous_buckets(all_subject_values)
+
+        categories = buckets.collect{|b| "#{b[0].round(1)} to #{b[1].round(1)}"}
+
+        chart_variable_domain.each do |display_name, value|
+          visit_subjects = subjects.select{ |s| s.send(chart_type) == value and s.send(method) != nil }.collect(&method.to_sym).sort
+          next unless visit_subjects.size > 0
+
+          data = []
+
+          visit_subjects.group_by{|v| get_bucket(buckets, v) }.each do |key, values|
+            data[categories.index(key)] = values.count if categories.index(key)
+          end
+
+          series << { name: display_name, data: data }
+        end
+
+        { title: title, subtitle: subtitle, categories: categories, units: units, series: series }
+      end
+
+    end
+  end
+end

data/lib/spout/helpers/table_formatting.rb

@@ -0,0 +1,52 @@
+module Spout
+  module Helpers
+    class TableFormatting
+
+      # def initialize(number)
+      #   @number = number
+      # end
+
+      def self.number_with_delimiter(number, delimiter = ",")
+        number.to_s.reverse.scan(/(?:\d*\.)?\d{1,3}-?/).join(',').reverse
+      end
+
+      # type:  :count    or   :decimal
+      def self.format_number(number, type, format = nil)
+        if number == nil
+          format_nil(number)
+        elsif type == :count
+          format_count(number)
+        else
+          format_decimal(number, format)
+        end
+      end
+
+      def self.format_nil(number)
+        '-'
+      end
+
+      #   count:
+      #        0          ->             '-'
+      #       10          ->            '10'
+      #     1000          ->         '1,000'
+      # Input (Numeric)   -> Output (String)
+      def self.format_count(number)
+        (number == 0 || number == nil) ? '-' : number_with_delimiter(number)
+      end
+
+
+      # decimal:
+      #        0          ->           '0.0'
+      #       10          ->          '10.0'
+      #      -50.2555     ->         '-50.3'
+      #     1000          ->       '1,000.0'
+      # 12412423.42252525 ->  '12,412,423.4'
+      # Input (Numeric)   -> Output (String)
+      def self.format_decimal(number, format)
+        number = self.number_with_delimiter(number.round(1))
+        number = format % number if format
+        number
+      end
+    end
+  end
+end

data/lib/spout/models/subject.rb

@@ -0,0 +1,17 @@
+# subject.rb
+
+module Spout
+  module Models
+
+    class Subject
+      attr_accessor :_visit
+
+      def self.create(&block)
+        subject = self.new
+        yield subject if block_given?
+        subject
+      end
+
+    end
+  end
+end

data/lib/spout/tasks/engine.rake

@@ -131,6 +131,13 @@ namespace :spout do
     Spout::Commands::Graphs.new(types, variable_ids, sizes)
   end
 
+  desc 'Generate JSON charts and tables'
+  task :json do
+    require 'spout/commands/json_charts_and_tables'
+    variables = ENV['variables'].to_s.split(',').collect{|s| s.to_s.downcase}
+    Spout::Commands::JsonChartsAndTables.new(variables)
+  end
+
 end
 
 class SpoutCoverageResult

data/lib/spout/templates/gitignore

@@ -9,3 +9,4 @@
 /coverage
 /csvs
 /graphs
+/charts

data/lib/spout/version.rb

@@ -1,9 +1,9 @@
 module Spout
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 7
+    MINOR = 8
     TINY = 0
-    BUILD = nil # nil, "pre", "rc", "rc2"
+    BUILD = "beta1" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

data/spout.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rake"
   spec.add_dependency "turn"
   spec.add_dependency "json"
-  spec.add_dependency "colorize", "~> 0.6.0"
+  spec.add_dependency "colorize", "~> 0.7.2"
 
   spec.add_development_dependency "bundler", "~> 1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0.beta1
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-16 00:00:00.000000000 Z
+date: 2014-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,12 @@ files:
 - lib/spout/actions.rb
 - lib/spout/application.rb
 - lib/spout/commands/graphs.rb
+- lib/spout/commands/json_charts_and_tables.rb
+- lib/spout/helpers/array_statistics.rb
+- lib/spout/helpers/chart_types.rb
+- lib/spout/helpers/table_formatting.rb
 - lib/spout/hidden_reporter.rb
+- lib/spout/models/subject.rb
 - lib/spout/support/javascripts/data.js
 - lib/spout/support/javascripts/highcharts-convert.js
 - lib/spout/support/javascripts/highcharts-more.js
@@ -142,9 +147,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2