spout 0.8.0.beta1 → 0.8.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55751ba1e111b174e9501fb52b1dd9a484b9ee1c
-  data.tar.gz: ca81a991a37e425a4166dd28e6aeffa183378b1c
+  metadata.gz: 9e4150e7322cba034877fabe6e4a6cec8bdcd444
+  data.tar.gz: 25d7aba466f6667316378a3582f6625c1f541bf7
 SHA512:
-  metadata.gz: 6fec0a99eeb2b26af88652c7b6b79b3d5de8b750593aac2196275e1a1d365dda0814f47343ac312adaf9bb41bd29065a5d6dee8092de8927bdd41136d6ffbca0
-  data.tar.gz: 1d80934213ba8633506850aeabc142764f578d6fa727ffe7ac5ac76cc07422b4e0759d53e7200c5c686075077d3a37cd1101d7b650294d147db5a2f38f1e43eb
+  metadata.gz: 04137f2579d7da8bf3bcc5ad4105d7ccfe78799656015bbdc0b3e0b34a2503522eb90312919969dad7ec0fbaa6f60231aec7f90bda1e8bbfd8817b28e06e6aea
+  data.tar.gz: 98fae07b2e79ee985c0ef849b6e0e2706c824d0f7d5c767ca2c97b24523fefd92da973b4ccb762273c814a2843f365dadbca5535a1d7f3f388aa1d5f9ead8336

data/CHANGELOG.md

@@ -1,10 +1,11 @@
 ## 0.8.0
 
 ### Enhancements
-- Added `spout json` command that generates JSON charts and tables of each variable in a dataset
+- Added `spout graphs` command that generates JSON charts and tables of each variable in a dataset
   - This command requires a .spout.yml file to be specified to identify the following variables:
     - `visit`: This variable is used to separate subject encounters in a histogram
     - `charts`: Array of choices, numeric, or integer variables for charts
+- The `spout pngs` command now renders the histogram form for each variable
 - **Gem Changes**
   - Updated to colorize 0.7.2
 - Use of Ruby 2.1.2 is now recommended
@@ -12,16 +13,16 @@
 ## 0.7.0 (April 16, 2014)
 
 ### Enhancements
-- Added `spout graphs` command that generates pie charts and histograms of each variable in a dataset
+- Added `spout pngs` command that generates pie charts and histograms of each variable in a dataset
   - The following flags are available:
-    - `spout g --type-numeric`
-    - `spout g --type-integer`
-    - `spout g --type-choices`
-    - `spout g --size-lg`
-    - `spout g --size-sm`
-    - `spout g --type-numeric --size-sm`
+    - `spout p --type-numeric`
+    - `spout p --type-integer`
+    - `spout p --type-choices`
+    - `spout p --size-lg`
+    - `spout p --size-sm`
+    - `spout p --type-numeric --size-sm`
   - For specific variables the following can be used:
-    - `spout g --id-<variable_id>`
+    - `spout p --id-<variable_id>`
 
 ## 0.6.0 (March 7, 2014)
 

data/README.md

@@ -146,35 +146,35 @@ You can optionally provide a version string
 spout export [1.0.0]
 ```
 
-### Generate graphs for data in your dataset
+### Generate images for data in your dataset
 
-Spout lets you generate graphs for each variable defined in your dataset. Make sure to run `spout coverage` first to validate that your data dictionary and dataset match.
+Spout lets you generate images for each variable defined in your dataset. Make sure to run `spout coverage` first to validate that your data dictionary and dataset match.
 
 This command will take some time, and requires [PhantomJS](http://phantomjs.org/) to be installed on your system.
 
 ```
-spout graphs
+spout pngs
 ```
 
-The following flags can be passed to the `spout graphs` command:
+The following flags can be passed to the `spout pngs` command:
 
-- `spout g --type-numeric`
-- `spout g --type-integer`
-- `spout g --type-choices`
-- `spout g --size-lg`
-- `spout g --size-sm`
-- `spout g --type-numeric --size-sm`
+- `spout p --type-numeric`
+- `spout p --type-integer`
+- `spout p --type-choices`
+- `spout p --size-lg`
+- `spout p --size-sm`
+- `spout p --type-numeric --size-sm`
 
 For specific variables the following can be used:
-  - `spout g --id-<variable_id>`
+  - `spout p --id-<variable_id>`
 
-Generated graphs are placed in: `./graphs/`
+Generated images are placed in: `./images/`
 
 
 ### Generate charts and tables for data in your dataset
 
 ```
-spout json
+spout graphs
 ```
 
 This command generates JSON charts and tables of each variable in a dataset
@@ -194,6 +194,23 @@ charts:
   - race
 ```
 
+To only generate graphs for a few select variables, add the variable names after the `spout graphs` command.
+
+For example, the command below will only generate graphs for the two variables `ahi` and `bmi`.
+
+```
+spout g ahi bmi
+```
+
+You can also specify a limit to the amount of rows to read in from the CSV files by specifying the `-rows` flag
+
+```
+spout -rows=10 ahi
+```
+
+This will generate a graph for ahi for the first 10 rows of each dataset CSV.
+
+
 This will generate charts and tables for each variable in the dataset plotted against the variables listed under `charts`.
 
 

data/lib/spout/actions.rb

@@ -21,9 +21,9 @@ module Spout
         new_data_dictionary_export(argv, 'hybrid')
       when 'coverage', '-coverage', '--coverage', 'c', '-c'
         coverage_report(argv)
+      when 'pngs', '-pngs', '--pngs', 'p', '-p'
+        generate_images(argv.last(argv.size - 1))
       when 'graphs', '-graphs', '--graphs', 'g', '-g'
-        generate_graphs(argv.last(argv.size - 1))
-      when 'json', 'j'
         generate_charts_and_tables(argv.last(argv.size - 1))
       else
         help
@@ -70,9 +70,12 @@ The most common spout commands are:
                     dictionary format
   [c]overage        Coverage report, requires dataset CSVs
                     in `<project_name>/csvs/`
-  [g]raphs          Generates graphs for each variable in a
+  [p]ngs            Generates images for each variable in a
                     dataset and places them
-                    in `<project_name>/graphs/`
+                    in `<project_name>/images/<version>/`
+  [g]raphs          Generates JSON graphs for each variable
+                    in a dataset and places them
+                    in `<project_name>/graphs/<version>/`
   [v]ersion         Returns the version of Spout
 
 Commands can be referenced by the first letter:
@@ -131,7 +134,7 @@ EOT
         flags.select{|f| f[0..((param.size + 3) - 1)] == "--#{param}-" and f.length > param.size + 3}.collect{|f| f[(param.size + 3)..-1]}
       end
 
-      def generate_graphs(flags)
+      def generate_images(flags)
         params = {}
         params['types']        = flag_values(flags, 'type')
         params['variable_ids'] = flag_values(flags, 'id')
@@ -139,7 +142,7 @@ EOT
 
         params_string = params.collect{|key, values| "#{key}=#{values.join(',')}"}.join(' ')
 
-        system "bundle exec rake spout:graphs #{params_string}"
+        system "bundle exec rake spout:images #{params_string}"
       end
 
       def generate_charts_and_tables(variables)

data/lib/spout/commands/coverage.rb

@@ -0,0 +1,75 @@
+require 'yaml'
+
+require 'spout/helpers/subject_loader'
+require 'spout/models/coverage_result'
+
+module Spout
+  module Commands
+    class Coverage
+      def initialize(standard_version)
+        @standard_version = standard_version
+
+        @variable_files = []
+        @valid_ids = []
+        @number_of_rows = 100
+
+        spout_config = YAML.load_file('.spout.yml')
+        @visit = (spout_config.kind_of?(Hash) ? spout_config['visit'].to_s.strip : '')
+
+        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
+        @subject_loader.load_subjects_from_csvs_part_one! # Not Part Two which is essentially cleaning the data
+        @subjects = @subject_loader.subjects
+
+        run_coverage_report!
+      end
+
+      def run_coverage_report!
+        choice_variables = []
+
+        Dir.glob("variables/**/*.json").each do |file|
+          if json = JSON.parse(File.read(file)) rescue false
+            choice_variables << json['id'] if json['type'] == 'choices'
+          end
+        end
+
+        @matching_results = []
+        csv_names = ['tmp_csv_file.csv']
+
+        all_column_headers = @subjects.size > 0 ? @subjects.first.class.instance_methods(false).select{|k| k != :_visit}.reject{|k| k.to_s[-1] == '='} : []
+        all_column_headers.each do |column|
+          csv = 'tmp_csv_file.csv'
+          scr = Spout::Models::CoverageResult.new(csv, column.to_s, @subjects.collect(&column))
+          @matching_results << [ csv, column, scr ]
+        end
+
+
+        @matching_results.sort!{|a,b| [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]}
+
+        @coverage_results = []
+
+        csv_names.each do |csv_name|
+          total_column_count = @matching_results.select{|mr| mr[0] == csv_name}.count
+          mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2].number_of_errors == 0}.count
+          @coverage_results << [ csv_name, total_column_count, mapped_column_count ]
+        end
+
+        coverage_folder = File.join(Dir.pwd, 'coverage')
+        FileUtils.mkpath coverage_folder
+        coverage_file = File.join(coverage_folder, 'index.html')
+
+        print "\nGenerating: index.html\n\n"
+
+        File.open(coverage_file, 'w+') do |file|
+          erb_location = File.join( File.dirname(__FILE__), '../views/index.html.erb' )
+          file.puts ERB.new(File.read(erb_location)).result(binding)
+        end
+
+        open_command = 'open'  if RUBY_PLATFORM.match(/darwin/) != nil
+        open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
+
+        system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
+        puts "#{coverage_file}\n\n"
+      end
+    end
+  end
+end

data/lib/spout/commands/graphs.rb

@@ -2,179 +2,127 @@ require 'csv'
 require 'fileutils'
 require 'rubygems'
 require 'json'
+require 'yaml'
+
+require 'spout/helpers/subject_loader'
+require 'spout/helpers/chart_types'
 
 module Spout
   module Commands
     class Graphs
+      def initialize(variables, standard_version)
+        @standard_version = standard_version
 
-      def initialize(types, variable_ids, sizes)
-
-        total_index_count = Dir.glob("variables/**/*.json").count
-
-        last_completed = 0
-
-        options_folder = 'graphs'
-        FileUtils.mkpath( options_folder )
-        tmp_options_file = File.join( options_folder, 'options.json' )
-
-        Dir.glob("csvs/*.csv").each do |csv_file|
-          puts "Working on: #{csv_file}"
-          t = Time.now
-          csv_table = CSV.table(csv_file, encoding: 'iso-8859-1').by_col!
-          puts "Loaded #{csv_file} in #{Time.now - t} seconds."
-
-          total_header_count = csv_table.headers.count
-          csv_table.headers.each_with_index do |header, index|
-            puts "Column #{ index + 1 } of #{ total_header_count } for #{header} in #{csv_file}"
-            if variable_file = Dir.glob("variables/**/#{header.downcase}.json", File::FNM_CASEFOLD).first
-              json = JSON.parse(File.read(variable_file)) rescue json = nil
-              next unless json
-              next unless ["choices", "numeric", "integer"].include?(json["type"])
-              next unless types.size == 0 or types.include?(json['type'])
-              next unless variable_ids.size == 0 or variable_ids.include?(json['id'].to_s.downcase)
-
-              basename = File.basename(variable_file).gsub(/\.json$/, '').downcase
-              col_data = csv_table[header]
-
-              case json["type"] when "choices"
-                domain_file = Dir.glob("domains/**/#{json['domain']}.json").first
-                domain_json = JSON.parse(File.read(domain_file)) rescue domain_json = nil
-                next unless domain_json
-
-                create_pie_chart_options_file(col_data, tmp_options_file, domain_json)
-              when 'numeric', 'integer'
-                create_line_chart_options_file(col_data, tmp_options_file, json["units"])
-              else
-                next
-              end
-
-              run_phantom_js("#{basename}-lg.png", 600, tmp_options_file) if sizes.size == 0 or sizes.include?('lg')
-              run_phantom_js("#{basename}.png",     75, tmp_options_file) if sizes.size == 0 or sizes.include?('sm')
-            end
-          end
-        end
-        File.delete(tmp_options_file) if File.exists?(tmp_options_file)
-      end
+        spout_config = YAML.load_file('.spout.yml')
 
-      def graph_values(col_data)
-        categories = []
+        @visit = ''
 
-        col_data = col_data.select{|v| !['', 'null'].include?(v.to_s.strip.downcase)}.collect(&:to_f)
+        if spout_config.kind_of?(Hash)
+          @visit = spout_config['visit'].to_s.strip
 
-        all_integers = false
-        all_integers = (col_data.count{|i| i.denominator != 1} == 0)
-
-        minimum = col_data.min || 0
-        maximum = col_data.max || 100
-
-        default_max_buckets = 30
-        max_buckets = all_integers ? [maximum - minimum + 1, default_max_buckets].min : default_max_buckets
-        bucket_size = (maximum - minimum + 1).to_f / max_buckets
-
-        (0..(max_buckets-1)).each do |bucket|
-          val_min = (bucket_size * bucket) + minimum
-          val_max = bucket_size * (bucket + 1) + minimum
-          # Greater or equal to val_min, less than val_max
-          # categories << "'#{val_min} to #{val_max}'"
-          categories << "#{all_integers || (maximum - minimum) > (default_max_buckets / 2)  ? val_min.round : "%0.02f" % val_min}"
+          chart_variables = if spout_config['charts'].kind_of?(Array)
+            spout_config['charts'].select{|c| c.kind_of?(Hash)}
+          else
+            []
+          end
+        else
+          puts "The YAML file needs to be in the following format:"
+          puts "---\nvisit: visit_variable_name\ncharts:\n- chart: age_variable_name\n  title: Age\n- chart: gender_variable_name\n  title: Gender\n- chart: race_variable_name\n  title: Race\n"
+          exit
         end
 
-        new_values = []
-        (0..max_buckets-1).each do |bucket|
-          val_min = (bucket_size * bucket) + minimum
-          val_max = bucket_size * (bucket + 1) + minimum
-          # Greater or equal to val_min, less than val_max
-          new_values << col_data.count{|i| i >= val_min and i < val_max}
+        if Spout::Helpers::ChartTypes::get_json(@visit, 'variable') == nil
+          if @visit == ''
+            puts "The visit variable in .spout.yml can't be blank."
+          else
+            puts "Could not find the following visit variable: #{@visit}"
+          end
+          exit
+        end
+        missing_variables = chart_variables.select{|c| Spout::Helpers::ChartTypes::get_json(c['chart'], 'variable') == nil}
+        if missing_variables.count > 0
+          puts "Could not find the following chart variable#{'s' unless missing_variables.size == 1}: #{missing_variables.join(', ')}"
+          exit
         end
 
-        values = []
-
-        values << { name: '', data: new_values, showInLegend: false }
+        argv_string = variables.join(',')
+        @number_of_rows = nil
 
-        [ values, categories ]
-      end
+        if match_data = argv_string.match(/-rows=(\d*)/)
+          @number_of_rows = match_data[1].to_i
+          argv_string.gsub!(match_data[0], '')
+        end
 
+        @valid_ids = argv_string.split(',').compact.reject{|s| s == ''}
 
-      def create_pie_chart_options_file(values, options_file, domain_json)
+        @chart_variables = chart_variables.unshift( { "chart" => @visit, "title" => 'Histogram' } )
 
-        values.select!{|v| !['', 'null'].include?(v.to_s.strip.downcase) }
-        counts = values.group_by{|a| a}.collect{|k,v| [(domain_json.select{|h| h['value'] == k.to_s}.first['display_name'] rescue (k.to_s == '' ? 'NULL' : k)), v.count]}
+        @variable_files = Dir.glob('variables/**/*.json')
 
-        total_count = counts.collect(&:last).inject(&:+)
+        t = Time.now
+        FileUtils.mkpath "graphs/#{@standard_version}"
 
-        data = counts.collect{|value, count| [value, (count * 100.0 / total_count)]}
+        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
 
-        File.open(options_file, "w") do |outfile|
-          outfile.puts <<-eos
-            {
-              "title": {
-                "text": ""
-              },
+        @subject_loader.load_subjects_from_csvs!
+        @subjects = @subject_loader.subjects
+        compute_tables_and_charts
 
-              "credits": {
-                  "enabled": false,
-              },
-              "series": [{
-                        "type": "pie",
-                        "name": "",
-                        "data": #{data.to_json}
-              }]
-            }
-          eos
-        end
+        puts "Took #{Time.now - t} seconds."
       end
 
+      def compute_tables_and_charts
+        variable_files_count = @variable_files.count
+        @variable_files.each_with_index do |variable_file, file_index|
+          json = JSON.parse(File.read(variable_file)) rescue json = nil
+          next unless json
+          next unless @valid_ids.include?(json["id"].to_s.downcase) or @valid_ids.size == 0
+          next unless ["numeric", "integer", "choices"].include?(json["type"])
+          variable_name  = json['id'].to_s.downcase
+          next unless Spout::Models::Subject.method_defined?(variable_name)
+
+          puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
+
+
+          stats = {
+            charts: {},
+            tables: {}
+          }
+
+          @chart_variables.each do |chart_type_hash|
+            chart_type = chart_type_hash["chart"]
+            chart_title = chart_type_hash["title"].downcase
+
+            if chart_type == @visit
+              filtered_subjects = @subjects.select{ |s| s.send(chart_type) != nil }  # and s.send(variable_name) != nil
+              if filtered_subjects.count > 0
+                stats[:charts][chart_title] = Spout::Helpers::ChartTypes::chart_histogram(chart_type, filtered_subjects, json, variable_name)
+                stats[:tables][chart_title] = Spout::Helpers::ChartTypes::table_arbitrary(chart_type, filtered_subjects, json, variable_name)
+              end
+            else
+              filtered_subjects = @subjects.select{ |s| s.send(chart_type) != nil } # and s.send(variable_name) != nil
+              if filtered_subjects.count > 0
+                stats[:charts][chart_title] = Spout::Helpers::ChartTypes::chart_arbitrary(chart_type, filtered_subjects, json, variable_name, visits)
+                stats[:tables][chart_title] = visits.collect do |visit_display_name, visit_value|
+                  visit_subjects = filtered_subjects.select{ |s| s._visit == visit_value }
+                  unknown_subjects = visit_subjects.select{ |s| s.send(variable_name) == nil }
+                  (visit_subjects.count > 0 && visit_subjects.count != unknown_subjects.count) ? Spout::Helpers::ChartTypes::table_arbitrary(chart_type, visit_subjects, json, variable_name, visit_display_name) : nil
+                end.compact
+              end
+            end
+          end
+
+          chart_json_file = File.join('graphs', @standard_version, "#{json['id']}.json")
+          File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
 
-      def create_line_chart_options_file(values, options_file, units)
-        ( series, categories ) = graph_values(values)
-
-        File.open(options_file, "w") do |outfile|
-          outfile.puts <<-eos
-            {
-              "chart": {
-                  "type": "areaspline"
-              },
-              "title": {
-                "text": ""
-              },
-              "credits": {
-                  "enabled": false,
-              },
-              "xAxis": {
-                  "categories": #{categories.to_json},
-                  "labels": {
-                      "step": #{(categories.size.to_f / 12).ceil}
-                  },
-                  "title": {
-                      "text": #{units.to_json}
-                  }
-              },
-              "yAxis": {
-                  "maxPadding": 0,
-                  "minPadding": 0,
-                  "title": {
-                    "text": "Count"
-                  }
-              },
-              "series": #{series.to_json}
-            }
-          eos
         end
       end
 
-      def run_phantom_js(png_name, width, tmp_options_file)
-        graph_path = File.join(Dir.pwd, 'graphs', png_name)
-        directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
-
-        open_command = if RUBY_PLATFORM.match(/mingw/) != nil
-          'phantomjs.exe'
-        else
-          'phantomjs'
+      # [["Visit 1", "1"], ["Visit 2", "2"], ["CVD Outcomes", "3"]]
+      def visits
+        @visits ||= begin
+          Spout::Helpers::ChartTypes::domain_array(@visit)
         end
-
-        phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
-        # puts phantomjs_command
-        `#{phantomjs_command}`
       end
 
     end