spout 0.8.0.beta5 → 0.8.0.beta6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fc36d1e60674ee2067a6ae4456b4bb00352eb34
-  data.tar.gz: 67655d5dc2f69d095eba794a4868a0b59542e03e
+  metadata.gz: e6045b21354f8f0af4c207823913ef0a75a12fb7
+  data.tar.gz: aa6e0e604deeec131d8641370c674df7d411065f
 SHA512:
-  metadata.gz: dce743be43983a5594023bbb459afdbcb7bba1605f62d49a320ba0344c3f70a6a296dec0a6aff7333d3f928a6e9dc2ac11183b80fd8158bb76983c776d48b094
-  data.tar.gz: 8bb42025917af18b7e3253864a82bb3164c6c14c9e6bbdd5c37f0bf575b4a44c7d4be1555a58d111e33ed5fbc07d79a80f0670cc383069b9cf9fdb4715e5efcb
+  metadata.gz: 5a5c0432dd78583502043ad987fdf75727b5322de35bf799b1d9ea6c114129a823cc057463785c662c4867ede530a8425fcddd1c81788014d459f85837e95af5
+  data.tar.gz: 0a95fe2f9d085d2f7108f7fa2d22f5ac33c43a9361cdaf54d226913a2d085dd718f21ad0c1c4006b61effe9950d4d54c8f234c6e7c44da23f0accb146a94c220

data/CHANGELOG.md

@@ -7,6 +7,7 @@
     - `charts`: Array of choices, numeric, or integer variables for charts
 - The `spout pngs` command now renders the histogram form for each variable
 - The `spout coverage` command now lists variables that are defined in the data dictionary and that do not exist in any CSV dataset
+- Added `spout outliers` command that returns a list of integer or numeric variables that contain major and minor outliers
 - **Gem Changes**
   - Updated to colorize 0.7.2
 - Use of Ruby 2.1.2 is now recommended

data/README.md

@@ -132,6 +132,18 @@ This will generate an `index.html` file that can be opened and viewed in any bro
 
 Spout coverage validates that values stored in your dataset match up with variables and domains defined in your data dictionary.
 
+### Identify outliers in your dataset
+
+Spout lets you generate detect outliers in your underlying datasets. Place your dataset csvs into `./csvs/<version>/` and then run the following Spout command:
+
+```
+spout outliers
+```
+
+This will generate an `outliers.html` file that can be opened and viewed in any browser.
+
+Spout outliers computes the [inner and outer fences](http://www.wikihow.com/Calculate-Outliers) to identify minor and major outliers in the dataset.
+
 ### Create a CSV Data Dictionary from your JSON repository
 
 Provide an optional version parameter to name the folder the CSVs will be generated in, defaults to what is in `VERSION` file, or if that does not exist `1.0.0`.

data/lib/spout/actions.rb

@@ -25,6 +25,8 @@ module Spout
         generate_images(argv.last(argv.size - 1))
       when 'graphs', '-graphs', '--graphs', 'g', '-g'
         generate_charts_and_tables(argv.last(argv.size - 1))
+      when 'outliers', '-outliers', '--outliers', 'o', '-o'
+        outliers_report(argv)
       else
         help
       end
@@ -69,7 +71,9 @@ The most common spout commands are:
  h[y]brid  [1.0.0]  Export the JSON dictionary in the Hybrid
                     dictionary format
   [c]overage        Coverage report, requires dataset CSVs
-                    in `<project_name>/csvs/`
+                    in `<project_name>/csvs/<version>`
+  [o]utliers        Outlier report, requires dataset CSVs
+                    in `<project_name>/csvs/<version>`
   [p]ngs            Generates images for each variable in a
                     dataset and places them
                     in `<project_name>/images/<version>/`
@@ -130,6 +134,10 @@ EOT
         system "bundle exec rake spout:coverage"
       end
 
+      def outliers_report(argv)
+        system "bundle exec rake spout:outliers"
+      end
+
       def flag_values(flags, param)
         flags.select{|f| f[0..((param.size + 3) - 1)] == "--#{param}-" and f.length > param.size + 3}.collect{|f| f[(param.size + 3)..-1]}
       end

data/lib/spout/commands/outliers.rb

@@ -0,0 +1,59 @@
+require 'yaml'
+
+require 'spout/helpers/subject_loader'
+require 'spout/models/outlier_result'
+
+module Spout
+  module Commands
+    class Outliers
+      def initialize(standard_version)
+        @standard_version = standard_version
+
+        @variable_files = Dir.glob('variables/**/*.json')
+        @valid_ids = []
+        @number_of_rows = nil
+
+        spout_config = YAML.load_file('.spout.yml')
+        @visit = (spout_config.kind_of?(Hash) ? spout_config['visit'].to_s.strip : '')
+
+        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
+        @subject_loader.load_subjects_from_csvs!
+        @subjects = @subject_loader.subjects
+      end
+
+      def run_outliers_report!
+        @outlier_results = @subject_loader.all_methods.collect do |method, csv_files|
+          Spout::Models::OutlierResult.new(@subjects, method, csv_files)
+        end
+
+        @outlier_results.sort!{|a,b| [a.weight, a.method] <=> [b.weight, b.method]}
+
+        @csv_files = Dir.glob("csvs/#{@standard_version}/*.csv")
+        @overall_results = @csv_files.collect do |csv_file|
+          major_outliers = @outlier_results.select{|outlier_result| outlier_result.csv_files.include?(csv_file) and outlier_result.weight == 0 }.count
+          minor_outliers = @outlier_results.select{|outlier_result| outlier_result.csv_files.include?(csv_file) and outlier_result.weight == 1 }.count
+          total_outliers = major_outliers + minor_outliers
+          [ csv_file, major_outliers, minor_outliers, total_outliers ]
+        end
+
+        coverage_folder = File.join(Dir.pwd, 'coverage')
+        FileUtils.mkpath coverage_folder
+        html_file = File.join(coverage_folder, 'outliers.html')
+
+        print "\nGenerating: outliers.html\n\n"
+
+        File.open(html_file, 'w+') do |file|
+          erb_location = File.join( File.dirname(__FILE__), '../views/outliers.html.erb' )
+          file.puts ERB.new(File.read(erb_location)).result(binding)
+        end
+
+        open_command = 'open'  if RUBY_PLATFORM.match(/darwin/) != nil
+        open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
+
+        system "#{open_command} #{html_file}" if ['start', 'open'].include?(open_command)
+        puts "#{html_file}\n\n"
+      end
+
+    end
+  end
+end

data/lib/spout/helpers/array_statistics.rb

@@ -80,6 +80,36 @@ class Array
     self.compact.max
   end
 
+  def outliers
+    array = self.compact.sort.select{|v| v.kind_of?(Numeric)}
+    q1 = (array.quartile_one + array.quartile_two).median
+    q3 = (array.quartile_three + array.quartile_four).median
+    return [] if q1 == nil or q3 == nil
+    iq_range = q3 - q1
+    inner_fence_lower = q1 - iq_range * 1.5
+    inner_fence_upper = q3 + iq_range * 1.5
+    outer_fence_lower = q1 - iq_range * 3
+    outer_fence_upper = q3 + iq_range * 3
+    array.select{ |v| v > inner_fence_upper or v < inner_fence_lower }
+  end
+
+  def major_outliers
+    array = self.compact.sort.select{|v| v.kind_of?(Numeric)}
+    q1 = (array.quartile_one + array.quartile_two).median
+    q3 = (array.quartile_three + array.quartile_four).median
+    return [] if q1 == nil or q3 == nil
+    iq_range = q3 - q1
+    inner_fence_lower = q1 - iq_range * 1.5
+    inner_fence_upper = q3 + iq_range * 1.5
+    outer_fence_lower = q1 - iq_range * 3
+    outer_fence_upper = q3 + iq_range * 3
+    array.select{ |v| v > outer_fence_upper or v < outer_fence_lower }
+  end
+
+  def minor_outliers
+    self.outliers - self.major_outliers
+  end
+
 end
 
 module Spout

data/lib/spout/helpers/json_loader.rb

@@ -0,0 +1,17 @@
+module Spout
+  module Helpers
+    class JsonLoader
+
+      def self.get_json(file_name, file_type)
+        file = Dir.glob("#{file_type}s/**/#{file_name}.json").first
+        json = JSON.parse(File.read(file)) rescue json = nil
+        json
+      end
+
+      def self.get_variable(variable_name)
+        get_json(variable_name, 'variable')
+      end
+
+    end
+  end
+end

data/lib/spout/models/outlier_result.rb

@@ -0,0 +1,37 @@
+require 'spout/helpers/array_statistics'
+require 'spout/helpers/json_loader'
+
+module Spout
+  module Models
+    class OutlierResult
+      attr_reader :csv_files, :method, :major_outliers, :minor_outliers, :outliers, :weight, :units
+
+      def initialize(subjects, method, csv_files)
+        @values = subjects.collect(&method.to_sym)
+        @csv_files = csv_files
+        @method = method
+
+
+        calculate_outliers!
+
+        @weight = if @major_outliers.count > 0
+          0
+        elsif @minor_outliers.count > 0
+          1
+        else
+          2
+        end
+        variable = Spout::Helpers::JsonLoader::get_variable(method)
+        @units = (variable.kind_of?(Hash) ? variable['units'] : nil)
+      end
+
+      def calculate_outliers!
+        @major_outliers = @values.major_outliers.uniq
+        @minor_outliers = @values.minor_outliers.uniq
+        @outliers = @values.outliers.uniq
+      end
+
+    end
+
+  end
+end

data/lib/spout/tasks/engine.rake

@@ -50,6 +50,13 @@ namespace :spout do
     Spout::Commands::Coverage.new(standard_version)
   end
 
+  desc 'Identify Outliers in CSV dataset'
+  task :outliers do
+    require 'spout/commands/outliers'
+    outliers = Spout::Commands::Outliers.new(standard_version)
+    outliers.run_outliers_report!
+  end
+
   desc 'Match CSV dataset with JSON repository'
   task :images do
     require 'spout/commands/images'

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 8
     TINY = 0
-    BUILD = "beta5" # nil, "pre", "rc", "rc2"
+    BUILD = "beta6" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

data/lib/spout/views/outliers.html.erb

@@ -0,0 +1,150 @@
+<html lang="en"><head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="">
+    <link rel="shortcut icon" href="data:image/vnd.microsoft.icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABMLAAATCwAAAAAAAAAAAAAAAAAAAAAAIAAAAMIAAAAiAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAcAAAAwQAAACEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAADCAAAAIgAAAAAAAAAAAAAAAAAAAEQAAABBAAAAHAAAAMEAAAAhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAwwAAACIAAAAAAAAAAAAAAAAAAABEAAAAQQAAABwAAADBAAAAIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAAKoAAAAeAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAYAAAAqwAAAB4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAWAAAACAAAAB8AAAAfAAAAAAAAAEQAAABDAAAAAAAAABgAAAAIAAAAHQAAAB0AAAAEAAAAAAAAAAAAAAAAAAAAAAAAABkAAACuAAAArQAAABQAAABCAAAAQwAAAAAAAAAAAAAAGAAAAK0AAACrAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAADAAAAGQAAABkAAAAAAAAARAAAAEMAAAAAAAAAAAAAAAMAAAAZAAAAGQAAAAMAAAAAAAAATAAAAEwAAABOAAAATAAAAEoAAABKAAAASQAAAHwAAAB7AAAASgAAAEwAAABMAAAASgAAAEoAAABMAAAATAAAAFAAAABnAAAA0AAAAGcAAABRAAAAUgAAAE8AAACAAAAAfgAAAE0AAABQAAAAUAAAAFAAAABSAAAAUgAAAFIAAAAAAAAAIAAAAL8AAAAfAAAAAAAAAAAAAAAAAAAARQAAAFkAAAAbAAAAHgAAAB8AAAAbAAAABAAAAAAAAAAAAAAAAAAAACAAAAC/AAAAHwAAAAIAAAAXAAAAAAAAAEgAAADPAAAAwwAAAMMAAADFAAAArAAAABgAAAAAAAAAAAAAAAAAAAAgAAAAvwAAAB4AAAAcAAAAqwAAABkAAABDAAAAWgAAAB0AAAAgAAAAHwAAABsAAAAEAAAAAAAAAAAAAAAAAAAAHAAAAKgAAAAaAAAAIAAAAMMAAAAdAAAAQgAAAEMAAAAAAAAABAAAABsAAAAfAAAAIAAAACEAAAAhAAAAAAAAAAQAAAAXAAAAAgAAACIAAADDAAAAHQAAAEIAAABDAAAAAAAAABkAAACrAAAAwwAAAMIAAADCAAAAwQAAAAAAAAAAAAAAAAAAAAAAAAAiAAAAwwAAAB0AAABCAAAAQwAAAAAAAAAEAAAAHQAAACEAAAAgAAAAIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAMMAAAAdAAAAQgAAAEMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAjg8AAI4PAACODwAAjg8AAIABAADgYQAA4mEAAAAAAAAAAAAAjgMAAIIDAACAAwAAgEAAAIBAAADwQAAA8H8AAA==">
+
+    <title>Spout Coverage</title>
+
+    <!-- Bootstrap core CSS -->
+    <link href="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
+
+    <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+      <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+    <![endif]-->
+
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+    <script src="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
+    <style type="text/css">
+html {
+  position: relative;
+  min-height: 100%;
+}
+body {
+  padding-top: 50px;
+  /* Margin bottom by footer height */
+  margin-bottom: 60px;
+}
+#footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  /* Set the fixed height of the footer here */
+  height: 60px;
+  background-color: #f5f5f5;
+  padding: 0px 30px;
+}
+code.success {
+  color: #468847;
+  background-color: #dff0d8;
+}
+code.default {
+  color: #333;
+  background-color: #f5f5f5;
+}
+code.danger {
+  color: #FFF;
+  background-color: #d9534f;
+}
+code.warning {
+  color: #333;
+  background-color: #f0ad4e;
+}
+tfoot td {
+  vertical-align: middle !important;
+  padding-bottom: 0px !important;
+}
+    </style>
+  </head>
+
+  <body>
+
+    <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="navbar-header" style="padding: 0px 30px">
+        <a class="navbar-brand" href="#">Spout Outliers</a>
+      </div>
+    </div>
+
+    <% if @subject_loader.all_methods.size == 0 %>
+      <div class="container" style="margin-top:30px">
+        <div class="jumbotron">
+          <h1>You made Spout cry... :'-(</h1>
+          <p>No CSVs found in <code><%= Dir.pwd %>/csvs/<%= @standard_version %>/</code></p>
+        </div>
+      </div>
+    <% else %>
+      <div style="padding: 30px 30px 10px 30px;">
+
+        <table class="table">
+          <thead>
+            <tr>
+              <th>CSV</th>
+              <th>Major Outliers</th>
+              <th>Minor Outliers</th>
+              <th>Total Outliers</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @overall_results.each do |csv_file, major_outliers, minor_outliers, total_outliers| %>
+              <tr>
+                <td><%= csv_file %></td>
+                <td><%= major_outliers %></td>
+                <td><%= minor_outliers %></td>
+                <td><%= total_outliers %></td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+
+
+        <table class="table table-bordered table-hover">
+          <thead>
+            <tr>
+              <th>CSV</th>
+              <th>Column</th>
+              <th>Units</th>
+              <th>Major Outliers</th>
+              <th>Minor Outliers</th>
+              <th>Outliers</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @outlier_results.each do |outlier_result| %>
+              <tr>
+                <td>
+                  <% outlier_result.csv_files.each do |csv_file| %>
+                    <code class="<%= 'success' if outlier_result.outliers.size == 0 %>"><%= csv_file %><code><br />
+                  <% end %></td>
+                <td><%= outlier_result.method %></td>
+                <td><%= outlier_result.units %></td>
+                <td>
+                  <% outlier_result.major_outliers.collect{|v| v.round(2)}.uniq.each do |value| %>
+                    <code class="danger"><%= value %></code>
+                  <% end %>
+                </td>
+                <td>
+                  <% outlier_result.minor_outliers.collect{|v| v.round(2)}.uniq.each do |value| %>
+                    <code class="warning"><%= value %></code>
+                  <% end %>
+                </td>
+                <td>
+                  <% outlier_result.outliers.collect{|v| v.round(2)}.uniq.each do |value| %>
+                    <code class="default"><%= value %></code>
+                  <% end %>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    <% end %>
+
+    <div id="footer">
+      <p class="text-muted" style="margin: 20px 0;">Generated by <a href="https://github.com/sleepepi/spout">Spout</a> v<%= Spout::VERSION::STRING %></p>
+    </div>
+
+</body></html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.8.0.beta5
+  version: 0.8.0.beta6
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -100,12 +100,15 @@ files:
 - lib/spout/commands/coverage.rb
 - lib/spout/commands/graphs.rb
 - lib/spout/commands/images.rb
+- lib/spout/commands/outliers.rb
 - lib/spout/helpers/array_statistics.rb
 - lib/spout/helpers/chart_types.rb
+- lib/spout/helpers/json_loader.rb
 - lib/spout/helpers/subject_loader.rb
 - lib/spout/helpers/table_formatting.rb
 - lib/spout/hidden_reporter.rb
 - lib/spout/models/coverage_result.rb
+- lib/spout/models/outlier_result.rb
 - lib/spout/models/subject.rb
 - lib/spout/support/javascripts/data.js
 - lib/spout/support/javascripts/highcharts-convert.js
@@ -134,6 +137,7 @@ files:
 - lib/spout/tests/variable_type_validation.rb
 - lib/spout/version.rb
 - lib/spout/views/index.html.erb
+- lib/spout/views/outliers.html.erb
 - spout.gemspec
 homepage: https://github.com/sleepepi
 licenses: