rails-data-explorer 0.0.1 → 0.1.0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
-## 0.0.1
+## 0.1.0
+
+* Is now being used in a production app, however no documentation yet.
+
+### 0.0.1
 
 * Initial release

data/README.md

@@ -7,6 +7,10 @@ data in your app using charts and statistics.
 Make sure to go to the thorough [documentation](http://rails-data-explorer.clearcove.ca)
 to find out more!
 
+## IMPORTANT NOTE
+
+This gem is under active development and will see significant changes until it's
+officially released.
 
 ### Installation
 
@@ -32,6 +36,13 @@ or with bundler in your Gemfile:
 * Chart -
 
 
+### Dependencies
+
+* ActionView >= 3.0
+* ActiveSupport >= 3.0
+* Asset pipeline (for batteries included, otherwise you'll have to pull in a number of assets manually)
+* jQuery??
+
 ### Resources
 
 * [Documentation](http://rails-data-explorer.clearcove.ca)

data/Rakefile

@@ -16,3 +16,65 @@ Rake::TestTask.new do |t|
 end
 
 task :default => ['test']
+
+require 'yui/compressor'
+
+namespace :rde do
+
+  def minify_js(output_file_name, file_list)
+    uncompressed_source = ''
+    file_list.each do |file|
+      uncompressed_source << File.read(file)
+    end
+    compressor = YUI::JavaScriptCompressor.new
+    compressed_output = compressor.compress(uncompressed_source)
+    File.write(output_file_name, compressed_output)
+  end
+
+  def minify_css(output_file_name, file_list)
+    uncompressed_source = ''
+    file_list.each do |file|
+      uncompressed_source << File.read(file)
+    end
+    compressor = YUI::CssCompressor.new
+    compressed_output = compressor.compress(uncompressed_source)
+    File.write(output_file_name, compressed_output)
+  end
+
+  desc "minify"
+  task :minify => [:minify_js, :minify_css]
+
+  desc "minify javascript"
+  task :minify_js do
+    base_path = File.expand_path("../vendor/assets/javascripts", __FILE__)
+    minify_js(
+      File.join(base_path, 'packaged/rails-data-explorer.min.js'),
+      [
+        File.join(base_path, 'sources/d3.v3.js'),
+        # File.join(base_path, 'sources/nv.d3.js'),
+        File.join(base_path, 'sources/d3.boxplot.js'),
+        File.join(base_path, 'sources/d3.parcoords.js'),
+        File.join(base_path, 'sources/d3.parsets.js'),
+        File.join(base_path, 'sources/vega.js'),
+      ]
+    )
+  end
+
+  desc "minify css"
+  task :minify_css do
+    base_path = File.expand_path("../vendor/assets/stylesheets", __FILE__)
+    minify_css(
+      File.join(base_path, 'packaged/rails-data-explorer.min.css'),
+      [
+        File.join(base_path, 'sources/bootstrap-theme.css'),
+        File.join(base_path, 'sources/bootstrap.css'),
+        File.join(base_path, 'sources/d3.boxplot.css'),
+        File.join(base_path, 'sources/d3.parcoords.css'),
+        File.join(base_path, 'sources/d3.parsets.css'),
+        # File.join(base_path, 'sources/nv.d3.css'),
+        File.join(base_path, 'sources/rde-default-style.css'),
+      ]
+    )
+  end
+
+end

data/doc/how_to/release.md

@@ -0,0 +1,23 @@
+Workflow to Maintain This Gem
+=============================
+
+I use the gem-release gem
+
+For more info see: https://github.com/svenfuchs/gem-release#usage
+
+Steps for an update
+-------------------
+
+1. Update code and commit it.
+2. Add entry to CHANGELOG and commit it:
+   * h1 for major release (1.0.0)
+   * h2 for minor release (0.1.0)
+   * h3 for patch release (0.0.1)
+3. Update version in .gemspec and commit
+4. Release it.
+   * `gem release`
+5. Create a git tag and push to origin.
+   `gem tag`
+
+
+http://prioritized.net/blog/gemify-assets-for-rails/

data/doc/how_to/trouble_when_packaging_assets.md

@@ -0,0 +1,8 @@
+Trouble when packaging assets
+=============================
+
+I had syntax and dot errors when packaging the JS assets. Turns out that the
+JS libraries used reserved words as method accessors. Here is an example how I
+fixed it (`in`) is the reserved word:
+
+`c.in` => `c["in"]`

data/lib/rails-data-explorer-no-rails.rb

@@ -0,0 +1,42 @@
+# Require this file for fast testing outside of rails. Don't use this for normal
+# usage.
+require 'action_view'
+require 'active_support/all' # TODO: once the dust settles, only require the modules we need
+require 'color'
+require 'descriptive-statistics'
+require 'distribution'
+require 'interpolate'
+
+# require 'rails-data-explorer/engine'
+
+require 'rails_data_explorer'
+require 'rails-data-explorer/chart'
+require 'rails-data-explorer/data_series'
+require 'rails-data-explorer/data_set'
+require 'rails-data-explorer/data_type'
+require 'rails-data-explorer/exploration'
+require 'rails-data-explorer/statistics/rng_category'
+require 'rails-data-explorer/statistics/rng_gaussian'
+require 'rails-data-explorer/statistics/rng_power_law'
+require 'rails-data-explorer/utils/color_scale'
+require 'rails-data-explorer/utils/data_binner'
+require 'rails-data-explorer/utils/data_quantizer'
+require 'rails-data-explorer/utils/value_formatter'
+
+require 'rails-data-explorer/chart/box_plot'
+require 'rails-data-explorer/chart/box_plot_group'
+require 'rails-data-explorer/chart/contingency_table'
+require 'rails-data-explorer/chart/descriptive_statistics_table'
+require 'rails-data-explorer/chart/histogram_categorical'
+require 'rails-data-explorer/chart/histogram_quantitative'
+require 'rails-data-explorer/chart/histogram_temporal'
+require 'rails-data-explorer/chart/parallel_coordinates'
+require 'rails-data-explorer/chart/parallel_set'
+require 'rails-data-explorer/chart/pie_chart'
+require 'rails-data-explorer/chart/scatterplot'
+require 'rails-data-explorer/chart/stacked_bar_chart_categorical_percent'
+require 'rails-data-explorer/data_type/categorical'
+require 'rails-data-explorer/data_type/quantitative'
+require 'rails-data-explorer/data_type/quantitative/decimal'
+require 'rails-data-explorer/data_type/quantitative/integer'
+require 'rails-data-explorer/data_type/quantitative/temporal'

data/lib/rails-data-explorer.rb

@@ -1,3 +1,5 @@
+require 'action_view'
+require 'active_support/all' # TODO: once the dust settles, only require the modules we need
 require 'color'
 require 'descriptive-statistics'
 require 'distribution'
@@ -5,6 +7,7 @@ require 'interpolate'
 
 require 'rails-data-explorer/engine'
 
+require 'rails_data_explorer'
 require 'rails-data-explorer/chart'
 require 'rails-data-explorer/data_series'
 require 'rails-data-explorer/data_set'
@@ -14,6 +17,8 @@ require 'rails-data-explorer/statistics/rng_category'
 require 'rails-data-explorer/statistics/rng_gaussian'
 require 'rails-data-explorer/statistics/rng_power_law'
 require 'rails-data-explorer/utils/color_scale'
+require 'rails-data-explorer/utils/data_binner'
+require 'rails-data-explorer/utils/data_quantizer'
 require 'rails-data-explorer/utils/value_formatter'
 
 require 'rails-data-explorer/chart/box_plot'
@@ -33,12 +38,3 @@ require 'rails-data-explorer/data_type/quantitative'
 require 'rails-data-explorer/data_type/quantitative/decimal'
 require 'rails-data-explorer/data_type/quantitative/integer'
 require 'rails-data-explorer/data_type/quantitative/temporal'
-
-class RailsDataExplorer
-
-  # Convenience method to instantiate new Exploration
-  def self.new(*args)
-    RailsDataExplorer::Exploration.new(*args)
-  end
-
-end

data/lib/rails-data-explorer/chart/box_plot.rb

@@ -11,12 +11,14 @@ class RailsDataExplorer
 
       def compute_chart_attrs
         x_ds = @data_set.data_series.first
+        return false  if x_ds.nil?
+
         {
           values: [x_ds.values],
           min: x_ds.min_val,
           max: x_ds.max_val,
           base_width: 120,
-          base_height: 1334,
+          base_height: 800,
           axis_tick_format: x_ds.axis_tick_format,
         }
       end
@@ -24,6 +26,8 @@ class RailsDataExplorer
       def render
         return ''  unless render?
         ca = compute_chart_attrs
+        return ''  unless ca
+
         %(
           <div id="#{ dom_id }" class="rde-chart rde-box-plot">
             <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>

data/lib/rails-data-explorer/chart/box_plot_group.rb

@@ -9,6 +9,9 @@ class RailsDataExplorer
   class Chart
     class BoxPlotGroup < Chart
 
+# TODO: imitate this:
+# http://www.stata.com/support/faqs/graphics/gph/graphdocs/horizontal-box-plot-of-variable-by-values-of-categorical-variable/
+
       def initialize(_data_set, options = {})
         @data_set = _data_set
         @options = {}.merge(options)
@@ -24,7 +27,6 @@ class RailsDataExplorer
 
         x_ds = x_candidates.first
         y_ds = (y_candidates - [x_ds]).first
-
         return false  if x_ds.nil? || y_ds.nil?
 
         min = x_ds.min_val # get global min
@@ -36,16 +38,20 @@ class RailsDataExplorer
         }
 
         y_ds.values.each_with_index { |y_val, idx|
+          next  if (y_val.nil? || Float::NAN == y_val)
           values_hash[y_val] << x_ds.values[idx]
         }
 
         {
           values: values_hash.values,
+          category_labels: values_hash.keys,
           min: min,
           max: max,
           base_width: 120,
-          base_height: 1334,
+          base_height: 800,
           axis_tick_format: x_ds.axis_tick_format,
+          num_box_plots: y_ds.uniq_vals_count,
+          axis_scale: DataSeries.new('_', [min, max]).axis_scale
         }
       end
 
@@ -53,10 +59,20 @@ class RailsDataExplorer
         return ''  unless render?
         ca = compute_chart_attrs
         return ''  unless ca
+
+        svg_trs = ca[:category_labels].map { |cat_label|
+          %(
+            <tr>
+              <td style="vertical-align: middle;">#{ cat_label }</td>
+              <td style="vertical-align: middle; width: 100%">
+                <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
+              </td>
+            </tr>
+          )
+        }.join.html_safe
         %(
-          <div id="#{ dom_id }" class="rde-chart rde-box-plot">
-            <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
-            <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
+          <div id="#{ dom_id }" class="rde-chart rde-box-plot-group">
+            <table>#{ svg_trs }</table>
 
             <script type="text/javascript">
               (function() {
@@ -78,6 +94,7 @@ class RailsDataExplorer
                 var data = #{ ca[:values].to_json };
 
                 chart.domain([min, max]);
+                chart.scale(#{ ca[:axis_scale] });
 
                 var svg = d3.select("##{ dom_id }").selectAll("svg")
                             .data(data)

data/lib/rails-data-explorer/chart/contingency_table.rb

@@ -5,6 +5,8 @@
 # * http://www.quora.com/What-is-the-most-intuitive-explanation-for-the-chi-square-test
 # * http://people.revoledu.com/kardi/tutorial/Questionnaire/Chi-Square%20IndependentTest.html
 # * http://stattrek.com/chi-square-test/independence.aspx?Tutorial=AP
+
+# Contingency table and chi squared test is a good tool for interpreting A/B tests.
 class RailsDataExplorer
   class Chart
     class ContingencyTable < Chart
@@ -24,16 +26,23 @@ class RailsDataExplorer
 
         x_ds = x_candidates.first
         y_ds = (y_candidates - [x_ds]).first
+        return false  if x_ds.nil? || y_ds.nil?
 
         # Compute @observed_vals, @expected_vals, etc.
         compute_contingency_and_chi_squared!(x_ds, y_ds)
 
-        x_sorted_keys = x_ds.uniq_vals.sort { |a,b|
-          @observed_vals[b][:_sum] <=> @observed_vals[a][:_sum]
-        }
-        y_sorted_keys = y_ds.uniq_vals.sort { |a,b|
-          @observed_vals[:_sum][b] <=> @observed_vals[:_sum][a]
-        }
+        x_sorted_keys = x_ds.uniq_vals.sort(
+          &x_ds.label_sorter(
+            nil,
+            lambda { |a,b| @observed_vals[b][:_sum] <=> @observed_vals[a][:_sum] }
+          )
+        )
+        y_sorted_keys = y_ds.uniq_vals.sort(
+          &y_ds.label_sorter(
+            nil,
+            lambda { |a,b| @observed_vals[:_sum][b] <=> @observed_vals[:_sum][a] }
+          )
+        )
 
         ca = case @data_set.dimensions_count
         when 2
@@ -97,6 +106,8 @@ class RailsDataExplorer
       def render
         return ''  unless render?
         ca = compute_chart_attrs
+        return ''  unless ca
+
         content_tag(:div, :class => 'rde-chart rde-contingency-table', :id => dom_id) do
           content_tag(:h3, "Contingency Table", :class => 'rde-chart-title') +
           render_html_table(ca)
@@ -176,11 +187,35 @@ class RailsDataExplorer
         # Set significance_level
         @significance_level = 0.05
         # Compute conclusion
-        @conclusion = %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> suggests that )
-        @conclusion << if @p_value <= @significance_level
-          "#{ x_ds.name } and #{ y_ds.name } are dependent variables (p_value: #{ number_with_precision(@p_value) })"
+        all_observed_vals = []
+        x_ds.uniq_vals.each { |x_val|
+          y_ds.uniq_vals.each { |y_val|
+            all_observed_vals << @observed_vals[x_val][y_val]
+          }
+        }
+        observed_vals_less_than_five = all_observed_vals.find_all { |e| e < 5 }
+        ratio_of_observed_vals_below_five = observed_vals_less_than_five.length / all_observed_vals.length.to_f
+
+        if ratio_of_observed_vals_below_five > 0.2
+          @conclusion = [
+            "We did not run the ",
+            %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> ),
+            "since #{ number_to_percentage(ratio_of_observed_vals_below_five * 100, :precision => 0) } ",
+            "of observed values in the contingency table are below 5 (cutoff is 20%)."
+          ].join
+        elsif([x_ds, y_ds].any? { |e| e.uniq_vals.length < 2 })
+          @conclusion = [
+            "We did not run the ",
+            %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> ),
+            "since there are not enough observed values in the contingency table."
+          ].join
         else
-          "#{ x_ds.name } and #{ y_ds.name } are independent variables (p_value: #{ number_with_precision(@p_value) })"
+          @conclusion = %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> suggests that )
+          @conclusion << if @p_value <= @significance_level
+            %("#{ x_ds.name }" and "#{ y_ds.name }" are dependent variables (p_value: #{ number_with_precision(@p_value) }))
+          else
+            %("#{ x_ds.name }" and "#{ y_ds.name }" are independent variables (p_value: #{ number_with_precision(@p_value) }))
+          end
         end
         @conclusion = @conclusion.html_safe
       end

data/lib/rails-data-explorer/chart/histogram_categorical.rb

@@ -9,12 +9,21 @@ class RailsDataExplorer
 
       def compute_chart_attrs
         x_ds = @data_set.data_series.first
+        return false  if x_ds.nil?
+
         # compute histogram
         h = x_ds.values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
         {
-          values: h.map { |k,v| { x: k, y: v } }.sort { |a,b| b[:y] <=> a[:y] },
+          values: h.map { |k,v|
+            { x: k, y: v }
+          }.sort(
+            &x_ds.label_sorter(
+              :x,
+              lambda { |a,b| b[:y] <=> a[:y] }
+            )
+          ),
           x_axis_label: x_ds.name,
-          x_axis_tick_format: "",
+          x_axis_tick_format: "d3.format('r')",
           y_axis_label: 'Frequency',
           y_axis_tick_format: "d3.format('r')",
         }
@@ -23,8 +32,88 @@ class RailsDataExplorer
       def render
         return ''  unless render?
         ca = compute_chart_attrs
+        return ''  unless ca
+        render_vega(ca)
+      end
+
+      def render_vega(ca)
         %(
-          <div class="rde-chart rde-histogram">
+          <div class="rde-chart rde-histogram-categorical">
+            <h3 class="rde-chart-title">Histogram</h3>
+            <div id="#{ dom_id }"></div>
+            <script type="text/javascript">
+              (function() {
+                var spec = {
+                  "width": 800,
+                  "height": 200,
+                  "padding": {"top": 10, "left": 70, "bottom": 50, "right": 10},
+                  "data": [
+                    {
+                      "name": "table",
+                      "values": #{ ca[:values].to_json }
+                    }
+                  ],
+                  "scales": [
+                    {
+                      "name": "x",
+                      "type": "ordinal",
+                      "range": "width",
+                      "domain": {"data": "table", "field": "data.x"}
+                    },
+                    {
+                      "name": "y",
+                      "range": "height",
+                      "nice": true,
+                      "domain": {"data": "table", "field": "data.y"}
+                    }
+                  ],
+                  "axes": [
+                    {
+                      "type": "x",
+                      "scale": "x",
+                      "title": "#{ ca[:x_axis_label] }",
+                      "format": #{ ca[:x_axis_tick_format] },
+                    },
+                    {
+                      "type": "y",
+                      "scale": "y",
+                      "title": "#{ ca[:y_axis_label] }",
+                      "titleOffset": 60,
+                      "format": #{ ca[:y_axis_tick_format] },
+                    }
+                  ],
+                  "marks": [
+                    {
+                      "type": "rect",
+                      "from": {"data": "table"},
+                      "properties": {
+                        "enter": {
+                          "x": {"scale": "x", "field": "data.x"},
+                          "width": {"scale": "x", "band": true, "offset": -1},
+                          "y": {"scale": "y", "field": "data.y"},
+                          "y2": {"scale": "y", "value": 0}
+                        },
+                        "update": {
+                          "fill": {"value": "#1F77B4"}
+                        },
+                      }
+                    }
+                  ]
+                };
+
+                vg.parse.spec(spec, function(chart) {
+                  var view = chart({ el:"##{ dom_id }" }).update();
+                });
+
+              })();
+            </script>
+          </div>
+        )
+      end
+
+      def render_nvd3(ca)
+        %(
+          <div class="rde-chart rde-histogram-categorical">
             <h3 class="rde-chart-title">Histogram</h3>
             <div id="#{ dom_id }", style="height: 200px;">
               <svg></svg>
@@ -52,6 +141,12 @@ class RailsDataExplorer
                     .tickFormat(#{ ca[:y_axis_tick_format] })
                     ;
 
+                  chart.tooltipContent(
+                    function(key, x, y, e, graph) {
+                      return '<p>' + x + '</p>' + '<p>' +  y + '</p>'
+                    }
+                  );
+
                   d3.select('##{ dom_id } svg')
                     .datum(data)
                     .transition().duration(100)
@@ -68,6 +163,12 @@ class RailsDataExplorer
         )
       end
 
+      # Render HistogramCategorical only if there is a fairly small number of
+      # distinct values.
+      def render?
+        !@data_set.data_series.first.has_many_uniq_vals?
+      end
+
     end
   end
 end