rails-data-explorer 0.2.3 → 1.0.0

data/lib/rails_data_explorer/chart/anova.rb

@@ -0,0 +1 @@
+# http://en.wikipedia.org/wiki/Analysis_of_variance

data/lib/{rails-data-explorer → rails_data_explorer}/chart/box_plot.rb

@@ -1,7 +1,16 @@
-# http://johan.github.io/d3/ex/box.html
-# http://bl.ocks.org/mbostock/4061502
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a box plot for univariate analysis of a quantitative data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    # http://johan.github.io/d3/ex/box.html
+    # http://bl.ocks.org/mbostock/4061502
     class BoxPlot < Chart
 
       def initialize(_data_set, options = {})
@@ -18,7 +27,7 @@ class RailsDataExplorer
           min: x_ds.min_val,
           max: x_ds.max_val,
           base_width: 120,
-          base_height: 800,
+          base_height: 960,
           axis_tick_format: x_ds.axis_tick_format,
         }
       end

data/lib/{rails-data-explorer → rails_data_explorer}/chart/box_plot_group.rb

@@ -1,16 +1,26 @@
-# http://bl.ocks.org/jensgrubert/7789216
-# http://www.datavizcatalogue.com/methods/box_plot.html#.U0S8Ra1dUyE
-# http://mbostock.github.io/protovis/ex/box-and-whisker.html
-# http://bl.ocks.org/mbostock/4061502
-# http://johan.github.io/d3/ex/box.html
-# http://johan.github.io/d3/ex/box.html
-# http://bl.ocks.org/mbostock/4061502
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
-    class BoxPlotGroup < Chart
 
-# TODO: imitate this:
-# http://www.stata.com/support/faqs/graphics/gph/graphdocs/horizontal-box-plot-of-variable-by-values-of-categorical-variable/
+    # Responsibilities:
+    #  * Render a group of box plots for bivariate analysis of a categorical and
+    #    a numerical data series. One box plot is rendered for each distinct
+    #    categorical value.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    # http://bl.ocks.org/jensgrubert/7789216
+    # http://www.datavizcatalogue.com/methods/box_plot.html#.U0S8Ra1dUyE
+    # http://mbostock.github.io/protovis/ex/box-and-whisker.html
+    # http://bl.ocks.org/mbostock/4061502
+    # http://johan.github.io/d3/ex/box.html
+    # http://johan.github.io/d3/ex/box.html
+    # http://bl.ocks.org/mbostock/4061502
+    # TODO: imitate this:
+    # http://www.stata.com/support/faqs/graphics/gph/graphdocs/horizontal-box-plot-of-variable-by-values-of-categorical-variable/
+    class BoxPlotGroup < Chart
 
       def initialize(_data_set, options = {})
         @data_set = _data_set
@@ -24,19 +34,16 @@ class RailsDataExplorer
         y_candidates = @data_set.data_series.find_all { |ds|
           (ds.chart_roles[Chart::BoxPlotGroup] & [:y, :any]).any?
         }
-
         x_ds = x_candidates.first
         y_ds = (y_candidates - [x_ds]).first
         return false  if x_ds.nil? || y_ds.nil?
 
-        min = x_ds.min_val # get global min
-        max = x_ds.max_val # get global max
-
+        # initialize values_hash
         values_hash = y_ds.uniq_vals.inject({}) { |m,y_val|
           m[y_val] = []
           m
         }
-
+        # populate values hash
         y_ds.values.each_with_index { |y_val, idx|
           next  if (y_val.nil? || Float::NAN == y_val)
           values_hash[y_val] << x_ds.values[idx]
@@ -49,16 +56,36 @@ class RailsDataExplorer
         )
         sorted_values = y_sorted_keys.map { |y_val| values_hash[y_val] }
 
+        # Compute min and max values based on interquartile range of each
+        # boxplot. Objective is to normalize boxplots so that the widest chart
+        # uses almost the entire space available.
+        # Iterate over all individual boxplots
+        global_min = Float::INFINITY
+        global_max = -Float::INFINITY
+        sorted_values.each { |x_vals|
+          ds = DataSeries.new('_', x_vals)
+          desc_stats = ds.descriptive_statistics
+          # compute first and third quartile. Use min and max if they are nil
+          # for very small data series with only one or two entries.
+          q1 = desc_stats.detect { |e| '25%ile' == e[:label] }[:value] || x_vals.min
+          q3 = desc_stats.detect { |e| '75%ile' == e[:label] }[:value] || x_vals.max
+          iqr = (q3 - q1) * 1.5
+          local_min = [x_vals.min, q1 - iqr].max
+          global_min = [global_min, local_min].min
+          local_max = [x_vals.max, q3 + iqr].min
+          global_max = [global_max, local_max].max
+        }
+
         {
           values: sorted_values,
           category_labels: y_sorted_keys,
-          min: min,
-          max: max,
-          base_width: 120,
-          base_height: 800,
+          min: global_min,
+          max: global_max,
+          base_width: 100,
+          base_height: 960,
           axis_tick_format: x_ds.axis_tick_format,
           num_box_plots: y_ds.uniq_vals_count,
-          axis_scale: DataSeries.new('_', [min, max]).axis_scale(:d3)
+          axis_scale: DataSeries.new('_', [global_min, global_max]).axis_scale(:d3)
         }
       end
 
@@ -85,7 +112,7 @@ class RailsDataExplorer
               (function() {
                 var base_width = #{ ca[:base_width] },
                     base_height = #{ ca[:base_height] },
-                    margin = { top: 10, right: 50, bottom: 95, left: 50 },
+                    margin = { top: 10, right: 40, bottom: 10, left: 40 },
                     width = base_width - margin.left - margin.right,
                     height = base_height - margin.top - margin.bottom;
 
@@ -106,7 +133,7 @@ class RailsDataExplorer
                 var svg = d3.select("##{ dom_id }").selectAll("svg")
                             .data(data)
                           .append("g")
-                            .attr("transform", "rotate(90) translate(" + (width + margin.left) + " -" + (height + margin.bottom) + ")")
+                            .attr("transform", "rotate(90) translate(" + margin.left + " -" + (height + margin.bottom) + ")")
                             .call(chart);
 
                 // Function to compute the interquartile range.

data/lib/{rails-data-explorer → rails_data_explorer}/chart/contingency_table.rb

@@ -1,14 +1,25 @@
-# See this project for code to compute chi_square and contingency_coefficient
-# https://github.com/bioruby/bioruby/blob/master/lib/bio/util/contingency_table.rb
-#
-# Resources for Chi Squared Test
-# * http://www.quora.com/What-is-the-most-intuitive-explanation-for-the-chi-square-test
-# * http://people.revoledu.com/kardi/tutorial/Questionnaire/Chi-Square%20IndependentTest.html
-# * http://stattrek.com/chi-square-test/independence.aspx?Tutorial=AP
+# -*- coding: utf-8 -*-
 
-# Contingency table and chi squared test is a good tool for interpreting A/B tests.
 class RailsDataExplorer
   class Chart
+
+    # Contingency table and chi squared test are great tools for interpreting
+    # A/B tests.
+    #
+    # Responsibilities:
+    #  * Render a contingency table for bivariate analysis of two categorical
+    #    data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    # See this project for code to compute chi_square and contingency_coefficient
+    # https://github.com/bioruby/bioruby/blob/master/lib/bio/util/contingency_table.rb
+    #
+    # Resources for Chi Squared Test
+    # * http://www.quora.com/What-is-the-most-intuitive-explanation-for-the-chi-square-test
+    # * http://people.revoledu.com/kardi/tutorial/Questionnaire/Chi-Square%20IndependentTest.html
+    # * http://stattrek.com/chi-square-test/independence.aspx?Tutorial=AP
     class ContingencyTable < Chart
 
       def initialize(_data_set, options = {})

data/lib/{rails-data-explorer → rails_data_explorer}/chart/descriptive_statistics_table.rb

@@ -1,5 +1,14 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a table with descriptive statistics for a data series of any type.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class DescriptiveStatisticsTable < Chart
 
       def initialize(_data_set, options = {})

data/lib/rails_data_explorer/chart/descriptive_statistics_table_group.rb

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

data/lib/{rails-data-explorer → rails_data_explorer}/chart/histogram_categorical.rb

@@ -1,5 +1,14 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a histogram for univariate analysis of a categorical data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class HistogramCategorical < Chart
 
       def initialize(_data_set, options = {})
@@ -12,7 +21,8 @@ class RailsDataExplorer
         return false  if x_ds.nil?
 
         # compute histogram
-        h = x_ds.values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
+        val_mod = { name: :limit_distinct_values }
+        h = x_ds.values(val_mod).inject(Hash.new(0)) { |m,e| m[e] += 1; m }
         histogram_values_ds = DataSeries.new('_', h.values)
         y_scale_type = histogram_values_ds.axis_scale(:vega)
         bar_y2_val = 'log' == y_scale_type ? histogram_values_ds.min_val / 10.0 : 0
@@ -50,7 +60,7 @@ class RailsDataExplorer
             <script type="text/javascript">
               (function() {
                 var spec = {
-                  "width": 800,
+                  "width": 960,
                   "height": 200,
                   "padding": {"top": 10, "left": 70, "bottom": 50, "right": 10},
                   "data": [
@@ -169,12 +179,6 @@ class RailsDataExplorer
         )
       end
 
-      # Render HistogramCategorical only if there is a fairly small number of
-      # distinct values.
-      def render?
-        !@data_set.data_series.first.has_many_uniq_vals?
-      end
-
     end
   end
 end

data/lib/{rails-data-explorer → rails_data_explorer}/chart/histogram_quantitative.rb

@@ -1,5 +1,14 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a histogram for univariate analysis of a quantitative data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class HistogramQuantitative < Chart
 
       def initialize(_data_set, options = {})
@@ -15,7 +24,7 @@ class RailsDataExplorer
         quantizer = Utils::DataQuantizer.new(x_ds, max_number_of_bins: 100)
         quantized_values = quantizer.values
         number_of_bars = quantizer.number_of_bins
-        width = 800
+        width = 960
         h = quantized_values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
         histogram_values_ds = DataSeries.new('_', h.values)
         y_scale_type = histogram_values_ds.axis_scale(:vega)
@@ -33,6 +42,7 @@ class RailsDataExplorer
           y_scale_type: y_scale_type,
           y_scale_domain: [bar_y2_val, histogram_values_ds.max_val],
           bar_y2_val: bar_y2_val,
+          css_class: 'rde-histogram-quantitative',
         }
       end
 
@@ -45,7 +55,7 @@ class RailsDataExplorer
 
       def render_vega(ca)
         %(
-          <div class="rde-chart rde-histogram-quantitative">
+          <div class="rde-chart #{ ca[:css_class] }">
             <h3 class="rde-chart-title">Histogram</h3>
             <div id="#{ dom_id }"></div>
             <script type="text/javascript">

data/lib/{rails-data-explorer → rails_data_explorer}/chart/histogram_temporal.rb

@@ -1,6 +1,14 @@
-# TODO: could I use histogram_quantitative instead and just tweak the tick mark format?
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a histogram for univariate analysis of a temporal data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class HistogramTemporal < HistogramQuantitative
 
       def compute_chart_attrs
@@ -17,7 +25,7 @@ class RailsDataExplorer
         histogram_values_ds = DataSeries.new('_', h.values)
         y_scale_type = histogram_values_ds.axis_scale(:vega)
         bar_y2_val = 'log' == y_scale_type ? histogram_values_ds.min_val / 10.0 : 0
-        width = 800
+        width = 960
         {
           values: h.map { |k,v| { x: k, y: v } },
           width: width,
@@ -31,6 +39,7 @@ class RailsDataExplorer
           y_scale_type: y_scale_type,
           y_scale_domain: [bar_y2_val, histogram_values_ds.max_val],
           bar_y2_val: bar_y2_val,
+          css_class: 'rde-histogram-temporal',
         }
       end
 

data/lib/{rails-data-explorer → rails_data_explorer}/chart/multi_dimensional_charts.rb

@@ -1 +1,3 @@
+# -*- coding: utf-8 -*-
+
 # http://dc-js.github.io/dc.js/

data/lib/{rails-data-explorer → rails_data_explorer}/chart/parallel_coordinates.rb

@@ -1,6 +1,16 @@
-# TODO: add :color chart_role (test first if it makes sense, e.g., for 'pay')
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a parallel coordinates chart for multivariate analysis of
+    #    a mix of quantitative, temporal, and categorical data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    # TODO: add :color chart_role (test first if it makes sense, e.g., for 'pay')
     class ParallelCoordinates < Chart
 
       def initialize(_data_set, options = {})

data/lib/{rails-data-explorer → rails_data_explorer}/chart/parallel_set.rb

@@ -1,7 +1,16 @@
-# http://www.jasondavies.com/parallel-sets/
-# Suitable when all data series are categorical
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a parallel set chart for multivariate analysis of categorical
+    #    data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    # http://www.jasondavies.com/parallel-sets/
     class ParallelSet < Chart
 
       def initialize(_data_set, options = {})

data/lib/{rails-data-explorer → rails_data_explorer}/chart/pie_chart.rb

@@ -1,5 +1,14 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a pie chart for univariate analysis of a categorical data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class PieChart < Chart
 
       def initialize(_data_set, options = {})
@@ -11,9 +20,10 @@ class RailsDataExplorer
         x_ds = @data_set.data_series.first
         return false  if x_ds.nil?
 
-        total_count = x_ds.values.length
+        val_mod = { name: :limit_distinct_values }
+        total_count = x_ds.values(val_mod).length
         # compute histogram
-        h = x_ds.values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
+        h = x_ds.values(val_mod).inject(Hash.new(0)) { |m,e| m[e] += 1; m }
         {
           values: h.map { |k,v|
             { key: k, value: (v / total_count.to_f) }
@@ -137,12 +147,6 @@ class RailsDataExplorer
         )
       end
 
-      # Render PieChart only if there is a fairly small number of
-      # distinct values.
-      def render?
-        !@data_set.data_series.first.has_many_uniq_vals?
-      end
-
     end
   end
 end

data/lib/{rails-data-explorer → rails_data_explorer}/chart/scatterplot.rb

@@ -1,5 +1,17 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
+
+    # Responsibilities:
+    #  * Render a scatter plot for either
+    #      * bivariate analysis of two quantitative data series or
+    #      * multivariate analysis of two quantitative and one categorical
+    #        data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
     class Scatterplot < Chart
 
       def initialize(_data_set, options = {})
@@ -81,7 +93,7 @@ class RailsDataExplorer
             <script type="text/javascript">
               (function() {
                 var spec = {
-                  "width": 800,
+                  "width": 960,
                   "height": 200,
                   "data": [
                     {

data/lib/{rails-data-explorer → rails_data_explorer}/chart/scatterplot_matrix.rb

@@ -1 +1,3 @@
+# -*- coding: utf-8 -*-
+
 # http://benjiec.github.io/scatter-matrix/demo/demo.html#

data/lib/{rails-data-explorer/chart/stacked_bar_chart_categorical_percent.rb → rails_data_explorer/chart/stacked_bar_chart_categorical.rb}

@@ -1,6 +1,16 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
   class Chart
-    class StackedBarChartCategoricalPercent < Chart
+
+    # Responsibilities:
+    #  * Render a stacked bar chart for bivariate analysis of two categorical
+    #    data series. Renders absolute frequencies of y-data series.
+    #
+    # Collaborators:
+    #  * DataSet
+    #
+    class StackedBarChartCategorical < Chart
 
       def initialize(_data_set, options = {})
         @data_set = _data_set
@@ -8,9 +18,11 @@ class RailsDataExplorer
       end
 
       def compute_chart_attrs
+        val_mod = { name: :limit_distinct_values }
+
         x_candidates = @data_set.data_series.find_all { |ds|
           (ds.chart_roles[Chart::StackedBarChartCategoricalPercent] & [:x, :any]).any?
-        }.sort { |a,b| b.uniq_vals.length <=> a.uniq_vals.length }
+        }.sort { |a,b| b.uniq_vals_count(val_mod) <=> a.uniq_vals_count(val_mod) }
         y_candidates = @data_set.data_series.find_all { |ds|
           (ds.chart_roles[Chart::StackedBarChartCategoricalPercent] & [:y, :any]).any?
         }
@@ -21,31 +33,30 @@ class RailsDataExplorer
 
         # initialize data_matrix
         data_matrix = { _sum: { _sum: 0 } }
-        x_ds.uniq_vals.each { |x_val|
+        x_ds.uniq_vals(val_mod).each { |x_val|
           data_matrix[x_val] = {}
           data_matrix[x_val][:_sum] = 0
-          y_ds.uniq_vals.each { |y_val|
+          y_ds.uniq_vals(val_mod).each { |y_val|
             data_matrix[x_val][y_val] = 0
             data_matrix[:_sum][y_val] = 0
           }
         }
         # populate data_matrix
-        x_ds.values.length.times { |idx|
-          x_val = x_ds.values[idx]
-          y_val = y_ds.values[idx]
+        x_ds.values(val_mod).length.times { |idx|
+          x_val = x_ds.values(val_mod)[idx]
+          y_val = y_ds.values(val_mod)[idx]
           data_matrix[x_val][y_val] += 1
           data_matrix[:_sum][y_val] += 1
           data_matrix[x_val][:_sum] += 1
           data_matrix[:_sum][:_sum] += 1
         }
-
-        x_sorted_keys = x_ds.uniq_vals.sort(
+        x_sorted_keys = x_ds.uniq_vals(val_mod).sort(
           &x_ds.label_sorter(
             nil,
             lambda { |a,b| data_matrix[b][:_sum] <=> data_matrix[a][:_sum] }
           )
         )
-        y_sorted_keys = y_ds.uniq_vals.sort(
+        y_sorted_keys = y_ds.uniq_vals(val_mod).sort(
           &y_ds.label_sorter(
             nil,
             lambda { |a,b| data_matrix[:_sum][b] <=> data_matrix[:_sum][a] }
@@ -58,7 +69,7 @@ class RailsDataExplorer
             x_sorted_keys.map { |x_val|
               {
                 x: x_val,
-                y: (data_matrix[x_val][y_val] / data_matrix[x_val][:_sum].to_f) * 100,
+                y: compute_y_value(data_matrix, x_val, y_val),
                 c: y_val
               }
             }
@@ -70,11 +81,22 @@ class RailsDataExplorer
           values: values,
           x_axis_label: x_ds.name,
           x_axis_tick_format: 'function(d) { return d }',
-          y_axis_label: "#{ y_ds.name } distribution [%]",
+          y_axis_label: compute_y_axis_label(y_ds.name),
           y_axis_tick_format: "d3.format('.1%')",
         }
       end
 
+      # Override this method to change how the y value is computed. E.g., to
+      # change from absolute values to percentages.
+      def compute_y_value(data_matrix, x_val, y_val)
+        data_matrix[x_val][y_val]
+      end
+
+      # @param y_ds_name [String] name of the y data series
+      def compute_y_axis_label(y_ds_name)
+        "Frequency"
+      end
+
       def render
         return ''  unless render?
         ca = compute_chart_attrs
@@ -90,9 +112,9 @@ class RailsDataExplorer
             <script type="text/javascript">
               (function() {
                 var spec = {
-                  "width": 800,
+                  "width": 960,
                   "height": 200,
-                  "padding": {"top": 10, "left": 50, "bottom": 50, "right": 100},
+                  "padding": {"top": 10, "left": 70, "bottom": 50, "right": 100},
                   "data": [
                     {
                       "name": "table",
@@ -139,6 +161,7 @@ class RailsDataExplorer
                       "scale": "y",
                       "title": "#{ ca[:y_axis_label] }",
                       "format": #{ ca[:y_axis_tick_format] },
+                      "titleOffset": 60,
                     }
                   ],
                   "marks": [