rails-data-explorer 0.0.1

data/.gitignore

@@ -0,0 +1,10 @@
+*.gem
+.bundle
+.DS_Store
+.ruby-version
+.ruby-gemset
+pkg/*
+tmp
+
+Gemfile.lock
+gemfiles/*.lock

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* Initial release

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gemspec
+
+group :development, :test do
+  gem 'gem-release'
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Jo Hund
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+rails-data-explorer
+===================
+
+rails-data-explorer is a Rails Engine plugin that makes it easy to explore the
+data in your app using charts and statistics.
+
+Make sure to go to the thorough [documentation](http://rails-data-explorer.clearcove.ca)
+to find out more!
+
+
+### Installation
+
+`gem install rails-data-explorer`
+
+or with bundler in your Gemfile:
+
+`gem 'rails-data-explorer'`
+
+
+### Concepts
+
+* Exploration - top level container
+* DataSet - like a spreadsheet with one or more columns of data
+* DataSeries - like a column in a spreadsheet, with multiple rows of data
+* DataType - Each DataSeries contains data of a certain type.
+    * Categorical
+    * Quantitative
+        * Integer
+        * Decimal
+        * Temporal
+    * Geo
+* Chart -
+
+
+### Resources
+
+* [Documentation](http://rails-data-explorer.clearcove.ca)
+* [Live demo](http://rails-data-explorer-demo.herokuapp.com)
+* [Changelog](https://github.com/jhund/rails-data-explorer/blob/master/CHANGELOG.md)
+* [Source code (github)](https://github.com/jhund/rails-data-explorer)
+* [Issues](https://github.com/jhund/rails-data-explorer/issues)
+* [Rubygems.org](http://rubygems.org/gems/rails-data-explorer)
+
+### License
+
+[MIT licensed](https://github.com/jhund/rails-data-explorer/blob/master/MIT-LICENSE).
+
+
+
+### Copyright
+
+Copyright (c) 2014 Jo Hund. See [(MIT) LICENSE](https://github.com/jhund/rails-data-explorer/blob/master/MIT-LICENSE) for details.

data/Rakefile

@@ -0,0 +1,18 @@
+# encoding: utf-8
+begin
+  require 'bundler'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs.push "lib"
+  t.libs.push "spec"
+  t.pattern = "spec/**/*_spec.rb"
+  t.verbose = true
+end
+
+task :default => ['test']

data/lib/rails-data-explorer.rb

@@ -0,0 +1,44 @@
+require 'color'
+require 'descriptive-statistics'
+require 'distribution'
+require 'interpolate'
+
+require 'rails-data-explorer/engine'
+
+require 'rails-data-explorer/chart'
+require 'rails-data-explorer/data_series'
+require 'rails-data-explorer/data_set'
+require 'rails-data-explorer/data_type'
+require 'rails-data-explorer/exploration'
+require 'rails-data-explorer/statistics/rng_category'
+require 'rails-data-explorer/statistics/rng_gaussian'
+require 'rails-data-explorer/statistics/rng_power_law'
+require 'rails-data-explorer/utils/color_scale'
+require 'rails-data-explorer/utils/value_formatter'
+
+require 'rails-data-explorer/chart/box_plot'
+require 'rails-data-explorer/chart/box_plot_group'
+require 'rails-data-explorer/chart/contingency_table'
+require 'rails-data-explorer/chart/descriptive_statistics_table'
+require 'rails-data-explorer/chart/histogram_categorical'
+require 'rails-data-explorer/chart/histogram_quantitative'
+require 'rails-data-explorer/chart/histogram_temporal'
+require 'rails-data-explorer/chart/parallel_coordinates'
+require 'rails-data-explorer/chart/parallel_set'
+require 'rails-data-explorer/chart/pie_chart'
+require 'rails-data-explorer/chart/scatterplot'
+require 'rails-data-explorer/chart/stacked_bar_chart_categorical_percent'
+require 'rails-data-explorer/data_type/categorical'
+require 'rails-data-explorer/data_type/quantitative'
+require 'rails-data-explorer/data_type/quantitative/decimal'
+require 'rails-data-explorer/data_type/quantitative/integer'
+require 'rails-data-explorer/data_type/quantitative/temporal'
+
+class RailsDataExplorer
+
+  # Convenience method to instantiate new Exploration
+  def self.new(*args)
+    RailsDataExplorer::Exploration.new(*args)
+  end
+
+end

data/lib/rails-data-explorer/action_view_extension.rb

@@ -0,0 +1,12 @@
+#
+# Adds view helpers to ActionView
+#
+class RailsDataExplorer
+  module ActionViewExtension
+
+    def render_explorations(explorations)
+      explorations.map { |e| e.render }
+    end
+
+  end
+end

data/lib/rails-data-explorer/active_record_extension.rb

@@ -0,0 +1,14 @@
+#
+# Adds rails-data-explorer methods to ActiveRecord::Base and sub classes.
+#
+class RailsDataExplorer
+  module ActiveRecordExtension
+
+    module ClassMethods
+
+    protected
+
+    end
+
+  end
+end

data/lib/rails-data-explorer/chart.rb

@@ -0,0 +1,52 @@
+class RailsDataExplorer
+  class Chart
+
+    include ActionView::Helpers::NumberHelper
+    attr_accessor :output_buffer # required for content_tag
+    include ActionView::Helpers::TagHelper
+
+    def dom_id
+      "rde-chart-#{ object_id }"
+    end
+
+    # Returns true if this chart will be rendered. Sometimes we can't make that
+    # decision until render time. Override this method in sub classes, e.g.,
+    # to avoid rendering ParallelCoordinates when all data series are categorical.
+    def render?
+      true
+    end
+
+  protected
+
+    # Renders an HTML table
+    # @param[OpenStruct, Struct] table_struct
+    def render_html_table(table_struct)
+      content_tag(:table, :class => 'table rde-table') do
+        table_struct.rows.map { |row|
+          content_tag(row.tag, :class => row.css_class) do
+            row.cells.map { |cell|
+              if cell.ruby_formatter
+                content_tag(
+                  cell.tag,
+                  instance_exec(cell.value, &cell.ruby_formatter),
+                  :class => cell.css_class,
+                  :title => cell.title,
+                  :style => cell.style,
+                )
+              else
+                content_tag(
+                  cell.tag,
+                  cell.value,
+                  :class => cell.css_class,
+                  :title => cell.title,
+                  :style => cell.style,
+                )
+              end
+            }.join.html_safe
+          end
+        }.join.html_safe
+      end
+    end
+
+  end
+end

data/lib/rails-data-explorer/chart/box_plot.rb

@@ -0,0 +1,79 @@
+# http://johan.github.io/d3/ex/box.html
+# http://bl.ocks.org/mbostock/4061502
+class RailsDataExplorer
+  class Chart
+    class BoxPlot < Chart
+
+      def initialize(_data_set, options = {})
+        @data_set = _data_set
+        @options = {}.merge(options)
+      end
+
+      def compute_chart_attrs
+        x_ds = @data_set.data_series.first
+        {
+          values: [x_ds.values],
+          min: x_ds.min_val,
+          max: x_ds.max_val,
+          base_width: 120,
+          base_height: 1334,
+          axis_tick_format: x_ds.axis_tick_format,
+        }
+      end
+
+      def render
+        return ''  unless render?
+        ca = compute_chart_attrs
+        %(
+          <div id="#{ dom_id }" class="rde-chart rde-box-plot">
+            <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
+
+            <script type="text/javascript">
+              (function() {
+                var base_width = #{ ca[:base_width] },
+                    base_height = #{ ca[:base_height] },
+                    margin = { top: 10, right: 50, bottom: 95, left: 50 },
+                    width = base_width - margin.left - margin.right,
+                    height = base_height - margin.top - margin.bottom;
+
+                var min = #{ ca[:min] },
+                    max = #{ ca[:max] };
+
+                var chart = d3.box()
+                              .whiskers(iqr(1.5))
+                              .width(width)
+                              .height(height)
+                              .tickFormat(#{ ca[:axis_tick_format] });
+
+                var data = #{ ca[:values].to_json };
+
+                chart.domain([min, max]);
+
+                var svg = d3.select("##{ dom_id }").selectAll("svg")
+                            .data(data)
+                          .append("g")
+                            .attr("transform", "rotate(90) translate(" + (width + margin.left) + " -" + (height + margin.bottom) + ")")
+                            .call(chart);
+
+                // Function to compute the interquartile range.
+                function iqr(k) {
+                  return function(d, i) {
+                    var q1 = d.quartiles[0],
+                        q3 = d.quartiles[2],
+                        iqr = (q3 - q1) * k,
+                        i = -1,
+                        j = d.length;
+                    while (d[++i] < q1 - iqr);
+                    while (d[--j] > q3 + iqr);
+                    return [i, j];
+                  };
+                }
+              })();
+            </script>
+          </div>
+        )
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/chart/box_plot_group.rb

@@ -0,0 +1,109 @@
+# http://bl.ocks.org/jensgrubert/7789216
+# http://www.datavizcatalogue.com/methods/box_plot.html#.U0S8Ra1dUyE
+# http://mbostock.github.io/protovis/ex/box-and-whisker.html
+# http://bl.ocks.org/mbostock/4061502
+# http://johan.github.io/d3/ex/box.html
+# http://johan.github.io/d3/ex/box.html
+# http://bl.ocks.org/mbostock/4061502
+class RailsDataExplorer
+  class Chart
+    class BoxPlotGroup < Chart
+
+      def initialize(_data_set, options = {})
+        @data_set = _data_set
+        @options = {}.merge(options)
+      end
+
+      def compute_chart_attrs
+        x_candidates = @data_set.data_series.find_all { |ds|
+          (ds.chart_roles[Chart::BoxPlotGroup] & [:x, :any]).any?
+        }
+        y_candidates = @data_set.data_series.find_all { |ds|
+          (ds.chart_roles[Chart::BoxPlotGroup] & [:y, :any]).any?
+        }
+
+        x_ds = x_candidates.first
+        y_ds = (y_candidates - [x_ds]).first
+
+        return false  if x_ds.nil? || y_ds.nil?
+
+        min = x_ds.min_val # get global min
+        max = x_ds.max_val # get global max
+
+        values_hash = y_ds.uniq_vals.inject({}) { |m,y_val|
+          m[y_val] = []
+          m
+        }
+
+        y_ds.values.each_with_index { |y_val, idx|
+          values_hash[y_val] << x_ds.values[idx]
+        }
+
+        {
+          values: values_hash.values,
+          min: min,
+          max: max,
+          base_width: 120,
+          base_height: 1334,
+          axis_tick_format: x_ds.axis_tick_format,
+        }
+      end
+
+      def render
+        return ''  unless render?
+        ca = compute_chart_attrs
+        return ''  unless ca
+        %(
+          <div id="#{ dom_id }" class="rde-chart rde-box-plot">
+            <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
+            <svg class="box" style="height: #{ ca[:base_width] }px;"></svg>
+
+            <script type="text/javascript">
+              (function() {
+                var base_width = #{ ca[:base_width] },
+                    base_height = #{ ca[:base_height] },
+                    margin = { top: 10, right: 50, bottom: 95, left: 50 },
+                    width = base_width - margin.left - margin.right,
+                    height = base_height - margin.top - margin.bottom;
+
+                var min = #{ ca[:min] },
+                    max = #{ ca[:max] };
+
+                var chart = d3.box()
+                              .whiskers(iqr(1.5))
+                              .width(width)
+                              .height(height)
+                              .tickFormat(#{ ca[:axis_tick_format] });
+
+                var data = #{ ca[:values].to_json };
+
+                chart.domain([min, max]);
+
+                var svg = d3.select("##{ dom_id }").selectAll("svg")
+                            .data(data)
+                          .append("g")
+                            .attr("transform", "rotate(90) translate(" + (width + margin.left) + " -" + (height + margin.bottom) + ")")
+                            .call(chart);
+
+                // Function to compute the interquartile range.
+                function iqr(k) {
+                  return function(d, i) {
+                    var q1 = d.quartiles[0],
+                        q3 = d.quartiles[2],
+                        iqr = (q3 - q1) * k,
+                        i = -1,
+                        j = d.length;
+                    while (d[++i] < q1 - iqr);
+                    while (d[--j] > q3 + iqr);
+                    return [i, j];
+                  };
+                }
+              })();
+            </script>
+          </div>
+        )
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/chart/contingency_table.rb

@@ -0,0 +1,189 @@
+# See this project for code to compute chi_square and contingency_coefficient
+# https://github.com/bioruby/bioruby/blob/master/lib/bio/util/contingency_table.rb
+#
+# Resources for Chi Squared Test
+# * http://www.quora.com/What-is-the-most-intuitive-explanation-for-the-chi-square-test
+# * http://people.revoledu.com/kardi/tutorial/Questionnaire/Chi-Square%20IndependentTest.html
+# * http://stattrek.com/chi-square-test/independence.aspx?Tutorial=AP
+class RailsDataExplorer
+  class Chart
+    class ContingencyTable < Chart
+
+      def initialize(_data_set, options = {})
+        @data_set = _data_set
+        @options = {}.merge(options)
+      end
+
+      def compute_chart_attrs
+        x_candidates = @data_set.data_series.find_all { |ds|
+          (ds.chart_roles[Chart::ContingencyTable] & [:x, :any]).any?
+        }
+        y_candidates = @data_set.data_series.find_all { |ds|
+          (ds.chart_roles[Chart::ContingencyTable] & [:y, :any]).any?
+        }
+
+        x_ds = x_candidates.first
+        y_ds = (y_candidates - [x_ds]).first
+
+        # Compute @observed_vals, @expected_vals, etc.
+        compute_contingency_and_chi_squared!(x_ds, y_ds)
+
+        x_sorted_keys = x_ds.uniq_vals.sort { |a,b|
+          @observed_vals[b][:_sum] <=> @observed_vals[a][:_sum]
+        }
+        y_sorted_keys = y_ds.uniq_vals.sort { |a,b|
+          @observed_vals[:_sum][b] <=> @observed_vals[:_sum][a]
+        }
+
+        ca = case @data_set.dimensions_count
+        when 2
+          # Table
+          OpenStruct.new(
+            # Top header row
+            :rows => [
+              OpenStruct.new(
+                :css_class => 'rde-column_header',
+                :tag => :tr,
+                :cells => [
+                  OpenStruct.new(:tag => :th, :value => '')
+                ] +
+                x_sorted_keys.map { |x_val|
+                  OpenStruct.new(:tag => :th, :value => x_val)
+                } +
+                [OpenStruct.new(:tag => :th, :value => 'Totals')]
+              )
+            ] +
+            # Data rows
+            y_sorted_keys.map { |y_val|
+              OpenStruct.new(
+                :css_class => 'rde-data_row',
+                :tag => :tr,
+                :cells => [
+                  OpenStruct.new(:tag => :th, :value => y_val, :css_class => 'rde-row_header')
+                ] +
+                x_sorted_keys.map { |x_val|
+                  OpenStruct.new(
+                    :tag => :td,
+                    :value => @observed_vals[x_val][y_val],
+                    :css_class => 'rde-numerical',
+                    :title => "Expected value: #{ number_with_precision(@expected_vals[x_val][y_val]) }",
+                    :style => "color: #{ @delta_attrs[x_val][y_val][:color] };",
+                  )
+                } +
+                [OpenStruct.new(:tag => :th, :value => @observed_vals[:_sum][y_val])]
+              )
+            } +
+            # Footer row
+            [
+              OpenStruct.new(
+                :css_class => 'rde-column_header',
+                :tag => :tr,
+                :cells => [
+                  OpenStruct.new(:tag => :th, :value => 'Totals', :css_class => 'rde-row_header')
+                ] +
+                x_sorted_keys.map { |x_val|
+                  OpenStruct.new(:tag => :th, :value => @observed_vals[x_val][:_sum])
+                } +
+                [OpenStruct.new(:tag => :th, :value => @observed_vals[:_sum][:_sum])]
+              )
+            ]
+          )
+        else
+          raise(ArgumentError.new("Exactly two data series required for contingency table."))
+        end
+        ca
+      end
+
+      def render
+        return ''  unless render?
+        ca = compute_chart_attrs
+        content_tag(:div, :class => 'rde-chart rde-contingency-table', :id => dom_id) do
+          content_tag(:h3, "Contingency Table", :class => 'rde-chart-title') +
+          render_html_table(ca)
+        end +
+        content_tag(:p, @conclusion)
+      end
+
+      def render?
+        # http://en.wikipedia.org/wiki/Pearson's_chi-squared_test#Assumptions
+        true
+      end
+
+    private
+
+      # Computes @observed_vals, @expected_vals, @chi_squared, etc.
+      # @param[DataSeries] x_ds
+      # @param[DataSeries] y_ds
+      def compute_contingency_and_chi_squared!(x_ds, y_ds)
+        # Compute the observed values table
+        @observed_vals = { :_sum => { :_sum => 0 } }
+        x_ds.uniq_vals.each { |x_val|
+          @observed_vals[x_val] = {}
+          @observed_vals[x_val][:_sum] = 0
+          y_ds.uniq_vals.each { |y_val|
+            @observed_vals[x_val][y_val] = 0
+            @observed_vals[:_sum][y_val] = 0
+          }
+        }
+        x_ds.values.length.times { |idx|
+          x_val = x_ds.values[idx]
+          y_val = y_ds.values[idx]
+          @observed_vals[x_val][y_val] += 1
+          @observed_vals[:_sum][y_val] += 1
+          @observed_vals[x_val][:_sum] += 1
+          @observed_vals[:_sum][:_sum] += 1
+        }
+        # Compute degrees of freedom
+        @degrees_of_freedom = (x_ds.uniq_vals_count - 1) * (y_ds.uniq_vals_count - 1)
+        # Compute the expected values table
+        @expected_vals = {}
+        x_ds.uniq_vals.each { |x_val|
+          @expected_vals[x_val] = {}
+          y_ds.uniq_vals.each { |y_val|
+            @expected_vals[x_val][y_val] = (
+              @observed_vals[:_sum][y_val] * @observed_vals[x_val][:_sum]
+            ) / (@observed_vals[:_sum][:_sum]).to_f
+          }
+        }
+        # Compute Chi squared
+        @chi_squared = 0
+        x_ds.uniq_vals.each { |x_val|
+          y_ds.uniq_vals.each { |y_val|
+            @chi_squared += (
+              (@observed_vals[x_val][y_val] - @expected_vals[x_val][y_val]) ** 2
+            ) / @expected_vals[x_val][y_val]
+          }
+        }
+        # Compute deltas
+        @delta_attrs = {}
+        color_scale = RailsDataExplorer::Utils::ColorScale.new
+        x_ds.uniq_vals.each { |x_val|
+          @delta_attrs[x_val] = {}
+          y_ds.uniq_vals.each { |y_val|
+            delta = @observed_vals[x_val][y_val] - @expected_vals[x_val][y_val]
+            delta_factor = delta / @expected_vals[x_val][y_val].to_f
+            @delta_attrs[x_val][y_val] = {
+              :expected => @expected_vals[x_val][y_val],
+              :color => color_scale.compute(delta_factor),
+              :delta => delta,
+              :delta_factor => delta_factor,
+            }
+          }
+        }
+        # Compute probability of observing a sample statistic as extreme as the
+        # observed test statistic.
+        @p_value = 1 - Distribution::ChiSquare.cdf(@chi_squared, @degrees_of_freedom)
+        # Set significance_level
+        @significance_level = 0.05
+        # Compute conclusion
+        @conclusion = %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> suggests that )
+        @conclusion << if @p_value <= @significance_level
+          "#{ x_ds.name } and #{ y_ds.name } are dependent variables (p_value: #{ number_with_precision(@p_value) })"
+        else
+          "#{ x_ds.name } and #{ y_ds.name } are independent variables (p_value: #{ number_with_precision(@p_value) })"
+        end
+        @conclusion = @conclusion.html_safe
+      end
+    end
+  end
+end