rails-data-explorer 0.0.1

data/lib/rails-data-explorer/statistics/pearsons_chi_squared_independence_test.rb

@@ -0,0 +1,75 @@
+=begin
+
+From http://en.wikipedia.org/wiki/Pearson's_chi-squared_test
+
+Pearson's chi-squared test is used to assess whether paired observations on two
+variables, expressed in a contingency table, are independent of each other.
+
+An "observation" consists of the values of two outcomes and the null hypothesis
+is that the occurrence of these outcomes is statistically independent. Each
+observation is allocated to one cell of a two-dimensional array of cells (called
+a contingency table) according to the values of the two outcomes.
+
+Assumptions
+-----------
+
+The chi-squared test, when used with the standard approximation that a chi-
+squared distribution is applicable, has the following assumptions:
+
+* Simple random sample – The sample data is a random sampling from a fixed
+  distribution or population where every collection of members of the population
+  of the given sample size has an equal probability of selection. Variants of
+  the test have been developed for complex samples, such as where the data is
+  weighted. Other forms can be used such as purposive sampling.
+* Sample size (whole table) – A sample with a sufficiently large size is assumed.
+  If a chi squared test is conducted on a sample with a smaller size, then the
+  chi squared test will yield an inaccurate inference. The researcher, by using
+  chi squared test on small samples, might end up committing a Type II error.
+* Expected cell count – Adequate expected cell counts. Some require 5 or more,
+  and others require 10 or more. A common rule is 5 or more in all cells of a
+  2-by-2 table, and 5 or more in 80% of cells in larger tables, but no cells
+  with zero expected count. When this assumption is not met, Yates's Correction
+  is applied.
+* Independence – The observations are always assumed to be independent of each
+  other. This means chi-squared cannot be used to test correlated data
+  (like matched pairs or panel data). In those cases you might want to turn to
+  McNemar's test.
+
+Problems
+--------
+
+The approximation to the chi-squared distribution breaks down if expected
+frequencies are too low. It will normally be acceptable so long as no more than
+20% of the events have expected frequencies below 5. Where there is only 1
+degree of freedom, the approximation is not reliable if expected frequencies are
+below 10. In this case, a better approximation can be obtained by reducing the
+absolute value of each difference between observed and expected frequencies by
+0.5 before squaring; this is called Yates's correction for continuity.
+
+In cases where the expected value, E, is found to be small (indicating a small
+underlying population probability, and/or a small number of observations), the
+normal approximation of the multinomial distribution can fail, and in such cases
+it is found to be more appropriate to use the G-test, a likelihood ratio-based
+test statistic. Where the total sample size is small, it is necessary to use an
+appropriate exact test, typically either the binomial test or (for contingency
+tables) Fisher's exact test. This test uses the conditional distribution of the
+test statistic given the marginal totals; however, it does not assume that the
+data were generated from an experiment in which the marginal totals are fixed
+and is valid whether or not that is the case.
+
+=end
+
+class RailsDataExplorer
+  module Statistics
+    class PearsonsChiSquaredIndependenceTest
+
+      #
+      def initialize(data_matrix, min_probability = 0.05)
+      end
+
+      def compute
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/statistics/rng_category.rb

@@ -0,0 +1,37 @@
+class RailsDataExplorer
+  module Statistics
+    class RngCategory
+
+      def initialize(categories, category_probabilities = nil, rng = lambda { Kernel.rand })
+        @categories, @category_probabilities, @rng = categories, category_probabilities, rng
+        @category_probabilities ||= @categories.map { |e| @rng.call }
+        @category_probabilities = normalize_category_probabilities
+        @category_order = compute_category_order
+      end
+
+      def rand
+        r_v = @rng.call
+        rnd = @category_order.detect { |e|
+          e[:threshold] >= r_v
+        }
+        rnd[:category]
+      end
+
+      def normalize_category_probabilities
+        total = @category_probabilities.inject(0) { |m,e| m += e }
+        @category_probabilities.map { |e| e / total.to_f }
+      end
+
+      def compute_category_order
+        category_order = []
+        running_sum = 0
+        @categories.each_with_index { |e, idx|
+          running_sum += @category_probabilities[idx]
+          category_order << { :category => e, :threshold => running_sum }
+        }
+        category_order
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/statistics/rng_gaussian.rb

@@ -0,0 +1,24 @@
+# From http://stackoverflow.com/a/9266488
+class RailsDataExplorer
+  module Statistics
+    class RngGaussian
+      def initialize(mean = 0.0, sd = 1.0, rng = lambda { Kernel.rand })
+        @mean, @sd, @rng = mean, sd, rng
+        @compute_next_pair = false
+      end
+
+      def rand
+        if (@compute_next_pair = !@compute_next_pair)
+          # Compute a pair of random values with normal distribution.
+          # See http://en.wikipedia.org/wiki/Box-Muller_transform
+          theta = 2 * Math::PI * @rng.call
+          scale = @sd * Math.sqrt(-2 * Math.log(1 - @rng.call))
+          @g1 = @mean + scale * Math.sin(theta)
+          @g0 = @mean + scale * Math.cos(theta)
+        else
+          @g1
+        end
+      end
+    end
+  end
+end

data/lib/rails-data-explorer/statistics/rng_power_law.rb

@@ -0,0 +1,21 @@
+class RailsDataExplorer
+  module Statistics
+    class RngPowerLaw
+
+      def initialize(min = 1, max = 1000, pow = 2, rng = lambda { Kernel.rand })
+        @min, @max, @pow, @rng = min, max, pow, rng
+        @max += 1
+      end
+
+      def rand
+        y = (
+          (
+            (@max ** (@pow + 1) - @min ** (@pow + 1)) * @rng.call + @min ** (@pow + 1)
+          ) ** (1.0 / (@pow + 1))
+        ).to_i
+        (@max - 1 - y) + @min
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/utils/color_scale.rb

@@ -0,0 +1,33 @@
+class RailsDataExplorer
+  module Utils
+    class ColorScale
+
+      def initialize
+        @points = {
+          -1 => Color::RGB::Red,
+          -0.1 =>  Color::RGB::Black,
+          0.1 =>  Color::RGB::Black,
+          1 => Color::RGB::Green,
+        }
+        @gradient = Interpolate::Points.new(@points)
+        @gradient.blend_with {|color, other, balance|
+          other.mix_with(color, balance * 100.0)
+        }
+      end
+
+      def compute(input)
+        @gradient.at(input).html
+      end
+
+      def demo
+        "<ul>".html_safe +
+        (-1).step(1, 0.1).map { |e|
+          color = compute(e)
+          %(<li style="color: #{ color }">#{ e } (#{ color })</li>)
+        }.join.html_safe +
+        "</ul>".html_safe
+      end
+
+    end
+  end
+end

data/lib/rails-data-explorer/utils/data_binner.rb

@@ -0,0 +1,8 @@
+# Convert quantitative data to categorical data.
+# http://saedsayad.com/binning.htm
+# E.g., ages in years to 5 groups:
+# * < 10
+# * 11 - 20
+# * 21 - 30
+# * 31 - 40
+# * > 40

data/lib/rails-data-explorer/utils/data_encoder.rb

@@ -0,0 +1,2 @@
+# Convert categorical data to quantitative data.
+# http://saedsayad.com/encoding.htm

data/lib/rails-data-explorer/utils/data_quantizer.rb

@@ -0,0 +1,2 @@
+# Map a large set of quantitative/temporal/geo input values to a (countable)
+# smaller set – such as rounding values to some unit of precision.

data/lib/rails-data-explorer/utils/value_formatter.rb

@@ -0,0 +1,41 @@
+# Add ValueFormat to DataSeries and to individual data points
+# Good resource on significant figures:
+# * http://www.edu.pe.ca/gray/class_pages/krcutcliffe/physics521/sigfigs/sigfigRULES.htm
+# * http://en.wikipedia.org/wiki/Significant_figures
+class RailsDataExplorer
+  module Utils
+    class ValueFormatter
+
+      attr_accessor :d3_format, :ruby_formatter, :significant_figures
+
+      # @param[Object] context
+      def initialize(context)
+        case context
+        when DataSeries
+          initialize_from_data_series(context)
+        when Hash
+          initialize_from_options(context)
+        when Numeric
+          initialize_from_single_value(context)
+        else
+          raise "Handle this context: #{ context.inspect }"
+        end
+      end
+
+    private
+
+      def initialize_from_data_series(data_series)
+      end
+
+      def initialize_from_options(options)
+        @d3_format = options[:d3_format]
+        @significant_figures = options[:significant_figures]
+        @ruby_formatter = options[:ruby_formatter]
+      end
+
+      def initialize_from_single_value(options)
+      end
+
+    end
+  end
+end

data/rails-data-explorer.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name = 'rails-data-explorer'
+  gem.version = '0.0.1'
+  gem.platform = Gem::Platform::RUBY
+
+  gem.authors = ['Jo Hund']
+  gem.email = 'jhund@clearcove.ca'
+  gem.homepage = 'http://rails-data-explorer.clearcove.ca'
+  gem.licenses = ['MIT']
+  gem.summary = 'A Rails engine plugin for exploring data in your app with charts and statistics.'
+  gem.description = %(rails-data-explorer is a Rails Engine plugin that makes it easy to explore the data in your app using charts and statistics.)
+
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  # really it's only ActiveSupport and ActionView
+  gem.add_dependency 'actionview', '>= 3.0.0'
+  gem.add_dependency 'color'
+  gem.add_dependency 'descriptive-statistics'
+  gem.add_dependency 'distribution'
+  gem.add_dependency 'interpolate'
+
+  gem.add_development_dependency 'bundler', ['>= 1.0.0']
+  gem.add_development_dependency 'rake', ['>= 0']
+  gem.add_development_dependency 'minitest'
+  gem.add_development_dependency 'minitest-spec-expect'
+end

data/vendor/assets/javascripts/d3.boxplot.js

@@ -0,0 +1,302 @@
+// From http://bl.ocks.org/mbostock/4061502
+(function() {
+
+// Inspired by http://informationandvisualization.de/blog/box-plot
+d3.box = function() {
+  var width = 1,
+      height = 1,
+      duration = 0,
+      domain = null,
+      value = Number,
+      whiskers = boxWhiskers,
+      quartiles = boxQuartiles,
+      tickFormat = null;
+
+  // For each small multiple…
+  function box(g) {
+    g.each(function(d, i) {
+      d = d.map(value).sort(d3.ascending);
+      var g = d3.select(this),
+          n = d.length,
+          min = d[0],
+          max = d[n - 1];
+
+      // Compute quartiles. Must return exactly 3 elements.
+      var quartileData = d.quartiles = quartiles(d);
+
+      // Compute whiskers. Must return exactly 2 elements, or null.
+      var whiskerIndices = whiskers && whiskers.call(this, d, i),
+          whiskerData = whiskerIndices && whiskerIndices.map(function(i) { return d[i]; });
+
+      // Compute outliers. If no whiskers are specified, all data are "outliers".
+      // We compute the outliers as indices, so that we can join across transitions!
+      var outlierIndices = whiskerIndices
+          ? d3.range(0, whiskerIndices[0]).concat(d3.range(whiskerIndices[1] + 1, n))
+          : d3.range(n);
+
+      // Compute the new x-scale.
+      var x1 = d3.scale.linear()
+          .domain(domain && domain.call(this, d, i) || [min, max])
+          .range([height, 0]);
+
+      // Retrieve the old x-scale, if this is an update.
+      var x0 = this.__chart__ || d3.scale.linear()
+          .domain([0, Infinity])
+          .range(x1.range());
+
+      // Stash the new scale.
+      this.__chart__ = x1;
+
+      // Note: the box, median, and box tick elements are fixed in number,
+      // so we only have to handle enter and update. In contrast, the outliers
+      // and other elements are variable, so we need to exit them! Variable
+      // elements also fade in and out.
+
+      // Update center line: the vertical line spanning the whiskers.
+      var center = g.selectAll("line.center")
+          .data(whiskerData ? [whiskerData] : []);
+
+      center.enter().insert("line", "rect")
+          .attr("class", "center")
+          .attr("x1", width / 2)
+          .attr("y1", function(d) { return x0(d[0]); })
+          .attr("x2", width / 2)
+          .attr("y2", function(d) { return x0(d[1]); })
+          .style("opacity", 1e-6)
+        .transition()
+          .duration(duration)
+          .style("opacity", 1)
+          .attr("y1", function(d) { return x1(d[0]); })
+          .attr("y2", function(d) { return x1(d[1]); });
+
+      center.transition()
+          .duration(duration)
+          .style("opacity", 1)
+          .attr("y1", function(d) { return x1(d[0]); })
+          .attr("y2", function(d) { return x1(d[1]); });
+
+      center.exit().transition()
+          .duration(duration)
+          .style("opacity", 1e-6)
+          .attr("y1", function(d) { return x1(d[0]); })
+          .attr("y2", function(d) { return x1(d[1]); })
+          .remove();
+
+      // Update innerquartile box.
+      var box = g.selectAll("rect.box")
+          .data([quartileData]);
+
+      box.enter().append("rect")
+          .attr("class", "box")
+          .attr("x", 0)
+          .attr("y", function(d) { return x0(d[2]); })
+          .attr("width", width)
+          .attr("height", function(d) { return x0(d[0]) - x0(d[2]); })
+        .transition()
+          .duration(duration)
+          .attr("y", function(d) { return x1(d[2]); })
+          .attr("height", function(d) { return x1(d[0]) - x1(d[2]); });
+
+      box.transition()
+          .duration(duration)
+          .attr("y", function(d) { return x1(d[2]); })
+          .attr("height", function(d) { return x1(d[0]) - x1(d[2]); });
+
+      // Update median line.
+      var medianLine = g.selectAll("line.median")
+          .data([quartileData[1]]);
+
+      medianLine.enter().append("line")
+          .attr("class", "median")
+          .attr("x1", 0)
+          .attr("y1", x0)
+          .attr("x2", width)
+          .attr("y2", x0)
+        .transition()
+          .duration(duration)
+          .attr("y1", x1)
+          .attr("y2", x1);
+
+      medianLine.transition()
+          .duration(duration)
+          .attr("y1", x1)
+          .attr("y2", x1);
+
+      // Update whiskers.
+      var whisker = g.selectAll("line.whisker")
+          .data(whiskerData || []);
+
+      whisker.enter().insert("line", "circle, text")
+          .attr("class", "whisker")
+          .attr("x1", 0)
+          .attr("y1", x0)
+          .attr("x2", width)
+          .attr("y2", x0)
+          .style("opacity", 1e-6)
+        .transition()
+          .duration(duration)
+          .attr("y1", x1)
+          .attr("y2", x1)
+          .style("opacity", 1);
+
+      whisker.transition()
+          .duration(duration)
+          .attr("y1", x1)
+          .attr("y2", x1)
+          .style("opacity", 1);
+
+      whisker.exit().transition()
+          .duration(duration)
+          .attr("y1", x1)
+          .attr("y2", x1)
+          .style("opacity", 1e-6)
+          .remove();
+
+      // Update outliers.
+      var outlier = g.selectAll("circle.outlier")
+          .data(outlierIndices, Number);
+
+      outlier.enter().insert("circle", "text")
+          .attr("class", "outlier")
+          .attr("r", 5)
+          .attr("cx", width / 2)
+          .attr("cy", function(i) { return x0(d[i]); })
+          .style("opacity", 1e-6)
+        .transition()
+          .duration(duration)
+          .attr("cy", function(i) { return x1(d[i]); })
+          .style("opacity", 1);
+
+      outlier.transition()
+          .duration(duration)
+          .attr("cy", function(i) { return x1(d[i]); })
+          .style("opacity", 1);
+
+      outlier.exit().transition()
+          .duration(duration)
+          .attr("cy", function(i) { return x1(d[i]); })
+          .style("opacity", 1e-6)
+          .remove();
+
+      // Compute the tick format.
+      var format = tickFormat || x1.tickFormat(8);
+
+      // Update box ticks.
+      var boxTick = g.selectAll("text.box")
+          .data(quartileData);
+
+      boxTick.enter().append("text")
+          .attr("class", "box")
+          .attr("dy", ".3em")
+          .attr("dx", function(d, i) { return i & 1 ? 6 : -6 })
+          .attr("x", function(d, i) { return i & 1 ? width : 0 })
+          .attr("y", x0)
+          .attr("text-anchor", function(d, i) { return i & 1 ? "start" : "end"; })
+          .text(format)
+        .transition()
+          .duration(duration)
+          .attr("y", x1);
+
+      boxTick.transition()
+          .duration(duration)
+          .text(format)
+          .attr("y", x1);
+
+      // Update whisker ticks. These are handled separately from the box
+      // ticks because they may or may not exist, and we want don't want
+      // to join box ticks pre-transition with whisker ticks post-.
+      var whiskerTick = g.selectAll("text.whisker")
+          .data(whiskerData || []);
+
+      whiskerTick.enter().append("text")
+          .attr("class", "whisker")
+          .attr("dy", ".3em")
+          .attr("dx", 6)
+          .attr("x", width)
+          .attr("y", x0)
+          .text(format)
+          .style("opacity", 1e-6)
+        .transition()
+          .duration(duration)
+          .attr("y", x1)
+          .style("opacity", 1);
+
+      whiskerTick.transition()
+          .duration(duration)
+          .text(format)
+          .attr("y", x1)
+          .style("opacity", 1);
+
+      whiskerTick.exit().transition()
+          .duration(duration)
+          .attr("y", x1)
+          .style("opacity", 1e-6)
+          .remove();
+    });
+    d3.timer.flush();
+  }
+
+  box.width = function(x) {
+    if (!arguments.length) return width;
+    width = x;
+    return box;
+  };
+
+  box.height = function(x) {
+    if (!arguments.length) return height;
+    height = x;
+    return box;
+  };
+
+  box.tickFormat = function(x) {
+    if (!arguments.length) return tickFormat;
+    tickFormat = x;
+    return box;
+  };
+
+  box.duration = function(x) {
+    if (!arguments.length) return duration;
+    duration = x;
+    return box;
+  };
+
+  box.domain = function(x) {
+    if (!arguments.length) return domain;
+    domain = x == null ? x : d3.functor(x);
+    return box;
+  };
+
+  box.value = function(x) {
+    if (!arguments.length) return value;
+    value = x;
+    return box;
+  };
+
+  box.whiskers = function(x) {
+    if (!arguments.length) return whiskers;
+    whiskers = x;
+    return box;
+  };
+
+  box.quartiles = function(x) {
+    if (!arguments.length) return quartiles;
+    quartiles = x;
+    return box;
+  };
+
+  return box;
+};
+
+function boxWhiskers(d) {
+  return [0, d.length - 1];
+}
+
+function boxQuartiles(d) {
+  return [
+    d3.quantile(d, .25),
+    d3.quantile(d, .5),
+    d3.quantile(d, .75)
+  ];
+}
+
+})();