rails-data-explorer 0.1.0 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 111102a24d7fb013d746b9a0dba6df3f7a881c1d
+  data.tar.gz: a2a99dd6d1b00c8f62143c725fdb7eb0bd6bb404
+SHA512:
+  metadata.gz: 75fd9d7a12c1e32bc06fad41a4e662165aed8909290bf5c7fd17270a9d4e7dd23d780c5611bf42bd835d0052648cf17e3aa6a07f70e0b899bcf58772b7a9ea3c
+  data.tar.gz: c5c5236cb2fbcb441be1a4a449b405f02ec06741880bedeeb1cd5e9bdcb74fbdc774da84baaccf2a12b932ebbc1b272405f4bdec51203333e6f3d5d49285642c

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.2.0
+
+* Improvements to charts.
+* Sort labels for categorical data.
+* Improved table of contents at top of page.
+* Improved presentation of contingency table.
+* Avoid division by zero when given a data series with all '0' values.
+* Improved performance of table rendering.
+* Added `#number_of_values` for integration with Filterrific gem.
+
 ## 0.1.0
 
 * Is now being used in a production app, however no documentation yet.

data/README.md

@@ -36,6 +36,24 @@ or with bundler in your Gemfile:
 * Chart -
 
 
+### Ways to shoot yourself in the foot
+
+* Loading too many DB rows at once: Remember that you are loading ActiveRecord
+  objects, and they can use a lot of ram. It's a cartesian product of number of
+  rows times columns per record. As a rule of thumb, for a medium sized model with
+  30 columns, you can load up to 10,000 rows.
+* Using expensive operations in the :data_method option for a given data series.
+  As a rule of thumb, it should be ok to run simple methods that don't require
+  DB access. Examples: `#.to_s`, `if` and `case`, and math operations.
+* Declaring too many charts: The charts rendered are the sum of the following:
+    * univariate: one or more charts for each data series
+    * bivariate: the cartesian product of all data series in a bivariate group
+    * multivariate: one or more charts for each multivariate group
+  I have tested it with 70 charts on a single page. More are probably ok, I
+  just haven't tested it.
+* Drowning in detail. rde makes it easy to generate a large number of charts.
+  Make sure you don't miss the important data in the noise.
+
 ### Dependencies
 
 * ActionView >= 3.0

data/doc/how_to/release.md

@@ -1,12 +1,10 @@
-Workflow to Maintain This Gem
-=============================
+# Workflow to Maintain This Gem
 
 I use the gem-release gem
 
 For more info see: https://github.com/svenfuchs/gem-release#usage
 
-Steps for an update
--------------------
+## Steps for an update
 
 1. Update code and commit it.
 2. Add entry to CHANGELOG and commit it:
@@ -19,5 +17,4 @@ Steps for an update
 5. Create a git tag and push to origin.
    `gem tag`
 
-
 http://prioritized.net/blog/gemify-assets-for-rails/

data/lib/rails-data-explorer-no-rails.rb

@@ -11,6 +11,7 @@ require 'interpolate'
 
 require 'rails_data_explorer'
 require 'rails-data-explorer/chart'
+require 'rails-data-explorer/constants'
 require 'rails-data-explorer/data_series'
 require 'rails-data-explorer/data_set'
 require 'rails-data-explorer/data_type'

data/lib/rails-data-explorer.rb

@@ -1,3 +1,5 @@
+
+
 require 'action_view'
 require 'active_support/all' # TODO: once the dust settles, only require the modules we need
 require 'color'
@@ -9,6 +11,7 @@ require 'rails-data-explorer/engine'
 
 require 'rails_data_explorer'
 require 'rails-data-explorer/chart'
+require 'rails-data-explorer/constants'
 require 'rails-data-explorer/data_series'
 require 'rails-data-explorer/data_set'
 require 'rails-data-explorer/data_type'
@@ -19,6 +22,7 @@ require 'rails-data-explorer/statistics/rng_power_law'
 require 'rails-data-explorer/utils/color_scale'
 require 'rails-data-explorer/utils/data_binner'
 require 'rails-data-explorer/utils/data_quantizer'
+require 'rails-data-explorer/utils/rde_table'
 require 'rails-data-explorer/utils/value_formatter'
 
 require 'rails-data-explorer/chart/box_plot'
@@ -33,6 +37,7 @@ require 'rails-data-explorer/chart/parallel_set'
 require 'rails-data-explorer/chart/pie_chart'
 require 'rails-data-explorer/chart/scatterplot'
 require 'rails-data-explorer/chart/stacked_bar_chart_categorical_percent'
+require 'rails-data-explorer/chart/stacked_histogram_temporal'
 require 'rails-data-explorer/data_type/categorical'
 require 'rails-data-explorer/data_type/quantitative'
 require 'rails-data-explorer/data_type/quantitative/decimal'

data/lib/rails-data-explorer/chart.rb

@@ -19,27 +19,27 @@ class RailsDataExplorer
   protected
 
     # Renders an HTML table
-    # @param[OpenStruct, Struct] table_struct
-    def render_html_table(table_struct)
-      content_tag(:table, :class => 'table rde-table') do
-        table_struct.rows.map { |row|
-          content_tag(row.tag, :class => row.css_class) do
+    # @param rde_table [RdeTable]
+    def render_html_table(rde_table)
+      content_tag(:table, class: 'table rde-table') do
+        rde_table.rows.map { |row|
+          content_tag(row.tag, class: row.css_class) do
             row.cells.map { |cell|
               if cell.ruby_formatter
                 content_tag(
                   cell.tag,
                   instance_exec(cell.value, &cell.ruby_formatter),
-                  :class => cell.css_class,
-                  :title => cell.title,
-                  :style => cell.style,
+                  class: cell.css_class,
+                  title: cell.title,
+                  style: cell.style,
                 )
               else
                 content_tag(
                   cell.tag,
                   cell.value,
-                  :class => cell.css_class,
-                  :title => cell.title,
-                  :style => cell.style,
+                  class: cell.css_class,
+                  title: cell.title,
+                  style: cell.style,
                 )
               end
             }.join.html_safe

data/lib/rails-data-explorer/chart/box_plot_group.rb

@@ -41,17 +41,24 @@ class RailsDataExplorer
           next  if (y_val.nil? || Float::NAN == y_val)
           values_hash[y_val] << x_ds.values[idx]
         }
+        y_sorted_keys = y_ds.uniq_vals.sort(
+          &y_ds.label_sorter(
+            nil,
+            lambda { |a,b| a <=> b }
+          )
+        )
+        sorted_values = y_sorted_keys.map { |y_val| values_hash[y_val] }
 
         {
-          values: values_hash.values,
-          category_labels: values_hash.keys,
+          values: sorted_values,
+          category_labels: y_sorted_keys,
           min: min,
           max: max,
           base_width: 120,
           base_height: 800,
           axis_tick_format: x_ds.axis_tick_format,
           num_box_plots: y_ds.uniq_vals_count,
-          axis_scale: DataSeries.new('_', [min, max]).axis_scale
+          axis_scale: DataSeries.new('_', [min, max]).axis_scale(:d3)
         }
       end
 

data/lib/rails-data-explorer/chart/contingency_table.rb

@@ -46,54 +46,61 @@ class RailsDataExplorer
 
         ca = case @data_set.dimensions_count
         when 2
-          # Table
-          OpenStruct.new(
+          Utils::RdeTable.new(
             # Top header row
-            :rows => [
-              OpenStruct.new(
-                :css_class => 'rde-column_header',
-                :tag => :tr,
-                :cells => [
-                  OpenStruct.new(:tag => :th, :value => '')
-                ] +
-                x_sorted_keys.map { |x_val|
-                  OpenStruct.new(:tag => :th, :value => x_val)
-                } +
-                [OpenStruct.new(:tag => :th, :value => 'Totals')]
+            [
+              Utils::RdeTableRow.new(
+                :tr,
+                [Utils::RdeTableCell.new(:th, '')] +
+                x_sorted_keys.map { |x_val| Utils::RdeTableCell.new(:th, x_val) } +
+                [Utils::RdeTableCell.new(:th, 'Totals')],
+                css_class: 'rde-column_header'
               )
             ] +
             # Data rows
             y_sorted_keys.map { |y_val|
-              OpenStruct.new(
-                :css_class => 'rde-data_row',
-                :tag => :tr,
-                :cells => [
-                  OpenStruct.new(:tag => :th, :value => y_val, :css_class => 'rde-row_header')
+              Utils::RdeTableRow.new(
+                :tr,
+                [
+                  Utils::RdeTableCell.new(:th, y_val, css_class: 'rde-row_header')
                 ] +
                 x_sorted_keys.map { |x_val|
-                  OpenStruct.new(
-                    :tag => :td,
-                    :value => @observed_vals[x_val][y_val],
-                    :css_class => 'rde-numerical',
-                    :title => "Expected value: #{ number_with_precision(@expected_vals[x_val][y_val]) }",
-                    :style => "color: #{ @delta_attrs[x_val][y_val][:color] };",
+                  Utils::RdeTableCell.new(
+                    :td,
+                    @observed_vals[x_val][y_val],
+                    css_class: 'rde-numerical',
+                    title: [
+                      "Expected value: #{ number_with_precision(@expected_vals[x_val][y_val], precision: 3, significant: true) }",
+                      "Percentage of row: #{ number_to_percentage(@delta_attrs[x_val][y_val][:percentage_of_row], precision: 3, significant: true) }",
+                      "Percentage of col: #{ number_to_percentage(@delta_attrs[x_val][y_val][:percentage_of_col], precision: 3, significant: true) }",
+                    ].join("\n"),
+                    style: "color: #{ @delta_attrs[x_val][y_val][:color] };",
                   )
                 } +
-                [OpenStruct.new(:tag => :th, :value => @observed_vals[:_sum][y_val])]
+                [
+                  Utils::RdeTableCell.new(
+                    :th,
+                    @observed_vals[:_sum][y_val],
+                    title: "Percentage of col: #{ number_to_percentage(@delta_attrs[:_sum][y_val][:percentage_of_col], precision: 3, significant: true) }"
+                  )
+                ],
+                css_class: 'rde-data_row'
               )
             } +
             # Footer row
             [
-              OpenStruct.new(
-                :css_class => 'rde-column_header',
-                :tag => :tr,
-                :cells => [
-                  OpenStruct.new(:tag => :th, :value => 'Totals', :css_class => 'rde-row_header')
-                ] +
+              Utils::RdeTableRow.new(
+                :tr,
+                [Utils::RdeTableCell.new(:th, 'Totals', css_class: 'rde-row_header')] +
                 x_sorted_keys.map { |x_val|
-                  OpenStruct.new(:tag => :th, :value => @observed_vals[x_val][:_sum])
+                  Utils::RdeTableCell.new(
+                    :th,
+                    @observed_vals[x_val][:_sum],
+                    title: "Percentage of row: #{ number_to_percentage(@delta_attrs[x_val][:_sum][:percentage_of_row], precision: 3, significant: true) }"
+                  )
                 } +
-                [OpenStruct.new(:tag => :th, :value => @observed_vals[:_sum][:_sum])]
+                [Utils::RdeTableCell.new(:th, @observed_vals[:_sum][:_sum])],
+                css_class: 'rde-column_header'
               )
             ]
           )
@@ -108,8 +115,8 @@ class RailsDataExplorer
         ca = compute_chart_attrs
         return ''  unless ca
 
-        content_tag(:div, :class => 'rde-chart rde-contingency-table', :id => dom_id) do
-          content_tag(:h3, "Contingency Table", :class => 'rde-chart-title') +
+        content_tag(:div, class: 'rde-chart rde-contingency-table', id: dom_id) do
+          content_tag(:h3, "Contingency Table", class: 'rde-chart-title') +
           render_html_table(ca)
         end +
         content_tag(:p, @conclusion)
@@ -127,7 +134,7 @@ class RailsDataExplorer
       # @param[DataSeries] y_ds
       def compute_contingency_and_chi_squared!(x_ds, y_ds)
         # Compute the observed values table
-        @observed_vals = { :_sum => { :_sum => 0 } }
+        @observed_vals = { _sum: { _sum: 0 } }
         x_ds.uniq_vals.each { |x_val|
           @observed_vals[x_val] = {}
           @observed_vals[x_val][:_sum] = 0
@@ -166,18 +173,24 @@ class RailsDataExplorer
           }
         }
         # Compute deltas
-        @delta_attrs = {}
+        @delta_attrs = { _sum: {} }
         color_scale = RailsDataExplorer::Utils::ColorScale.new
         x_ds.uniq_vals.each { |x_val|
-          @delta_attrs[x_val] = {}
+          @delta_attrs[x_val] = { _sum: {} }
+          @delta_attrs[x_val][:_sum][:percentage_of_row] = (@observed_vals[x_val][:_sum] / @observed_vals[:_sum][:_sum].to_f) * 100
           y_ds.uniq_vals.each { |y_val|
             delta = @observed_vals[x_val][y_val] - @expected_vals[x_val][y_val]
             delta_factor = delta / @expected_vals[x_val][y_val].to_f
             @delta_attrs[x_val][y_val] = {
-              :expected => @expected_vals[x_val][y_val],
-              :color => color_scale.compute(delta_factor),
-              :delta => delta,
-              :delta_factor => delta_factor,
+              :expected @expected_vals[x_val][y_val],
+              color: color_scale.compute(delta_factor),
+              delta: delta,
+              delta_factor: delta_factor,
+              percentage_of_row: (@observed_vals[x_val][y_val] / @observed_vals[:_sum][y_val].to_f) * 100,
+              percentage_of_col: (@observed_vals[x_val][y_val] / @observed_vals[x_val][:_sum].to_f) * 100,
+            }
+            @delta_attrs[:_sum][y_val] ||= {
+              percentage_of_col: (@observed_vals[:_sum][y_val] / @observed_vals[:_sum][:_sum].to_f) * 100
             }
           }
         }
@@ -200,7 +213,7 @@ class RailsDataExplorer
           @conclusion = [
             "We did not run the ",
             %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> ),
-            "since #{ number_to_percentage(ratio_of_observed_vals_below_five * 100, :precision => 0) } ",
+            "since #{ number_to_percentage(ratio_of_observed_vals_below_five * 100, precision: 0) } ",
             "of observed values in the contingency table are below 5 (cutoff is 20%)."
           ].join
         elsif([x_ds, y_ds].any? { |e| e.uniq_vals.length < 2 })
@@ -212,9 +225,9 @@ class RailsDataExplorer
         else
           @conclusion = %(<a href="http://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence">Pearson chi squared test of independence</a> suggests that )
           @conclusion << if @p_value <= @significance_level
-            %("#{ x_ds.name }" and "#{ y_ds.name }" are dependent variables (p_value: #{ number_with_precision(@p_value) }))
+            %("#{ x_ds.name }" and "#{ y_ds.name }" are dependent variables (p_value of #{ number_with_precision(@p_value) } <= #{ number_with_precision(@significance_level )}))
           else
-            %("#{ x_ds.name }" and "#{ y_ds.name }" are independent variables (p_value: #{ number_with_precision(@p_value) }))
+            %("#{ x_ds.name }" and "#{ y_ds.name }" are independent variables (p_value of #{ number_with_precision(@p_value) } > #{ number_with_precision(@significance_level )}))
           end
         end
         @conclusion = @conclusion.html_safe

data/lib/rails-data-explorer/chart/descriptive_statistics_table.rb

@@ -9,9 +9,9 @@ class RailsDataExplorer
 
       def render
         return ''  unless render?
-        content_tag(:div, :id => dom_id, :class => 'rde-chart rde-descriptive-statistics-table') do
+        content_tag(:div, id: dom_id, class: 'rde-chart rde-descriptive-statistics-table') do
           @data_set.data_series.map { |data_series|
-            content_tag(:h3, "Descriptive Statistics", :class => 'rde-chart-title') +
+            content_tag(:h3, "Descriptive Statistics", class: 'rde-chart-title') +
             render_html_table(data_series.descriptive_statistics_table)
           }.join.html_safe
         end

data/lib/rails-data-explorer/chart/histogram_categorical.rb

@@ -13,6 +13,9 @@ class RailsDataExplorer
 
         # compute histogram
         h = x_ds.values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
+        histogram_values_ds = DataSeries.new('_', h.values)
+        y_scale_type = histogram_values_ds.axis_scale(:vega)
+        bar_y2_val = 'log' == y_scale_type ? histogram_values_ds.min_val / 10.0 : 0
         {
           values: h.map { |k,v|
             { x: k, y: v }
@@ -25,7 +28,10 @@ class RailsDataExplorer
           x_axis_label: x_ds.name,
           x_axis_tick_format: "d3.format('r')",
           y_axis_label: 'Frequency',
-          y_axis_tick_format: "d3.format('r')",
+          y_axis_tick_format: "d3.format(',g')", #histogram_values_ds.axis_tick_format,
+          y_scale_type: y_scale_type,
+          y_scale_domain: [bar_y2_val, histogram_values_ds.max_val],
+          bar_y2_val: bar_y2_val,
         }
       end
 
@@ -62,9 +68,9 @@ class RailsDataExplorer
                     },
                     {
                       "name": "y",
+                      "type": "#{ ca[:y_scale_type] }",
                       "range": "height",
-                      "nice": true,
-                      "domain": {"data": "table", "field": "data.y"}
+                      "domain": #{ ca[:y_scale_domain].to_json },
                     }
                   ],
                   "axes": [
@@ -91,7 +97,7 @@ class RailsDataExplorer
                           "x": {"scale": "x", "field": "data.x"},
                           "width": {"scale": "x", "band": true, "offset": -1},
                           "y": {"scale": "y", "field": "data.y"},
-                          "y2": {"scale": "y", "value": 0}
+                          "y2": {"scale": "y", "value": #{ ca[:bar_y2_val] }},
                         },
                         "update": {
                           "fill": {"value": "#1F77B4"}

data/lib/rails-data-explorer/chart/histogram_quantitative.rb

@@ -12,11 +12,14 @@ class RailsDataExplorer
         return false  if x_ds.nil?
 
         # compute histogram
-        quantizer = Utils::DataQuantizer.new(x_ds, :max_number_of_bins => 100)
+        quantizer = Utils::DataQuantizer.new(x_ds, max_number_of_bins: 100)
         quantized_values = quantizer.values
         number_of_bars = quantizer.number_of_bins
         width = 800
         h = quantized_values.inject(Hash.new(0)) { |m,e| m[e] += 1; m }
+        histogram_values_ds = DataSeries.new('_', h.values)
+        y_scale_type = histogram_values_ds.axis_scale(:vega)
+        bar_y2_val = 'log' == y_scale_type ? histogram_values_ds.min_val / 10.0 : 0
         {
           values: h.map { |k,v| { x: k, y: v } },
           width: width,
@@ -27,6 +30,9 @@ class RailsDataExplorer
           bar_width: (width / number_of_bars.to_f) - 3,
           y_axis_label: 'Frequency',
           y_axis_tick_format: "d3.format('r')",
+          y_scale_type: y_scale_type,
+          y_scale_domain: [bar_y2_val, histogram_values_ds.max_val],
+          bar_y2_val: bar_y2_val,
         }
       end
 
@@ -51,7 +57,7 @@ class RailsDataExplorer
                   "data": [
                     {
                       "name": "table",
-                      "values": #{ ca[:values].to_json }
+                      "values": #{ ca[:values].to_json },
                     }
                   ],
                   "scales": [
@@ -61,12 +67,13 @@ class RailsDataExplorer
                       "range": "width",
                       "zero": false,
                       "nice": #{ ca[:x_scale_nice] },
-                      "domain": {"data": "table", "field": "data.x"}
+                      "domain": {"data": "table", "field": "data.x"},
                     },
                     {
                       "name": "y",
+                      "type": "#{ ca[:y_scale_type] }",
                       "range": "height",
-                      "domain": {"data": "table", "field": "data.y"}
+                      "domain": #{ ca[:y_scale_domain].to_json },
                     }
                   ],
                   "axes": [
@@ -93,7 +100,7 @@ class RailsDataExplorer
                           "x": {"scale": "x", "field": "data.x"},
                           "width": { "value": #{ ca[:bar_width] } },
                           "y": {"scale": "y", "field": "data.y"},
-                          "y2": {"scale": "y", "value": 0},
+                          "y2": {"scale": "y", "value": #{ ca[:bar_y2_val] }},
                         },
                         "update": {
                           "fill": {"value": "#1F77B4"}