rails-data-explorer 0.2.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b417c02b65c9c3f0fed2ddb85c5f6e992671aea
-  data.tar.gz: b0ff6ff26221174cf12ba739f5a6029ac636a0d1
+  metadata.gz: 411afabc4250e907fbdbe7e70805be898219eb5a
+  data.tar.gz: 6a95cd6abc7828634bfd88ac69ca9b2a805a4970
 SHA512:
-  metadata.gz: 7badacd27cd2ed6059094d5aba6027918e342437c79ae938504b451c6572632c42d0abf1817c0ea7dc0e0c3083f037dd11acd455ab6e23784349908faf5a0c00
-  data.tar.gz: 597a4a2857756a8226582d69b3a2154ea1708e2c19a396a2cc94f2947d63a8126a8517b7242923d9b328384730bde98b3172a10a472ef8bc35d1d52d08daae20
+  metadata.gz: 598df5a92a7f39dbb213283cfd93cbf0a7c72fdc3c12aaa03fd875d924b2160eacfcdbec079045c8a8eca67da6fa336939ed58e8f4d8b6e839ce48ba5ad866d9
+  data.tar.gz: 8f9ee3dd2de65400978ea82a3445c40974bac8e5a5e8a54137fe062a25eed16ecd6b10e92e5a2ffcdf57b7ad37f651451b12f6e4d23a5b93271630f9f7a1226e

data/CHANGELOG.md

@@ -1,6 +1,22 @@
-there seems a huge inefficiency in RailsDataExplorer#build_exploration_from_data_series_specs
-where we iterate over the entire data collection for every exploration.
-
+TODO: add sampling for large ActiveRecord tables
+
+# 1.0.0
+
+* New chart type: StackedBarChartCategorical to go with its percentage cousing.
+* Improved display of table of contents.
+* Improved display of box plot groups: Auto scale charts to make best use of
+  screen space when there are extreme outliers.
+* Automatically limit the number of distinct values in a categorical data series
+  to reduce number of items in categorical charts. Group less frequent observations
+  under '[Other]'.
+* Made charts wider to use more screen space.
+* Scroll contingency tables horizontally if they are too wide.
+* Improved label sorter for categorical values. When working with a mix of
+  numeric and non-numeric labels, sort non-numeric ones at the end.
+* Improved performance by caching a lot of computed data.
+* Removed obsolete files.
+* Improved documentation.
+* Various bug fixes.
 
 ### 0.2.3
 

data/README.md

@@ -27,7 +27,9 @@ named 'signups' in `app/controllers/users_controller.rb`:
 
 ~~~ ruby
 def signups
+  # DANGER! Make sure not to load too much data into memory!
   user_data = User.all.to_a
+
   c_binner = RailsDataExplorer::Utils::DataBinner.new(
     '0' => 0,
     '1' => 1,

data/lib/rails-data-explorer-no-rails.rb

@@ -7,37 +7,41 @@ require 'descriptive-statistics'
 require 'distribution'
 require 'interpolate'
 
-# require 'rails-data-explorer/engine'
+# require 'rails_data_explorer/action_view_extension.rb'
 
-require 'rails_data_explorer'
-require 'rails-data-explorer/chart'
-require 'rails-data-explorer/constants'
-require 'rails-data-explorer/data_series'
-require 'rails-data-explorer/data_set'
-require 'rails-data-explorer/data_type'
-require 'rails-data-explorer/exploration'
-require 'rails-data-explorer/statistics/rng_category'
-require 'rails-data-explorer/statistics/rng_gaussian'
-require 'rails-data-explorer/statistics/rng_power_law'
-require 'rails-data-explorer/utils/color_scale'
-require 'rails-data-explorer/utils/data_binner'
-require 'rails-data-explorer/utils/data_quantizer'
-require 'rails-data-explorer/utils/value_formatter'
+# require 'rails_data_explorer/engine.rb'
 
-require 'rails-data-explorer/chart/box_plot'
-require 'rails-data-explorer/chart/box_plot_group'
-require 'rails-data-explorer/chart/contingency_table'
-require 'rails-data-explorer/chart/descriptive_statistics_table'
-require 'rails-data-explorer/chart/histogram_categorical'
-require 'rails-data-explorer/chart/histogram_quantitative'
-require 'rails-data-explorer/chart/histogram_temporal'
-require 'rails-data-explorer/chart/parallel_coordinates'
-require 'rails-data-explorer/chart/parallel_set'
-require 'rails-data-explorer/chart/pie_chart'
-require 'rails-data-explorer/chart/scatterplot'
-require 'rails-data-explorer/chart/stacked_bar_chart_categorical_percent'
-require 'rails-data-explorer/data_type/categorical'
-require 'rails-data-explorer/data_type/quantitative'
-require 'rails-data-explorer/data_type/quantitative/decimal'
-require 'rails-data-explorer/data_type/quantitative/integer'
-require 'rails-data-explorer/data_type/quantitative/temporal'
+require 'rails_data_explorer.rb'
+require 'rails_data_explorer/chart.rb'
+require 'rails_data_explorer/data_series.rb'
+require 'rails_data_explorer/data_set.rb'
+require 'rails_data_explorer/data_type.rb'
+require 'rails_data_explorer/exploration.rb'
+require 'rails_data_explorer/statistics/rng_category.rb'
+require 'rails_data_explorer/statistics/rng_gaussian.rb'
+require 'rails_data_explorer/statistics/rng_power_law.rb'
+require 'rails_data_explorer/utils/color_scale.rb'
+require 'rails_data_explorer/utils/data_binner.rb'
+require 'rails_data_explorer/utils/data_quantizer.rb'
+require 'rails_data_explorer/utils/rde_table.rb'
+require 'rails_data_explorer/utils/value_formatter.rb'
+
+require 'rails_data_explorer/chart/box_plot.rb'
+require 'rails_data_explorer/chart/box_plot_group.rb'
+require 'rails_data_explorer/chart/contingency_table.rb'
+require 'rails_data_explorer/chart/descriptive_statistics_table.rb'
+require 'rails_data_explorer/chart/histogram_categorical.rb'
+require 'rails_data_explorer/chart/histogram_quantitative.rb'
+require 'rails_data_explorer/chart/histogram_temporal.rb'
+require 'rails_data_explorer/chart/parallel_coordinates.rb'
+require 'rails_data_explorer/chart/parallel_set.rb'
+require 'rails_data_explorer/chart/pie_chart.rb'
+require 'rails_data_explorer/chart/scatterplot.rb'
+require 'rails_data_explorer/chart/stacked_bar_chart_categorical.rb'
+require 'rails_data_explorer/data_type/categorical.rb'
+require 'rails_data_explorer/data_type/quantitative.rb'
+require 'rails_data_explorer/data_type/quantitative/decimal.rb'
+require 'rails_data_explorer/data_type/quantitative/integer.rb'
+require 'rails_data_explorer/data_type/quantitative/temporal.rb'
+
+require 'rails_data_explorer/chart/stacked_bar_chart_categorical_percent.rb'

data/lib/rails-data-explorer.rb

@@ -1,3 +1,5 @@
+
+
 require 'action_view'
 require 'active_support/all' # TODO: once the dust settles, only require the modules we need
 require 'color'
@@ -5,41 +7,42 @@ require 'descriptive-statistics'
 require 'distribution'
 require 'interpolate'
 
-require 'rails-data-explorer/action_view_extension'
+require 'rails_data_explorer/action_view_extension.rb'
+
+require 'rails_data_explorer/engine.rb'
 
-require 'rails-data-explorer/engine'
+require 'rails_data_explorer.rb'
+require 'rails_data_explorer/chart.rb'
+require 'rails_data_explorer/data_series.rb'
+require 'rails_data_explorer/data_set.rb'
+require 'rails_data_explorer/data_type.rb'
+require 'rails_data_explorer/exploration.rb'
+require 'rails_data_explorer/statistics/rng_category.rb'
+require 'rails_data_explorer/statistics/rng_gaussian.rb'
+require 'rails_data_explorer/statistics/rng_power_law.rb'
+require 'rails_data_explorer/utils/color_scale.rb'
+require 'rails_data_explorer/utils/data_binner.rb'
+require 'rails_data_explorer/utils/data_quantizer.rb'
+require 'rails_data_explorer/utils/rde_table.rb'
+require 'rails_data_explorer/utils/value_formatter.rb'
 
-require 'rails_data_explorer'
-require 'rails-data-explorer/chart'
-require 'rails-data-explorer/constants'
-require 'rails-data-explorer/data_series'
-require 'rails-data-explorer/data_set'
-require 'rails-data-explorer/data_type'
-require 'rails-data-explorer/exploration'
-require 'rails-data-explorer/statistics/rng_category'
-require 'rails-data-explorer/statistics/rng_gaussian'
-require 'rails-data-explorer/statistics/rng_power_law'
-require 'rails-data-explorer/utils/color_scale'
-require 'rails-data-explorer/utils/data_binner'
-require 'rails-data-explorer/utils/data_quantizer'
-require 'rails-data-explorer/utils/rde_table'
-require 'rails-data-explorer/utils/value_formatter'
+require 'rails_data_explorer/chart/box_plot.rb'
+require 'rails_data_explorer/chart/box_plot_group.rb'
+require 'rails_data_explorer/chart/contingency_table.rb'
+require 'rails_data_explorer/chart/descriptive_statistics_table.rb'
+require 'rails_data_explorer/chart/histogram_categorical.rb'
+require 'rails_data_explorer/chart/histogram_quantitative.rb'
+require 'rails_data_explorer/chart/histogram_temporal.rb'
+require 'rails_data_explorer/chart/parallel_coordinates.rb'
+require 'rails_data_explorer/chart/parallel_set.rb'
+require 'rails_data_explorer/chart/pie_chart.rb'
+require 'rails_data_explorer/chart/scatterplot.rb'
+require 'rails_data_explorer/chart/stacked_bar_chart_categorical.rb'
+# require 'rails_data_explorer/chart/stacked_histogram_temporal.rb'
+require 'rails_data_explorer/data_type/categorical.rb'
+require 'rails_data_explorer/data_type/quantitative.rb'
+require 'rails_data_explorer/data_type/quantitative/decimal.rb'
+require 'rails_data_explorer/data_type/quantitative/integer.rb'
+require 'rails_data_explorer/data_type/quantitative/temporal.rb'
 
-require 'rails-data-explorer/chart/box_plot'
-require 'rails-data-explorer/chart/box_plot_group'
-require 'rails-data-explorer/chart/contingency_table'
-require 'rails-data-explorer/chart/descriptive_statistics_table'
-require 'rails-data-explorer/chart/histogram_categorical'
-require 'rails-data-explorer/chart/histogram_quantitative'
-require 'rails-data-explorer/chart/histogram_temporal'
-require 'rails-data-explorer/chart/parallel_coordinates'
-require 'rails-data-explorer/chart/parallel_set'
-require 'rails-data-explorer/chart/pie_chart'
-require 'rails-data-explorer/chart/scatterplot'
-require 'rails-data-explorer/chart/stacked_bar_chart_categorical_percent'
-# require 'rails-data-explorer/chart/stacked_histogram_temporal'
-require 'rails-data-explorer/data_type/categorical'
-require 'rails-data-explorer/data_type/quantitative'
-require 'rails-data-explorer/data_type/quantitative/decimal'
-require 'rails-data-explorer/data_type/quantitative/integer'
-require 'rails-data-explorer/data_type/quantitative/temporal'
+require 'rails_data_explorer/chart/stacked_bar_chart_categorical_percent.rb'

data/lib/rails_data_explorer.rb

@@ -1,7 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Responsibilities:
+#  * Integrate all the pieces required for this gem
+#  * Initialize a collection of Explorations
+#
+# Collaborators:
+#  * Exploration
 class RailsDataExplorer
 
-  attr_accessor :output_buffer # required for content_tag
-  include ActionView::Helpers::TagHelper
+  GREATER_ZERO = 1.0 / 1_000_000 # The smallest value to use if we have to avoid zero (div by zero)
 
   attr_reader :explorations
   attr_reader :data_series_names
@@ -28,6 +35,16 @@ class RailsDataExplorer
       multivariate: {}
     }
     @data_series_names = data_series_specs.map { |e| e[:name] }
+
+    @cached_data_series = data_series_specs.inject({}) { |m, ds_spec|
+      m[ds_spec[:name]] = DataSeries.new(
+        ds_spec[:name],
+        data_collection.map(&ds_spec[:data_method]),
+        ds_spec
+      )
+      m
+    }
+
     # Default to all univariate explorations
     explorations_to_render = (
       explorations_to_render || @data_series_names.inject({ univariate: {}}) { |m,e|
@@ -60,7 +77,10 @@ class RailsDataExplorer
     charts[:univariate].uniq.compact.each { |data_series_spec|
       @explorations << Exploration.new(
         data_series_spec[:name],
-        data_collection.map(&data_series_spec[:data_method]),
+        DataSet.new(
+          [@cached_data_series[data_series_spec[:name]]],
+          data_series_spec[:name]
+        ),
         render_exploration_for?(
           explorations_to_render,
           :univariate,
@@ -114,14 +134,13 @@ class RailsDataExplorer
 private
 
   def build_exploration_from_data_series_specs(data_collection, ds_specs, render_charts)
+    expl_title = ds_specs.map { |e| e[:name] }.sort.join(' vs. ')
     Exploration.new(
-      ds_specs.map { |e| e[:name] }.sort.join(' vs. '),
-      ds_specs.map { |ds_spec|
-        {
-          name: ds_spec[:name],
-          values: data_collection.map(&ds_spec[:data_method])
-        }
-      },
+      expl_title,
+      DataSet.new(
+        ds_specs.map { |ds_spec| @cached_data_series[ds_spec[:name]] },
+        expl_title
+      ),
       render_charts
     )
   end

data/lib/{rails-data-explorer → rails_data_explorer}/action_view_extension.rb

@@ -1,12 +1,19 @@
 # -*- coding: utf-8 -*-
-#
-# Adds view helpers to ActionView
-#
+
 class RailsDataExplorer
+
+  # Responsibilities:
+  #  * Add rails-data-explorer view helpers to ActionView
+  #  * Render rails-data-explorer generated content
+  #
+  # Collaborators:
+  #  * RailsDataExplorer
+  #  * Exploration
+  #
   module ActionViewExtension
 
     # Renders the entire RailsDataExplorer view
-    # @param rails_data_explorer [RailsDataExplorer]
+    # @param rde [RailsDataExplorer]
     def rails_data_explorer(rde)
       content_tag(:div, class: 'rails-data-explorer') do
         [
@@ -16,7 +23,8 @@ class RailsDataExplorer
       end
     end
 
-    # Returns a url that can be used to reset the Filterrific params
+    # Returns a url that can be used to reset the Filterrific params when
+    # rails-data-explorer is used in combination with Filterrific.
     def reset_filterrific_url(opts = {})
       url_for(
         { filterrific: { reset_filterrific: true } }.merge(opts)
@@ -37,11 +45,6 @@ class RailsDataExplorer
         end +
         content_tag(:div, class: 'panel-body') do
           content_tag(:table, class: 'table rde_toc-matrix') do
-            # render header row for analysis types
-            content_tag(:tr) do
-              content_tag(:th, 'Univariate', colspan: '2') +
-              content_tag(:th, 'Bivariate', colspan: ds_names.length)
-            end +
             # render uni-/bi-variate analysis column headers (with reversed data_series names)
             content_tag(:tr) do
               content_tag(
@@ -57,7 +60,8 @@ class RailsDataExplorer
                     },
                     anchor: 'rails_data_explorer-toc',
                   ),
-                  class: 'btn btn-default btn-xs'
+                  class: 'btn btn-default btn-xs',
+                  title: 'Load univariate explorations for all data series.'
                 ),
                 colspan: 2
               ) +
@@ -74,7 +78,7 @@ class RailsDataExplorer
                   )
                 }
                 encountered_bv_with_self = false
-                # cell with name
+                # row header with data_series name
                 content_tag(
                   :th,
                   ds_name.truncate(12),
@@ -83,6 +87,7 @@ class RailsDataExplorer
                 ) +
                 # cell with link to univariate analysis
                 if uv_expl
+                  tooltip_suffix = "univariate exploration for #{ ds_name.inspect }."
                   if uv_expl.render_charts?
                     # Link to anchor on current page (chart is currently rendered)
                     content_tag(
@@ -93,6 +98,7 @@ class RailsDataExplorer
                         class: 'btn btn-default btn-xs'
                       ),
                       class: 'rde_toc-currently_rendered',
+                      title: 'Jump to ' + tooltip_suffix,
                     )
                   else
                     # Load new page
@@ -106,12 +112,18 @@ class RailsDataExplorer
                         ),
                         class: 'btn btn-default btn-xs'
                       ),
-                      class: 'rde_toc-available_not_rendered'
+                      class: 'rde_toc-available_not_rendered',
+                      title: 'Load ' + tooltip_suffix,
                     )
                   end
                 else
                   # show that no exploration exists
-                  content_tag(:td, 'N/A', class: 'rde_toc-not_available')
+                  content_tag(
+                    :td,
+                    'N/A',
+                    class: 'rde_toc-not_available',
+                    title: 'There is no ' + tooltip_suffix,
+                  )
                 end +
                 # iterate over reversed data_series names
                 reversed_ds_names.map { |r_ds_name|
@@ -120,6 +132,8 @@ class RailsDataExplorer
                       [ds_name, r_ds_name]
                     )
                   }
+                  bv_ds_names = [ds_name.inspect, r_ds_name.inspect].sort.join(' vs. ')
+                  tooltip_suffix = "bivariate exploration for #{ bv_ds_names }."
                   if encountered_bv_with_self
                     # blank cell
                     '<td class="rde_toc-oso_diagonal"></td>'.html_safe
@@ -144,7 +158,8 @@ class RailsDataExplorer
                         ),
                         class: 'btn btn-default btn-xs'
                       ),
-                      class: 'rde_toc-available_not_rendered'
+                      class: 'rde_toc-available_not_rendered',
+                      title: "Load all bivariate explorations for #{ ds_name.inspect }.",
                     )
                   elsif bv_expl
                     # bivariate analysis exists
@@ -158,6 +173,7 @@ class RailsDataExplorer
                           class: 'btn btn-default btn-xs'
                         ),
                         class: 'rde_toc-currently_rendered',
+                        title: 'Jump to ' + tooltip_suffix,
                       )
                     else
                       # Link to anchor on new page (chart is currently not rendered)
@@ -171,12 +187,18 @@ class RailsDataExplorer
                           ),
                           class: 'btn btn-default btn-xs'
                         ),
-                        class: 'rde_toc-available_not_rendered'
+                        class: 'rde_toc-available_not_rendered',
+                        title: 'Load ' + tooltip_suffix,
                       )
                     end
                   else
                     # show that no exploration exists
-                    content_tag(:td, 'N/A', class: 'rde_toc-not_available')
+                    content_tag(
+                      :td,
+                      'N/A',
+                      class: 'rde_toc-not_available',
+                      title: 'There is no ' + tooltip_suffix,
+                    )
                   end
                 }.join.html_safe
               end

data/lib/rails_data_explorer/active_record_extension.rb

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+class RailsDataExplorer
+
+  # Responsibilities:
+  #  * Add rails-data-explorer methods to ActiveRecord::Base and sub classes
+  #
+  # Collaborators:
+  #
+  module ActiveRecordExtension
+
+    module ClassMethods
+
+    protected
+
+    end
+
+  end
+end

data/lib/{rails-data-explorer → rails_data_explorer}/chart.rb

@@ -1,4 +1,14 @@
+# -*- coding: utf-8 -*-
+
 class RailsDataExplorer
+
+  # Responsibilities:
+  #  * Visualize data
+  #  * Integrate with front end visualization libraries
+  #
+  # Collaborators:
+  #  * RdeTable
+  #
   class Chart
 
     include ActionView::Helpers::NumberHelper