rails-data-explorer 0.0.1 → 0.1.0

data/lib/rails-data-explorer/chart/stacked_bar_chart_categorical_percent.rb

@@ -17,6 +17,7 @@ class RailsDataExplorer
 
         x_ds = x_candidates.first
         y_ds = (y_candidates - [x_ds]).first
+        return false  if x_ds.nil? || y_ds.nil?
 
         # initialize data_matrix
         data_matrix = { :_sum => { :_sum => 0 } }
@@ -38,25 +39,30 @@ class RailsDataExplorer
           data_matrix[:_sum][:_sum] += 1
         }
 
-        x_sorted_keys = x_ds.uniq_vals.sort { |a,b|
-          data_matrix[b][:_sum] <=> data_matrix[a][:_sum]
-        }
-        y_sorted_keys = y_ds.uniq_vals.sort { |a,b|
-          data_matrix[:_sum][b] <=> data_matrix[:_sum][a]
-        }
+        x_sorted_keys = x_ds.uniq_vals.sort(
+          &x_ds.label_sorter(
+            nil,
+            lambda { |a,b| data_matrix[b][:_sum] <=> data_matrix[a][:_sum] }
+          )
+        )
+        y_sorted_keys = y_ds.uniq_vals.sort(
+          &y_ds.label_sorter(
+            nil,
+            lambda { |a,b| data_matrix[:_sum][b] <=> data_matrix[:_sum][a] }
+          )
+        )
 
         values = case @data_set.dimensions_count
         when 2
           y_sorted_keys.map { |y_val|
-            {
-              key: y_val,
-              values: x_sorted_keys.map { |x_val|
-                {
-                  x: x_val,
-                  y: (data_matrix[x_val][y_val] / data_matrix[x_val][:_sum].to_f) }
+            x_sorted_keys.map { |x_val|
+              {
+                x: x_val,
+                y: (data_matrix[x_val][y_val] / data_matrix[x_val][:_sum].to_f) * 100,
+                c: y_val
               }
             }
-          }
+          }.flatten
         else
           raise(ArgumentError.new("Exactly two data series required for contingency table."))
         end
@@ -72,8 +78,115 @@ class RailsDataExplorer
       def render
         return ''  unless render?
         ca = compute_chart_attrs
+        return ''  unless ca
+        render_vega(ca)
+      end
+
+      def render_vega(ca)
         %(
-          <div class="rde-chart rde-bar-chart">
+          <div class="rde-chart rde-stacked-bar-chart-categorical-percent">
+            <h3 class="rde-chart-title">Stacked Bar Chart</h3>
+            <div id="#{ dom_id }"></div>
+            <script type="text/javascript">
+              (function() {
+                var spec = {
+                  "width": 800,
+                  "height": 200,
+                  "padding": {"top": 10, "left": 50, "bottom": 50, "right": 100},
+                  "data": [
+                    {
+                      "name": "table",
+                      "values": #{ ca[:values].to_json }
+                    },
+                    {
+                      "name": "stats",
+                      "source": "table",
+                      "transform": [
+                        {"type": "facet", "keys": ["data.x"]},
+                        {"type": "stats", "value": "data.y"}
+                      ]
+                    }
+                  ],
+                  "scales": [
+                    {
+                      "name": "x",
+                      "type": "ordinal",
+                      "range": "width",
+                      "domain": {"data": "table", "field": "data.x"}
+                    },
+                    {
+                      "name": "y",
+                      "type": "linear",
+                      "range": "height",
+                      "nice": true,
+                      "domain": {"data": "stats", "field": "sum"}
+                    },
+                    {
+                      "name": "color",
+                      "type": "ordinal",
+                      "range": "category10"
+                    }
+                  ],
+                  "axes": [
+                    {
+                      "type": "x",
+                      "scale": "x",
+                      "title": "#{ ca[:x_axis_label] }",
+                      "format": #{ ca[:x_axis_tick_format] },
+                    },
+                    {
+                      "type": "y",
+                      "scale": "y",
+                      "title": "#{ ca[:y_axis_label] }",
+                      "format": #{ ca[:y_axis_tick_format] },
+                    }
+                  ],
+                  "marks": [
+                    {
+                      "type": "group",
+                      "from": {
+                        "data": "table",
+                        "transform": [
+                          {"type": "facet", "keys": ["data.c"]},
+                          {"type": "stack", "point": "data.x", "height": "data.y"}
+                        ]
+                      },
+                      "marks": [
+                        {
+                          "type": "rect",
+                          "properties": {
+                            "enter": {
+                              "x": {"scale": "x", "field": "data.x"},
+                              "width": {"scale": "x", "band": true, "offset": -1},
+                              "y": {"scale": "y", "field": "y"},
+                              "y2": {"scale": "y", "field": "y2"},
+                              "fill": {"scale": "color", "field": "data.c"}
+                            },
+                          }
+                        }
+                      ]
+                    }
+                  ],
+                  "legends": [
+                    {
+                      "fill": "color",
+                    }
+                  ],
+                };
+
+                vg.parse.spec(spec, function(chart) {
+                  var view = chart({ el:"##{ dom_id }" }).update();
+                });
+
+              })();
+            </script>
+          </div>
+        )
+      end
+
+      def render_nvd3(ca)
+        %(
+          <div class="rde-chart rde-stacked-bar-chart-categorical-percent">
             <h3 class="rde-chart-title">Stacked Bar Chart</h3>
             <div id="#{ dom_id }", style="height: 200px;">
               <svg></svg>
@@ -98,6 +211,12 @@ class RailsDataExplorer
 
                   chart.multibar.stacked(true);
                   chart.showControls(false);
+                  chart.tooltipContent(
+                    function(key, x, y, e, graph) {
+                      return '<p>' + key + '</p>' + '<p>' +  y + ' of ' + x + '</p>'
+                    }
+                  );
+
 
                   d3.select('##{ dom_id } svg')
                     .datum(data)

data/lib/rails-data-explorer/data_series.rb

@@ -8,6 +8,19 @@ class RailsDataExplorer
     delegate :available_chart_types, :to => :data_type, :prefix => false
     delegate :available_chart_roles, :to => :data_type, :prefix => false
 
+    # Any data series with a dynamic range greater than this is considered
+    # having a large dynamic range
+    # We consider dynamic range the ratio between the largest and the smallest value.
+    def self.large_dynamic_range_cutoff
+      1000.0
+    end
+
+    # Any data series with more than this uniq vals is considered having many
+    # uniq values.
+    def self.many_uniq_vals_cutoff
+      20
+    end
+
     # options: :chart_roles, :data_type (all optional)
     def initialize(_name, _values, options={})
       options = { chart_roles: [], data_type: nil }.merge(options)
@@ -53,6 +66,10 @@ class RailsDataExplorer
       data_type.axis_tick_format(values)
     end
 
+    def axis_scale
+      data_type.axis_scale(self)
+    end
+
     def uniq_vals
       @uniq_vals = values.uniq
     end
@@ -69,6 +86,23 @@ class RailsDataExplorer
       @max_val = values.compact.max
     end
 
+    # Used to decide whether we can render certain chart types
+    def has_many_uniq_vals?
+      uniq_vals_count > self.class.many_uniq_vals_cutoff
+    end
+
+    def dynamic_range
+      max_val / [min_val, max_val].min.to_f
+    end
+
+    def has_large_dynamic_range?
+      dynamic_range > self.class.large_dynamic_range_cutoff
+    end
+
+    def label_sorter(label_val_key, value_sorter)
+      data_type.label_sorter(label_val_key, self, value_sorter)
+    end
+
   private
 
     # @param[Array<Symbol>] chart_role_overrides, :x, :y, :color
@@ -94,14 +128,15 @@ class RailsDataExplorer
 
     def init_data_type(data_type_override)
       if data_type_override.nil?
-        case values.first
+        first_value = values.detect { |e| !e.nil? }
+        case first_value
         when Integer, Bignum, Fixnum
           DataType::Quantitative::Integer.new
         when Float
           DataType::Quantitative::Decimal.new
         when String
           DataType::Categorical.new
-        when DateTime, ActiveSupport::TimeWithZone
+        when Time, DateTime, ActiveSupport::TimeWithZone
           DataType::Quantitative::Temporal.new
         else
           raise(ArgumentError.new("Can't infer data type for value: #{ values.first.class.inspect }"))

data/lib/rails-data-explorer/data_type/categorical.rb

@@ -70,14 +70,22 @@ class RailsDataExplorer
           m << { :label => "#{ k.to_s }_percent", :value => (v / total_count.to_f) * 100, :ruby_formatter => ruby_formatters[:percent] }
           m
         }.sort { |a,b| b[:value] <=> a[:value] }
-        r.insert(0, { :label => 'Total_count', :value => total_count, :ruby_formatter => ruby_formatters[:integer] })
-        r.insert(0, { :label => 'Total_percent', :value => 100, :ruby_formatter => ruby_formatters[:percent] })
+        r.insert(0, { :label => '[Total]_count', :value => total_count, :ruby_formatter => ruby_formatters[:integer] })
+        r.insert(0, { :label => '[Total]_percent', :value => 100, :ruby_formatter => ruby_formatters[:percent] })
         r
       end
 
       # Returns an OpenStruct that describes a statistics table.
       def descriptive_statistics_table(values)
         desc_stats = descriptive_statistics(values)
+        if desc_stats.length < DataSeries.many_uniq_vals_cutoff
+          descriptive_statistics_table_horizontal(desc_stats)
+        else
+          descriptive_statistics_table_vertical(desc_stats)
+        end
+      end
+
+      def descriptive_statistics_table_horizontal(desc_stats)
         labels = desc_stats.map { |e| e[:label].gsub(/_count|_percent/, '') }.uniq
         table = OpenStruct.new(
           :rows => []
@@ -108,10 +116,72 @@ class RailsDataExplorer
         table
       end
 
+      def descriptive_statistics_table_vertical(desc_stats)
+        labels = desc_stats.map { |e| e[:label].gsub(/_count|_percent/, '') }.uniq
+        table = OpenStruct.new(
+          :rows => []
+        )
+        table.rows << OpenStruct.new(
+          :css_class => 'rde-column_header',
+          :tag => :tr,
+          :cells => %w[Value Count Percent].map { |label|
+            OpenStruct.new(:value => label, :tag => :th, :css_class => 'rde-cell-label')
+          }
+        )
+        labels.each { |label|
+          count_stat = desc_stats.detect { |e| "#{ label }_count" == e[:label] }
+          percent_stat = desc_stats.detect { |e| "#{ label }_percent" == e[:label] }
+          table.rows << OpenStruct.new(
+            :css_class => 'rde-data_row',
+            :tag => :tr,
+            :cells => [
+              OpenStruct.new(:value => label, :tag => :td, :css_class => 'rde-cell-value'),
+              OpenStruct.new(
+                :value => count_stat[:value],
+                :ruby_formatter => count_stat[:ruby_formatter],
+                :tag => :td,
+                :css_class => 'rde-cell-value'
+              ),
+              OpenStruct.new(
+                :value => percent_stat[:value],
+                :ruby_formatter => percent_stat[:ruby_formatter],
+                :tag => :td,
+                :css_class => 'rde-cell-value'
+              ),
+            ]
+          )
+        }
+        table
+      end
+
       def axis_tick_format(values)
         %(function(d) { return d })
       end
 
+      # @param[Symbol, nil] label_val_key the hash key to use to get the label value during sort (sent to a,b)
+      # @param[DataSeries] data_series the ds that contains the uniq vals
+      # @param[Proc] value_sorter the sorting proc to use if not sorted numerically
+      # @return[Proc] a Proc that will be used by #sort
+      def label_sorter(label_val_key, data_series, value_sorter)
+        if data_series.uniq_vals.any? { |e| e.to_s =~ /^\d+/ }
+          # Sort numerical categories by key ASC
+          lambda { |a,b|
+            number_extractor = lambda { |val|
+              str = label_val_key ? val[label_val_key] : val
+              number = str.gsub(/^[^\d]*/, '').to_f
+              number += 1  if str =~ /^>/ # increase highest threshold by one for proper sorting
+              number
+            }
+            a_number = number_extractor.call(a)
+            b_number = number_extractor.call(b)
+            a_number <=> b_number
+          }
+        else
+          # Use provided value sorter
+          value_sorter
+        end
+      end
+
     end
   end
 end

data/lib/rails-data-explorer/data_type/quantitative.rb

@@ -6,12 +6,12 @@ class RailsDataExplorer
 
       def all_available_chart_types
         [
-          {
-            chart_class: Chart::BoxPlot,
-            chart_roles: [:y],
-            dimensions_count_min: 1,
-            dimensions_count_max: 1
-          },
+          # {
+          #   chart_class: Chart::BoxPlot,
+          #   chart_roles: [:y],
+          #   dimensions_count_min: 1,
+          #   dimensions_count_max: 1
+          # },
           {
             chart_class: Chart::HistogramQuantitative,
             chart_roles: [:x],
@@ -44,34 +44,54 @@ class RailsDataExplorer
       end
 
       def descriptive_statistics(values)
-        stats = ::DescriptiveStatistics::Stats.new(values)
+        non_nil_values = values.find_all { |e| !(e.nil? || Float::NAN == e) }
+        stats = ::DescriptiveStatistics::Stats.new(non_nil_values)
         ruby_formatters = {
-          :integer => Proc.new { |v| number_with_delimiter(v.round) },
-          :decimal => Proc.new { |v| number_with_precision(v, :precision => 3, :significant => true, :strip_insignificant_zeros => true, :delimiter => ',') },
-          :pass_through => Proc.new { |v| v },
+          :integer => Proc.new { |v|
+            v.nil? ? 'Null' : number_with_delimiter(v.round)
+          },
+          :decimal => Proc.new { |v|
+            case
+            when v.nil?
+              'Null'
+            when v.is_a?(Float) && v.nan?
+              'NaN'
+            else
+              number_with_precision(
+                v,
+                :precision => 3,
+                :significant => true,
+                :strip_insignificant_zeros => true,
+                :delimiter => ','
+              )
+            end
+          },
+          :pass_through => Proc.new { |v| (v.nil? || Float::NAN == v) ? 'NaN' : v },
         }
         [
           { :label => 'Min', :value => stats.min, :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '1%ile', :value => stats.value_from_percentile(1), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
+          { :label => '5%ile', :value => stats.value_from_percentile(5), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '10%ile', :value => stats.value_from_percentile(10), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '25%ile', :value => stats.value_from_percentile(25), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => 'Median', :value => stats.median, :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '75%ile', :value => stats.value_from_percentile(75), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '90%ile', :value => stats.value_from_percentile(90), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
+          { :label => '95%ile', :value => stats.value_from_percentile(95), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => '99%ile', :value => stats.value_from_percentile(99), :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
           { :label => 'Max', :value => stats.max, :ruby_formatter => ruby_formatters[:decimal], :table_row => 1 },
-          { :label => '', :value => '', :ruby_formatter => ruby_formatters[:pass_through], :table_row => 1 },
 
           { :label => 'Range', :value => stats.range, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Mean', :value => stats.mean, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Mode', :value => stats.mode, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Count', :value => values.length, :ruby_formatter => ruby_formatters[:integer], :table_row => 2 },
-          { :label => 'Sum', :value => values.inject(0) { |m,e| m += e }, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
+          { :label => 'Sum', :value => non_nil_values.inject(0) { |m,e| m += e }, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Variance', :value => stats.variance, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Std. dev.', :value => stats.standard_deviation, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Rel. std. dev.', :value => stats.relative_standard_deviation, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Skewness', :value => stats.skewness, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
           { :label => 'Kurtosis', :value => stats.kurtosis, :ruby_formatter => ruby_formatters[:decimal], :table_row => 2 },
+          { :label => '', :value => '', :ruby_formatter => ruby_formatters[:pass_through], :table_row => 2 },
         ]
       end
 
@@ -104,6 +124,15 @@ class RailsDataExplorer
         raise "Implement me in sub_class"
       end
 
+      def axis_scale(data_series)
+        # Log scales can't handle 0 values
+        if data_series.min_val > 0.0 && data_series.has_large_dynamic_range?
+          'd3.scale.log'
+        else
+          'd3.scale.linear'
+        end
+      end
+
     end
   end
 end

data/lib/rails-data-explorer/data_type/quantitative/temporal.rb

@@ -26,10 +26,11 @@ class RailsDataExplorer
         end
 
         def descriptive_statistics(values)
+          non_nil_values = values.find_all { |e| !e.nil? }
           ruby_formatter = Proc.new { |v| v.nil? ? '' : v.strftime('%a, %b %e, %Y, %l:%M:%S %p %Z') }
           [
-            { :label => 'Min', :value => values.min, :ruby_formatter => ruby_formatter },
-            { :label => 'Max', :value => values.max, :ruby_formatter => ruby_formatter },
+            { :label => 'Min', :value => non_nil_values.min, :ruby_formatter => ruby_formatter },
+            { :label => 'Max', :value => non_nil_values.max, :ruby_formatter => ruby_formatter },
             { :label => 'Count', :value => values.length, :ruby_formatter => Proc.new { |e| number_with_delimiter(e) } },
           ]
         end