blazer 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71bd121d2bace5612677090372e36ebdd89eb09398bd7510ba38c666351c1eb4
-  data.tar.gz: 4b109076d6af5aee545f4c19311b431feb68bc7e31f45e28704a99fe7692595b
+  metadata.gz: c9f173935d9c63e3537014420daf0af3017f6f9c51615d102fb6f0b4b3b45ba8
+  data.tar.gz: 8743bc1783f5181bd946d6263a70ac7ae02cbb26d3663b59c8915cdc9495e80f
 SHA512:
-  metadata.gz: 035bcb0b2252cd10514b5ab3f28c78b4ae6b6b6b534d99948f20815076661cf7fe85200debcf64ab11c6e5307fc24c326daea76f361f6c474e8216b69ebbbe35
-  data.tar.gz: 4838d7ee13b888d005602916d309825f716cd868287b0f1843aa92bf1a96d5f43168f22de3430c7b2ba6b4ab25fc71e1ee6bd6b45588729c0a1b44897d508052
+  metadata.gz: bfe97805e455f1ec96cc825563460a303006c6c379f9adfb3b24ed2f9f3b86f57f9600648a8fbc01a564ba62f78346582659dab9c121d239f91383211a3b8ad3
+  data.tar.gz: 52a1ce2a7a489105ba1c1ece23e412117de868bf6c50a07b968056b6ccbbe40d90031087c836f7831d04f12bd7d88593260a0a98bfdcb15659e1158c3fc070d6

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.4.0 (2020-12-15)
+
+- Added cohorts
+- Fixed broken routes for some applications
+- Forecasting and uploads are no longer experimental
+
 ## 2.3.1 (2020-11-23)
 
 - Improved column names for uploads

data/README.md

@@ -25,6 +25,7 @@ Blazer is also available as a [Docker image](https://github.com/ankane/blazer-do
 - [Charts](#charts)
 - [Dashboards](#dashboards)
 - [Checks](#checks)
+- [Cohorts](#cohorts)
 - [Anomaly Detection](#anomaly-detection)
 - [Forecasting](#forecasting)
 - [Uploads](#uploads)
@@ -388,6 +389,31 @@ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
 
 Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
 
+## Cohorts
+
+Create a cohort analysis from a simple SQL query. [Example](https://blazer.dokkuapp.com/queries/19-cohort-analysis-from-first-order)
+
+Create a query with the comment `/* cohort analysis */`. The result should have columns named `user_id` and `conversion_time` and optionally `cohort_time`.
+
+You can generate cohorts from the first conversion time:
+
+```sql
+/* cohort analysis */
+SELECT user_id, created_at AS conversion_time FROM orders
+```
+
+(the first conversion isn’t counted in the first time period with this format)
+
+Or from another time, like sign up:
+
+```sql
+/* cohort analysis */
+SELECT users.id AS user_id, orders.created_at AS conversion_time, users.created_at AS cohort_time
+FROM users LEFT JOIN orders ON orders.user_id = users.id
+```
+
+This feature requires PostgreSQL.
+
 ## Anomaly Detection
 
 Blazer supports two different approaches to anomaly detection.
@@ -456,7 +482,7 @@ Commit and deploy away. The first deploy may take a few minutes.
 
 ## Forecasting
 
-Blazer has experimental support for two different forecasting methods. [Example](https://blazer.dokkuapp.com/queries/18-forecast?forecast=t)
+Blazer supports for two different forecasting methods. [Example](https://blazer.dokkuapp.com/queries/18-forecast?forecast=t)
 
 A forecast link will appear for queries that return 2 columns with types timestamp and numeric.
 
@@ -492,7 +518,7 @@ forecasting: trend
 
 ## Uploads
 
-Blazer has experimental support for creating database tables from CSV files. [Example](https://blazer.dokkuapp.com/uploads)
+Creating database tables from CSV files. [Example](https://blazer.dokkuapp.com/uploads)
 
 Run:
 
@@ -657,8 +683,6 @@ data_sources:
 
 ### InfluxDB
 
-*Experimental*
-
 Add [influxdb](https://github.com/influxdata/influxdb-ruby) to your Gemfile and set:
 
 ```yml
@@ -692,8 +716,6 @@ data_sources:
 
 ### Neo4j
 
-*Experimental*
-
 Add [neo4j-core](https://github.com/neo4jrb/neo4j-core) to your Gemfile and set:
 
 ```yml
@@ -735,8 +757,6 @@ data_sources:
 
 ### Salesforce
 
-*Experimental*
-
 Add [restforce](https://github.com/restforce/restforce) to your Gemfile and set:
 
 ```yml
@@ -760,8 +780,6 @@ Supports [SOQL](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta
 
 ### Socrata Open Data API (SODA)
 
-*Experimental*
-
 Set:
 
 ```yml

data/app/assets/stylesheets/blazer/application.css

@@ -16,6 +16,10 @@ body {
   padding-bottom: 15px;
 }
 
+table.results-table {
+  margin-bottom: 0;
+}
+
 .results-table th {
   cursor: pointer;
 }

data/app/controllers/blazer/base_controller.rb

@@ -6,6 +6,8 @@ module Blazer
     skip_after_action(*filters, raise: false)
     skip_around_action(*filters, raise: false)
 
+    clear_helpers
+
     protect_from_forgery with: :exception
 
     if ENV["BLAZER_PASSWORD"]
@@ -65,6 +67,12 @@ module Blazer
         end
       end
 
+      def add_cohort_analysis_vars
+        @bind_vars << "cohort_period" unless @bind_vars.include?("cohort_period")
+        @smart_vars["cohort_period"] = ["day", "week", "month"]
+        params[:cohort_period] ||= "week"
+      end
+
       def parse_smart_variables(var, data_source)
         smart_var_data_source =
           ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_variables[var] }

data/app/controllers/blazer/dashboards_controller.rb

@@ -39,6 +39,8 @@ module Blazer
           @sql_errors << error if error
         end
       end
+
+      add_cohort_analysis_vars if @queries.any?(&:cohort_analysis?)
     end
 
     def edit

data/app/controllers/blazer/queries_controller.rb

@@ -74,6 +74,8 @@ module Blazer
       @query.update!(status: "active") if @query.try(:status) == "archived"
 
       Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+
+      add_cohort_analysis_vars if @query.cohort_analysis?
     end
 
     def edit
@@ -81,6 +83,8 @@ module Blazer
 
     def run
       @statement = params[:statement]
+      # before process_vars
+      @cohort_analysis = Query.new(statement: @statement).cohort_analysis?
       data_source = params[:data_source]
       process_vars(@statement, data_source)
       @only_chart = params[:only_chart]
@@ -89,6 +93,8 @@ module Blazer
       data_source = @query.data_source if @query && @query.data_source
       @data_source = Blazer.data_sources[data_source]
 
+      run_cohort_analysis if @cohort_analysis
+
       # ensure viewable
       if !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)
         render_forbidden
@@ -164,6 +170,7 @@ module Blazer
       @statement = @query.statement.dup
       process_vars(@statement, @query.data_source)
       Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+      @statement = cohort_analysis_statement(data_source, @statement) if @query.cohort_analysis?
       data_source.clear_cache(@statement)
       redirect_to query_path(@query, variable_params(@query))
     end
@@ -241,7 +248,6 @@ module Blazer
           end
         end
 
-        @filename = @query.name.parameterize if @query
         @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
 
         @boom = @result.boom if @result
@@ -268,6 +274,8 @@ module Blazer
           end
         end
 
+        render_cohort_analysis if @cohort_analysis && !@error
+
         respond_to do |format|
           format.html do
             render layout: false
@@ -352,5 +360,104 @@ module Blazer
       def blazer_run_id
         params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
       end
+
+      def run_cohort_analysis
+        unless @data_source.supports_cohort_analysis?
+          @cohort_error = "This data source does not support cohort analysis"
+        end
+
+        @show_cohort_rows = !params[:query_id] || @cohort_error
+
+        unless @show_cohort_rows
+          @statement = cohort_analysis_statement(@data_source, @statement)
+        end
+      end
+
+      def cohort_analysis_statement(data_source, statement)
+        @cohort_period = params["cohort_period"] || "week"
+        @cohort_period = "week" unless ["day", "week", "month"].include?(@cohort_period)
+
+        # for now
+        @conversion_period = @cohort_period
+        @cohort_days =
+          case @cohort_period
+          when "day"
+            1
+          when "week"
+            7
+          when "month"
+            30
+          end
+
+        data_source.cohort_analysis_statement(statement, period: @cohort_period, days: @cohort_days)
+      end
+
+      def render_cohort_analysis
+        if @show_cohort_rows
+          @cohort_analysis = false
+
+          @row_limit = 1000
+
+          # check results
+          unless @cohort_error
+            # check names
+            expected_columns = ["user_id", "conversion_time"]
+            missing_columns = expected_columns - @result.columns
+            if missing_columns.any?
+              @cohort_error = "Expected user_id and conversion_time columns"
+            end
+
+            # check types (user_id can be any type)
+            unless @cohort_error
+              column_types = @result.columns.zip(@result.column_types).to_h
+
+              if !column_types["cohort_time"].in?(["time", nil])
+                @cohort_error = "cohort_time must be time column"
+              elsif !column_types["conversion_time"].in?(["time", nil])
+                @cohort_error = "conversion_time must be time column"
+              end
+            end
+          end
+        else
+          @today = Blazer.time_zone.today
+          @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
+          @buckets = {}
+          @rows.each do |r|
+            @buckets[[r[0], r[1]]] = r[2]
+          end
+
+          @cohort_dates = []
+          current_date = @max_cohort_date
+          while current_date && current_date >= @min_cohort_date
+            @cohort_dates << current_date
+            current_date =
+              case @cohort_period
+              when "day"
+                current_date - 1
+              when "week"
+                current_date - 7
+              else
+                current_date.prev_month
+              end
+          end
+
+          num_cols = @cohort_dates.size
+          @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
+          rows = []
+          date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
+          @cohort_dates.each do |date|
+            row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
+
+            num_cols.times do |i|
+              if @today >= date + (@cohort_days * i)
+                row << (@buckets[[date, i + 1]] || 0)
+              end
+            end
+
+            rows << row
+          end
+          @rows = rows
+        end
+      end
   end
 end

data/app/models/blazer/query.rb

@@ -35,7 +35,13 @@ module Blazer
     end
 
     def variables
-      Blazer.extract_vars(statement)
+      variables = Blazer.extract_vars(statement)
+      variables += ["cohort_period"] if cohort_analysis?
+      variables
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
     end
   end
 end

data/app/views/blazer/dashboards/show.html.erb

@@ -39,7 +39,7 @@
     </div>
   </div>
   <script>
-    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true} %>
+    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true, cohort_period: params[:cohort_period]} %>
 
     runQuery(data, function (data) {
       $("#chart-<%= i %>").html(data)

data/app/views/blazer/queries/_caching.html.erb

@@ -0,0 +1,16 @@
+<% if @cached_at || @just_cached %>
+  <p class="text-muted" style="float: right;">
+    <% if @cached_at %>
+      Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
+    <% elsif params[:query_id] %>
+      Cached just now
+      <% if @data_source.cache_mode == "slow" %>
+        (over <%= "%g" % @data_source.cache_slow_threshold %>s)
+      <% end %>
+    <% end %>
+
+    <% if @query && params[:query_id] %>
+      <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
+    <% end %>
+  </p>
+<% end %>

data/app/views/blazer/queries/_cohorts.html.erb

@@ -0,0 +1,48 @@
+<% unless @only_chart %>
+  <%= render partial: "caching" %>
+  <p class="text-muted" style="margin-bottom: 10px;">
+    <%= pluralize(@rows.size, "cohort") %>
+  </p>
+<% end %>
+<% if @rows.any? %>
+  <div class="results-container">
+    <table class="table results-table">
+      <thead>
+        <tr>
+          <th style="min-width: 100px;">Cohort</th>
+          <% 12.times do |i| %>
+            <th style="width: 7.5%; text-align: right;"><%= @conversion_period.titleize %> <%= i + 1 %></th>
+          <% end %>
+        </tr>
+      </thead>
+      <tbody>
+        <% @rows.each do |row| %>
+          <tr>
+            <td>
+              <%= row[0] %>
+              <div style="font-size: 12px; color: #999;"><%= row[1] == 1 ? "1 user" : "#{number_with_delimiter(row[1])} users" %></div>
+            </td>
+            <% 12.times do |i| %>
+              <td style="text-align: right;">
+                <% num = row[i + 2] %>
+                <% if num %>
+                  <% denom = row[1] %>
+                  <% if denom > 0 %>
+                    <%= (100.0 * num / denom).round %>%
+                  <% else %>
+                    -
+                  <% end %>
+                  <div style="font-size: 12px; color: #999;"><%= number_with_delimiter(num) %></div>
+                <% else %>
+                  -
+                <% end %>
+              </td>
+            <% end %>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% elsif @only_chart %>
+  <p class="text-muted">No cohorts</p>
+<% end %>

data/app/views/blazer/queries/docs.html.erb

@@ -129,3 +129,9 @@
 </table>
 
 <p>Use the column name <code>target</code> to draw a line for goals.</p>
+
+<% if @data_source.supports_cohort_analysis? %>
+  <h2>Cohort Analysis</h2>
+
+  <p>Create a query with the comment <code>/* cohort analysis */</code>. The result should have columns named <code>user_id</code> and <code>conversion_time</code> and optionally <code>cohort_time</code>.</p>
+<% end %>

data/app/views/blazer/queries/run.html.erb

@@ -6,25 +6,20 @@
   <% else %>
     <div class="alert alert-info">Can’t preview queries with variables...yet!</div>
   <% end %>
+<% elsif @cohort_analysis %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% else %>
+    <%= render partial: "cohorts" %>
+  <% end %>
 <% else %>
   <% unless @only_chart %>
-    <% if @cached_at || @just_cached %>
-      <p class="text-muted" style="float: right;">
-        <% if @cached_at %>
-          Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
-        <% elsif params[:query_id] %>
-          Cached just now
-          <% if @data_source.cache_mode == "slow" %>
-            (over <%= "%g" % @data_source.cache_slow_threshold %>s)
-          <% end %>
-        <% end %>
-
-        <% if @query && params[:query_id] %>
-          <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
-        <% end %>
-      </p>
-    <% end %>
+    <%= render partial: "caching" %>
     <p class="text-muted" style="margin-bottom: 10px;">
+      <% if @row_limit && @rows.size > @row_limit %>
+        First
+        <% @rows = @rows.first(@row_limit) %>
+      <% end %>
       <%= pluralize(@rows.size, "row") %>
 
       <% @checks.select(&:state).each do |check| %>
@@ -43,6 +38,9 @@
   <% if @forecast_error %>
     <div class="alert alert-danger"><%= @forecast_error %></div>
   <% end %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% end %>
   <% if @rows.any? %>
     <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
@@ -147,7 +145,7 @@
         <% elsif @columns == ["PLAN"] && @data_source.adapter == "druid" %>
           <pre><code><%= @rows[0][0] %></code></pre>
         <% else %>
-          <table class="table results-table" style="margin-bottom: 0;">
+          <table class="table results-table">
             <thead>
               <tr>
                 <% @columns.each_with_index do |key, i| %>

data/app/views/blazer/queries/show.html.erb

@@ -14,7 +14,7 @@
         <%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
 
         <% if !@error && @success %>
-          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
+          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast], cohort_period: params[:cohort_period]), params: {statement: @statement}, class: "btn btn-primary" %>
         <% end %>
       </div>
     </div>

data/lib/blazer/adapters/base_adapter.rb

@@ -43,6 +43,14 @@ module Blazer
         true # optional
       end
 
+      def supports_cohort_analysis?
+        false # optional
+      end
+
+      def cohort_analysis_statement(statement, period:, days:)
+        # optional
+      end
+
       protected
 
       def settings

data/lib/blazer/adapters/sql_adapter.rb

@@ -122,6 +122,47 @@ module Blazer
         !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
       end
 
+      def supports_cohort_analysis?
+        postgresql?
+      end
+
+      # TODO treat date columns as already in time zone
+      def cohort_analysis_statement(statement, period:, days:)
+        raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+        cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+
+        # WITH not an optimization fence in Postgres 12+
+        statement = <<~SQL
+          WITH query AS (
+            #{statement}
+          ),
+          cohorts AS (
+            SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+            WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+            GROUP BY 1
+          )
+          SELECT
+            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+            0 AS bucket,
+            COUNT(DISTINCT cohorts.user_id)
+          FROM cohorts GROUP BY 1
+          UNION ALL
+          SELECT
+            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+            CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int AS bucket,
+            COUNT(DISTINCT query.user_id)
+          FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+          WHERE query.conversion_time IS NOT NULL
+          AND query.conversion_time >= cohorts.cohort_time
+          #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+          GROUP BY 1, 2
+        SQL
+        tzname = Blazer.time_zone.tzinfo.name
+        params = [statement, period, tzname, period, tzname, days.to_i * 86400]
+        connection_model.send(:sanitize_sql_array, params)
+      end
+
       protected
 
       def select_all(statement, params = [])

data/lib/blazer/data_source.rb

@@ -6,7 +6,7 @@ module Blazer
 
     attr_reader :id, :settings
 
-    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
+    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
 
     def initialize(id, settings)
       @id = id

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.3.1"
+  VERSION = "2.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.4.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-24 00:00:00.000000000 Z
+date: 2020-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -170,6 +170,8 @@ files:
 - app/views/blazer/dashboards/edit.html.erb
 - app/views/blazer/dashboards/new.html.erb
 - app/views/blazer/dashboards/show.html.erb
+- app/views/blazer/queries/_caching.html.erb
+- app/views/blazer/queries/_cohorts.html.erb
 - app/views/blazer/queries/_form.html.erb
 - app/views/blazer/queries/docs.html.erb
 - app/views/blazer/queries/edit.html.erb