blazer 2.2.6 → 2.4.0

data/app/models/blazer/uploads_connection.rb

@@ -0,0 +1,7 @@
+module Blazer
+  class UploadsConnection < ActiveRecord::Base
+    self.abstract_class = true
+
+    establish_connection Blazer.settings["uploads"]["url"] if Blazer.uploads?
+  end
+end

data/app/views/blazer/_nav.html.erb

@@ -6,6 +6,9 @@
   </button>
   <ul class="dropdown-menu">
     <li><%= link_to "Checks", checks_path %></li>
+    <% if Blazer.uploads? %>
+      <li><%= link_to "Uploads", uploads_path %></li>
+    <% end %>
     <li role="separator" class="divider"></li>
     <li><%= link_to "New Query", new_query_path %></li>
     <li><%= link_to "New Dashboard", new_dashboard_path %></li>

data/app/views/blazer/_variables.html.erb

@@ -42,6 +42,7 @@
               singleDatePicker: true,
               locale: {format: format},
               autoUpdateInput: false,
+              autoApply: true,
               startDate: input.val().length > 0 ? moment.tz(input.val(), timeZone) : now
             })
             // hack to start with empty date
@@ -95,7 +96,8 @@
             },
             startDate: dateStr(29),
             endDate: dateStr(),
-            opens: "right"
+            opens: "right",
+            alwaysShowCalendars: true
           },
           function(start, end) {
             setTimeInputs(start, end)

data/app/views/blazer/checks/_form.html.erb

@@ -13,7 +13,7 @@
       <%= f.select :query_id, [], {include_blank: true} %>
     </div>
     <script>
-      <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+      <%= blazer_js_var "queries", Blazer::Query.active.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
       <%= blazer_js_var "items", [@check.query_id].compact %>
 
       $("#check_query_id").selectize({options: queries, items: items, highlight: false, maxOptions: 100}).parents(".hide").removeClass("hide");

data/app/views/blazer/checks/index.html.erb

@@ -10,6 +10,9 @@
       </button>
       <ul class="dropdown-menu">
         <li><%= link_to "Home", root_path %></li>
+        <% if Blazer.uploads? %>
+          <li><%= link_to "Uploads", uploads_path %></li>
+        <% end %>
         <li role="separator" class="divider"></li>
         <li><%= link_to "New Query", new_query_path %></li>
         <li><%= link_to "New Dashboard", new_dashboard_path %></li>

data/app/views/blazer/dashboards/_form.html.erb

@@ -31,7 +31,7 @@
 <% end %>
 
 <script>
-  <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+  <%= blazer_js_var "queries", Blazer::Query.active.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
   <%= blazer_js_var "dashboardQueries", @queries || @dashboard.dashboard_queries.order(:position).map(&:query) %>
 
   var app = new Vue({

data/app/views/blazer/dashboards/show.html.erb

@@ -39,7 +39,7 @@
     </div>
   </div>
   <script>
-    <%= blazer_js_var "data", {statement: query.statement, query_id: query.id, data_source: query.data_source, only_chart: true} %>
+    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true, cohort_period: params[:cohort_period]} %>
 
     runQuery(data, function (data) {
       $("#chart-<%= i %>").html(data)

data/app/views/blazer/queries/_caching.html.erb

@@ -0,0 +1,16 @@
+<% if @cached_at || @just_cached %>
+  <p class="text-muted" style="float: right;">
+    <% if @cached_at %>
+      Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
+    <% elsif params[:query_id] %>
+      Cached just now
+      <% if @data_source.cache_mode == "slow" %>
+        (over <%= "%g" % @data_source.cache_slow_threshold %>s)
+      <% end %>
+    <% end %>
+
+    <% if @query && params[:query_id] %>
+      <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
+    <% end %>
+  </p>
+<% end %>

data/app/views/blazer/queries/_cohorts.html.erb

@@ -0,0 +1,48 @@
+<% unless @only_chart %>
+  <%= render partial: "caching" %>
+  <p class="text-muted" style="margin-bottom: 10px;">
+    <%= pluralize(@rows.size, "cohort") %>
+  </p>
+<% end %>
+<% if @rows.any? %>
+  <div class="results-container">
+    <table class="table results-table">
+      <thead>
+        <tr>
+          <th style="min-width: 100px;">Cohort</th>
+          <% 12.times do |i| %>
+            <th style="width: 7.5%; text-align: right;"><%= @conversion_period.titleize %> <%= i + 1 %></th>
+          <% end %>
+        </tr>
+      </thead>
+      <tbody>
+        <% @rows.each do |row| %>
+          <tr>
+            <td>
+              <%= row[0] %>
+              <div style="font-size: 12px; color: #999;"><%= row[1] == 1 ? "1 user" : "#{number_with_delimiter(row[1])} users" %></div>
+            </td>
+            <% 12.times do |i| %>
+              <td style="text-align: right;">
+                <% num = row[i + 2] %>
+                <% if num %>
+                  <% denom = row[1] %>
+                  <% if denom > 0 %>
+                    <%= (100.0 * num / denom).round %>%
+                  <% else %>
+                    -
+                  <% end %>
+                  <div style="font-size: 12px; color: #999;"><%= number_with_delimiter(num) %></div>
+                <% else %>
+                  -
+                <% end %>
+              </td>
+            <% end %>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% elsif @only_chart %>
+  <p class="text-muted">No cohorts</p>
+<% end %>

data/app/views/blazer/queries/docs.html.erb

@@ -129,3 +129,9 @@
 </table>
 
 <p>Use the column name <code>target</code> to draw a line for goals.</p>
+
+<% if @data_source.supports_cohort_analysis? %>
+  <h2>Cohort Analysis</h2>
+
+  <p>Create a query with the comment <code>/* cohort analysis */</code>. The result should have columns named <code>user_id</code> and <code>conversion_time</code> and optionally <code>cohort_time</code>.</p>
+<% end %>

data/app/views/blazer/queries/home.html.erb

@@ -19,6 +19,9 @@
         </button>
         <ul class="dropdown-menu">
           <li><%= link_to "Checks", checks_path %></li>
+          <% if Blazer.uploads? %>
+            <li><%= link_to "Uploads", uploads_path %></li>
+          <% end %>
           <li role="separator" class="divider"></li>
           <li><%= link_to "New Dashboard", new_dashboard_path %></li>
           <li><%= link_to "New Check", new_check_path %></li>

data/app/views/blazer/queries/run.html.erb

@@ -6,25 +6,20 @@
   <% else %>
     <div class="alert alert-info">Can’t preview queries with variables...yet!</div>
   <% end %>
+<% elsif @cohort_analysis %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% else %>
+    <%= render partial: "cohorts" %>
+  <% end %>
 <% else %>
   <% unless @only_chart %>
-    <% if @cached_at || @just_cached %>
-      <p class="text-muted" style="float: right;">
-        <% if @cached_at %>
-          Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
-        <% elsif params[:query_id] %>
-          Cached just now
-          <% if @data_source.cache_mode == "slow" %>
-            (over <%= "%g" % @data_source.cache_slow_threshold %>s)
-          <% end %>
-        <% end %>
-
-        <% if @query && params[:query_id] %>
-          <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
-        <% end %>
-      </p>
-    <% end %>
+    <%= render partial: "caching" %>
     <p class="text-muted" style="margin-bottom: 10px;">
+      <% if @row_limit && @rows.size > @row_limit %>
+        First
+        <% @rows = @rows.first(@row_limit) %>
+      <% end %>
       <%= pluralize(@rows.size, "row") %>
 
       <% @checks.select(&:state).each do |check| %>
@@ -43,6 +38,9 @@
   <% if @forecast_error %>
     <div class="alert alert-danger"><%= @forecast_error %></div>
   <% end %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% end %>
   <% if @rows.any? %>
     <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
@@ -107,13 +105,13 @@
       </script>
     <% elsif chart_type == "line" %>
       <% chart_data = @columns[1..-1].each_with_index.map{ |k, i| {name: blazer_series_name(k), data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} } %>
-      <%= line_chart chart_data, chart_options %>
+      <%= line_chart chart_data, **chart_options %>
     <% elsif chart_type == "line2" %>
-      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
+      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, **chart_options %>
     <% elsif chart_type == "pie" %>
-      <%= pie_chart @rows.map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[1]] }, chart_options %>
+      <%= pie_chart @rows.map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[1]] }, **chart_options %>
     <% elsif chart_type == "bar" %>
-      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, chart_options %>
+      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, **chart_options %>
     <% elsif chart_type == "bar2" %>
       <% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %>
       <% labels = first_20.map { |r| r[0] }.uniq %>
@@ -123,7 +121,7 @@
           <% first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } %>
         <% end %>
       <% end %>
-      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, chart_options %>
+      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, **chart_options %>
     <% elsif chart_type == "scatter" %>
       <%= scatter_chart @rows, xtitle: @columns[0], ytitle: @columns[1], **chart_options %>
     <% elsif @only_chart %>
@@ -147,7 +145,7 @@
         <% elsif @columns == ["PLAN"] && @data_source.adapter == "druid" %>
           <pre><code><%= @rows[0][0] %></code></pre>
         <% else %>
-          <table class="table results-table" style="margin-bottom: 0;">
+          <table class="table results-table">
             <thead>
               <tr>
                 <% @columns.each_with_index do |key, i| %>

data/app/views/blazer/queries/show.html.erb

@@ -14,7 +14,7 @@
         <%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
 
         <% if !@error && @success %>
-          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
+          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast], cohort_period: params[:cohort_period]), params: {statement: @statement}, class: "btn btn-primary" %>
         <% end %>
       </div>
     </div>

data/app/views/blazer/uploads/_form.html.erb

@@ -0,0 +1,27 @@
+<%= form_for @upload, html: {class: "small-form"} do |f| %>
+  <% if @upload.errors.any? %>
+    <div class="alert alert-danger"><%= @upload.errors.full_messages.first %></div>
+  <% elsif !@upload.persisted? %>
+    <p>Create a database table from a CSV file. The table will be created in the <code><%= Blazer.settings["uploads"]["schema"] %></code> schema.</p>
+  <% end %>
+
+  <div class="form-group">
+    <%= f.label :table %>
+    <%= f.text_field :table, class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :description %>
+    <%= f.text_area :description, placeholder: "Optional", style: "height: 60px;", class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :file %>
+    <%= f.file_field :file, accept: "text/csv", style: "margin-top: 6px; margin-bottom: 21px;" %>
+  </div>
+  <p>
+    <% if @upload.persisted? %>
+      <%= link_to "Delete", upload_path(@upload), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+    <% end %>
+    <%= f.submit "Save", class: "btn btn-success" %>
+    <%= link_to "Back", :back, class: "btn btn-link" %>
+  </p>
+<% end %>

data/app/views/blazer/uploads/edit.html.erb

@@ -0,0 +1,3 @@
+<% blazer_title "Edit Upload" %>
+
+<%= render partial: "form" %>

data/app/views/blazer/uploads/index.html.erb

@@ -0,0 +1,55 @@
+<% blazer_title "Uploads" %>
+
+<div id="header">
+  <div class="pull-right" style="line-height: 34px;">
+    <div class="btn-group">
+      <%= link_to "New Upload", new_upload_path, class: "btn btn-info" %>
+      <button type="button" class="btn btn-info dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+        <span class="caret"></span>
+        <span class="sr-only">Toggle Dropdown</span>
+      </button>
+      <ul class="dropdown-menu">
+        <li><%= link_to "Home", root_path %></li>
+        <li><%= link_to "Checks", checks_path %></li>
+        <li role="separator" class="divider"></li>
+        <li><%= link_to "New Query", new_query_path %></li>
+        <li><%= link_to "New Dashboard", new_dashboard_path %></li>
+        <li><%= link_to "New Check", new_check_path %></li>
+      </ul>
+    </div>
+  </div>
+
+  <input id="search" type="text" placeholder="Start typing a table or person" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
+
+<table id="uploads" class="table">
+  <thead>
+    <tr>
+      <th>Table</th>
+      <th style="width: 60%;"></th>
+      <% if Blazer.user_class %>
+        <th style="width: 20%; text-align: right;">Mastermind</th>
+      <% end%>
+    </tr>
+  </thead>
+  <tbody>
+    <% @uploads.each do |upload| %>
+      <tr>
+        <td><%= link_to upload.table, edit_upload_path(upload) %></td>
+        <td><%= truncate(upload.description, length: 100, separator: " ") %></td>
+        <% if Blazer.user_class %>
+          <td class="creator"><%= blazer_user && upload.creator == blazer_user ? "You" : upload.creator.try(Blazer.user_name) %></td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<script>
+  $("#search").on("keyup", function() {
+    var value = $(this).val().toLowerCase()
+    $("#uploads tbody tr").filter( function() {
+      $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+    })
+  }).focus()
+</script>

data/app/views/blazer/uploads/new.html.erb

@@ -0,0 +1,3 @@
+<% blazer_title "New Upload" %>
+
+<%= render partial: "form" %>

data/config/routes.rb

@@ -16,5 +16,10 @@ Blazer::Engine.routes.draw do
     post :refresh, on: :member
   end
 
+  if Blazer.uploads?
+    resources :uploads do
+    end
+  end
+
   root to: "queries#home"
 end

data/lib/blazer.rb

@@ -32,6 +32,7 @@ require "blazer/engine"
 
 module Blazer
   class Error < StandardError; end
+  class UploadError < Error; end
   class TimeoutNotSupported < Error; end
 
   class << self
@@ -206,6 +207,23 @@ module Blazer
     slack_webhook_url.present?
   end
 
+  def self.uploads?
+    settings.key?("uploads")
+  end
+
+  def self.uploads_connection
+    raise "Empty url for uploads" unless settings.dig("uploads", "url")
+    Blazer::UploadsConnection.connection
+  end
+
+  def self.uploads_schema
+    settings.dig("uploads", "schema") || "uploads"
+  end
+
+  def self.uploads_table_name(name)
+    uploads_connection.quote_table_name("#{uploads_schema}.#{name}")
+  end
+
   def self.adapters
     @adapters ||= {}
   end

data/lib/blazer/adapters/base_adapter.rb

@@ -43,6 +43,14 @@ module Blazer
         true # optional
       end
 
+      def supports_cohort_analysis?
+        false # optional
+      end
+
+      def cohort_analysis_statement(statement, period:, days:)
+        # optional
+      end
+
       protected
 
       def settings

data/lib/blazer/adapters/sql_adapter.rb

@@ -27,7 +27,7 @@ module Blazer
             result = select_all("#{statement} /*#{comment}*/")
             columns = result.columns
             result.rows.each do |untyped_row|
-              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
+              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
             end
           end
         rescue => e
@@ -122,6 +122,47 @@ module Blazer
         !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
       end
 
+      def supports_cohort_analysis?
+        postgresql?
+      end
+
+      # TODO treat date columns as already in time zone
+      def cohort_analysis_statement(statement, period:, days:)
+        raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+        cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+
+        # WITH not an optimization fence in Postgres 12+
+        statement = <<~SQL
+          WITH query AS (
+            #{statement}
+          ),
+          cohorts AS (
+            SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+            WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+            GROUP BY 1
+          )
+          SELECT
+            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+            0 AS bucket,
+            COUNT(DISTINCT cohorts.user_id)
+          FROM cohorts GROUP BY 1
+          UNION ALL
+          SELECT
+            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+            CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int AS bucket,
+            COUNT(DISTINCT query.user_id)
+          FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+          WHERE query.conversion_time IS NOT NULL
+          AND query.conversion_time >= cohorts.cohort_time
+          #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+          GROUP BY 1, 2
+        SQL
+        tzname = Blazer.time_zone.tzinfo.name
+        params = [statement, period, tzname, period, tzname, days.to_i * 86400]
+        connection_model.send(:sanitize_sql_array, params)
+      end
+
       protected
 
       def select_all(statement, params = [])