blazer 2.2.7 → 2.4.1

data/app/views/blazer/dashboards/_form.html.erb

@@ -31,7 +31,7 @@
 <% end %>
 
 <script>
-  <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+  <%= blazer_js_var "queries", Blazer::Query.active.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
   <%= blazer_js_var "dashboardQueries", @queries || @dashboard.dashboard_queries.order(:position).map(&:query) %>
 
   var app = new Vue({

data/app/views/blazer/dashboards/show.html.erb

@@ -39,7 +39,7 @@
     </div>
   </div>
   <script>
-    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true} %>
+    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true, cohort_period: params[:cohort_period]} %>
 
     runQuery(data, function (data) {
       $("#chart-<%= i %>").html(data)

data/app/views/blazer/queries/_caching.html.erb

@@ -0,0 +1,16 @@
+<% if @cached_at || @just_cached %>
+  <p class="text-muted" style="float: right;">
+    <% if @cached_at %>
+      Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
+    <% elsif params[:query_id] %>
+      Cached just now
+      <% if @data_source.cache_mode == "slow" %>
+        (over <%= "%g" % @data_source.cache_slow_threshold %>s)
+      <% end %>
+    <% end %>
+
+    <% if @query && params[:query_id] %>
+      <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
+    <% end %>
+  </p>
+<% end %>

data/app/views/blazer/queries/_cohorts.html.erb

@@ -0,0 +1,48 @@
+<% unless @only_chart %>
+  <%= render partial: "caching" %>
+  <p class="text-muted" style="margin-bottom: 10px;">
+    <%= pluralize(@rows.size, "cohort") %>
+  </p>
+<% end %>
+<% if @rows.any? %>
+  <div class="results-container">
+    <table class="table results-table">
+      <thead>
+        <tr>
+          <th style="min-width: 100px;">Cohort</th>
+          <% 12.times do |i| %>
+            <th style="width: 7.5%; text-align: right;"><%= @conversion_period.titleize %> <%= i + 1 %></th>
+          <% end %>
+        </tr>
+      </thead>
+      <tbody>
+        <% @rows.each do |row| %>
+          <tr>
+            <td>
+              <%= row[0] %>
+              <div style="font-size: 12px; color: #999;"><%= row[1] == 1 ? "1 user" : "#{number_with_delimiter(row[1])} users" %></div>
+            </td>
+            <% 12.times do |i| %>
+              <td style="text-align: right;">
+                <% num = row[i + 2] %>
+                <% if num %>
+                  <% denom = row[1] %>
+                  <% if denom > 0 %>
+                    <%= (100.0 * num / denom).round %>%
+                  <% else %>
+                    -
+                  <% end %>
+                  <div style="font-size: 12px; color: #999;"><%= number_with_delimiter(num) %></div>
+                <% else %>
+                  -
+                <% end %>
+              </td>
+            <% end %>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% elsif @only_chart %>
+  <p class="text-muted">No cohorts</p>
+<% end %>

data/app/views/blazer/queries/docs.html.erb

@@ -129,3 +129,9 @@
 </table>
 
 <p>Use the column name <code>target</code> to draw a line for goals.</p>
+
+<% if @data_source.supports_cohort_analysis? %>
+  <h2>Cohort Analysis</h2>
+
+  <p>Create a query with the comment <code>/* cohort analysis */</code>. The result should have columns named <code>user_id</code> and <code>conversion_time</code> and optionally <code>cohort_time</code>.</p>
+<% end %>

data/app/views/blazer/queries/home.html.erb

@@ -19,6 +19,9 @@
         </button>
         <ul class="dropdown-menu">
           <li><%= link_to "Checks", checks_path %></li>
+          <% if Blazer.uploads? %>
+            <li><%= link_to "Uploads", uploads_path %></li>
+          <% end %>
           <li role="separator" class="divider"></li>
           <li><%= link_to "New Dashboard", new_dashboard_path %></li>
           <li><%= link_to "New Check", new_check_path %></li>

data/app/views/blazer/queries/run.html.erb

@@ -6,25 +6,20 @@
   <% else %>
     <div class="alert alert-info">Can’t preview queries with variables...yet!</div>
   <% end %>
+<% elsif @cohort_analysis %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% else %>
+    <%= render partial: "cohorts" %>
+  <% end %>
 <% else %>
   <% unless @only_chart %>
-    <% if @cached_at || @just_cached %>
-      <p class="text-muted" style="float: right;">
-        <% if @cached_at %>
-          Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
-        <% elsif params[:query_id] %>
-          Cached just now
-          <% if @data_source.cache_mode == "slow" %>
-            (over <%= "%g" % @data_source.cache_slow_threshold %>s)
-          <% end %>
-        <% end %>
-
-        <% if @query && params[:query_id] %>
-          <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
-        <% end %>
-      </p>
-    <% end %>
+    <%= render partial: "caching" %>
     <p class="text-muted" style="margin-bottom: 10px;">
+      <% if @row_limit && @rows.size > @row_limit %>
+        First
+        <% @rows = @rows.first(@row_limit) %>
+      <% end %>
       <%= pluralize(@rows.size, "row") %>
 
       <% @checks.select(&:state).each do |check| %>
@@ -43,6 +38,9 @@
   <% if @forecast_error %>
     <div class="alert alert-danger"><%= @forecast_error %></div>
   <% end %>
+  <% if @cohort_error %>
+    <div class="alert alert-info"><%= @cohort_error %></div>
+  <% end %>
   <% if @rows.any? %>
     <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
@@ -147,7 +145,7 @@
         <% elsif @columns == ["PLAN"] && @data_source.adapter == "druid" %>
           <pre><code><%= @rows[0][0] %></code></pre>
         <% else %>
-          <table class="table results-table" style="margin-bottom: 0;">
+          <table class="table results-table">
             <thead>
               <tr>
                 <% @columns.each_with_index do |key, i| %>

data/app/views/blazer/queries/show.html.erb

@@ -14,7 +14,7 @@
         <%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
 
         <% if !@error && @success %>
-          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
+          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast], cohort_period: params[:cohort_period]), params: {statement: @statement}, class: "btn btn-primary" %>
         <% end %>
       </div>
     </div>

data/app/views/blazer/uploads/_form.html.erb

@@ -0,0 +1,27 @@
+<%= form_for @upload, html: {class: "small-form"} do |f| %>
+  <% if @upload.errors.any? %>
+    <div class="alert alert-danger"><%= @upload.errors.full_messages.first %></div>
+  <% elsif !@upload.persisted? %>
+    <p>Create a database table from a CSV file. The table will be created in the <code><%= Blazer.settings["uploads"]["schema"] %></code> schema.</p>
+  <% end %>
+
+  <div class="form-group">
+    <%= f.label :table %>
+    <%= f.text_field :table, class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :description %>
+    <%= f.text_area :description, placeholder: "Optional", style: "height: 60px;", class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :file %>
+    <%= f.file_field :file, accept: "text/csv", style: "margin-top: 6px; margin-bottom: 21px;" %>
+  </div>
+  <p>
+    <% if @upload.persisted? %>
+      <%= link_to "Delete", upload_path(@upload), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+    <% end %>
+    <%= f.submit "Save", class: "btn btn-success" %>
+    <%= link_to "Back", :back, class: "btn btn-link" %>
+  </p>
+<% end %>

data/app/views/blazer/uploads/edit.html.erb

@@ -0,0 +1,3 @@
+<% blazer_title "Edit Upload" %>
+
+<%= render partial: "form" %>

data/app/views/blazer/uploads/index.html.erb

@@ -0,0 +1,55 @@
+<% blazer_title "Uploads" %>
+
+<div id="header">
+  <div class="pull-right" style="line-height: 34px;">
+    <div class="btn-group">
+      <%= link_to "New Upload", new_upload_path, class: "btn btn-info" %>
+      <button type="button" class="btn btn-info dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+        <span class="caret"></span>
+        <span class="sr-only">Toggle Dropdown</span>
+      </button>
+      <ul class="dropdown-menu">
+        <li><%= link_to "Home", root_path %></li>
+        <li><%= link_to "Checks", checks_path %></li>
+        <li role="separator" class="divider"></li>
+        <li><%= link_to "New Query", new_query_path %></li>
+        <li><%= link_to "New Dashboard", new_dashboard_path %></li>
+        <li><%= link_to "New Check", new_check_path %></li>
+      </ul>
+    </div>
+  </div>
+
+  <input id="search" type="text" placeholder="Start typing a table or person" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
+
+<table id="uploads" class="table">
+  <thead>
+    <tr>
+      <th>Table</th>
+      <th style="width: 60%;"></th>
+      <% if Blazer.user_class %>
+        <th style="width: 20%; text-align: right;">Mastermind</th>
+      <% end%>
+    </tr>
+  </thead>
+  <tbody>
+    <% @uploads.each do |upload| %>
+      <tr>
+        <td><%= link_to upload.table, edit_upload_path(upload) %></td>
+        <td><%= truncate(upload.description, length: 100, separator: " ") %></td>
+        <% if Blazer.user_class %>
+          <td class="creator"><%= blazer_user && upload.creator == blazer_user ? "You" : upload.creator.try(Blazer.user_name) %></td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<script>
+  $("#search").on("keyup", function() {
+    var value = $(this).val().toLowerCase()
+    $("#uploads tbody tr").filter( function() {
+      $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+    })
+  }).focus()
+</script>

data/app/views/blazer/uploads/new.html.erb

@@ -0,0 +1,3 @@
+<% blazer_title "New Upload" %>
+
+<%= render partial: "form" %>

data/config/routes.rb

@@ -16,5 +16,10 @@ Blazer::Engine.routes.draw do
     post :refresh, on: :member
   end
 
+  if Blazer.uploads?
+    resources :uploads do
+    end
+  end
+
   root to: "queries#home"
 end

data/lib/blazer.rb

@@ -18,12 +18,14 @@ require "blazer/adapters/cassandra_adapter"
 require "blazer/adapters/drill_adapter"
 require "blazer/adapters/druid_adapter"
 require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/hive_adapter"
 require "blazer/adapters/influxdb_adapter"
 require "blazer/adapters/mongodb_adapter"
 require "blazer/adapters/neo4j_adapter"
 require "blazer/adapters/presto_adapter"
 require "blazer/adapters/salesforce_adapter"
 require "blazer/adapters/soda_adapter"
+require "blazer/adapters/spark_adapter"
 require "blazer/adapters/sql_adapter"
 require "blazer/adapters/snowflake_adapter"
 
@@ -32,6 +34,7 @@ require "blazer/engine"
 
 module Blazer
   class Error < StandardError; end
+  class UploadError < Error; end
   class TimeoutNotSupported < Error; end
 
   class << self
@@ -206,6 +209,23 @@ module Blazer
     slack_webhook_url.present?
   end
 
+  def self.uploads?
+    settings.key?("uploads")
+  end
+
+  def self.uploads_connection
+    raise "Empty url for uploads" unless settings.dig("uploads", "url")
+    Blazer::UploadsConnection.connection
+  end
+
+  def self.uploads_schema
+    settings.dig("uploads", "schema") || "uploads"
+  end
+
+  def self.uploads_table_name(name)
+    uploads_connection.quote_table_name("#{uploads_schema}.#{name}")
+  end
+
   def self.adapters
     @adapters ||= {}
   end
@@ -221,11 +241,13 @@ Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
 Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
 Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
 Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
 Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
 Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
 Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
 Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
 Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
+Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
 Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
 Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/adapters/base_adapter.rb

@@ -43,6 +43,14 @@ module Blazer
         true # optional
       end
 
+      def supports_cohort_analysis?
+        false # optional
+      end
+
+      def cohort_analysis_statement(statement, period:, days:)
+        # optional
+      end
+
       protected
 
       def settings

data/lib/blazer/adapters/hive_adapter.rb

@@ -0,0 +1,45 @@
+module Blazer
+  module Adapters
+    class HiveAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          result = client.execute("#{statement} /*#{comment}*/")
+          columns = result.any? ? result.first.keys : []
+          rows = result.map(&:values)
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        client.execute("SHOW TABLES").map { |r| r["tab_name"] }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      protected
+
+      def client
+        @client ||= begin
+          uri = URI.parse(settings["url"])
+          Hexspace::Client.new(
+            host: uri.host,
+            port: uri.port,
+            username: uri.user,
+            password: uri.password,
+            database: uri.path.sub(/\A\//, ""),
+            mode: uri.scheme.to_sym
+          )
+        end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/spark_adapter.rb

@@ -0,0 +1,9 @@
+module Blazer
+  module Adapters
+    class SparkAdapter < HiveAdapter
+      def tables
+        client.execute("SHOW TABLES").map { |r| r["tableName"] }
+      end
+    end
+  end
+end

data/lib/blazer/adapters/sql_adapter.rb

@@ -27,7 +27,7 @@ module Blazer
             result = select_all("#{statement} /*#{comment}*/")
             columns = result.columns
             result.rows.each do |untyped_row|
-              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
+              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
             end
           end
         rescue => e
@@ -122,6 +122,67 @@ module Blazer
         !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
       end
 
+      def supports_cohort_analysis?
+        postgresql? || mysql?
+      end
+
+      # TODO treat date columns as already in time zone
+      def cohort_analysis_statement(statement, period:, days:)
+        raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+        cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+        tzname = Blazer.time_zone.tzinfo.name
+
+        if mysql?
+          time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
+          case period
+          when "day"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname]
+          when "week"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname, tzname]
+          else
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
+            date_params = [tzname]
+          end
+          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS INTEGER)"
+        else
+          date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
+          date_params = [period, tzname]
+          bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
+        end
+
+        # WITH not an optimization fence in Postgres 12+
+        statement = <<~SQL
+          WITH query AS (
+            #{statement}
+          ),
+          cohorts AS (
+            SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+            WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+            GROUP BY 1
+          )
+          SELECT
+            #{date_sql} AS period,
+            0 AS bucket,
+            COUNT(DISTINCT cohorts.user_id)
+          FROM cohorts GROUP BY 1
+          UNION ALL
+          SELECT
+            #{date_sql} AS period,
+            #{bucket_sql} AS bucket,
+            COUNT(DISTINCT query.user_id)
+          FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+          WHERE query.conversion_time IS NOT NULL
+          AND query.conversion_time >= cohorts.cohort_time
+          #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+          GROUP BY 1, 2
+        SQL
+        params = [statement] + date_params + date_params + [days.to_i * 86400]
+        connection_model.send(:sanitize_sql_array, params)
+      end
+
       protected
 
       def select_all(statement, params = [])
@@ -165,6 +226,8 @@ module Blazer
             "public"
           elsif sqlserver?
             "dbo"
+          elsif connection_model.respond_to?(:connection_db_config)
+            connection_model.connection_db_config.database
           else
             connection_model.connection_config[:database]
           end