blazer 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ebec2b00f0b97033272541d144c26ad98379be5
-  data.tar.gz: db8f009f5cb2c12d4a960c2ce5cab96f33e287b0
+  metadata.gz: 63faea5b7fa9ad8d4365931c80851e33d80553e8
+  data.tar.gz: 30b6a8d2dd631512a5cd82a6fabe119e7e9d5963
 SHA512:
-  metadata.gz: e267ad7981a3ed4d8774e1ef255ebecea5d43edfe97addbe07f2a95a43c1af8344e69834a9978563400294f6ff5012061e0a832cf0bb99d50acf67aa64a1d331
-  data.tar.gz: cdea706a25e45acea0cfb660d31bfb6f3d2163319c781ce91bca213b1df0efa69124f339b374d4b7703ba275f486b8c757043889237e3eb6982767538b8ec140
+  metadata.gz: 69a08a57180385ed46ae3bd776fa3e67bf885ed9845dc7508f70d1d7d883d940120c4b03167800726fe6d0dcb728518e4cfbdc32b6fc2647813d381eea0d1670
+  data.tar.gz: 681fdbe806cfc7675335802166199ed11124e6c443c4d8da31e279c47b48ed847a5d2061d24d657d5c343c47a284b929dc93f4188ca7fa7fcae4d47bcd225d67

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.5.0
+
+- Added new bar chart format
+- Added anomaly detection checks
+- Added `async` option for polling
+
 ## 1.4.0
 
 - Added `slow` cache mode

data/README.md

@@ -254,12 +254,20 @@ SELECT date_trunc('week', created_at), gender, COUNT(*) FROM users GROUP BY 1, 2
 
 ### Column Chart
 
+There are also two ways to generate column charts.
+
 2+ columns - string, numeric(s) - [Example](https://blazerme.herokuapp.com/queries/2-top-genres)
 
 ```sql
 SELECT gender, COUNT(*) FROM users GROUP BY 1
 ```
 
+3 columns - string, string, numeric
+
+```sql
+SELECT gender, zip_code, COUNT(*) FROM users GROUP BY 1, 2
+```
+
 ### Maps
 
 Columns named `latitude` and `longitude` or `lat` and `lon` - [Example](https://blazerme.herokuapp.com/queries/11-airports-in-pacific-time-zone)
@@ -270,6 +278,14 @@ SELECT name, latitude, longitude FROM cities
 
 To enable, get an access token from [Mapbox](https://www.mapbox.com/) and set `ENV["MAPBOX_ACCESS_TOKEN"]`.
 
+### Targets
+
+Use the column name `target` to draw a line for goals.
+
+```sql
+SELECT date_trunc('week', created_at), COUNT(*) AS new_users, 100000 AS target FROM users GROUP BY 1
+```
+
 ## Dashboards
 
 Create a dashboard with multiple queries. [Example](https://blazerme.herokuapp.com/dashboards/1-movielens)
@@ -290,6 +306,25 @@ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
 
 Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
 
+## Anomaly Detection
+
+Anomaly detection is supported thanks to Twitter’s [AnomalyDetection](https://github.com/twitter/AnomalyDetection) library.
+
+First, [install R](https://cloud.r-project.org/). Then, run:
+
+```R
+install.packages("devtools")
+devtools::install_github("twitter/AnomalyDetection")
+```
+
+And add to `config/blazer.yml`:
+
+```yml
+anomaly_checks: true
+```
+
+If upgrading from version 1.4 or below, also follow the [upgrade instructions](#15).
+
 ## Data Sources
 
 Blazer supports multiple data sources :tada:
@@ -330,6 +365,27 @@ For an easy way to group by day, week, month, and more with correct time zones,
 
 ## Upgrading
 
+### 1.5
+
+To take advantage of the anomaly detection, create a migration
+
+```sh
+rails g migration upgrade_blazer_to_1_5
+```
+
+with:
+
+```ruby
+add_column(:blazer_checks, :check_type, :string)
+add_column(:blazer_checks, :message, :text)
+commit_db_transaction
+
+Blazer::Check.reset_column_information
+
+Blazer::Check.where(invert: true).update_all(check_type: "missing_data")
+Blazer::Check.where(check_type: nil).update_all(check_type: "bad_data")
+```
+
 ### 1.3
 
 To take advantage of the latest features, create a migration

data/app/assets/javascripts/blazer/application.js

@@ -31,7 +31,17 @@ function runQuery(data, success, error) {
     method: "POST",
     data: data,
     dataType: "html"
-  }).done(success).fail( function(jqXHR, textStatus, errorThrown) {
+  }).done( function (d) {
+    if (d[0] == "{") {
+      var response = $.parseJSON(d);
+      data.blazer = response;
+      setTimeout( function () {
+        runQuery(data, success, error);
+      }, 1000);
+    } else {
+      success(d);
+    }
+  }).fail( function(jqXHR, textStatus, errorThrown) {
     var message = (typeof errorThrown === "string") ? errorThrown : errorThrown.message;
     error(message);
   });

data/app/assets/javascripts/blazer/selectize.js

@@ -2236,7 +2236,7 @@
      */
     registerOption: function(data) {
       var key = hash_key(data[this.settings.valueField]);
-      if ((!key || this.options.hasOwnProperty(key)) && !this.settings.allowEmptyOption) return false;
+      if (!key || this.options.hasOwnProperty(key)) return false;
       data.$order = data.$order || ++this.order;
       this.options[key] = data;
       return key;

data/app/assets/stylesheets/blazer/application.css

@@ -100,3 +100,31 @@ input.search:focus {
 .vars {
   color: #ccc;
 }
+
+.check-state {
+  font-weight: bold;
+}
+
+.check-state a {
+  color: inherit;
+}
+
+.check-state.failing {
+  color: red;
+}
+
+.check-state.passing {
+  color: #5cb85c;
+}
+
+.check-state.error {
+  color: #666;
+}
+
+.check-state.timed_out {
+  color: orange;
+}
+
+.check-state.disabled {
+  color: #000;
+}

data/app/controllers/blazer/base_controller.rb

@@ -51,7 +51,7 @@ module Blazer
     helper_method :extract_vars
 
     def variable_params
-      params.except(:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id).permit!
+      params.except(:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer).permit!
     end
     helper_method :variable_params
 

data/app/controllers/blazer/checks_controller.rb

@@ -9,7 +9,7 @@ module Blazer
     end
 
     def new
-      @check = Blazer::Check.new
+      @check = Blazer::Check.new(query_id: params[:query_id])
     end
 
     def create
@@ -40,12 +40,13 @@ module Blazer
 
     def run
       @query = @check.query
+      redirect_to query_path(@query)
     end
 
     private
 
     def check_params
-      params.require(:check).permit(:query_id, :emails, :invert, :schedule)
+      params.require(:check).permit(:query_id, :emails, :invert, :check_type, :schedule)
     end
 
     def set_check

data/app/controllers/blazer/dashboards_controller.rb

@@ -37,9 +37,9 @@ module Blazer
         @data_sources.each do |data_source|
           query = data_source.smart_variables[var]
           if query
-            columns, rows, error, cached_at = data_source.run_statement(query)
-            ((@smart_vars[var] ||= []).concat(rows.map { |v| v.reverse })).uniq!
-            @sql_errors << error if error
+            result = data_source.run_statement(query)
+            ((@smart_vars[var] ||= []).concat(result.rows.map { |v| v.reverse })).uniq!
+            @sql_errors << result.error if result.error
           end
         end
       end

data/app/controllers/blazer/queries_controller.rb

@@ -4,8 +4,11 @@ module Blazer
 
     def home
       set_queries(1000)
+
+      @dashboards = Blazer::Dashboard.order(:name)
+      @dashboards = @dashboards.includes(:creator) if Blazer.user_class
       @dashboards =
-        Blazer::Dashboard.order(:name).map do |d|
+        @dashboards.map do |d|
           {
             name: "<strong>#{view_context.link_to(d.name, d)}</strong>",
             creator: blazer_user && d.try(:creator) == blazer_user ? "You" : d.try(:creator).try(Blazer.user_name),
@@ -51,9 +54,9 @@ module Blazer
       @bind_vars.each do |var|
         query = data_source.smart_variables[var]
         if query
-          columns, rows, error, cached_at = data_source.run_statement(query)
-          @smart_vars[var] = rows.map { |v| v.reverse }
-          @sql_errors << error if error
+          result = data_source.run_statement(query)
+          @smart_vars[var] = result.rows.map { |v| v.reverse }
+          @sql_errors << result.error if result.error
         end
       end
 
@@ -68,25 +71,68 @@ module Blazer
       data_source = params[:data_source]
       process_vars(@statement, data_source)
       @only_chart = params[:only_chart]
+      @run_id = blazer_params[:run_id]
+      @query = Query.find_by(id: params[:query_id]) if params[:query_id]
+      data_source = @query.data_source if @query && @query.data_source
+      @data_source = Blazer.data_sources[data_source]
+
+      if @run_id
+        @timestamp = blazer_params[:timestamp].to_i
+
+        @result = @data_source.run_results(@run_id)
+        @success = !@result.nil?
+
+        if @success
+          @data_source.delete_results(@run_id)
+          @columns = @result.columns
+          @rows = @result.rows
+          @error = @result.error
+          @just_cached = !@result.error && @result.cached_at.present?
+          @cached_at = nil
+          params[:data_source] = nil
+          render_run
+        elsif Time.now > Time.at(@timestamp + (@data_source.timeout || 120).to_i)
+          # timed out
+          @error = Blazer::TIMEOUT_MESSAGE
+          @rows = []
+          @columns = []
+          render_run
+        else
+          continue_run
+        end
+      elsif @success
+        @run_id = Blazer.async ? SecureRandom.uuid : nil
+
+        options = {user: blazer_user, query: @query, refresh_cache: params[:check], run_id: @run_id}
+        if Blazer.async && request.format.symbol != :csv
+          result = []
+          Blazer::RunStatementJob.perform_async(result, @data_source, @statement, options)
+          wait_start = Time.now
+          loop do
+            sleep(0.02)
+            break if result.any? || Time.now - wait_start > 3
+          end
+          @result = result.first
+        else
+          @result = @data_source.run_main_statement(@statement, options)
+        end
 
-      if @success
-        @query = Query.find_by(id: params[:query_id]) if params[:query_id]
-
-        data_source = @query.data_source if @query && @query.data_source
-        @data_source = Blazer.data_sources[data_source]
-
-        @columns, @rows, @error, @cached_at, @just_cached = @data_source.run_main_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])
+        if @result
+          @data_source.delete_results(@run_id) if @run_id
 
-        render_run
-      end
+          @columns = @result.columns
+          @rows = @result.rows
+          @error = @result.error
+          @cached_at = @result.cached_at
+          @just_cached = @result.just_cached
 
-      respond_to do |format|
-        format.html do
-          render layout: false
-        end
-        format.csv do
-          send_data csv_data(@columns, @rows, @data_source), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
+          render_run
+        else
+          @timestamp = Time.now.to_i
+          continue_run
         end
+      else
+        render layout: false
       end
     end
 
@@ -126,7 +172,13 @@ module Blazer
 
     private
 
+    def continue_run
+      render json: {run_id: @run_id, timestamp: @timestamp}, status: :accepted
+    end
+
     def render_run
+      @checks = @query ? @query.checks : []
+
       @first_row = @rows.first || []
       @column_types = []
       if @rows.any?
@@ -145,17 +197,9 @@ module Blazer
       end
 
       @filename = @query.name.parameterize if @query
-      @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }
+      @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
 
-      @boom = {}
-      @columns.each_with_index do |key, i|
-        query = @data_source.smart_columns[key]
-        if query
-          values = @rows.map { |r| r[i] }.compact.uniq
-          columns, rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
-          @boom[key] = Hash[rows.map { |k, v| [k.to_s, v] }]
-        end
-      end
+      @boom = @result.boom
 
       @linked_columns = @data_source.linked_columns
 
@@ -176,6 +220,15 @@ module Blazer
             end
         end
       end
+
+      respond_to do |format|
+        format.html do
+          render layout: false
+        end
+        format.csv do
+          send_data csv_data(@columns, @rows, @data_source), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
+        end
+      end
     end
 
     def set_queries(limit = nil)
@@ -215,6 +268,10 @@ module Blazer
       params.require(:query).permit(:name, :description, :statement, :data_source)
     end
 
+    def blazer_params
+      params[:blazer] || {}
+    end
+
     def csv_data(columns, rows, data_source)
       CSV.generate do |csv|
         csv << columns

data/app/helpers/blazer/base_helper.rb

@@ -16,23 +16,6 @@ module Blazer
       end
     end
 
-    def blazer_column_types(columns, rows, boom)
-      columns.each_with_index.map do |k, i|
-        v = (rows.find { |r| r[i] } || {})[i]
-        if boom[k]
-          "string"
-        elsif v.is_a?(Numeric)
-          "numeric"
-        elsif v.is_a?(Time) || v.is_a?(Date)
-          "time"
-        elsif v.nil?
-          nil
-        else
-          "string"
-        end
-      end
-    end
-
     def blazer_maps?
       ENV["MAPBOX_ACCESS_TOKEN"].present?
     end

data/app/mailers/blazer/check_mailer.rb

@@ -10,7 +10,7 @@ module Blazer
       @state_was = state_was
       @rows_count = rows_count
       @error = error
-      mail to: check.emails, subject: "Check #{state.titleize}: #{check.query.name}"
+      mail to: check.emails, reply_to: check.emails, subject: "Check #{state.titleize}: #{check.query.name}"
     end
 
     def failing_checks(email, checks)

data/app/models/blazer/check.rb

@@ -1,32 +1,57 @@
 module Blazer
   class Check < ActiveRecord::Base
+    belongs_to :creator, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s) if Blazer.user_class
     belongs_to :query
 
     validates :query_id, presence: true
 
+    before_validation :set_state
+
+    def set_state
+      self.state ||= "new"
+    end
+
     def split_emails
       emails.to_s.downcase.split(",").map(&:strip)
     end
 
-    def update_state(rows, error)
-      invert = respond_to?(:invert) && self.invert
+    def update_state(result)
+      check_type =
+        if respond_to?(:check_type)
+          self.check_type
+        elsif respond_to?(:invert)
+          invert ? "missing_data" : "bad_data"
+        else
+          "bad_data"
+        end
+
+      message = result.error
+
       self.state =
-        if error
-          if error == Blazer::TIMEOUT_MESSAGE
-            "timed out"
-          else
+        if result.timed_out?
+          "timed out"
+        elsif result.error
+          "error"
+        elsif check_type == "anomaly"
+          anomaly, message = result.detect_anomaly
+          if anomaly.nil?
             "error"
+          elsif anomaly
+            "failing"
+          else
+            "passing"
           end
-        elsif rows.any?
-          invert ? "passing" : "failing"
+        elsif result.rows.any?
+          check_type == "missing_data" ? "passing" : "failing"
         else
-          invert ? "failing" : "passing"
+          check_type == "missing_data" ? "failing" : "passing"
         end
 
       self.last_run_at = Time.now if respond_to?(:last_run_at=)
+      self.message = message if respond_to?(:message=)
 
       if respond_to?(:timeouts=)
-        if state == "timed out"
+        if result.timed_out?
           self.timeouts += 1
           self.state = "disabled" if timeouts >= 3
         else
@@ -36,7 +61,7 @@ module Blazer
 
       # do not notify on creation, except when not passing
       if (state_was || state != "passing") && state != state_was && emails.present?
-        Blazer::CheckMailer.state_change(self, state, state_was, rows.size, error).deliver_later
+        Blazer::CheckMailer.state_change(self, state, state_was, result.rows.size, message).deliver_later
       end
       save! if changed?
     end

data/app/models/blazer/dashboard.rb

@@ -1,5 +1,6 @@
 module Blazer
   class Dashboard < ActiveRecord::Base
+    belongs_to :creator, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s) if Blazer.user_class
     has_many :dashboard_queries, dependent: :destroy
     has_many :queries, through: :dashboard_queries
 

data/app/views/blazer/_nav.html.erb

@@ -10,6 +10,7 @@
     <li role="separator" class="divider"></li>
     <li><%= link_to "New Query", new_query_path %></li>
     <li><%= link_to "New Dashboard", new_dashboard_path %></li>
-    <li><%= link_to "New Check", new_check_path %></li>
+    <% check_params = @query ? {query_id: @query.id} : {} %>
+    <li><%= link_to "New Check", new_check_path(check_params) %></li>
   </ul>
 </div>

data/app/views/blazer/checks/_form.html.erb

@@ -10,14 +10,29 @@
   <div class="form-group">
     <%= f.label :query_id, "Query" %>
     <div class="hide">
-      <%= f.select :query_id, Blazer::Query.named.order(:name).map { |q| [q.name, q.id] }, {include_blank: true} %>
+      <%= f.select :query_id, [], {include_blank: true} %>
     </div>
     <script>
-      $("#check_query_id").selectize().parents(".hide").removeClass("hide");
+      var queries = <%= blazer_json_escape(Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} }.to_json).html_safe %>;
+      var items = <%= blazer_json_escape([@check.query_id].compact.to_json).html_safe %>;
+
+      $("#check_query_id").selectize({options: queries, items: items, highlight: false, maxOptions: 100}).parents(".hide").removeClass("hide");
     </script>
   </div>
 
-  <% if @check.respond_to?(:invert) %>
+  <% if @check.respond_to?(:check_type) %>
+    <div class="form-group">
+      <%= f.label :check_type, "Alert if" %>
+      <div class="hide">
+        <% check_options = [["Any results (bad data)", "bad_data"], ["No results (missing data)", "missing_data"]] %>
+        <% check_options << ["Anomaly (most recent data point)", "anomaly"] if Blazer.anomaly_checks %>
+        <%= f.select :check_type, check_options %>
+      </div>
+      <script>
+        $("#check_check_type").selectize({}).parent().removeClass("hide");
+      </script>
+    </div>
+  <% elsif @check.respond_to?(:invert) %>
     <div class="form-group">
       <%= f.label :invert, "Fails if" %>
       <div class="hide">

data/app/views/blazer/checks/index.html.erb

@@ -3,7 +3,6 @@
 <p style="float: right;"><%= link_to "New Check", new_check_path, class: "btn btn-info" %></p>
 <%= render partial: "blazer/nav" %>
 
-<% colors = {failing: "red", passing: "#5cb85c", error: "#666", timed_out: "orange", disabled: "#000"} %>
 <table class="table">
   <thead>
     <tr>
@@ -20,7 +19,7 @@
         <td><%= link_to check.query.name, check.query %></td>
         <td>
           <% if check.state %>
-            <small style="font-weight: bold; color: <%= colors[check.state.parameterize("_").to_sym] %>;"><%= check.state.upcase %></small>
+            <small class="check-state <%= check.state.parameterize("_") %>"><%= check.state.upcase %></small>
           <% end %>
         </td>
         <td><%= check.schedule if check.respond_to?(:schedule) %></td>
@@ -33,7 +32,7 @@
         </td>
         <td style="text-align: right; padding: 1px;">
           <%= link_to "Edit", edit_check_path(check), class: "btn btn-info" %>
-          <%= link_to "Run Now", run_check_path(check), class: "btn btn-primary" %>
+          <%= link_to "Run Now", run_check_path(check), target: "_blank", class: "btn btn-primary" %>
         </td>
       </tr>
     <% end %>

data/app/views/blazer/dashboards/_form.html.erb

@@ -38,10 +38,11 @@ li:hover .glyphicon-remove {
   <div class="form-group">
     <%= f.label :query_id, "Add Chart" %>
     <div class="hide">
-      <%= select_tag :query_id, options_for_select(Blazer::Query.named.order(:name).map { |q| [q.name, q.id] }), {include_blank: true, placeholder: "Select chart"} %>
+      <%= select_tag :query_id, nil, {include_blank: true, placeholder: "Select chart"} %>
     </div>
     <script>
-      $("#query_id").selectize().parents(".hide").removeClass("hide");
+      var queries = <%= blazer_json_escape(Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} }.to_json).html_safe %>;
+      $("#query_id").selectize({options: queries, highlight: false, maxOptions: 100}).parents(".hide").removeClass("hide");
       $("#query_id").change( function () {
         var $option = $(this).find("option:selected");
         if ($option.val() !== "") {

data/app/views/blazer/dashboards/show.html.erb

@@ -40,7 +40,7 @@
 <% end %>
 
 <% if @bind_vars.any? %>
-  <form id="bind" method="get" action="<%= url_for(params) %>" class="form-inline" style="margin-bottom: 10px;">
+  <form id="bind" method="get" action="<%= dashboard_path(@dashboard, variable_params) %>" class="form-inline" style="margin-bottom: 10px;">
     <% date_vars = ["start_time", "end_time"] %>
     <% if (date_vars - @bind_vars).empty? %>
       <% @bind_vars = @bind_vars - date_vars %>

data/app/views/blazer/queries/_form.html.erb

@@ -14,9 +14,6 @@
       <div class="form-group text-right">
         <div class="pull-left" style="margin-top: 6px;">
           <%= link_to "Back", :back %>
-          <% if Blazer.data_sources.size == 1 %>
-            <span class="text-muted" style="margin-left: 20px;"> Use {start_time} and {end_time} for time ranges</span>
-          <% end %>
         </div>
         <%= f.select :data_source, Blazer.data_sources.values.map { |ds| [ds.name, ds.id] }, {}, class: ("hide" if Blazer.data_sources.size == 1), style: "width: 140px;" %>
         <div id="tables" style="display: inline-block; width: 260px; margin-right: 10px;" class="hide">
@@ -103,6 +100,12 @@
     editor.resize();
   };
 
+  function getSQL() {
+    var selectedText = editor.getSelectedText();
+    var text = selectedText.length == 0 ? editor.getValue() : selectedText;
+    return text.replace(/\n/g, "\r\n");
+  }
+
   editor.getSession().on("change", adjustHeight);
   adjustHeight();
   $("#editor").show();
@@ -125,7 +128,7 @@
       xhr.abort();
     }
 
-    var data = $.extend({}, params, {statement: editor.getValue().replace(/\n/g, "\r\n"), data_source: $("#query_data_source").val()});
+    var data = $.extend({}, params, {statement: getSQL(), data_source: $("#query_data_source").val()});
 
     xhr = runQuery(data, function (data) {
       $("#results").html(data);

data/app/views/blazer/queries/run.html.erb

@@ -24,12 +24,22 @@
         <% end %>
       </p>
     <% end %>
-    <p class="text-muted"><%= pluralize(@rows.size, "row") %></p>
+    <p class="text-muted">
+      <%= pluralize(@rows.size, "row") %>
+
+      <% @checks.select(&:state).each do |check| %>
+        &middot; <small class="check-state <%= check.state.parameterize("_") %>"><%= link_to check.state.upcase, edit_check_path(check) %></small>
+        <% if check.try(:message) %>
+          &middot; <%= check.message %>
+        <% end %>
+      <% end %>
+    </p>
   <% end %>
   <% if @rows.any? %>
     <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
-    <% column_types = blazer_column_types(@columns, @rows, @boom) %>
+    <% column_types = @result.column_types %>
+    <% chart_type = @result.chart_type %>
     <% chart_options = {id: chart_id, min: nil} %>
     <% series_library = {} %>
     <% target_index = @columns.index { |k| k.downcase == "target" } %>
@@ -65,12 +75,22 @@
         featureLayer.setGeoJSON(geojson);
         map.fitBounds(featureLayer.getBounds());
       </script>
-    <% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>
+    <% elsif chart_type == "line" %>
       <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: k, data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
-    <% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>
+    <% elsif chart_type == "line2" %>
       <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
-    <% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>
+    <% elsif chart_type == "bar" %>
       <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
+    <% elsif chart_type == "bar2" %>
+      <% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %>
+      <% labels = first_20.map { |r| r[0] }.uniq %>
+      <% series = first_20.map { |r| r[1] }.uniq %>
+      <% labels.each do |l| %>
+        <% series.each do |s| %>
+          <% first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } %>
+        <% end %>
+      <% end %>
+      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, id: chart_id %>
     <% elsif @only_chart %>
       <% if @rows.size == 1 && @rows.first.size == 1 %>
         <% v = @rows.first.first %>
@@ -96,7 +116,7 @@
                 <% @columns.each_with_index do |key, i| %>
                   <% type = @column_types[i] %>
                   <th style="width: <%= header_width %>%;" data-sort="<%= type %>">
-                    <div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
+                    <div style="min-width: <%= @min_width_types.include?(i) ? 180 : 60 %>px;">
                       <%= key %>
                     </div>
                   </th>

data/app/views/blazer/queries/show.html.erb

@@ -52,7 +52,7 @@
 <% end %>
 
 <% if @bind_vars.any? %>
-  <form id="bind" method="get" action="<%= url_for(params) %>" class="form-inline" style="margin-bottom: 10px;">
+  <form id="bind" method="get" action="<%= query_path(@query, variable_params) %>" class="form-inline" style="margin-bottom: 10px;">
     <% date_vars = ["start_time", "end_time"] %>
     <% if (date_vars - @bind_vars).empty? %>
       <% @bind_vars = @bind_vars - date_vars %>

data/lib/blazer.rb

@@ -1,10 +1,11 @@
 require "csv"
 require "yaml"
 require "chartkick"
+require "safely/core"
 require "blazer/version"
 require "blazer/data_source"
+require "blazer/result"
 require "blazer/engine"
-require "safely/core"
 
 module Blazer
   class Error < StandardError; end
@@ -21,14 +22,19 @@ module Blazer
     attr_accessor :cache
     attr_accessor :transform_statement
     attr_accessor :check_schedules
+    attr_accessor :anomaly_checks
+    attr_accessor :async
   end
   self.audit = true
   self.user_name = :name
   self.check_schedules = ["5 minutes", "1 hour", "1 day"]
+  self.anomaly_checks = false
+  self.async = false
 
   TIMEOUT_MESSAGE = "Query timed out :("
   TIMEOUT_ERRORS = [
     "canceling statement due to statement timeout", # postgres
+    "canceling statement due to conflict with recovery", # postgres
     "cancelled on user's request", # redshift
     "canceled on user's request", # redshift
     "system requested abort", # redshift
@@ -68,6 +74,7 @@ module Blazer
     checks = Blazer::Check.includes(:query)
     checks = checks.where(schedule: schedule) if schedule
     checks.find_each do |check|
+      next if check.state == "disabled"
       Safely.safely { run_check(check) }
     end
   end
@@ -84,13 +91,13 @@ module Blazer
       Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
 
       while tries <= 3
-        columns, rows, error, cached_at = data_source.run_statement(statement, refresh_cache: true)
-        if error == Blazer::TIMEOUT_MESSAGE
+        result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
+        if result.timed_out?
           Rails.logger.info "[blazer timeout] query=#{check.query.name}"
           tries += 1
           sleep(10)
-        elsif error.to_s.start_with?("PG::ConnectionBad")
-          data_sources[check.query.data_source].reconnect
+        elsif result.error.to_s.start_with?("PG::ConnectionBad")
+          data_source.reconnect
           Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
           tries += 1
           sleep(10)
@@ -98,15 +105,15 @@ module Blazer
           break
         end
       end
-      check.update_state(rows, error)
+      check.update_state(result)
       # TODO use proper logfmt
-      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"
+      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
 
       instrument[:statement] = statement
       instrument[:data_source] = data_source
       instrument[:state] = check.state
-      instrument[:rows] = rows.try(:size)
-      instrument[:error] = error
+      instrument[:rows] = result.rows.try(:size)
+      instrument[:error] = result.error
       instrument[:tries] = tries
     end
   end

data/lib/blazer/data_source.rb

@@ -110,42 +110,52 @@ module Blazer
       end
 
       start_time = Time.now
-      columns, rows, error, cached_at, just_cached = run_statement(statement, options.merge(with_just_cached: true))
+      result = run_statement(statement, options)
       duration = Time.now - start_time
 
       if Blazer.audit
         audit.duration = duration if audit.respond_to?(:duration=)
-        audit.error = error if audit.respond_to?(:error=)
-        audit.timed_out = error == Blazer::TIMEOUT_MESSAGE if audit.respond_to?(:timed_out=)
-        audit.cached = cached_at.present? if audit.respond_to?(:cached=)
-        if !cached_at && duration >= 10
+        audit.error = result.error if audit.respond_to?(:error=)
+        audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
+        audit.cached = result.cached? if audit.respond_to?(:cached=)
+        if !result.cached? && duration >= 10
           audit.cost = cost(statement) if audit.respond_to?(:cost=)
         end
         audit.save! if audit.changed?
       end
 
-      if query && error != Blazer::TIMEOUT_MESSAGE
+      if query && !result.timed_out?
         query.checks.each do |check|
-          check.update_state(rows, error)
+          check.update_state(result)
         end
       end
 
-      [columns, rows, error, cached_at, just_cached]
+      result
+    end
+
+    def read_cache(cache_key)
+      value = Blazer.cache.read(cache_key)
+      if value
+        Blazer::Result.new(self, *Marshal.load(value), nil)
+      end
+    end
+
+    def run_results(run_id)
+      read_cache(run_cache_key(run_id))
+    end
+
+    def delete_results(run_id)
+      Blazer.cache.delete(run_cache_key(run_id))
     end
 
     def run_statement(statement, options = {})
-      columns = nil
-      rows = nil
-      error = nil
-      cached_at = nil
-      just_cached = false
-      cache_key = self.cache_key(statement) if cache
-      if cache && !options[:refresh_cache]
-        value = Blazer.cache.read(cache_key)
-        columns, rows, cached_at = Marshal.load(value) if value
+      run_id = options[:run_id]
+      result = nil
+      if cache_mode != "off" && !options[:refresh_cache]
+        result = read_cache(statement_cache_key(statement))
       end
 
-      unless rows
+      unless result
         comment = "blazer"
         if options[:user].respond_to?(:id)
           comment << ",user_id:#{options[:user].id}"
@@ -157,20 +167,29 @@ module Blazer
         if options[:query].respond_to?(:id)
           comment << ",query_id:#{options[:query].id}"
         end
-        columns, rows, error, just_cached = run_statement_helper(statement, comment)
+        if options[:check]
+          comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"
+        end
+        result = run_statement_helper(statement, comment, options[:run_id])
       end
 
-      output = [columns, rows, error, cached_at]
-      output << just_cached if options[:with_just_cached]
-      output
+      result
     end
 
     def clear_cache(statement)
-      Blazer.cache.delete(cache_key(statement))
+      Blazer.cache.delete(statement_cache_key(statement))
+    end
+
+    def cache_key(key)
+      (["blazer", "v4"] + key).join("/")
+    end
+
+    def statement_cache_key(statement)
+      cache_key(["statement", id, Digest::MD5.hexdigest(statement)])
     end
 
-    def cache_key(statement)
-      ["blazer", "v3", id, Digest::MD5.hexdigest(statement)].join("/")
+    def run_cache_key(run_id)
+      cache_key(["run", run_id])
     end
 
     def schemas
@@ -179,8 +198,8 @@ module Blazer
     end
 
     def tables
-      columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))
-      rows.map(&:first)
+      result = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))
+      result.rows.map(&:first)
     end
 
     def postgresql?
@@ -201,7 +220,7 @@ module Blazer
 
     protected
 
-    def run_statement_helper(statement, comment)
+    def run_statement_helper(statement, comment, run_id)
       columns = []
       rows = []
       error = nil
@@ -238,14 +257,24 @@ module Blazer
       end
 
       cache_data = nil
-      if !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
-        cache_data = Marshal.dump([columns, rows, Time.now]) rescue nil
-        if cache_data
-          Blazer.cache.write(cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
+      cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
+      if cache || run_id
+        cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
+      end
+
+      if cache && cache_data
+        Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
+      end
+
+      if run_id
+        unless cache_data
+          error = "Error storing the results of this query :("
+          cache_data = Marshal.dump([[], [], error, nil])
         end
+        Blazer.cache.write(run_cache_key(run_id), cache_data, expires_in: 30.seconds)
       end
 
-      [columns, rows, error, !cache_data.nil?]
+      Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
     end
 
     def adapter_name

data/lib/blazer/detect_anomalies.R

@@ -0,0 +1,14 @@
+tryCatch({
+  library(AnomalyDetection)
+
+  args <- commandArgs(trailingOnly = TRUE)
+
+  con <- textConnection(args[1])
+  data <- read.csv(con, stringsAsFactors = FALSE)
+  data$timestamp <- as.POSIXct(data$timestamp)
+
+  res = AnomalyDetectionTs(data, direction = "both", alpha = 0.05)
+  write.csv(res$anoms)
+}, error = function (e) {
+  write.csv(geterrmessage())
+})

data/lib/blazer/engine.rb

@@ -29,6 +29,12 @@ module Blazer
       end
 
       Blazer.cache ||= Rails.cache
+
+      Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
+      Blazer.async = Blazer.settings["async"] || false
+      if Blazer.async
+        require "blazer/run_statement_job"
+      end
     end
   end
 end

data/lib/blazer/result.rb

@@ -0,0 +1,147 @@
+module Blazer
+  class Result
+    attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached
+
+    def initialize(data_source, columns, rows, error, cached_at, just_cached)
+      @data_source = data_source
+      @columns = columns
+      @rows = rows
+      @error = error
+      @cached_at = cached_at
+      @just_cached = just_cached
+    end
+
+    def timed_out?
+      error == Blazer::TIMEOUT_MESSAGE
+    end
+
+    def cached?
+      cached_at.present?
+    end
+
+    def boom
+      @boom ||= begin
+        boom = {}
+        columns.each_with_index do |key, i|
+          query = data_source.smart_columns[key]
+          if query
+            values = rows.map { |r| r[i] }.compact.uniq
+            result = data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+            boom[key] = Hash[result.rows.map { |k, v| [k.to_s, v] }]
+          end
+        end
+        boom
+      end
+    end
+
+    def column_types
+      @column_types ||= begin
+        columns.each_with_index.map do |k, i|
+          v = (rows.find { |r| r[i] } || {})[i]
+          if boom[k]
+            "string"
+          elsif v.is_a?(Numeric)
+            "numeric"
+          elsif v.is_a?(Time) || v.is_a?(Date)
+            "time"
+          elsif v.nil?
+            nil
+          else
+            "string"
+          end
+        end
+      end
+    end
+
+    def chart_type
+      @chart_type ||= begin
+        if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "line"
+        elsif column_types == ["time", "string", "numeric"]
+          "line2"
+        elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "bar"
+        elsif column_types == ["string", "string", "numeric"]
+          "bar2"
+        end
+      end
+    end
+
+    def detect_anomaly
+      anomaly = nil
+      message = nil
+
+      if rows.empty?
+        message = "No data"
+      else
+        if chart_type == "line" || chart_type == "line2"
+          series = []
+
+          if chart_type == "line"
+            columns[1..-1].each_with_index.each do |k, i|
+              series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
+            end
+          else
+            rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
+              series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
+            end
+          end
+
+          current_series = nil
+          begin
+            anomalies = []
+            series.each do |s|
+              current_series = s[:name]
+              anomalies << s[:name] if anomaly?(s[:data])
+            end
+            anomaly = anomalies.any?
+            if anomaly
+              if anomalies.size == 1
+                message = "Anomaly detected in #{anomalies.first}"
+              else
+                message = "Anomalies detected in #{anomalies.to_sentence}"
+              end
+            else
+              message = "No anomalies detected"
+            end
+          rescue => e
+            message = "#{current_series}: #{e.message}"
+          end
+        else
+          message = "Bad format"
+        end
+      end
+
+      [anomaly, message]
+    end
+
+    def anomaly?(series)
+      series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
+
+      csv_str =
+        CSV.generate do |csv|
+          csv << ["timestamp", "count"]
+          series.each do |row|
+            csv << row
+          end
+        end
+
+      timestamps = []
+      r_script = %x[which Rscript].chomp
+      raise "R not found" if r_script.empty?
+      output = %x[#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{Shellwords.escape(csv_str)}]
+      if output.empty?
+        raise "Unknown R error"
+      end
+
+      rows = CSV.parse(output, headers: true)
+      error = rows.first && rows.first["x"]
+      raise error if error
+
+      rows.each do |row|
+        timestamps << Time.parse(row["timestamp"])
+      end
+      timestamps.include?(series.last[0].to_time)
+    end
+  end
+end

data/lib/blazer/run_statement_job.rb

@@ -0,0 +1,16 @@
+require "sucker_punch"
+
+module Blazer
+  class RunStatementJob
+    include SuckerPunch::Job
+    workers 4
+
+    def perform(result, data_source, statement, options)
+      ActiveRecord::Base.connection_pool.with_connection do
+        data_source.connection_model.connection_pool.with_connection do
+          result << data_source.run_main_statement(statement, options)
+        end
+      end
+    end
+  end
+end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

data/lib/generators/blazer/templates/install.rb

@@ -36,7 +36,8 @@ class <%= migration_class_name %> < ActiveRecord::Migration
       t.string :state
       t.string :schedule
       t.text :emails
-      t.boolean :invert
+      t.string :check_type
+      t.text :message
       t.timestamp :last_run_at
       t.timestamps
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-10 00:00:00.000000000 Z
+date: 2016-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -143,7 +143,6 @@ files:
 - app/views/blazer/checks/edit.html.erb
 - app/views/blazer/checks/index.html.erb
 - app/views/blazer/checks/new.html.erb
-- app/views/blazer/checks/run.html.erb
 - app/views/blazer/dashboards/_form.html.erb
 - app/views/blazer/dashboards/edit.html.erb
 - app/views/blazer/dashboards/index.html.erb
@@ -161,7 +160,10 @@ files:
 - config/routes.rb
 - lib/blazer.rb
 - lib/blazer/data_source.rb
+- lib/blazer/detect_anomalies.R
 - lib/blazer/engine.rb
+- lib/blazer/result.rb
+- lib/blazer/run_statement_job.rb
 - lib/blazer/version.rb
 - lib/generators/blazer/install_generator.rb
 - lib/generators/blazer/templates/config.yml
@@ -187,9 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.1
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Share data effortlessly with your team
 test_files: []
-has_rdoc: 

data/app/views/blazer/checks/run.html.erb

@@ -1,11 +0,0 @@
-<p style="text-muted">Running check...</p>
-
-<script>
-  var data = <%= blazer_json_escape({statement: @query.statement, query_id: @query.id, check: true}.to_json).html_safe %>;
-
-  runQuery(data, function (data) {
-    setTimeout( function () {
-      window.location.href = "<%= checks_path %>";
-    }, 200);
-  });
-</script>