blazer 1.2.1 → 1.3.0

data/app/assets/stylesheets/blazer/application.css

@@ -87,3 +87,16 @@ input.search:focus {
 .dashboard a {
   font-weight: bold;
 }
+
+#search-item {
+  display: none;
+}
+
+.creator {
+  text-align: right;
+  color: #999;
+}
+
+.vars {
+  color: #ccc;
+}

data/app/controllers/blazer/checks_controller.rb

@@ -13,6 +13,9 @@ module Blazer
 
     def create
       @check = Blazer::Check.new(check_params)
+      # use creator_id instead of creator
+      # since we setup association without checking if column exists
+      @check.creator = blazer_user if @check.respond_to?(:creator_id=)
 
       if @check.save
         redirect_to run_check_path(@check)
@@ -41,7 +44,7 @@ module Blazer
     private
 
     def check_params
-      params.require(:check).permit(:query_id, :emails, :invert)
+      params.require(:check).permit(:query_id, :emails, :invert, :schedule)
     end
 
     def set_check

data/app/controllers/blazer/dashboards_controller.rb

@@ -12,6 +12,9 @@ module Blazer
 
     def create
       @dashboard = Blazer::Dashboard.new
+      # use creator_id instead of creator
+      # since we setup association without checking if column exists
+      @dashboard.creator = blazer_user if @dashboard.respond_to?(:creator_id=)
 
       if update_dashboard(@dashboard)
         redirect_to dashboard_path(@dashboard)

data/app/controllers/blazer/queries_controller.rb

@@ -4,11 +4,20 @@ module Blazer
 
     def home
       set_queries(1000)
+      @dashboards =
+        Blazer::Dashboard.order(:name).map do |d|
+          {
+            name: "<strong>#{view_context.link_to(d.name, d)}</strong>",
+            creator: blazer_user && d.try(:creator) == blazer_user ? "You" : d.try(:creator).try(Blazer.user_name),
+            hide: d.name.gsub(/\s+/, ""),
+            vars: nil
+          }
+        end
     end
 
     def index
       set_queries
-      render partial: "index", layout: false
+      render json: @queries
     end
 
     def new
@@ -180,7 +189,7 @@ module Blazer
 
     def set_queries(limit = nil)
       @my_queries =
-        if blazer_user
+        if limit && blazer_user
           favorite_query_ids = Blazer::Audit.where(user_id: blazer_user.id).where("created_at > ?", 30.days.ago).where("query_id IS NOT NULL").group(:query_id).order("count_all desc").count.keys
           queries = Blazer::Query.named.where(id: favorite_query_ids)
           queries = queries.includes(:creator) if Blazer.user_class
@@ -194,9 +203,17 @@ module Blazer
       @queries = @queries.where("id NOT IN (?)", @my_queries.map(&:id)) if @my_queries.any?
       @queries = @queries.includes(:creator) if Blazer.user_class
       @queries = @queries.limit(limit) if limit
-      @trending_queries = Blazer::Audit.group(:query_id).where("created_at > ?", 2.days.ago).having("COUNT(DISTINCT user_id) >= 3").uniq.count(:user_id)
-      @checks = Blazer::Check.group(:query_id).count
-      @dashboards = Blazer::Dashboard.order(:name)
+
+      @queries =
+        @queries.map do |q|
+          {
+            id: q.id,
+            name: view_context.link_to(q.name, q),
+            creator: blazer_user && q.try(:creator) == blazer_user ? "You" : q.try(:creator).try(Blazer.user_name),
+            hide: q.name.gsub(/\s+/, ""),
+            vars: extract_vars(q.statement).join(", ")
+          }
+        end
     end
 
     def set_query

data/app/models/blazer/check.rb

@@ -19,6 +19,8 @@ module Blazer
           invert ? "failing" : "passing"
         end
 
+      self.last_run_at = Time.now if self.respond_to?(:last_run_at=)
+
       # do not notify on creation, except when not passing
       if (state_was || state != "passing") && state != state_was && emails.present?
         Blazer::CheckMailer.state_change(self, state, state_was, rows.size, error).deliver_later

data/app/views/blazer/checks/_form.html.erb

@@ -17,17 +17,29 @@
     </script>
   </div>
 
-<% if @check.respond_to?(:invert) %>
-  <div class="form-group">
-    <%= f.label "Fails if" %>
-    <div class="hide">
-      <%= f.select :invert, [["Any results (bad data)", false], ["No results (missing data)", true]], {}, {id: "select-invert"} %>
+  <% if @check.respond_to?(:invert) %>
+    <div class="form-group">
+      <%= f.label :invert, "Fails if" %>
+      <div class="hide">
+        <%= f.select :invert, [["Any results (bad data)", false], ["No results (missing data)", true]] %>
+      </div>
+      <script>
+        $("#check_invert").selectize({}).parent().removeClass("hide");
+      </script>
     </div>
-    <script>
-      $("#select-invert").selectize({}).parent().removeClass("hide");
-    </script>
-  </div>
-<% end %>
+  <% end %>
+
+  <% if @check.respond_to?(:schedule) && Blazer.check_schedules %>
+    <div class="form-group">
+      <%= f.label :schedule, "Run every" %>
+      <div class="hide">
+        <%= f.select :schedule, Blazer.check_schedules.map { |v| [v, v] } %>
+      </div>
+      <script>
+        $("#check_schedule").selectize({}).parent().removeClass("hide");
+      </script>
+    </div>
+  <% end %>
 
   <div class="form-group">
     <%= f.label :emails %>

data/app/views/blazer/checks/index.html.erb

@@ -8,7 +8,8 @@
   <thead>
     <tr>
       <th>Query</th>
-      <th style="width: 15%;">State</th>
+      <th style="width: 10%;">State</th>
+      <th style="width: 10%;">Run</th>
       <th style="width: 20%;">Emails</th>
       <th style="width: 15%;"></th>
     </tr>
@@ -22,6 +23,7 @@
             <small style="font-weight: bold; color: <%= colors[check.state.to_sym] %>;"><%= check.state.upcase %></small>
           <% end %>
         </td>
+        <td><%= check.schedule if check.respond_to?(:schedule) %></td>
         <td>
           <ul class="list-unstyled" style="margin-bottom: 0;">
             <% check.split_emails.each do |email| %>

data/app/views/blazer/dashboards/show.html.erb

@@ -19,7 +19,7 @@
     <div class="row" style="padding-top: 13px;">
       <div class="col-sm-9">
         <%= render partial: "blazer/nav" %>
-        <h3 style="margin: 0; line-height: 34px; display: inline-block;">
+        <h3 style="margin: 0; line-height: 34px; display: inline;">
           <%= @dashboard.name %>
         </h3>
       </div>

data/app/views/blazer/queries/_form.html.erb

@@ -48,8 +48,17 @@
         <% end %>
         <%= f.submit @query.persisted? ? "Update" : "Create", class: "btn btn-success" %>
       </div>
-      <% if @query.persisted? && (dashboards_count = @query.dashboards.count) > 0 %>
-        <div class="alert alert-info" style="margin-top: 10px; padding: 8px 12px;">Part of <%= pluralize(dashboards_count, "dashboard") %>. Be careful when editing.</div>
+      <% if @query.persisted? %>
+        <% dashboards_count = @query.dashboards.count %>
+        <% checks_count = @query.checks.count %>
+        <% words = [] %>
+        <% words << pluralize(dashboards_count, "dashboard") if dashboards_count > 0 %>
+        <% words << pluralize(checks_count, "check") if checks_count > 0 %>
+        <% if words.any? %>
+          <div class="alert alert-info" style="margin-top: 10px; padding: 8px 12px;">
+            Part of <%= words.to_sentence %>. Be careful when editing.
+          </div>
+        <% end %>
       <% end %>
     </div>
   </div>

data/app/views/blazer/queries/home.html.erb

@@ -25,30 +25,55 @@
       </tr>
     </thead>
     <tbody class="list">
-      <%= render partial: "index" %>
+      <tr id="search-item">
+        <td>
+          <span class="name"></span>
+          <span class="vars"></span>
+          <span class="hide"></span>
+        </td>
+        <td class="creator"></td>
+      </tr>
     </tbody>
   </table>
 </div>
 
 <script>
   var options = {
-    valueNames: ['query', 'creator'],
-    page: 200
+    valueNames: ["name", "vars", "hide", "creator"],
+    item: "search-item",
+    page: 200,
+    indexAsync: true
   };
+  var dashboardValues = <%= blazer_json_escape(@dashboards.to_json).html_safe %>;
+  var queryValues = <%= blazer_json_escape(@queries.to_json).html_safe %>;
+  var queryList = new List("queries", options, dashboardValues);
+  queryList.add(queryValues);
 
-  function updateList() {
-    var userList = new List('queries', options);
-    userList.search($(".search").val());
+  var queryIds = {};
+  for (var i = 0; i < queryValues.length; i++) {
+    queryIds[queryValues[i].id] = true;
   }
-  updateList();
 </script>
 
-<% if @queries.size == 1000 %>
+<% if @queries.size >= 200 %>
   <p id="loading" class="text-muted">Loading...</p>
   <script>
-    $(".list").load("<%= queries_path %>", function () {
-      updateList();
+    $.getJSON("<%= queries_path %>", function (data) {
+      var i, j, newValues, val, size = 500;
       $("#loading").remove();
+      for (i = 0; i < data.length / size; i++) {
+        newValues = [];
+        for (j = 0; j < size; j++) {
+          val = data[i * size + j];
+          if (!val) {
+            break;
+          }
+          if (!queryIds[val.id]) {
+            newValues.push(val);
+          }
+        }
+        queryList.add(newValues);
+      }
     });
   </script>
 <% end %>

data/app/views/blazer/queries/run.html.erb

@@ -28,8 +28,9 @@
     <% chart_id = SecureRandom.hex %>
     <% column_types = blazer_column_types(@columns, @rows, @boom) %>
     <% chart_options = {id: chart_id, min: nil} %>
-    <% if (target_index = @columns.keys.map(&:downcase).index("target")) %>
-      <% chart_options[:library] = {series: {(target_index - 1) => {pointSize: 0, lineWidth: 3, color: "#109618"}}} %>
+    <% series_library = {} %>
+    <% @columns.keys.select { |k| k.downcase == "target" }.each do |key| %>
+      <% series_library[key] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
     <% end %>
     <% if blazer_maps? && @markers.any? %>
       <div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
@@ -62,10 +63,10 @@
       </script>
     <% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>
       <% time_k = @columns.keys.first %>
-      <%= line_chart @columns.keys[1..-1].map{|k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }} }, chart_options %>
+      <%= line_chart @columns.keys[1..-1].map{ |k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }, library: series_library[k]} }, chart_options %>
     <% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>
       <% keys = @columns.keys %>
-      <%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] } } }, chart_options %>
+      <%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] }, library: series_library[name]} }, chart_options %>
     <% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>
       <% keys = @columns.keys %>
       <%= column_chart (values.size - 1).times.map { |i| name = @columns.keys[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[keys[0]] || {})[r[keys[0]].to_s] || r[keys[0]], r[keys[i + 1]]] } } }, id: chart_id %>
@@ -80,46 +81,50 @@
     <% unless @only_chart && !@no_chart %>
       <% header_width = 100 / @rows.first.keys.size.to_f %>
       <div class="results-container">
-        <table class="table results-table" style="margin-bottom: 0;">
-          <thead>
-            <tr>
-              <% @columns.each do |key, type| %>
-                <th style="width: <%= header_width %>%;" data-sort="<%= type %>">
-                  <div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
-                    <%= key %>
-                  </div>
-                </th>
-              <% end %>
-            </tr>
-          </thead>
-          <tbody>
-            <% @rows.each do |row| %>
+        <% if @columns.keys == ["QUERY PLAN"] %>
+          <pre><code><%= @rows.map { |r| r["QUERY PLAN"] }.join("\n") %></code></pre>
+        <% else %>
+          <table class="table results-table" style="margin-bottom: 0;">
+            <thead>
               <tr>
-                <% row.each do |k, v| %>
-                  <td>
-                    <% if v.is_a?(Time) %>
-                      <% v = blazer_time_value(k, v) %>
-                    <% end %>
-
-                    <% unless v.nil? %>
-                      <% if v == "" %>
-                        <div class="text-muted">empty string</div>
-                      <% elsif @linked_columns[k] %>
-                        <%= link_to blazer_format_value(k, v), @linked_columns[k].gsub("{value}", u(v.to_s)), target: "_blank" %>
-                      <% else %>
-                        <%= blazer_format_value(k, v) %>
+                <% @columns.each do |key, type| %>
+                  <th style="width: <%= header_width %>%;" data-sort="<%= type %>">
+                    <div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
+                      <%= key %>
+                    </div>
+                  </th>
+                <% end %>
+              </tr>
+            </thead>
+            <tbody>
+              <% @rows.each do |row| %>
+                <tr>
+                  <% row.each do |k, v| %>
+                    <td>
+                      <% if v.is_a?(Time) %>
+                        <% v = blazer_time_value(k, v) %>
                       <% end %>
 
-                      <% if v2 = (@boom[k] || {})[v.to_s] %>
-                        <div class="text-muted"><%= v2 %></div>
+                      <% unless v.nil? %>
+                        <% if v.is_a?(String) && v == "" %>
+                          <div class="text-muted">empty string</div>
+                        <% elsif @linked_columns[k] %>
+                          <%= link_to blazer_format_value(k, v), @linked_columns[k].gsub("{value}", u(v.to_s)), target: "_blank" %>
+                        <% else %>
+                          <%= blazer_format_value(k, v) %>
+                        <% end %>
+
+                        <% if v2 = (@boom[k] || {})[v.to_s] %>
+                          <div class="text-muted"><%= v2 %></div>
+                        <% end %>
                       <% end %>
-                    <% end %>
-                  </td>
-                <% end %>
-              </tr>
-            <% end %>
-          </tbody>
-        </table>
+                    </td>
+                  <% end %>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        <% end %>
       </div>
     <% end %>
   <% elsif @only_chart %>

data/app/views/blazer/queries/show.html.erb

@@ -19,7 +19,7 @@
     <div class="row" style="padding-top: 13px;">
       <div class="col-sm-9">
         <%= render partial: "blazer/nav" %>
-        <h3 style="margin: 0; line-height: 34px; display: inline-block;">
+        <h3 style="margin: 0; line-height: 34px; display: inline;">
           <%= @query.name %>
         </h3>
       </div>
@@ -149,7 +149,7 @@
   </form>
 <% end %>
 
-<pre><code><%= @statement %></code></pre>
+<pre style="max-height: 236px; overflow: hidden;" onclick="this.style.maxHeight = 'none';"><code><%= @statement %></code></pre>
 
 <% if @success %>
   <div id="results">

data/app/views/layouts/blazer/application.html.erb

@@ -6,10 +6,13 @@
     <meta charset="utf-8" />
 
     <%= stylesheet_link_tag "blazer/application" %>
-    <%= javascript_include_tag "https://www.google.com/jsapi", "blazer/application" %>
+    <script>
+      var Chartkick = {smarterDates: true, smarterDiscrete: true};
+    </script>
+    <%= javascript_include_tag "blazer/application" %>
     <% if blazer_maps? %>
-      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v2.2.2/mapbox.css" %>
-      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v2.2.2/mapbox.js" %>
+      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.css" %>
+      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.js" %>
     <% end %>
     <%= csrf_meta_tags %>
   </head>

data/lib/blazer.rb

@@ -4,9 +4,10 @@ require "chartkick"
 require "blazer/version"
 require "blazer/data_source"
 require "blazer/engine"
-require "blazer/tasks"
 
 module Blazer
+  class TimeoutNotSupported < StandardError; end
+
   class << self
     attr_accessor :audit
     attr_reader :time_zone
@@ -17,11 +18,18 @@ module Blazer
     attr_accessor :from_email
     attr_accessor :cache
     attr_accessor :transform_statement
+    attr_accessor :check_schedules
   end
   self.audit = true
   self.user_name = :name
+  self.check_schedules = ["5 minutes", "1 hour", "1 day"]
 
   TIMEOUT_MESSAGE = "Query timed out :("
+  TIMEOUT_ERRORS = [
+    "canceling statement due to statement timeout", # postgres
+    "cancelled on user's request", # redshift
+    "system requested abort" # redshift
+  ]
 
   def self.time_zone=(time_zone)
     @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
@@ -50,25 +58,36 @@ module Blazer
     end
   end
 
-  def self.run_checks
-    Blazer::Check.includes(:query).find_each do |check|
+  def self.run_checks(schedule: nil)
+    checks = Blazer::Check.includes(:query)
+    checks = checks.where(schedule: schedule) if schedule
+    checks.find_each do |check|
       rows = nil
       error = nil
-      tries = 0
-      # try 3 times on timeout errors
-      while tries < 3
-        rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
-        if error == Blazer::TIMEOUT_MESSAGE
-          Rails.logger.info "[blazer timeout] query=#{check.query.name}"
-          tries += 1
-          sleep(10)
-        else
-          break
+      tries = 1
+
+      ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
+        # try 3 times on timeout errors
+        while tries <= 3
+          rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
+          if error == Blazer::TIMEOUT_MESSAGE
+            Rails.logger.info "[blazer timeout] query=#{check.query.name}"
+            tries += 1
+            sleep(10)
+          else
+            break
+          end
         end
+        check.update_state(rows, error)
+        # TODO use proper logfmt
+        Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"
+
+        instrument[:state] = check.state
+        instrument[:rows] = rows.try(:size)
+        instrument[:error] = error
+        instrument[:tries] = tries
       end
-      check.update_state(rows, error)
-      # TODO use proper logfmt
-      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"
+
     end
   end