sql-jarvis 1.8.0

data/app/helpers/blazer/base_helper.rb

@@ -0,0 +1,57 @@
+module Blazer
+  module BaseHelper
+    def blazer_title(title = nil)
+      if title
+        content_for(:title) { title }
+      else
+        content_for?(:title) ? content_for(:title) : nil
+      end
+    end
+
+    BLAZER_URL_REGEX = /\Ahttps?:\/\/[\S]+\z/
+    BLAZER_IMAGE_EXT = %w[png jpg jpeg gif]
+
+    def blazer_format_value(key, value)
+      if value.is_a?(Integer) && !key.to_s.end_with?("id") && !key.to_s.start_with?("id")
+        number_with_delimiter(value)
+      elsif value =~ BLAZER_URL_REGEX
+        # see if image or link
+        if Blazer.images && (key.include?("image") || BLAZER_IMAGE_EXT.include?(value.split(".").last.split("?").first.try(:downcase)))
+          link_to value, target: "_blank" do
+            image_tag value, referrerpolicy: "no-referrer"
+          end
+        else
+          link_to value, value, target: "_blank"
+        end
+      else
+        value
+      end
+    end
+
+    def blazer_maps?
+      Blazer.mapbox_access_token.present?
+    end
+
+    def blazer_js_var(name, value)
+      "var #{name} = #{blazer_json_escape(value.to_json(root: false))};".html_safe
+    end
+
+    JSON_ESCAPE = { '&' => '\u0026', '>' => '\u003e', '<' => '\u003c', "\u2028" => '\u2028', "\u2029" => '\u2029' }
+    JSON_ESCAPE_REGEXP = /[\u2028\u2029&><]/u
+
+    # Prior to version 4.1 of rails double quotes were inadventently removed in json_escape.
+    # This adds the correct json_escape functionality to rails versions < 4.1
+    def blazer_json_escape(s)
+      if Rails::VERSION::STRING < "4.1"
+        result = s.to_s.gsub(JSON_ESCAPE_REGEXP, JSON_ESCAPE)
+        s.html_safe? ? result.html_safe : result
+      else
+        json_escape(s)
+      end
+    end
+
+    def blazer_series_name(k)
+      k.nil? ? "null" : k.to_s
+    end
+  end
+end

data/app/mailers/blazer/check_mailer.rb

@@ -0,0 +1,27 @@
+module Blazer
+  class CheckMailer < ActionMailer::Base
+    include ActionView::Helpers::TextHelper
+
+    default from: Blazer.from_email if Blazer.from_email
+    layout false
+
+    def state_change(check, state, state_was, rows_count, error, columns, rows, column_types, check_type)
+      @check = check
+      @state = state
+      @state_was = state_was
+      @rows_count = rows_count
+      @error = error
+      @columns = columns
+      @rows = rows
+      @column_types = column_types
+      @check_type = check_type
+      mail to: check.emails, reply_to: check.emails, subject: "Check #{state.titleize}: #{check.query.name}"
+    end
+
+    def failing_checks(email, checks)
+      @checks = checks
+      # add reply_to for mailing lists
+      mail to: email, reply_to: email, subject: "#{pluralize(checks.size, "Check")} Failing"
+    end
+  end
+end

data/app/models/blazer/audit.rb

@@ -0,0 +1,6 @@
+module Blazer
+  class Audit < Record
+    belongs_to :user, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s)
+    belongs_to :query, Blazer::BELONGS_TO_OPTIONAL
+  end
+end

data/app/models/blazer/check.rb

@@ -0,0 +1,95 @@
+module Blazer
+  class Check < Record
+    belongs_to :creator, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s) if Blazer.user_class
+    belongs_to :query
+
+    validates :query_id, presence: true
+    validate :validate_emails
+    validate :validate_variables, if: -> { query_id_changed? }
+
+    before_validation :set_state
+    before_validation :fix_emails
+
+    def split_emails
+      emails.to_s.downcase.split(",").map(&:strip)
+    end
+
+    def update_state(result)
+      check_type =
+        if respond_to?(:check_type)
+          self.check_type
+        elsif respond_to?(:invert)
+          invert ? "missing_data" : "bad_data"
+        else
+          "bad_data"
+        end
+
+      message = result.error
+
+      self.state =
+        if result.timed_out?
+          "timed out"
+        elsif result.error
+          "error"
+        elsif check_type == "anomaly"
+          anomaly, message = result.detect_anomaly
+          if anomaly.nil?
+            "error"
+          elsif anomaly
+            "failing"
+          else
+            "passing"
+          end
+        elsif result.rows.any?
+          check_type == "missing_data" ? "passing" : "failing"
+        else
+          check_type == "missing_data" ? "failing" : "passing"
+        end
+
+      self.last_run_at = Time.now if respond_to?(:last_run_at=)
+      self.message = message if respond_to?(:message=)
+
+      if respond_to?(:timeouts=)
+        if result.timed_out?
+          self.timeouts += 1
+          self.state = "disabled" if timeouts >= 3
+        else
+          self.timeouts = 0
+        end
+      end
+
+      # do not notify on creation, except when not passing
+      if (state_was != "new" || state != "passing") && state != state_was && emails.present?
+        Blazer::CheckMailer.state_change(self, state, state_was, result.rows.size, message, result.columns, result.rows.first(10).as_json, result.column_types, check_type).deliver_now
+      end
+      save! if changed?
+    end
+
+    private
+
+      def set_state
+        self.state ||= "new"
+      end
+
+      def fix_emails
+        # some people like doing ; instead of ,
+        # but we know what they mean, so let's fix it
+        # also, some people like to use whitespace
+        if emails.present?
+          self.emails = emails.strip.gsub(/[;\s]/, ",").gsub(/,+/, ", ")
+        end
+      end
+
+      def validate_emails
+        unless split_emails.all? { |e| e =~ /\A\S+@\S+\.\S+\z/ }
+          errors.add(:base, "Invalid emails")
+        end
+      end
+
+      def validate_variables
+        if query.variables.any?
+          errors.add(:base, "Query can't have variables")
+        end
+      end
+  end
+end

data/app/models/blazer/connection.rb

@@ -0,0 +1,5 @@
+module Blazer
+  class Connection < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/blazer/dashboard.rb

@@ -0,0 +1,13 @@
+module Blazer
+  class Dashboard < Record
+    belongs_to :creator, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s) if Blazer.user_class
+    has_many :dashboard_queries, dependent: :destroy
+    has_many :queries, through: :dashboard_queries
+
+    validates :name, presence: true
+
+    def to_param
+      [id, name.gsub("'", "").parameterize].join("-")
+    end
+  end
+end

data/app/models/blazer/dashboard_query.rb

@@ -0,0 +1,9 @@
+module Blazer
+  class DashboardQuery < Record
+    belongs_to :dashboard
+    belongs_to :query
+
+    validates :dashboard_id, presence: true
+    validates :query_id, presence: true
+  end
+end

data/app/models/blazer/query.rb

@@ -0,0 +1,31 @@
+module Blazer
+  class Query < Record
+    belongs_to :creator, Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s) if Blazer.user_class
+    has_many :checks, dependent: :destroy
+    has_many :dashboard_queries, dependent: :destroy
+    has_many :dashboards, through: :dashboard_queries
+    has_many :audits
+
+    validates :statement, presence: true
+
+    scope :named, -> { where("blazer_queries.name <> ''") }
+
+    def to_param
+      [id, name].compact.join("-").gsub("'", "").parameterize
+    end
+
+    def friendly_name
+      name.to_s.sub(/\A[#\*]/, "").gsub(/\[.+\]/, "").strip
+    end
+
+    def editable?(user)
+      editable = !persisted? || (name.present? && name.first != "*" && name.first != "#") || user == try(:creator)
+      editable &&= Blazer.query_editable.call(self, user) if Blazer.query_editable
+      editable
+    end
+
+    def variables
+      Blazer.extract_vars(statement)
+    end
+  end
+end

data/app/models/blazer/record.rb

@@ -0,0 +1,5 @@
+module Blazer
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/views/blazer/_nav.html.erb

@@ -0,0 +1,16 @@
+<div class="btn-group" style="vertical-align: top; margin-right: 5px;">
+  <%= link_to "Home", root_path, class: "btn btn-primary" %>
+  <button type="button" class="btn btn-primary dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+    <span class="caret"></span>
+    <span class="sr-only">Toggle Dropdown</span>
+  </button>
+  <ul class="dropdown-menu">
+    <li><%= link_to "Dashboards", dashboards_path %></li>
+    <li><%= link_to "Checks", checks_path %></li>
+    <li role="separator" class="divider"></li>
+    <li><%= link_to "New Query", new_query_path %></li>
+    <li><%= link_to "New Dashboard", new_dashboard_path %></li>
+    <% check_params = @query ? {query_id: @query.id} : {} %>
+    <li><%= link_to "New Check", new_check_path(check_params) %></li>
+  </ul>
+</div>

data/app/views/blazer/_variables.html.erb

@@ -0,0 +1,102 @@
+<% if @bind_vars.any? %>
+  <form id="bind" method="get" action="<%= action %>" class="form-inline" style="margin-bottom: 10px;">
+    <% date_vars = ["start_time", "end_time"] %>
+    <% if (date_vars - @bind_vars).empty? %>
+      <% @bind_vars = @bind_vars - date_vars %>
+    <% else %>
+      <% date_vars = nil %>
+    <% end %>
+
+    <% @bind_vars.each_with_index do |var, i| %>
+      <%= label_tag var, var %>
+      <% if (data = @smart_vars[var]) %>
+        <%= select_tag var, options_for_select([[nil, nil]] + data, selected: params[var]), style: "margin-right: 20px; width: 200px; display: none;" %>
+        <script>
+          $("#<%= var %>").selectize({
+            create: true
+          });
+        </script>
+      <% else %>
+        <%= text_field_tag var, params[var], style: "width: 120px; margin-right: 20px;", autofocus: i == 0 && !var.end_with?("_at") && !params[var], class: "form-control" %>
+        <% if var.end_with?("_at") %>
+          <script>
+            $("#<%= var %>").daterangepicker({singleDatePicker: true, locale: {format: "YYYY-MM-DD"}, autoUpdateInput: false});
+            // hack to start with empty date
+            $("#<%= var %>").on("apply.daterangepicker", function(ev, picker) {
+              $(this).val(picker.startDate.format("YYYY-MM-DD"));
+              $(this).change();
+            });
+          </script>
+        <% end %>
+      <% end %>
+    <% end %>
+
+    <% if date_vars %>
+      <% date_vars.each do |var| %>
+        <%= hidden_field_tag var, params[var] %>
+      <% end %>
+
+      <%= label_tag nil, date_vars.join(" & ") %>
+      <div class="selectize-control single" style="width: 300px;">
+        <div id="reportrange" class="selectize-input" style="display: inline-block;">
+          <span>Select a time range</span>
+        </div>
+      </div>
+
+      <script>
+        <%= blazer_js_var "timeZone", Blazer.time_zone.tzinfo.name %>
+        var format = "YYYY-MM-DD"
+        var now = moment.tz(timeZone)
+
+        function dateStr(daysAgo) {
+          return now.clone().subtract(daysAgo || 0, "days").format(format)
+        }
+
+        function toDate(time) {
+          return moment.tz(time.format(format), timeZone)
+        }
+
+        function setTimeInputs(start, end) {
+          $("#start_time").val(toDate(start).utc().format())
+          $("#end_time").val(toDate(end).endOf("day").utc().format())
+        }
+
+        $("#reportrange").daterangepicker(
+          {
+            ranges: {
+             "Today": [dateStr(), dateStr()],
+             "Last 7 Days": [dateStr(6), dateStr()],
+             "Last 30 Days": [dateStr(29), dateStr()]
+            },
+            locale: {
+              format: format
+            },
+            startDate: dateStr(29),
+            endDate: dateStr(),
+            opens: "right"
+          },
+          function(start, end) {
+            setTimeInputs(start, end)
+            submitIfCompleted($("#start_time").closest("form"))
+          }
+        ).on("apply.daterangepicker", function(ev, picker) {
+          setTimeInputs(picker.startDate, picker.endDate)
+          $("#reportrange span").html(toDate(picker.startDate).format("MMMM D, YYYY") + " - " + toDate(picker.endDate).format("MMMM D, YYYY"))
+        })
+
+        if ($("#start_time").val().length > 0) {
+          var picker = $("#reportrange").data("daterangepicker")
+          picker.setStartDate(moment.tz($("#start_time").val(), timeZone))
+          picker.setEndDate(moment.tz($("#end_time").val(), timeZone))
+          $("#reportrange").trigger("apply.daterangepicker", picker)
+        } else {
+          var picker = $("#reportrange").data("daterangepicker")
+          $("#reportrange").trigger("apply.daterangepicker", picker)
+          submitIfCompleted($("#start_time").closest("form"))
+        }
+      </script>
+    <% end %>
+
+    <input type="submit" class="btn btn-success" value="Run" style="vertical-align: top;" />
+  </form>
+<% end %>

data/app/views/blazer/check_mailer/failing_checks.html.erb

@@ -0,0 +1,6 @@
+<ul>
+  <% @checks.each do |check| %>
+    <li><%= link_to check.query.name, query_url(check.query_id) %> <%= check.state %></li>
+  <% end %>
+</ul>
+<p><%= link_to "Manage checks", checks_url %></p>

data/app/views/blazer/check_mailer/state_change.html.erb

@@ -0,0 +1,47 @@
+<html>
+  <head>
+  </head>
+  <body style="font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;">
+    <p><%= link_to "View", query_url(@check.query_id) %></p>
+    <% if @error %>
+      <p><%= @error %></p>
+    <% elsif @rows_count > 0 && @check_type == "bad_data" %>
+      <p>
+        <% if @rows_count <= 10 %>
+          <%= pluralize(@rows_count, "row") %>
+        <% else %>
+          Showing 10 of <%= @rows_count %> rows
+        <% end %>
+      </p>
+      <table style="width: 100%; border-spacing: 0; border-collapse: collapse;">
+        <thead>
+          <tr>
+            <% @columns.first(5).each do |column| %>
+              <th style="padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: <%= (100 / @columns.size).round(2) %>%;">
+                <%= column %>
+              </th>
+            <% end %>
+          </tr>
+        </thead>
+        <tbody>
+          <% @rows.first(10).each do |row| %>
+            <tr>
+              <% @columns.first(5).each_with_index do |column, i| %>
+                <td style="padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;">
+                  <% value = row[i] %>
+                  <% if @column_types[i] == "time" && value.to_s.length > 10 %>
+                    <% value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value %>
+                  <% end %>
+                  <%= value %>
+                </td>
+              <% end %>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+      <% if @columns.size > 5 %>
+        <p style="color: #999;">Only first 5 columns shown</p>
+      <% end %>
+    <% end %>
+  </body>
+</html>

data/app/views/blazer/checks/_form.html.erb

@@ -0,0 +1,71 @@
+<% unless @check.respond_to?(:invert) %>
+  <p class="text-muted">Checks are designed to identify bad data. A check fails if there are any results.</p>
+<% end %>
+
+<% if @check.errors.any? %>
+  <div class="alert alert-danger"><%= @check.errors.full_messages.first %></div>
+<% end %>
+
+<%= form_for @check do |f| %>
+  <div class="form-group">
+    <%= f.label :query_id, "Query" %>
+    <div class="hide">
+      <%= f.select :query_id, [], {include_blank: true} %>
+    </div>
+    <script>
+      <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+      <%= blazer_js_var "items", [@check.query_id].compact %>
+
+      $("#check_query_id").selectize({options: queries, items: items, highlight: false, maxOptions: 100}).parents(".hide").removeClass("hide");
+    </script>
+  </div>
+
+  <% if @check.respond_to?(:check_type) %>
+    <div class="form-group">
+      <%= f.label :check_type, "Alert if" %>
+      <div class="hide">
+        <% check_options = [["Any results (bad data)", "bad_data"], ["No results (missing data)", "missing_data"]] %>
+        <% check_options << ["Anomaly (most recent data point)", "anomaly"] if Blazer.anomaly_checks %>
+        <%= f.select :check_type, check_options %>
+      </div>
+      <script>
+        $("#check_check_type").selectize({}).parent().removeClass("hide");
+      </script>
+    </div>
+  <% elsif @check.respond_to?(:invert) %>
+    <div class="form-group">
+      <%= f.label :invert, "Fails if" %>
+      <div class="hide">
+        <%= f.select :invert, [["Any results (bad data)", false], ["No results (missing data)", true]] %>
+      </div>
+      <script>
+        $("#check_invert").selectize({}).parent().removeClass("hide");
+      </script>
+    </div>
+  <% end %>
+
+  <% if @check.respond_to?(:schedule) && Blazer.check_schedules %>
+    <div class="form-group">
+      <%= f.label :schedule, "Run every" %>
+      <div class="hide">
+        <%= f.select :schedule, Blazer.check_schedules.map { |v| [v, v] } %>
+      </div>
+      <script>
+        $("#check_schedule").selectize({}).parent().removeClass("hide");
+      </script>
+    </div>
+  <% end %>
+
+  <div class="form-group">
+    <%= f.label :emails %>
+    <%= f.text_field :emails, placeholder: "Optional, comma separated", class: "form-control" %>
+  </div>
+  <p class="text-muted">Emails are sent when a check starts failing, and when it starts passing again.
+  <p>
+    <% if @check.persisted? %>
+      <%= link_to "Delete", check_path(@check), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+    <% end %>
+    <%= f.submit "Save", class: "btn btn-success" %>
+    <%= link_to "Back", :back, class: "btn btn-link" %>
+  </p>
+<% end %>