sql-jarvis 2.0.1 → 2.0.2

data/app/views/blazer/queries/home.html.erb

@@ -1,6 +1,6 @@
 <div id="queries">
-  <div id="header" style="margin-bottom: 20px;">
-    <div class="pull-right">
+  <div id="header">
+    <div class="pull-right" style="line-height: 34px;">
       <% if blazer_user %>
         <div class='btn-group' style='margin-right: 20px;'>
           <%= link_to "All", root_path, class: !params[:filter] ? 'active btn btn-default btn-xs' : 'btn btn-default btn-xs' %>
@@ -21,12 +21,14 @@
           <span class="sr-only">Toggle Dropdown</span>
         </button>
         <ul class="dropdown-menu">
+          <li><%= link_to "Checks", checks_path %></li>
+          <li role="separator" class="divider"></li>
           <li><%= link_to "New Dashboard", new_dashboard_path %></li>
           <li><%= link_to "New Check", new_check_path %></li>
         </ul>
       </div>
     </div>
-    <input type="text" v-model="searchTerm" placeholder="Start typing a query or person" style="width: 300px; display: inline-block;" autofocus=true class="search form-control" />
+    <input type="text" v-model="searchTerm" placeholder="Start typing a query, dashboard, or person" style="width: 300px; display: inline-block;" v-focus class="search form-control" />
   </div>
 
   <table class="table">
@@ -148,6 +150,13 @@
           return Routes.query_path(item.to_param)
         }
       }
+    },
+    directives: {
+      focus: {
+        inserted: function (el) {
+          el.focus()
+        }
+      }
     }
   })
 </script>

data/app/views/blazer/queries/run.html.erb

@@ -24,7 +24,7 @@
         <% end %>
       </p>
     <% end %>
-    <p class="text-muted">
+    <p class="text-muted" style="margin-bottom: 10px;">
       <%= pluralize(@rows.size, "row") %>
 
       <% @checks.select(&:state).each do |check| %>
@@ -33,19 +33,46 @@
           &middot; <%= check.message %>
         <% end %>
       <% end %>
+
+      <% if @query && @result.forecastable? && !params[:forecast] %>
+        &middot;
+        <%= link_to "Forecast", query_path(@query, {forecast: "t"}.merge(variable_params)) %>
+      <% end %>
     </p>
   <% end %>
+  <% if @forecast_error %>
+    <div class="alert alert-danger"><%= @forecast_error %></div>
+  <% end %>
   <% if @rows.any? %>
     <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
     <% column_types = @result.column_types %>
     <% chart_type = @result.chart_type %>
-    <% chart_options = {id: chart_id, min: nil} %>
+    <% chart_options = {id: chart_id} %>
+    <% if ["line", "line2"].include?(chart_type) %>
+      <% chart_options.merge!(min: nil) %>
+    <% end %>
+    <% if chart_type == "scatter" %>
+      <% chart_options.merge!(library: {tooltips: {intersect: false}}) %>
+    <% elsif ["bar", "bar2"].include?(chart_type) %>
+      <% chart_options.merge!(library: {tooltips: {intersect: false, axis: 'x'}}) %>
+    <% elsif chart_type != "pie" %>
+      <% if column_types.size == 2 || @forecast %>
+        <% chart_options.merge!(library: {tooltips: {intersect: false, axis: 'x'}}) %>
+      <% else %>
+        <%# chartjs axis: 'x' has poor behavior with multiple series %>
+        <% chart_options.merge!(library: {tooltips: {intersect: false}}) %>
+      <% end %>
+    <% end %>
     <% series_library = {} %>
     <% target_index = @columns.index { |k| k.downcase == "target" } %>
     <% if target_index %>
       <% series_library[target_index - 1] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
     <% end %>
+    <% if @forecast %>
+      <% color = "#54a3ee" %>
+      <% series_library[1] = {borderDash: [8], borderColor: color, pointBackgroundColor: color, backgroundColor: color, pointHoverBackgroundColor: color} %>
+    <% end %>
     <% if blazer_maps? && @markers.any? %>
       <div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
       <script>
@@ -76,11 +103,14 @@
         map.fitBounds(featureLayer.getBounds());
       </script>
     <% elsif chart_type == "line" %>
-      <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: blazer_series_name(k), data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
+      <% chart_data = @columns[1..-1].each_with_index.map{ |k, i| {name: blazer_series_name(k), data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} } %>
+      <%= line_chart chart_data, chart_options %>
     <% elsif chart_type == "line2" %>
       <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
+    <% elsif chart_type == "pie" %>
+      <%= pie_chart @rows.map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[1]] }, chart_options %>
     <% elsif chart_type == "bar" %>
-      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
+      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, chart_options %>
     <% elsif chart_type == "bar2" %>
       <% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %>
       <% labels = first_20.map { |r| r[0] }.uniq %>
@@ -90,9 +120,9 @@
           <% first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } %>
         <% end %>
       <% end %>
-      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, id: chart_id %>
+      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, chart_options %>
     <% elsif chart_type == "scatter" %>
-      <%= scatter_chart @rows, xtitle: @columns[0], ytitle: @columns[1], id: chart_id %>
+      <%= scatter_chart @rows, xtitle: @columns[0], ytitle: @columns[1], **chart_options %>
     <% elsif @only_chart %>
       <% if @rows.size == 1 && @rows.first.size == 1 %>
         <% v = @rows.first.first %>

data/app/views/blazer/queries/schema.html.erb

@@ -1,13 +1,25 @@
-<% blazer_title "Database Schema" %>
+<% blazer_title "Schema: #{@data_source.name}" %>
+
+<h1>Schema: <%= @data_source.name %></h1>
+
+<hr />
+
+<div id="header">
+  <input id="search" type="text" placeholder="Start typing a table or column" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
 
 <% @schema.each do |table| %>
-  <h4>
-    <%= table[:table] %>
-    <% if table[:schema] != "public" %>
-      <small><%= table[:schema] %></small>
-    <% end %>
-  </h4>
-  <table class="table" style="max-width: 500px;">
+  <table class="table schema-table">
+    <thead>
+      <tr>
+        <th colspan="2">
+          <%= table[:table] %>
+          <% if table[:schema] != "public" %>
+            <span class="text-muted" style="font-weight: normal;"><%= table[:schema] %></span>
+          <% end %>
+        </th>
+      </tr>
+    </thead>
     <tbody>
       <% table[:columns].each do |column| %>
         <tr>
@@ -18,3 +30,29 @@
     </tbody>
   </table>
 <% end %>
+
+<script>
+  $("#search").on("keyup", function() {
+    var value = $(this).val().toLowerCase()
+    $(".schema-table").filter(function() {
+      // if found in table name, show entire table
+      // if just found in rows, show row
+
+      var found = $(this).find("thead").text().toLowerCase().indexOf(value) > -1
+
+      if (found) {
+        $(this).find("tbody tr").toggle(true)
+      } else {
+        $(this).find("tbody tr").filter(function() {
+          var found2 = $(this).text().toLowerCase().indexOf(value) > -1
+          $(this).toggle(found2)
+          if (found2) {
+            found = true
+          }
+        })
+      }
+
+      $(this).toggle(found)
+    })
+  }).focus()
+</script>

data/app/views/blazer/queries/show.html.erb

@@ -5,7 +5,7 @@
     <div class="row" style="padding-top: 13px;">
       <div class="col-sm-8">
         <%= render partial: "blazer/nav" %>
-        <h3 style="margin: 0; line-height: 34px; display: inline;">
+        <h3 style="line-height: 34px; display: inline; margin-left: 5px;">
           <%= @query.name %>
         </h3>
       </div>
@@ -14,7 +14,7 @@
         <%= link_to "Fork", new_query_path(variable_params.merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
 
         <% if !@error && @success %>
-          <%= button_to "⤓ .csv", run_queries_path(query_id: @query.id, format: "csv"), params: {statement: @statement}, class: "btn btn-primary" %>
+          <%= button_to "⤓ .csv", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
           <%= button_to "⤓ .xlsx", run_queries_path(query_id: @query.id, format: "xlsx"), params: {statement: @statement}, class: "btn btn-primary" %>
         <% end %>
       </div>
@@ -57,13 +57,13 @@
       $("#results").addClass("query-error").html(message)
     }
 
-    <%= blazer_js_var "data", variable_params.merge(statement: @statement, query_id: @query.id) %>
+    <%= blazer_js_var "data", variable_params.merge(statement: @statement, query_id: @query.id, data_source: @query.data_source) %>
 
     runQuery(data, showRun, showError)
   </script>
 <% end %>
 
-<% unless %w(mongodb elasticsearch).include?(Blazer.data_sources[@query.data_source].adapter) %>
+<% unless %w(mongodb).include?(Blazer.data_sources[@query.data_source].adapter) %>
   <script>
     // do not highlight really long queries
     // this can lead to performance issues

data/app/views/layouts/blazer/application.html.erb

@@ -4,15 +4,15 @@
     <title><%= blazer_title ? blazer_title : "Blazer" %></title>
 
     <meta charset="utf-8" />
-
+    <%= favicon_link_tag "blazer/favicon.png" %>
     <%= stylesheet_link_tag "blazer/application" %>
     <%= javascript_include_tag "blazer/application" %>
     <script>
       <%= blazer_js_var "rootPath", root_path %>
     </script>
     <% if blazer_maps? %>
-      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.css" %>
-      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.js" %>
+      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v3.1.1/mapbox.css", integrity: "sha384-o8tecBIfqi9yU5yYK2Ne/9A9hlOGFV9MBvCpmemvJH1XxqOe6h8Bl4mLxMi6PgjG", crossorigin: "anonymous" %>
+      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v3.1.1/mapbox.js", integrity: "sha384-j81LqvtvYigFzGSUgAoFijpvoq4yGoCJSOXI9DFaUEpenR029MBE3E/X5Gr+WdO0", crossorigin: "anonymous" %>
     <% end %>
     <%= csrf_meta_tags %>
   </head>

data/config/routes.rb

@@ -6,12 +6,16 @@ Blazer::Engine.routes.draw do
     get :tables, on: :collection
     get :columns, on: :collection
     get :schema, on: :collection
+    get :docs, on: :collection
   end
+
   resources :checks, except: [:show] do
     get :run, on: :member
   end
-  resources :dashboards do
+
+  resources :dashboards, except: [:index] do
     post :refresh, on: :member
   end
+
   root to: "queries#home"
 end

data/lib/blazer.rb

@@ -1,14 +1,19 @@
+# dependencies
 require "csv"
 require "yaml"
 require "chartkick"
 require "zip/zip"
 require "axlsx"
 require "safely/core"
+
+# modules
 require "blazer/version"
 require "blazer/data_source"
 require "blazer/result"
 require "blazer/excel_parser"
 require "blazer/run_statement"
+
+# adapters
 require "blazer/adapters/base_adapter"
 require "blazer/adapters/athena_adapter"
 require "blazer/adapters/bigquery_adapter"
@@ -20,6 +25,8 @@ require "blazer/adapters/mongodb_adapter"
 require "blazer/adapters/presto_adapter"
 require "blazer/adapters/sql_adapter"
 require "blazer/adapters/snowflake_adapter"
+
+# engine
 require "blazer/engine"
 
 module Blazer
@@ -37,14 +44,18 @@ module Blazer
     attr_accessor :subdomain
     attr_accessor :cache
     attr_accessor :transform_statement
+    attr_accessor :transform_variable
     attr_accessor :check_schedules
     attr_accessor :mapbox_access_token
     attr_accessor :assignees
     attr_accessor :anomaly_checks
+    attr_accessor :forecasting
     attr_accessor :async
     attr_accessor :images
     attr_accessor :query_viewable
     attr_accessor :query_editable
+    attr_accessor :override_csp
+    attr_accessor :slack_webhook_url
     attr_accessor :query_creatable
   end
   self.audit = true
@@ -54,8 +65,10 @@ module Blazer
   self.assignees = []
   self.subdomain = nil
   self.anomaly_checks = false
+  self.forecasting = false
   self.async = false
   self.images = false
+  self.override_csp = false
 
   TIMEOUT_MESSAGE = "Query timed out :("
   TIMEOUT_ERRORS = [
@@ -103,20 +116,11 @@ module Blazer
 
   def self.data_sources
     @data_sources ||= begin
-      ds = Hash[
-        settings["data_sources"].map do |id, s|
-          [id, Blazer::DataSource.new(id, s)]
-        end
-      ]
-      ds.default = ds.values.first
+      ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      settings["data_sources"].each do |id, s|
+        ds[id] = Blazer::DataSource.new(id, s)
+      end
       ds
-
-      # TODO Blazer 2.0
-      # ds2 = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
-      # ds.each do |k, v|
-      #   ds2[k] = v
-      # end
-      # ds2
     end
   end
 
@@ -186,10 +190,15 @@ module Blazer
 
   def self.send_failing_checks
     emails = {}
+    slack_channels = {}
+
     Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
       check.split_emails.each do |email|
         (emails[email] ||= []) << check
       end
+      check.split_slack_channels.each do |channel|
+        (slack_channels[channel] ||= []) << check
+      end
     end
 
     emails.each do |email, checks|
@@ -197,6 +206,16 @@ module Blazer
         Blazer::CheckMailer.failing_checks(email, checks).deliver_now
       end
     end
+
+    slack_channels.each do |channel, checks|
+      Safely.safely do
+        Blazer::SlackNotifier.failing_checks(channel, checks)
+      end
+    end
+  end
+
+  def self.slack?
+    slack_webhook_url.present?
   end
 
   def self.adapters

data/lib/blazer/adapters/athena_adapter.rb

@@ -13,7 +13,7 @@ module Blazer
             client.start_query_execution(
               query_string: statement,
               # use token so we fetch cached results after query is run
-              client_request_token: Digest::MD5.hexdigest(statement),
+              client_request_token: Digest::MD5.hexdigest([statement,data_source.id].join("/")),
               query_execution_context: {
                 database: database,
               },

data/lib/blazer/adapters/elasticsearch_adapter.rb

@@ -7,22 +7,17 @@ module Blazer
         error = nil
 
         begin
-          header, body = statement.gsub(/\/\/.+/, "").strip.split("\n", 2)
-          body = JSON.parse(body)
-          body["timeout"] ||= data_source.timeout if data_source.timeout
-          response = client.msearch(body: [JSON.parse(header), body])["responses"].first
-          if response["error"]
-            error = response["error"]
-          else
-            hits = response["hits"]["hits"]
-            source_keys = hits.flat_map { |r| r["_source"].keys }.uniq
-            hit_keys = (hits.first.try(:keys) || []) - ["_source"]
-            columns = source_keys + hit_keys
-            rows =
-              hits.map do |r|
-                source = r["_source"]
-                source_keys.map { |k| source[k] } + hit_keys.map { |k| r[k] }
+          response = client.xpack.sql.query(body: {query: "#{statement} /*#{comment}*/"})
+          columns = response["columns"].map { |v| v["name"] }
+          # Elasticsearch does not differentiate between dates and times
+          date_indexes = response["columns"].each_index.select { |i| response["columns"][i]["type"] == "date" }
+          if columns.any?
+            rows = response["rows"]
+            date_indexes.each do |i|
+              rows.each do |row|
+                row[i] = Time.parse(row[i])
               end
+            end
           end
         rescue => e
           error = e.message
@@ -32,11 +27,13 @@ module Blazer
       end
 
       def tables
-        client.indices.get_aliases(name: "*").map { |k, v| [k, v["aliases"].keys] }.flatten.uniq.sort
+        indices = client.cat.indices(format: "json").map { |v| v["index"] }
+        aliases = client.cat.aliases(format: "json").map { |v| v["alias"] }
+        (indices + aliases).uniq.sort
       end
 
       def preview_statement
-        %!// header\n{"index": "{table}"}\n\n// body\n{"query": {"match_all": {}}, "size": 10}!
+        "SELECT * FROM \"{table}\" LIMIT 10"
       end
 
       protected

data/lib/blazer/adapters/mongodb_adapter.rb

@@ -7,7 +7,7 @@ module Blazer
         error = nil
 
         begin
-          documents = db.command({:$eval => "#{statement.strip}.toArray()"}).documents.first["retval"]
+          documents = db.command({:$eval => "#{statement.strip}.toArray()", nolock: true}).documents.first["retval"]
           columns = documents.flat_map { |r| r.keys }.uniq
           rows = documents.map { |r| columns.map { |c| r[c] } }
         rescue => e

data/lib/blazer/adapters/sql_adapter.rb

@@ -155,7 +155,13 @@ module Blazer
         if postgresql? || redshift?
           execute("SET #{use_transaction? ? "LOCAL " : ""}statement_timeout = #{timeout.to_i * 1000}")
         elsif mysql?
-          execute("SET max_execution_time = #{timeout.to_i * 1000}")
+          # use send as this method is private in Rails 4.2
+          mariadb = connection_model.connection.send(:mariadb?) rescue false
+          if mariadb
+            execute("SET max_statement_time = #{timeout.to_i * 1000}")
+          else
+            execute("SET max_execution_time = #{timeout.to_i * 1000}")
+          end
         else
           raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
         end