blazer 2.2.6

data/app/views/blazer/queries/schema.html.erb

@@ -0,0 +1,55 @@
+<% blazer_title "Schema: #{@data_source.name}" %>
+
+<h1>Schema: <%= @data_source.name %></h1>
+
+<hr />
+
+<div id="header">
+  <input id="search" type="text" placeholder="Start typing a table or column" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
+
+<% @schema.each do |table| %>
+  <table class="table schema-table">
+    <thead>
+      <tr>
+        <th colspan="2">
+          <% if table[:schema] != "public" %><%= table[:schema] %>.<% end %><%= table[:table] %>
+        </th>
+      </tr>
+    </thead>
+    <tbody>
+      <% table[:columns].each do |column| %>
+        <tr>
+          <td style="width: 60%;"><%= column[:name] %></td>
+          <td class="text-muted"><%= column[:data_type] %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>
+
+<script>
+  $("#search").on("keyup", function() {
+    var value = $(this).val().toLowerCase()
+    $(".schema-table").filter(function() {
+      // if found in table name, show entire table
+      // if just found in rows, show row
+
+      var found = $(this).find("thead").text().toLowerCase().indexOf(value) > -1
+
+      if (found) {
+        $(this).find("tbody tr").toggle(true)
+      } else {
+        $(this).find("tbody tr").filter(function() {
+          var found2 = $(this).text().toLowerCase().indexOf(value) > -1
+          $(this).toggle(found2)
+          if (found2) {
+            found = true
+          }
+        })
+      }
+
+      $(this).toggle(found)
+    })
+  }).focus()
+</script>

data/app/views/blazer/queries/show.html.erb

@@ -0,0 +1,75 @@
+<% blazer_title @query.name %>
+
+<div class="topbar">
+  <div class="container">
+    <div class="row" style="padding-top: 13px;">
+      <div class="col-sm-9">
+        <%= render partial: "blazer/nav" %>
+        <h3 style="line-height: 34px; display: inline; margin-left: 5px;">
+          <%= @query.name %>
+        </h3>
+      </div>
+      <div class="col-sm-3 text-right">
+        <%= link_to "Edit", edit_query_path(@query, variable_params(@query)), class: "btn btn-default", disabled: !@query.editable?(blazer_user) %>
+        <%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
+
+        <% if !@error && @success %>
+          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</div>
+
+<div style="margin-bottom: 60px;"></div>
+
+<% if @sql_errors.any? %>
+  <div class="alert alert-danger">
+    <ul>
+      <% @sql_errors.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+    </ul>
+  </div>
+<% end %>
+
+<% if @query.description.present? %>
+  <p><%= @query.description %></p>
+<% end %>
+
+<%= render partial: "blazer/variables", locals: {action: query_path(@query)} %>
+
+<pre id="code"><code><%= @statement %></code></pre>
+
+<% if @success %>
+  <div id="results">
+    <p class="text-muted">Loading...</p>
+  </div>
+
+  <script>
+    function showRun(data) {
+      $("#results").html(data)
+      $("#results table").stupidtable(stupidtableCustomSettings).stickyTableHeaders({fixedOffset: 60})
+    }
+
+    function showError(message) {
+      $("#results").addClass("query-error").html(message)
+    }
+
+    <% data = variable_params(@query).merge(statement: @statement, query_id: @query.id, data_source: @query.data_source) %>
+    <% data.merge!(forecast: "t") if params[:forecast] %>
+    <%= blazer_js_var "data", data %>
+
+    runQuery(data, showRun, showError)
+  </script>
+<% end %>
+
+<% unless %w(mongodb).include?(Blazer.data_sources[@query.data_source].adapter) %>
+  <script>
+    // do not highlight really long queries
+    // this can lead to performance issues
+    if ($("code").text().length < 10000) {
+      hljs.highlightBlock(document.getElementById("code"));
+    }
+  </script>
+<% end %>

data/app/views/layouts/blazer/application.html.erb

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title><%= blazer_title ? blazer_title : "Blazer" %></title>
+
+    <meta charset="utf-8" />
+    <%= favicon_link_tag "blazer/favicon.png" %>
+    <%= stylesheet_link_tag "blazer/application" %>
+    <%= javascript_include_tag "blazer/application" %>
+    <script>
+      <%= blazer_js_var "rootPath", root_path %>
+    </script>
+    <% if blazer_maps? %>
+      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v3.3.1/mapbox.css", integrity: "sha384-vxzdEt+wZRPNQbhChjmiaFMLWg86IGuq1NGDehJHsD2mphYkxXll/eSs16WWi6Dq", crossorigin: "anonymous" %>
+      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v3.3.1/mapbox.js", integrity: "sha384-CTBEiDLiZJ8gkAQ3fYGoeiRp81/ecNiBkGz11jXFALOZ6++rbnqmdo6OImkmr1MO", crossorigin: "anonymous" %>
+    <% end %>
+    <%= csrf_meta_tags %>
+  </head>
+  <body>
+    <div class="container">
+      <%= yield %>
+    </div>
+  </body>
+</html>

data/config/routes.rb

@@ -0,0 +1,20 @@
+Blazer::Engine.routes.draw do
+  resources :queries do
+    post :run, on: :collection # err on the side of caution
+    post :cancel, on: :collection
+    post :refresh, on: :member
+    get :tables, on: :collection
+    get :schema, on: :collection
+    get :docs, on: :collection
+  end
+
+  resources :checks, except: [:show] do
+    get :run, on: :member
+  end
+
+  resources :dashboards, except: [:index] do
+    post :refresh, on: :member
+  end
+
+  root to: "queries#home"
+end

data/lib/blazer.rb

@@ -0,0 +1,231 @@
+# dependencies
+require "csv"
+require "yaml"
+require "chartkick"
+require "safely/core"
+
+# modules
+require "blazer/version"
+require "blazer/data_source"
+require "blazer/result"
+require "blazer/run_statement"
+
+# adapters
+require "blazer/adapters/base_adapter"
+require "blazer/adapters/athena_adapter"
+require "blazer/adapters/bigquery_adapter"
+require "blazer/adapters/cassandra_adapter"
+require "blazer/adapters/drill_adapter"
+require "blazer/adapters/druid_adapter"
+require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/influxdb_adapter"
+require "blazer/adapters/mongodb_adapter"
+require "blazer/adapters/neo4j_adapter"
+require "blazer/adapters/presto_adapter"
+require "blazer/adapters/salesforce_adapter"
+require "blazer/adapters/soda_adapter"
+require "blazer/adapters/sql_adapter"
+require "blazer/adapters/snowflake_adapter"
+
+# engine
+require "blazer/engine"
+
+module Blazer
+  class Error < StandardError; end
+  class TimeoutNotSupported < Error; end
+
+  class << self
+    attr_accessor :audit
+    attr_reader :time_zone
+    attr_accessor :user_name
+    attr_writer :user_class
+    attr_writer :user_method
+    attr_accessor :before_action
+    attr_accessor :from_email
+    attr_accessor :cache
+    attr_accessor :transform_statement
+    attr_accessor :transform_variable
+    attr_accessor :check_schedules
+    attr_accessor :anomaly_checks
+    attr_accessor :forecasting
+    attr_accessor :async
+    attr_accessor :images
+    attr_accessor :query_viewable
+    attr_accessor :query_editable
+    attr_accessor :override_csp
+    attr_accessor :slack_webhook_url
+    attr_accessor :mapbox_access_token
+  end
+  self.audit = true
+  self.user_name = :name
+  self.check_schedules = ["5 minutes", "1 hour", "1 day"]
+  self.anomaly_checks = false
+  self.forecasting = false
+  self.async = false
+  self.images = false
+  self.override_csp = false
+
+  TIMEOUT_MESSAGE = "Query timed out :("
+  TIMEOUT_ERRORS = [
+    "canceling statement due to statement timeout", # postgres
+    "canceling statement due to conflict with recovery", # postgres
+    "cancelled on user's request", # redshift
+    "canceled on user's request", # redshift
+    "system requested abort", # redshift
+    "maximum statement execution time exceeded" # mysql
+  ]
+
+  def self.time_zone=(time_zone)
+    @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+  end
+
+  def self.user_class
+    if !defined?(@user_class)
+      @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil)
+    end
+    @user_class
+  end
+
+  def self.user_method
+    if !defined?(@user_method)
+      @user_method = settings["user_method"]
+      if user_class
+        @user_method ||= "current_#{user_class.to_s.downcase.singularize}"
+      end
+    end
+    @user_method
+  end
+
+  def self.settings
+    @settings ||= begin
+      path = Rails.root.join("config", "blazer.yml").to_s
+      if File.exist?(path)
+        YAML.load(ERB.new(File.read(path)).result)
+      else
+        {}
+      end
+    end
+  end
+
+  def self.data_sources
+    @data_sources ||= begin
+      ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      settings["data_sources"].each do |id, s|
+        ds[id] = Blazer::DataSource.new(id, s)
+      end
+      ds
+    end
+  end
+
+  def self.extract_vars(statement)
+    # strip commented out lines
+    # and regex {1} or {1,2}
+    statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+  end
+
+  def self.run_checks(schedule: nil)
+    checks = Blazer::Check.includes(:query)
+    checks = checks.where(schedule: schedule) if schedule
+    checks.find_each do |check|
+      next if check.state == "disabled"
+      Safely.safely { run_check(check) }
+    end
+  end
+
+  def self.run_check(check)
+    tries = 1
+
+    ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
+      # try 3 times on timeout errors
+      data_source = data_sources[check.query.data_source]
+      statement = check.query.statement
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      while tries <= 3
+        result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
+        if result.timed_out?
+          Rails.logger.info "[blazer timeout] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        elsif result.error.to_s.start_with?("PG::ConnectionBad")
+          data_source.reconnect
+          Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        else
+          break
+        end
+      end
+
+      begin
+        check.reload # in case state has changed since job started
+        check.update_state(result)
+      rescue ActiveRecord::RecordNotFound
+        # check deleted
+      end
+
+      # TODO use proper logfmt
+      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
+
+      instrument[:statement] = statement
+      instrument[:data_source] = data_source
+      instrument[:state] = check.state
+      instrument[:rows] = result.rows.try(:size)
+      instrument[:error] = result.error
+      instrument[:tries] = tries
+    end
+  end
+
+  def self.send_failing_checks
+    emails = {}
+    slack_channels = {}
+
+    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
+      check.split_emails.each do |email|
+        (emails[email] ||= []) << check
+      end
+      check.split_slack_channels.each do |channel|
+        (slack_channels[channel] ||= []) << check
+      end
+    end
+
+    emails.each do |email, checks|
+      Safely.safely do
+        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+      end
+    end
+
+    slack_channels.each do |channel, checks|
+      Safely.safely do
+        Blazer::SlackNotifier.failing_checks(channel, checks)
+      end
+    end
+  end
+
+  def self.slack?
+    slack_webhook_url.present?
+  end
+
+  def self.adapters
+    @adapters ||= {}
+  end
+
+  def self.register_adapter(name, adapter)
+    adapters[name] = adapter
+  end
+end
+
+Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
+Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
+Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
+Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
+Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
+Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
+Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
+Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
+Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
+Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/adapters/athena_adapter.rb

@@ -0,0 +1,129 @@
+module Blazer
+  module Adapters
+    class AthenaAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        require "digest/md5"
+
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          resp =
+            client.start_query_execution(
+              query_string: statement,
+              # use token so we fetch cached results after query is run
+              client_request_token: Digest::MD5.hexdigest([statement,data_source.id].join("/")),
+              query_execution_context: {
+                database: database,
+              },
+              result_configuration: {
+                output_location: settings["output_location"]
+              }
+            )
+          query_execution_id = resp.query_execution_id
+
+          timeout = data_source.timeout || 300
+          stop_at = Time.now + timeout
+          resp = nil
+
+          begin
+            resp = client.get_query_results(
+              query_execution_id: query_execution_id
+            )
+          rescue Aws::Athena::Errors::InvalidRequestException => e
+            unless e.message.start_with?("Query has not yet finished.")
+              raise e
+            end
+            if Time.now < stop_at
+              sleep(3)
+              retry
+            end
+          end
+
+          if resp && resp.result_set
+            column_info = resp.result_set.result_set_metadata.column_info
+            columns = column_info.map(&:name)
+            column_types = column_info.map(&:type)
+
+            untyped_rows = []
+
+            # paginated
+            resp.each do |page|
+              untyped_rows.concat page.result_set.rows.map { |r| r.data.map(&:var_char_value) }
+            end
+
+            utc = ActiveSupport::TimeZone['Etc/UTC']
+
+            rows = untyped_rows[1..-1] || []
+            rows = untyped_rows[0..-1] unless column_info.present?
+            column_types.each_with_index do |ct, i|
+              # TODO more column_types
+              case ct
+              when "timestamp"
+                rows.each do |row|
+                  row[i] = utc.parse(row[i])
+                end
+              when "date"
+                rows.each do |row|
+                  row[i] = Date.parse(row[i])
+                end
+              when "bigint"
+                rows.each do |row|
+                  row[i] = row[i].to_i
+                end
+              when "double"
+                rows.each do |row|
+                  row[i] = row[i].to_f
+                end
+              end
+            end
+          elsif resp
+            error = fetch_error(query_execution_id)
+          else
+            error = Blazer::TIMEOUT_MESSAGE
+          end
+        rescue Aws::Athena::Errors::InvalidRequestException => e
+          error = e.message
+          if error == "Query did not finish successfully. Final query state: FAILED"
+            error = fetch_error(query_execution_id)
+          end
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        glue.get_tables(database_name: database).table_list.map(&:name).sort
+      end
+
+      def schema
+        glue.get_tables(database_name: database).table_list.map { |t| {table: t.name, columns: t.storage_descriptor.columns.map { |c| {name: c.name, data_type: c.type} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      private
+
+      def database
+        @database ||= settings["database"] || "default"
+      end
+
+      def fetch_error(query_execution_id)
+        client.get_query_execution(
+          query_execution_id: query_execution_id
+        ).query_execution.status.state_change_reason
+      end
+
+      def client
+        @client ||= Aws::Athena::Client.new
+      end
+
+      def glue
+        @glue ||= Aws::Glue::Client.new
+      end
+    end
+  end
+end