sql-jarvis 1.8.0

data/app/views/blazer/queries/new.html.erb

@@ -0,0 +1,2 @@
+<% blazer_title "New Query" %>
+<%= render partial: "form" %>

data/app/views/blazer/queries/run.html.erb

@@ -0,0 +1,163 @@
+<% if @error %>
+  <div class="alert alert-danger"><%= @error.first(200) %></div>
+<% elsif !@success %>
+  <% if @only_chart %>
+    <p class="text-muted">Select variables</p>
+  <% else %>
+    <div class="alert alert-info">Can’t preview queries with variables...yet!</div>
+  <% end %>
+<% else %>
+  <% unless @only_chart %>
+    <% if @cached_at || @just_cached %>
+      <p class="text-muted" style="float: right;">
+        <% if @cached_at %>
+          Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
+        <% elsif !params[:data_source] %>
+          Cached just now
+          <% if @data_source.cache_mode == "slow" %>
+            (over <%= "%g" % @data_source.cache_slow_threshold %>s)
+          <% end %>
+        <% end %>
+
+        <% if @query && !params[:data_source] %>
+          <%= link_to "Refresh", refresh_query_path(@query, variable_params), method: :post %>
+        <% end %>
+      </p>
+    <% end %>
+    <p class="text-muted">
+      <%= pluralize(@rows.size, "row") %>
+
+      <% @checks.select(&:state).each do |check| %>
+        &middot; <small class="check-state <%= check.state.parameterize.gsub("-", "_") %>"><%= link_to check.state.upcase, edit_check_path(check) %></small>
+        <% if check.try(:message) %>
+          &middot; <%= check.message %>
+        <% end %>
+      <% end %>
+    </p>
+  <% end %>
+  <% if @rows.any? %>
+    <% values = @rows.first %>
+    <% chart_id = SecureRandom.hex %>
+    <% column_types = @result.column_types %>
+    <% chart_type = @result.chart_type %>
+    <% chart_options = {id: chart_id, min: nil} %>
+    <% series_library = {} %>
+    <% target_index = @columns.index { |k| k.downcase == "target" } %>
+    <% if target_index %>
+      <% series_library[target_index - 1] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
+    <% end %>
+    <% if blazer_maps? && @markers.any? %>
+      <div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
+      <script>
+        L.mapbox.accessToken = '<%= Blazer.mapbox_access_token %>';
+        var map = L.mapbox.map('map', 'ankane.ioo8nki0');
+        <%= blazer_js_var "markers", @markers %>
+        var featureLayer = L.mapbox.featureLayer().addTo(map);
+        var geojson = [];
+        for (var i = 0; i < markers.length; i++) {
+          var marker = markers[i];
+          geojson.push({
+            type: 'Feature',
+            geometry: {
+              type: 'Point',
+              coordinates: [
+                marker.longitude,
+                marker.latitude
+              ]
+            },
+            properties: {
+              description: marker.title,
+              'marker-color': '#f86767',
+              'marker-size': 'medium'
+            }
+          });
+        }
+        featureLayer.setGeoJSON(geojson);
+        map.fitBounds(featureLayer.getBounds());
+      </script>
+    <% elsif chart_type == "line" %>
+      <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: blazer_series_name(k), data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
+    <% elsif chart_type == "line2" %>
+      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
+    <% elsif chart_type == "bar" %>
+      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
+    <% elsif chart_type == "bar2" %>
+      <% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %>
+      <% labels = first_20.map { |r| r[0] }.uniq %>
+      <% series = first_20.map { |r| r[1] }.uniq %>
+      <% labels.each do |l| %>
+        <% series.each do |s| %>
+          <% first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } %>
+        <% end %>
+      <% end %>
+      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, id: chart_id %>
+    <% elsif chart_type == "scatter" %>
+      <%= scatter_chart @rows, xtitle: @columns[0], ytitle: @columns[1], id: chart_id %>
+    <% elsif @only_chart %>
+      <% if @rows.size == 1 && @rows.first.size == 1 %>
+        <% v = @rows.first.first %>
+        <% if v.is_a?(String) && v == "" %>
+          <div class="text-muted">empty string</div>
+        <% else %>
+          <p style="font-size: 160px;"><%= blazer_format_value(@columns.first, v) %></p>
+        <% end %>
+      <% else %>
+        <% @no_chart = true %>
+      <% end %>
+    <% end %>
+
+    <% unless @only_chart && !@no_chart %>
+      <% header_width = 100 / @columns.size.to_f %>
+      <div class="results-container">
+        <% if @columns == ["QUERY PLAN"] %>
+          <pre><code><%= @rows.map { |r| r[0] }.join("\n") %></code></pre>
+        <% else %>
+          <table class="table results-table" style="margin-bottom: 0;">
+            <thead>
+              <tr>
+                <% @columns.each_with_index do |key, i| %>
+                  <% type = @column_types[i] %>
+                  <th style="width: <%= header_width %>%;" data-sort="<%= type %>">
+                    <div style="min-width: <%= @min_width_types.include?(i) ? 180 : 60 %>px;">
+                      <%= key %>
+                    </div>
+                  </th>
+                <% end %>
+              </tr>
+            </thead>
+            <tbody>
+              <% @rows.each do |row| %>
+                <tr>
+                  <% row.each_with_index do |v, i| %>
+                    <% k = @columns[i] %>
+                    <td>
+                      <% if v.is_a?(Time) %>
+                        <% v = blazer_time_value(@data_source, k, v) %>
+                      <% end %>
+
+                      <% unless v.nil? %>
+                        <% if v.is_a?(String) && v == "" %>
+                          <div class="text-muted">empty string</div>
+                        <% elsif @linked_columns[k] %>
+                          <%= link_to blazer_format_value(k, v), @linked_columns[k].gsub("{value}", u(v.to_s)), target: "_blank" %>
+                        <% else %>
+                          <%= blazer_format_value(k, v) %>
+                        <% end %>
+                      <% end %>
+
+                      <% if v2 = (@boom[k] || {})[v.nil? ? v : v.to_s] %>
+                        <div class="text-muted"><%= v2 %></div>
+                      <% end %>
+                    </td>
+                  <% end %>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        <% end %>
+      </div>
+    <% end %>
+  <% elsif @only_chart %>
+    <p class="text-muted">No rows</p>
+  <% end %>
+<% end %>

data/app/views/blazer/queries/schema.html.erb

@@ -0,0 +1,18 @@
+<% @schema.each do |table| %>
+  <h4>
+    <%= table[:table] %>
+    <% if table[:schema] != "public" %>
+      <small><%= table[:schema] %></small>
+    <% end %>
+  </h4>
+  <table class="table" style="max-width: 500px;">
+    <tbody>
+      <% table[:columns].each do |column| %>
+        <tr>
+          <td style="width: 60%;"><%= column[:name] %></td>
+          <td class="text-muted"><%= column[:data_type] %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

data/app/views/blazer/queries/show.html.erb

@@ -0,0 +1,73 @@
+<% blazer_title @query.name %>
+
+<div class="topbar">
+  <div class="container">
+    <div class="row" style="padding-top: 13px;">
+      <div class="col-sm-9">
+        <%= render partial: "blazer/nav" %>
+        <h3 style="margin: 0; line-height: 34px; display: inline;">
+          <%= @query.name %>
+        </h3>
+      </div>
+      <div class="col-sm-3 text-right">
+        <%= link_to "Edit", edit_query_path(@query, variable_params), class: "btn btn-default", disabled: !@query.editable?(blazer_user) %>
+        <%= link_to "Fork", new_query_path(variable_params.merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
+
+        <% if !@error && @success %>
+          <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv"), params: {statement: @statement}, class: "btn btn-primary" %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</div>
+
+<div style="margin-bottom: 60px;"></div>
+
+<% if @sql_errors.any? %>
+  <div class="alert alert-danger">
+    <ul>
+      <% @sql_errors.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+    </ul>
+  </div>
+<% end %>
+
+<% if @query.description.present? %>
+  <p><%= @query.description %></p>
+<% end %>
+
+<%= render partial: "blazer/variables", locals: {action: query_path(@query)} %>
+
+<pre id="code"><code><%= @statement %></code></pre>
+
+<% if @success %>
+  <div id="results">
+    <p class="text-muted">Loading...</p>
+  </div>
+
+  <script>
+    function showRun(data) {
+      $("#results").html(data)
+      $("#results table").stupidtable().stickyTableHeaders({fixedOffset: 60})
+    }
+
+    function showError(message) {
+      $("#results").addClass("query-error").html(message)
+    }
+
+    <%= blazer_js_var "data", variable_params.merge(statement: @statement, query_id: @query.id) %>
+
+    runQuery(data, showRun, showError)
+  </script>
+<% end %>
+
+<% if %w[sql presto drill bigquery athena].include?(Blazer.data_sources[@query.data_source].adapter) %>
+  <script>
+    // do not highlight really long queries
+    // this can lead to performance issues
+    if ($("code").text().length < 10000) {
+      hljs.highlightBlock(document.getElementById("code"));
+    }
+  </script>
+<% end %>

data/app/views/layouts/blazer/application.html.erb

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title><%= blazer_title ? blazer_title : "Blazer" %></title>
+
+    <meta charset="utf-8" />
+
+    <%= stylesheet_link_tag "blazer/application" %>
+    <%= javascript_include_tag "blazer/application" %>
+    <script>
+      <%= blazer_js_var "rootPath", root_path %>
+    </script>
+    <% if blazer_maps? %>
+      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.css" %>
+      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.js" %>
+    <% end %>
+    <%= csrf_meta_tags %>
+  </head>
+  <body>
+    <div class="container">
+      <%= yield %>
+    </div>
+  </body>
+</html>

data/blazer.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "blazer/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "sql-jarvis"
+  spec.version       = Blazer::VERSION
+  spec.authors       = ["ThanhKhoaIT"]
+  spec.email         = ["thanhkhoait@gmail.com"]
+  spec.summary       = "Fork from ankane! Explore your data with SQL. Easily create charts and dashboards, and share them with your team."
+  spec.homepage      = "https://github.com/ThanhKhoaIT/blazer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", ">= 4"
+  spec.add_dependency "chartkick"
+  spec.add_dependency "safely_block", ">= 0.1.1"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/config/routes.rb

@@ -0,0 +1,16 @@
+Blazer::Engine.routes.draw do
+  resources :queries do
+    post :run, on: :collection # err on the side of caution
+    post :cancel, on: :collection
+    post :refresh, on: :member
+    get :tables, on: :collection
+    get :schema, on: :collection
+  end
+  resources :checks, except: [:show] do
+    get :run, on: :member
+  end
+  resources :dashboards do
+    post :refresh, on: :member
+  end
+  root to: "queries#home"
+end

data/lib/blazer.rb

@@ -0,0 +1,185 @@
+require "csv"
+require "yaml"
+require "chartkick"
+require "safely/core"
+require "blazer/version"
+require "blazer/data_source"
+require "blazer/result"
+require "blazer/run_statement"
+require "blazer/adapters/base_adapter"
+require "blazer/adapters/athena_adapter"
+require "blazer/adapters/bigquery_adapter"
+require "blazer/adapters/drill_adapter"
+require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/mongodb_adapter"
+require "blazer/adapters/presto_adapter"
+require "blazer/adapters/sql_adapter"
+require "blazer/engine"
+
+module Blazer
+  class Error < StandardError; end
+  class TimeoutNotSupported < Error; end
+
+  class << self
+    attr_accessor :audit
+    attr_reader :time_zone
+    attr_accessor :user_name
+    attr_accessor :user_class
+    attr_accessor :user_method
+    attr_accessor :before_action
+    attr_accessor :from_email
+    attr_accessor :cache
+    attr_accessor :transform_statement
+    attr_accessor :check_schedules
+    attr_accessor :mapbox_access_token
+    attr_accessor :anomaly_checks
+    attr_accessor :async
+    attr_accessor :images
+    attr_accessor :query_editable
+  end
+  self.audit = true
+  self.user_name = :name
+  self.check_schedules = ["5 minutes", "1 hour", "1 day"]
+  self.mapbox_access_token = nil
+  self.anomaly_checks = false
+  self.async = false
+  self.images = false
+
+  TIMEOUT_MESSAGE = "Query timed out :("
+  TIMEOUT_ERRORS = [
+    "canceling statement due to statement timeout", # postgres
+    "canceling statement due to conflict with recovery", # postgres
+    "cancelled on user's request", # redshift
+    "canceled on user's request", # redshift
+    "system requested abort", # redshift
+    "maximum statement execution time exceeded" # mysql
+  ]
+  BELONGS_TO_OPTIONAL = {}
+  BELONGS_TO_OPTIONAL[:optional] = true if Rails::VERSION::MAJOR >= 5
+
+  def self.time_zone=(time_zone)
+    @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+  end
+
+  def self.settings
+    @settings ||= begin
+      path = Rails.root.join("config", "blazer.yml").to_s
+      if File.exist?(path)
+        YAML.load(ERB.new(File.read(path)).result)
+      else
+        {}
+      end
+    end
+  end
+
+  def self.data_sources
+    @data_sources ||= begin
+      ds = Hash[
+        settings["data_sources"].map do |id, s|
+          [id, Blazer::DataSource.new(id, s)]
+        end
+      ]
+      ds.default = ds.values.first
+      ds
+
+      # TODO Blazer 2.0
+      # ds2 = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      # ds.each do |k, v|
+      #   ds2[k] = v
+      # end
+      # ds2
+    end
+  end
+
+  def self.extract_vars(statement)
+    # strip commented out lines
+    # and regex {1} or {1,2}
+    statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+  end
+
+  def self.run_checks(schedule: nil)
+    checks = Blazer::Check.includes(:query)
+    checks = checks.where(schedule: schedule) if schedule
+    checks.find_each do |check|
+      next if check.state == "disabled"
+      Safely.safely { run_check(check) }
+    end
+  end
+
+  def self.run_check(check)
+    rows = nil
+    error = nil
+    tries = 1
+
+    ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
+      # try 3 times on timeout errors
+      data_source = data_sources[check.query.data_source]
+      statement = check.query.statement
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      while tries <= 3
+        result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
+        if result.timed_out?
+          Rails.logger.info "[blazer timeout] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        elsif result.error.to_s.start_with?("PG::ConnectionBad")
+          data_source.reconnect
+          Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        else
+          break
+        end
+      end
+
+      begin
+        check.reload # in case state has changed since job started
+        check.update_state(result)
+      rescue ActiveRecord::RecordNotFound
+        # check deleted
+      end
+
+      # TODO use proper logfmt
+      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
+
+      instrument[:statement] = statement
+      instrument[:data_source] = data_source
+      instrument[:state] = check.state
+      instrument[:rows] = result.rows.try(:size)
+      instrument[:error] = result.error
+      instrument[:tries] = tries
+    end
+  end
+
+  def self.send_failing_checks
+    emails = {}
+    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
+      check.split_emails.each do |email|
+        (emails[email] ||= []) << check
+      end
+    end
+
+    emails.each do |email, checks|
+      Safely.safely do
+        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+      end
+    end
+  end
+
+  def self.adapters
+    @adapters ||= {}
+  end
+
+  def self.register_adapter(name, adapter)
+    adapters[name] = adapter
+  end
+end
+
+Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
+Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
+Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
+Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
+Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter