railsblazer 2.0.0

data/blazer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "blazer/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "railsblazer"
+  spec.version       = Blazer::VERSION
+  spec.authors       = ["Andrew Kane"]
+  spec.email         = ["andrew@chartkick.com"]
+  spec.summary       = "Explore your data with SQL. Easily create charts and dashboards, and share them with your team."
+  spec.homepage      = "https://github.com/ankane/blazer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "railties", ">= 4"
+  spec.add_dependency "activerecord", ">= 4"
+  spec.add_dependency "chartkick"
+  spec.add_dependency "safely_block", ">= 0.1.1"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/config/routes.rb

@@ -0,0 +1,16 @@
+Blazer::Engine.routes.draw do
+  resources :queries do
+    post :run, on: :collection # err on the side of caution
+    post :cancel, on: :collection
+    post :refresh, on: :member
+    get :tables, on: :collection
+    get :schema, on: :collection
+  end
+  resources :checks, except: [:show] do
+    get :run, on: :member
+  end
+  resources :dashboards do
+    post :refresh, on: :member
+  end
+  root to: "queries#home"
+end

data/lib/blazer.rb

@@ -0,0 +1,223 @@
+require "csv"
+require "yaml"
+require "chartkick"
+require "safely/core"
+require "blazer/version"
+require "blazer/data_source"
+require "blazer/result"
+require "blazer/run_statement"
+require "blazer/adapters/base_adapter"
+require "blazer/adapters/athena_adapter"
+require "blazer/adapters/bigquery_adapter"
+require "blazer/adapters/cassandra_adapter"
+require "blazer/adapters/drill_adapter"
+require "blazer/adapters/druid_adapter"
+require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/mongodb_adapter"
+require "blazer/adapters/presto_adapter"
+require "blazer/adapters/sql_adapter"
+require "blazer/adapters/snowflake_adapter"
+require "blazer/engine"
+
+module Blazer
+  class Error < StandardError; end
+  class TimeoutNotSupported < Error; end
+
+  class << self
+    attr_accessor :audit
+    attr_reader :time_zone
+    attr_accessor :user_name
+    attr_writer :user_class
+    attr_writer :user_method
+    attr_accessor :before_action
+    attr_accessor :from_email
+    attr_accessor :cache
+    attr_accessor :transform_statement
+    attr_accessor :check_schedules
+    attr_accessor :anomaly_checks
+    attr_accessor :async
+    attr_accessor :images
+    attr_accessor :query_viewable
+    attr_accessor :query_editable
+    attr_accessor :override_csp
+    attr_accessor :slack_webhook_url
+  end
+  self.audit = true
+  self.user_name = :name
+  self.check_schedules = ["5 minutes", "1 hour", "1 day"]
+  self.anomaly_checks = false
+  self.async = false
+  self.images = false
+  self.override_csp = false
+
+  TIMEOUT_MESSAGE = "Query timed out :("
+  TIMEOUT_ERRORS = [
+    "canceling statement due to statement timeout", # postgres
+    "canceling statement due to conflict with recovery", # postgres
+    "cancelled on user's request", # redshift
+    "canceled on user's request", # redshift
+    "system requested abort", # redshift
+    "maximum statement execution time exceeded" # mysql
+  ]
+  BELONGS_TO_OPTIONAL = {}
+  BELONGS_TO_OPTIONAL[:optional] = true if Rails::VERSION::MAJOR >= 5
+
+  def self.time_zone=(time_zone)
+    @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+  end
+
+  def self.user_class
+    if !defined?(@user_class)
+      @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil)
+    end
+    @user_class
+  end
+
+  def self.user_method
+    if !defined?(@user_method)
+      @user_method = settings["user_method"]
+      if user_class
+        @user_method ||= "current_#{user_class.to_s.downcase.singularize}"
+      end
+    end
+    @user_method
+  end
+
+  def self.settings
+    @settings ||= begin
+      path = Rails.root.join("config", "blazer.yml").to_s
+      if File.exist?(path)
+        YAML.load(ERB.new(File.read(path)).result)
+      else
+        {}
+      end
+    end
+  end
+
+  def self.data_sources
+    @data_sources ||= begin
+      ds = Hash[
+        settings["data_sources"].map do |id, s|
+          [id, Blazer::DataSource.new(id, s)]
+        end
+      ]
+      ds.default = ds.values.first
+      ds
+
+      # TODO Blazer 2.0
+      # ds2 = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      # ds.each do |k, v|
+      #   ds2[k] = v
+      # end
+      # ds2
+    end
+  end
+
+  def self.extract_vars(statement)
+    # strip commented out lines
+    # and regex {1} or {1,2}
+    statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+  end
+
+  def self.run_checks(schedule: nil)
+    checks = Blazer::Check.includes(:query)
+    checks = checks.where(schedule: schedule) if schedule
+    checks.find_each do |check|
+      next if check.state == "disabled"
+      Safely.safely { run_check(check) }
+    end
+  end
+
+  def self.run_check(check)
+    tries = 1
+
+    ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
+      # try 3 times on timeout errors
+      data_source = data_sources[check.query.data_source]
+      statement = check.query.statement
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      while tries <= 3
+        result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
+        if result.timed_out?
+          Rails.logger.info "[blazer timeout] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        elsif result.error.to_s.start_with?("PG::ConnectionBad")
+          data_source.reconnect
+          Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        else
+          break
+        end
+      end
+
+      begin
+        check.reload # in case state has changed since job started
+        check.update_state(result)
+      rescue ActiveRecord::RecordNotFound
+        # check deleted
+      end
+
+      # TODO use proper logfmt
+      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
+
+      instrument[:statement] = statement
+      instrument[:data_source] = data_source
+      instrument[:state] = check.state
+      instrument[:rows] = result.rows.try(:size)
+      instrument[:error] = result.error
+      instrument[:tries] = tries
+    end
+  end
+
+  def self.send_failing_checks
+    emails = {}
+    slack_channels = {}
+
+    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
+      check.split_emails.each do |email|
+        (emails[email] ||= []) << check
+      end
+      check.split_slack_channels.each do |channel|
+        (slack_channels[channel] ||= []) << check
+      end
+    end
+
+    emails.each do |email, checks|
+      Safely.safely do
+        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+      end
+    end
+
+    slack_channels.each do |channel, checks|
+      Safely.safely do
+        Blazer::SlackNotifier.failing_checks(channel, checks)
+      end
+    end
+  end
+
+  def self.slack?
+    slack_webhook_url.present?
+  end
+
+  def self.adapters
+    @adapters ||= {}
+  end
+
+  def self.register_adapter(name, adapter)
+    adapters[name] = adapter
+  end
+end
+
+Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
+Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
+Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
+Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
+Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
+Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/adapters/athena_adapter.rb

@@ -0,0 +1,128 @@
+module Blazer
+  module Adapters
+    class AthenaAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        require "digest/md5"
+
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          resp =
+            client.start_query_execution(
+              query_string: statement,
+              # use token so we fetch cached results after query is run
+              client_request_token: Digest::MD5.hexdigest(statement),
+              query_execution_context: {
+                database: database,
+              },
+              result_configuration: {
+                output_location: settings["output_location"]
+              }
+            )
+          query_execution_id = resp.query_execution_id
+
+          timeout = data_source.timeout || 300
+          stop_at = Time.now + timeout
+          resp = nil
+
+          begin
+            resp = client.get_query_results(
+              query_execution_id: query_execution_id
+            )
+          rescue Aws::Athena::Errors::InvalidRequestException => e
+            if e.message != "Query has not yet finished. Current state: RUNNING"
+              raise e
+            end
+            if Time.now < stop_at
+              sleep(3)
+              retry
+            end
+          end
+
+          if resp && resp.result_set
+            column_info = resp.result_set.result_set_metadata.column_info
+            columns = column_info.map(&:name)
+            column_types = column_info.map(&:type)
+
+            untyped_rows = []
+
+            # paginated
+            resp.each do |page|
+              untyped_rows.concat page.result_set.rows.map { |r| r.data.map(&:var_char_value) }
+            end
+
+            utc = ActiveSupport::TimeZone['Etc/UTC']
+
+            rows = untyped_rows[1..-1] || []
+            column_types.each_with_index do |ct, i|
+              # TODO more column_types
+              case ct
+              when "timestamp"
+                rows.each do |row|
+                  row[i] = utc.parse(row[i])
+                end
+              when "date"
+                rows.each do |row|
+                  row[i] = Date.parse(row[i])
+                end
+              when "bigint"
+                rows.each do |row|
+                  row[i] = row[i].to_i
+                end
+              when "double"
+                rows.each do |row|
+                  row[i] = row[i].to_f
+                end
+              end
+            end
+          elsif resp
+            error = fetch_error(query_execution_id)
+          else
+            error = Blazer::TIMEOUT_MESSAGE
+          end
+        rescue Aws::Athena::Errors::InvalidRequestException => e
+          error = e.message
+          if error == "Query did not finish successfully. Final query state: FAILED"
+            error = fetch_error(query_execution_id)
+          end
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        glue.get_tables(database_name: database).table_list.map(&:name).sort
+      end
+
+      def schema
+        glue.get_tables(database_name: database).table_list.map { |t| {table: t.name, columns: t.storage_descriptor.columns.map { |c| {name: c.name, data_type: c.type} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      private
+
+      def database
+        @database ||= settings["database"] || "default"
+      end
+
+      def fetch_error(query_execution_id)
+        client.get_query_execution(
+          query_execution_id: query_execution_id
+        ).query_execution.status.state_change_reason
+      end
+
+      def client
+        @client ||= Aws::Athena::Client.new
+      end
+
+      def glue
+        @glue ||= Aws::Glue::Client.new
+      end
+    end
+  end
+end

data/lib/blazer/adapters/base_adapter.rb

@@ -0,0 +1,53 @@
+module Blazer
+  module Adapters
+    class BaseAdapter
+      attr_reader :data_source
+
+      def initialize(data_source)
+        @data_source = data_source
+      end
+
+      def run_statement(statement, comment)
+        # the one required method
+      end
+
+      def tables
+        [] # optional, but nice to have
+      end
+
+      def schema
+        [] # optional, but nice to have
+      end
+
+      def preview_statement
+        "" # also optional, but nice to have
+      end
+
+      def reconnect
+        # optional
+      end
+
+      def cost(statement)
+        # optional
+      end
+
+      def explain(statement)
+        # optional
+      end
+
+      def cancel(run_id)
+        # optional
+      end
+
+      def cachable?(statement)
+        true # optional
+      end
+
+      protected
+
+      def settings
+        @data_source.settings
+      end
+    end
+  end
+end