blazer 1.7.7 → 2.6.4

data/lib/blazer/result.rb

@@ -1,6 +1,6 @@
 module Blazer
   class Result
-    attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached
+    attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached, :forecast_error
 
     def initialize(data_source, columns, rows, error, cached_at, just_cached)
       @data_source = data_source
@@ -56,6 +56,8 @@ module Blazer
             "time"
           elsif v.nil?
             nil
+          elsif v.is_a?(String) && v.encoding == Encoding::BINARY
+            "binary"
           else
             "string"
           end
@@ -69,6 +71,8 @@ module Blazer
           "line"
         elsif column_types == ["time", "string", "numeric"]
           "line2"
+        elsif column_types == ["string", "numeric"] && @columns.last == "pie"
+          "pie"
         elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
           "bar"
         elsif column_types == ["string", "string", "numeric"]
@@ -79,6 +83,47 @@ module Blazer
       end
     end
 
+    def forecastable?
+      @forecastable ||= Blazer.forecasting && column_types == ["time", "numeric"] && @rows.size >= 10
+    end
+
+    # TODO cache it?
+    # don't want to put result data (even hashed version)
+    # into cache without developer opt-in
+    def forecast
+      count = (@rows.size * 0.25).round.clamp(30, 365)
+
+      case Blazer.forecasting
+      when "prophet"
+        require "prophet"
+        forecast = Prophet.forecast(@rows.to_h, count: count)
+      else
+        require "trend"
+        forecast = Trend.forecast(@rows.to_h, count: count)
+      end
+
+      # round integers
+      if @rows[0][1].is_a?(Integer)
+        forecast = forecast.map { |k, v| [k, v.round] }.to_h
+      end
+
+      @rows.each do |row|
+        row[2] = nil
+      end
+      @rows.unshift(*forecast.map { |k, v| [k, nil, v] })
+      @columns << "forecast"
+
+      # reset cache
+      @column_types = nil
+      @chart_type = nil
+
+      forecast
+    rescue => e
+      @forecast_error = String.new("Error generating forecast")
+      @forecast_error << ": #{e.message.sub("Invalid parameter: ", "")}"
+      nil
+    end
+
     def detect_anomaly
       anomaly = nil
       message = nil
@@ -131,39 +176,60 @@ module Blazer
     def anomaly?(series)
       series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
 
-      csv_str =
-        CSV.generate do |csv|
-          csv << ["timestamp", "count"]
-          series.each do |row|
-            csv << row
+      case Blazer.anomaly_checks
+      when "prophet"
+        df = Rover::DataFrame.new(series[0..-2].map { |v| {"ds" => v[0], "y" => v[1]} })
+        m = Prophet.new(interval_width: 0.99)
+        m.logger.level = ::Logger::FATAL # no logging
+        m.fit(df)
+        future = Rover::DataFrame.new(series[-1..-1].map { |v| {"ds" => v[0]} })
+        forecast = m.predict(future).to_a[0]
+        lower = forecast["yhat_lower"]
+        upper = forecast["yhat_upper"]
+        value = series.last[1]
+        value < lower || value > upper
+      when "trend"
+        anomalies = Trend.anomalies(Hash[series])
+        anomalies.include?(series.last[0])
+      when "anomaly_detection"
+        period = 7 # TODO determine period
+        anomalies = AnomalyDetection.detect(Hash[series], period: period)
+        anomalies.include?(series.last[0])
+      else
+        csv_str =
+          CSV.generate do |csv|
+            csv << ["timestamp", "count"]
+            series.each do |row|
+              csv << row
+            end
           end
-        end
 
-      r_script = %x[which Rscript].chomp
-      type = series.any? && series.last.first.to_time - series.first.first.to_time >= 2.weeks ? "ts" : "vec"
-      args = [type, csv_str]
-      raise "R not found" if r_script.empty?
-      command = "#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{args.map { |a| Shellwords.escape(a) }.join(" ")}"
-      output = %x[#{command}]
-      if output.empty?
-        raise "Unknown R error"
-      end
+        r_script = %x[which Rscript].chomp
+        type = series.any? && series.last.first.to_time - series.first.first.to_time >= 2.weeks ? "ts" : "vec"
+        args = [type, csv_str]
+        raise "R not found" if r_script.empty?
+        command = "#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{args.map { |a| Shellwords.escape(a) }.join(" ")}"
+        output = %x[#{command}]
+        if output.empty?
+          raise "Unknown R error"
+        end
 
-      rows = CSV.parse(output, headers: true)
-      error = rows.first && rows.first["x"]
-      raise error if error
+        rows = CSV.parse(output, headers: true)
+        error = rows.first && rows.first["x"]
+        raise error if error
 
-      timestamps = []
-      if type == "ts"
-        rows.each do |row|
-          timestamps << Time.parse(row["timestamp"])
-        end
-        timestamps.include?(series.last[0].to_time)
-      else
-        rows.each do |row|
-          timestamps << row["index"].to_i
+        timestamps = []
+        if type == "ts"
+          rows.each do |row|
+            timestamps << Time.parse(row["timestamp"])
+          end
+          timestamps.include?(series.last[0].to_time)
+        else
+          rows.each do |row|
+            timestamps << row["index"].to_i
+          end
+          timestamps.include?(series.length)
         end
-        timestamps.include?(series.length)
       end
     end
   end

data/lib/blazer/run_statement.rb

@@ -1,12 +1,16 @@
 module Blazer
   class RunStatement
-    def perform(data_source, statement, options = {})
+    def perform(statement, options = {})
       query = options[:query]
-      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      data_source = statement.data_source
+      statement.bind
 
       # audit
       if Blazer.audit
-        audit = Blazer::Audit.new(statement: statement)
+        audit_statement = statement.bind_statement
+        audit_statement += "\n\n#{statement.bind_values.to_json}" if statement.bind_values.any?
+        audit = Blazer::Audit.new(statement: audit_statement)
         audit.query = query
         audit.data_source = data_source.id
         audit.user = options[:user]
@@ -28,7 +32,7 @@ module Blazer
         audit.save! if audit.changed?
       end
 
-      if query && !result.timed_out?
+      if query && !result.timed_out? && !query.variables.any?
         query.checks.each do |check|
           check.update_state(result)
         end

data/lib/blazer/run_statement_job.rb

@@ -1,18 +1,17 @@
-require "sucker_punch"
-
 module Blazer
-  class RunStatementJob
-    include SuckerPunch::Job
-    workers 4
+  class RunStatementJob < ActiveJob::Base
+    self.queue_adapter = :async
 
-    def perform(result, data_source, statement, options)
+    def perform(data_source_id, statement, options)
+      statement = Blazer::Statement.new(statement, data_source_id)
+      statement.values = options.delete(:values)
+      data_source = statement.data_source
       begin
         ActiveRecord::Base.connection_pool.with_connection do
-          result << Blazer::RunStatement.new.perform(data_source, statement, options)
+          Blazer::RunStatement.new.perform(statement, options)
         end
       rescue Exception => e
-        result.clear
-        result << Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
+        Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
         Blazer.cache.write(data_source.run_cache_key(options[:run_id]), Marshal.dump([[], [], "Unknown error", nil]), expires_in: 30.seconds)
         raise e
       end

data/lib/blazer/slack_notifier.rb

@@ -0,0 +1,94 @@
+require "net/http"
+
+module Blazer
+  class SlackNotifier
+    def self.state_change(check, state, state_was, rows_count, error, check_type)
+      check.split_slack_channels.each do |channel|
+        text =
+          if error
+            error
+          elsif rows_count > 0 && check_type == "bad_data"
+            pluralize(rows_count, "row")
+          end
+
+        payload = {
+          channel: channel,
+          attachments: [
+            {
+              title: escape("Check #{state.titleize}: #{check.query.name}"),
+              title_link: query_url(check.query_id),
+              text: escape(text),
+              color: state == "passing" ? "good" : "danger"
+            }
+          ]
+        }
+
+        post(payload)
+      end
+    end
+
+    def self.failing_checks(channel, checks)
+      text =
+        checks.map do |check|
+          "<#{query_url(check.query_id)}|#{escape(check.query.name)}> #{escape(check.state)}"
+        end
+
+      payload = {
+        channel: channel,
+        attachments: [
+          {
+            title: escape("#{pluralize(checks.size, "Check")} Failing"),
+            text: text.join("\n"),
+            color: "warning"
+          }
+        ]
+      }
+
+      post(payload)
+    end
+
+    # https://api.slack.com/docs/message-formatting#how_to_escape_characters
+    # - Replace the ampersand, &, with &amp;
+    # - Replace the less-than sign, < with &lt;
+    # - Replace the greater-than sign, > with &gt;
+    # That's it. Don't HTML entity-encode the entire message.
+    def self.escape(str)
+      str.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;") if str
+    end
+
+    def self.pluralize(*args)
+      ActionController::Base.helpers.pluralize(*args)
+    end
+
+    # checks shouldn't have variables, but in any case,
+    # avoid passing variable params to url helpers
+    # (known unsafe parameters are removed, but still not ideal)
+    def self.query_url(id)
+      Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
+    end
+
+    # TODO use return value
+    def self.post(payload)
+      if Blazer.slack_webhook_url.present?
+        response = post_api(Blazer.slack_webhook_url, payload, {})
+        response.is_a?(Net::HTTPSuccess) && response.body == "ok"
+      else
+        headers = {
+          "Authorization" => "Bearer #{Blazer.slack_oauth_token}",
+          "Content-type" => "application/json"
+        }
+        response = post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response.is_a?(Net::HTTPSuccess) && (JSON.parse(response.body)["ok"] rescue false)
+      end
+    end
+
+    def self.post_api(url, payload, headers)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = 3
+      http.read_timeout = 5
+      http.post(uri.request_uri, payload.to_json, headers)
+    end
+  end
+end

data/lib/blazer/statement.rb

@@ -0,0 +1,75 @@
+module Blazer
+  class Statement
+    attr_reader :statement, :data_source, :bind_statement, :bind_values
+    attr_accessor :values
+
+    def initialize(statement, data_source = nil)
+      @statement = statement
+      @data_source = data_source.is_a?(String) ? Blazer.data_sources[data_source] : data_source
+      @values = {}
+    end
+
+    def variables
+      @variables ||= Blazer.extract_vars(statement)
+    end
+
+    def add_values(var_params)
+      variables.each do |var|
+        value = var_params[var].presence
+        value = nil unless value.is_a?(String) # ignore arrays and hashes
+        if value
+          if ["start_time", "end_time"].include?(var)
+            value = value.to_s.gsub(" ", "+") # fix for Quip bug
+          end
+
+          if var.end_with?("_at")
+            begin
+              value = Blazer.time_zone.parse(value)
+            rescue
+              # do nothing
+            end
+          end
+
+          unless value.is_a?(ActiveSupport::TimeWithZone)
+            if value =~ /\A\d+\z/
+              value = value.to_i
+            elsif value =~ /\A\d+\.\d+\z/
+              value = value.to_f
+            end
+          end
+        end
+        value = Blazer.transform_variable.call(var, value) if Blazer.transform_variable
+        @values[var] = value
+      end
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
+    end
+
+    def apply_cohort_analysis(period:, days:)
+      @statement = data_source.cohort_analysis_statement(statement, period: period, days: days).sub("{placeholder}") { statement }
+    end
+
+    # should probably transform before cohort analysis
+    # but keep previous order for now
+    def transformed_statement
+      statement = self.statement.dup
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement
+    end
+
+    def bind
+      @bind_statement, @bind_values = data_source.bind_params(transformed_statement, values)
+    end
+
+    def display_statement
+      data_source.sub_variables(transformed_statement, values)
+    end
+
+    def clear_cache
+      bind if bind_statement.nil?
+      data_source.clear_cache(self)
+    end
+  end
+end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "1.7.7"
+  VERSION = "2.6.4"
 end

data/lib/blazer.rb

@@ -1,45 +1,88 @@
-require "csv"
-require "yaml"
+# dependencies
 require "chartkick"
 require "safely/core"
+
+# stdlib
+require "csv"
+require "json"
+require "yaml"
+
+# modules
 require "blazer/version"
 require "blazer/data_source"
 require "blazer/result"
 require "blazer/run_statement"
+require "blazer/statement"
+
+# adapters
 require "blazer/adapters/base_adapter"
+require "blazer/adapters/athena_adapter"
+require "blazer/adapters/bigquery_adapter"
+require "blazer/adapters/cassandra_adapter"
+require "blazer/adapters/drill_adapter"
+require "blazer/adapters/druid_adapter"
 require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/hive_adapter"
+require "blazer/adapters/ignite_adapter"
+require "blazer/adapters/influxdb_adapter"
 require "blazer/adapters/mongodb_adapter"
+require "blazer/adapters/neo4j_adapter"
+require "blazer/adapters/opensearch_adapter"
 require "blazer/adapters/presto_adapter"
+require "blazer/adapters/salesforce_adapter"
+require "blazer/adapters/soda_adapter"
+require "blazer/adapters/spark_adapter"
 require "blazer/adapters/sql_adapter"
+require "blazer/adapters/snowflake_adapter"
+
+# engine
 require "blazer/engine"
 
 module Blazer
   class Error < StandardError; end
+  class UploadError < Error; end
   class TimeoutNotSupported < Error; end
 
+  # actionmailer optional
+  autoload :CheckMailer, "blazer/check_mailer"
+  # net/http optional
+  autoload :SlackNotifier, "blazer/slack_notifier"
+  # activejob optional
+  autoload :RunStatementJob, "blazer/run_statement_job"
+
   class << self
     attr_accessor :audit
     attr_reader :time_zone
     attr_accessor :user_name
-    attr_accessor :user_class
-    attr_accessor :user_method
+    attr_writer :user_class
+    attr_writer :user_method
     attr_accessor :before_action
     attr_accessor :from_email
     attr_accessor :cache
     attr_accessor :transform_statement
+    attr_accessor :transform_variable
     attr_accessor :check_schedules
     attr_accessor :anomaly_checks
+    attr_accessor :forecasting
     attr_accessor :async
     attr_accessor :images
+    attr_accessor :query_viewable
     attr_accessor :query_editable
+    attr_accessor :override_csp
+    attr_accessor :slack_oauth_token
+    attr_accessor :slack_webhook_url
+    attr_accessor :mapbox_access_token
   end
   self.audit = true
   self.user_name = :name
   self.check_schedules = ["5 minutes", "1 hour", "1 day"]
   self.anomaly_checks = false
+  self.forecasting = false
   self.async = false
   self.images = false
+  self.override_csp = false
 
+  VARIABLE_MESSAGE = "Variable cannot be used in this position"
   TIMEOUT_MESSAGE = "Query timed out :("
   TIMEOUT_ERRORS = [
     "canceling statement due to statement timeout", # postgres
@@ -49,13 +92,28 @@ module Blazer
     "system requested abort", # redshift
     "maximum statement execution time exceeded" # mysql
   ]
-  BELONGS_TO_OPTIONAL = {}
-  BELONGS_TO_OPTIONAL[:optional] = true if Rails::VERSION::MAJOR >= 5
 
   def self.time_zone=(time_zone)
     @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
   end
 
+  def self.user_class
+    if !defined?(@user_class)
+      @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil)
+    end
+    @user_class
+  end
+
+  def self.user_method
+    if !defined?(@user_method)
+      @user_method = settings["user_method"]
+      if user_class
+        @user_method ||= "current_#{user_class.to_s.downcase.singularize}"
+      end
+    end
+    @user_method
+  end
+
   def self.settings
     @settings ||= begin
       path = Rails.root.join("config", "blazer.yml").to_s
@@ -69,23 +127,21 @@ module Blazer
 
   def self.data_sources
     @data_sources ||= begin
-      ds = Hash[
-        settings["data_sources"].map do |id, s|
-          [id, Blazer::DataSource.new(id, s)]
-        end
-      ]
-      ds.default = ds.values.first
+      ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      settings["data_sources"].each do |id, s|
+        ds[id] = Blazer::DataSource.new(id, s)
+      end
       ds
-
-      # TODO Blazer 2.0
-      # ds2 = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
-      # ds.each do |k, v|
-      #   ds2[k] = v
-      # end
-      # ds2
     end
   end
 
+  # TODO move to Statement and remove in 3.0.0
+  def self.extract_vars(statement)
+    # strip commented out lines
+    # and regex {1} or {1,2}
+    statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+  end
+
   def self.run_checks(schedule: nil)
     checks = Blazer::Check.includes(:query)
     checks = checks.where(schedule: schedule) if schedule
@@ -96,15 +152,12 @@ module Blazer
   end
 
   def self.run_check(check)
-    rows = nil
-    error = nil
     tries = 1
 
     ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
       # try 3 times on timeout errors
-      data_source = data_sources[check.query.data_source]
-      statement = check.query.statement
-      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement = check.query.statement_object
+      data_source = statement.data_source
 
       while tries <= 3
         result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
@@ -121,11 +174,19 @@ module Blazer
           break
         end
       end
-      check.update_state(result)
+
+      begin
+        check.reload # in case state has changed since job started
+        check.update_state(result)
+      rescue ActiveRecord::RecordNotFound
+        # check deleted
+      end
+
       # TODO use proper logfmt
       Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
 
-      instrument[:statement] = statement
+      # should be no variables
+      instrument[:statement] = statement.bind_statement
       instrument[:data_source] = data_source
       instrument[:state] = check.state
       instrument[:rows] = result.rows.try(:size)
@@ -136,10 +197,15 @@ module Blazer
 
   def self.send_failing_checks
     emails = {}
+    slack_channels = {}
+
     Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
       check.split_emails.each do |email|
         (emails[email] ||= []) << check
       end
+      check.split_slack_channels.each do |channel|
+        (slack_channels[channel] ||= []) << check
+      end
     end
 
     emails.each do |email, checks|
@@ -147,5 +213,67 @@ module Blazer
         Blazer::CheckMailer.failing_checks(email, checks).deliver_now
       end
     end
+
+    slack_channels.each do |channel, checks|
+      Safely.safely do
+        Blazer::SlackNotifier.failing_checks(channel, checks)
+      end
+    end
+  end
+
+  def self.slack?
+    slack_oauth_token.present? || slack_webhook_url.present?
+  end
+
+  def self.uploads?
+    settings.key?("uploads")
+  end
+
+  def self.uploads_connection
+    raise "Empty url for uploads" unless settings.dig("uploads", "url")
+    Blazer::UploadsConnection.connection
+  end
+
+  def self.uploads_schema
+    settings.dig("uploads", "schema") || "uploads"
+  end
+
+  def self.uploads_table_name(name)
+    uploads_connection.quote_table_name("#{uploads_schema}.#{name}")
+  end
+
+  def self.adapters
+    @adapters ||= {}
+  end
+
+  def self.register_adapter(name, adapter)
+    adapters[name] = adapter
+  end
+
+  def self.archive_queries
+    raise "Audits must be enabled to archive" unless Blazer.audit
+    raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
+
+    viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact
+    Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived")
   end
 end
+
+Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
+Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
+Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
+Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapter
+Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
+Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
+Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
+Blazer.register_adapter "opensearch", Blazer::Adapters::OpensearchAdapter
+Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
+Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
+Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
+Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
+Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
+Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter