blazer 2.4.2 → 2.6.0

data/lib/blazer/adapters/spark_adapter.rb

@@ -4,6 +4,11 @@ module Blazer
       def tables
         client.execute("SHOW TABLES").map { |r| r["tableName"] }
       end
+
+      # https://spark.apache.org/docs/latest/sql-ref-literals.html
+      def quoting
+        :backslash_escape
+      end
     end
   end
 end

data/lib/blazer/adapters/sql_adapter.rb

@@ -15,7 +15,7 @@ module Blazer
           end
       end
 
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params = [])
         columns = []
         rows = []
         error = nil
@@ -24,7 +24,8 @@ module Blazer
           in_transaction do
             set_timeout(data_source.timeout) if data_source.timeout
 
-            result = select_all("#{statement} /*#{comment}*/")
+            binds = bind_params.map { |v| ActiveRecord::Relation::QueryAttribute.new(nil, v, ActiveRecord::Type::Value.new) }
+            result = connection_model.connection.select_all("#{statement} /*#{comment}*/", nil, binds)
             columns = result.columns
             result.rows.each do |untyped_row|
               rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
@@ -33,6 +34,7 @@ module Blazer
         rescue => e
           error = e.message.sub(/.+ERROR: /, "")
           error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
+          error = Blazer::VARIABLE_MESSAGE if error.include?("syntax error at or near \"$") || error.include?("Incorrect syntax near '@") || error.include?("your MySQL server version for the right syntax to use near '?")
           reconnect if error.include?("PG::ConnectionBad")
         end
 
@@ -146,7 +148,7 @@ module Blazer
             date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
             date_params = [tzname]
           end
-          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS INTEGER)"
+          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS SIGNED)"
         else
           date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
           date_params = [period, tzname]
@@ -156,7 +158,7 @@ module Blazer
         # WITH not an optimization fence in Postgres 12+
         statement = <<~SQL
           WITH query AS (
-            #{statement}
+            {placeholder}
           ),
           cohorts AS (
             SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
@@ -183,6 +185,27 @@ module Blazer
         connection_model.send(:sanitize_sql_array, params)
       end
 
+      def quoting
+        ->(value) { connection_model.connection.quote(value) }
+      end
+
+      # Redshift adapter silently ignores binds
+      def parameter_binding
+        if postgresql? || sqlite?
+          :numeric
+        elsif mysql? && connection_model.connection.prepared_statements?
+          # Active Record silently ignores binds with MySQL when prepared statements are disabled
+          :positional
+        elsif sqlserver?
+          proc do |statement, variables|
+            variables.each_with_index do |(k, _), i|
+              statement = statement.gsub("{#{k}}", "@#{i} ")
+            end
+            [statement, variables.values]
+          end
+        end
+      end
+
       protected
 
       def select_all(statement, params = [])
@@ -207,6 +230,10 @@ module Blazer
         ["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
       end
 
+      def sqlite?
+        ["SQLite"].include?(adapter_name)
+      end
+
       def sqlserver?
         ["SQLServer", "tinytds", "mssql"].include?(adapter_name)
       end
@@ -238,6 +265,9 @@ module Blazer
         if settings["schemas"]
           where = "table_schema IN (?)"
           schemas = settings["schemas"]
+        elsif mysql?
+          where = "table_schema IN (?)"
+          schemas = [default_schema]
         else
           where = "table_schema NOT IN (?)"
           schemas = ["information_schema"]

data/lib/blazer/data_source.rb

@@ -89,7 +89,19 @@ module Blazer
       Blazer.cache.delete(run_cache_key(run_id))
     end
 
+    def sub_variables(statement, vars)
+      statement = statement.dup
+      vars.each do |var, value|
+        # use block form to disable back-references
+        statement.gsub!("{#{var}}") { quote(value) }
+      end
+      statement
+    end
+
     def run_statement(statement, options = {})
+      statement = Statement.new(statement, self) if statement.is_a?(String)
+      statement.bind unless statement.bind_statement
+
       async = options[:async]
       result = nil
       if cache_mode != "off"
@@ -118,7 +130,7 @@ module Blazer
         if options[:run_id]
           comment << ",run_id:#{options[:run_id]}"
         end
-        result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
+        result = run_statement_helper(statement, comment, async ? options[:run_id] : nil, options)
       end
 
       result
@@ -133,17 +145,73 @@ module Blazer
     end
 
     def statement_cache_key(statement)
-      cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
+      cache_key(["statement", id, Digest::MD5.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.sort_by { |k, _| k }.to_json)])
     end
 
     def run_cache_key(run_id)
       cache_key(["run", run_id])
     end
 
+    def quote(value)
+      if quoting == :backslash_escape || quoting == :single_quote_escape
+        # only need to support types generated by process_vars
+        if value.is_a?(Integer) || value.is_a?(Float)
+          value.to_s
+        elsif value.nil?
+          "NULL"
+        else
+          value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
+
+          if quoting == :backslash_escape
+            "'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
+          else
+            "'#{value.gsub("'", "''")}'"
+          end
+        end
+      elsif quoting.respond_to?(:call)
+        quoting.call(value)
+      elsif quoting.nil?
+        raise Blazer::Error, "Quoting not specified"
+      else
+        raise Blazer::Error, "Unknown quoting"
+      end
+    end
+
+    def bind_params(statement, variables)
+      if parameter_binding == :positional
+        locations = []
+        variables.each do |k, v|
+          i = 0
+          while (idx = statement.index("{#{k}}", i))
+            locations << [v, idx]
+            i = idx + 1
+          end
+        end
+        variables.each do |k, v|
+          statement = statement.gsub("{#{k}}", "?")
+        end
+        [statement, locations.sort_by(&:last).map(&:first)]
+      elsif parameter_binding == :numeric
+        variables.each_with_index do |(k, v), i|
+          # add trailing space if followed by digit
+          # try to keep minimal to avoid fixing invalid queries like SELECT{var}
+          statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
+        end
+        [statement, variables.values]
+      elsif parameter_binding.respond_to?(:call)
+        parameter_binding.call(statement, variables)
+      elsif parameter_binding.nil?
+        [sub_variables(statement, variables), []]
+      else
+        raise Blazer::Error, "Unknown bind parameters"
+      end
+    end
+
     protected
 
     def adapter_instance
       @adapter_instance ||= begin
+        # TODO add required settings to adapters
         unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
           raise Blazer::Error, "Empty url for data source: #{id}"
         end
@@ -156,9 +224,22 @@ module Blazer
       end
     end
 
-    def run_statement_helper(statement, comment, run_id)
+    def quoting
+      @quoting ||= adapter_instance.quoting
+    end
+
+    def parameter_binding
+      @parameter_binding ||= adapter_instance.parameter_binding
+    end
+
+    def run_statement_helper(statement, comment, run_id, options)
       start_time = Time.now
-      columns, rows, error = adapter_instance.run_statement(statement, comment)
+      columns, rows, error =
+        if adapter_instance.parameter_binding
+          adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
+        else
+          adapter_instance.run_statement(statement.bind_statement, comment)
+        end
       duration = Time.now - start_time
 
       cache_data = nil
@@ -167,7 +248,7 @@ module Blazer
         cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
       end
 
-      if cache && cache_data && adapter_instance.cachable?(statement)
+      if cache && cache_data && adapter_instance.cachable?(statement.bind_statement)
         Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
       end
 
@@ -182,11 +263,12 @@ module Blazer
       Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
     end
 
+    # TODO check for adapter with same name, default to sql
     def detect_adapter
-      schema = settings["url"].to_s.split("://").first
-      case schema
+      scheme = settings["url"].to_s.split("://").first
+      case scheme
       when "mongodb", "presto", "cassandra", "ignite"
-        schema
+        scheme
       else
         "sql"
       end

data/lib/blazer/engine.rb

@@ -30,12 +30,9 @@ module Blazer
       Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
       Blazer.forecasting = Blazer.settings["forecasting"] || false
       Blazer.async = Blazer.settings["async"] || false
-      if Blazer.async
-        require "blazer/run_statement_job"
-      end
-
       Blazer.images = Blazer.settings["images"] || false
       Blazer.override_csp = Blazer.settings["override_csp"] || false
+      Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]
       Blazer.slack_webhook_url = Blazer.settings["slack_webhook_url"] || ENV["BLAZER_SLACK_WEBHOOK_URL"]
       Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"] || ENV["MAPBOX_ACCESS_TOKEN"]
     end

data/lib/blazer/result.rb

@@ -174,9 +174,25 @@ module Blazer
     def anomaly?(series)
       series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
 
-      if Blazer.anomaly_checks == "trend"
+      case Blazer.anomaly_checks
+      when "prophet"
+        df = Rover::DataFrame.new(series[0..-2].map { |v| {"ds" => v[0], "y" => v[1]} })
+        m = Prophet.new(interval_width: 0.99)
+        m.logger.level = ::Logger::FATAL # no logging
+        m.fit(df)
+        future = Rover::DataFrame.new(series[-1..-1].map { |v| {"ds" => v[0]} })
+        forecast = m.predict(future).to_a[0]
+        lower = forecast["yhat_lower"]
+        upper = forecast["yhat_upper"]
+        value = series.last[1]
+        value < lower || value > upper
+      when "trend"
         anomalies = Trend.anomalies(Hash[series])
         anomalies.include?(series.last[0])
+      when "anomaly_detection"
+        period = 7 # TODO determine period
+        anomalies = AnomalyDetection.detect(Hash[series], period: period)
+        anomalies.include?(series.last[0])
       else
         csv_str =
           CSV.generate do |csv|

data/lib/blazer/run_statement.rb

@@ -1,12 +1,16 @@
 module Blazer
   class RunStatement
-    def perform(data_source, statement, options = {})
+    def perform(statement, options = {})
       query = options[:query]
-      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      data_source = statement.data_source
+      statement.bind
 
       # audit
       if Blazer.audit
-        audit = Blazer::Audit.new(statement: statement)
+        audit_statement = statement.bind_statement
+        audit_statement += "\n\n#{statement.bind_values.to_json}" if statement.bind_values.any?
+        audit = Blazer::Audit.new(statement: audit_statement)
         audit.query = query
         audit.data_source = data_source.id
         audit.user = options[:user]

data/lib/blazer/run_statement_job.rb

@@ -3,10 +3,12 @@ module Blazer
     self.queue_adapter = :async
 
     def perform(data_source_id, statement, options)
-      data_source = Blazer.data_sources[data_source_id]
+      statement = Blazer::Statement.new(statement, data_source_id)
+      statement.values = options.delete(:values)
+      data_source = statement.data_source
       begin
         ActiveRecord::Base.connection_pool.with_connection do
-          Blazer::RunStatement.new.perform(data_source, statement, options)
+          Blazer::RunStatement.new.perform(statement, options)
         end
       rescue Exception => e
         Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)

data/{app/mailers → lib}/blazer/slack_notifier.rb

@@ -23,7 +23,7 @@ module Blazer
           ]
         }
 
-        post(Blazer.slack_webhook_url, payload)
+        post(payload)
       end
     end
 
@@ -44,7 +44,7 @@ module Blazer
         ]
       }
 
-      post(Blazer.slack_webhook_url, payload)
+      post(payload)
     end
 
     # https://api.slack.com/docs/message-formatting#how_to_escape_characters
@@ -67,13 +67,28 @@ module Blazer
       Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
     end
 
-    def self.post(url, payload)
+    # TODO use return value
+    def self.post(payload)
+      if Blazer.slack_webhook_url.present?
+        response = post_api(Blazer.slack_webhook_url, payload, {})
+        response.is_a?(Net::HTTPSuccess) && response.body == "ok"
+      else
+        headers = {
+          "Authorization" => "Bearer #{Blazer.slack_oauth_token}",
+          "Content-type" => "application/json"
+        }
+        response = post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response.is_a?(Net::HTTPSuccess) && (JSON.parse(response.body)["ok"] rescue false)
+      end
+    end
+
+    def self.post_api(url, payload, headers)
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = true
       http.open_timeout = 3
       http.read_timeout = 5
-      http.post(uri.request_uri, payload.to_json)
+      http.post(uri.request_uri, payload.to_json, headers)
     end
   end
 end

data/lib/blazer/statement.rb

@@ -0,0 +1,75 @@
+module Blazer
+  class Statement
+    attr_reader :statement, :data_source, :bind_statement, :bind_values
+    attr_accessor :values
+
+    def initialize(statement, data_source = nil)
+      @statement = statement
+      @data_source = data_source.is_a?(String) ? Blazer.data_sources[data_source] : data_source
+      @values = {}
+    end
+
+    def variables
+      @variables ||= Blazer.extract_vars(statement)
+    end
+
+    def add_values(var_params)
+      variables.each do |var|
+        value = var_params[var].presence
+        value = nil unless value.is_a?(String) # ignore arrays and hashes
+        if value
+          if ["start_time", "end_time"].include?(var)
+            value = value.to_s.gsub(" ", "+") # fix for Quip bug
+          end
+
+          if var.end_with?("_at")
+            begin
+              value = Blazer.time_zone.parse(value)
+            rescue
+              # do nothing
+            end
+          end
+
+          unless value.is_a?(ActiveSupport::TimeWithZone)
+            if value =~ /\A\d+\z/
+              value = value.to_i
+            elsif value =~ /\A\d+\.\d+\z/
+              value = value.to_f
+            end
+          end
+        end
+        value = Blazer.transform_variable.call(var, value) if Blazer.transform_variable
+        @values[var] = value
+      end
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
+    end
+
+    def apply_cohort_analysis(period:, days:)
+      @statement = data_source.cohort_analysis_statement(statement, period: period, days: days).sub("{placeholder}") { statement }
+    end
+
+    # should probably transform before cohort analysis
+    # but keep previous order for now
+    def transformed_statement
+      statement = self.statement.dup
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement
+    end
+
+    def bind
+      @bind_statement, @bind_values = data_source.bind_params(transformed_statement, values)
+    end
+
+    def display_statement
+      data_source.sub_variables(transformed_statement, values)
+    end
+
+    def clear_cache
+      bind if bind_statement.nil?
+      data_source.clear_cache(self)
+    end
+  end
+end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.4.2"
+  VERSION = "2.6.0"
 end

data/lib/blazer.rb

@@ -1,14 +1,18 @@
 # dependencies
-require "csv"
-require "yaml"
 require "chartkick"
 require "safely/core"
 
+# stdlib
+require "csv"
+require "json"
+require "yaml"
+
 # modules
 require "blazer/version"
 require "blazer/data_source"
 require "blazer/result"
 require "blazer/run_statement"
+require "blazer/statement"
 
 # adapters
 require "blazer/adapters/base_adapter"
@@ -23,6 +27,7 @@ require "blazer/adapters/ignite_adapter"
 require "blazer/adapters/influxdb_adapter"
 require "blazer/adapters/mongodb_adapter"
 require "blazer/adapters/neo4j_adapter"
+require "blazer/adapters/opensearch_adapter"
 require "blazer/adapters/presto_adapter"
 require "blazer/adapters/salesforce_adapter"
 require "blazer/adapters/soda_adapter"
@@ -38,6 +43,13 @@ module Blazer
   class UploadError < Error; end
   class TimeoutNotSupported < Error; end
 
+  # actionmailer optional
+  autoload :CheckMailer, "blazer/check_mailer"
+  # net/http optional
+  autoload :SlackNotifier, "blazer/slack_notifier"
+  # activejob optional
+  autoload :RunStatementJob, "blazer/run_statement_job"
+
   class << self
     attr_accessor :audit
     attr_reader :time_zone
@@ -57,6 +69,7 @@ module Blazer
     attr_accessor :query_viewable
     attr_accessor :query_editable
     attr_accessor :override_csp
+    attr_accessor :slack_oauth_token
     attr_accessor :slack_webhook_url
     attr_accessor :mapbox_access_token
   end
@@ -69,6 +82,7 @@ module Blazer
   self.images = false
   self.override_csp = false
 
+  VARIABLE_MESSAGE = "Variable cannot be used in this position"
   TIMEOUT_MESSAGE = "Query timed out :("
   TIMEOUT_ERRORS = [
     "canceling statement due to statement timeout", # postgres
@@ -121,6 +135,7 @@ module Blazer
     end
   end
 
+  # TODO move to Statement and remove in 3.0.0
   def self.extract_vars(statement)
     # strip commented out lines
     # and regex {1} or {1,2}
@@ -141,9 +156,8 @@ module Blazer
 
     ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
       # try 3 times on timeout errors
-      data_source = data_sources[check.query.data_source]
-      statement = check.query.statement
-      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement = check.query.statement_object
+      data_source = statement.data_source
 
       while tries <= 3
         result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
@@ -171,7 +185,8 @@ module Blazer
       # TODO use proper logfmt
       Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
 
-      instrument[:statement] = statement
+      # should be no variables
+      instrument[:statement] = statement.bind_statement
       instrument[:data_source] = data_source
       instrument[:state] = check.state
       instrument[:rows] = result.rows.try(:size)
@@ -207,7 +222,7 @@ module Blazer
   end
 
   def self.slack?
-    slack_webhook_url.present?
+    slack_oauth_token.present? || slack_webhook_url.present?
   end
 
   def self.uploads?
@@ -234,6 +249,14 @@ module Blazer
   def self.register_adapter(name, adapter)
     adapters[name] = adapter
   end
+
+  def self.archive_queries
+    raise "Audits must be enabled to archive" unless Blazer.audit
+    raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
+
+    viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact
+    Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived")
+  end
 end
 
 Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
@@ -245,9 +268,10 @@ Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
 Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
 Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapter
 Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
+Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
+Blazer.register_adapter "opensearch", Blazer::Adapters::OpensearchAdapter
 Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
-Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
 Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
 Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter

data/lib/generators/blazer/templates/config.yml.tt

@@ -63,11 +63,11 @@ check_schedules:
 
 # enable anomaly detection
 # note: with trend, time series are sent to https://trendapi.org
-# anomaly_checks: trend / r
+# anomaly_checks: prophet / trend / anomaly_detection
 
 # enable forecasting
 # note: with trend, time series are sent to https://trendapi.org
-# forecasting: trend / prophet
+# forecasting: prophet / trend
 
 # enable map
 # mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>

data/lib/tasks/blazer.rake

@@ -11,10 +11,10 @@ namespace :blazer do
 
   desc "archive queries"
   task archive_queries: :environment do
-    abort "Audits must be enabled to archive" unless Blazer.audit
-    abort "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
-
-    viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact
-    Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived")
+    begin
+      Blazer.archive_queries
+    rescue => e
+      abort e.message
+    end
   end
 end

data/licenses/LICENSE-bootstrap.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2011-2016 Twitter, Inc.
+Copyright (c) 2011-2019 Twitter, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal