blazer 2.5.0 → 2.6.4

data/lib/blazer/adapters/druid_adapter.rb

@@ -3,15 +3,37 @@ module Blazer
     class DruidAdapter < BaseAdapter
       TIMESTAMP_REGEX = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\z/
 
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params)
+        require "json"
+        require "net/http"
+        require "uri"
+
         columns = []
         rows = []
         error = nil
 
+        params =
+          bind_params.map do |v|
+            type =
+              case v
+              when Integer
+                "BIGINT"
+              when Float
+                "DOUBLE"
+              when ActiveSupport::TimeWithZone
+                v = (v.to_f * 1000).round
+                "TIMESTAMP"
+              else
+                "VARCHAR"
+              end
+            {type: type, value: v}
+          end
+
         header = {"Content-Type" => "application/json", "Accept" => "application/json"}
         timeout = data_source.timeout ? data_source.timeout.to_i : 300
         data = {
           query: statement,
+          parameters: params,
           context: {
             timeout: timeout * 1000
           }
@@ -27,6 +49,8 @@ module Blazer
             error = response["errorMessage"] || "Unknown error: #{response.inspect}"
             if error.include?("timed out")
               error = Blazer::TIMEOUT_MESSAGE
+            elsif error.include?("Encountered \"?\" at")
+              error = Blazer::VARIABLE_MESSAGE
             end
           else
             columns = (response.first || {}).keys
@@ -62,6 +86,17 @@ module Blazer
       def preview_statement
         "SELECT * FROM {table} LIMIT 10"
       end
+
+      # https://druid.apache.org/docs/latest/querying/sql.html#identifiers-and-literals
+      # docs only mention double quotes
+      def quoting
+        :single_quote_escape
+      end
+
+      # https://druid.apache.org/docs/latest/querying/sql.html#dynamic-parameters
+      def parameter_binding
+        :positional
+      end
     end
   end
 end

data/lib/blazer/adapters/elasticsearch_adapter.rb

@@ -1,13 +1,13 @@
 module Blazer
   module Adapters
     class ElasticsearchAdapter < BaseAdapter
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params)
         columns = []
         rows = []
         error = nil
 
         begin
-          response = client.transport.perform_request("POST", endpoint, {}, {query: "#{statement} /*#{comment}*/"}).body
+          response = client.transport.perform_request("POST", endpoint, {}, {query: "#{statement} /*#{comment}*/", params: bind_params}).body
           columns = response["columns"].map { |v| v["name"] }
           # Elasticsearch does not differentiate between dates and times
           date_indexes = response["columns"].each_index.select { |i| ["date", "datetime"].include?(response["columns"][i]["type"]) }
@@ -21,6 +21,7 @@ module Blazer
           end
         rescue => e
           error = e.message
+          error = Blazer::VARIABLE_MESSAGE if error.include?("mismatched input '?'")
         end
 
         [columns, rows, error]
@@ -36,6 +37,16 @@ module Blazer
         "SELECT * FROM \"{table}\" LIMIT 10"
       end
 
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-lexical-structure.html#sql-syntax-string-literals
+      def quoting
+        :single_quote_escape
+      end
+
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-rest-params.html
+      def parameter_binding
+        :positional
+      end
+
       protected
 
       def endpoint

data/lib/blazer/adapters/hive_adapter.rb

@@ -25,6 +25,16 @@ module Blazer
         "SELECT * FROM {table} LIMIT 10"
       end
 
+      # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-StringsstringStrings
+      def quoting
+        :backslash_escape
+      end
+
+      # has variable substitution, but sets for session
+      # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+VariableSubstitution
+      def parameter_binding
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/ignite_adapter.rb

@@ -1,13 +1,13 @@
 module Blazer
   module Adapters
     class IgniteAdapter < BaseAdapter
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params)
         columns = []
         rows = []
         error = nil
 
         begin
-          result = client.query("#{statement} /*#{comment}*/", schema: default_schema, statement_type: :select, timeout: data_source.timeout)
+          result = client.query("#{statement} /*#{comment}*/", bind_params, schema: default_schema, statement_type: :select, timeout: data_source.timeout)
           columns = result.any? ? result.first.keys : []
           rows = result.map(&:values)
         rescue => e
@@ -37,6 +37,16 @@ module Blazer
       #   result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }
       # end
 
+      def quoting
+        :single_quote_escape
+      end
+
+      # query arguments
+      # https://ignite.apache.org/docs/latest/binary-client-protocol/sql-and-scan-queries#op_query_sql
+      def parameter_binding
+        :positional
+      end
+
       private
 
       def default_schema

data/lib/blazer/adapters/influxdb_adapter.rb

@@ -8,16 +8,19 @@ module Blazer
 
         begin
           result = client.query(statement, denormalize: false).first
-          columns = result["columns"]
-          rows = result["values"]
-
-          # parse time columns
-          # current approach isn't ideal, but result doesn't include types
-          # another approach would be to check the format
-          time_index = columns.index("time")
-          if time_index
-            rows.each do |row|
-              row[time_index] = Time.parse(row[time_index]) if row[time_index]
+
+          if result
+            columns = result["columns"]
+            rows = result["values"]
+
+            # parse time columns
+            # current approach isn't ideal, but result doesn't include types
+            # another approach would be to check the format
+            time_index = columns.index("time")
+            if time_index
+              rows.each do |row|
+                row[time_index] = Time.parse(row[time_index]) if row[time_index]
+              end
             end
           end
         rescue => e
@@ -35,6 +38,15 @@ module Blazer
         "SELECT * FROM {table} LIMIT 10"
       end
 
+      # https://docs.influxdata.com/influxdb/v1.8/query_language/spec/#strings
+      def quoting
+        :backslash_escape
+      end
+
+      def parameter_binding
+        # not supported
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/mongodb_adapter.rb

@@ -25,6 +25,10 @@ module Blazer
         "db.{table}.find().limit(10)"
       end
 
+      def quoting
+        :backslash_escape
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/neo4j_adapter.rb

@@ -1,13 +1,13 @@
 module Blazer
   module Adapters
     class Neo4jAdapter < BaseAdapter
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params)
         columns = []
         rows = []
         error = nil
 
         begin
-          result = session.query("#{statement} /*#{comment}*/")
+          result = session.query("#{statement} /*#{comment}*/", bind_params)
           columns = result.columns.map(&:to_s)
           rows = []
           result.each do |row|
@@ -19,6 +19,7 @@ module Blazer
           end
         rescue => e
           error = e.message
+          error = Blazer::VARIABLE_MESSAGE if error.include?("Invalid input '$'")
         end
 
         [columns, rows, error]
@@ -33,6 +34,20 @@ module Blazer
         "MATCH (n:{table}) RETURN n LIMIT 10"
       end
 
+      # https://neo4j.com/docs/cypher-manual/current/syntax/expressions/#cypher-expressions-string-literals
+      def quoting
+        :backslash_escape
+      end
+
+      def parameter_binding
+        proc do |statement, variables|
+          variables.each_key do |k|
+            statement = statement.gsub("{#{k}}") { "$#{k} " }
+          end
+          [statement, variables]
+        end
+      end
+
       protected
 
       def session

data/lib/blazer/adapters/opensearch_adapter.rb

@@ -38,6 +38,10 @@ module Blazer
         "SELECT * FROM `{table}` LIMIT 10"
       end
 
+      def quoting
+        # unknown
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/presto_adapter.rb

@@ -25,6 +25,15 @@ module Blazer
         "SELECT * FROM {table} LIMIT 10"
       end
 
+      def quoting
+        :single_quote_escape
+      end
+
+      # TODO support prepared statements - https://prestodb.io/docs/current/sql/prepare.html
+      # feature request for variables - https://github.com/prestodb/presto/issues/5918
+      def parameter_binding
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/salesforce_adapter.rb

@@ -35,6 +35,11 @@ module Blazer
         "SELECT Id FROM {table} LIMIT 10"
       end
 
+      # https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql_select_quotedstringescapes.htm
+      def quoting
+        :backslash_escape
+      end
+
       protected
 
       def client

data/lib/blazer/adapters/snowflake_adapter.rb

@@ -68,6 +68,15 @@ module Blazer
       def cancel(run_id)
         # todo
       end
+
+      # https://docs.snowflake.com/en/sql-reference/data-types-text.html#escape-sequences
+      def quoting
+        :backslash_escape
+      end
+
+      def parameter_binding
+        # TODO
+      end
     end
   end
 end

data/lib/blazer/adapters/soda_adapter.rb

@@ -2,6 +2,10 @@ module Blazer
   module Adapters
     class SodaAdapter < BaseAdapter
       def run_statement(statement, comment)
+        require "json"
+        require "net/http"
+        require "uri"
+
         columns = []
         rows = []
         error = nil
@@ -91,6 +95,11 @@ module Blazer
       def tables
         ["all"]
       end
+
+      # https://dev.socrata.com/docs/datatypes/text.html
+      def quoting
+        :single_quote_escape
+      end
     end
   end
 end

data/lib/blazer/adapters/spark_adapter.rb

@@ -4,6 +4,11 @@ module Blazer
       def tables
         client.execute("SHOW TABLES").map { |r| r["tableName"] }
       end
+
+      # https://spark.apache.org/docs/latest/sql-ref-literals.html
+      def quoting
+        :backslash_escape
+      end
     end
   end
 end

data/lib/blazer/adapters/sql_adapter.rb

@@ -15,7 +15,7 @@ module Blazer
           end
       end
 
-      def run_statement(statement, comment)
+      def run_statement(statement, comment, bind_params = [])
         columns = []
         rows = []
         error = nil
@@ -24,7 +24,8 @@ module Blazer
           in_transaction do
             set_timeout(data_source.timeout) if data_source.timeout
 
-            result = select_all("#{statement} /*#{comment}*/")
+            binds = bind_params.map { |v| ActiveRecord::Relation::QueryAttribute.new(nil, v, ActiveRecord::Type::Value.new) }
+            result = connection_model.connection.select_all("#{statement} /*#{comment}*/", nil, binds)
             columns = result.columns
             result.rows.each do |untyped_row|
               rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
@@ -33,6 +34,7 @@ module Blazer
         rescue => e
           error = e.message.sub(/.+ERROR: /, "")
           error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
+          error = Blazer::VARIABLE_MESSAGE if error.include?("syntax error at or near \"$") || error.include?("Incorrect syntax near '@") || error.include?("your MySQL server version for the right syntax to use near '?")
           reconnect if error.include?("PG::ConnectionBad")
         end
 
@@ -156,7 +158,7 @@ module Blazer
         # WITH not an optimization fence in Postgres 12+
         statement = <<~SQL
           WITH query AS (
-            #{statement}
+            {placeholder}
           ),
           cohorts AS (
             SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
@@ -183,6 +185,30 @@ module Blazer
         connection_model.send(:sanitize_sql_array, params)
       end
 
+      def quoting
+        ->(value) { connection_model.connection.quote(value) }
+      end
+
+      # Redshift adapter silently ignores binds
+      def parameter_binding
+        if postgresql? && (ActiveRecord::VERSION::STRING.to_f >= 6.1 || prepared_statements?)
+          # Active Record < 6.1 silently ignores binds with Postgres when prepared statements are disabled
+          :numeric
+        elsif sqlite?
+          :numeric
+        elsif mysql? && prepared_statements?
+          # Active Record silently ignores binds with MySQL when prepared statements are disabled
+          :positional
+        elsif sqlserver?
+          proc do |statement, variables|
+            variables.each_with_index do |(k, _), i|
+              statement = statement.gsub("{#{k}}", "@#{i} ")
+            end
+            [statement, variables.values]
+          end
+        end
+      end
+
       protected
 
       def select_all(statement, params = [])
@@ -207,6 +233,10 @@ module Blazer
         ["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
       end
 
+      def sqlite?
+        ["SQLite"].include?(adapter_name)
+      end
+
       def sqlserver?
         ["SQLServer", "tinytds", "mssql"].include?(adapter_name)
       end
@@ -282,6 +312,10 @@ module Blazer
           end
         end
       end
+
+      def prepared_statements?
+        connection_model.connection.prepared_statements
+      end
     end
   end
 end

data/lib/blazer/data_source.rb

@@ -89,7 +89,19 @@ module Blazer
       Blazer.cache.delete(run_cache_key(run_id))
     end
 
+    def sub_variables(statement, vars)
+      statement = statement.dup
+      vars.each do |var, value|
+        # use block form to disable back-references
+        statement.gsub!("{#{var}}") { quote(value) }
+      end
+      statement
+    end
+
     def run_statement(statement, options = {})
+      statement = Statement.new(statement, self) if statement.is_a?(String)
+      statement.bind unless statement.bind_statement
+
       async = options[:async]
       result = nil
       if cache_mode != "off"
@@ -118,7 +130,7 @@ module Blazer
         if options[:run_id]
           comment << ",run_id:#{options[:run_id]}"
         end
-        result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
+        result = run_statement_helper(statement, comment, async ? options[:run_id] : nil, options)
       end
 
       result
@@ -133,13 +145,68 @@ module Blazer
     end
 
     def statement_cache_key(statement)
-      cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
+      cache_key(["statement", id, Digest::MD5.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
     end
 
     def run_cache_key(run_id)
       cache_key(["run", run_id])
     end
 
+    def quote(value)
+      if quoting == :backslash_escape || quoting == :single_quote_escape
+        # only need to support types generated by process_vars
+        if value.is_a?(Integer) || value.is_a?(Float)
+          value.to_s
+        elsif value.nil?
+          "NULL"
+        else
+          value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
+
+          if quoting == :backslash_escape
+            "'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
+          else
+            "'#{value.gsub("'", "''")}'"
+          end
+        end
+      elsif quoting.respond_to?(:call)
+        quoting.call(value)
+      elsif quoting.nil?
+        raise Blazer::Error, "Quoting not specified"
+      else
+        raise Blazer::Error, "Unknown quoting"
+      end
+    end
+
+    def bind_params(statement, variables)
+      if parameter_binding == :positional
+        locations = []
+        variables.each do |k, v|
+          i = 0
+          while (idx = statement.index("{#{k}}", i))
+            locations << [v, idx]
+            i = idx + 1
+          end
+        end
+        variables.each do |k, v|
+          statement = statement.gsub("{#{k}}", "?")
+        end
+        [statement, locations.sort_by(&:last).map(&:first)]
+      elsif parameter_binding == :numeric
+        variables.each_with_index do |(k, v), i|
+          # add trailing space if followed by digit
+          # try to keep minimal to avoid fixing invalid queries like SELECT{var}
+          statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
+        end
+        [statement, variables.values]
+      elsif parameter_binding.respond_to?(:call)
+        parameter_binding.call(statement, variables)
+      elsif parameter_binding.nil?
+        [sub_variables(statement, variables), []]
+      else
+        raise Blazer::Error, "Unknown bind parameters"
+      end
+    end
+
     protected
 
     def adapter_instance
@@ -157,9 +224,22 @@ module Blazer
       end
     end
 
-    def run_statement_helper(statement, comment, run_id)
+    def quoting
+      @quoting ||= adapter_instance.quoting
+    end
+
+    def parameter_binding
+      @parameter_binding ||= adapter_instance.parameter_binding
+    end
+
+    def run_statement_helper(statement, comment, run_id, options)
       start_time = Time.now
-      columns, rows, error = adapter_instance.run_statement(statement, comment)
+      columns, rows, error =
+        if adapter_instance.parameter_binding
+          adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
+        else
+          adapter_instance.run_statement(statement.bind_statement, comment)
+        end
       duration = Time.now - start_time
 
       cache_data = nil
@@ -168,7 +248,7 @@ module Blazer
         cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
       end
 
-      if cache && cache_data && adapter_instance.cachable?(statement)
+      if cache && cache_data && adapter_instance.cachable?(statement.bind_statement)
         Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
       end
 

data/lib/blazer/engine.rb

@@ -30,10 +30,6 @@ module Blazer
       Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
       Blazer.forecasting = Blazer.settings["forecasting"] || false
       Blazer.async = Blazer.settings["async"] || false
-      if Blazer.async
-        require "blazer/run_statement_job"
-      end
-
       Blazer.images = Blazer.settings["images"] || false
       Blazer.override_csp = Blazer.settings["override_csp"] || false
       Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]

data/lib/blazer/result.rb

@@ -56,6 +56,8 @@ module Blazer
             "time"
           elsif v.nil?
             nil
+          elsif v.is_a?(String) && v.encoding == Encoding::BINARY
+            "binary"
           else
             "string"
           end

data/lib/blazer/run_statement.rb

@@ -1,12 +1,16 @@
 module Blazer
   class RunStatement
-    def perform(data_source, statement, options = {})
+    def perform(statement, options = {})
       query = options[:query]
-      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      data_source = statement.data_source
+      statement.bind
 
       # audit
       if Blazer.audit
-        audit = Blazer::Audit.new(statement: statement)
+        audit_statement = statement.bind_statement
+        audit_statement += "\n\n#{statement.bind_values.to_json}" if statement.bind_values.any?
+        audit = Blazer::Audit.new(statement: audit_statement)
         audit.query = query
         audit.data_source = data_source.id
         audit.user = options[:user]

data/lib/blazer/run_statement_job.rb

@@ -3,10 +3,12 @@ module Blazer
     self.queue_adapter = :async
 
     def perform(data_source_id, statement, options)
-      data_source = Blazer.data_sources[data_source_id]
+      statement = Blazer::Statement.new(statement, data_source_id)
+      statement.values = options.delete(:values)
+      data_source = statement.data_source
       begin
         ActiveRecord::Base.connection_pool.with_connection do
-          Blazer::RunStatement.new.perform(data_source, statement, options)
+          Blazer::RunStatement.new.perform(statement, options)
         end
       rescue Exception => e
         Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)

data/lib/blazer/slack_notifier.rb

@@ -67,15 +67,18 @@ module Blazer
       Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
     end
 
+    # TODO use return value
     def self.post(payload)
       if Blazer.slack_webhook_url.present?
-        post_api(Blazer.slack_webhook_url, payload, {})
+        response = post_api(Blazer.slack_webhook_url, payload, {})
+        response.is_a?(Net::HTTPSuccess) && response.body == "ok"
       else
         headers = {
           "Authorization" => "Bearer #{Blazer.slack_oauth_token}",
           "Content-type" => "application/json"
         }
-        post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response = post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response.is_a?(Net::HTTPSuccess) && (JSON.parse(response.body)["ok"] rescue false)
       end
     end