blazer 3.3.0 → 3.5.0

data/lib/blazer/adapters/clickhouse_adapter.rb

@@ -0,0 +1,136 @@
+module Blazer
+  module Adapters
+    class ClickhouseAdapter < BaseAdapter
+      def run_statement(statement, comment, bind_params)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          res = execute("#{statement} /*#{comment}*/", bind_params)
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            columns = data["meta"].map { |v| v["name"] }
+            rows = data["data"]
+            data["meta"].each_with_index do |c, i|
+              type = c["type"]
+              type = type[9..-2] if type.start_with?("Nullable(")
+              if type.start_with?("DateTime")
+                utc = ActiveSupport::TimeZone["Etc/UTC"]
+                rows.each do |row|
+                  row[i] &&= utc.parse(row[i])
+                end
+              elsif ["Date", "Date32"].include?(type)
+                rows.each do |row|
+                  row[i] &&= Date.parse(row[i])
+                end
+              end
+            end
+          else
+            error = res.body
+            error = Blazer::TIMEOUT_MESSAGE if error.include?("TIMEOUT_EXCEEDED")
+          end
+        rescue Errno::ECONNREFUSED => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        result = data_source.run_statement("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = currentDatabase() ORDER BY table_name")
+        result.rows.map(&:first)
+      end
+
+      def schema
+        result = data_source.run_statement("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = currentDatabase() ORDER BY 1, 2")
+        result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      # https://clickhouse.com/docs/sql-reference/syntax#string
+      def quoting
+        :backslash_escape
+      end
+
+      def parameter_binding
+        proc do |statement, variables|
+          variables.each do |k, v|
+            if v.nil?
+              statement = statement.gsub("{#{k}}") { "NULL" }
+            else
+              type =
+                case v
+                when Integer
+                  # TODO improve
+                  "Int64"
+                when Float
+                  "Float64"
+                when ActiveSupport::TimeWithZone
+                  "DateTime64(9, 'UTC')"
+                else
+                  "String"
+                end
+              statement = statement.gsub("{#{k}}") { "{#{k}: #{type}}" }
+            end
+          end
+          [statement, variables]
+        end
+      end
+
+      def cancel(run_id)
+        execute("KILL QUERY WHERE query LIKE {query: String} AND query NOT LIKE '%system.processes%' SYNC", {"query" => "%,run_id:#{run_id}%"})
+      end
+
+      private
+
+      def database
+        @database ||= settings["database"] || "default"
+      end
+
+      def execute(statement, bind_params)
+        require "net/http"
+
+        query_params = {
+          "database" => database,
+          "default_format" => "JSONCompact",
+          "output_format_json_quote_64bit_integers" => 0, # for ClickHouse < 25.8
+          "readonly" => 1
+        }
+        query_params["max_execution_time"] = data_source.timeout if data_source.timeout
+
+        post_data = {
+          "query" => statement
+        }
+        bind_params.each do |k, v|
+          v = v.utc.strftime("%Y-%m-%d %H:%M:%S.%N") if v.is_a?(ActiveSupport::TimeWithZone)
+          # https://github.com/ClickHouse/ClickHouse/issues/69656
+          v = v.gsub("\\") { "\\\\" } if v.is_a?(String)
+          post_data["param_#{k}"] = v.to_s unless v.nil?
+        end
+
+        uri = URI(settings["url"])
+        raise "Not supported" if uri.query
+        uri.query = URI.encode_www_form(query_params)
+
+        req = Net::HTTP::Post.new(uri)
+        req.basic_auth(uri.user, uri.password) if uri.user || uri.password
+        req.set_form(post_data, "multipart/form-data")
+
+        options = {
+          use_ssl: uri.scheme == "https",
+          open_timeout: 3,
+          read_timeout: data_source.timeout ? data_source.timeout + 3 : 30
+        }
+
+        Net::HTTP.start(uri.hostname, uri.port, options) do |http|
+          http.request(req)
+        end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/druid_adapter.rb

@@ -4,9 +4,7 @@ module Blazer
       TIMESTAMP_REGEX = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\z/
 
       def run_statement(statement, comment, bind_params)
-        require "json"
         require "net/http"
-        require "uri"
 
         columns = []
         rows = []

data/lib/blazer/adapters/snowflake2_adapter.rb

@@ -0,0 +1,198 @@
+module Blazer
+  module Adapters
+    class Snowflake2Adapter < BaseAdapter
+      def run_statement(statement, comment, bind_params)
+        require "net/http"
+
+        columns = []
+        rows = []
+        error = nil
+
+        api_prefix = "https://#{settings.fetch("account_id")}.snowflakecomputing.com/api/v2/statements/"
+        authorization = "Bearer #{settings.fetch("access_token")}"
+
+        submit_uri = URI(api_prefix)
+        # for testing
+        # submit_uri.query = URI.encode_www_form({"async" => true})
+
+        timeout = data_source.timeout || 300
+        stop_at = Time.now + timeout
+
+        post_data = {
+          statement: "#{statement} /*#{comment}*/",
+          timeout: timeout
+        }
+        # use empty? since any? doesn't work for [nil]
+        unless bind_params.empty?
+          post_data[:bindings] =
+            bind_params.map.with_index.to_h do |v, i|
+              type =
+                case v
+                when Integer
+                  "FIXED"
+                when Float
+                  "REAL"
+                when ActiveSupport::TimeWithZone
+                  "TIMESTAMP_NTZ"
+                else
+                  "TEXT"
+                end
+              v = v.to_i * 1000000000 + v.nsec if v.is_a?(ActiveSupport::TimeWithZone)
+              [i + 1, {type: type, value: v&.to_s}]
+            end
+        end
+        post_data[:database] = settings["database"] if settings["database"]
+        post_data[:schema] = settings["schema"] if settings["schema"]
+        post_data[:warehouse] = settings["warehouse"] if settings["warehouse"]
+        post_data[:role] = settings["role"] if settings["role"]
+
+        req = Net::HTTP::Post.new(submit_uri)
+        req["Authorization"] = authorization
+        req.body = post_data.to_json
+
+        options = {
+          use_ssl: true,
+          open_timeout: 3,
+          read_timeout: timeout + 3
+        }
+
+        begin
+          res = Net::HTTP.start(submit_uri.hostname, submit_uri.port, options) do |http|
+            http.request(req)
+          end
+
+          while res.is_a?(Net::HTTPAccepted)
+            if Time.now > stop_at
+              error = Blazer::TIMEOUT_MESSAGE
+              break
+            end
+
+            sleep(3)
+
+            data = JSON.parse(res.body)
+            statement_uri = URI("#{api_prefix}#{CGI.escape(data["statementHandle"])}")
+            req = Net::HTTP::Get.new(statement_uri)
+            req["Authorization"] = authorization
+
+            res = Net::HTTP.start(statement_uri.hostname, statement_uri.port, options) do |http|
+              http.request(req)
+            end
+          end
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            metadata = data["resultSetMetaData"]
+            columns = metadata["rowType"].map { |v| v["name"].downcase }
+            column_types = metadata["rowType"].map { |v| v["type"] }
+            rows = data["data"]
+
+            if metadata["partitionInfo"]
+              1.upto(metadata["partitionInfo"].size - 1) do |i|
+                statement_uri.query = URI.encode_www_form({"partition" => i})
+                req = Net::HTTP::Get.new(statement_uri)
+                req["Authorization"] = authorization
+
+                res = Net::HTTP.start(statement_uri.hostname, statement_uri.port, options) do |http|
+                  http.request(req)
+                end
+
+                if res.is_a?(Net::HTTPSuccess)
+                  data = JSON.parse(res.body)
+                  rows += data["data"]
+                else
+                  data = JSON.parse(res.body)
+                  error = data["message"]
+                  break
+                end
+              end
+            end
+
+            if error
+              columns.clear
+              rows.clear
+            else
+              column_types.each_with_index do |c, i|
+                # TODO handle more types
+                case c
+                when "fixed"
+                  rows.each do |row|
+                    row[i] &&= row[i].to_i
+                  end
+                when "real"
+                  rows.each do |row|
+                    row[i] &&= row[i].to_f
+                  end
+                when "timestamp_ntz"
+                  utc = ActiveSupport::TimeZone["Etc/UTC"]
+                  rows.each do |row|
+                    row[i] &&= utc.strptime(row[i], "%s.%N")
+                  end
+                end
+              end
+            end
+          elsif !error
+            data = JSON.parse(res.body)
+            error = data["message"]
+            error = Blazer::TIMEOUT_MESSAGE if error.include?("Statement reached its statement or warehouse timeout")
+          end
+        rescue Errno::ECONNREFUSED => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        sql = "SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema != 'INFORMATION_SCHEMA'"
+        result = data_source.run_statement(sql)
+        result.rows.sort_by { |r| [r[0] == default_schema ? "" : r[0], r[1]] }.map do |row|
+          table =
+            if row[0] == default_schema
+              row[1]
+            else
+              "#{row[0]}.#{row[1]}"
+            end
+
+          # TODO quote if needed
+          table.downcase
+        end
+      end
+
+      def schema
+        sql = "SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema != 'INFORMATION_SCHEMA'"
+        result = data_source.run_statement(sql)
+        result.rows.group_by { |r| [r[0], r[1]] }.sort_by { |k, _| [k[0] == default_schema ? "" : k[0], k[1]] }.map { |k, vs| {schema: k[0].downcase, table: k[1].downcase, columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2].downcase, data_type: v[3].downcase} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      # https://docs.snowflake.com/en/sql-reference/data-types-text#escape-sequences-in-single-quoted-string-constants
+      def quoting
+        :backslash_escape
+      end
+
+      # https://docs.snowflake.com/en/developer-guide/sql-api/submitting-requests#label-sql-api-bind-variables
+      def parameter_binding
+        :positional
+      end
+
+      # https://docs.snowflake.com/en/developer-guide/sql-api/cancelling-requests
+      def cancel(run_id)
+        # TODO
+      end
+
+      private
+
+      def default_schema
+        @default_schema ||=
+          if settings["schema"]
+            settings["schema"]
+          else
+            data_source.run_statement("SELECT CURRENT_SCHEMA()").rows[0][0]
+          end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/soda_adapter.rb

@@ -2,9 +2,7 @@ module Blazer
   module Adapters
     class SodaAdapter < BaseAdapter
       def run_statement(statement, comment)
-        require "json"
         require "net/http"
-        require "uri"
 
         columns = []
         rows = []

data/lib/blazer/adapters/sql_adapter.rb

@@ -74,6 +74,8 @@ module Blazer
         sql =
           if sqlite?
             "SELECT NULL, name FROM sqlite_master WHERE type IN ('table', 'view') ORDER BY name"
+          elsif postgresql?
+            add_schemas("SELECT * FROM (SELECT table_schema, table_name FROM information_schema.tables UNION ALL SELECT n.nspname AS table_schema, c.relname AS table_name FROM pg_class c INNER JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.relkind = 'm' AND has_any_column_privilege(c.oid, 'SELECT'))")
           else
             add_schemas("SELECT table_schema, table_name FROM information_schema.tables")
           end
@@ -103,6 +105,8 @@ module Blazer
         sql =
           if sqlite?
             "SELECT NULL, t.name, c.name, c.type, c.cid FROM sqlite_master t INNER JOIN pragma_table_info(t.name) c WHERE t.type IN ('table', 'view')"
+          elsif postgresql?
+            add_schemas("SELECT * FROM (SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns UNION ALL SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name, format_type(a.atttypid, a.atttypmod) AS data_type, a.attnum AS ordinal_position FROM pg_attribute a INNER JOIN pg_class c ON a.attrelid = c.oid INNER JOIN pg_namespace n ON c.relnamespace = n.oid WHERE a.attnum > 0 AND NOT a.attisdropped AND c.relkind = 'm' AND has_column_privilege(c.oid, a.attnum, 'SELECT'))")
           else
             add_schemas("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns")
           end
@@ -255,7 +259,7 @@ module Blazer
         connection_model.connection.select_all(statement)
       end
 
-      # seperate from select_all to prevent mysql error
+      # separate from select_all to prevent mysql error
       def execute(statement)
         connection_model.connection.execute(statement)
       end

data/lib/blazer/adapters.rb

@@ -1,6 +1,7 @@
 Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
 Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
 Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "clickhouse", Blazer::Adapters::ClickhouseAdapter
 Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
 Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
 Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
@@ -15,4 +16,5 @@ Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
 Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
 Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
 Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter
+Blazer.register_adapter "snowflake2", Blazer::Adapters::Snowflake2Adapter
 Blazer.register_adapter "trino", Blazer::Adapters::PrestoAdapter

data/lib/blazer/check_types.rb

@@ -0,0 +1,20 @@
+Blazer.register_check_type "bad_data", "Any results (bad data)" do |result|
+  result.rows.any? ? "failing" : "passing"
+end
+
+Blazer.register_check_type "missing_data", "No results (missing data)" do |result|
+  result.rows.any? ? "passing" : "failing"
+end
+
+Blazer.register_check_type "anomaly", "Anomaly (most recent data point)" do |result|
+  anomaly, message = result.detect_anomaly
+  state =
+    if anomaly.nil?
+      "error"
+    elsif anomaly
+      "failing"
+    else
+      "passing"
+    end
+  [state, message]
+end

data/lib/blazer/data_source.rb

@@ -204,7 +204,7 @@ module Blazer
     def adapter_instance
       @adapter_instance ||= begin
         # TODO add required settings to adapters
-        unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
+        unless settings["url"] || Rails.env.local? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
           raise Blazer::Error, "Empty url for data source: #{id}"
         end
 

data/lib/blazer/email_notifier.rb

@@ -0,0 +1,49 @@
+module Blazer
+  class EmailNotifier
+    def self.failing_checks(checks)
+      emails = {}
+      checks.each do |check|
+        split_emails(check).each do |email|
+          (emails[email] ||= []) << check
+        end
+      end
+      emails.each do |email, checks|
+        Safely.safely do
+          Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+        end
+      end
+    end
+
+    def self.state_change(check:, state:, state_was:, result:, message:, check_type:)
+      Blazer::CheckMailer.state_change(check, state, state_was, result.rows.size, message, result.columns, result.rows.first(10).as_json, result.column_types, check_type).deliver_now if check.emails.present?
+    end
+
+    # TODO improve name
+    def self.notify_list(check)
+      split_emails(check)
+    end
+
+    def self.before_validation(check)
+      # some people like doing ; instead of ,
+      # but we know what they mean, so let's fix it
+      # also, some people like to use whitespace
+      if check.emails.present?
+        check.emails = check.emails.strip.gsub(/[;\s]/, ",").gsub(/,+/, ", ")
+      end
+    end
+
+    def self.validate(check)
+      unless split_emails(check).all? { |e| e =~ /\A\S+@\S+\.\S+\z/ }
+        check.errors.add(:base, "Invalid emails")
+      end
+    end
+
+    def self.fields
+      [:emails]
+    end
+
+    def self.split_emails(check)
+      check.emails.to_s.downcase.split(",").map(&:strip)
+    end
+  end
+end

data/lib/blazer/result.rb

@@ -20,6 +20,21 @@ module Blazer
       cached_at.present?
     end
 
+    def explain
+      # TODO move to data source adapters
+      case @data_source.adapter
+      when "sql"
+        # TODO limit to Postgres and add other databases
+        if @columns == ["QUERY PLAN"]
+          @rows.map { |r| r[0] }.join("\n")
+        end
+      when "druid"
+        if @columns == ["PLAN"] && @rows.size == 1
+          @rows[0][0]
+        end
+      end
+    end
+
     def smart_values
       @smart_values ||= begin
         smart_values = {}

data/lib/blazer/slack_notifier.rb

@@ -1,12 +1,25 @@
-require "net/http"
-
 module Blazer
   class SlackNotifier
-    def self.state_change(check, state, state_was, rows_count, error, check_type)
-      check.split_slack_channels.each do |channel|
+    def self.failing_checks(checks)
+      slack_channels = {}
+      checks.each do |check|
+        split_slack_channels(check).each do |channel|
+          (slack_channels[channel] ||= []) << check
+        end
+      end
+      slack_channels.each do |channel, checks|
+        Safely.safely do
+          channel_failing_checks(channel, checks)
+        end
+      end
+    end
+
+    def self.state_change(check:, state:, state_was:, result:, message:, check_type:)
+      rows_count = result.rows.size
+      split_slack_channels(check).each do |channel|
         text =
-          if error
-            error
+          if message
+            message
           elsif rows_count > 0 && check_type == "bad_data"
             pluralize(rows_count, "row")
           end
@@ -27,7 +40,24 @@ module Blazer
       end
     end
 
-    def self.failing_checks(channel, checks)
+    # TODO improve name
+    def self.notify_list(check)
+      split_slack_channels(check)
+    end
+
+    def self.fields
+      Blazer.slack? ? [:slack_channels] : []
+    end
+
+    def self.split_slack_channels(check)
+      if Blazer.slack?
+        check.slack_channels.to_s.downcase.split(",").map(&:strip)
+      else
+        []
+      end
+    end
+
+    def self.channel_failing_checks(channel, checks)
       text =
         checks.map do |check|
           "<#{query_url(check.query_id)}|#{escape(check.query.name)}> #{escape(check.state)}"
@@ -69,6 +99,8 @@ module Blazer
 
     # TODO use return value
     def self.post(payload)
+      require "net/http"
+
       if Blazer.slack_webhook_url.present?
         response = post_api(Blazer.slack_webhook_url, payload, {})
         response.is_a?(Net::HTTPSuccess) && response.body == "ok"

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "3.3.0"
+  VERSION = "3.5.0"
 end

data/lib/blazer.rb

@@ -1,9 +1,8 @@
 # dependencies
-require "chartkick"
+require "csv"
 require "safely/core"
 
 # stdlib
-require "csv"
 require "digest/sha2"
 require "json"
 require "yaml"
@@ -21,6 +20,7 @@ require_relative "blazer/adapters/base_adapter"
 require_relative "blazer/adapters/athena_adapter"
 require_relative "blazer/adapters/bigquery_adapter"
 require_relative "blazer/adapters/cassandra_adapter"
+require_relative "blazer/adapters/clickhouse_adapter"
 require_relative "blazer/adapters/drill_adapter"
 require_relative "blazer/adapters/druid_adapter"
 require_relative "blazer/adapters/elasticsearch_adapter"
@@ -35,6 +35,11 @@ require_relative "blazer/adapters/soda_adapter"
 require_relative "blazer/adapters/spark_adapter"
 require_relative "blazer/adapters/sql_adapter"
 require_relative "blazer/adapters/snowflake_adapter"
+require_relative "blazer/adapters/snowflake2_adapter"
+
+# notifiers
+require_relative "blazer/email_notifier"
+require_relative "blazer/slack_notifier"
 
 # engine
 require_relative "blazer/engine"
@@ -46,8 +51,6 @@ module Blazer
 
   # actionmailer optional
   autoload :CheckMailer, "blazer/check_mailer"
-  # net/http optional
-  autoload :SlackNotifier, "blazer/slack_notifier"
   # activejob optional
   autoload :RunStatementJob, "blazer/run_statement_job"
 
@@ -83,6 +86,8 @@ module Blazer
 
   VARIABLE_MESSAGE = "Variable cannot be used in this position"
   TIMEOUT_MESSAGE = "Query timed out :("
+
+  # TODO move to SqlAdapter
   TIMEOUT_ERRORS = [
     "canceling statement due to statement timeout", # postgres
     "canceling statement due to conflict with recovery", # postgres
@@ -188,28 +193,9 @@ module Blazer
   end
 
   def self.send_failing_checks
-    emails = {}
-    slack_channels = {}
-
-    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
-      check.split_emails.each do |email|
-        (emails[email] ||= []) << check
-      end
-      check.split_slack_channels.each do |channel|
-        (slack_channels[channel] ||= []) << check
-      end
-    end
-
-    emails.each do |email, checks|
-      Safely.safely do
-        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
-      end
-    end
-
-    slack_channels.each do |channel, checks|
-      Safely.safely do
-        Blazer::SlackNotifier.failing_checks(channel, checks)
-      end
+    checks = Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).to_a
+    Blazer.notifiers.each do |notifier|
+      notifier.failing_checks(checks)
     end
   end
 
@@ -263,6 +249,22 @@ module Blazer
     forecasters[name] = forecaster
   end
 
+  def self.notifiers
+    @notifiers ||= []
+  end
+
+  def self.register_notifier(notifier)
+    self.notifiers << notifier
+  end
+
+  def self.check_types
+    @check_types ||= {}
+  end
+
+  def self.register_check_type(name, label, &block)
+    check_types[name] = {label: label, block: block}
+  end
+
   def self.archive_queries
     raise "Audits must be enabled to archive" unless Blazer.audit
     raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
@@ -279,4 +281,8 @@ end
 
 require_relative "blazer/adapters"
 require_relative "blazer/anomaly_detectors"
+require_relative "blazer/check_types"
 require_relative "blazer/forecasters"
+
+Blazer.register_notifier Blazer::EmailNotifier
+Blazer.register_notifier Blazer::SlackNotifier