blazer 3.4.0 → 3.5.0

data/lib/blazer/adapters/clickhouse_adapter.rb

@@ -0,0 +1,136 @@
+module Blazer
+  module Adapters
+    class ClickhouseAdapter < BaseAdapter
+      def run_statement(statement, comment, bind_params)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          res = execute("#{statement} /*#{comment}*/", bind_params)
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            columns = data["meta"].map { |v| v["name"] }
+            rows = data["data"]
+            data["meta"].each_with_index do |c, i|
+              type = c["type"]
+              type = type[9..-2] if type.start_with?("Nullable(")
+              if type.start_with?("DateTime")
+                utc = ActiveSupport::TimeZone["Etc/UTC"]
+                rows.each do |row|
+                  row[i] &&= utc.parse(row[i])
+                end
+              elsif ["Date", "Date32"].include?(type)
+                rows.each do |row|
+                  row[i] &&= Date.parse(row[i])
+                end
+              end
+            end
+          else
+            error = res.body
+            error = Blazer::TIMEOUT_MESSAGE if error.include?("TIMEOUT_EXCEEDED")
+          end
+        rescue Errno::ECONNREFUSED => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        result = data_source.run_statement("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = currentDatabase() ORDER BY table_name")
+        result.rows.map(&:first)
+      end
+
+      def schema
+        result = data_source.run_statement("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = currentDatabase() ORDER BY 1, 2")
+        result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      # https://clickhouse.com/docs/sql-reference/syntax#string
+      def quoting
+        :backslash_escape
+      end
+
+      def parameter_binding
+        proc do |statement, variables|
+          variables.each do |k, v|
+            if v.nil?
+              statement = statement.gsub("{#{k}}") { "NULL" }
+            else
+              type =
+                case v
+                when Integer
+                  # TODO improve
+                  "Int64"
+                when Float
+                  "Float64"
+                when ActiveSupport::TimeWithZone
+                  "DateTime64(9, 'UTC')"
+                else
+                  "String"
+                end
+              statement = statement.gsub("{#{k}}") { "{#{k}: #{type}}" }
+            end
+          end
+          [statement, variables]
+        end
+      end
+
+      def cancel(run_id)
+        execute("KILL QUERY WHERE query LIKE {query: String} AND query NOT LIKE '%system.processes%' SYNC", {"query" => "%,run_id:#{run_id}%"})
+      end
+
+      private
+
+      def database
+        @database ||= settings["database"] || "default"
+      end
+
+      def execute(statement, bind_params)
+        require "net/http"
+
+        query_params = {
+          "database" => database,
+          "default_format" => "JSONCompact",
+          "output_format_json_quote_64bit_integers" => 0, # for ClickHouse < 25.8
+          "readonly" => 1
+        }
+        query_params["max_execution_time"] = data_source.timeout if data_source.timeout
+
+        post_data = {
+          "query" => statement
+        }
+        bind_params.each do |k, v|
+          v = v.utc.strftime("%Y-%m-%d %H:%M:%S.%N") if v.is_a?(ActiveSupport::TimeWithZone)
+          # https://github.com/ClickHouse/ClickHouse/issues/69656
+          v = v.gsub("\\") { "\\\\" } if v.is_a?(String)
+          post_data["param_#{k}"] = v.to_s unless v.nil?
+        end
+
+        uri = URI(settings["url"])
+        raise "Not supported" if uri.query
+        uri.query = URI.encode_www_form(query_params)
+
+        req = Net::HTTP::Post.new(uri)
+        req.basic_auth(uri.user, uri.password) if uri.user || uri.password
+        req.set_form(post_data, "multipart/form-data")
+
+        options = {
+          use_ssl: uri.scheme == "https",
+          open_timeout: 3,
+          read_timeout: data_source.timeout ? data_source.timeout + 3 : 30
+        }
+
+        Net::HTTP.start(uri.hostname, uri.port, options) do |http|
+          http.request(req)
+        end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/druid_adapter.rb

@@ -4,9 +4,7 @@ module Blazer
       TIMESTAMP_REGEX = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\z/
 
       def run_statement(statement, comment, bind_params)
-        require "json"
         require "net/http"
-        require "uri"
 
         columns = []
         rows = []

data/lib/blazer/adapters/snowflake2_adapter.rb

@@ -0,0 +1,198 @@
+module Blazer
+  module Adapters
+    class Snowflake2Adapter < BaseAdapter
+      def run_statement(statement, comment, bind_params)
+        require "net/http"
+
+        columns = []
+        rows = []
+        error = nil
+
+        api_prefix = "https://#{settings.fetch("account_id")}.snowflakecomputing.com/api/v2/statements/"
+        authorization = "Bearer #{settings.fetch("access_token")}"
+
+        submit_uri = URI(api_prefix)
+        # for testing
+        # submit_uri.query = URI.encode_www_form({"async" => true})
+
+        timeout = data_source.timeout || 300
+        stop_at = Time.now + timeout
+
+        post_data = {
+          statement: "#{statement} /*#{comment}*/",
+          timeout: timeout
+        }
+        # use empty? since any? doesn't work for [nil]
+        unless bind_params.empty?
+          post_data[:bindings] =
+            bind_params.map.with_index.to_h do |v, i|
+              type =
+                case v
+                when Integer
+                  "FIXED"
+                when Float
+                  "REAL"
+                when ActiveSupport::TimeWithZone
+                  "TIMESTAMP_NTZ"
+                else
+                  "TEXT"
+                end
+              v = v.to_i * 1000000000 + v.nsec if v.is_a?(ActiveSupport::TimeWithZone)
+              [i + 1, {type: type, value: v&.to_s}]
+            end
+        end
+        post_data[:database] = settings["database"] if settings["database"]
+        post_data[:schema] = settings["schema"] if settings["schema"]
+        post_data[:warehouse] = settings["warehouse"] if settings["warehouse"]
+        post_data[:role] = settings["role"] if settings["role"]
+
+        req = Net::HTTP::Post.new(submit_uri)
+        req["Authorization"] = authorization
+        req.body = post_data.to_json
+
+        options = {
+          use_ssl: true,
+          open_timeout: 3,
+          read_timeout: timeout + 3
+        }
+
+        begin
+          res = Net::HTTP.start(submit_uri.hostname, submit_uri.port, options) do |http|
+            http.request(req)
+          end
+
+          while res.is_a?(Net::HTTPAccepted)
+            if Time.now > stop_at
+              error = Blazer::TIMEOUT_MESSAGE
+              break
+            end
+
+            sleep(3)
+
+            data = JSON.parse(res.body)
+            statement_uri = URI("#{api_prefix}#{CGI.escape(data["statementHandle"])}")
+            req = Net::HTTP::Get.new(statement_uri)
+            req["Authorization"] = authorization
+
+            res = Net::HTTP.start(statement_uri.hostname, statement_uri.port, options) do |http|
+              http.request(req)
+            end
+          end
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            metadata = data["resultSetMetaData"]
+            columns = metadata["rowType"].map { |v| v["name"].downcase }
+            column_types = metadata["rowType"].map { |v| v["type"] }
+            rows = data["data"]
+
+            if metadata["partitionInfo"]
+              1.upto(metadata["partitionInfo"].size - 1) do |i|
+                statement_uri.query = URI.encode_www_form({"partition" => i})
+                req = Net::HTTP::Get.new(statement_uri)
+                req["Authorization"] = authorization
+
+                res = Net::HTTP.start(statement_uri.hostname, statement_uri.port, options) do |http|
+                  http.request(req)
+                end
+
+                if res.is_a?(Net::HTTPSuccess)
+                  data = JSON.parse(res.body)
+                  rows += data["data"]
+                else
+                  data = JSON.parse(res.body)
+                  error = data["message"]
+                  break
+                end
+              end
+            end
+
+            if error
+              columns.clear
+              rows.clear
+            else
+              column_types.each_with_index do |c, i|
+                # TODO handle more types
+                case c
+                when "fixed"
+                  rows.each do |row|
+                    row[i] &&= row[i].to_i
+                  end
+                when "real"
+                  rows.each do |row|
+                    row[i] &&= row[i].to_f
+                  end
+                when "timestamp_ntz"
+                  utc = ActiveSupport::TimeZone["Etc/UTC"]
+                  rows.each do |row|
+                    row[i] &&= utc.strptime(row[i], "%s.%N")
+                  end
+                end
+              end
+            end
+          elsif !error
+            data = JSON.parse(res.body)
+            error = data["message"]
+            error = Blazer::TIMEOUT_MESSAGE if error.include?("Statement reached its statement or warehouse timeout")
+          end
+        rescue Errno::ECONNREFUSED => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        sql = "SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema != 'INFORMATION_SCHEMA'"
+        result = data_source.run_statement(sql)
+        result.rows.sort_by { |r| [r[0] == default_schema ? "" : r[0], r[1]] }.map do |row|
+          table =
+            if row[0] == default_schema
+              row[1]
+            else
+              "#{row[0]}.#{row[1]}"
+            end
+
+          # TODO quote if needed
+          table.downcase
+        end
+      end
+
+      def schema
+        sql = "SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema != 'INFORMATION_SCHEMA'"
+        result = data_source.run_statement(sql)
+        result.rows.group_by { |r| [r[0], r[1]] }.sort_by { |k, _| [k[0] == default_schema ? "" : k[0], k[1]] }.map { |k, vs| {schema: k[0].downcase, table: k[1].downcase, columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2].downcase, data_type: v[3].downcase} }} }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      # https://docs.snowflake.com/en/sql-reference/data-types-text#escape-sequences-in-single-quoted-string-constants
+      def quoting
+        :backslash_escape
+      end
+
+      # https://docs.snowflake.com/en/developer-guide/sql-api/submitting-requests#label-sql-api-bind-variables
+      def parameter_binding
+        :positional
+      end
+
+      # https://docs.snowflake.com/en/developer-guide/sql-api/cancelling-requests
+      def cancel(run_id)
+        # TODO
+      end
+
+      private
+
+      def default_schema
+        @default_schema ||=
+          if settings["schema"]
+            settings["schema"]
+          else
+            data_source.run_statement("SELECT CURRENT_SCHEMA()").rows[0][0]
+          end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/soda_adapter.rb

@@ -2,9 +2,7 @@ module Blazer
   module Adapters
     class SodaAdapter < BaseAdapter
       def run_statement(statement, comment)
-        require "json"
         require "net/http"
-        require "uri"
 
         columns = []
         rows = []

data/lib/blazer/adapters/sql_adapter.rb

@@ -74,6 +74,8 @@ module Blazer
         sql =
           if sqlite?
             "SELECT NULL, name FROM sqlite_master WHERE type IN ('table', 'view') ORDER BY name"
+          elsif postgresql?
+            add_schemas("SELECT * FROM (SELECT table_schema, table_name FROM information_schema.tables UNION ALL SELECT n.nspname AS table_schema, c.relname AS table_name FROM pg_class c INNER JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.relkind = 'm' AND has_any_column_privilege(c.oid, 'SELECT'))")
           else
             add_schemas("SELECT table_schema, table_name FROM information_schema.tables")
           end
@@ -103,6 +105,8 @@ module Blazer
         sql =
           if sqlite?
             "SELECT NULL, t.name, c.name, c.type, c.cid FROM sqlite_master t INNER JOIN pragma_table_info(t.name) c WHERE t.type IN ('table', 'view')"
+          elsif postgresql?
+            add_schemas("SELECT * FROM (SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns UNION ALL SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name, format_type(a.atttypid, a.atttypmod) AS data_type, a.attnum AS ordinal_position FROM pg_attribute a INNER JOIN pg_class c ON a.attrelid = c.oid INNER JOIN pg_namespace n ON c.relnamespace = n.oid WHERE a.attnum > 0 AND NOT a.attisdropped AND c.relkind = 'm' AND has_column_privilege(c.oid, a.attnum, 'SELECT'))")
           else
             add_schemas("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns")
           end

data/lib/blazer/adapters.rb

@@ -1,6 +1,7 @@
 Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
 Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
 Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "clickhouse", Blazer::Adapters::ClickhouseAdapter
 Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
 Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
 Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
@@ -15,4 +16,5 @@ Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
 Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
 Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
 Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter
+Blazer.register_adapter "snowflake2", Blazer::Adapters::Snowflake2Adapter
 Blazer.register_adapter "trino", Blazer::Adapters::PrestoAdapter

data/lib/blazer/check_types.rb

@@ -0,0 +1,20 @@
+Blazer.register_check_type "bad_data", "Any results (bad data)" do |result|
+  result.rows.any? ? "failing" : "passing"
+end
+
+Blazer.register_check_type "missing_data", "No results (missing data)" do |result|
+  result.rows.any? ? "passing" : "failing"
+end
+
+Blazer.register_check_type "anomaly", "Anomaly (most recent data point)" do |result|
+  anomaly, message = result.detect_anomaly
+  state =
+    if anomaly.nil?
+      "error"
+    elsif anomaly
+      "failing"
+    else
+      "passing"
+    end
+  [state, message]
+end

data/lib/blazer/email_notifier.rb

@@ -0,0 +1,49 @@
+module Blazer
+  class EmailNotifier
+    def self.failing_checks(checks)
+      emails = {}
+      checks.each do |check|
+        split_emails(check).each do |email|
+          (emails[email] ||= []) << check
+        end
+      end
+      emails.each do |email, checks|
+        Safely.safely do
+          Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+        end
+      end
+    end
+
+    def self.state_change(check:, state:, state_was:, result:, message:, check_type:)
+      Blazer::CheckMailer.state_change(check, state, state_was, result.rows.size, message, result.columns, result.rows.first(10).as_json, result.column_types, check_type).deliver_now if check.emails.present?
+    end
+
+    # TODO improve name
+    def self.notify_list(check)
+      split_emails(check)
+    end
+
+    def self.before_validation(check)
+      # some people like doing ; instead of ,
+      # but we know what they mean, so let's fix it
+      # also, some people like to use whitespace
+      if check.emails.present?
+        check.emails = check.emails.strip.gsub(/[;\s]/, ",").gsub(/,+/, ", ")
+      end
+    end
+
+    def self.validate(check)
+      unless split_emails(check).all? { |e| e =~ /\A\S+@\S+\.\S+\z/ }
+        check.errors.add(:base, "Invalid emails")
+      end
+    end
+
+    def self.fields
+      [:emails]
+    end
+
+    def self.split_emails(check)
+      check.emails.to_s.downcase.split(",").map(&:strip)
+    end
+  end
+end

data/lib/blazer/result.rb

@@ -20,6 +20,21 @@ module Blazer
       cached_at.present?
     end
 
+    def explain
+      # TODO move to data source adapters
+      case @data_source.adapter
+      when "sql"
+        # TODO limit to Postgres and add other databases
+        if @columns == ["QUERY PLAN"]
+          @rows.map { |r| r[0] }.join("\n")
+        end
+      when "druid"
+        if @columns == ["PLAN"] && @rows.size == 1
+          @rows[0][0]
+        end
+      end
+    end
+
     def smart_values
       @smart_values ||= begin
         smart_values = {}

data/lib/blazer/slack_notifier.rb

@@ -1,12 +1,25 @@
-require "net/http"
-
 module Blazer
   class SlackNotifier
-    def self.state_change(check, state, state_was, rows_count, error, check_type)
-      check.split_slack_channels.each do |channel|
+    def self.failing_checks(checks)
+      slack_channels = {}
+      checks.each do |check|
+        split_slack_channels(check).each do |channel|
+          (slack_channels[channel] ||= []) << check
+        end
+      end
+      slack_channels.each do |channel, checks|
+        Safely.safely do
+          channel_failing_checks(channel, checks)
+        end
+      end
+    end
+
+    def self.state_change(check:, state:, state_was:, result:, message:, check_type:)
+      rows_count = result.rows.size
+      split_slack_channels(check).each do |channel|
         text =
-          if error
-            error
+          if message
+            message
           elsif rows_count > 0 && check_type == "bad_data"
             pluralize(rows_count, "row")
           end
@@ -27,7 +40,24 @@ module Blazer
       end
     end
 
-    def self.failing_checks(channel, checks)
+    # TODO improve name
+    def self.notify_list(check)
+      split_slack_channels(check)
+    end
+
+    def self.fields
+      Blazer.slack? ? [:slack_channels] : []
+    end
+
+    def self.split_slack_channels(check)
+      if Blazer.slack?
+        check.slack_channels.to_s.downcase.split(",").map(&:strip)
+      else
+        []
+      end
+    end
+
+    def self.channel_failing_checks(channel, checks)
       text =
         checks.map do |check|
           "<#{query_url(check.query_id)}|#{escape(check.query.name)}> #{escape(check.state)}"
@@ -69,6 +99,8 @@ module Blazer
 
     # TODO use return value
     def self.post(payload)
+      require "net/http"
+
       if Blazer.slack_webhook_url.present?
         response = post_api(Blazer.slack_webhook_url, payload, {})
         response.is_a?(Net::HTTPSuccess) && response.body == "ok"

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "3.4.0"
+  VERSION = "3.5.0"
 end

data/lib/blazer.rb

@@ -1,9 +1,8 @@
 # dependencies
-require "chartkick"
+require "csv"
 require "safely/core"
 
 # stdlib
-require "csv"
 require "digest/sha2"
 require "json"
 require "yaml"
@@ -21,6 +20,7 @@ require_relative "blazer/adapters/base_adapter"
 require_relative "blazer/adapters/athena_adapter"
 require_relative "blazer/adapters/bigquery_adapter"
 require_relative "blazer/adapters/cassandra_adapter"
+require_relative "blazer/adapters/clickhouse_adapter"
 require_relative "blazer/adapters/drill_adapter"
 require_relative "blazer/adapters/druid_adapter"
 require_relative "blazer/adapters/elasticsearch_adapter"
@@ -35,6 +35,11 @@ require_relative "blazer/adapters/soda_adapter"
 require_relative "blazer/adapters/spark_adapter"
 require_relative "blazer/adapters/sql_adapter"
 require_relative "blazer/adapters/snowflake_adapter"
+require_relative "blazer/adapters/snowflake2_adapter"
+
+# notifiers
+require_relative "blazer/email_notifier"
+require_relative "blazer/slack_notifier"
 
 # engine
 require_relative "blazer/engine"
@@ -46,8 +51,6 @@ module Blazer
 
   # actionmailer optional
   autoload :CheckMailer, "blazer/check_mailer"
-  # net/http optional
-  autoload :SlackNotifier, "blazer/slack_notifier"
   # activejob optional
   autoload :RunStatementJob, "blazer/run_statement_job"
 
@@ -83,6 +86,8 @@ module Blazer
 
   VARIABLE_MESSAGE = "Variable cannot be used in this position"
   TIMEOUT_MESSAGE = "Query timed out :("
+
+  # TODO move to SqlAdapter
   TIMEOUT_ERRORS = [
     "canceling statement due to statement timeout", # postgres
     "canceling statement due to conflict with recovery", # postgres
@@ -188,28 +193,9 @@ module Blazer
   end
 
   def self.send_failing_checks
-    emails = {}
-    slack_channels = {}
-
-    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
-      check.split_emails.each do |email|
-        (emails[email] ||= []) << check
-      end
-      check.split_slack_channels.each do |channel|
-        (slack_channels[channel] ||= []) << check
-      end
-    end
-
-    emails.each do |email, checks|
-      Safely.safely do
-        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
-      end
-    end
-
-    slack_channels.each do |channel, checks|
-      Safely.safely do
-        Blazer::SlackNotifier.failing_checks(channel, checks)
-      end
+    checks = Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).to_a
+    Blazer.notifiers.each do |notifier|
+      notifier.failing_checks(checks)
     end
   end
 
@@ -263,6 +249,22 @@ module Blazer
     forecasters[name] = forecaster
   end
 
+  def self.notifiers
+    @notifiers ||= []
+  end
+
+  def self.register_notifier(notifier)
+    self.notifiers << notifier
+  end
+
+  def self.check_types
+    @check_types ||= {}
+  end
+
+  def self.register_check_type(name, label, &block)
+    check_types[name] = {label: label, block: block}
+  end
+
   def self.archive_queries
     raise "Audits must be enabled to archive" unless Blazer.audit
     raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
@@ -279,4 +281,8 @@ end
 
 require_relative "blazer/adapters"
 require_relative "blazer/anomaly_detectors"
+require_relative "blazer/check_types"
 require_relative "blazer/forecasters"
+
+Blazer.register_notifier Blazer::EmailNotifier
+Blazer.register_notifier Blazer::SlackNotifier

data/lib/generators/blazer/templates/config.yml.tt

@@ -62,11 +62,9 @@ check_schedules:
   - "5 minutes"
 
 # enable anomaly detection
-# note: with trend, time series are sent to https://trendapi.org
 # anomaly_checks: prophet / trend / anomaly_detection
 
 # enable forecasting
-# note: with trend, time series are sent to https://trendapi.org
 # forecasting: prophet / trend
 
 # enable map

data/licenses/LICENSE-chart.js.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2022 Chart.js Contributors
+Copyright (c) 2014-2024 Chart.js Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: