finery 3.0.0

data/lib/blazer/slack_notifier.rb

@@ -0,0 +1,94 @@
+require "net/http"
+
+module Blazer
+  class SlackNotifier
+    def self.state_change(check, state, state_was, rows_count, error, check_type)
+      check.split_slack_channels.each do |channel|
+        text =
+          if error
+            error
+          elsif rows_count > 0 && check_type == "bad_data"
+            pluralize(rows_count, "row")
+          end
+
+        payload = {
+          channel: channel,
+          attachments: [
+            {
+              title: escape("Check #{state.titleize}: #{check.query.name}"),
+              title_link: query_url(check.query_id),
+              text: escape(text),
+              color: state == "passing" ? "good" : "danger"
+            }
+          ]
+        }
+
+        post(payload)
+      end
+    end
+
+    def self.failing_checks(channel, checks)
+      text =
+        checks.map do |check|
+          "<#{query_url(check.query_id)}|#{escape(check.query.name)}> #{escape(check.state)}"
+        end
+
+      payload = {
+        channel: channel,
+        attachments: [
+          {
+            title: escape("#{pluralize(checks.size, "Check")} Failing"),
+            text: text.join("\n"),
+            color: "warning"
+          }
+        ]
+      }
+
+      post(payload)
+    end
+
+    # https://api.slack.com/docs/message-formatting#how_to_escape_characters
+    # - Replace the ampersand, &, with &amp;
+    # - Replace the less-than sign, < with &lt;
+    # - Replace the greater-than sign, > with &gt;
+    # That's it. Don't HTML entity-encode the entire message.
+    def self.escape(str)
+      str.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;") if str
+    end
+
+    def self.pluralize(*args)
+      ActionController::Base.helpers.pluralize(*args)
+    end
+
+    # checks shouldn't have variables, but in any case,
+    # avoid passing variable params to url helpers
+    # (known unsafe parameters are removed, but still not ideal)
+    def self.query_url(id)
+      Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
+    end
+
+    # TODO use return value
+    def self.post(payload)
+      if Blazer.slack_webhook_url.present?
+        response = post_api(Blazer.slack_webhook_url, payload, {})
+        response.is_a?(Net::HTTPSuccess) && response.body == "ok"
+      else
+        headers = {
+          "Authorization" => "Bearer #{Blazer.slack_oauth_token}",
+          "Content-type" => "application/json"
+        }
+        response = post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response.is_a?(Net::HTTPSuccess) && (JSON.parse(response.body)["ok"] rescue false)
+      end
+    end
+
+    def self.post_api(url, payload, headers)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = 3
+      http.read_timeout = 5
+      http.post(uri.request_uri, payload.to_json, headers)
+    end
+  end
+end

data/lib/blazer/statement.rb

@@ -0,0 +1,77 @@
+module Blazer
+  class Statement
+    attr_reader :statement, :data_source, :bind_statement, :bind_values
+    attr_accessor :values
+
+    def initialize(statement, data_source = nil)
+      @statement = statement
+      @data_source = data_source.is_a?(String) ? Blazer.data_sources[data_source] : data_source
+      @values = {}
+    end
+
+    def variables
+      # strip commented out lines
+      # and regex {1} or {1,2}
+      @variables ||= statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+    end
+
+    def add_values(var_params)
+      variables.each do |var|
+        value = var_params[var].presence
+        value = nil unless value.is_a?(String) # ignore arrays and hashes
+        if value
+          if ["start_time", "end_time"].include?(var)
+            value = value.to_s.gsub(" ", "+") # fix for Quip bug
+          end
+
+          if var.end_with?("_at")
+            begin
+              value = Blazer.time_zone.parse(value)
+            rescue
+              # do nothing
+            end
+          end
+
+          unless value.is_a?(ActiveSupport::TimeWithZone)
+            if value =~ /\A\d+\z/
+              value = value.to_i
+            elsif value =~ /\A\d+\.\d+\z/
+              value = value.to_f
+            end
+          end
+        end
+        value = Blazer.transform_variable.call(var, value) if Blazer.transform_variable
+        @values[var] = value
+      end
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
+    end
+
+    def apply_cohort_analysis(period:, days:)
+      @statement = data_source.cohort_analysis_statement(statement, period: period, days: days).sub("{placeholder}") { statement }
+    end
+
+    # should probably transform before cohort analysis
+    # but keep previous order for now
+    def transformed_statement
+      statement = self.statement.dup
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement
+    end
+
+    def bind
+      @bind_statement, @bind_values = data_source.bind_params(transformed_statement, values)
+    end
+
+    def display_statement
+      data_source.sub_variables(transformed_statement, values)
+    end
+
+    def clear_cache
+      bind if bind_statement.nil?
+      data_source.clear_cache(self)
+    end
+  end
+end

data/lib/blazer/version.rb

@@ -0,0 +1,3 @@
+module Blazer
+  VERSION = "3.0.0"
+end

data/lib/blazer.rb

@@ -0,0 +1,286 @@
+# dependencies
+require "chartkick"
+require "safely/core"
+
+# stdlib
+require "csv"
+require "digest/sha2"
+require "json"
+require "yaml"
+
+# modules
+require_relative "blazer/version"
+require_relative "blazer/data_source"
+require_relative "blazer/result"
+require_relative "blazer/result_cache"
+require_relative "blazer/run_statement"
+require_relative "blazer/statement"
+require_relative "blazer/annotations"
+
+# adapters
+require_relative "blazer/adapters/base_adapter"
+require_relative "blazer/adapters/athena_adapter"
+require_relative "blazer/adapters/bigquery_adapter"
+require_relative "blazer/adapters/cassandra_adapter"
+require_relative "blazer/adapters/clickhouse_adapter"
+require_relative "blazer/adapters/drill_adapter"
+require_relative "blazer/adapters/druid_adapter"
+require_relative "blazer/adapters/elasticsearch_adapter"
+require_relative "blazer/adapters/hive_adapter"
+require_relative "blazer/adapters/ignite_adapter"
+require_relative "blazer/adapters/influxdb_adapter"
+require_relative "blazer/adapters/neo4j_adapter"
+require_relative "blazer/adapters/opensearch_adapter"
+require_relative "blazer/adapters/presto_adapter"
+require_relative "blazer/adapters/salesforce_adapter"
+require_relative "blazer/adapters/soda_adapter"
+require_relative "blazer/adapters/spark_adapter"
+require_relative "blazer/adapters/sql_adapter"
+require_relative "blazer/adapters/snowflake_adapter"
+
+# engine
+require_relative "blazer/engine"
+
+module Blazer
+  class Error < StandardError; end
+  class UploadError < Error; end
+  class TimeoutNotSupported < Error; end
+
+  # actionmailer optional
+  autoload :CheckMailer, "blazer/check_mailer"
+  # net/http optional
+  autoload :SlackNotifier, "blazer/slack_notifier"
+  # activejob optional
+  autoload :RunStatementJob, "blazer/run_statement_job"
+
+  class << self
+    attr_accessor :audit
+    attr_reader :time_zone
+    attr_accessor :user_name
+    attr_writer :user_class
+    attr_writer :user_method
+    attr_accessor :before_action
+    attr_accessor :from_email
+    attr_accessor :cache
+    attr_accessor :transform_statement
+    attr_accessor :transform_variable
+    attr_accessor :check_schedules
+    attr_accessor :anomaly_checks
+    attr_accessor :forecasting
+    attr_accessor :async
+    attr_accessor :images
+    attr_accessor :annotations
+    attr_accessor :override_csp
+    attr_accessor :slack_oauth_token
+    attr_accessor :slack_webhook_url
+    attr_accessor :mapbox_access_token
+  end
+  self.audit = true
+  self.user_name = :name
+  self.check_schedules = ["5 minutes", "1 hour", "1 day"]
+  self.anomaly_checks = false
+  self.forecasting = false
+  self.async = false
+  self.images = false
+  self.override_csp = false
+  self.annotations = Blazer::Annotations
+
+  VARIABLE_MESSAGE = "Variable cannot be used in this position"
+  TIMEOUT_MESSAGE = "Query timed out :("
+  TIMEOUT_ERRORS = [
+    "canceling statement due to statement timeout", # postgres
+    "canceling statement due to conflict with recovery", # postgres
+    "cancelled on user's request", # redshift
+    "canceled on user's request", # redshift
+    "system requested abort", # redshift
+    "maximum statement execution time exceeded" # mysql
+  ]
+
+  def self.time_zone=(time_zone)
+    @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+  end
+
+  def self.user_class
+    if !defined?(@user_class)
+      @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil)
+    end
+    @user_class
+  end
+
+  def self.user_method
+    if !defined?(@user_method)
+      @user_method = settings["user_method"]
+      if user_class
+        @user_method ||= "current_#{user_class.to_s.downcase.singularize}"
+      end
+    end
+    @user_method
+  end
+
+  def self.settings
+    @settings ||= begin
+      path = Rails.root.join("config", "blazer.yml").to_s
+      if File.exist?(path)
+        YAML.safe_load(ERB.new(File.read(path)).result, aliases: true)
+      else
+        {}
+      end
+    end
+  end
+
+  def self.data_sources
+    @data_sources ||= begin
+      ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" }
+      settings["data_sources"].each do |id, s|
+        ds[id] = Blazer::DataSource.new(id, s)
+      end
+      ds
+    end
+  end
+
+  def self.run_checks(schedule: nil)
+    checks = Blazer::Check.includes(:query)
+    checks = checks.where(schedule: schedule) if schedule
+    checks.find_each do |check|
+      next if check.state == "disabled"
+      Safely.safely { run_check(check) }
+    end
+  end
+
+  def self.run_check(check)
+    tries = 1
+
+    ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
+      # try 3 times on timeout errors
+      statement = check.query.statement_object
+      data_source = statement.data_source
+
+      while tries <= 3
+        result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)
+        if result.timed_out?
+          Rails.logger.info "[blazer timeout] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        elsif result.error.to_s.start_with?("PG::ConnectionBad")
+          data_source.reconnect
+          Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
+          tries += 1
+          sleep(10)
+        else
+          break
+        end
+      end
+
+      begin
+        check.reload # in case state has changed since job started
+        check.update_state(result)
+      rescue ActiveRecord::RecordNotFound
+        # check deleted
+      end
+
+      # TODO use proper logfmt
+      Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"
+
+      # should be no variables
+      instrument[:statement] = statement.bind_statement
+      instrument[:data_source] = data_source
+      instrument[:state] = check.state
+      instrument[:rows] = result.rows.try(:size)
+      instrument[:error] = result.error
+      instrument[:tries] = tries
+    end
+  end
+
+  def self.send_failing_checks
+    emails = {}
+    slack_channels = {}
+
+    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
+      check.split_emails.each do |email|
+        (emails[email] ||= []) << check
+      end
+      check.split_slack_channels.each do |channel|
+        (slack_channels[channel] ||= []) << check
+      end
+    end
+
+    emails.each do |email, checks|
+      Safely.safely do
+        Blazer::CheckMailer.failing_checks(email, checks).deliver_now
+      end
+    end
+
+    slack_channels.each do |channel, checks|
+      Safely.safely do
+        Blazer::SlackNotifier.failing_checks(channel, checks)
+      end
+    end
+  end
+
+  def self.slack?
+    slack_oauth_token.present? || slack_webhook_url.present?
+  end
+
+  # TODO show warning on invalid access token
+  def self.maps?
+    mapbox_access_token.present? && mapbox_access_token.start_with?("pk.")
+  end
+
+  def self.uploads?
+    settings.key?("uploads")
+  end
+
+  def self.uploads_connection
+    raise "Empty url for uploads" unless settings.dig("uploads", "url")
+    Blazer::UploadsConnection.connection
+  end
+
+  def self.uploads_schema
+    settings.dig("uploads", "schema") || "uploads"
+  end
+
+  def self.uploads_table_name(name)
+    uploads_connection.quote_table_name("#{uploads_schema}.#{name}")
+  end
+
+  def self.adapters
+    @adapters ||= {}
+  end
+
+  def self.register_adapter(name, adapter)
+    adapters[name] = adapter
+  end
+
+  def self.anomaly_detectors
+    @anomaly_detectors ||= {}
+  end
+
+  def self.register_anomaly_detector(name, &anomaly_detector)
+    anomaly_detectors[name] = anomaly_detector
+  end
+
+  def self.forecasters
+    @forecasters ||= {}
+  end
+
+  def self.register_forecaster(name, &forecaster)
+    forecasters[name] = forecaster
+  end
+
+  def self.archive_queries
+    raise "Audits must be enabled to archive" unless Blazer.audit
+    raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
+
+    viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact
+    Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived")
+  end
+
+  # private
+  def self.monotonic_time
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end
+
+require_relative "blazer/adapters"
+require_relative "blazer/anomaly_detectors"
+require_relative "blazer/forecasters"

data/lib/finery.rb

@@ -0,0 +1,3 @@
+require "blazer"
+
+Finery = Blazer

data/lib/generators/blazer/install_generator.rb

@@ -0,0 +1,22 @@
+require "rails/generators/active_record"
+
+module Blazer
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "install.rb", "db/migrate/install_blazer.rb", migration_version: migration_version
+      end
+
+      def copy_config
+        template "config.yml", "config/finery.yml"
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/blazer/templates/config.yml.tt

@@ -0,0 +1,83 @@
+# see https://github.com/finery-bi/finery for more info
+
+data_sources:
+  main:
+    url: <%%= ENV["BLAZER_DATABASE_URL"] %>
+
+    # statement timeout, in seconds
+    # none by default
+    # timeout: 15
+
+    # caching settings
+    # can greatly improve speed
+    # off by default
+    # cache:
+    #   mode: slow # or all
+    #   expires_in: 60 # min
+    #   slow_threshold: 15 # sec, only used in slow mode
+
+    # wrap queries in a transaction for safety
+    # not necessary if you use a read-only user
+    # true by default
+    # use_transaction: false
+
+    smart_variables:
+      # zone_id: "SELECT id, name FROM zones ORDER BY name ASC"
+      # period: ["day", "week", "month"]
+      # status: {0: "Active", 1: "Archived"}
+
+    linked_columns:
+      # user_id: "/admin/users/{value}"
+
+    smart_columns:
+      # user_id: "SELECT id, name FROM users WHERE id IN {value}"
+
+    annotations:
+      # holiday_boxes: SELECT min_date, max_date, label FROM holidays WHERE (min_date, max_date) OVERLAPS ({min_date}, {max_date})
+      # deployment_lines: SELECT date, label FROM deployments WHERE date BETWEEN {min_date} and {max_date}
+
+# create audits
+audit: true
+
+# change the time zone
+# time_zone: "Pacific Time (US & Canada)"
+
+# class name of the user model
+# user_class: User
+
+# method name for the current user
+# user_method: current_user
+
+# method name for the display name
+# user_name: name
+
+# custom before_action to use for auth
+# before_action_method: require_admin
+
+# email to send checks from
+# from_email: finery@example.org
+
+# webhook for Slack
+# slack_webhook_url: <%%= ENV["BLAZER_SLACK_WEBHOOK_URL"] %>
+
+check_schedules:
+  - "1 day"
+  - "1 hour"
+  - "5 minutes"
+
+# enable anomaly detection
+# note: with trend, time series are sent to https://trendapi.org
+# anomaly_checks: prophet / trend / anomaly_detection
+
+# enable forecasting
+# note: with trend, time series are sent to https://trendapi.org
+# forecasting: prophet / trend
+
+# enable map
+# mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>
+
+# enable uploads
+# uploads:
+#   url: <%%= ENV["BLAZER_UPLOADS_URL"] %>
+#   schema: uploads
+#   data_source: main

data/lib/generators/blazer/templates/install.rb.tt

@@ -0,0 +1,47 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :blazer_queries do |t|
+      t.references :creator
+      t.string :name
+      t.text :description
+      t.text :statement
+      t.string :data_source
+      t.string :status
+      t.timestamps null: false
+    end
+
+    create_table :blazer_audits do |t|
+      t.references :user
+      t.references :query
+      t.text :statement
+      t.string :data_source
+      t.datetime :created_at
+    end
+
+    create_table :blazer_dashboards do |t|
+      t.references :creator
+      t.string :name
+      t.timestamps null: false
+    end
+
+    create_table :blazer_dashboard_queries do |t|
+      t.references :dashboard
+      t.references :query
+      t.integer :position
+      t.timestamps null: false
+    end
+
+    create_table :blazer_checks do |t|
+      t.references :creator
+      t.references :query
+      t.string :state
+      t.string :schedule
+      t.text :emails
+      t.text :slack_channels
+      t.string :check_type
+      t.text :message
+      t.datetime :last_run_at
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/blazer/templates/uploads.rb.tt

@@ -0,0 +1,10 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :blazer_uploads do |t|
+      t.references :creator
+      t.string :table
+      t.text :description
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/blazer/uploads_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Blazer
+  module Generators
+    class UploadsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "uploads.rb", "db/migrate/create_blazer_uploads.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/tasks/blazer.rake

@@ -0,0 +1,20 @@
+namespace :blazer do
+  desc "run checks"
+  task :run_checks, [:schedule] => :environment do |_, args|
+    Blazer.run_checks(schedule: args[:schedule] || ENV["SCHEDULE"])
+  end
+
+  desc "send failing checks"
+  task send_failing_checks: :environment do
+    Blazer.send_failing_checks
+  end
+
+  desc "archive queries"
+  task archive_queries: :environment do
+    begin
+      Blazer.archive_queries
+    rescue => e
+      abort e.message
+    end
+  end
+end

data/lib/tasks/finery.rake

@@ -0,0 +1,20 @@
+namespace :finery do
+  desc "run checks"
+  task :run_checks, [:schedule] => :environment do |_, args|
+    Finery.run_checks(schedule: args[:schedule] || ENV["SCHEDULE"])
+  end
+
+  desc "send failing checks"
+  task send_failing_checks: :environment do
+    Finery.send_failing_checks
+  end
+
+  desc "archive queries"
+  task archive_queries: :environment do
+    begin
+      Finery.archive_queries
+    rescue => e
+      abort e.message
+    end
+  end
+end