blazer_xlsx 3.0.5

data/lib/blazer/engine.rb

@@ -0,0 +1,42 @@
+module Blazer
+  class Engine < ::Rails::Engine
+    isolate_namespace Blazer
+
+    initializer "blazer" do |app|
+      if defined?(Sprockets) && Sprockets::VERSION.to_i >= 4
+        app.config.assets.precompile += [
+          "blazer/application.js",
+          "blazer/application.css",
+          "blazer/glyphicons-halflings-regular.eot",
+          "blazer/glyphicons-halflings-regular.svg",
+          "blazer/glyphicons-halflings-regular.ttf",
+          "blazer/glyphicons-halflings-regular.woff",
+          "blazer/glyphicons-halflings-regular.woff2",
+          "blazer/favicon.png"
+        ]
+      else
+        # use a proc instead of a string
+        app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
+        app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff|woff2)\z/ }
+        app.config.assets.precompile << proc { |path| path == "blazer/favicon.png" }
+      end
+
+      Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
+      Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
+      Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
+      Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
+      Blazer.before_action = Blazer.settings["before_action_method"] if Blazer.settings["before_action_method"]
+      Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
+      Blazer.cache ||= Rails.cache
+
+      Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
+      Blazer.forecasting = Blazer.settings["forecasting"] || false
+      Blazer.async = Blazer.settings["async"] || false
+      Blazer.images = Blazer.settings["images"] || false
+      Blazer.override_csp = Blazer.settings["override_csp"] || false
+      Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]
+      Blazer.slack_webhook_url = Blazer.settings["slack_webhook_url"] || ENV["BLAZER_SLACK_WEBHOOK_URL"]
+      Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"] || ENV["MAPBOX_ACCESS_TOKEN"]
+    end
+  end
+end

data/lib/blazer/forecasters.rb

@@ -0,0 +1,7 @@
+Blazer.register_forecaster "prophet" do |series, count:|
+  Prophet.forecast(series, count: count)
+end
+
+Blazer.register_forecaster "trend" do |series, count:|
+  Trend.forecast(series, count: count)
+end

data/lib/blazer/result.rb

@@ -0,0 +1,178 @@
+module Blazer
+  class Result
+    attr_reader :data_source, :columns, :rows, :error, :forecast_error
+    attr_accessor :cached_at, :just_cached
+
+    def initialize(data_source, columns, rows, error, cached_at, just_cached)
+      @data_source = data_source
+      @columns = columns
+      @rows = rows
+      @error = error
+      @cached_at = cached_at
+      @just_cached = just_cached
+    end
+
+    def timed_out?
+      error == Blazer::TIMEOUT_MESSAGE
+    end
+
+    def cached?
+      cached_at.present?
+    end
+
+    def smart_values
+      @smart_values ||= begin
+        smart_values = {}
+        columns.each_with_index do |key, i|
+          smart_columns_data_source =
+            ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_columns[key] }
+
+          if smart_columns_data_source
+            query = smart_columns_data_source.smart_columns[key]
+            res =
+              if query.is_a?(Hash)
+                query
+              else
+                values = rows.map { |r| r[i] }.compact.uniq
+                result = smart_columns_data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+                result.rows
+              end
+
+            smart_values[key] = res.to_h { |k, v| [k.nil? ? k : k.to_s, v] }
+          end
+        end
+        smart_values
+      end
+    end
+
+    def column_types
+      @column_types ||= begin
+        columns.each_with_index.map do |k, i|
+          v = (rows.find { |r| r[i] } || {})[i]
+          if smart_values[k]
+            "string"
+          elsif v.is_a?(Numeric)
+            "numeric"
+          elsif v.is_a?(Time) || v.is_a?(Date)
+            "time"
+          elsif v.nil?
+            nil
+          elsif v.is_a?(String) && v.encoding == Encoding::BINARY
+            "binary"
+          else
+            "string"
+          end
+        end
+      end
+    end
+
+    def chart_type
+      @chart_type ||= begin
+        if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "line"
+        elsif column_types == ["time", "string", "numeric"]
+          "line2"
+        elsif column_types == ["string", "numeric"] && @columns.last == "pie"
+          "pie"
+        elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "bar"
+        elsif column_types == ["string", "string", "numeric"]
+          "bar2"
+        elsif column_types == ["numeric", "numeric"]
+          "scatter"
+        end
+      end
+    end
+
+    def forecastable?
+      @forecastable ||= Blazer.forecasting && column_types == ["time", "numeric"] && @rows.size >= 10
+    end
+
+    # TODO cache it?
+    # don't want to put result data (even hashed version)
+    # into cache without developer opt-in
+    def forecast
+      count = (@rows.size * 0.25).round.clamp(30, 365)
+
+      forecaster = Blazer.forecasters.fetch(Blazer.forecasting)
+      forecast = forecaster.call(@rows.to_h, count: count)
+
+      # round integers
+      if @rows[0][1].is_a?(Integer)
+        forecast = forecast.map { |k, v| [k, v.round] }.to_h
+      end
+
+      @rows.each do |row|
+        row[2] = nil
+      end
+      @rows.unshift(*forecast.map { |k, v| [k, nil, v] })
+      @columns << "forecast"
+
+      # reset cache
+      @column_types = nil
+      @chart_type = nil
+
+      forecast
+    rescue => e
+      @forecast_error = String.new("Error generating forecast")
+      @forecast_error << ": #{e.message.sub("Invalid parameter: ", "")}"
+      nil
+    end
+
+    def detect_anomaly
+      anomaly = nil
+      message = nil
+
+      if rows.empty?
+        message = "No data"
+      else
+        if chart_type == "line" || chart_type == "line2"
+          series = []
+
+          if chart_type == "line"
+            columns[1..-1].each_with_index.each do |k, i|
+              series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
+            end
+          else
+            rows.group_by { |r| v = r[1]; (smart_values[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
+              series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
+            end
+          end
+
+          current_series = nil
+          begin
+            anomalies = []
+            series.each do |s|
+              current_series = s[:name]
+              anomalies << s[:name] if anomaly?(s[:data])
+            end
+            anomaly = anomalies.any?
+            if anomaly
+              if anomalies.size == 1
+                message = "Anomaly detected in #{anomalies.first}"
+              else
+                message = "Anomalies detected in #{anomalies.to_sentence}"
+              end
+            else
+              message = "No anomalies detected"
+            end
+          rescue => e
+            message = "#{current_series}: #{e.message}"
+            raise e if Rails.env.development?
+          end
+        else
+          message = "Bad format"
+        end
+      end
+
+      [anomaly, message]
+    end
+
+    def anomaly?(series)
+      series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
+
+      anomaly_detector = Blazer.anomaly_detectors.fetch(Blazer.anomaly_checks)
+      anomaly_detector.call(series)
+    end
+  end
+end

data/lib/blazer/result_cache.rb

@@ -0,0 +1,71 @@
+module Blazer
+  class ResultCache
+    def initialize(data_source)
+      @data_source = data_source
+    end
+
+    def write_run(run_id, result)
+      write(run_cache_key(run_id), result, expires_in: 30.seconds)
+    end
+
+    def read_run(run_id)
+      read(run_cache_key(run_id))
+    end
+
+    def delete_run(run_id)
+      delete(run_cache_key(run_id))
+    end
+
+    def write_statement(statement, result, expires_in:)
+      write(statement_cache_key(statement), result, expires_in: expires_in) if caching?
+    end
+
+    def read_statement(statement)
+      read(statement_cache_key(statement)) if caching?
+    end
+
+    def delete_statement(statement)
+      delete(statement_cache_key(statement)) if caching?
+    end
+
+    private
+
+    def write(key, result, expires_in:)
+      raise ArgumentError, "expected Blazer::Result" unless result.is_a?(Blazer::Result)
+      value = [result.columns, result.rows, result.error, result.cached_at, result.just_cached]
+      cache.write(key, value, expires_in: expires_in)
+    end
+
+    def read(key)
+      value = cache.read(key)
+      if value
+        columns, rows, error, cached_at, just_cached = value
+        Blazer::Result.new(@data_source, columns, rows, error, cached_at, just_cached)
+      end
+    end
+
+    def delete(key)
+      cache.delete(key)
+    end
+
+    def caching?
+      @data_source.cache_mode != "off"
+    end
+
+    def cache_key(key)
+      (["blazer", "v5", @data_source.id] + key).join("/")
+    end
+
+    def statement_cache_key(statement)
+      cache_key(["statement", Digest::SHA256.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
+    end
+
+    def run_cache_key(run_id)
+      cache_key(["run", run_id])
+    end
+
+    def cache
+      Blazer.cache
+    end
+  end
+end

data/lib/blazer/run_statement.rb

@@ -0,0 +1,45 @@
+module Blazer
+  class RunStatement
+    def perform(statement, options = {})
+      query = options[:query]
+
+      data_source = statement.data_source
+      statement.bind
+
+      # audit
+      if Blazer.audit
+        audit_statement = statement.bind_statement
+        audit_statement += "\n\n#{statement.bind_values.to_json}" if statement.bind_values.any?
+        audit = Blazer::Audit.new(statement: audit_statement)
+        audit.query = query
+        audit.data_source = data_source.id
+        # only set user if present to avoid error with Rails 7.1 when no user model
+        audit.user = options[:user] unless options[:user].nil?
+        audit.save!
+      end
+
+      start_time = Blazer.monotonic_time
+      result = data_source.run_statement(statement, options)
+      duration = Blazer.monotonic_time - start_time
+
+      if Blazer.audit
+        audit.duration = duration if audit.respond_to?(:duration=)
+        audit.error = result.error if audit.respond_to?(:error=)
+        audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
+        audit.cached = result.cached? if audit.respond_to?(:cached=)
+        if !result.cached? && duration >= 10
+          audit.cost = data_source.cost(statement) if audit.respond_to?(:cost=)
+        end
+        audit.save! if audit.changed?
+      end
+
+      if query && !result.timed_out? && !result.cached? && !query.variables.any?
+        query.checks.each do |check|
+          check.update_state(result)
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/blazer/run_statement_job.rb

@@ -0,0 +1,20 @@
+module Blazer
+  class RunStatementJob < ActiveJob::Base
+    self.queue_adapter = :async
+
+    def perform(data_source_id, statement, options)
+      statement = Blazer::Statement.new(statement, data_source_id)
+      statement.values = options.delete(:values)
+      data_source = statement.data_source
+      begin
+        ActiveRecord::Base.connection_pool.with_connection do
+          Blazer::RunStatement.new.perform(statement, options)
+        end
+      rescue Exception => e
+        result = Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
+        data_source.result_cache.write_run(options[:run_id], result)
+        raise e
+      end
+    end
+  end
+end

data/lib/blazer/slack_notifier.rb

@@ -0,0 +1,94 @@
+require "net/http"
+
+module Blazer
+  class SlackNotifier
+    def self.state_change(check, state, state_was, rows_count, error, check_type)
+      check.split_slack_channels.each do |channel|
+        text =
+          if error
+            error
+          elsif rows_count > 0 && check_type == "bad_data"
+            pluralize(rows_count, "row")
+          end
+
+        payload = {
+          channel: channel,
+          attachments: [
+            {
+              title: escape("Check #{state.titleize}: #{check.query.name}"),
+              title_link: query_url(check.query_id),
+              text: escape(text),
+              color: state == "passing" ? "good" : "danger"
+            }
+          ]
+        }
+
+        post(payload)
+      end
+    end
+
+    def self.failing_checks(channel, checks)
+      text =
+        checks.map do |check|
+          "<#{query_url(check.query_id)}|#{escape(check.query.name)}> #{escape(check.state)}"
+        end
+
+      payload = {
+        channel: channel,
+        attachments: [
+          {
+            title: escape("#{pluralize(checks.size, "Check")} Failing"),
+            text: text.join("\n"),
+            color: "warning"
+          }
+        ]
+      }
+
+      post(payload)
+    end
+
+    # https://api.slack.com/docs/message-formatting#how_to_escape_characters
+    # - Replace the ampersand, &, with &amp;
+    # - Replace the less-than sign, < with &lt;
+    # - Replace the greater-than sign, > with &gt;
+    # That's it. Don't HTML entity-encode the entire message.
+    def self.escape(str)
+      str.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;") if str
+    end
+
+    def self.pluralize(*args)
+      ActionController::Base.helpers.pluralize(*args)
+    end
+
+    # checks shouldn't have variables, but in any case,
+    # avoid passing variable params to url helpers
+    # (known unsafe parameters are removed, but still not ideal)
+    def self.query_url(id)
+      Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
+    end
+
+    # TODO use return value
+    def self.post(payload)
+      if Blazer.slack_webhook_url.present?
+        response = post_api(Blazer.slack_webhook_url, payload, {})
+        response.is_a?(Net::HTTPSuccess) && response.body == "ok"
+      else
+        headers = {
+          "Authorization" => "Bearer #{Blazer.slack_oauth_token}",
+          "Content-type" => "application/json"
+        }
+        response = post_api("https://slack.com/api/chat.postMessage", payload, headers)
+        response.is_a?(Net::HTTPSuccess) && (JSON.parse(response.body)["ok"] rescue false)
+      end
+    end
+
+    def self.post_api(url, payload, headers)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = 3
+      http.read_timeout = 5
+      http.post(uri.request_uri, payload.to_json, headers)
+    end
+  end
+end

data/lib/blazer/statement.rb

@@ -0,0 +1,77 @@
+module Blazer
+  class Statement
+    attr_reader :statement, :data_source, :bind_statement, :bind_values
+    attr_accessor :values
+
+    def initialize(statement, data_source = nil)
+      @statement = statement
+      @data_source = data_source.is_a?(String) ? Blazer.data_sources[data_source] : data_source
+      @values = {}
+    end
+
+    def variables
+      # strip commented out lines
+      # and regex {1} or {1,2}
+      @variables ||= statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+    end
+
+    def add_values(var_params)
+      variables.each do |var|
+        value = var_params[var].presence
+        value = nil unless value.is_a?(String) # ignore arrays and hashes
+        if value
+          if ["start_time", "end_time"].include?(var)
+            value = value.to_s.gsub(" ", "+") # fix for Quip bug
+          end
+
+          if var.end_with?("_at")
+            begin
+              value = Blazer.time_zone.parse(value)
+            rescue
+              # do nothing
+            end
+          end
+
+          unless value.is_a?(ActiveSupport::TimeWithZone)
+            if value =~ /\A\d+\z/
+              value = value.to_i
+            elsif value =~ /\A\d+\.\d+\z/
+              value = value.to_f
+            end
+          end
+        end
+        value = Blazer.transform_variable.call(var, value) if Blazer.transform_variable
+        @values[var] = value
+      end
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
+    end
+
+    def apply_cohort_analysis(period:, days:)
+      @statement = data_source.cohort_analysis_statement(statement, period: period, days: days).sub("{placeholder}") { statement }
+    end
+
+    # should probably transform before cohort analysis
+    # but keep previous order for now
+    def transformed_statement
+      statement = self.statement.dup
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+      statement
+    end
+
+    def bind
+      @bind_statement, @bind_values = data_source.bind_params(transformed_statement, values)
+    end
+
+    def display_statement
+      data_source.sub_variables(transformed_statement, values)
+    end
+
+    def clear_cache
+      bind if bind_statement.nil?
+      data_source.clear_cache(self)
+    end
+  end
+end

data/lib/blazer/version.rb

@@ -0,0 +1,3 @@
+module Blazer
+  VERSION = "3.0.5"
+end