blazer 2.6.5 → 3.0.0

data/lib/blazer/result.rb

@@ -1,6 +1,7 @@
 module Blazer
   class Result
-    attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached, :forecast_error
+    attr_reader :data_source, :columns, :rows, :error, :forecast_error
+    attr_accessor :cached_at, :just_cached
 
     def initialize(data_source, columns, rows, error, cached_at, just_cached)
       @data_source = data_source
@@ -19,9 +20,9 @@ module Blazer
       cached_at.present?
     end
 
-    def boom
-      @boom ||= begin
-        boom = {}
+    def smart_values
+      @smart_values ||= begin
+        smart_values = {}
         columns.each_with_index do |key, i|
           smart_columns_data_source =
             ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_columns[key] }
@@ -37,10 +38,10 @@ module Blazer
                 result.rows
               end
 
-            boom[key] = Hash[res.map { |k, v| [k.nil? ? k : k.to_s, v] }]
+            smart_values[key] = res.to_h { |k, v| [k.nil? ? k : k.to_s, v] }
           end
         end
-        boom
+        smart_values
       end
     end
 
@@ -48,7 +49,7 @@ module Blazer
       @column_types ||= begin
         columns.each_with_index.map do |k, i|
           v = (rows.find { |r| r[i] } || {})[i]
-          if boom[k]
+          if smart_values[k]
             "string"
           elsif v.is_a?(Numeric)
             "numeric"
@@ -93,14 +94,8 @@ module Blazer
     def forecast
       count = (@rows.size * 0.25).round.clamp(30, 365)
 
-      case Blazer.forecasting
-      when "prophet"
-        require "prophet"
-        forecast = Prophet.forecast(@rows.to_h, count: count)
-      else
-        require "trend"
-        forecast = Trend.forecast(@rows.to_h, count: count)
-      end
+      forecaster = Blazer.forecasters.fetch(Blazer.forecasting)
+      forecast = forecaster.call(@rows.to_h, count: count)
 
       # round integers
       if @rows[0][1].is_a?(Integer)
@@ -139,7 +134,7 @@ module Blazer
               series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
             end
           else
-            rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
+            rows.group_by { |r| v = r[1]; (smart_values[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
               series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
             end
           end
@@ -176,61 +171,8 @@ module Blazer
     def anomaly?(series)
       series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
 
-      case Blazer.anomaly_checks
-      when "prophet"
-        df = Rover::DataFrame.new(series[0..-2].map { |v| {"ds" => v[0], "y" => v[1]} })
-        m = Prophet.new(interval_width: 0.99)
-        m.logger.level = ::Logger::FATAL # no logging
-        m.fit(df)
-        future = Rover::DataFrame.new(series[-1..-1].map { |v| {"ds" => v[0]} })
-        forecast = m.predict(future).to_a[0]
-        lower = forecast["yhat_lower"]
-        upper = forecast["yhat_upper"]
-        value = series.last[1]
-        value < lower || value > upper
-      when "trend"
-        anomalies = Trend.anomalies(Hash[series])
-        anomalies.include?(series.last[0])
-      when "anomaly_detection"
-        period = 7 # TODO determine period
-        anomalies = AnomalyDetection.detect(Hash[series], period: period)
-        anomalies.include?(series.last[0])
-      else
-        csv_str =
-          CSV.generate do |csv|
-            csv << ["timestamp", "count"]
-            series.each do |row|
-              csv << row
-            end
-          end
-
-        r_script = %x[which Rscript].chomp
-        type = series.any? && series.last.first.to_time - series.first.first.to_time >= 2.weeks ? "ts" : "vec"
-        args = [type, csv_str]
-        raise "R not found" if r_script.empty?
-        command = "#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{args.map { |a| Shellwords.escape(a) }.join(" ")}"
-        output = %x[#{command}]
-        if output.empty?
-          raise "Unknown R error"
-        end
-
-        rows = CSV.parse(output, headers: true)
-        error = rows.first && rows.first["x"]
-        raise error if error
-
-        timestamps = []
-        if type == "ts"
-          rows.each do |row|
-            timestamps << Time.parse(row["timestamp"])
-          end
-          timestamps.include?(series.last[0].to_time)
-        else
-          rows.each do |row|
-            timestamps << row["index"].to_i
-          end
-          timestamps.include?(series.length)
-        end
-      end
+      anomaly_detector = Blazer.anomaly_detectors.fetch(Blazer.anomaly_checks)
+      anomaly_detector.call(series)
     end
   end
 end

data/lib/blazer/result_cache.rb

@@ -0,0 +1,71 @@
+module Blazer
+  class ResultCache
+    def initialize(data_source)
+      @data_source = data_source
+    end
+
+    def write_run(run_id, result)
+      write(run_cache_key(run_id), result, expires_in: 30.seconds)
+    end
+
+    def read_run(run_id)
+      read(run_cache_key(run_id))
+    end
+
+    def delete_run(run_id)
+      delete(run_cache_key(run_id))
+    end
+
+    def write_statement(statement, result, expires_in:)
+      write(statement_cache_key(statement), result, expires_in: expires_in) if caching?
+    end
+
+    def read_statement(statement)
+      read(statement_cache_key(statement)) if caching?
+    end
+
+    def delete_statement(statement)
+      delete(statement_cache_key(statement)) if caching?
+    end
+
+    private
+
+    def write(key, result, expires_in:)
+      raise ArgumentError, "expected Blazer::Result" unless result.is_a?(Blazer::Result)
+      value = [result.columns, result.rows, result.error, result.cached_at, result.just_cached]
+      cache.write(key, value, expires_in: expires_in)
+    end
+
+    def read(key)
+      value = cache.read(key)
+      if value
+        columns, rows, error, cached_at, just_cached = value
+        Blazer::Result.new(@data_source, columns, rows, error, cached_at, just_cached)
+      end
+    end
+
+    def delete(key)
+      cache.delete(key)
+    end
+
+    def caching?
+      @data_source.cache_mode != "off"
+    end
+
+    def cache_key(key)
+      (["blazer", "v5", @data_source.id] + key).join("/")
+    end
+
+    def statement_cache_key(statement)
+      cache_key(["statement", Digest::SHA256.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
+    end
+
+    def run_cache_key(run_id)
+      cache_key(["run", run_id])
+    end
+
+    def cache
+      Blazer.cache
+    end
+  end
+end

data/lib/blazer/run_statement.rb

@@ -32,7 +32,7 @@ module Blazer
         audit.save! if audit.changed?
       end
 
-      if query && !result.timed_out? && !query.variables.any?
+      if query && !result.timed_out? && !result.cached? && !query.variables.any?
         query.checks.each do |check|
           check.update_state(result)
         end

data/lib/blazer/run_statement_job.rb

@@ -11,8 +11,8 @@ module Blazer
           Blazer::RunStatement.new.perform(statement, options)
         end
       rescue Exception => e
-        Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
-        Blazer.cache.write(data_source.run_cache_key(options[:run_id]), Marshal.dump([[], [], "Unknown error", nil]), expires_in: 30.seconds)
+        result = Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
+        data_source.result_cache.write_run(options[:run_id], result)
         raise e
       end
     end

data/lib/blazer/statement.rb

@@ -10,7 +10,9 @@ module Blazer
     end
 
     def variables
-      @variables ||= Blazer.extract_vars(statement)
+      # strip commented out lines
+      # and regex {1} or {1,2}
+      @variables ||= statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
     end
 
     def add_values(var_params)

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.6.5"
+  VERSION = "3.0.0"
 end

data/lib/blazer.rb

@@ -4,39 +4,40 @@ require "safely/core"
 
 # stdlib
 require "csv"
+require "digest/sha2"
 require "json"
 require "yaml"
 
 # modules
-require "blazer/version"
-require "blazer/data_source"
-require "blazer/result"
-require "blazer/run_statement"
-require "blazer/statement"
+require_relative "blazer/version"
+require_relative "blazer/data_source"
+require_relative "blazer/result"
+require_relative "blazer/result_cache"
+require_relative "blazer/run_statement"
+require_relative "blazer/statement"
 
 # adapters
-require "blazer/adapters/base_adapter"
-require "blazer/adapters/athena_adapter"
-require "blazer/adapters/bigquery_adapter"
-require "blazer/adapters/cassandra_adapter"
-require "blazer/adapters/drill_adapter"
-require "blazer/adapters/druid_adapter"
-require "blazer/adapters/elasticsearch_adapter"
-require "blazer/adapters/hive_adapter"
-require "blazer/adapters/ignite_adapter"
-require "blazer/adapters/influxdb_adapter"
-require "blazer/adapters/mongodb_adapter"
-require "blazer/adapters/neo4j_adapter"
-require "blazer/adapters/opensearch_adapter"
-require "blazer/adapters/presto_adapter"
-require "blazer/adapters/salesforce_adapter"
-require "blazer/adapters/soda_adapter"
-require "blazer/adapters/spark_adapter"
-require "blazer/adapters/sql_adapter"
-require "blazer/adapters/snowflake_adapter"
+require_relative "blazer/adapters/base_adapter"
+require_relative "blazer/adapters/athena_adapter"
+require_relative "blazer/adapters/bigquery_adapter"
+require_relative "blazer/adapters/cassandra_adapter"
+require_relative "blazer/adapters/drill_adapter"
+require_relative "blazer/adapters/druid_adapter"
+require_relative "blazer/adapters/elasticsearch_adapter"
+require_relative "blazer/adapters/hive_adapter"
+require_relative "blazer/adapters/ignite_adapter"
+require_relative "blazer/adapters/influxdb_adapter"
+require_relative "blazer/adapters/neo4j_adapter"
+require_relative "blazer/adapters/opensearch_adapter"
+require_relative "blazer/adapters/presto_adapter"
+require_relative "blazer/adapters/salesforce_adapter"
+require_relative "blazer/adapters/soda_adapter"
+require_relative "blazer/adapters/spark_adapter"
+require_relative "blazer/adapters/sql_adapter"
+require_relative "blazer/adapters/snowflake_adapter"
 
 # engine
-require "blazer/engine"
+require_relative "blazer/engine"
 
 module Blazer
   class Error < StandardError; end
@@ -66,8 +67,6 @@ module Blazer
     attr_accessor :forecasting
     attr_accessor :async
     attr_accessor :images
-    attr_accessor :query_viewable
-    attr_accessor :query_editable
     attr_accessor :override_csp
     attr_accessor :slack_oauth_token
     attr_accessor :slack_webhook_url
@@ -118,7 +117,7 @@ module Blazer
     @settings ||= begin
       path = Rails.root.join("config", "blazer.yml").to_s
       if File.exist?(path)
-        YAML.load(ERB.new(File.read(path)).result)
+        YAML.safe_load(ERB.new(File.read(path)).result, aliases: true)
       else
         {}
       end
@@ -135,13 +134,6 @@ module Blazer
     end
   end
 
-  # TODO move to Statement and remove in 3.0.0
-  def self.extract_vars(statement)
-    # strip commented out lines
-    # and regex {1} or {1,2}
-    statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
-  end
-
   def self.run_checks(schedule: nil)
     checks = Blazer::Check.includes(:query)
     checks = checks.where(schedule: schedule) if schedule
@@ -225,6 +217,11 @@ module Blazer
     slack_oauth_token.present? || slack_webhook_url.present?
   end
 
+  # TODO show warning on invalid access token
+  def self.maps?
+    mapbox_access_token.present? && mapbox_access_token.start_with?("pk.")
+  end
+
   def self.uploads?
     settings.key?("uploads")
   end
@@ -250,6 +247,22 @@ module Blazer
     adapters[name] = adapter
   end
 
+  def self.anomaly_detectors
+    @anomaly_detectors ||= {}
+  end
+
+  def self.register_anomaly_detector(name, &anomaly_detector)
+    anomaly_detectors[name] = anomaly_detector
+  end
+
+  def self.forecasters
+    @forecasters ||= {}
+  end
+
+  def self.register_forecaster(name, &forecaster)
+    forecasters[name] = forecaster
+  end
+
   def self.archive_queries
     raise "Audits must be enabled to archive" unless Blazer.audit
     raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
@@ -264,21 +277,6 @@ module Blazer
   end
 end
 
-Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
-Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
-Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
-Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
-Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
-Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
-Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
-Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapter
-Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
-Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
-Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
-Blazer.register_adapter "opensearch", Blazer::Adapters::OpensearchAdapter
-Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
-Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
-Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
-Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
-Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
-Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter
+require_relative "blazer/adapters"
+require_relative "blazer/anomaly_detectors"
+require_relative "blazer/forecasters"

data/licenses/LICENSE-chart.js.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Chart.js Contributors
+Copyright (c) 2014-2022 Chart.js Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

data/licenses/LICENSE-chartjs-adapter-date-fns.txt

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Chart.js Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/licenses/LICENSE-chartkick.js.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013-2019 Andrew Kane
+Copyright (c) 2013-2023 Andrew Kane
 
 MIT License
 

data/licenses/LICENSE-date-fns.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Sasha Koss and Lesha Koss https://kossnocorp.mit-license.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/licenses/LICENSE-kurkle-color.txt

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) 2018-2021 Jukka Kurkela
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.