sql-jarvis 1.8.0

data/lib/blazer/data_source.rb

@@ -0,0 +1,193 @@
+require "digest/md5"
+
+module Blazer
+  class DataSource
+    extend Forwardable
+
+    attr_reader :id, :settings, :adapter, :adapter_instance
+
+    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
+
+    def initialize(id, settings)
+      @id = id
+      @settings = settings
+
+      unless settings["url"] || Rails.env.development? || ["bigquery", "athena"].include?(settings["adapter"])
+        raise Blazer::Error, "Empty url for data source: #{id}"
+      end
+
+      @adapter_instance =
+        if Blazer.adapters[adapter]
+          Blazer.adapters[adapter].new(self)
+        else
+          raise Blazer::Error, "Unknown adapter"
+        end
+    end
+
+    def adapter
+      settings["adapter"] || detect_adapter
+    end
+
+    def name
+      settings["name"] || @id
+    end
+
+    def linked_columns
+      settings["linked_columns"] || {}
+    end
+
+    def smart_columns
+      settings["smart_columns"] || {}
+    end
+
+    def smart_variables
+      settings["smart_variables"] || {}
+    end
+
+    def variable_defaults
+      settings["variable_defaults"] || {}
+    end
+
+    def timeout
+      settings["timeout"]
+    end
+
+    def cache
+      @cache ||= begin
+        if settings["cache"].is_a?(Hash)
+          settings["cache"]
+        elsif settings["cache"]
+          {
+            "mode" => "all",
+            "expires_in" => settings["cache"]
+          }
+        else
+          {
+            "mode" => "off"
+          }
+        end
+      end
+    end
+
+    def cache_mode
+      cache["mode"]
+    end
+
+    def cache_expires_in
+      (cache["expires_in"] || 60).to_f
+    end
+
+    def cache_slow_threshold
+      (cache["slow_threshold"] || 15).to_f
+    end
+
+    def local_time_suffix
+      @local_time_suffix ||= Array(settings["local_time_suffix"])
+    end
+
+    def read_cache(cache_key)
+      value = Blazer.cache.read(cache_key)
+      if value
+        Blazer::Result.new(self, *Marshal.load(value), nil)
+      end
+    end
+
+    def run_results(run_id)
+      read_cache(run_cache_key(run_id))
+    end
+
+    def delete_results(run_id)
+      Blazer.cache.delete(run_cache_key(run_id))
+    end
+
+    def run_statement(statement, options = {})
+      run_id = options[:run_id]
+      async = options[:async]
+      result = nil
+      if cache_mode != "off"
+        if options[:refresh_cache]
+          clear_cache(statement) # for checks
+        else
+          result = read_cache(statement_cache_key(statement))
+        end
+      end
+
+      unless result
+        comment = "blazer"
+        if options[:user].respond_to?(:id)
+          comment << ",user_id:#{options[:user].id}"
+        end
+        if options[:user].respond_to?(Blazer.user_name)
+          # only include letters, numbers, and spaces to prevent injection
+          comment << ",user_name:#{options[:user].send(Blazer.user_name).to_s.gsub(/[^a-zA-Z0-9 ]/, "")}"
+        end
+        if options[:query].respond_to?(:id)
+          comment << ",query_id:#{options[:query].id}"
+        end
+        if options[:check]
+          comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"
+        end
+        if options[:run_id]
+          comment << ",run_id:#{options[:run_id]}"
+        end
+        result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
+      end
+
+      result
+    end
+
+    def clear_cache(statement)
+      Blazer.cache.delete(statement_cache_key(statement))
+    end
+
+    def cache_key(key)
+      (["blazer", "v4"] + key).join("/")
+    end
+
+    def statement_cache_key(statement)
+      cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
+    end
+
+    def run_cache_key(run_id)
+      cache_key(["run", run_id])
+    end
+
+    protected
+
+    def run_statement_helper(statement, comment, run_id)
+      start_time = Time.now
+      columns, rows, error = @adapter_instance.run_statement(statement, comment)
+      duration = Time.now - start_time
+
+      cache_data = nil
+      cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
+      if cache || run_id
+        cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
+      end
+
+      if cache && cache_data && @adapter_instance.cachable?(statement)
+        Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
+      end
+
+      if run_id
+        unless cache_data
+          error = "Error storing the results of this query :("
+          cache_data = Marshal.dump([[], [], error, nil])
+        end
+        Blazer.cache.write(run_cache_key(run_id), cache_data, expires_in: 30.seconds)
+      end
+
+      Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
+    end
+
+    def detect_adapter
+      schema = settings["url"].to_s.split("://").first
+      case schema
+      when "mongodb", "presto"
+        schema
+      else
+        "sql"
+      end
+    end
+  end
+end

data/lib/blazer/detect_anomalies.R

@@ -0,0 +1,19 @@
+tryCatch({
+  library(AnomalyDetection)
+
+  args <- commandArgs(trailingOnly = TRUE)
+
+  con <- textConnection(args[2])
+  data <- read.csv(con, stringsAsFactors = FALSE)
+  data$timestamp <- as.POSIXct(data$timestamp)
+
+  if (identical(args[1], "ts")) {
+    res <- AnomalyDetectionTs(data, direction = "both", alpha = 0.05, max_anoms = 0.2)
+  } else {
+    res <- AnomalyDetectionVec(data$count, direction = "both", alpha = 0.05, max_anoms = 0.2, period = length(data$count) / 2 - 1)
+  }
+
+  write.csv(res$anoms)
+}, error = function (e) {
+  write.csv(geterrmessage())
+})

data/lib/blazer/engine.rb

@@ -0,0 +1,47 @@
+module Blazer
+  class Engine < ::Rails::Engine
+    isolate_namespace Blazer
+
+    initializer "blazer" do |app|
+      # use a proc instead of a string
+      app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
+      app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff)\z/ }
+
+      Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
+      Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
+      Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
+      Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
+      Blazer.before_action = Blazer.settings["before_action"] if Blazer.settings["before_action"]
+
+      Blazer.user_class ||= Blazer.settings.key?("user_class") ? Blazer.settings["user_class"] : (User rescue nil)
+      Blazer.user_method = Blazer.settings["user_method"]
+      if Blazer.user_class
+        Blazer.user_method ||= "current_#{Blazer.user_class.to_s.downcase.singularize}"
+      end
+
+      Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
+      if Blazer.settings.key?("mapbox_access_token")
+        Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"]
+      elsif ENV["MAPBOX_ACCESS_TOKEN"].present?
+        Blazer.mapbox_access_token = ENV["MAPBOX_ACCESS_TOKEN"]
+      end
+
+      if Blazer.user_class
+        options = Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s)
+        Blazer::Query.belongs_to :creator, options
+        Blazer::Dashboard.belongs_to :creator, options
+        Blazer::Check.belongs_to :creator, options
+      end
+
+      Blazer.cache ||= Rails.cache
+
+      Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
+      Blazer.async = Blazer.settings["async"] || false
+      if Blazer.async
+        require "blazer/run_statement_job"
+      end
+
+      Blazer.images = Blazer.settings["images"] || false
+    end
+  end
+end

data/lib/blazer/result.rb

@@ -0,0 +1,170 @@
+module Blazer
+  class Result
+    attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached
+
+    def initialize(data_source, columns, rows, error, cached_at, just_cached)
+      @data_source = data_source
+      @columns = columns
+      @rows = rows
+      @error = error
+      @cached_at = cached_at
+      @just_cached = just_cached
+    end
+
+    def timed_out?
+      error == Blazer::TIMEOUT_MESSAGE
+    end
+
+    def cached?
+      cached_at.present?
+    end
+
+    def boom
+      @boom ||= begin
+        boom = {}
+        columns.each_with_index do |key, i|
+          smart_columns_data_source =
+            ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_columns[key] }
+
+          if smart_columns_data_source
+            query = smart_columns_data_source.smart_columns[key]
+            res =
+              if query.is_a?(Hash)
+                query
+              else
+                values = rows.map { |r| r[i] }.compact.uniq
+                result = smart_columns_data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+                result.rows
+              end
+
+            boom[key] = Hash[res.map { |k, v| [k.nil? ? k : k.to_s, v] }]
+          end
+        end
+        boom
+      end
+    end
+
+    def column_types
+      @column_types ||= begin
+        columns.each_with_index.map do |k, i|
+          v = (rows.find { |r| r[i] } || {})[i]
+          if boom[k]
+            "string"
+          elsif v.is_a?(Numeric)
+            "numeric"
+          elsif v.is_a?(Time) || v.is_a?(Date)
+            "time"
+          elsif v.nil?
+            nil
+          else
+            "string"
+          end
+        end
+      end
+    end
+
+    def chart_type
+      @chart_type ||= begin
+        if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "line"
+        elsif column_types == ["time", "string", "numeric"]
+          "line2"
+        elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
+          "bar"
+        elsif column_types == ["string", "string", "numeric"]
+          "bar2"
+        elsif column_types == ["numeric", "numeric"]
+          "scatter"
+        end
+      end
+    end
+
+    def detect_anomaly
+      anomaly = nil
+      message = nil
+
+      if rows.empty?
+        message = "No data"
+      else
+        if chart_type == "line" || chart_type == "line2"
+          series = []
+
+          if chart_type == "line"
+            columns[1..-1].each_with_index.each do |k, i|
+              series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
+            end
+          else
+            rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
+              series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
+            end
+          end
+
+          current_series = nil
+          begin
+            anomalies = []
+            series.each do |s|
+              current_series = s[:name]
+              anomalies << s[:name] if anomaly?(s[:data])
+            end
+            anomaly = anomalies.any?
+            if anomaly
+              if anomalies.size == 1
+                message = "Anomaly detected in #{anomalies.first}"
+              else
+                message = "Anomalies detected in #{anomalies.to_sentence}"
+              end
+            else
+              message = "No anomalies detected"
+            end
+          rescue => e
+            message = "#{current_series}: #{e.message}"
+            raise e if Rails.env.development?
+          end
+        else
+          message = "Bad format"
+        end
+      end
+
+      [anomaly, message]
+    end
+
+    def anomaly?(series)
+      series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
+
+      csv_str =
+        CSV.generate do |csv|
+          csv << ["timestamp", "count"]
+          series.each do |row|
+            csv << row
+          end
+        end
+
+      r_script = %x[which Rscript].chomp
+      type = series.any? && series.last.first.to_time - series.first.first.to_time >= 2.weeks ? "ts" : "vec"
+      args = [type, csv_str]
+      raise "R not found" if r_script.empty?
+      command = "#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{args.map { |a| Shellwords.escape(a) }.join(" ")}"
+      output = %x[#{command}]
+      if output.empty?
+        raise "Unknown R error"
+      end
+
+      rows = CSV.parse(output, headers: true)
+      error = rows.first && rows.first["x"]
+      raise error if error
+
+      timestamps = []
+      if type == "ts"
+        rows.each do |row|
+          timestamps << Time.parse(row["timestamp"])
+        end
+        timestamps.include?(series.last[0].to_time)
+      else
+        rows.each do |row|
+          timestamps << row["index"].to_i
+        end
+        timestamps.include?(series.length)
+      end
+    end
+  end
+end

data/lib/blazer/run_statement.rb

@@ -0,0 +1,40 @@
+module Blazer
+  class RunStatement
+    def perform(data_source, statement, options = {})
+      query = options[:query]
+      Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+      # audit
+      if Blazer.audit
+        audit = Blazer::Audit.new(statement: statement)
+        audit.query = query
+        audit.data_source = data_source.id
+        audit.user = options[:user]
+        audit.save!
+      end
+
+      start_time = Time.now
+      result = data_source.run_statement(statement, options)
+      duration = Time.now - start_time
+
+      if Blazer.audit
+        audit.duration = duration if audit.respond_to?(:duration=)
+        audit.error = result.error if audit.respond_to?(:error=)
+        audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
+        audit.cached = result.cached? if audit.respond_to?(:cached=)
+        if !result.cached? && duration >= 10
+          audit.cost = data_source.cost(statement) if audit.respond_to?(:cost=)
+        end
+        audit.save! if audit.changed?
+      end
+
+      if query && !result.timed_out? && !query.variables.any?
+        query.checks.each do |check|
+          check.update_state(result)
+        end
+      end
+
+      result
+    end
+  end
+end