blazer_xlsx 3.0.5

data/lib/blazer/adapters/sql_adapter.rb

@@ -0,0 +1,353 @@
+module Blazer
+  module Adapters
+    class SqlAdapter < BaseAdapter
+      attr_reader :connection_model
+
+      def initialize(data_source)
+        super
+
+        @connection_model =
+          Class.new(Blazer::Connection) do
+            def self.name
+              "Blazer::Connection::Adapter#{object_id}"
+            end
+            establish_connection(data_source.settings["url"]) if data_source.settings["url"]
+          end
+      end
+
+      def run_statement(statement, comment, bind_params = [])
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          result = nil
+          in_transaction do
+            set_timeout(data_source.timeout) if data_source.timeout
+            binds = bind_params.map { |v| ActiveRecord::Relation::QueryAttribute.new(nil, v, ActiveRecord::Type::Value.new) }
+            result = connection_model.connection.select_all("#{statement} /*#{comment}*/", nil, binds)
+          end
+
+          columns = result.columns
+          rows = result.rows
+
+          # cast values
+          if result.column_types.any?
+            types = columns.map { |c| result.column_types[c] }
+            rows =
+              rows.map do |row|
+                row.map.with_index do |v, i|
+                  v && (t = types[i]) ? t.send(:cast_value, v) : v
+                end
+              end
+          end
+
+          # fix for non-ASCII column names and charts
+          if adapter_name == "Trilogy"
+            columns.map! { |k| k.dup.force_encoding(Encoding::UTF_8) }
+          end
+
+          # fix for binary data
+          if mysql?
+            rows =
+              rows.map do |row|
+                row.map do |v|
+                  if v.is_a?(String) && v.encoding == Encoding::BINARY
+                    "0x#{v.unpack1("H*").upcase}"
+                  else
+                    v
+                  end
+                end
+              end
+          end
+        rescue => e
+          error = e.message.sub(/.+ERROR: /, "")
+          error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
+          error = Blazer::VARIABLE_MESSAGE if error.include?("syntax error at or near \"$") || error.include?("Incorrect syntax near '@") || error.include?("your MySQL server version for the right syntax to use near '?")
+          if error.include?("could not determine data type of parameter")
+            error += " - try adding casting to variables and make sure none are inside a string literal"
+          end
+          reconnect if error.include?("PG::ConnectionBad")
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        sql = add_schemas("SELECT table_schema, table_name FROM information_schema.tables")
+        result = data_source.run_statement(sql, refresh_cache: true)
+        if postgresql? || redshift? || snowflake?
+          result.rows.sort_by { |r| [r[0] == default_schema ? "" : r[0], r[1]] }.map do |row|
+            table =
+              if row[0] == default_schema
+                row[1]
+              else
+                "#{row[0]}.#{row[1]}"
+              end
+
+            table = table.downcase if snowflake?
+
+            {
+              table: table,
+              value: connection_model.connection.quote_table_name(table)
+            }
+          end
+        else
+          result.rows.map(&:second).sort
+        end
+      end
+
+      def schema
+        sql = add_schemas("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns")
+        result = data_source.run_statement(sql)
+        result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }
+      end
+
+      def preview_statement
+        if sqlserver?
+          "SELECT TOP (10) * FROM {table}"
+        else
+          "SELECT * FROM {table} LIMIT 10"
+        end
+      end
+
+      def reconnect
+        connection_model.establish_connection(settings["url"])
+      end
+
+      def cost(statement)
+        result = explain(statement)
+        if sqlserver?
+          result["TotalSubtreeCost"]
+        else
+          match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
+          match[1] if match
+        end
+      end
+
+      def explain(statement)
+        if postgresql? || redshift?
+          select_all("EXPLAIN #{statement}").rows.first.first
+        elsif sqlserver?
+          begin
+            execute("SET SHOWPLAN_ALL ON")
+            result = select_all(statement).each.first
+          ensure
+            execute("SET SHOWPLAN_ALL OFF")
+          end
+          result
+        end
+      rescue
+        nil
+      end
+
+      def cancel(run_id)
+        if postgresql?
+          select_all("SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND query LIKE ?", ["%,run_id:#{run_id}%"])
+        elsif redshift?
+          first_row = select_all("SELECT pid FROM stv_recents WHERE status = 'Running' AND query LIKE ?", ["%,run_id:#{run_id}%"]).first
+          if first_row
+            select_all("CANCEL #{first_row["pid"].to_i}")
+          end
+        end
+      end
+
+      def cachable?(statement)
+        !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
+      end
+
+      def supports_cohort_analysis?
+        postgresql? || mysql?
+      end
+
+      # TODO treat date columns as already in time zone
+      def cohort_analysis_statement(statement, period:, days:)
+        raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+        cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+        tzname = Blazer.time_zone.tzinfo.name
+
+        if mysql?
+          time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
+          case period
+          when "day"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname]
+          when "week"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname, tzname]
+          else
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
+            date_params = [tzname]
+          end
+          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS SIGNED)"
+        else
+          date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
+          date_params = [period, tzname]
+          bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
+        end
+
+        # WITH not an optimization fence in Postgres 12+
+        statement = <<~SQL
+          WITH query AS (
+            {placeholder}
+          ),
+          cohorts AS (
+            SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+            WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+            GROUP BY 1
+          )
+          SELECT
+            #{date_sql} AS period,
+            0 AS bucket,
+            COUNT(DISTINCT cohorts.user_id)
+          FROM cohorts GROUP BY 1
+          UNION ALL
+          SELECT
+            #{date_sql} AS period,
+            #{bucket_sql} AS bucket,
+            COUNT(DISTINCT query.user_id)
+          FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+          WHERE query.conversion_time IS NOT NULL
+          AND query.conversion_time >= cohorts.cohort_time
+          #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+          GROUP BY 1, 2
+        SQL
+        params = [statement] + date_params + date_params + [days.to_i * 86400]
+        connection_model.send(:sanitize_sql_array, params)
+      end
+
+      def quoting
+        ->(value) { connection_model.connection.quote(value) }
+      end
+
+      # Redshift adapter silently ignores binds
+      def parameter_binding
+        if postgresql?
+          :numeric
+        elsif sqlite? && prepared_statements?
+          # Active Record silently ignores binds with SQLite when prepared statements are disabled
+          :numeric
+        elsif mysql? && prepared_statements?
+          # Active Record silently ignores binds with MySQL when prepared statements are disabled
+          :positional
+        elsif sqlserver?
+          proc do |statement, variables|
+            variables.each_with_index do |(k, _), i|
+              statement = statement.gsub("{#{k}}", "@#{i} ")
+            end
+            [statement, variables.values]
+          end
+        end
+      end
+
+      protected
+
+      def select_all(statement, params = [])
+        statement = connection_model.send(:sanitize_sql_array, [statement] + params) if params.any?
+        connection_model.connection.select_all(statement)
+      end
+
+      # seperate from select_all to prevent mysql error
+      def execute(statement)
+        connection_model.connection.execute(statement)
+      end
+
+      def postgresql?
+        ["PostgreSQL", "PostGIS"].include?(adapter_name)
+      end
+
+      def redshift?
+        ["Redshift"].include?(adapter_name)
+      end
+
+      def mysql?
+        ["MySQL", "Mysql2", "Mysql2Spatial", "Trilogy"].include?(adapter_name)
+      end
+
+      def sqlite?
+        ["SQLite"].include?(adapter_name)
+      end
+
+      def sqlserver?
+        ["SQLServer", "tinytds", "mssql"].include?(adapter_name)
+      end
+
+      def snowflake?
+        data_source.adapter == "snowflake"
+      end
+
+      def adapter_name
+        # prevent bad data source from taking down queries/new
+        connection_model.connection.adapter_name rescue nil
+      end
+
+      def default_schema
+        @default_schema ||= begin
+          if postgresql? || redshift?
+            "public"
+          elsif sqlserver?
+            "dbo"
+          elsif connection_model.respond_to?(:connection_db_config)
+            connection_model.connection_db_config.database
+          else
+            connection_model.connection_config[:database]
+          end
+        end
+      end
+
+      def add_schemas(query)
+        if settings["schemas"]
+          where = "table_schema IN (?)"
+          schemas = settings["schemas"]
+        elsif mysql?
+          where = "table_schema IN (?)"
+          schemas = [default_schema]
+        else
+          where = "table_schema NOT IN (?)"
+          schemas = ["information_schema"]
+          schemas.map!(&:upcase) if snowflake?
+          schemas << "pg_catalog" if postgresql? || redshift?
+        end
+        connection_model.send(:sanitize_sql_array, ["#{query} WHERE #{where}", schemas])
+      end
+
+      def set_timeout(timeout)
+        if postgresql? || redshift?
+          execute("SET #{use_transaction? ? "LOCAL " : ""}statement_timeout = #{timeout.to_i * 1000}")
+        elsif mysql?
+          # use send as this method is private in Rails 4.2
+          mariadb = connection_model.connection.send(:mariadb?) rescue false
+          if mariadb
+            execute("SET max_statement_time = #{timeout.to_i * 1000}")
+          else
+            execute("SET max_execution_time = #{timeout.to_i * 1000}")
+          end
+        else
+          raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
+        end
+      end
+
+      def use_transaction?
+        settings.key?("use_transaction") ? settings["use_transaction"] : true
+      end
+
+      def in_transaction
+        connection_model.connection_pool.with_connection do
+          if use_transaction?
+            connection_model.transaction do
+              yield
+              raise ActiveRecord::Rollback
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      def prepared_statements?
+        connection_model.connection.prepared_statements
+      end
+    end
+  end
+end

data/lib/blazer/adapters.rb

@@ -0,0 +1,17 @@
+Blazer.register_adapter "athena", Blazer::Adapters::AthenaAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
+Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
+Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
+Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
+Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapter
+Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
+Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
+Blazer.register_adapter "opensearch", Blazer::Adapters::OpensearchAdapter
+Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
+Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
+Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
+Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
+Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
+Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/anomaly_detectors.rb

@@ -0,0 +1,22 @@
+Blazer.register_anomaly_detector "anomaly_detection" do |series|
+  anomalies = AnomalyDetection.detect(series.to_h, period: :auto)
+  anomalies.include?(series.last[0])
+end
+
+Blazer.register_anomaly_detector "prophet" do |series|
+  df = Rover::DataFrame.new(series[0..-2].map { |v| {"ds" => v[0], "y" => v[1]} })
+  m = Prophet.new(interval_width: 0.99)
+  m.logger.level = ::Logger::FATAL # no logging
+  m.fit(df)
+  future = Rover::DataFrame.new(series[-1..-1].map { |v| {"ds" => v[0]} })
+  forecast = m.predict(future).to_a[0]
+  lower = forecast["yhat_lower"]
+  upper = forecast["yhat_upper"]
+  value = series.last[1]
+  value < lower || value > upper
+end
+
+Blazer.register_anomaly_detector "trend" do |series|
+  anomalies = Trend.anomalies(series.to_h)
+  anomalies.include?(series.last[0])
+end

data/lib/blazer/check_mailer.rb

@@ -0,0 +1,27 @@
+module Blazer
+  class CheckMailer < ActionMailer::Base
+    include ActionView::Helpers::TextHelper
+
+    default from: Blazer.from_email if Blazer.from_email
+    layout false
+
+    def state_change(check, state, state_was, rows_count, error, columns, rows, column_types, check_type)
+      @check = check
+      @state = state
+      @state_was = state_was
+      @rows_count = rows_count
+      @error = error
+      @columns = columns
+      @rows = rows
+      @column_types = column_types
+      @check_type = check_type
+      mail to: check.emails, reply_to: check.emails, subject: "Check #{state.titleize}: #{check.query.name}"
+    end
+
+    def failing_checks(email, checks)
+      @checks = checks
+      # add reply_to for mailing lists
+      mail to: email, reply_to: email, subject: "#{pluralize(checks.size, "Check")} Failing"
+    end
+  end
+end

data/lib/blazer/data_source.rb

@@ -0,0 +1,266 @@
+module Blazer
+  class DataSource
+    extend Forwardable
+
+    attr_reader :id, :settings
+
+    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
+
+    def initialize(id, settings)
+      @id = id
+      @settings = settings
+    end
+
+    def adapter
+      settings["adapter"] || detect_adapter
+    end
+
+    def name
+      settings["name"] || @id
+    end
+
+    def linked_columns
+      settings["linked_columns"] || {}
+    end
+
+    def smart_columns
+      settings["smart_columns"] || {}
+    end
+
+    def smart_variables
+      settings["smart_variables"] || {}
+    end
+
+    def variable_defaults
+      settings["variable_defaults"] || {}
+    end
+
+    def timeout
+      settings["timeout"]
+    end
+
+    def cache
+      @cache ||= begin
+        if settings["cache"].is_a?(Hash)
+          settings["cache"]
+        elsif settings["cache"]
+          {
+            "mode" => "all",
+            "expires_in" => settings["cache"]
+          }
+        else
+          {
+            "mode" => "off"
+          }
+        end
+      end
+    end
+
+    def cache_mode
+      cache["mode"]
+    end
+
+    def cache_expires_in
+      (cache["expires_in"] || 60).to_f
+    end
+
+    def cache_slow_threshold
+      (cache["slow_threshold"] || 15).to_f
+    end
+
+    def local_time_suffix
+      @local_time_suffix ||= Array(settings["local_time_suffix"])
+    end
+
+    def result_cache
+      @result_cache ||= Blazer::ResultCache.new(self)
+    end
+
+    def run_results(run_id)
+      result_cache.read_run(run_id)
+    end
+
+    def delete_results(run_id)
+      result_cache.delete_run(run_id)
+    end
+
+    def sub_variables(statement, vars)
+      statement = statement.dup
+      vars.each do |var, value|
+        # use block form to disable back-references
+        statement.gsub!("{#{var}}") { quote(value) }
+      end
+      statement
+    end
+
+    def run_statement(statement, options = {})
+      statement = Statement.new(statement, self) if statement.is_a?(String)
+      statement.bind unless statement.bind_statement
+
+      result = nil
+      if cache_mode != "off"
+        if options[:refresh_cache]
+          clear_cache(statement) # for checks
+        else
+          result = result_cache.read_statement(statement)
+        end
+      end
+
+      unless result
+        comment = "blazer"
+        if options[:user].respond_to?(:id)
+          comment << ",user_id:#{options[:user].id}"
+        end
+        if options[:user].respond_to?(Blazer.user_name)
+          # only include letters, numbers, and spaces to prevent injection
+          comment << ",user_name:#{options[:user].send(Blazer.user_name).to_s.gsub(/[^a-zA-Z0-9 ]/, "")}"
+        end
+        if options[:query].respond_to?(:id)
+          comment << ",query_id:#{options[:query].id}"
+        end
+        if options[:check]
+          comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"
+        end
+        if options[:run_id]
+          comment << ",run_id:#{options[:run_id]}"
+        end
+        result = run_statement_helper(statement, comment, options)
+      end
+
+      if options[:async] && options[:run_id]
+        run_id = options[:run_id]
+        begin
+          result_cache.write_run(run_id, result)
+        rescue
+          result = Blazer::Result.new(self, [], [], "Error storing the results of this query :(", nil, false)
+          result_cache.write_run(run_id, result)
+        end
+      end
+
+      result
+    end
+
+    def clear_cache(statement)
+      result_cache.delete_statement(statement)
+    end
+
+    def quote(value)
+      if quoting == :backslash_escape || quoting == :single_quote_escape
+        # only need to support types generated by process_vars
+        if value.is_a?(Integer) || value.is_a?(Float)
+          value.to_s
+        elsif value.nil?
+          "NULL"
+        else
+          value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
+
+          if quoting == :backslash_escape
+            "'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
+          else
+            "'#{value.gsub("'", "''")}'"
+          end
+        end
+      elsif quoting.respond_to?(:call)
+        quoting.call(value)
+      elsif quoting.nil?
+        raise Blazer::Error, "Quoting not specified"
+      else
+        raise Blazer::Error, "Unknown quoting"
+      end
+    end
+
+    def bind_params(statement, variables)
+      if parameter_binding == :positional
+        locations = []
+        variables.each do |k, v|
+          i = 0
+          while (idx = statement.index("{#{k}}", i))
+            locations << [v, idx]
+            i = idx + 1
+          end
+        end
+        variables.each do |k, v|
+          statement = statement.gsub("{#{k}}", "?")
+        end
+        [statement, locations.sort_by(&:last).map(&:first)]
+      elsif parameter_binding == :numeric
+        variables.each_with_index do |(k, v), i|
+          # add trailing space if followed by digit
+          # try to keep minimal to avoid fixing invalid queries like SELECT{var}
+          statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
+        end
+        [statement, variables.values]
+      elsif parameter_binding.respond_to?(:call)
+        parameter_binding.call(statement, variables)
+      elsif parameter_binding.nil?
+        [sub_variables(statement, variables), []]
+      else
+        raise Blazer::Error, "Unknown bind parameters"
+      end
+    end
+
+    protected
+
+    def adapter_instance
+      @adapter_instance ||= begin
+        # TODO add required settings to adapters
+        unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
+          raise Blazer::Error, "Empty url for data source: #{id}"
+        end
+
+        unless Blazer.adapters[adapter]
+          raise Blazer::Error, "Unknown adapter"
+        end
+
+        Blazer.adapters[adapter].new(self)
+      end
+    end
+
+    def quoting
+      @quoting ||= adapter_instance.quoting
+    end
+
+    def parameter_binding
+      @parameter_binding ||= adapter_instance.parameter_binding
+    end
+
+    def run_statement_helper(statement, comment, options)
+      start_time = Blazer.monotonic_time
+      columns, rows, error =
+        if adapter_instance.parameter_binding
+          adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
+        else
+          adapter_instance.run_statement(statement.bind_statement, comment)
+        end
+      duration = Blazer.monotonic_time - start_time
+
+      cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
+
+      result = Blazer::Result.new(self, columns, rows, error, cache ? Time.now : nil, false)
+
+      if cache && adapter_instance.cachable?(statement.bind_statement)
+        begin
+          result_cache.write_statement(statement, result, expires_in: cache_expires_in.to_f * 60)
+          # set just_cached after caching
+          result.just_cached = true
+        rescue
+          # do nothing
+        end
+      end
+
+      result.cached_at = nil
+      result
+    end
+
+    # TODO check for adapter with same name, default to sql
+    def detect_adapter
+      scheme = settings["url"].to_s.split("://").first
+      case scheme
+      when "presto", "cassandra", "ignite"
+        scheme
+      else
+        "sql"
+      end
+    end
+  end
+end