blazer 2.2.8 → 2.4.2

data/lib/blazer/adapters/spark_adapter.rb

@@ -0,0 +1,9 @@
+module Blazer
+  module Adapters
+    class SparkAdapter < HiveAdapter
+      def tables
+        client.execute("SHOW TABLES").map { |r| r["tableName"] }
+      end
+    end
+  end
+end

data/lib/blazer/adapters/sql_adapter.rb

@@ -27,7 +27,7 @@ module Blazer
             result = select_all("#{statement} /*#{comment}*/")
             columns = result.columns
             result.rows.each do |untyped_row|
-              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
+              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
             end
           end
         rescue => e
@@ -122,6 +122,67 @@ module Blazer
         !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
       end
 
+      def supports_cohort_analysis?
+        postgresql? || mysql?
+      end
+
+      # TODO treat date columns as already in time zone
+      def cohort_analysis_statement(statement, period:, days:)
+        raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+        cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+        tzname = Blazer.time_zone.tzinfo.name
+
+        if mysql?
+          time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
+          case period
+          when "day"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname]
+          when "week"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname, tzname]
+          else
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
+            date_params = [tzname]
+          end
+          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS INTEGER)"
+        else
+          date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
+          date_params = [period, tzname]
+          bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
+        end
+
+        # WITH not an optimization fence in Postgres 12+
+        statement = <<~SQL
+          WITH query AS (
+            #{statement}
+          ),
+          cohorts AS (
+            SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+            WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+            GROUP BY 1
+          )
+          SELECT
+            #{date_sql} AS period,
+            0 AS bucket,
+            COUNT(DISTINCT cohorts.user_id)
+          FROM cohorts GROUP BY 1
+          UNION ALL
+          SELECT
+            #{date_sql} AS period,
+            #{bucket_sql} AS bucket,
+            COUNT(DISTINCT query.user_id)
+          FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+          WHERE query.conversion_time IS NOT NULL
+          AND query.conversion_time >= cohorts.cohort_time
+          #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+          GROUP BY 1, 2
+        SQL
+        params = [statement] + date_params + date_params + [days.to_i * 86400]
+        connection_model.send(:sanitize_sql_array, params)
+      end
+
       protected
 
       def select_all(statement, params = [])
@@ -165,6 +226,8 @@ module Blazer
             "public"
           elsif sqlserver?
             "dbo"
+          elsif connection_model.respond_to?(:connection_db_config)
+            connection_model.connection_db_config.database
           else
             connection_model.connection_config[:database]
           end

data/lib/blazer/data_source.rb

@@ -6,7 +6,7 @@ module Blazer
 
     attr_reader :id, :settings
 
-    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
+    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
 
     def initialize(id, settings)
       @id = id
@@ -185,7 +185,7 @@ module Blazer
     def detect_adapter
       schema = settings["url"].to_s.split("://").first
       case schema
-      when "mongodb", "presto", "cassandra"
+      when "mongodb", "presto", "cassandra", "ignite"
         schema
       else
         "sql"

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.2.8"
+  VERSION = "2.4.2"
 end

data/lib/generators/blazer/templates/config.yml.tt

@@ -71,3 +71,9 @@ check_schedules:
 
 # enable map
 # mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>
+
+# enable uploads
+# uploads:
+#   url: <%%= ENV["BLAZER_UPLOADS_URL"] %>
+#   schema: uploads
+#   data_source: main

data/lib/generators/blazer/templates/install.rb.tt

@@ -6,6 +6,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.text :description
       t.text :statement
       t.string :data_source
+      t.string :status
       t.timestamps null: false
     end
 

data/lib/generators/blazer/templates/uploads.rb.tt

@@ -0,0 +1,10 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :blazer_uploads do |t|
+      t.references :creator
+      t.string :table
+      t.text :description
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/blazer/uploads_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Blazer
+  module Generators
+    class UploadsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "uploads.rb", "db/migrate/create_blazer_uploads.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/tasks/blazer.rake

@@ -8,4 +8,13 @@ namespace :blazer do
   task send_failing_checks: :environment do
     Blazer.send_failing_checks
   end
+
+  desc "archive queries"
+  task archive_queries: :environment do
+    abort "Audits must be enabled to archive" unless Blazer.audit
+    abort "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status")
+
+    viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact
+    Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived")
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 2.2.8
+  version: 2.4.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-01 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -66,36 +66,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.1
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -145,6 +117,7 @@ files:
 - app/controllers/blazer/checks_controller.rb
 - app/controllers/blazer/dashboards_controller.rb
 - app/controllers/blazer/queries_controller.rb
+- app/controllers/blazer/uploads_controller.rb
 - app/helpers/blazer/base_helper.rb
 - app/mailers/blazer/check_mailer.rb
 - app/mailers/blazer/slack_notifier.rb
@@ -155,6 +128,8 @@ files:
 - app/models/blazer/dashboard_query.rb
 - app/models/blazer/query.rb
 - app/models/blazer/record.rb
+- app/models/blazer/upload.rb
+- app/models/blazer/uploads_connection.rb
 - app/views/blazer/_nav.html.erb
 - app/views/blazer/_variables.html.erb
 - app/views/blazer/check_mailer/failing_checks.html.erb
@@ -167,6 +142,8 @@ files:
 - app/views/blazer/dashboards/edit.html.erb
 - app/views/blazer/dashboards/new.html.erb
 - app/views/blazer/dashboards/show.html.erb
+- app/views/blazer/queries/_caching.html.erb
+- app/views/blazer/queries/_cohorts.html.erb
 - app/views/blazer/queries/_form.html.erb
 - app/views/blazer/queries/docs.html.erb
 - app/views/blazer/queries/edit.html.erb
@@ -175,6 +152,10 @@ files:
 - app/views/blazer/queries/run.html.erb
 - app/views/blazer/queries/schema.html.erb
 - app/views/blazer/queries/show.html.erb
+- app/views/blazer/uploads/_form.html.erb
+- app/views/blazer/uploads/edit.html.erb
+- app/views/blazer/uploads/index.html.erb
+- app/views/blazer/uploads/new.html.erb
 - app/views/layouts/blazer/application.html.erb
 - config/routes.rb
 - lib/blazer.rb
@@ -185,6 +166,8 @@ files:
 - lib/blazer/adapters/drill_adapter.rb
 - lib/blazer/adapters/druid_adapter.rb
 - lib/blazer/adapters/elasticsearch_adapter.rb
+- lib/blazer/adapters/hive_adapter.rb
+- lib/blazer/adapters/ignite_adapter.rb
 - lib/blazer/adapters/influxdb_adapter.rb
 - lib/blazer/adapters/mongodb_adapter.rb
 - lib/blazer/adapters/neo4j_adapter.rb
@@ -192,6 +175,7 @@ files:
 - lib/blazer/adapters/salesforce_adapter.rb
 - lib/blazer/adapters/snowflake_adapter.rb
 - lib/blazer/adapters/soda_adapter.rb
+- lib/blazer/adapters/spark_adapter.rb
 - lib/blazer/adapters/sql_adapter.rb
 - lib/blazer/data_source.rb
 - lib/blazer/detect_anomalies.R
@@ -203,6 +187,8 @@ files:
 - lib/generators/blazer/install_generator.rb
 - lib/generators/blazer/templates/config.yml.tt
 - lib/generators/blazer/templates/install.rb.tt
+- lib/generators/blazer/templates/uploads.rb.tt
+- lib/generators/blazer/uploads_generator.rb
 - lib/tasks/blazer.rake
 - licenses/LICENSE-ace.txt
 - licenses/LICENSE-bootstrap.txt
@@ -239,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Explore your data with SQL. Easily create charts and dashboards, and share