blazer 2.4.0 → 2.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9f173935d9c63e3537014420daf0af3017f6f9c51615d102fb6f0b4b3b45ba8
-  data.tar.gz: 8743bc1783f5181bd946d6263a70ac7ae02cbb26d3663b59c8915cdc9495e80f
+  metadata.gz: b2a6413e7d7d49e280cbb8d6a880ec32c6c49452e69f980183b592bd927fa90a
+  data.tar.gz: 2e988e2010ae919dbfbb7c4c922753425b16eb0ee1e6fe309789cb44a450d8f7
 SHA512:
-  metadata.gz: bfe97805e455f1ec96cc825563460a303006c6c379f9adfb3b24ed2f9f3b86f57f9600648a8fbc01a564ba62f78346582659dab9c121d239f91383211a3b8ad3
-  data.tar.gz: 52a1ce2a7a489105ba1c1ece23e412117de868bf6c50a07b968056b6ccbbe40d90031087c836f7831d04f12bd7d88593260a0a98bfdcb15659e1158c3fc070d6
+  metadata.gz: 50fc2e2d4c430935e05d14df65aea648df86c4792e58007ff0f6015a5c58a5842db4f9d22d430ac7f84b094325d45c41406f67676d6ec3d3535266e47ebd53a0
+  data.tar.gz: 3118830b6ab4198fe383615c09319d98619480e2f88dca117b92e99c228da82a5556f7a9948ad57dd45e788cacc20efec370e0eb68770919e5dab9909bd4f1ef

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.4.1 (2021-01-25)
+
+- Added cohorts for MySQL
+- Added support for Apache Hive and Apache Spark
+- Fixed deprecation warning with Active Record 6.1
+
 ## 2.4.0 (2020-12-15)
 
 - Added cohorts

data/README.md

@@ -4,7 +4,7 @@ Explore your data with SQL. Easily create charts and dashboards, and share them
 
 [Try it out](https://blazer.dokkuapp.com)
 
-[![Screenshot](https://blazer.dokkuapp.com/assets/blazer-90bd7acc9fdf1f5fc2bb25bfe5506f746ec8c9d2e0730388debfd697e32f75b8.png)](https://blazer.dokkuapp.com)
+[![Screenshot](https://blazer.dokkuapp.com/assets/blazer-a10baa40fef1ca2f5bb25fc97bcf261a6a54192fb1ad0f893c0f562b8c7c4697.png)](https://blazer.dokkuapp.com)
 
 Blazer is also available as a [Docker image](https://github.com/ankane/blazer-docker).
 
@@ -412,7 +412,7 @@ SELECT users.id AS user_id, orders.created_at AS conversion_time, users.created_
 FROM users LEFT JOIN orders ON orders.user_id = users.id
 ```
 
-This feature requires PostgreSQL.
+This feature requires PostgreSQL or MySQL.
 
 ## Anomaly Detection
 
@@ -566,6 +566,8 @@ data_sources:
 - [Amazon Athena](#amazon-athena)
 - [Amazon Redshift](#amazon-redshift)
 - [Apache Drill](#apache-drill)
+- [Apache Hive](#apache-hive)
+- [Apache Spark](#apache-spark)
 - [Cassandra](#cassandra)
 - [Druid](#druid)
 - [Elasticsearch](#elasticsearch)
@@ -627,6 +629,32 @@ data_sources:
     url: http://hostname:8047
 ```
 
+### Apache Hive
+
+Add [hexspace](https://github.com/ankane/hexspace) to your Gemfile and set:
+
+```yml
+data_sources:
+  my_source:
+    adapter: hive
+    url: sasl://user:password@hostname:10000/database
+```
+
+Use a read-only user. Requires [HiveServer2](https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2).
+
+### Apache Spark
+
+Add [hexspace](https://github.com/ankane/hexspace) to your Gemfile and set:
+
+```yml
+data_sources:
+  my_source:
+    adapter: spark
+    url: sasl://user:password@hostname:10000/database
+```
+
+Use a read-only user. Requires the [Thrift server](https://spark.apache.org/docs/latest/sql-distributed-sql-engine.html).
+
 ### Cassandra
 
 Add [cassandra-driver](https://github.com/datastax/ruby-driver) to your Gemfile and set:
@@ -949,152 +977,6 @@ with:
 add_column :blazer_checks, :slack_channels, :text
 ```
 
-### 1.5
-
-To take advantage of the anomaly detection, create a migration
-
-```sh
-rails g migration upgrade_blazer_to_1_5
-```
-
-with:
-
-```ruby
-add_column :blazer_checks, :check_type, :string
-add_column :blazer_checks, :message, :text
-commit_db_transaction
-
-Blazer::Check.reset_column_information
-
-Blazer::Check.where(invert: true).update_all(check_type: "missing_data")
-Blazer::Check.where(check_type: nil).update_all(check_type: "bad_data")
-```
-
-### 1.3
-
-To take advantage of the latest features, create a migration
-
-```sh
-rails g migration upgrade_blazer_to_1_3
-```
-
-with:
-
-```ruby
-add_column :blazer_dashboards, :creator_id, :integer
-add_column :blazer_checks, :creator_id, :integer
-add_column :blazer_checks, :invert, :boolean
-add_column :blazer_checks, :schedule, :string
-add_column :blazer_checks, :last_run_at, :timestamp
-commit_db_transaction
-
-Blazer::Check.update_all schedule: "1 hour"
-```
-
-### 1.0
-
-Blazer 1.0 brings a number of new features:
-
-- multiple data sources, including Redshift
-- dashboards
-- checks
-
-To upgrade, run:
-
-```sh
-bundle update blazer
-```
-
-Create a migration
-
-```sh
-rails g migration upgrade_blazer_to_1_0
-```
-
-with:
-
-```ruby
-add_column :blazer_queries, :data_source, :string
-add_column :blazer_audits, :data_source, :string
-
-create_table :blazer_dashboards do |t|
-  t.text :name
-  t.timestamps
-end
-
-create_table :blazer_dashboard_queries do |t|
-  t.references :dashboard
-  t.references :query
-  t.integer :position
-  t.timestamps
-end
-
-create_table :blazer_checks do |t|
-  t.references :query
-  t.string :state
-  t.text :emails
-  t.timestamps
-end
-```
-
-And run:
-
-```sh
-rails db:migrate
-```
-
-Update `config/blazer.yml` with:
-
-```yml
-# see https://github.com/ankane/blazer for more info
-
-data_sources:
-  main:
-    url: <%= ENV["BLAZER_DATABASE_URL"] %>
-
-    # statement timeout, in seconds
-    # applies to PostgreSQL only
-    # none by default
-    # timeout: 15
-
-    # time to cache results, in minutes
-    # can greatly improve speed
-    # none by default
-    # cache: 60
-
-    # wrap queries in a transaction for safety
-    # not necessary if you use a read-only user
-    # true by default
-    # use_transaction: false
-
-    smart_variables:
-      # zone_id: "SELECT id, name FROM zones ORDER BY name ASC"
-
-    linked_columns:
-      # user_id: "/admin/users/{value}"
-
-    smart_columns:
-      # user_id: "SELECT id, name FROM users WHERE id IN {value}"
-
-# create audits
-audit: true
-
-# change the time zone
-# time_zone: "Pacific Time (US & Canada)"
-
-# class name of the user model
-# user_class: User
-
-# method name for the current user
-# user_method: current_user
-
-# method name for the display name
-# user_name: name
-
-# email to send checks from
-# from_email: blazer@example.org
-```
-
 ## History
 
 View the [changelog](https://github.com/ankane/blazer/blob/master/CHANGELOG.md)

data/app/models/blazer/query.rb

@@ -9,7 +9,7 @@ module Blazer
     validates :statement, presence: true
 
     scope :active, -> { column_names.include?("status") ? where(status: "active") : all }
-    scope :named, -> { where("blazer_queries.name <> ''") }
+    scope :named, -> { where.not(name: "") }
 
     def to_param
       [id, name].compact.join("-").gsub("'", "").parameterize

data/lib/blazer.rb

@@ -18,12 +18,14 @@ require "blazer/adapters/cassandra_adapter"
 require "blazer/adapters/drill_adapter"
 require "blazer/adapters/druid_adapter"
 require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/hive_adapter"
 require "blazer/adapters/influxdb_adapter"
 require "blazer/adapters/mongodb_adapter"
 require "blazer/adapters/neo4j_adapter"
 require "blazer/adapters/presto_adapter"
 require "blazer/adapters/salesforce_adapter"
 require "blazer/adapters/soda_adapter"
+require "blazer/adapters/spark_adapter"
 require "blazer/adapters/sql_adapter"
 require "blazer/adapters/snowflake_adapter"
 
@@ -239,11 +241,13 @@ Blazer.register_adapter "cassandra", Blazer::Adapters::CassandraAdapter
 Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
 Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
 Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
+Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
 Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
 Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
 Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
 Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
 Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
+Blazer.register_adapter "spark", Blazer::Adapters::SparkAdapter
 Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
 Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/adapters/hive_adapter.rb

@@ -0,0 +1,45 @@
+module Blazer
+  module Adapters
+    class HiveAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          result = client.execute("#{statement} /*#{comment}*/")
+          columns = result.any? ? result.first.keys : []
+          rows = result.map(&:values)
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        client.execute("SHOW TABLES").map { |r| r["tab_name"] }
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      protected
+
+      def client
+        @client ||= begin
+          uri = URI.parse(settings["url"])
+          Hexspace::Client.new(
+            host: uri.host,
+            port: uri.port,
+            username: uri.user,
+            password: uri.password,
+            database: uri.path.sub(/\A\//, ""),
+            mode: uri.scheme.to_sym
+          )
+        end
+      end
+    end
+  end
+end

data/lib/blazer/adapters/spark_adapter.rb

@@ -0,0 +1,9 @@
+module Blazer
+  module Adapters
+    class SparkAdapter < HiveAdapter
+      def tables
+        client.execute("SHOW TABLES").map { |r| r["tableName"] }
+      end
+    end
+  end
+end

data/lib/blazer/adapters/sql_adapter.rb

@@ -123,7 +123,7 @@ module Blazer
       end
 
       def supports_cohort_analysis?
-        postgresql?
+        postgresql? || mysql?
       end
 
       # TODO treat date columns as already in time zone
@@ -131,6 +131,27 @@ module Blazer
         raise "Cohort analysis not supported" unless supports_cohort_analysis?
 
         cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+        tzname = Blazer.time_zone.tzinfo.name
+
+        if mysql?
+          time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
+          case period
+          when "day"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname]
+          when "week"
+            date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
+            date_params = [tzname, tzname]
+          else
+            date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
+            date_params = [tzname]
+          end
+          bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS INTEGER)"
+        else
+          date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
+          date_params = [period, tzname]
+          bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
+        end
 
         # WITH not an optimization fence in Postgres 12+
         statement = <<~SQL
@@ -143,14 +164,14 @@ module Blazer
             GROUP BY 1
           )
           SELECT
-            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+            #{date_sql} AS period,
             0 AS bucket,
             COUNT(DISTINCT cohorts.user_id)
           FROM cohorts GROUP BY 1
           UNION ALL
           SELECT
-            date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
-            CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int AS bucket,
+            #{date_sql} AS period,
+            #{bucket_sql} AS bucket,
             COUNT(DISTINCT query.user_id)
           FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
           WHERE query.conversion_time IS NOT NULL
@@ -158,8 +179,7 @@ module Blazer
           #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
           GROUP BY 1, 2
         SQL
-        tzname = Blazer.time_zone.tzinfo.name
-        params = [statement, period, tzname, period, tzname, days.to_i * 86400]
+        params = [statement] + date_params + date_params + [days.to_i * 86400]
         connection_model.send(:sanitize_sql_array, params)
       end
 
@@ -206,6 +226,8 @@ module Blazer
             "public"
           elsif sqlserver?
             "dbo"
+          elsif connection_model.respond_to?(:connection_db_config)
+            connection_model.connection_db_config.database
           else
             connection_model.connection_config[:database]
           end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.4.0"
+  VERSION = "2.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-15 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -194,6 +194,7 @@ files:
 - lib/blazer/adapters/drill_adapter.rb
 - lib/blazer/adapters/druid_adapter.rb
 - lib/blazer/adapters/elasticsearch_adapter.rb
+- lib/blazer/adapters/hive_adapter.rb
 - lib/blazer/adapters/influxdb_adapter.rb
 - lib/blazer/adapters/mongodb_adapter.rb
 - lib/blazer/adapters/neo4j_adapter.rb
@@ -201,6 +202,7 @@ files:
 - lib/blazer/adapters/salesforce_adapter.rb
 - lib/blazer/adapters/snowflake_adapter.rb
 - lib/blazer/adapters/soda_adapter.rb
+- lib/blazer/adapters/spark_adapter.rb
 - lib/blazer/adapters/sql_adapter.rb
 - lib/blazer/data_source.rb
 - lib/blazer/detect_anomalies.R
@@ -250,7 +252,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Explore your data with SQL. Easily create charts and dashboards, and share