blazer 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af24e4d2d3a7574a1b6e03186ea77d54dcb4f4b5
-  data.tar.gz: 3d9b34fd4cb0c26cb7516d75cfe2eab85a2d3b8f
+  metadata.gz: 04d14bae947237ea5450d0c8104572d41030e2ba
+  data.tar.gz: 1cf2e2ecad47ab458a653382728911dfaa82b1d9
 SHA512:
-  metadata.gz: 5c9590d8f9798d5cef1ca335ba38f990ae51d021d2a04d4fa01929b43a032a823988ddc39b697e36888d62a84fe663d9a974fdbd9c6339fa00d50b15688aa899
-  data.tar.gz: 55fe87e6539b8f9b776c68bb18dd9b20564151f3b5dbb47ad3a0380e811914f4c4b5e6f8202dd891406e6c790c4f17ddfc21abd1f97ff05b65fd160950231cef
+  metadata.gz: 6331237e96ba16d02561cfeb1f2de6e95abdbc2b6f2069cdfa2a72309e92720137ae899498b4ac177abc9b3fb663293950006ff84e186560101b0786c9a6c75a
+  data.tar.gz: b71679dded9d87c6637c082da7a41bda1ff448122fbfd3de62b1a8d8b5604b14196a26a96d8b2a14b0db38e65b77a8f2e0299761b3c25f411955541ce276e865

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.6.0
+
+- Added support for MongoDB [beta]
+- Added support for Elasticsearch [beta]
+- Fixed deprecation warning in Rails 5
+
 ## 1.5.1
 
 - Added anomaly detection for data less than 2 weeks

data/README.md

@@ -12,7 +12,7 @@ Explore your data with SQL. Easily create charts and dashboards, and share them
 
 ## Features
 
-- **Multiple data sources** - works with PostgreSQL, MySQL, and Redshift
+- **Multiple data sources** - PostgreSQL, MySQL, Redshift, and [many more](#full-list)
 - **Variables** - run the same queries with different values
 - **Checks & alerts** - get emailed when bad data appears
 - **Audits** - all queries are tracked
@@ -346,12 +346,57 @@ data_sources:
     # ...
 ```
 
+### Full List
+
+- PostgreSQL
+- MySQL
+- SQL Server
+- Oracle
+- IBM DB2 and Informix
+- SQLite
+- [Redshift](#redshift)
+- [MongoDB](#mongodb) [beta]
+- [Elasticsearch](#elasticsearch) [beta]
+
+You can also create an adapter for any other data store.
+
+**Note:** In the examples below, we recommend using environment variables for urls.
+
+```yml
+data_sources:
+  my_source:
+    url: <%= ENV["BLAZER_MY_SOURCE_URL"] %>
+```
+
 ### Redshift
 
 Add [activerecord4-redshift-adapter](https://github.com/aamine/activerecord4-redshift-adapter) to your Gemfile and set:
 
-```ruby
-ENV["BLAZER_DATABASE_URL"] = "redshift://user:password@hostname:5439/database"
+```yml
+data_sources:
+  my_source:
+    url: redshift://user:password@hostname:5439/database
+```
+
+### MongoDB
+
+Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
+
+```yml
+data_sources:
+  my_source:
+    url: mongodb://user:password@hostname:27017/database
+```
+
+### Elasticsearch
+
+Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) to your Gemfile and set:
+
+```yml
+data_sources:
+  my_source:
+    adapter: elasticsearch
+    url: http://user:password@hostname:9200/
 ```
 
 ## Learn SQL

data/app/controllers/blazer/base_controller.rb

@@ -1,7 +1,15 @@
 module Blazer
   class BaseController < ApplicationController
     # skip all filters
-    skip_filter *_process_action_callbacks.map(&:filter)
+    filters = _process_action_callbacks.map(&:filter)
+    if Rails::VERSION::MAJOR >= 5
+      skip_before_action(*filters, raise: false)
+      skip_after_action(*filters, raise: false)
+      skip_around_action(*filters, raise: false)
+      before_action :verify_request_size
+    else
+      skip_action_callback *filters
+    end
 
     protect_from_forgery with: :exception
 
@@ -46,7 +54,7 @@ module Blazer
     def extract_vars(statement)
       # strip commented out lines
       # and regex {1} or {1,2}
-      statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{.*?\}/).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) }.uniq
+      statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
     end
     helper_method :extract_vars
 

data/app/controllers/blazer/queries_controller.rb

@@ -114,7 +114,7 @@ module Blazer
           end
           @result = result.first
         else
-          @result = @data_source.run_main_statement(@statement, options)
+          @result = RunStatement.new.perform(@data_source, @statement, options)
         end
 
         if @result

data/app/views/blazer/check_mailer/failing_checks.html.erb

@@ -1,6 +1,6 @@
 <ul>
   <% @checks.each do |check| %>
-    <li><%= link_to check.query.name, query_url(check.query_id) %></li>
+    <li><%= link_to check.query.name, query_url(check.query_id) %> <%= check.state %></li>
   <% end %>
 </ul>
 <p><%= link_to "Manage checks", checks_url %></p>

data/app/views/blazer/queries/_form.html.erb

@@ -126,6 +126,7 @@
   var error_line = null;
   var xhr;
   var params = <%= raw blazer_json_escape(variable_params.to_json) %>;
+  var previewStatement = <%= raw blazer_json_escape(Hash[Blazer.data_sources.map { |k, v| [k, v.preview_statement] }].to_json) %>;
 
   $("#run").click(function (e) {
     e.preventDefault();
@@ -162,7 +163,8 @@
   $(document).on("change", "#table_names", function () {
     var val = $(this).val();
     if (val.length > 0) {
-      editor.setValue("SELECT * FROM " + val + " LIMIT 10");
+      var dataSource = $("#query_data_source").val();
+      editor.setValue(previewStatement[dataSource].replace("{table}", val));
       $("#run").click();
     }
   });

data/app/views/blazer/queries/show.html.erb

@@ -172,13 +172,17 @@
   </script>
 <% end %>
 
-<script>
-  // do not highlight really long queries
-  // this can lead to performance issues
-  if ($("code").text().length < 10000) {
-    hljs.initHighlightingOnLoad();
-  }
+<% if Blazer.data_sources[@query.data_source].adapter == "activerecord" %>
+  <script>
+    // do not highlight really long queries
+    // this can lead to performance issues
+    if ($("code").text().length < 10000) {
+      hljs.initHighlightingOnLoad();
+    }
+  </script>
+<% end %>
 
+<script>
   $(".form-inline input, .form-inline select").change( function () {
     submitIfCompleted($(this).closest("form"));
   });

data/lib/blazer.rb

@@ -5,6 +5,11 @@ require "safely/core"
 require "blazer/version"
 require "blazer/data_source"
 require "blazer/result"
+require "blazer/run_statement"
+require "blazer/adapters/base_adapter"
+require "blazer/adapters/elasticsearch_adapter"
+require "blazer/adapters/mongodb_adapter"
+require "blazer/adapters/sql_adapter"
 require "blazer/engine"
 
 module Blazer

data/lib/blazer/adapters/base_adapter.rb

@@ -0,0 +1,41 @@
+module Blazer
+  module Adapters
+    class BaseAdapter
+      attr_reader :data_source
+
+      def initialize(data_source)
+        @data_source = data_source
+      end
+
+      def run_statement(statement, comment)
+        # the one required method
+      end
+
+      def tables
+        [] # optional, but nice to have
+      end
+
+      def preview_statement
+        "" # also optional, but nice to have
+      end
+
+      def reconnect
+        # optional
+      end
+
+      def cost(statement)
+        # optional
+      end
+
+      def explain(statement)
+        # optional
+      end
+
+      protected
+
+      def settings
+        @data_source.settings
+      end
+    end
+  end
+end

data/lib/blazer/adapters/elasticsearch_adapter.rb

@@ -0,0 +1,43 @@
+module Blazer
+  module Adapters
+    class ElasticsearchAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          header, body = statement.gsub(/\/\/.+/, "").strip.split("\n", 2)
+          response = client.msearch(body: [JSON.parse(header), JSON.parse(body)])["responses"].first
+          hits = response["hits"]["hits"]
+          source_keys = hits.flat_map { |r| r["_source"].keys }.uniq
+          hit_keys = (hits.first.try(:keys) || []) - ["_source"]
+          columns = source_keys + hit_keys
+          rows =
+            hits.map do |r|
+              source = r["_source"]
+              source_keys.map { |k| source[k] } + hit_keys.map { |k| r[k] }
+            end
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        client.indices.get_aliases.map { |k, v| [k, v["aliases"].keys] }.flatten.uniq.sort
+      end
+
+      def preview_statement
+        %!// header\n{"index": "{table}"}\n\n// body\n{"query": {"match_all": {}}, "size": 10}!
+      end
+
+      protected
+
+      def client
+        @client ||= Elasticsearch::Client.new(url: settings["url"])
+      end
+    end
+  end
+end

data/lib/blazer/adapters/mongodb_adapter.rb

@@ -0,0 +1,39 @@
+module Blazer
+  module Adapters
+    class MongodbAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          documents = db.command({:$eval => "#{statement.strip}.toArray()"}).documents.first["retval"]
+          columns = documents.flat_map { |r| r.keys }.uniq
+          rows = documents.map { |r| columns.map { |c| r[c] } }
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        db.collection_names
+      end
+
+      def preview_statement
+        "db.{table}.find().limit(10)"
+      end
+
+      protected
+
+      def client
+        @client ||= Mongo::Client.new(settings["url"])
+      end
+
+      def db
+        @db ||= client.database
+      end
+    end
+  end
+end

data/lib/blazer/adapters/sql_adapter.rb

@@ -0,0 +1,120 @@
+module Blazer
+  module Adapters
+    class SqlAdapter < BaseAdapter
+      attr_reader :connection_model
+
+      def initialize(data_source)
+        super
+
+        @connection_model =
+          Class.new(Blazer::Connection) do
+            def self.name
+              "Blazer::Connection::#{object_id}"
+            end
+            establish_connection(data_source.settings["url"]) if data_source.settings["url"]
+          end
+      end
+
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          in_transaction do
+            set_timeout(data_source.timeout) if data_source.timeout
+
+            result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
+            columns = result.columns
+            cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
+            result.rows.each do |untyped_row|
+              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(cast_method, untyped_row[i]) : nil })
+            end
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          error = e.message.sub(/.+ERROR: /, "")
+          error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        result = data_source.run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))
+        result.rows.map(&:first)
+      end
+
+      def preview_statement
+        "SELECT * FROM {table} LIMIT 10"
+      end
+
+      def reconnect
+        connection_model.establish_connection(settings["url"])
+      end
+
+      def cost(statement)
+        result = explain(statement)
+        match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
+        match[1] if match
+      end
+
+      def explain(statement)
+        if postgresql? || redshift?
+          connection_model.connection.select_all("EXPLAIN #{statement}").rows.first.first
+        end
+      rescue
+        nil
+      end
+
+      protected
+
+      def postgresql?
+        ["PostgreSQL", "PostGIS"].include?(adapter_name)
+      end
+
+      def redshift?
+        ["Redshift"].include?(adapter_name)
+      end
+
+      def mysql?
+        ["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
+      end
+
+      def adapter_name
+        connection_model.connection.adapter_name
+      end
+
+      def schemas
+        default_schema = (postgresql? || redshift?) ? "public" : connection_model.connection_config[:database]
+        settings["schemas"] || [connection_model.connection_config[:schema] || default_schema]
+      end
+
+      def set_timeout(timeout)
+        if postgresql? || redshift?
+          connection_model.connection.execute("SET statement_timeout = #{timeout.to_i * 1000}")
+        elsif mysql?
+          connection_model.connection.execute("SET max_execution_time = #{timeout.to_i * 1000}")
+        else
+          raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
+        end
+      end
+
+      def use_transaction?
+        settings.key?("use_transaction") ? settings["use_transaction"] : true
+      end
+
+      def in_transaction
+        connection_model.connection_pool.with_connection do
+          if use_transaction?
+            connection_model.transaction do
+              yield
+              raise ActiveRecord::Rollback
+            end
+          else
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/blazer/data_source.rb

@@ -2,7 +2,11 @@ require "digest/md5"
 
 module Blazer
   class DataSource
-    attr_reader :id, :settings, :connection_model
+    extend Forwardable
+
+    attr_reader :id, :settings, :adapter, :adapter_instance
+
+    def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain
 
     def initialize(id, settings)
       @id = id
@@ -12,15 +16,23 @@ module Blazer
         raise Blazer::Error, "Empty url"
       end
 
-      @connection_model =
-        Class.new(Blazer::Connection) do
-          def self.name
-            "Blazer::Connection::#{object_id}"
-          end
-          establish_connection(settings["url"]) if settings["url"]
+      @adapter_instance =
+        case adapter
+        when "sql"
+          Blazer::Adapters::SqlAdapter.new(self)
+        when "elasticsearch"
+          Blazer::Adapters::ElasticsearchAdapter.new(self)
+        when "mongodb"
+          Blazer::Adapters::MongodbAdapter.new(self)
+        else
+          raise Blazer::Error, "Unknown adapter"
         end
     end
 
+    def adapter
+      settings["adapter"] || detect_adapter
+    end
+
     def name
       settings["name"] || @id
     end
@@ -78,61 +90,6 @@ module Blazer
       @local_time_suffix ||= Array(settings["local_time_suffix"])
     end
 
-    def use_transaction?
-      settings.key?("use_transaction") ? settings["use_transaction"] : true
-    end
-
-    def cost(statement)
-      result = explain(statement)
-      match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
-      match[1] if match
-    end
-
-    def explain(statement)
-      if postgresql? || redshift?
-        connection_model.connection.select_all("EXPLAIN #{statement}").rows.first.first
-      end
-    rescue
-      nil
-    end
-
-    def run_main_statement(statement, options = {})
-      query = options[:query]
-      Blazer.transform_statement.call(self, statement) if Blazer.transform_statement
-
-      # audit
-      if Blazer.audit
-        audit = Blazer::Audit.new(statement: statement)
-        audit.query = query
-        audit.data_source = id
-        audit.user = options[:user]
-        audit.save!
-      end
-
-      start_time = Time.now
-      result = run_statement(statement, options)
-      duration = Time.now - start_time
-
-      if Blazer.audit
-        audit.duration = duration if audit.respond_to?(:duration=)
-        audit.error = result.error if audit.respond_to?(:error=)
-        audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
-        audit.cached = result.cached? if audit.respond_to?(:cached=)
-        if !result.cached? && duration >= 10
-          audit.cost = cost(statement) if audit.respond_to?(:cost=)
-        end
-        audit.save! if audit.changed?
-      end
-
-      if query && !result.timed_out?
-        query.checks.each do |check|
-          check.update_state(result)
-        end
-      end
-
-      result
-    end
-
     def read_cache(cache_key)
       value = Blazer.cache.read(cache_key)
       if value
@@ -192,70 +149,13 @@ module Blazer
       cache_key(["run", run_id])
     end
 
-    def schemas
-      default_schema = (postgresql? || redshift?) ? "public" : connection_model.connection_config[:database]
-      settings["schemas"] || [connection_model.connection_config[:schema] || default_schema]
-    end
-
-    def tables
-      result = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))
-      result.rows.map(&:first)
-    end
-
-    def postgresql?
-      ["PostgreSQL", "PostGIS"].include?(adapter_name)
-    end
-
-    def redshift?
-      ["Redshift"].include?(adapter_name)
-    end
-
-    def mysql?
-      ["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
-    end
-
-    def reconnect
-      connection_model.establish_connection(settings["url"])
-    end
-
     protected
 
     def run_statement_helper(statement, comment, run_id)
-      columns = []
-      rows = []
-      error = nil
       start_time = Time.now
-      result = nil
-
-      begin
-        in_transaction do
-          if timeout
-            if postgresql? || redshift?
-              connection_model.connection.execute("SET statement_timeout = #{timeout.to_i * 1000}")
-            elsif mysql?
-              connection_model.connection.execute("SET max_execution_time = #{timeout.to_i * 1000}")
-            else
-              raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
-            end
-          end
-
-          result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
-        end
-      rescue ActiveRecord::StatementInvalid => e
-        error = e.message.sub(/.+ERROR: /, "")
-        error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
-      end
-
+      columns, rows, error = @adapter_instance.run_statement(statement, comment)
       duration = Time.now - start_time
 
-      if result
-        columns = result.columns
-        cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
-        result.rows.each do |untyped_row|
-          rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(cast_method, untyped_row[i]) : nil })
-        end
-      end
-
       cache_data = nil
       cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
       if cache || run_id
@@ -277,18 +177,11 @@ module Blazer
       Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
     end
 
-    def adapter_name
-      connection_model.connection.adapter_name
-    end
-
-    def in_transaction
-      if use_transaction?
-        connection_model.transaction do
-          yield
-          raise ActiveRecord::Rollback
-        end
+    def detect_adapter
+      if settings["url"].to_s.start_with?("mongodb://")
+        "mongodb"
       else
-        yield
+        "sql"
       end
     end
   end

data/lib/blazer/run_statement.rb

@@ -0,0 +1,38 @@
+class RunStatement
+  def perform(data_source, statement, options = {})
+    query = options[:query]
+    Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
+
+    # audit
+    if Blazer.audit
+      audit = Blazer::Audit.new(statement: statement)
+      audit.query = query
+      audit.data_source = data_source.id
+      audit.user = options[:user]
+      audit.save!
+    end
+
+    start_time = Time.now
+    result = data_source.run_statement(statement, options)
+    duration = Time.now - start_time
+
+    if Blazer.audit
+      audit.duration = duration if audit.respond_to?(:duration=)
+      audit.error = result.error if audit.respond_to?(:error=)
+      audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
+      audit.cached = result.cached? if audit.respond_to?(:cached=)
+      if !result.cached? && duration >= 10
+        audit.cost = data_source.cost(statement) if audit.respond_to?(:cost=)
+      end
+      audit.save! if audit.changed?
+    end
+
+    if query && !result.timed_out?
+      query.checks.each do |check|
+        check.update_state(result)
+      end
+    end
+
+    result
+  end
+end

data/lib/blazer/run_statement_job.rb

@@ -6,10 +6,15 @@ module Blazer
     workers 4
 
     def perform(result, data_source, statement, options)
-      ActiveRecord::Base.connection_pool.with_connection do
-        data_source.connection_model.connection_pool.with_connection do
-          result << data_source.run_main_statement(statement, options)
+      begin
+        ActiveRecord::Base.connection_pool.with_connection do
+          result << RunStatement.new.perform(data_source, statement, options)
         end
+      rescue Exception => e
+        result.clear
+        result << Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
+        Blazer.cache.write(data_source.run_cache_key(options[:run_id]), Marshal.dump([[], [], "Unknown error", nil]), expires_in: 30.seconds)
+        raise e
       end
     end
   end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "1.5.1"
+  VERSION = "1.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-25 00:00:00.000000000 Z
+date: 2016-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -159,10 +159,15 @@ files:
 - blazer.gemspec
 - config/routes.rb
 - lib/blazer.rb
+- lib/blazer/adapters/base_adapter.rb
+- lib/blazer/adapters/elasticsearch_adapter.rb
+- lib/blazer/adapters/mongodb_adapter.rb
+- lib/blazer/adapters/sql_adapter.rb
 - lib/blazer/data_source.rb
 - lib/blazer/detect_anomalies.R
 - lib/blazer/engine.rb
 - lib/blazer/result.rb
+- lib/blazer/run_statement.rb
 - lib/blazer/run_statement_job.rb
 - lib/blazer/version.rb
 - lib/generators/blazer/install_generator.rb