blazer 1.7.9 → 1.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c856080eb1bc2dc870a499745d4eba632a36ff3
-  data.tar.gz: 39e040fd3054a54d5a46369646d416efab0e1b87
+  metadata.gz: f47130dee076e979384bf983f7faff420979a002
+  data.tar.gz: 3e0aa371bf99fb223b0db1bc5a85c97c51353cfc
 SHA512:
-  metadata.gz: d03c45609d8a09648f949f300571fe9ecd4c8f3c3ffbab1be6f1668efc22dc7e4892dcdc441a2dd9c0a018a40b797f6bd0283672bce17f205e80237cd465435d
-  data.tar.gz: 29f45987e40c2e0e03f005caf703231300ed1bc8ea55cf2344e84e313566ba5f1b683606f3bbe5eb187e5cac85dc90acfa144c971e3d951778100c3eae0523b2
+  metadata.gz: 248be46d44525be01771768364029b0e0376a368c433957ef53dacdebae45d7d5b9114a36a16d48398e63860b29050d9346f8c52153ef3308e65854aa54a7f85
+  data.tar.gz: a4db6ce496c4b7fd20adc85ffc16b792867833f2d1f1340f5fae73ba2f227520904defd0371c3d34cad6339180888d278a2822447fc69c5e74ab8ca06d97894f

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.7.10
+
+- Added support for Google BigQuery
+- Require `drill-sergeant` gem for Apache Drill
+- Better handling of checks with variables
+
 ## 1.7.9
 
 - Added beta support for Apache Drill

data/README.md

@@ -391,9 +391,10 @@ data_sources:
 - [SQLite](#sqlite)
 - [Redshift](#redshift)
 - [Presto](#presto)
-- [MongoDB](#mongodb-1) [beta]
+- [Apache Drill](#apache-drill)
+- [Google BigQuery](#google-bigquery-master)
+- [MongoDB](#mongodb-1)
 - [Elasticsearch](#elasticsearch) [beta]
-- [Apache Drill](#apache-drill-master) [beta]
 
 You can also [create an adapter](#creating-an-adapter) for any other data store.
 
@@ -473,36 +474,48 @@ data_sources:
     url: presto://user@hostname:8080/catalog
 ```
 
-### MongoDB
+### Apache Drill
 
-Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
+Add [drill-sergeant](https://github.com/ankane/drill-sergeant) to your Gemfile and set:
 
 ```yml
 data_sources:
   my_source:
-    url: mongodb://user:password@hostname:27017/database
+    adapter: drill
+    url: http://hostname:8047
 ```
 
-### Elasticsearch
+### Google BigQuery [master]
 
-Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) to your Gemfile and set:
+Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:
 
 ```yml
 data_sources:
   my_source:
-    adapter: elasticsearch
-    url: http://user:password@hostname:9200/
+    adapter: bigquery
+    project: your-project
+    keyfile: path/to/keyfile.json
 ```
 
-### Apache Drill [master]
+### MongoDB
 
-Set:
+Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
 
 ```yml
 data_sources:
   my_source:
-    adapter: drill
-    url: http://hostname:8047
+    url: mongodb://user:password@hostname:27017/database
+```
+
+### Elasticsearch
+
+Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) to your Gemfile and set:
+
+```yml
+data_sources:
+  my_source:
+    adapter: elasticsearch
+    url: http://user:password@hostname:9200
 ```
 
 ## Creating an Adapter

data/app/controllers/blazer/base_controller.rb

@@ -1,7 +1,7 @@
 module Blazer
   class BaseController < ApplicationController
-    # skip all filters
-    filters = _process_action_callbacks.map(&:filter)
+    # skip filters
+    filters = _process_action_callbacks.map(&:filter) - [:activate_authlogic]
     if Rails::VERSION::MAJOR >= 5
       skip_before_action(*filters, raise: false)
       skip_after_action(*filters, raise: false)
@@ -25,7 +25,7 @@ module Blazer
     private
 
       def process_vars(statement, data_source)
-        (@bind_vars ||= []).concat(extract_vars(statement)).uniq!
+        (@bind_vars ||= []).concat(Blazer.extract_vars(statement)).uniq!
         @bind_vars.each do |var|
           params[var] ||= Blazer.data_sources[data_source].variable_defaults[var]
         end
@@ -79,13 +79,6 @@ module Blazer
         [smart_var, error]
       end
 
-      def extract_vars(statement)
-        # strip commented out lines
-        # and regex {1} or {1,2}
-        statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
-      end
-      helper_method :extract_vars
-
       def variable_params
         params.except(:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id).permit!
       end

data/app/controllers/blazer/queries_controller.rb

@@ -279,7 +279,7 @@ module Blazer
               id: q.id,
               name: q.name,
               creator: blazer_user && q.try(:creator) == blazer_user ? "You" : q.try(:creator).try(Blazer.user_name),
-              vars: extract_vars(q.statement).join(", "),
+              vars: q.variables.join(", "),
               to_param: q.to_param
             }
           end

data/app/models/blazer/check.rb

@@ -5,6 +5,7 @@ module Blazer
 
     validates :query_id, presence: true
     validate :validate_emails
+    validate :validate_variables, if: -> { query_id_changed? }
 
     before_validation :set_state
     before_validation :fix_emails
@@ -84,5 +85,11 @@ module Blazer
           errors.add(:base, "Invalid emails")
         end
       end
+
+      def validate_variables
+        if query.variables.any?
+          errors.add(:base, "Query can't have variables")
+        end
+      end
   end
 end

data/app/models/blazer/query.rb

@@ -23,5 +23,9 @@ module Blazer
       editable &&= Blazer.query_editable.call(self, user) if Blazer.query_editable
       editable
     end
+
+    def variables
+      Blazer.extract_vars(statement)
+    end
   end
 end

data/app/views/blazer/queries/show.html.erb

@@ -62,7 +62,7 @@
   </script>
 <% end %>
 
-<% if %w[sql presto].include?(Blazer.data_sources[@query.data_source].adapter) %>
+<% if %w[sql presto drill bigquery].include?(Blazer.data_sources[@query.data_source].adapter) %>
   <script>
     // do not highlight really long queries
     // this can lead to performance issues

data/lib/blazer.rb

@@ -7,6 +7,7 @@ require "blazer/data_source"
 require "blazer/result"
 require "blazer/run_statement"
 require "blazer/adapters/base_adapter"
+require "blazer/adapters/bigquery_adapter"
 require "blazer/adapters/drill_adapter"
 require "blazer/adapters/elasticsearch_adapter"
 require "blazer/adapters/mongodb_adapter"
@@ -87,6 +88,12 @@ module Blazer
     end
   end
 
+  def self.extract_vars(statement)
+    # strip commented out lines
+    # and regex {1} or {1,2}
+    statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+  end
+
   def self.run_checks(schedule: nil)
     checks = Blazer::Check.includes(:query)
     checks = checks.where(schedule: schedule) if schedule
@@ -160,6 +167,7 @@ module Blazer
 end
 
 Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapter
 Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
 Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter

data/lib/blazer/adapters/bigquery_adapter.rb

@@ -0,0 +1,67 @@
+module Blazer
+  module Adapters
+    class BigQueryAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        begin
+          options = {}
+          options[:timeout] = data_source.timeout.to_i * 1000 if data_source.timeout
+          results = bigquery.query(statement, options) # ms
+          if results.complete?
+            columns = results.first.keys.map(&:to_s) if results.size > 0
+            rows = results.map(&:values)
+          else
+            error = Blazer::TIMEOUT_MESSAGE
+          end
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def tables
+        table_refs.map { |t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }
+      end
+
+      def schema
+        table_refs.map do |table_ref|
+          {
+            schema: table_ref.dataset_id,
+            table: table_ref.table_id,
+            columns: table_columns(table_ref)
+          }
+        end
+      end
+
+      def preview_statement
+        "SELECT * FROM `{table}` LIMIT 10"
+      end
+
+      private
+
+      def bigquery
+        @bigquery ||= begin
+          require "google/cloud/bigquery"
+          Google::Cloud::Bigquery.new(
+            project: settings["project"],
+            keyfile: settings["keyfile"]
+          )
+        end
+      end
+
+      def table_refs
+        bigquery.datasets.map(&:tables).flat_map { |table_list| table_list.map(&:table_ref) }
+      end
+
+      def table_columns(table_ref)
+        schema = bigquery.service.get_table(table_ref.dataset_id, table_ref.table_id).schema
+        return [] if schema.nil?
+        schema.fields.map { |field| {name: field.name, data_type: field.type} }
+      end
+    end
+  end
+end

data/lib/blazer/adapters/drill_adapter.rb

@@ -6,29 +6,23 @@ module Blazer
         rows = []
         error = nil
 
-        header = {"Content-Type" => "application/json", "Accept" => "application/json"}
-        data = {
-          queryType: "sql",
-          query: statement
-        }
-
-        uri = URI.parse("#{settings["url"]}/query.json")
-        http = Net::HTTP.new(uri.host, uri.port)
-
         begin
-          response = JSON.parse(http.post(uri.request_uri, data.to_json, header).body)
-          if response["errorMessage"]
-            error = response["errorMessage"]
-          else
-            columns = response["columns"]
-            rows = response["rows"].map { |r| r.values }
-          end
+          # remove trailing semicolon
+          response = drill.query(statement.sub(/;\s*\z/, ""))
+          rows = response.map { |r| r.values }
+          columns = rows.any? ? response.first.keys : []
         rescue => e
           error = e.message
         end
 
         [columns, rows, error]
       end
+
+      private
+
+      def drill
+        @drill ||= ::Drill.new(url: settings["url"])
+      end
     end
   end
 end

data/lib/blazer/data_source.rb

@@ -12,7 +12,7 @@ module Blazer
       @id = id
       @settings = settings
 
-      unless settings["url"] || Rails.env.development?
+      unless settings["url"] || Rails.env.development? || settings["adapter"] == "bigquery"
         raise Blazer::Error, "Empty url for data source: #{id}"
       end
 

data/lib/blazer/run_statement.rb

@@ -28,7 +28,7 @@ module Blazer
         audit.save! if audit.changed?
       end
 
-      if query && !result.timed_out?
+      if query && !result.timed_out? && !query.variables.any?
         query.checks.each do |check|
           check.update_state(result)
         end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "1.7.9"
+  VERSION = "1.7.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 1.7.9
+  version: 1.7.10
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-21 00:00:00.000000000 Z
+date: 2017-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -164,6 +164,7 @@ files:
 - config/routes.rb
 - lib/blazer.rb
 - lib/blazer/adapters/base_adapter.rb
+- lib/blazer/adapters/bigquery_adapter.rb
 - lib/blazer/adapters/drill_adapter.rb
 - lib/blazer/adapters/elasticsearch_adapter.rb
 - lib/blazer/adapters/mongodb_adapter.rb