blazer 2.2.1 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eedba409f17c85953be66d2b295f1857fb61de146d5c88288509440f419853e7
-  data.tar.gz: ad0f8f433ad5be3052eb0237f86121c6a59147b2e240fb2a41d68611e69d3022
+  metadata.gz: 10aff02b2f4ebe93012af611e329c3a0a0be23c208d89d46db940c1bd21a97ed
+  data.tar.gz: b629339fcc448a079ce4759623a8906c9e4977435419ec2109c86a34d354f70a
 SHA512:
-  metadata.gz: 8e4236c254472fb2bdc9a8cb107306d7263a11ade5b8f223cefc2363431f6f9ffeb7087d373a95823bf53112c9869871d132764232754898f8740c72ea01ceea
-  data.tar.gz: bb12b6473ef4702adea12a9847ef020e0f8277d9714816cbfc511262345bffbbc8aa5530a7310dd4a263f634a493286f446bc8c4db48428f6c396109b2dce474
+  metadata.gz: ac3263567e689625741e0fa03a46fff1c1e8100f184245e130ed2e7b9b301173d5e8e6472a1fe6d5d7fb4e9bbc162ae2f3b4eeab5a14efc9bc7b59b605d96735
+  data.tar.gz: 800fa336c7c998345e0f1d047496ae7c6d39095714953d65a3ef5286b376da029fe885ae997afccb7f4adc86a3371e6658565e8bdd80cfb7bd7cb352fe64df72

data/CHANGELOG.md

@@ -1,27 +1,33 @@
-## 2.2.1
+## 2.2.2 (2020-04-13)
+
+- Added experimental support for the Socrata Open Data API (SODA)
+- Added experimental Prophet forecasting
+- Fixed query search for non-ASCII characters
+
+## 2.2.1 (2019-10-08)
 
 - Added support for Sprockets 4
 - Improved Snowflake table preview
 - Fixed bug with refresh link not showing
 
-## 2.2.0
+## 2.2.0 (2019-07-12)
 
 - Added schema to table preview for Postgres and Redshift
 - Fixed bug with Slack notifications not sending
 - Dropped support for Rails 4.2
 
-## 2.1.0
+## 2.1.0 (2019-06-04)
 
 - Require latest Chartkick to prevent possible XSS - see [#245](https://github.com/ankane/blazer/issues/245)
 
-## 2.0.2
+## 2.0.2 (2019-05-26)
 
 - Added support for variable transformation for blind indexing
 - Added experimental support for Neo4j
 - Added experimental support for Salesforce
 - Fixed JavaScript sorting for numbers with commas
 
-## 2.0.1
+## 2.0.1 (2019-01-07)
 
 - Added favicon
 - Added search for checks and schema
@@ -32,7 +38,7 @@
 - Improved docs for new installs
 - Fixed error with canceling queries
 
-## 2.0.0
+## 2.0.0 (2019-01-03)
 
 - Added support for Slack
 - Added `async` option
@@ -45,68 +51,67 @@ Breaking changes
 
 - Dropped support for Rails < 4.2
 
-## 1.9.0
+## 1.9.0 (2018-06-17)
 
 - Prompt developers to check custom `before_action`
 - Better ordering on home page
 - Added support for Snowflake
 
-## 1.8.2
+## 1.8.2 (2018-02-22)
 
 - Added support for Cassandra
 - Fixes for Druid
-
-## 1.8.1
-
 - Added support for Amazon Athena
 - Added support for Druid
 - Fixed query cancellation
 
-## 1.8.0
+There was no 1.8.1 release.
+
+## 1.8.0 (2017-05-01)
 
 - Added support for Rails 5.1
 
-## 1.7.10
+## 1.7.10 (2017-04-03)
 
 - Added support for Google BigQuery
 - Require `drill-sergeant` gem for Apache Drill
 - Better handling of checks with variables
 
-## 1.7.9
+## 1.7.9 (2017-03-20)
 
 - Added beta support for Apache Drill
 - Added email validation for checks
 - Updated Chart.js to 2.5.0
 
-## 1.7.8
+## 1.7.8 (2017-02-06)
 
 - Added support for custom adapters
 - Fixed bug with scatter charts on dashboards
 - Fixed table preview for SQL Server
 - Fixed issue when `default_url_options` set
 
-## 1.7.7
+## 1.7.7 (2016-12-17)
 
 - Fixed preview error for MySQL
 - Fixed error with timeouts for MySQL
 
-## 1.7.6
+## 1.7.6 (2016-12-13)
 
 - Added scatter chart
 - Fixed issue with false values showing up blank
 - Fixed preview for table names with certain characters
 
-## 1.7.5
+## 1.7.5 (2016-11-22)
 
 - Fixed issue with check emails sometimes failing for default Rails 5 ActiveJob adapter
 - Fixed sorting for new dashboards
 
-## 1.7.4
+## 1.7.4 (2016-11-06)
 
 - Removed extra dependencies added in 1.7.1
 - Fixed `send_failing_checks` for default Rails 5 ActiveJob adapter
 
-## 1.7.3
+## 1.7.3 (2016-11-01)
 
 - Fixed JavaScript errors
 - Fixed query cancel error
@@ -114,20 +119,20 @@ Breaking changes
 - Include sample data in email when bad data checks fail
 - Fixed deprecation warnings
 
-## 1.7.2
+## 1.7.2 (2016-10-30)
 
 - Cancel all queries on page nav
 - Prevent Ace from taking over find command
 - Added ability to use hashes for smart columns
 - Added ability to inherit smart variables and columns from other data sources
 
-## 1.7.1
+## 1.7.1 (2016-10-29)
 
 - Do not fork when enter key pressed
 - Use custom version of Chart.js to fix label overlap
 - Improved performance of home page
 
-## 1.7.0
+## 1.7.0 (2016-09-07)
 
 - Added ability to cancel queries on backend for Postgres and Redshift
 - Only run 3 queries at a time on dashboards
@@ -135,65 +140,65 @@ Breaking changes
 - Attempt to reconnect when connection issues
 - Fixed issues with caching
 
-## 1.6.2
+## 1.6.2 (2016-08-11)
 
 - Added basic query permissions
 - Added ability to use arrays and hashes for smart variables
 - Added cancel button for queries
 - Added `lat` and `lng` as map keys
 
-## 1.6.1
+## 1.6.1 (2016-07-30)
 
 - Added support for Presto [beta]
 - Added support for Elasticsearch timeouts
 - Fixed error in Rails 5
 
-## 1.6.0
+## 1.6.0 (2016-07-28)
 
 - Added support for MongoDB [beta]
 - Added support for Elasticsearch [beta]
 - Fixed deprecation warning in Rails 5
 
-## 1.5.1
+## 1.5.1 (2016-07-24)
 
 - Added anomaly detection for data less than 2 weeks
 - Added autolinking urls
 - Added support for images
 
-## 1.5.0
+## 1.5.0 (2016-06-29)
 
 - Added new bar chart format
 - Added anomaly detection checks
 - Added `async` option for polling
 
-## 1.4.0
+## 1.4.0 (2016-06-09)
 
 - Added `slow` cache mode
 - Fixed `BLAZER_DATABASE_URL required` error
 - Fixed issue with duplicate column names
 
-## 1.3.5
+## 1.3.5 (2016-05-11)
 
 - Fixed error with checks
 
-## 1.3.4
+## 1.3.4 (2016-05-11)
 
 - Fixed issue with missing queries
 
-## 1.3.3
+## 1.3.3 (2016-05-08)
 
 - Fixed error with Rails 4.1 and below
 
-## 1.3.2
+## 1.3.2 (2016-05-07)
 
 - Added support for Rails 5
 - Attempt to reconnect for checks
 
-## 1.3.1
+## 1.3.1 (2016-05-06)
 
 - Fixed migration error
 
-## 1.3.0
+## 1.3.0 (2016-05-06)
 
 - Added schedule for checks
 - Switched to Chart.js for charts
@@ -202,11 +207,11 @@ Breaking changes
 - Raise error when timeout not supported
 - Added creator to dashboards and checks
 
-## 1.2.1
+## 1.2.1 (2016-04-26)
 
 - Fixed checks
 
-## 1.2.0
+## 1.2.0 (2016-03-22)
 
 - Added non-editable queries
 - Added variable defaults
@@ -215,7 +220,7 @@ Breaking changes
 - Hide variables from commented out lines
 - Fixed regex as variable names
 
-## 1.1.1
+## 1.1.1 (2016-03-06)
 
 - Added `before_action` option
 - Added invert option for checks
@@ -224,7 +229,7 @@ Breaking changes
 - Fixed request URI too large
 - Prevent accidental backspace nav on query page
 
-## 1.1.0
+## 1.1.0 (2015-12-27)
 
 - Replaced pie charts with column charts
 - Fixed error with datepicker
@@ -232,55 +237,55 @@ Breaking changes
 - Added a notice when editing a query that is part of a dashboard
 - Added refresh for dashboards
 
-## 1.0.4
+## 1.0.4 (2015-11-04)
 
 - Added recently viewed queries and dashboards to home page
 - Fixed refresh when transform statement is used
 - Fixed error when no user model
 
-## 1.0.3
+## 1.0.3 (2015-10-18)
 
 - Added maps
 - Added support for Rails 4.0
 
-## 1.0.2
+## 1.0.2 (2015-10-11)
 
 - Fixed error when installing
 - Added `schemas` option
 
-## 1.0.1
+## 1.0.1 (2015-10-08)
 
 - Added comments to queries
 - Added `cache` option
 - Added `user_method` option
 - Added `use_transaction` option
 
-## 1.0.0
+## 1.0.0 (2015-10-04)
 
 - Added support for multiple data sources
 - Added dashboards
 - Added checks
 - Added support for Redshift
 
-## 0.0.8
+## 0.0.8 (2015-09-05)
 
 - Easier to edit queries with variables
 - Dynamically expand editor height as needed
 - No need for spaces in search
 
-## 0.0.7
+## 0.0.7 (2015-07-23)
 
 - Fixed error when no `User` class
 - Fixed forking a query with variables
 - Set time zone after Rails initializes
 
-## 0.0.6
+## 0.0.6 (2015-06-18)
 
 - Added fork button
 - Fixed trending
 - Fixed time zones for date select
 
-## 0.0.5
+## 0.0.5 (2015-01-31)
 
 - Added support for Rails 4.2
 - Fixed error with `mysql2` adapter

data/README.md

@@ -4,9 +4,9 @@ Explore your data with SQL. Easily create charts and dashboards, and share them
 
 [Try it out](https://blazer.dokkuapp.com)
 
-[![Screenshot](https://blazer.dokkuapp.com/assets/screenshot-6ca3115a518b488026e48be83ba0d4c9.png)](https://blazer.dokkuapp.com)
+[![Screenshot](https://blazer.dokkuapp.com/assets/blazer-90fb6ce8ad25614c6f9c3b4619cbfbd66fa3d6567dac34d83df540f6688665f1.png)](https://blazer.dokkuapp.com)
 
-Blazer 2.0 was recently released! See [instructions for upgrading](#20).
+Blazer is also available as a [Docker image](https://github.com/ankane/blazer-docker).
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
@@ -426,10 +426,32 @@ If you’re on Heroku, follow [these additional instructions](#anomaly-detection
 
 ## Forecasting
 
-Blazer has experimental support for forecasting through [Trend](https://trendapi.org/).
+Blazer has experimental support for two different forecasting methods.
+
+A forecast link will appear for queries that return 2 columns with types timestamp and numeric.
 
 [Example](https://blazer.dokkuapp.com/queries/18-forecast?forecast=t)
 
+### Prophet
+
+**Note:** Prophet only supports daily forecasts right now.
+
+Add [prophet](https://github.com/ankane/prophet) to your Gemfile:
+
+```ruby
+gem 'prophet-rb'
+```
+
+And add to `config/blazer.yml`:
+
+```yml
+forecasting: prophet
+```
+
+### Trend
+
+[Trend](https://trendapi.org/) uses an external service.
+
 Add [trend](https://github.com/ankane/trend) to your Gemfile:
 
 ```ruby
@@ -442,8 +464,6 @@ And add to `config/blazer.yml`:
 forecasting: trend
 ```
 
-A forecast link will appear for queries that return 2 columns with types timestamp and numeric.
-
 ## Data Sources
 
 Blazer supports multiple data sources :tada:
@@ -475,11 +495,12 @@ data_sources:
 - [IBM DB2 and Informix](#ibm-db2-and-informix)
 - [MongoDB](#mongodb-1)
 - [MySQL](#mysql-1)
-- [Neo4j](#neo4j-experimental)
+- [Neo4j](#neo4j)
 - [Oracle](#oracle)
 - [PostgreSQL](#postgresql-1)
 - [Presto](#presto)
-- [Salesforce](#salesforce-experimental)
+- [Salesforce](#salesforce)
+- [Socrata Open Data API (SODA)](#socrata-open-data-api-soda)
 - [Snowflake](#snowflake)
 - [SQLite](#sqlite)
 - [SQL Server](#sql-server)
@@ -595,7 +616,9 @@ data_sources:
     url: mysql2://user:password@hostname:3306/database
 ```
 
-### Neo4j [experimental]
+### Neo4j
+
+*Experimental*
 
 Add [neo4j-core](https://github.com/neo4jrb/neo4j-core) to your Gemfile and set:
 
@@ -630,7 +653,9 @@ data_sources:
     url: presto://user@hostname:8080/catalog
 ```
 
-### Salesforce [experimental]
+### Salesforce
+
+*Experimental*
 
 Add [restforce](https://github.com/restforce/restforce) to your Gemfile and set:
 
@@ -653,6 +678,22 @@ SALESFORCE_API_VERSION="41.0"
 
 Supports [SOQL](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)
 
+### Socrata Open Data API (SODA)
+
+*Experimental [master]*
+
+Set:
+
+```yml
+data_sources:
+  my_source:
+    adapter: soda
+    url: https://soda.demo.socrata.com/resource/4tka-6guv.json
+    app_token: ...
+```
+
+Supports [SoQL](https://dev.socrata.com/docs/functions/)
+
 ### Snowflake
 
 First, install ODBC. For Homebrew, use:

data/app/helpers/blazer/base_helper.rb

@@ -14,7 +14,7 @@ module Blazer
     def blazer_format_value(key, value)
       if value.is_a?(Numeric) && !key.to_s.end_with?("id") && !key.to_s.start_with?("id")
         number_with_delimiter(value)
-      elsif value =~ BLAZER_URL_REGEX
+      elsif value.is_a?(String) && value =~ BLAZER_URL_REGEX
         # see if image or link
         if Blazer.images && (key.include?("image") || BLAZER_IMAGE_EXT.include?(value.split(".").last.split("?").first.try(:downcase)))
           link_to value, target: "_blank" do

data/app/views/blazer/queries/home.html.erb

@@ -71,7 +71,7 @@
   }
 
   var prepareQuery = function (str) {
-    return str.toLowerCase().replace(/\W+/g, "")
+    return str.toLowerCase()
   }
 
   var app = new Vue({

data/app/views/blazer/queries/run.html.erb

@@ -80,7 +80,7 @@
         <%= blazer_js_var "mapboxAccessToken", Blazer.mapbox_access_token %>
         <%= blazer_js_var "markers", @markers %>
         L.mapbox.accessToken = mapboxAccessToken;
-        var map = L.mapbox.map('map', 'ankane.ioo8nki0');
+        var map = L.mapbox.map('map', 'mapbox.streets');
         var featureLayer = L.mapbox.featureLayer().addTo(map);
         var geojson = [];
         for (var i = 0; i < markers.length; i++) {

data/app/views/layouts/blazer/application.html.erb

@@ -11,8 +11,8 @@
       <%= blazer_js_var "rootPath", root_path %>
     </script>
     <% if blazer_maps? %>
-      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v3.1.1/mapbox.css", integrity: "sha384-o8tecBIfqi9yU5yYK2Ne/9A9hlOGFV9MBvCpmemvJH1XxqOe6h8Bl4mLxMi6PgjG", crossorigin: "anonymous" %>
-      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v3.1.1/mapbox.js", integrity: "sha384-j81LqvtvYigFzGSUgAoFijpvoq4yGoCJSOXI9DFaUEpenR029MBE3E/X5Gr+WdO0", crossorigin: "anonymous" %>
+      <%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v3.2.1/mapbox.css", integrity: "sha384-vxzdEt+wZRPNQbhChjmiaFMLWg86IGuq1NGDehJHsD2mphYkxXll/eSs16WWi6Dq", crossorigin: "anonymous" %>
+      <%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v3.2.1/mapbox.js", integrity: "sha384-dRMjy8ZlKk6czEwpaK3yySKItGPpGOplDM1+osgTwGhmxeTPpjB4kmRbDr2Y7lfQ", crossorigin: "anonymous" %>
     <% end %>
     <%= csrf_meta_tags %>
   </head>

data/lib/blazer.rb

@@ -22,6 +22,7 @@ require "blazer/adapters/mongodb_adapter"
 require "blazer/adapters/neo4j_adapter"
 require "blazer/adapters/presto_adapter"
 require "blazer/adapters/salesforce_adapter"
+require "blazer/adapters/soda_adapter"
 require "blazer/adapters/sql_adapter"
 require "blazer/adapters/snowflake_adapter"
 
@@ -223,5 +224,6 @@ Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
 Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
 Blazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapter
 Blazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter
+Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapter
 Blazer.register_adapter "sql", Blazer::Adapters::SqlAdapter
 Blazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapter

data/lib/blazer/adapters/soda_adapter.rb

@@ -0,0 +1,96 @@
+module Blazer
+  module Adapters
+    class SodaAdapter < BaseAdapter
+      def run_statement(statement, comment)
+        columns = []
+        rows = []
+        error = nil
+
+        # remove comments manually
+        statement = statement.gsub(/--.+/, "")
+        # only supports single line /* */ comments
+        # regex not perfect, but should be good enough
+        statement = statement.gsub(/\/\*.+\*\//, "")
+
+        # remove trailing semicolon
+        statement = statement.sub(/;\s*\z/, "")
+
+        # remove whitespace
+        statement = statement.squish
+
+        uri = URI(settings["url"])
+        uri.query = URI.encode_www_form("$query" => statement)
+
+        req = Net::HTTP::Get.new(uri)
+        req["X-App-Token"] = settings["app_token"] if settings["app_token"]
+
+        options = {
+          use_ssl: uri.scheme == "https",
+          open_timeout: 3,
+          read_timeout: 30
+        }
+
+        begin
+          # use Net::HTTP instead of soda-ruby for types and better error messages
+          res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|
+            http.request(req)
+          end
+
+          if res.is_a?(Net::HTTPSuccess)
+            body = JSON.parse(res.body)
+
+            columns = JSON.parse(res["x-soda2-fields"])
+            column_types = columns.zip(JSON.parse(res["x-soda2-types"])).to_h
+
+            columns.reject! { |f| f.start_with?(":@") }
+            # rows can be missing some keys in JSON, so need to map by column
+            rows = body.map { |r| columns.map { |c| r[c] } }
+
+            columns.each_with_index do |column, i|
+              # nothing to do for boolean
+              case column_types[column]
+              when "number"
+                # check if likely an integer column
+                if rows.all? { |r| r[i].to_i == r[i].to_f }
+                  rows.each do |row|
+                    row[i] = row[i].to_i
+                  end
+                else
+                  rows.each do |row|
+                    row[i] = row[i].to_f
+                  end
+                end
+              when "floating_timestamp"
+                # check if likely a date column
+                if rows.all? { |r| r[i].end_with?("T00:00:00.000") }
+                  rows.each do |row|
+                    row[i] = Date.parse(row[i])
+                  end
+                else
+                  utc = ActiveSupport::TimeZone["Etc/UTC"]
+                  rows.each do |row|
+                    row[i] = utc.parse(row[i])
+                  end
+                end
+              end
+            end
+          else
+            error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"
+          end
+        rescue => e
+          error = e.message
+        end
+
+        [columns, rows, error]
+      end
+
+      def preview_statement
+        "SELECT * LIMIT 10"
+      end
+
+      def tables
+        ["all"]
+      end
+    end
+  end
+end

data/lib/blazer/result.rb

@@ -85,11 +85,45 @@ module Blazer
       @forecastable ||= Blazer.forecasting && column_types == ["time", "numeric"] && @rows.size >= 10
     end
 
+    # TODO cache it?
+    # don't want to put result data (even hashed version)
+    # into cache without developer opt-in
     def forecast
-      # TODO cache it?
-      # don't want to put result data (even hashed version)
-      # into cache without developer opt-in
-      forecast = Trend.forecast(Hash[@rows], count: 30)
+      count = (@rows.size * 0.25).round.clamp(30, 365)
+
+      case Blazer.forecasting
+      when "prophet"
+        require "prophet"
+
+        df =
+          Daru::DataFrame.new(
+            "ds" => @rows.map { |r| r[0] },
+            "y" => @rows.map { |r| r[1] }
+          )
+
+        # TODO determine frequency
+        freq = "D"
+
+        m = Prophet.new
+        m.fit(df)
+        future = m.make_future_dataframe(periods: count, freq: freq, include_history: false)
+        fcst = m.predict(future)
+        ds = fcst["ds"]
+        if @rows[0][0].is_a?(Date)
+          ds = ds.map { |v| v.to_date }
+        end
+        forecast = ds.zip(fcst["yhat"]).to_h
+      else
+        require "trend"
+
+        forecast = Trend.forecast(Hash[@rows], count: count)
+      end
+
+      # round integers
+      if @rows[0][1].is_a?(Integer)
+        forecast = forecast.map { |k, v| [k, v.round] }.to_h
+      end
+
       @rows.each do |row|
         row[2] = nil
       end

data/lib/blazer/version.rb

@@ -1,3 +1,3 @@
 module Blazer
-  VERSION = "2.2.1"
+  VERSION = "2.2.2"
 end

data/lib/generators/blazer/templates/config.yml.tt

@@ -67,7 +67,7 @@ check_schedules:
 
 # enable forecasting
 # note: with trend, time series are sent to https://trendapi.org
-# forecasting: trend
+# forecasting: trend / prophet
 
 # enable map
 # mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blazer
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.2.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-09 00:00:00.000000000 Z
+date: 2020-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -190,6 +190,7 @@ files:
 - lib/blazer/adapters/presto_adapter.rb
 - lib/blazer/adapters/salesforce_adapter.rb
 - lib/blazer/adapters/snowflake_adapter.rb
+- lib/blazer/adapters/soda_adapter.rb
 - lib/blazer/adapters/sql_adapter.rb
 - lib/blazer/data_source.rb
 - lib/blazer/detect_anomalies.R
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Explore your data with SQL. Easily create charts and dashboards, and share