railsblazer 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/.gitattributes +1 -0
  3. data/.github/ISSUE_TEMPLATE.md +0 -0
  4. data/.gitignore +14 -0
  5. data/CHANGELOG.md +247 -0
  6. data/CONTRIBUTING.md +42 -0
  7. data/Gemfile +4 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.md +855 -0
  10. data/Rakefile +1 -0
  11. data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
  12. data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
  13. data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
  14. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
  15. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
  16. data/app/assets/javascripts/blazer/Chart.js +14145 -0
  17. data/app/assets/javascripts/blazer/Sortable.js +1144 -0
  18. data/app/assets/javascripts/blazer/ace.js +6 -0
  19. data/app/assets/javascripts/blazer/ace/ace.js +11 -0
  20. data/app/assets/javascripts/blazer/ace/ext-language_tools.js +5 -0
  21. data/app/assets/javascripts/blazer/ace/mode-sql.js +1 -0
  22. data/app/assets/javascripts/blazer/ace/snippets/sql.js +1 -0
  23. data/app/assets/javascripts/blazer/ace/snippets/text.js +1 -0
  24. data/app/assets/javascripts/blazer/ace/theme-twilight.js +1 -0
  25. data/app/assets/javascripts/blazer/application.js +79 -0
  26. data/app/assets/javascripts/blazer/bootstrap.js +2366 -0
  27. data/app/assets/javascripts/blazer/chartkick.js +1693 -0
  28. data/app/assets/javascripts/blazer/daterangepicker.js +1505 -0
  29. data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
  30. data/app/assets/javascripts/blazer/highlight.pack.js +1 -0
  31. data/app/assets/javascripts/blazer/jquery.js +10308 -0
  32. data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +263 -0
  33. data/app/assets/javascripts/blazer/jquery_ujs.js +469 -0
  34. data/app/assets/javascripts/blazer/moment-timezone.js +1007 -0
  35. data/app/assets/javascripts/blazer/moment.js +3043 -0
  36. data/app/assets/javascripts/blazer/queries.js +110 -0
  37. data/app/assets/javascripts/blazer/routes.js +23 -0
  38. data/app/assets/javascripts/blazer/selectize.js +3667 -0
  39. data/app/assets/javascripts/blazer/stupidtable.js +114 -0
  40. data/app/assets/javascripts/blazer/vue.js +7515 -0
  41. data/app/assets/stylesheets/blazer/application.css +198 -0
  42. data/app/assets/stylesheets/blazer/bootstrap.css.erb +6202 -0
  43. data/app/assets/stylesheets/blazer/daterangepicker-bs3.css +375 -0
  44. data/app/assets/stylesheets/blazer/github.css +125 -0
  45. data/app/assets/stylesheets/blazer/selectize.default.css +387 -0
  46. data/app/controllers/blazer/base_controller.rb +113 -0
  47. data/app/controllers/blazer/checks_controller.rb +56 -0
  48. data/app/controllers/blazer/dashboards_controller.rb +105 -0
  49. data/app/controllers/blazer/queries_controller.rb +337 -0
  50. data/app/helpers/blazer/base_helper.rb +57 -0
  51. data/app/mailers/blazer/check_mailer.rb +27 -0
  52. data/app/mailers/blazer/slack_notifier.rb +76 -0
  53. data/app/models/blazer/audit.rb +6 -0
  54. data/app/models/blazer/check.rb +104 -0
  55. data/app/models/blazer/connection.rb +5 -0
  56. data/app/models/blazer/dashboard.rb +13 -0
  57. data/app/models/blazer/dashboard_query.rb +9 -0
  58. data/app/models/blazer/query.rb +40 -0
  59. data/app/models/blazer/record.rb +5 -0
  60. data/app/views/blazer/_nav.html.erb +16 -0
  61. data/app/views/blazer/_variables.html.erb +102 -0
  62. data/app/views/blazer/check_mailer/failing_checks.html.erb +6 -0
  63. data/app/views/blazer/check_mailer/state_change.html.erb +47 -0
  64. data/app/views/blazer/checks/_form.html.erb +79 -0
  65. data/app/views/blazer/checks/edit.html.erb +1 -0
  66. data/app/views/blazer/checks/index.html.erb +43 -0
  67. data/app/views/blazer/checks/new.html.erb +1 -0
  68. data/app/views/blazer/dashboards/_form.html.erb +76 -0
  69. data/app/views/blazer/dashboards/edit.html.erb +1 -0
  70. data/app/views/blazer/dashboards/new.html.erb +1 -0
  71. data/app/views/blazer/dashboards/show.html.erb +47 -0
  72. data/app/views/blazer/queries/_form.html.erb +240 -0
  73. data/app/views/blazer/queries/edit.html.erb +2 -0
  74. data/app/views/blazer/queries/home.html.erb +152 -0
  75. data/app/views/blazer/queries/new.html.erb +2 -0
  76. data/app/views/blazer/queries/run.html.erb +165 -0
  77. data/app/views/blazer/queries/schema.html.erb +20 -0
  78. data/app/views/blazer/queries/show.html.erb +73 -0
  79. data/app/views/layouts/blazer/application.html.erb +24 -0
  80. data/blazer-0.0.1.gem +0 -0
  81. data/blazer.gemspec +27 -0
  82. data/config/routes.rb +16 -0
  83. data/lib/blazer.rb +223 -0
  84. data/lib/blazer/adapters/athena_adapter.rb +128 -0
  85. data/lib/blazer/adapters/base_adapter.rb +53 -0
  86. data/lib/blazer/adapters/bigquery_adapter.rb +68 -0
  87. data/lib/blazer/adapters/cassandra_adapter.rb +59 -0
  88. data/lib/blazer/adapters/drill_adapter.rb +28 -0
  89. data/lib/blazer/adapters/druid_adapter.rb +67 -0
  90. data/lib/blazer/adapters/elasticsearch_adapter.rb +46 -0
  91. data/lib/blazer/adapters/mongodb_adapter.rb +39 -0
  92. data/lib/blazer/adapters/presto_adapter.rb +45 -0
  93. data/lib/blazer/adapters/snowflake_adapter.rb +73 -0
  94. data/lib/blazer/adapters/sql_adapter.rb +182 -0
  95. data/lib/blazer/data_source.rb +195 -0
  96. data/lib/blazer/detect_anomalies.R +19 -0
  97. data/lib/blazer/engine.rb +30 -0
  98. data/lib/blazer/result.rb +170 -0
  99. data/lib/blazer/run_statement.rb +40 -0
  100. data/lib/blazer/run_statement_job.rb +21 -0
  101. data/lib/blazer/version.rb +3 -0
  102. data/lib/generators/blazer/install_generator.rb +39 -0
  103. data/lib/generators/blazer/templates/config.yml.tt +62 -0
  104. data/lib/generators/blazer/templates/install.rb.tt +46 -0
  105. data/lib/tasks/blazer.rake +11 -0
  106. data/railsblazer-0.0.1.gem +0 -0
  107. metadata +234 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: c75b58c2decb2b11cfefb09394dd8574a842125c
4
+ data.tar.gz: c46c2437488327e52ac707efa50cbc44eee95d18
5
+ SHA512:
6
+ metadata.gz: 7e41ac21fc598ac9a181bf26e90b32d490fce07aed4d69ab42c61652fc63020befb6f51041482598f9567441b5ac166e5a8d07b0383b25c6ed07e7d9526d6e38
7
+ data.tar.gz: 56c434f8b503fc5742eb0f472209c943ed8b14108cc46eb61ba8e79c91f207c549ad15c9fafcb48f2e9d29f66e0de2c78c7351ddebda2af1a59edc23ae1841ca
@@ -0,0 +1 @@
1
+ app/assets/* linguist-vendored
File without changes
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
@@ -0,0 +1,247 @@
1
+ ## 1.9.1 [unreleased]
2
+
3
+ - Added support for Slack
4
+ - Added `override_csp` option
5
+ - Use SQL for Elasticsearch
6
+ - Fixed error with latest `google-cloud-bigquery`
7
+
8
+ ## 1.9.0
9
+
10
+ - Prompt developers to check custom `before_action`
11
+ - Better ordering on home page
12
+ - Added support for Snowflake
13
+
14
+ ## 1.8.2
15
+
16
+ - Added support for Cassandra
17
+ - Fixes for Druid
18
+
19
+ ## 1.8.1
20
+
21
+ - Added support for Amazon Athena
22
+ - Added support for Druid
23
+ - Fixed query cancellation
24
+
25
+ ## 1.8.0
26
+
27
+ - Added support for Rails 5.1
28
+
29
+ ## 1.7.10
30
+
31
+ - Added support for Google BigQuery
32
+ - Require `drill-sergeant` gem for Apache Drill
33
+ - Better handling of checks with variables
34
+
35
+ ## 1.7.9
36
+
37
+ - Added beta support for Apache Drill
38
+ - Added email validation for checks
39
+ - Updated Chart.js to 2.5.0
40
+
41
+ ## 1.7.8
42
+
43
+ - Added support for custom adapters
44
+ - Fixed bug with scatter charts on dashboards
45
+ - Fixed table preview for SQL Server
46
+ - Fixed issue when `default_url_options` set
47
+
48
+ ## 1.7.7
49
+
50
+ - Fixed preview error for MySQL
51
+ - Fixed error with timeouts for MySQL
52
+
53
+ ## 1.7.6
54
+
55
+ - Added scatter chart
56
+ - Fixed issue with false values showing up blank
57
+ - Fixed preview for table names with certain characters
58
+
59
+ ## 1.7.5
60
+
61
+ - Fixed issue with check emails sometimes failing for default Rails 5 ActiveJob adapter
62
+ - Fixed sorting for new dashboards
63
+
64
+ ## 1.7.4
65
+
66
+ - Removed extra dependencies added in 1.7.1
67
+ - Fixed `send_failing_checks` for default Rails 5 ActiveJob adapter
68
+
69
+ ## 1.7.3
70
+
71
+ - Fixed JavaScript errors
72
+ - Fixed query cancel error
73
+ - Return search results for "me" or "mine"
74
+ - Include sample data in email when bad data checks fail
75
+ - Fixed deprecation warnings
76
+
77
+ ## 1.7.2
78
+
79
+ - Cancel all queries on page nav
80
+ - Prevent Ace from taking over find command
81
+ - Added ability to use hashes for smart columns
82
+ - Added ability to inherit smart variables and columns from other data sources
83
+
84
+ ## 1.7.1
85
+
86
+ - Do not fork when enter key pressed
87
+ - Use custom version of Chart.js to fix label overlap
88
+ - Improved performance of home page
89
+
90
+ ## 1.7.0
91
+
92
+ - Added ability to cancel queries on backend for Postgres and Redshift
93
+ - Only run 3 queries at a time on dashboards
94
+ - Better anomaly detection
95
+ - Attempt to reconnect when connection issues
96
+ - Fixed issues with caching
97
+
98
+ ## 1.6.2
99
+
100
+ - Added basic query permissions
101
+ - Added ability to use arrays and hashes for smart variables
102
+ - Added cancel button for queries
103
+ - Added `lat` and `lng` as map keys
104
+
105
+ ## 1.6.1
106
+
107
+ - Added support for Presto [beta]
108
+ - Added support for Elasticsearch timeouts
109
+ - Fixed error in Rails 5
110
+
111
+ ## 1.6.0
112
+
113
+ - Added support for MongoDB [beta]
114
+ - Added support for Elasticsearch [beta]
115
+ - Fixed deprecation warning in Rails 5
116
+
117
+ ## 1.5.1
118
+
119
+ - Added anomaly detection for data less than 2 weeks
120
+ - Added autolinking urls
121
+ - Added support for images
122
+
123
+ ## 1.5.0
124
+
125
+ - Added new bar chart format
126
+ - Added anomaly detection checks
127
+ - Added `async` option for polling
128
+
129
+ ## 1.4.0
130
+
131
+ - Added `slow` cache mode
132
+ - Fixed `BLAZER_DATABASE_URL required` error
133
+ - Fixed issue with duplicate column names
134
+
135
+ ## 1.3.5
136
+
137
+ - Fixed error with checks
138
+
139
+ ## 1.3.4
140
+
141
+ - Fixed issue with missing queries
142
+
143
+ ## 1.3.3
144
+
145
+ - Fixed error with Rails 4.1 and below
146
+
147
+ ## 1.3.2
148
+
149
+ - Added support for Rails 5
150
+ - Attempt to reconnect for checks
151
+
152
+ ## 1.3.1
153
+
154
+ - Fixed migration error
155
+
156
+ ## 1.3.0
157
+
158
+ - Added schedule for checks
159
+ - Switched to Chart.js for charts
160
+ - Better output for explain
161
+ - Support for MySQL timeouts
162
+ - Raise error when timeout not supported
163
+ - Added creator to dashboards and checks
164
+
165
+ ## 1.2.1
166
+
167
+ - Fixed checks
168
+
169
+ ## 1.2.0
170
+
171
+ - Added non-editable queries
172
+ - Added variable defaults
173
+ - Added `local_time_suffix` setting
174
+ - Better timeout message
175
+ - Hide variables from commented out lines
176
+ - Fixed regex as variable names
177
+
178
+ ## 1.1.1
179
+
180
+ - Added `before_action` option
181
+ - Added invert option for checks
182
+ - Added targets
183
+ - Friendlier error message for timeouts
184
+ - Fixed request URI too large
185
+ - Prevent accidental backspace nav on query page
186
+
187
+ ## 1.1.0
188
+
189
+ - Replaced pie charts with column charts
190
+ - Fixed error with datepicker
191
+ - Added fork button to edit query page
192
+ - Added a notice when editing a query that is part of a dashboard
193
+ - Added refresh for dashboards
194
+
195
+ ## 1.0.4
196
+
197
+ - Added recently viewed queries and dashboards to home page
198
+ - Fixed refresh when transform statement is used
199
+ - Fixed error when no user model
200
+
201
+ ## 1.0.3
202
+
203
+ - Added maps
204
+ - Added support for Rails 4.0
205
+
206
+ ## 1.0.2
207
+
208
+ - Fixed error when installing
209
+ - Added `schemas` option
210
+
211
+ ## 1.0.1
212
+
213
+ - Added comments to queries
214
+ - Added `cache` option
215
+ - Added `user_method` option
216
+ - Added `use_transaction` option
217
+
218
+ ## 1.0.0
219
+
220
+ - Added support for multiple data sources
221
+ - Added dashboards
222
+ - Added checks
223
+ - Added support for Redshift
224
+
225
+ ## 0.0.8
226
+
227
+ - Easier to edit queries with variables
228
+ - Dynamically expand editor height as needed
229
+ - No need for spaces in search
230
+
231
+ ## 0.0.7
232
+
233
+ - Fixed error when no `User` class
234
+ - Fixed forking a query with variables
235
+ - Set time zone after Rails initializes
236
+
237
+ ## 0.0.6
238
+
239
+ - Added fork button
240
+ - Fixed trending
241
+ - Fixed time zones for date select
242
+
243
+ ## 0.0.5
244
+
245
+ - Added support for Rails 4.2
246
+ - Fixed error with `mysql2` adapter
247
+ - Added `user_class` option
@@ -0,0 +1,42 @@
1
+ # Contributing
2
+
3
+ First, thanks for wanting to contribute. You’re awesome! :heart:
4
+
5
+ ## Help
6
+
7
+ We’re not able to provide support through GitHub Issues. If you’re looking for help with your code, try posting on [Stack Overflow](https://stackoverflow.com/).
8
+
9
+ All features should be documented. If you don’t see a feature in the docs, assume it doesn’t exist.
10
+
11
+ ## Bugs
12
+
13
+ Think you’ve discovered a bug?
14
+
15
+ 1. Search existing issues to see if it’s been reported.
16
+ 2. Try the `master` branch to make sure it hasn’t been fixed.
17
+
18
+ ```rb
19
+ gem "blazer", github: "ankane/blazer"
20
+ ```
21
+
22
+ If the above steps don’t help, create an issue. Include:
23
+
24
+ - Detailed steps to reproduce
25
+ - Complete backtraces for exceptions
26
+
27
+ ## New Features
28
+
29
+ If you’d like to discuss a new feature, create an issue and start the title with `[Idea]`.
30
+
31
+ ## Pull Requests
32
+
33
+ Fork the project and create a pull request. A few tips:
34
+
35
+ - Keep changes to a minimum. If you have multiple features or fixes, submit multiple pull requests.
36
+ - Follow the existing style. The code should read like it’s written by a single person.
37
+
38
+ Feel free to open an issue to get feedback on your idea before spending too much time on it.
39
+
40
+ ---
41
+
42
+ This contributing guide is released under [CCO](https://creativecommons.org/publicdomain/zero/1.0/) (public domain). Use it for your own project without attribution.
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014-2018 Andrew Kane
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,855 @@
1
+ ### Note: This is a just extention of existing blazer gem with the features I wanted.
2
+
3
+ # Blazer
4
+
5
+ Explore your data with SQL. Easily create charts and dashboards, and share them with your team.
6
+
7
+ [Try it out](https://blazer.dokkuapp.com)
8
+
9
+ [![Screenshot](https://blazer.dokkuapp.com/assets/screenshot-6ca3115a518b488026e48be83ba0d4c9.png)](https://blazer.dokkuapp.com)
10
+
11
+ :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
12
+
13
+ ## Features
14
+
15
+ - **Multiple data sources** - PostgreSQL, MySQL, Redshift, and [many more](#full-list)
16
+ - **Variables** - run the same queries with different values
17
+ - **Checks & alerts** - get emailed when bad data appears
18
+ - **Audits** - all queries are tracked
19
+ - **Security** - works with your authentication system
20
+
21
+ ## Docs
22
+
23
+ - [Installation](#installation)
24
+ - [Queries](#queries)
25
+ - [Charts](#charts)
26
+ - [Dashboards](#dashboards)
27
+ - [Checks](#checks)
28
+
29
+ ## Installation
30
+
31
+ Add this line to your application’s Gemfile:
32
+
33
+ ```ruby
34
+ gem 'blazer'
35
+ ```
36
+
37
+ Run:
38
+
39
+ ```sh
40
+ rails generate blazer:install
41
+ rails db:migrate
42
+ ```
43
+
44
+ And mount the dashboard in your `config/routes.rb`:
45
+
46
+ ```ruby
47
+ mount Blazer::Engine, at: "blazer"
48
+ ```
49
+
50
+ For production, specify your database:
51
+
52
+ ```ruby
53
+ ENV["BLAZER_DATABASE_URL"] = "postgres://user:password@hostname:5432/database"
54
+ ```
55
+
56
+ Blazer tries to protect against queries which modify data (by running each query in a transaction and rolling it back), but a safer approach is to use a read only user. [See how to create one](#permissions).
57
+
58
+ #### Checks (optional)
59
+
60
+ Be sure to set a host in `config/environments/production.rb` for emails to work.
61
+
62
+ ```ruby
63
+ config.action_mailer.default_url_options = {host: "blazer.dokkuapp.com"}
64
+ ```
65
+
66
+ Schedule checks to run (with cron, [Heroku Scheduler](https://elements.heroku.com/addons/scheduler), etc). The default options are every 5 minutes, 1 hour, or 1 day, which you can customize. For each of these options, set up a task to run.
67
+
68
+ ```sh
69
+ rake blazer:run_checks SCHEDULE="5 minutes"
70
+ rake blazer:run_checks SCHEDULE="1 hour"
71
+ rake blazer:run_checks SCHEDULE="1 day"
72
+ ```
73
+
74
+ You can also set up failing checks to be sent once a day (or whatever you prefer).
75
+
76
+ ```sh
77
+ rake blazer:send_failing_checks
78
+ ```
79
+
80
+ Here’s what it looks like with cron.
81
+
82
+ ```
83
+ */5 * * * * rake blazer:run_checks SCHEDULE="5 minutes"
84
+ 0 * * * * rake blazer:run_checks SCHEDULE="1 hour"
85
+ 30 7 * * * rake blazer:run_checks SCHEDULE="1 day"
86
+ 0 8 * * * rake blazer:send_failing_checks
87
+ ```
88
+
89
+ For Slack notifications, create an [incoming webhook](https://slack.com/apps/A0F7XDUAZ-incoming-webhooks) and set: [master]
90
+
91
+ ```sh
92
+ BLAZER_SLACK_WEBHOOK_URL=https://hooks.slack.com/...
93
+ ```
94
+
95
+ Name the webhook “Blazer” and add a cool icon.
96
+
97
+ ## Permissions
98
+
99
+ ### PostgreSQL
100
+
101
+ Create a user with read only permissions:
102
+
103
+ ```sql
104
+ BEGIN;
105
+ CREATE ROLE blazer LOGIN PASSWORD 'secret123';
106
+ GRANT CONNECT ON DATABASE database_name TO blazer;
107
+ GRANT USAGE ON SCHEMA public TO blazer;
108
+ GRANT SELECT ON ALL TABLES IN SCHEMA public TO blazer;
109
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO blazer;
110
+ COMMIT;
111
+ ```
112
+
113
+ ### MySQL
114
+
115
+ Create a user with read only permissions:
116
+
117
+ ```sql
118
+ GRANT SELECT, SHOW VIEW ON database_name.* TO blazer@’127.0.0.1′ IDENTIFIED BY ‘secret123‘;
119
+ FLUSH PRIVILEGES;
120
+ ```
121
+
122
+ ### MongoDB
123
+
124
+ Create a user with read only permissions:
125
+
126
+ ```
127
+ db.createUser({user: "blazer", pwd: "password", roles: ["read"]})
128
+ ```
129
+
130
+ Also, make sure authorization is enabled when you start the server.
131
+
132
+ ### Sensitive Data
133
+
134
+ Check out [Hypershield](https://github.com/ankane/hypershield) to shield sensitive data.
135
+
136
+ ## Authentication
137
+
138
+ Don’t forget to protect the dashboard in production.
139
+
140
+ ### Basic Authentication
141
+
142
+ Set the following variables in your environment or an initializer.
143
+
144
+ ```ruby
145
+ ENV["BLAZER_USERNAME"] = "andrew"
146
+ ENV["BLAZER_PASSWORD"] = "secret"
147
+ ```
148
+
149
+ ### Devise
150
+
151
+ ```ruby
152
+ authenticate :user, ->(user) { user.admin? } do
153
+ mount Blazer::Engine, at: "blazer"
154
+ end
155
+ ```
156
+
157
+ ### Other
158
+
159
+ Specify a `before_action` method to run in `blazer.yml`.
160
+
161
+ ```yml
162
+ before_action_method: require_admin
163
+ ```
164
+
165
+ You can define this method in your `ApplicationController`.
166
+
167
+ ```ruby
168
+ def require_admin
169
+ # depending on your auth, something like...
170
+ redirect_to root_path unless current_user && current_user.admin?
171
+ end
172
+ ```
173
+
174
+ Be sure to render or redirect for unauthorized users.
175
+
176
+ ## Queries
177
+
178
+ ### Variables
179
+
180
+ Create queries with variables.
181
+
182
+ ```sql
183
+ SELECT * FROM users WHERE gender = {gender}
184
+ ```
185
+
186
+ Use `{start_time}` and `{end_time}` for time ranges. [Example](https://blazer.dokkuapp.com/queries/9-time-range-selector?start_time=1997-10-03T05%3A00%3A00%2B00%3A00&end_time=1997-10-04T04%3A59%3A59%2B00%3A00)
187
+
188
+ ```sql
189
+ SELECT * FROM ratings WHERE rated_at >= {start_time} AND rated_at <= {end_time}
190
+ ```
191
+
192
+ ### Smart Variables
193
+
194
+ [Example](https://blazer.dokkuapp.com/queries/1-smart-variable)
195
+
196
+ Suppose you have the query:
197
+
198
+ ```sql
199
+ SELECT * FROM users WHERE occupation_id = {occupation_id}
200
+ ```
201
+
202
+ Instead of remembering each occupation’s id, users can select occupations by name.
203
+
204
+ Add a smart variable with:
205
+
206
+ ```yml
207
+ smart_variables:
208
+ occupation_id: "SELECT id, name FROM occupations ORDER BY name ASC"
209
+ ```
210
+
211
+ The first column is the value of the variable, and the second column is the label.
212
+
213
+ You can also use an array or hash for static data and enums.
214
+
215
+ ```yml
216
+ smart_variables:
217
+ period: ["day", "week", "month"]
218
+ status: {0: "Active", 1: "Archived"}
219
+ ```
220
+
221
+ ### Linked Columns
222
+
223
+ [Example](https://blazer.dokkuapp.com/queries/3-linked-column) - title column
224
+
225
+ Link results to other pages in your apps or around the web. Specify a column name and where it should link to. You can use the value of the result with `{value}`.
226
+
227
+ ```yml
228
+ linked_columns:
229
+ user_id: "/admin/users/{value}"
230
+ ip_address: "https://www.infosniper.net/index.php?ip_address={value}"
231
+ ```
232
+
233
+ ### Smart Columns
234
+
235
+ [Example](https://blazer.dokkuapp.com/queries/2-smart-column) - occupation_id column
236
+
237
+ Suppose you have the query:
238
+
239
+ ```sql
240
+ SELECT name, city_id FROM users
241
+ ```
242
+
243
+ See which city the user belongs to without a join.
244
+
245
+ ```yml
246
+ smart_columns:
247
+ city_id: "SELECT id, name FROM cities WHERE id IN {value}"
248
+ ```
249
+
250
+ You can also use a hash for static data and enums.
251
+
252
+ ```yml
253
+ smart_columns:
254
+ status: {0: "Active", 1: "Archived"}
255
+ ```
256
+
257
+ ### Caching
258
+
259
+ Blazer can automatically cache results to improve speed. It can cache slow queries:
260
+
261
+ ```yml
262
+ cache:
263
+ mode: slow
264
+ expires_in: 60 # min
265
+ slow_threshold: 15 # sec
266
+ ```
267
+
268
+ Or it can cache all queries:
269
+
270
+ ```yml
271
+ cache:
272
+ mode: all
273
+ expires_in: 60 # min
274
+ ```
275
+
276
+ Of course, you can force a refresh at any time.
277
+
278
+ ## Charts
279
+
280
+ Blazer will automatically generate charts based on the types of the columns returned in your query.
281
+
282
+ **Note:** The order of columns matters.
283
+
284
+ ### Line Chart
285
+
286
+ There are two ways to generate line charts.
287
+
288
+ 2+ columns - timestamp, numeric(s) - [Example](https://blazer.dokkuapp.com/queries/4-line-chart-format-1)
289
+
290
+ ```sql
291
+ SELECT date_trunc('week', created_at), COUNT(*) FROM users GROUP BY 1
292
+ ```
293
+
294
+ 3 columns - timestamp, string, numeric - [Example](https://blazer.dokkuapp.com/queries/5-line-chart-format-2)
295
+
296
+
297
+ ```sql
298
+ SELECT date_trunc('week', created_at), gender, COUNT(*) FROM users GROUP BY 1, 2
299
+ ```
300
+
301
+ ### Column Chart
302
+
303
+ There are also two ways to generate column charts.
304
+
305
+ 2+ columns - string, numeric(s) - [Example](https://blazer.dokkuapp.com/queries/6-column-chart-format-1)
306
+
307
+ ```sql
308
+ SELECT gender, COUNT(*) FROM users GROUP BY 1
309
+ ```
310
+
311
+ 3 columns - string, string, numeric - [Example](https://blazer.dokkuapp.com/queries/7-column-chart-format-2)
312
+
313
+ ```sql
314
+ SELECT gender, zip_code, COUNT(*) FROM users GROUP BY 1, 2
315
+ ```
316
+
317
+ ### Scatter Chart
318
+
319
+ 2 columns - both numeric
320
+
321
+ ```sql
322
+ SELECT x, y FROM table
323
+ ```
324
+
325
+ ### Maps
326
+
327
+ Columns named `latitude` and `longitude` or `lat` and `lon` or `lat` and `lng` - [Example](https://blazer.dokkuapp.com/queries/15-map)
328
+
329
+ ```sql
330
+ SELECT name, latitude, longitude FROM cities
331
+ ```
332
+
333
+ To enable, get an access token from [Mapbox](https://www.mapbox.com/) and set `ENV["MAPBOX_ACCESS_TOKEN"]`.
334
+
335
+ ### Targets
336
+
337
+ Use the column name `target` to draw a line for goals. [Example](https://blazer.dokkuapp.com/queries/8-target-line)
338
+
339
+ ```sql
340
+ SELECT date_trunc('week', created_at), COUNT(*) AS new_users, 100000 AS target FROM users GROUP BY 1
341
+ ```
342
+
343
+ ## Dashboards
344
+
345
+ Create a dashboard with multiple queries. [Example](https://blazer.dokkuapp.com/dashboards/1-dashboard-demo)
346
+
347
+ If the query has a chart, the chart is shown. Otherwise, you’ll see a table.
348
+
349
+ If any queries have variables, they will show up on the dashboard.
350
+
351
+ ## Checks
352
+
353
+ Checks give you a centralized place to see the health of your data. [Example](https://blazer.dokkuapp.com/checks)
354
+
355
+ Create a query to identify bad rows.
356
+
357
+ ```sql
358
+ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
359
+ ```
360
+
361
+ Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
362
+
363
+ ## Anomaly Detection
364
+
365
+ Anomaly detection is supported thanks to Twitter’s [AnomalyDetection](https://github.com/twitter/AnomalyDetection) library.
366
+
367
+ First, [install R](https://cloud.r-project.org/). Then, run:
368
+
369
+ ```R
370
+ install.packages("devtools")
371
+ devtools::install_github("twitter/AnomalyDetection")
372
+ ```
373
+
374
+ And add to `config/blazer.yml`:
375
+
376
+ ```yml
377
+ anomaly_checks: true
378
+ ```
379
+
380
+ If upgrading from version 1.4 or below, also follow the [upgrade instructions](#15).
381
+
382
+ If you’re on Heroku, follow [these additional instructions](#anomaly-detection-on-heroku).
383
+
384
+ ## Data Sources
385
+
386
+ Blazer supports multiple data sources :tada:
387
+
388
+ Add additional data sources in `config/blazer.yml`:
389
+
390
+ ```yml
391
+ data_sources:
392
+ main:
393
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
394
+ # timeout, smart_variables, linked_columns, smart_columns
395
+ catalog:
396
+ url: <%= ENV["CATALOG_DATABASE_URL"] %>
397
+ # ...
398
+ redshift:
399
+ url: <%= ENV["REDSHIFT_DATABASE_URL"] %>
400
+ # ...
401
+ ```
402
+
403
+ ### Full List
404
+
405
+ - [Amazon Athena](#amazon-athena)
406
+ - [Amazon Redshift](#amazon-redshift)
407
+ - [Apache Drill](#apache-drill)
408
+ - [Cassandra](#cassandra)
409
+ - [Druid](#druid)
410
+ - [Elasticsearch](#elasticsearch) [beta]
411
+ - [Google BigQuery](#google-bigquery)
412
+ - [IBM DB2 and Informix](#ibm-db2-and-informix)
413
+ - [MongoDB](#mongodb-1)
414
+ - [MySQL](#mysql-1)
415
+ - [Oracle](#oracle)
416
+ - [PostgreSQL](#postgresql-1)
417
+ - [Presto](#presto)
418
+ - [Snowflake](#snowflake)
419
+ - [SQLite](#sqlite)
420
+ - [SQL Server](#sql-server)
421
+
422
+ You can also [create an adapter](#creating-an-adapter) for any other data store.
423
+
424
+ **Note:** In the examples below, we recommend using environment variables for urls.
425
+
426
+ ```yml
427
+ data_sources:
428
+ my_source:
429
+ url: <%= ENV["BLAZER_MY_SOURCE_URL"] %>
430
+ ```
431
+
432
+ ### Amazon Athena
433
+
434
+ Add [aws-sdk-athena](https://github.com/aws/aws-sdk-ruby) and [aws-sdk-glue](https://github.com/aws/aws-sdk-ruby) to your Gemfile and set:
435
+
436
+ ```yml
437
+ data_sources:
438
+ my_source:
439
+ adapter: athena
440
+ database: database
441
+ output_location: s3://some-bucket/
442
+ ```
443
+
444
+ ### Amazon Redshift
445
+
446
+ Add [activerecord4-redshift-adapter](https://github.com/aamine/activerecord4-redshift-adapter) or [activerecord5-redshift-adapter](https://github.com/ConsultingMD/activerecord5-redshift-adapter) to your Gemfile and set:
447
+
448
+ ```yml
449
+ data_sources:
450
+ my_source:
451
+ url: redshift://user:password@hostname:5439/database
452
+ ```
453
+
454
+ ### Apache Drill
455
+
456
+ Add [drill-sergeant](https://github.com/ankane/drill-sergeant) to your Gemfile and set:
457
+
458
+ ```yml
459
+ data_sources:
460
+ my_source:
461
+ adapter: drill
462
+ url: http://hostname:8047
463
+ ```
464
+
465
+ ### Cassandra
466
+
467
+ Add [cassandra-driver](https://github.com/datastax/ruby-driver) to your Gemfile and set:
468
+
469
+ ```yml
470
+ data_sources:
471
+ my_source:
472
+ url: cassandra://user:password@hostname:9042/keyspace
473
+ ```
474
+
475
+ ### Druid
476
+
477
+ First, [enable SQL support](http://druid.io/docs/latest/querying/sql.html#configuration) on the broker.
478
+
479
+ Set:
480
+
481
+ ```yml
482
+ data_sources:
483
+ my_source:
484
+ adapter: druid
485
+ url: http://hostname:8082
486
+ ```
487
+
488
+ ### Elasticsearch
489
+
490
+ Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) and [elasticsearch-xpack](https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-xpack) to your Gemfile and set:
491
+
492
+ ```yml
493
+ data_sources:
494
+ my_source:
495
+ adapter: elasticsearch
496
+ url: http://user:password@hostname:9200
497
+ ```
498
+
499
+ ### Google BigQuery
500
+
501
+ Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:
502
+
503
+ ```yml
504
+ data_sources:
505
+ my_source:
506
+ adapter: bigquery
507
+ project: your-project
508
+ keyfile: path/to/keyfile.json
509
+ ```
510
+
511
+ ### IBM DB2 and Informix
512
+
513
+ Use [ibm_db](https://github.com/ibmdb/ruby-ibmdb).
514
+
515
+ ### MongoDB
516
+
517
+ Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
518
+
519
+ ```yml
520
+ data_sources:
521
+ my_source:
522
+ url: mongodb://user:password@hostname:27017/database
523
+ ```
524
+
525
+ ### MySQL
526
+
527
+ Add [mysql2](https://github.com/brianmario/mysql2) to your Gemfile (if it’s not there) and set:
528
+
529
+ ```yml
530
+ data_sources:
531
+ my_source:
532
+ url: mysql2://user:password@hostname:3306/database
533
+ ```
534
+
535
+ ### Oracle
536
+
537
+ Use [activerecord-oracle_enhanced-adapter](https://github.com/rsim/oracle-enhanced).
538
+
539
+ ### PostgreSQL
540
+
541
+ Add [pg](https://bitbucket.org/ged/ruby-pg/wiki/Home) to your Gemfile (if it’s not there) and set:
542
+
543
+ ```yml
544
+ data_sources:
545
+ my_source:
546
+ url: postgres://user:password@hostname:5432/database
547
+ ```
548
+
549
+ ### Presto
550
+
551
+ Add [presto-client](https://github.com/treasure-data/presto-client-ruby) to your Gemfile and set:
552
+
553
+ ```yml
554
+ data_sources:
555
+ my_source:
556
+ url: presto://user@hostname:8080/catalog
557
+ ```
558
+
559
+ ### Snowflake
560
+
561
+ First, install the [ODBC driver](https://docs.snowflake.net/manuals/user-guide/odbc.html). Add [odbc_adapter](https://github.com/localytics/odbc_adapter) to your Gemfile and set:
562
+
563
+ ```yml
564
+ data_sources:
565
+ my_source:
566
+ adapter: snowflake
567
+ dsn: ProductionSnowflake
568
+ ```
569
+
570
+ ### SQLite
571
+
572
+ Add [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) to your Gemfile and set:
573
+
574
+ ```yml
575
+ data_sources:
576
+ my_source:
577
+ url: sqlite3:path/to/database.sqlite3
578
+ ```
579
+
580
+ ### SQL Server
581
+
582
+ Add [tiny_tds](https://github.com/rails-sqlserver/tiny_tds) and [activerecord-sqlserver-adapter](https://github.com/rails-sqlserver/activerecord-sqlserver-adapter) to your Gemfile and set:
583
+
584
+ ```yml
585
+ data_sources:
586
+ my_source:
587
+ url: sqlserver://user:password@hostname:1433/database
588
+ ```
589
+
590
+ ## Creating an Adapter
591
+
592
+ Create an adapter for any data store with:
593
+
594
+ ```ruby
595
+ class FooAdapter < Blazer::Adapters::BaseAdapter
596
+ # code goes here
597
+ end
598
+
599
+ Blazer.register_adapter "foo", FooAdapter
600
+ ```
601
+
602
+ See the [Presto adapter](https://github.com/ankane/blazer/blob/master/lib/blazer/adapters/presto_adapter.rb) for a good example. Then use:
603
+
604
+ ```yml
605
+ data_sources:
606
+ my_source:
607
+ adapter: foo
608
+ url: http://user:password@hostname:9200/
609
+ ```
610
+
611
+ ## Query Permissions
612
+
613
+ Blazer supports a basic permissions model.
614
+
615
+ 1. Queries without a name are unlisted
616
+ 2. Queries whose name starts with `#` are only listed to the creator
617
+ 3. Queries whose name starts with `*` can only be edited by the creator
618
+
619
+ ## Learn SQL
620
+
621
+ Have team members who want to learn SQL? Here are a few great, free resources.
622
+
623
+ - [Khan Academy](https://www.khanacademy.org/computing/computer-programming/sql)
624
+ - [Codecademy](https://www.codecademy.com/learn/learn-sql)
625
+
626
+ ## Useful Tools
627
+
628
+ For an easy way to group by day, week, month, and more with correct time zones, check out [Groupdate](https://github.com/ankane/groupdate.sql).
629
+
630
+ ## Standalone Version
631
+
632
+ Looking for a standalone version? Check out [Ghost Blazer](https://github.com/buren/ghost_blazer).
633
+
634
+ ## Anomaly Detection on Heroku
635
+
636
+ Add the [R buildpack](https://github.com/virtualstaticvoid/heroku-buildpack-r) to your app.
637
+
638
+ ```sh
639
+ heroku buildpacks:add --index 1 https://github.com/virtualstaticvoid/heroku-buildpack-r.git\#cedar-14
640
+ ```
641
+
642
+ And create an `init.r` with:
643
+
644
+ ```sh
645
+ if (!"AnomalyDetection" %in% installed.packages()) {
646
+ install.packages("devtools")
647
+ devtools::install_github("twitter/AnomalyDetection")
648
+ }
649
+ ```
650
+
651
+ Commit and deploy away. The first deploy may take a few minutes.
652
+
653
+ ## Content Security Policy [master]
654
+
655
+ If views are stuck with a `Loading...` message, there might be a problem with strict CSP settings in your app. This can be checked with Firefox or Chrome dev tools. You can allow Blazer to override these settings for its controllers with:
656
+
657
+ ```yml
658
+ override_csp: true
659
+ ```
660
+
661
+ ## Assets Config
662
+ You can change assets path with following config:
663
+
664
+ ```yml
665
+ assets_prefix: "/my/custome/path"
666
+ ```
667
+
668
+ ## Upgrading
669
+
670
+ ### 1.9.1 [master]
671
+
672
+ To use Slack notifications, create a migration
673
+
674
+ ```sh
675
+ rails g migration add_slack_channels_to_blazer_checks
676
+ ```
677
+
678
+ with:
679
+
680
+ ```ruby
681
+ add_column :blazer_checks, :slack_channels, :text
682
+ ```
683
+
684
+ ### 1.5
685
+
686
+ To take advantage of the anomaly detection, create a migration
687
+
688
+ ```sh
689
+ rails g migration upgrade_blazer_to_1_5
690
+ ```
691
+
692
+ with:
693
+
694
+ ```ruby
695
+ add_column :blazer_checks, :check_type, :string
696
+ add_column :blazer_checks, :message, :text
697
+ commit_db_transaction
698
+
699
+ Blazer::Check.reset_column_information
700
+
701
+ Blazer::Check.where(invert: true).update_all(check_type: "missing_data")
702
+ Blazer::Check.where(check_type: nil).update_all(check_type: "bad_data")
703
+ ```
704
+
705
+ ### 1.3
706
+
707
+ To take advantage of the latest features, create a migration
708
+
709
+ ```sh
710
+ rails g migration upgrade_blazer_to_1_3
711
+ ```
712
+
713
+ with:
714
+
715
+ ```ruby
716
+ add_column :blazer_dashboards, :creator_id, :integer
717
+ add_column :blazer_checks, :creator_id, :integer
718
+ add_column :blazer_checks, :invert, :boolean
719
+ add_column :blazer_checks, :schedule, :string
720
+ add_column :blazer_checks, :last_run_at, :timestamp
721
+ commit_db_transaction
722
+
723
+ Blazer::Check.update_all schedule: "1 hour"
724
+ ```
725
+
726
+ ### 1.0
727
+
728
+ Blazer 1.0 brings a number of new features:
729
+
730
+ - multiple data sources, including Redshift
731
+ - dashboards
732
+ - checks
733
+
734
+ To upgrade, run:
735
+
736
+ ```sh
737
+ bundle update blazer
738
+ ```
739
+
740
+ Create a migration
741
+
742
+ ```sh
743
+ rails g migration upgrade_blazer_to_1_0
744
+ ```
745
+
746
+ with:
747
+
748
+ ```ruby
749
+ add_column :blazer_queries, :data_source, :string
750
+ add_column :blazer_audits, :data_source, :string
751
+
752
+ create_table :blazer_dashboards do |t|
753
+ t.text :name
754
+ t.timestamps
755
+ end
756
+
757
+ create_table :blazer_dashboard_queries do |t|
758
+ t.references :dashboard
759
+ t.references :query
760
+ t.integer :position
761
+ t.timestamps
762
+ end
763
+
764
+ create_table :blazer_checks do |t|
765
+ t.references :query
766
+ t.string :state
767
+ t.text :emails
768
+ t.timestamps
769
+ end
770
+ ```
771
+
772
+ And run:
773
+
774
+ ```sh
775
+ rails db:migrate
776
+ ```
777
+
778
+ Update `config/blazer.yml` with:
779
+
780
+ ```yml
781
+ # see https://github.com/ankane/blazer for more info
782
+
783
+ data_sources:
784
+ main:
785
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
786
+
787
+ # statement timeout, in seconds
788
+ # applies to PostgreSQL only
789
+ # none by default
790
+ # timeout: 15
791
+
792
+ # time to cache results, in minutes
793
+ # can greatly improve speed
794
+ # none by default
795
+ # cache: 60
796
+
797
+ # wrap queries in a transaction for safety
798
+ # not necessary if you use a read-only user
799
+ # true by default
800
+ # use_transaction: false
801
+
802
+ smart_variables:
803
+ # zone_id: "SELECT id, name FROM zones ORDER BY name ASC"
804
+
805
+ linked_columns:
806
+ # user_id: "/admin/users/{value}"
807
+
808
+ smart_columns:
809
+ # user_id: "SELECT id, name FROM users WHERE id IN {value}"
810
+
811
+ # create audits
812
+ audit: true
813
+
814
+ # change the time zone
815
+ # time_zone: "Pacific Time (US & Canada)"
816
+
817
+ # class name of the user model
818
+ # user_class: User
819
+
820
+ # method name for the current user
821
+ # user_method: current_user
822
+
823
+ # method name for the display name
824
+ # user_name: name
825
+
826
+ # email to send checks from
827
+ # from_email: blazer@example.org
828
+ ```
829
+
830
+ ## TODO
831
+
832
+ - advanced permissions
833
+ - standalone version
834
+ - better navigation
835
+
836
+ ## History
837
+
838
+ View the [changelog](https://github.com/ankane/blazer/blob/master/CHANGELOG.md)
839
+
840
+ ## Thanks
841
+
842
+ Blazer uses a number of awesome open source projects, including [Rails](https://github.com/rails/rails/), [Vue.js](https://github.com/vuejs/vue), [jQuery](https://github.com/jquery/jquery), [Bootstrap](https://github.com/twbs/bootstrap), [Selectize](https://github.com/brianreavis/selectize.js), [StickyTableHeaders](https://github.com/jmosbech/StickyTableHeaders), [Stupid jQuery Table Sort](https://github.com/joequery/Stupid-Table-Plugin), and [Date Range Picker](https://github.com/dangrossman/bootstrap-daterangepicker).
843
+
844
+ Demo data from [MovieLens](https://grouplens.org/datasets/movielens/).
845
+
846
+ ## Want to Make Blazer Better?
847
+
848
+ That’s awesome! Here are a few ways you can help:
849
+
850
+ - [Report bugs](https://github.com/ankane/blazer/issues)
851
+ - Fix bugs and [submit pull requests](https://github.com/ankane/blazer/pulls)
852
+ - Write, clarify, or fix documentation
853
+ - Suggest or add new features
854
+
855
+ Check out the [dev app](https://github.com/ankane/blazer-dev) to get started.