sql-jarvis 1.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +14 -0
  3. data/CHANGELOG.md +228 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +775 -0
  7. data/Rakefile +1 -0
  8. data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
  9. data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
  10. data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
  11. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
  12. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
  13. data/app/assets/javascripts/blazer/Chart.js +14145 -0
  14. data/app/assets/javascripts/blazer/Sortable.js +1144 -0
  15. data/app/assets/javascripts/blazer/ace.js +6 -0
  16. data/app/assets/javascripts/blazer/ace/ace.js +11 -0
  17. data/app/assets/javascripts/blazer/ace/ext-language_tools.js +5 -0
  18. data/app/assets/javascripts/blazer/ace/mode-sql.js +1 -0
  19. data/app/assets/javascripts/blazer/ace/snippets/sql.js +1 -0
  20. data/app/assets/javascripts/blazer/ace/snippets/text.js +1 -0
  21. data/app/assets/javascripts/blazer/ace/theme-twilight.js +1 -0
  22. data/app/assets/javascripts/blazer/application.js +79 -0
  23. data/app/assets/javascripts/blazer/bootstrap.js +2366 -0
  24. data/app/assets/javascripts/blazer/chartkick.js +1693 -0
  25. data/app/assets/javascripts/blazer/daterangepicker.js +1505 -0
  26. data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
  27. data/app/assets/javascripts/blazer/highlight.pack.js +1 -0
  28. data/app/assets/javascripts/blazer/jquery.js +10308 -0
  29. data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +263 -0
  30. data/app/assets/javascripts/blazer/jquery_ujs.js +469 -0
  31. data/app/assets/javascripts/blazer/moment-timezone.js +1007 -0
  32. data/app/assets/javascripts/blazer/moment.js +3043 -0
  33. data/app/assets/javascripts/blazer/queries.js +110 -0
  34. data/app/assets/javascripts/blazer/routes.js +23 -0
  35. data/app/assets/javascripts/blazer/selectize.js +3667 -0
  36. data/app/assets/javascripts/blazer/stupidtable.js +114 -0
  37. data/app/assets/javascripts/blazer/vue.js +7515 -0
  38. data/app/assets/stylesheets/blazer/application.css +198 -0
  39. data/app/assets/stylesheets/blazer/bootstrap.css.erb +6202 -0
  40. data/app/assets/stylesheets/blazer/daterangepicker-bs3.css +375 -0
  41. data/app/assets/stylesheets/blazer/github.css +125 -0
  42. data/app/assets/stylesheets/blazer/selectize.default.css +387 -0
  43. data/app/controllers/blazer/base_controller.rb +103 -0
  44. data/app/controllers/blazer/checks_controller.rb +56 -0
  45. data/app/controllers/blazer/dashboards_controller.rb +105 -0
  46. data/app/controllers/blazer/queries_controller.rb +325 -0
  47. data/app/helpers/blazer/base_helper.rb +57 -0
  48. data/app/mailers/blazer/check_mailer.rb +27 -0
  49. data/app/models/blazer/audit.rb +6 -0
  50. data/app/models/blazer/check.rb +95 -0
  51. data/app/models/blazer/connection.rb +5 -0
  52. data/app/models/blazer/dashboard.rb +13 -0
  53. data/app/models/blazer/dashboard_query.rb +9 -0
  54. data/app/models/blazer/query.rb +31 -0
  55. data/app/models/blazer/record.rb +5 -0
  56. data/app/views/blazer/_nav.html.erb +16 -0
  57. data/app/views/blazer/_variables.html.erb +102 -0
  58. data/app/views/blazer/check_mailer/failing_checks.html.erb +6 -0
  59. data/app/views/blazer/check_mailer/state_change.html.erb +47 -0
  60. data/app/views/blazer/checks/_form.html.erb +71 -0
  61. data/app/views/blazer/checks/edit.html.erb +1 -0
  62. data/app/views/blazer/checks/index.html.erb +40 -0
  63. data/app/views/blazer/checks/new.html.erb +1 -0
  64. data/app/views/blazer/dashboards/_form.html.erb +76 -0
  65. data/app/views/blazer/dashboards/edit.html.erb +1 -0
  66. data/app/views/blazer/dashboards/new.html.erb +1 -0
  67. data/app/views/blazer/dashboards/show.html.erb +47 -0
  68. data/app/views/blazer/queries/_form.html.erb +240 -0
  69. data/app/views/blazer/queries/edit.html.erb +2 -0
  70. data/app/views/blazer/queries/home.html.erb +152 -0
  71. data/app/views/blazer/queries/new.html.erb +2 -0
  72. data/app/views/blazer/queries/run.html.erb +163 -0
  73. data/app/views/blazer/queries/schema.html.erb +18 -0
  74. data/app/views/blazer/queries/show.html.erb +73 -0
  75. data/app/views/layouts/blazer/application.html.erb +24 -0
  76. data/blazer.gemspec +26 -0
  77. data/config/routes.rb +16 -0
  78. data/lib/blazer.rb +185 -0
  79. data/lib/blazer/adapters/athena_adapter.rb +128 -0
  80. data/lib/blazer/adapters/base_adapter.rb +53 -0
  81. data/lib/blazer/adapters/bigquery_adapter.rb +67 -0
  82. data/lib/blazer/adapters/drill_adapter.rb +28 -0
  83. data/lib/blazer/adapters/elasticsearch_adapter.rb +49 -0
  84. data/lib/blazer/adapters/mongodb_adapter.rb +39 -0
  85. data/lib/blazer/adapters/presto_adapter.rb +45 -0
  86. data/lib/blazer/adapters/sql_adapter.rb +182 -0
  87. data/lib/blazer/data_source.rb +193 -0
  88. data/lib/blazer/detect_anomalies.R +19 -0
  89. data/lib/blazer/engine.rb +47 -0
  90. data/lib/blazer/result.rb +170 -0
  91. data/lib/blazer/run_statement.rb +40 -0
  92. data/lib/blazer/run_statement_job.rb +21 -0
  93. data/lib/blazer/version.rb +3 -0
  94. data/lib/generators/blazer/install_generator.rb +39 -0
  95. data/lib/generators/blazer/templates/config.yml +62 -0
  96. data/lib/generators/blazer/templates/install.rb +45 -0
  97. data/lib/tasks/blazer.rake +10 -0
  98. metadata +211 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4170e0206ecc56c82faabe4c32081c9af2ceb0c3
4
+ data.tar.gz: 75e6b5926063b5035e8452dae7992230ed19cd2a
5
+ SHA512:
6
+ metadata.gz: 70ca90adf5d7b180a135d2dc8018c9436ef6bbb86135311f595e94bae58e30ba970d978fe71e149131e25985a6166e2dc73ec2d13341090e2a5b9291ac7316f9
7
+ data.tar.gz: 168aa4bde1f3bc5d730165fd45a65c288bc8e7acc74a0791e704610cd64d11b19b4968f39aa755efd67cee27c7c7df6a10e5cc6a40388d086ff03460cb08da25
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
@@ -0,0 +1,228 @@
1
+ ## 1.8.1 [unreleased]
2
+
3
+ - Added support for Amazon Athena
4
+ - Fixed query cancellation
5
+
6
+ ## 1.8.0
7
+
8
+ - Added support for Rails 5.1
9
+
10
+ ## 1.7.10
11
+
12
+ - Added support for Google BigQuery
13
+ - Require `drill-sergeant` gem for Apache Drill
14
+ - Better handling of checks with variables
15
+
16
+ ## 1.7.9
17
+
18
+ - Added beta support for Apache Drill
19
+ - Added email validation for checks
20
+ - Updated Chart.js to 2.5.0
21
+
22
+ ## 1.7.8
23
+
24
+ - Added support for custom adapters
25
+ - Fixed bug with scatter charts on dashboards
26
+ - Fixed table preview for SQL Server
27
+ - Fixed issue when `default_url_options` set
28
+
29
+ ## 1.7.7
30
+
31
+ - Fixed preview error for MySQL
32
+ - Fixed error with timeouts for MySQL
33
+
34
+ ## 1.7.6
35
+
36
+ - Added scatter chart
37
+ - Fixed issue with false values showing up blank
38
+ - Fixed preview for table names with certain characters
39
+
40
+ ## 1.7.5
41
+
42
+ - Fixed issue with check emails sometimes failing for default Rails 5 ActiveJob adapter
43
+ - Fixed sorting for new dashboards
44
+
45
+ ## 1.7.4
46
+
47
+ - Removed extra dependencies added in 1.7.1
48
+ - Fixed `send_failing_checks` for default Rails 5 ActiveJob adapter
49
+
50
+ ## 1.7.3
51
+
52
+ - Fixed JavaScript errors
53
+ - Fixed query cancel error
54
+ - Return search results for "me" or "mine"
55
+ - Include sample data in email when bad data checks fail
56
+ - Fixed deprecation warnings
57
+
58
+ ## 1.7.2
59
+
60
+ - Cancel all queries on page nav
61
+ - Prevent Ace from taking over find command
62
+ - Added ability to use hashes for smart columns
63
+ - Added ability to inherit smart variables and columns from other data sources
64
+
65
+ ## 1.7.1
66
+
67
+ - Do not fork when enter key pressed
68
+ - Use custom version of Chart.js to fix label overlap
69
+ - Improved performance of home page
70
+
71
+ ## 1.7.0
72
+
73
+ - Added ability to cancel queries on backend for Postgres and Redshift
74
+ - Only run 3 queries at a time on dashboards
75
+ - Better anomaly detection
76
+ - Attempt to reconnect when connection issues
77
+ - Fixed issues with caching
78
+
79
+ ## 1.6.2
80
+
81
+ - Added basic query permissions
82
+ - Added ability to use arrays and hashes for smart variables
83
+ - Added cancel button for queries
84
+ - Added `lat` and `lng` as map keys
85
+
86
+ ## 1.6.1
87
+
88
+ - Added support for Presto [beta]
89
+ - Added support for Elasticsearch timeouts
90
+ - Fixed error in Rails 5
91
+
92
+ ## 1.6.0
93
+
94
+ - Added support for MongoDB [beta]
95
+ - Added support for Elasticsearch [beta]
96
+ - Fixed deprecation warning in Rails 5
97
+
98
+ ## 1.5.1
99
+
100
+ - Added anomaly detection for data less than 2 weeks
101
+ - Added autolinking urls
102
+ - Added support for images
103
+
104
+ ## 1.5.0
105
+
106
+ - Added new bar chart format
107
+ - Added anomaly detection checks
108
+ - Added `async` option for polling
109
+
110
+ ## 1.4.0
111
+
112
+ - Added `slow` cache mode
113
+ - Fixed `BLAZER_DATABASE_URL required` error
114
+ - Fixed issue with duplicate column names
115
+
116
+ ## 1.3.5
117
+
118
+ - Fixed error with checks
119
+
120
+ ## 1.3.4
121
+
122
+ - Fixed issue with missing queries
123
+
124
+ ## 1.3.3
125
+
126
+ - Fixed error with Rails 4.1 and below
127
+
128
+ ## 1.3.2
129
+
130
+ - Added support for Rails 5
131
+ - Attempt to reconnect for checks
132
+
133
+ ## 1.3.1
134
+
135
+ - Fixed migration error
136
+
137
+ ## 1.3.0
138
+
139
+ - Added schedule for checks
140
+ - Switched to Chart.js for charts
141
+ - Better output for explain
142
+ - Support for MySQL timeouts
143
+ - Raise error when timeout not supported
144
+ - Added creator to dashboards and checks
145
+
146
+ ## 1.2.1
147
+
148
+ - Fixed checks
149
+
150
+ ## 1.2.0
151
+
152
+ - Added non-editable queries
153
+ - Added variable defaults
154
+ - Added `local_time_suffix` setting
155
+ - Better timeout message
156
+ - Hide variables from commented out lines
157
+ - Fixed regex as variable names
158
+
159
+ ## 1.1.1
160
+
161
+ - Added `before_action` option
162
+ - Added invert option for checks
163
+ - Added targets
164
+ - Friendlier error message for timeouts
165
+ - Fixed request URI too large
166
+ - Prevent accidental backspace nav on query page
167
+
168
+ ## 1.1.0
169
+
170
+ - Replaced pie charts with column charts
171
+ - Fixed error with datepicker
172
+ - Added fork button to edit query page
173
+ - Added a notice when editing a query that is part of a dashboard
174
+ - Added refresh for dashboards
175
+
176
+ ## 1.0.4
177
+
178
+ - Added recently viewed queries and dashboards to home page
179
+ - Fixed refresh when transform statement is used
180
+ - Fixed error when no user model
181
+
182
+ ## 1.0.3
183
+
184
+ - Added maps
185
+ - Added support for Rails 4.0
186
+
187
+ ## 1.0.2
188
+
189
+ - Fixed error when installing
190
+ - Added `schemas` option
191
+
192
+ ## 1.0.1
193
+
194
+ - Added comments to queries
195
+ - Added `cache` option
196
+ - Added `user_method` option
197
+ - Added `use_transaction` option
198
+
199
+ ## 1.0.0
200
+
201
+ - Added support for multiple data sources
202
+ - Added dashboards
203
+ - Added checks
204
+ - Added support for Redshift
205
+
206
+ ## 0.0.8
207
+
208
+ - Easier to edit queries with variables
209
+ - Dynamically expand editor height as needed
210
+ - No need for spaces in search
211
+
212
+ ## 0.0.7
213
+
214
+ - Fixed error when no `User` class
215
+ - Fixed forking a query with variables
216
+ - Set time zone after Rails initializes
217
+
218
+ ## 0.0.6
219
+
220
+ - Added fork button
221
+ - Fixed trending
222
+ - Fixed time zones for date select
223
+
224
+ ## 0.0.5
225
+
226
+ - Added support for Rails 4.2
227
+ - Fixed error with `mysql2` adapter
228
+ - Added `user_class` option
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in blazer.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Andrew Kane
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,775 @@
1
+ # Blazer
2
+
3
+ Explore your data with SQL. Easily create charts and dashboards, and share them with your team.
4
+
5
+ [Try it out](https://blazer.dokkuapp.com)
6
+
7
+ [![Screenshot](https://blazerme.herokuapp.com/assets/screenshot-6ca3115a518b488026e48be83ba0d4c9.png)](https://blazer.dokkuapp.com)
8
+
9
+ :envelope: [Get notified of updates](http://eepurl.com/cbUwsD)
10
+
11
+ :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
12
+
13
+ ## Features
14
+
15
+ - **Multiple data sources** - PostgreSQL, MySQL, Redshift, and [many more](#full-list)
16
+ - **Variables** - run the same queries with different values
17
+ - **Checks & alerts** - get emailed when bad data appears
18
+ - **Audits** - all queries are tracked
19
+ - **Security** - works with your authentication system
20
+
21
+ ## Docs
22
+
23
+ - [Installation](#installation)
24
+ - [Queries](#queries)
25
+ - [Charts](#charts)
26
+ - [Dashboards](#dashboards)
27
+ - [Checks](#checks)
28
+
29
+ ## Installation
30
+
31
+ Add this line to your application’s Gemfile:
32
+
33
+ ```ruby
34
+ gem 'blazer'
35
+ ```
36
+
37
+ Run:
38
+
39
+ ```sh
40
+ rails g blazer:install
41
+ rake db:migrate
42
+ ```
43
+
44
+ And mount the dashboard in your `config/routes.rb`:
45
+
46
+ ```ruby
47
+ mount Blazer::Engine, at: "blazer"
48
+ ```
49
+
50
+ For production, specify your database:
51
+
52
+ ```ruby
53
+ ENV["BLAZER_DATABASE_URL"] = "postgres://user:password@hostname:5432/database"
54
+ ```
55
+
56
+ Blazer tries to protect against queries which modify data (by running each query in a transaction and rolling it back), but a safer approach is to use a read only user. [See how to create one](#permissions).
57
+
58
+ #### Checks (optional)
59
+
60
+ Be sure to set a host in `config/environments/production.rb` for emails to work.
61
+
62
+ ```ruby
63
+ config.action_mailer.default_url_options = {host: "blazerme.herokuapp.com"}
64
+ ```
65
+
66
+ Schedule checks to run (with cron, [Heroku Scheduler](https://elements.heroku.com/addons/scheduler), etc). The default options are every 5 minutes, 1 hour, or 1 day, which you can customize. For each of these options, set up a task to run.
67
+
68
+ ```sh
69
+ rake blazer:run_checks SCHEDULE="5 minutes"
70
+ rake blazer:run_checks SCHEDULE="1 hour"
71
+ rake blazer:run_checks SCHEDULE="1 day"
72
+ ```
73
+
74
+ You can also set up failing checks to be sent once a day (or whatever you prefer).
75
+
76
+ ```sh
77
+ rake blazer:send_failing_checks
78
+ ```
79
+
80
+ Here’s what it looks like with cron.
81
+
82
+ ```
83
+ */5 * * * * rake blazer:run_checks SCHEDULE="5 minutes"
84
+ 0 * * * * rake blazer:run_checks SCHEDULE="1 hour"
85
+ 30 7 * * * rake blazer:run_checks SCHEDULE="1 day"
86
+ 0 8 * * * rake blazer:send_failing_checks
87
+ ```
88
+
89
+ ## Permissions
90
+
91
+ ### PostgreSQL
92
+
93
+ Create a user with read only permissions:
94
+
95
+ ```sql
96
+ BEGIN;
97
+ CREATE ROLE blazer LOGIN PASSWORD 'secret123';
98
+ GRANT CONNECT ON DATABASE database_name TO blazer;
99
+ GRANT USAGE ON SCHEMA public TO blazer;
100
+ GRANT SELECT ON ALL TABLES IN SCHEMA public TO blazer;
101
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO blazer;
102
+ COMMIT;
103
+ ```
104
+
105
+ ### MySQL
106
+
107
+ Create a user with read only permissions:
108
+
109
+ ```sql
110
+ GRANT SELECT, SHOW VIEW ON database_name.* TO blazer@’127.0.0.1′ IDENTIFIED BY ‘secret123‘;
111
+ FLUSH PRIVILEGES;
112
+ ```
113
+
114
+ ### MongoDB
115
+
116
+ Create a user with read only permissions:
117
+
118
+ ```
119
+ db.createUser({user: "blazer", pwd: "password", roles: ["read"]})
120
+ ```
121
+
122
+ Also, make sure authorization is enabled when you start the server.
123
+
124
+ ### Sensitive Data
125
+
126
+ To protect sensitive info like password hashes and access tokens, use views. Documentation coming soon.
127
+
128
+ ## Authentication
129
+
130
+ Don’t forget to protect the dashboard in production.
131
+
132
+ ### Basic Authentication
133
+
134
+ Set the following variables in your environment or an initializer.
135
+
136
+ ```ruby
137
+ ENV["BLAZER_USERNAME"] = "andrew"
138
+ ENV["BLAZER_PASSWORD"] = "secret"
139
+ ```
140
+
141
+ ### Devise
142
+
143
+ ```ruby
144
+ authenticate :user, -> (user) { user.admin? } do
145
+ mount Blazer::Engine, at: "blazer"
146
+ end
147
+ ```
148
+
149
+ ### Other
150
+
151
+ Specify a `before_action` method to run in `blazer.yml`.
152
+
153
+ ```yml
154
+ before_action: require_admin
155
+ ```
156
+
157
+ Then define the custom authentication method in your `application_controller.rb`.
158
+
159
+ ```ruby
160
+ def require_admin
161
+ # depending on your auth, maybe something like...
162
+ current_user && current_user.admin?
163
+ end
164
+ ```
165
+
166
+ ## Queries
167
+
168
+ ### Variables
169
+
170
+ Create queries with variables.
171
+
172
+ ```sql
173
+ SELECT * FROM users WHERE gender = {gender}
174
+ ```
175
+
176
+ Use `{start_time}` and `{end_time}` for time ranges. [Example](https://blazerme.herokuapp.com/queries/9-time-range-selector?start_time=1997-10-03T05%3A00%3A00%2B00%3A00&end_time=1997-10-04T04%3A59%3A59%2B00%3A00)
177
+
178
+ ```sql
179
+ SELECT * FROM ratings WHERE rated_at >= {start_time} AND rated_at <= {end_time}
180
+ ```
181
+
182
+ ### Smart Variables
183
+
184
+ [Example](https://blazerme.herokuapp.com/queries/1-smart-variable)
185
+
186
+ Suppose you have the query:
187
+
188
+ ```sql
189
+ SELECT * FROM users WHERE occupation_id = {occupation_id}
190
+ ```
191
+
192
+ Instead of remembering each occupation’s id, users can select occupations by name.
193
+
194
+ Add a smart variable with:
195
+
196
+ ```yml
197
+ smart_variables:
198
+ occupation_id: "SELECT id, name FROM occupations ORDER BY name ASC"
199
+ ```
200
+
201
+ The first column is the value of the variable, and the second column is the label.
202
+
203
+ You can also use an array or hash for static data and enums.
204
+
205
+ ```yml
206
+ smart_variables:
207
+ period: ["day", "week", "month"]
208
+ status: {0: "Active", 1: "Archived"}
209
+ ```
210
+
211
+ ### Linked Columns
212
+
213
+ [Example](https://blazerme.herokuapp.com/queries/3-linked-column) - title column
214
+
215
+ Link results to other pages in your apps or around the web. Specify a column name and where it should link to. You can use the value of the result with `{value}`.
216
+
217
+ ```yml
218
+ linked_columns:
219
+ user_id: "/admin/users/{value}"
220
+ ip_address: "http://www.infosniper.net/index.php?ip_address={value}"
221
+ ```
222
+
223
+ ### Smart Columns
224
+
225
+ [Example](https://blazerme.herokuapp.com/queries/2-smart-column) - occupation_id column
226
+
227
+ Suppose you have the query:
228
+
229
+ ```sql
230
+ SELECT name, city_id FROM users
231
+ ```
232
+
233
+ See which city the user belongs to without a join.
234
+
235
+ ```yml
236
+ smart_columns:
237
+ city_id: "SELECT id, name FROM cities WHERE id IN {value}"
238
+ ```
239
+
240
+ You can also use a hash for static data and enums.
241
+
242
+ ```yml
243
+ smart_columns:
244
+ status: {0: "Active", 1: "Archived"}
245
+ ```
246
+
247
+ ### Caching
248
+
249
+ Blazer can automatically cache results to improve speed. It can cache slow queries:
250
+
251
+ ```yml
252
+ cache:
253
+ mode: slow
254
+ expires_in: 60 # min
255
+ slow_threshold: 15 # sec
256
+ ```
257
+
258
+ Or it can cache all queries:
259
+
260
+ ```yml
261
+ cache:
262
+ mode: all
263
+ expires_in: 60 # min
264
+ ```
265
+
266
+ Of course, you can force a refresh at any time.
267
+
268
+ ## Charts
269
+
270
+ Blazer will automatically generate charts based on the types of the columns returned in your query.
271
+
272
+ **Note:** The order of columns matters.
273
+
274
+ ### Line Chart
275
+
276
+ There are two ways to generate line charts.
277
+
278
+ 2+ columns - timestamp, numeric(s) - [Example](https://blazerme.herokuapp.com/queries/4-line-chart-format-1)
279
+
280
+ ```sql
281
+ SELECT date_trunc('week', created_at), COUNT(*) FROM users GROUP BY 1
282
+ ```
283
+
284
+ 3 columns - timestamp, string, numeric - [Example](https://blazerme.herokuapp.com/queries/5-line-chart-format-2)
285
+
286
+
287
+ ```sql
288
+ SELECT date_trunc('week', created_at), gender, COUNT(*) FROM users GROUP BY 1, 2
289
+ ```
290
+
291
+ ### Column Chart
292
+
293
+ There are also two ways to generate column charts.
294
+
295
+ 2+ columns - string, numeric(s) - [Example](https://blazerme.herokuapp.com/queries/6-column-chart-format-1)
296
+
297
+ ```sql
298
+ SELECT gender, COUNT(*) FROM users GROUP BY 1
299
+ ```
300
+
301
+ 3 columns - string, string, numeric - [Example](https://blazerme.herokuapp.com/queries/7-column-chart-format-2)
302
+
303
+ ```sql
304
+ SELECT gender, zip_code, COUNT(*) FROM users GROUP BY 1, 2
305
+ ```
306
+
307
+ ### Scatter Chart
308
+
309
+ 2 columns - both numeric
310
+
311
+ ```sql
312
+ SELECT x, y FROM table
313
+ ```
314
+
315
+ ### Maps
316
+
317
+ Columns named `latitude` and `longitude` or `lat` and `lon` or `lat` and `lng` - [Example](https://blazerme.herokuapp.com/queries/15-map)
318
+
319
+ ```sql
320
+ SELECT name, latitude, longitude FROM cities
321
+ ```
322
+
323
+ To enable, get an access token from [Mapbox](https://www.mapbox.com/) and set `ENV["MAPBOX_ACCESS_TOKEN"]`.
324
+
325
+ ### Targets
326
+
327
+ Use the column name `target` to draw a line for goals. [Example](https://blazerme.herokuapp.com/queries/8-target-line)
328
+
329
+ ```sql
330
+ SELECT date_trunc('week', created_at), COUNT(*) AS new_users, 100000 AS target FROM users GROUP BY 1
331
+ ```
332
+
333
+ ## Dashboards
334
+
335
+ Create a dashboard with multiple queries. [Example](https://blazerme.herokuapp.com/dashboards/1-dashboard-demo)
336
+
337
+ If the query has a chart, the chart is shown. Otherwise, you’ll see a table.
338
+
339
+ If any queries have variables, they will show up on the dashboard.
340
+
341
+ ## Checks
342
+
343
+ Checks give you a centralized place to see the health of your data. [Example](https://blazerme.herokuapp.com/checks)
344
+
345
+ Create a query to identify bad rows.
346
+
347
+ ```sql
348
+ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
349
+ ```
350
+
351
+ Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
352
+
353
+ ## Anomaly Detection
354
+
355
+ Anomaly detection is supported thanks to Twitter’s [AnomalyDetection](https://github.com/twitter/AnomalyDetection) library.
356
+
357
+ First, [install R](https://cloud.r-project.org/). Then, run:
358
+
359
+ ```R
360
+ install.packages("devtools")
361
+ devtools::install_github("twitter/AnomalyDetection")
362
+ ```
363
+
364
+ And add to `config/blazer.yml`:
365
+
366
+ ```yml
367
+ anomaly_checks: true
368
+ ```
369
+
370
+ If upgrading from version 1.4 or below, also follow the [upgrade instructions](#15).
371
+
372
+ If you’re on Heroku, follow [these additional instructions](#anomaly-detection-on-heroku).
373
+
374
+ ## Data Sources
375
+
376
+ Blazer supports multiple data sources :tada:
377
+
378
+ Add additional data sources in `config/blazer.yml`:
379
+
380
+ ```yml
381
+ data_sources:
382
+ main:
383
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
384
+ # timeout, smart_variables, linked_columns, smart_columns
385
+ catalog:
386
+ url: <%= ENV["CATALOG_DATABASE_URL"] %>
387
+ # ...
388
+ redshift:
389
+ url: <%= ENV["REDSHIFT_DATABASE_URL"] %>
390
+ # ...
391
+ ```
392
+
393
+ ### Full List
394
+
395
+ - [PostgreSQL](#postgresql-1)
396
+ - [MySQL](#mysql-1)
397
+ - [SQL Server](#sql-server)
398
+ - [Oracle](#oracle)
399
+ - [IBM DB2 and Informix](#ibm-db2-and-informix)
400
+ - [SQLite](#sqlite)
401
+ - [Amazon Redshift](#amazon-redshift)
402
+ - [Amazon Athena](#amazon-athena-master) [master]
403
+ - [Presto](#presto)
404
+ - [Apache Drill](#apache-drill)
405
+ - [Google BigQuery](#google-bigquery)
406
+ - [MongoDB](#mongodb-1)
407
+ - [Elasticsearch](#elasticsearch-beta) [beta]
408
+
409
+ You can also [create an adapter](#creating-an-adapter) for any other data store.
410
+
411
+ **Note:** In the examples below, we recommend using environment variables for urls.
412
+
413
+ ```yml
414
+ data_sources:
415
+ my_source:
416
+ url: <%= ENV["BLAZER_MY_SOURCE_URL"] %>
417
+ ```
418
+
419
+ ### PostgreSQL
420
+
421
+ Add [pg](https://bitbucket.org/ged/ruby-pg/wiki/Home) to your Gemfile (if it’s not there) and set:
422
+
423
+ ```yml
424
+ data_sources:
425
+ my_source:
426
+ url: postgres://user:password@hostname:5432/database
427
+ ```
428
+
429
+ ### MySQL
430
+
431
+ Add [mysql2](https://github.com/brianmario/mysql2) to your Gemfile (if it’s not there) and set:
432
+
433
+ ```yml
434
+ data_sources:
435
+ my_source:
436
+ url: mysql2://user:password@hostname:3306/database
437
+ ```
438
+
439
+ ### SQL Server
440
+
441
+ Add [tiny_tds](https://github.com/rails-sqlserver/tiny_tds) and [activerecord-sqlserver-adapter](https://github.com/rails-sqlserver/activerecord-sqlserver-adapter) to your Gemfile and set:
442
+
443
+ ```yml
444
+ data_sources:
445
+ my_source:
446
+ url: sqlserver://user:password@hostname:1433/database
447
+ ```
448
+
449
+ ### Oracle
450
+
451
+ Use [activerecord-oracle_enhanced-adapter](https://github.com/rsim/oracle-enhanced).
452
+
453
+ ### IBM DB2 and Informix
454
+
455
+ Use [ibm_db](https://github.com/ibmdb/ruby-ibmdb).
456
+
457
+ ### SQLite
458
+
459
+ Add [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) to your Gemfile and set:
460
+
461
+ ```yml
462
+ data_sources:
463
+ my_source:
464
+ url: sqlite3:path/to/database.sqlite3
465
+ ```
466
+
467
+ ### Amazon Redshift
468
+
469
+ Add [activerecord4-redshift-adapter](https://github.com/aamine/activerecord4-redshift-adapter) or [activerecord5-redshift-adapter](https://github.com/ConsultingMD/activerecord5-redshift-adapter) to your Gemfile and set:
470
+
471
+ ```yml
472
+ data_sources:
473
+ my_source:
474
+ url: redshift://user:password@hostname:5439/database
475
+ ```
476
+
477
+ ### Amazon Athena [master]
478
+
479
+ Add [aws-sdk](https://github.com/aws/aws-sdk-ruby) `~> 2` to your Gemfile and set:
480
+
481
+ ```yml
482
+ data_sources:
483
+ my_source:
484
+ adapter: athena
485
+ database: database
486
+ output_location: s3://some-bucket/
487
+ ```
488
+
489
+ ### Presto
490
+
491
+ Add [presto-client](https://github.com/treasure-data/presto-client-ruby) to your Gemfile and set:
492
+
493
+ ```yml
494
+ data_sources:
495
+ my_source:
496
+ url: presto://user@hostname:8080/catalog
497
+ ```
498
+
499
+ ### Apache Drill
500
+
501
+ Add [drill-sergeant](https://github.com/ankane/drill-sergeant) to your Gemfile and set:
502
+
503
+ ```yml
504
+ data_sources:
505
+ my_source:
506
+ adapter: drill
507
+ url: http://hostname:8047
508
+ ```
509
+
510
+ ### Google BigQuery
511
+
512
+ Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:
513
+
514
+ ```yml
515
+ data_sources:
516
+ my_source:
517
+ adapter: bigquery
518
+ project: your-project
519
+ keyfile: path/to/keyfile.json
520
+ ```
521
+
522
+ ### MongoDB
523
+
524
+ Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
525
+
526
+ ```yml
527
+ data_sources:
528
+ my_source:
529
+ url: mongodb://user:password@hostname:27017/database
530
+ ```
531
+
532
+ ### Elasticsearch [beta]
533
+
534
+ Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) to your Gemfile and set:
535
+
536
+ ```yml
537
+ data_sources:
538
+ my_source:
539
+ adapter: elasticsearch
540
+ url: http://user:password@hostname:9200
541
+ ```
542
+
543
+ ## Creating an Adapter
544
+
545
+ Create an adapter for any data store with:
546
+
547
+ ```ruby
548
+ class FooAdapter < Blazer::Adapters::BaseAdapter
549
+ # code goes here
550
+ end
551
+
552
+ Blazer.register_adapter "foo", FooAdapter
553
+ ```
554
+
555
+ See the [Presto adapter](https://github.com/ankane/blazer/blob/master/lib/blazer/adapters/presto_adapter.rb) for a good example. Then use:
556
+
557
+ ```yml
558
+ data_sources:
559
+ my_source:
560
+ adapter: foo
561
+ url: http://user:password@hostname:9200/
562
+ ```
563
+
564
+ ## Query Permissions
565
+
566
+ Blazer supports a basic permissions model.
567
+
568
+ 1. Queries without a name are unlisted
569
+ 2. Queries whose name starts with `#` are only listed to the creator
570
+ 3. Queries whose name starts with `*` can only be edited by the creator
571
+
572
+ ## Learn SQL
573
+
574
+ Have team members who want to learn SQL? Here are a few great, free resources.
575
+
576
+ - [Khan Academy](https://www.khanacademy.org/computing/computer-programming/sql)
577
+ - [Codecademy](https://www.codecademy.com/learn/learn-sql)
578
+
579
+ ## Useful Tools
580
+
581
+ For an easy way to group by day, week, month, and more with correct time zones, check out [Groupdate](https://github.com/ankane/groupdate.sql).
582
+
583
+ ## Anomaly Detection on Heroku
584
+
585
+ Add the [R buildpack](https://github.com/virtualstaticvoid/heroku-buildpack-r) to your app.
586
+
587
+ ```sh
588
+ heroku buildpacks:add --index 1 https://github.com/virtualstaticvoid/heroku-buildpack-r.git\#cedar-14
589
+ ```
590
+
591
+ And create an `init.r` with:
592
+
593
+ ```sh
594
+ if (!"AnomalyDetection" %in% installed.packages()) {
595
+ install.packages("devtools")
596
+ devtools::install_github("twitter/AnomalyDetection")
597
+ }
598
+ ```
599
+
600
+ Commit and deploy away. The first deploy may take a few minutes.
601
+
602
+ ## Upgrading
603
+
604
+ ### 1.5
605
+
606
+ To take advantage of the anomaly detection, create a migration
607
+
608
+ ```sh
609
+ rails g migration upgrade_blazer_to_1_5
610
+ ```
611
+
612
+ with:
613
+
614
+ ```ruby
615
+ add_column(:blazer_checks, :check_type, :string)
616
+ add_column(:blazer_checks, :message, :text)
617
+ commit_db_transaction
618
+
619
+ Blazer::Check.reset_column_information
620
+
621
+ Blazer::Check.where(invert: true).update_all(check_type: "missing_data")
622
+ Blazer::Check.where(check_type: nil).update_all(check_type: "bad_data")
623
+ ```
624
+
625
+ ### 1.3
626
+
627
+ To take advantage of the latest features, create a migration
628
+
629
+ ```sh
630
+ rails g migration upgrade_blazer_to_1_3
631
+ ```
632
+
633
+ with:
634
+
635
+ ```ruby
636
+ add_column :blazer_dashboards, :creator_id, :integer
637
+ add_column :blazer_checks, :creator_id, :integer
638
+ add_column :blazer_checks, :invert, :boolean
639
+ add_column :blazer_checks, :schedule, :string
640
+ add_column :blazer_checks, :last_run_at, :timestamp
641
+ commit_db_transaction
642
+
643
+ Blazer::Check.update_all schedule: "1 hour"
644
+ ```
645
+
646
+ ### 1.0
647
+
648
+ Blazer 1.0 brings a number of new features:
649
+
650
+ - multiple data sources, including Redshift
651
+ - dashboards
652
+ - checks
653
+
654
+ To upgrade, run:
655
+
656
+ ```sh
657
+ bundle update blazer
658
+ ```
659
+
660
+ Create a migration
661
+
662
+ ```sh
663
+ rails g migration upgrade_blazer_to_1_0
664
+ ```
665
+
666
+ with:
667
+
668
+ ```ruby
669
+ add_column :blazer_queries, :data_source, :string
670
+ add_column :blazer_audits, :data_source, :string
671
+
672
+ create_table :blazer_dashboards do |t|
673
+ t.text :name
674
+ t.timestamps
675
+ end
676
+
677
+ create_table :blazer_dashboard_queries do |t|
678
+ t.references :dashboard
679
+ t.references :query
680
+ t.integer :position
681
+ t.timestamps
682
+ end
683
+
684
+ create_table :blazer_checks do |t|
685
+ t.references :query
686
+ t.string :state
687
+ t.text :emails
688
+ t.timestamps
689
+ end
690
+ ```
691
+
692
+ And run:
693
+
694
+ ```sh
695
+ rake db:migrate
696
+ ```
697
+
698
+ Update `config/blazer.yml` with:
699
+
700
+ ```yml
701
+ # see https://github.com/ankane/blazer for more info
702
+
703
+ data_sources:
704
+ main:
705
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
706
+
707
+ # statement timeout, in seconds
708
+ # applies to PostgreSQL only
709
+ # none by default
710
+ # timeout: 15
711
+
712
+ # time to cache results, in minutes
713
+ # can greatly improve speed
714
+ # none by default
715
+ # cache: 60
716
+
717
+ # wrap queries in a transaction for safety
718
+ # not necessary if you use a read-only user
719
+ # true by default
720
+ # use_transaction: false
721
+
722
+ smart_variables:
723
+ # zone_id: "SELECT id, name FROM zones ORDER BY name ASC"
724
+
725
+ linked_columns:
726
+ # user_id: "/admin/users/{value}"
727
+
728
+ smart_columns:
729
+ # user_id: "SELECT id, name FROM users WHERE id IN {value}"
730
+
731
+ # create audits
732
+ audit: true
733
+
734
+ # change the time zone
735
+ # time_zone: "Pacific Time (US & Canada)"
736
+
737
+ # class name of the user model
738
+ # user_class: User
739
+
740
+ # method name for the current user
741
+ # user_method: current_user
742
+
743
+ # method name for the display name
744
+ # user_name: name
745
+
746
+ # email to send checks from
747
+ # from_email: blazer@example.org
748
+ ```
749
+
750
+ ## TODO
751
+
752
+ - advanced permissions
753
+ - standalone version
754
+ - better navigation
755
+
756
+ ## History
757
+
758
+ View the [changelog](https://github.com/ankane/blazer/blob/master/CHANGELOG.md)
759
+
760
+ ## Thanks
761
+
762
+ Blazer uses a number of awesome open source projects, including [Rails](https://github.com/rails/rails/), [Vue.js](https://github.com/vuejs/vue), [jQuery](https://github.com/jquery/jquery), [Bootstrap](https://github.com/twbs/bootstrap), [Selectize](https://github.com/brianreavis/selectize.js), [StickyTableHeaders](https://github.com/jmosbech/StickyTableHeaders), [Stupid jQuery Table Sort](https://github.com/joequery/Stupid-Table-Plugin), and [Date Range Picker](https://github.com/dangrossman/bootstrap-daterangepicker).
763
+
764
+ Demo data from [MovieLens](http://grouplens.org/datasets/movielens/).
765
+
766
+ ## Want to Make Blazer Better?
767
+
768
+ That’s awesome! Here are a few ways you can help:
769
+
770
+ - [Report bugs](https://github.com/ankane/blazer/issues)
771
+ - Fix bugs and [submit pull requests](https://github.com/ankane/blazer/pulls)
772
+ - Write, clarify, or fix documentation
773
+ - Suggest or add new features
774
+
775
+ Check out the [dev app](https://github.com/ankane/blazer-dev) to get started.