sql-jarvis 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +14 -0
  3. data/CHANGELOG.md +228 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +775 -0
  7. data/Rakefile +1 -0
  8. data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
  9. data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
  10. data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
  11. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
  12. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
  13. data/app/assets/javascripts/blazer/Chart.js +14145 -0
  14. data/app/assets/javascripts/blazer/Sortable.js +1144 -0
  15. data/app/assets/javascripts/blazer/ace.js +6 -0
  16. data/app/assets/javascripts/blazer/ace/ace.js +11 -0
  17. data/app/assets/javascripts/blazer/ace/ext-language_tools.js +5 -0
  18. data/app/assets/javascripts/blazer/ace/mode-sql.js +1 -0
  19. data/app/assets/javascripts/blazer/ace/snippets/sql.js +1 -0
  20. data/app/assets/javascripts/blazer/ace/snippets/text.js +1 -0
  21. data/app/assets/javascripts/blazer/ace/theme-twilight.js +1 -0
  22. data/app/assets/javascripts/blazer/application.js +79 -0
  23. data/app/assets/javascripts/blazer/bootstrap.js +2366 -0
  24. data/app/assets/javascripts/blazer/chartkick.js +1693 -0
  25. data/app/assets/javascripts/blazer/daterangepicker.js +1505 -0
  26. data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
  27. data/app/assets/javascripts/blazer/highlight.pack.js +1 -0
  28. data/app/assets/javascripts/blazer/jquery.js +10308 -0
  29. data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +263 -0
  30. data/app/assets/javascripts/blazer/jquery_ujs.js +469 -0
  31. data/app/assets/javascripts/blazer/moment-timezone.js +1007 -0
  32. data/app/assets/javascripts/blazer/moment.js +3043 -0
  33. data/app/assets/javascripts/blazer/queries.js +110 -0
  34. data/app/assets/javascripts/blazer/routes.js +23 -0
  35. data/app/assets/javascripts/blazer/selectize.js +3667 -0
  36. data/app/assets/javascripts/blazer/stupidtable.js +114 -0
  37. data/app/assets/javascripts/blazer/vue.js +7515 -0
  38. data/app/assets/stylesheets/blazer/application.css +198 -0
  39. data/app/assets/stylesheets/blazer/bootstrap.css.erb +6202 -0
  40. data/app/assets/stylesheets/blazer/daterangepicker-bs3.css +375 -0
  41. data/app/assets/stylesheets/blazer/github.css +125 -0
  42. data/app/assets/stylesheets/blazer/selectize.default.css +387 -0
  43. data/app/controllers/blazer/base_controller.rb +103 -0
  44. data/app/controllers/blazer/checks_controller.rb +56 -0
  45. data/app/controllers/blazer/dashboards_controller.rb +105 -0
  46. data/app/controllers/blazer/queries_controller.rb +325 -0
  47. data/app/helpers/blazer/base_helper.rb +57 -0
  48. data/app/mailers/blazer/check_mailer.rb +27 -0
  49. data/app/models/blazer/audit.rb +6 -0
  50. data/app/models/blazer/check.rb +95 -0
  51. data/app/models/blazer/connection.rb +5 -0
  52. data/app/models/blazer/dashboard.rb +13 -0
  53. data/app/models/blazer/dashboard_query.rb +9 -0
  54. data/app/models/blazer/query.rb +31 -0
  55. data/app/models/blazer/record.rb +5 -0
  56. data/app/views/blazer/_nav.html.erb +16 -0
  57. data/app/views/blazer/_variables.html.erb +102 -0
  58. data/app/views/blazer/check_mailer/failing_checks.html.erb +6 -0
  59. data/app/views/blazer/check_mailer/state_change.html.erb +47 -0
  60. data/app/views/blazer/checks/_form.html.erb +71 -0
  61. data/app/views/blazer/checks/edit.html.erb +1 -0
  62. data/app/views/blazer/checks/index.html.erb +40 -0
  63. data/app/views/blazer/checks/new.html.erb +1 -0
  64. data/app/views/blazer/dashboards/_form.html.erb +76 -0
  65. data/app/views/blazer/dashboards/edit.html.erb +1 -0
  66. data/app/views/blazer/dashboards/new.html.erb +1 -0
  67. data/app/views/blazer/dashboards/show.html.erb +47 -0
  68. data/app/views/blazer/queries/_form.html.erb +240 -0
  69. data/app/views/blazer/queries/edit.html.erb +2 -0
  70. data/app/views/blazer/queries/home.html.erb +152 -0
  71. data/app/views/blazer/queries/new.html.erb +2 -0
  72. data/app/views/blazer/queries/run.html.erb +163 -0
  73. data/app/views/blazer/queries/schema.html.erb +18 -0
  74. data/app/views/blazer/queries/show.html.erb +73 -0
  75. data/app/views/layouts/blazer/application.html.erb +24 -0
  76. data/blazer.gemspec +26 -0
  77. data/config/routes.rb +16 -0
  78. data/lib/blazer.rb +185 -0
  79. data/lib/blazer/adapters/athena_adapter.rb +128 -0
  80. data/lib/blazer/adapters/base_adapter.rb +53 -0
  81. data/lib/blazer/adapters/bigquery_adapter.rb +67 -0
  82. data/lib/blazer/adapters/drill_adapter.rb +28 -0
  83. data/lib/blazer/adapters/elasticsearch_adapter.rb +49 -0
  84. data/lib/blazer/adapters/mongodb_adapter.rb +39 -0
  85. data/lib/blazer/adapters/presto_adapter.rb +45 -0
  86. data/lib/blazer/adapters/sql_adapter.rb +182 -0
  87. data/lib/blazer/data_source.rb +193 -0
  88. data/lib/blazer/detect_anomalies.R +19 -0
  89. data/lib/blazer/engine.rb +47 -0
  90. data/lib/blazer/result.rb +170 -0
  91. data/lib/blazer/run_statement.rb +40 -0
  92. data/lib/blazer/run_statement_job.rb +21 -0
  93. data/lib/blazer/version.rb +3 -0
  94. data/lib/generators/blazer/install_generator.rb +39 -0
  95. data/lib/generators/blazer/templates/config.yml +62 -0
  96. data/lib/generators/blazer/templates/install.rb +45 -0
  97. data/lib/tasks/blazer.rake +10 -0
  98. metadata +211 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4170e0206ecc56c82faabe4c32081c9af2ceb0c3
4
+ data.tar.gz: 75e6b5926063b5035e8452dae7992230ed19cd2a
5
+ SHA512:
6
+ metadata.gz: 70ca90adf5d7b180a135d2dc8018c9436ef6bbb86135311f595e94bae58e30ba970d978fe71e149131e25985a6166e2dc73ec2d13341090e2a5b9291ac7316f9
7
+ data.tar.gz: 168aa4bde1f3bc5d730165fd45a65c288bc8e7acc74a0791e704610cd64d11b19b4968f39aa755efd67cee27c7c7df6a10e5cc6a40388d086ff03460cb08da25
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
@@ -0,0 +1,228 @@
1
+ ## 1.8.1 [unreleased]
2
+
3
+ - Added support for Amazon Athena
4
+ - Fixed query cancellation
5
+
6
+ ## 1.8.0
7
+
8
+ - Added support for Rails 5.1
9
+
10
+ ## 1.7.10
11
+
12
+ - Added support for Google BigQuery
13
+ - Require `drill-sergeant` gem for Apache Drill
14
+ - Better handling of checks with variables
15
+
16
+ ## 1.7.9
17
+
18
+ - Added beta support for Apache Drill
19
+ - Added email validation for checks
20
+ - Updated Chart.js to 2.5.0
21
+
22
+ ## 1.7.8
23
+
24
+ - Added support for custom adapters
25
+ - Fixed bug with scatter charts on dashboards
26
+ - Fixed table preview for SQL Server
27
+ - Fixed issue when `default_url_options` set
28
+
29
+ ## 1.7.7
30
+
31
+ - Fixed preview error for MySQL
32
+ - Fixed error with timeouts for MySQL
33
+
34
+ ## 1.7.6
35
+
36
+ - Added scatter chart
37
+ - Fixed issue with false values showing up blank
38
+ - Fixed preview for table names with certain characters
39
+
40
+ ## 1.7.5
41
+
42
+ - Fixed issue with check emails sometimes failing for default Rails 5 ActiveJob adapter
43
+ - Fixed sorting for new dashboards
44
+
45
+ ## 1.7.4
46
+
47
+ - Removed extra dependencies added in 1.7.1
48
+ - Fixed `send_failing_checks` for default Rails 5 ActiveJob adapter
49
+
50
+ ## 1.7.3
51
+
52
+ - Fixed JavaScript errors
53
+ - Fixed query cancel error
54
+ - Return search results for "me" or "mine"
55
+ - Include sample data in email when bad data checks fail
56
+ - Fixed deprecation warnings
57
+
58
+ ## 1.7.2
59
+
60
+ - Cancel all queries on page nav
61
+ - Prevent Ace from taking over find command
62
+ - Added ability to use hashes for smart columns
63
+ - Added ability to inherit smart variables and columns from other data sources
64
+
65
+ ## 1.7.1
66
+
67
+ - Do not fork when enter key pressed
68
+ - Use custom version of Chart.js to fix label overlap
69
+ - Improved performance of home page
70
+
71
+ ## 1.7.0
72
+
73
+ - Added ability to cancel queries on backend for Postgres and Redshift
74
+ - Only run 3 queries at a time on dashboards
75
+ - Better anomaly detection
76
+ - Attempt to reconnect when connection issues
77
+ - Fixed issues with caching
78
+
79
+ ## 1.6.2
80
+
81
+ - Added basic query permissions
82
+ - Added ability to use arrays and hashes for smart variables
83
+ - Added cancel button for queries
84
+ - Added `lat` and `lng` as map keys
85
+
86
+ ## 1.6.1
87
+
88
+ - Added support for Presto [beta]
89
+ - Added support for Elasticsearch timeouts
90
+ - Fixed error in Rails 5
91
+
92
+ ## 1.6.0
93
+
94
+ - Added support for MongoDB [beta]
95
+ - Added support for Elasticsearch [beta]
96
+ - Fixed deprecation warning in Rails 5
97
+
98
+ ## 1.5.1
99
+
100
+ - Added anomaly detection for data less than 2 weeks
101
+ - Added autolinking urls
102
+ - Added support for images
103
+
104
+ ## 1.5.0
105
+
106
+ - Added new bar chart format
107
+ - Added anomaly detection checks
108
+ - Added `async` option for polling
109
+
110
+ ## 1.4.0
111
+
112
+ - Added `slow` cache mode
113
+ - Fixed `BLAZER_DATABASE_URL required` error
114
+ - Fixed issue with duplicate column names
115
+
116
+ ## 1.3.5
117
+
118
+ - Fixed error with checks
119
+
120
+ ## 1.3.4
121
+
122
+ - Fixed issue with missing queries
123
+
124
+ ## 1.3.3
125
+
126
+ - Fixed error with Rails 4.1 and below
127
+
128
+ ## 1.3.2
129
+
130
+ - Added support for Rails 5
131
+ - Attempt to reconnect for checks
132
+
133
+ ## 1.3.1
134
+
135
+ - Fixed migration error
136
+
137
+ ## 1.3.0
138
+
139
+ - Added schedule for checks
140
+ - Switched to Chart.js for charts
141
+ - Better output for explain
142
+ - Support for MySQL timeouts
143
+ - Raise error when timeout not supported
144
+ - Added creator to dashboards and checks
145
+
146
+ ## 1.2.1
147
+
148
+ - Fixed checks
149
+
150
+ ## 1.2.0
151
+
152
+ - Added non-editable queries
153
+ - Added variable defaults
154
+ - Added `local_time_suffix` setting
155
+ - Better timeout message
156
+ - Hide variables from commented out lines
157
+ - Fixed regex as variable names
158
+
159
+ ## 1.1.1
160
+
161
+ - Added `before_action` option
162
+ - Added invert option for checks
163
+ - Added targets
164
+ - Friendlier error message for timeouts
165
+ - Fixed request URI too large
166
+ - Prevent accidental backspace nav on query page
167
+
168
+ ## 1.1.0
169
+
170
+ - Replaced pie charts with column charts
171
+ - Fixed error with datepicker
172
+ - Added fork button to edit query page
173
+ - Added a notice when editing a query that is part of a dashboard
174
+ - Added refresh for dashboards
175
+
176
+ ## 1.0.4
177
+
178
+ - Added recently viewed queries and dashboards to home page
179
+ - Fixed refresh when transform statement is used
180
+ - Fixed error when no user model
181
+
182
+ ## 1.0.3
183
+
184
+ - Added maps
185
+ - Added support for Rails 4.0
186
+
187
+ ## 1.0.2
188
+
189
+ - Fixed error when installing
190
+ - Added `schemas` option
191
+
192
+ ## 1.0.1
193
+
194
+ - Added comments to queries
195
+ - Added `cache` option
196
+ - Added `user_method` option
197
+ - Added `use_transaction` option
198
+
199
+ ## 1.0.0
200
+
201
+ - Added support for multiple data sources
202
+ - Added dashboards
203
+ - Added checks
204
+ - Added support for Redshift
205
+
206
+ ## 0.0.8
207
+
208
+ - Easier to edit queries with variables
209
+ - Dynamically expand editor height as needed
210
+ - No need for spaces in search
211
+
212
+ ## 0.0.7
213
+
214
+ - Fixed error when no `User` class
215
+ - Fixed forking a query with variables
216
+ - Set time zone after Rails initializes
217
+
218
+ ## 0.0.6
219
+
220
+ - Added fork button
221
+ - Fixed trending
222
+ - Fixed time zones for date select
223
+
224
+ ## 0.0.5
225
+
226
+ - Added support for Rails 4.2
227
+ - Fixed error with `mysql2` adapter
228
+ - Added `user_class` option
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in blazer.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Andrew Kane
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,775 @@
1
+ # Blazer
2
+
3
+ Explore your data with SQL. Easily create charts and dashboards, and share them with your team.
4
+
5
+ [Try it out](https://blazer.dokkuapp.com)
6
+
7
+ [![Screenshot](https://blazerme.herokuapp.com/assets/screenshot-6ca3115a518b488026e48be83ba0d4c9.png)](https://blazer.dokkuapp.com)
8
+
9
+ :envelope: [Get notified of updates](http://eepurl.com/cbUwsD)
10
+
11
+ :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
12
+
13
+ ## Features
14
+
15
+ - **Multiple data sources** - PostgreSQL, MySQL, Redshift, and [many more](#full-list)
16
+ - **Variables** - run the same queries with different values
17
+ - **Checks & alerts** - get emailed when bad data appears
18
+ - **Audits** - all queries are tracked
19
+ - **Security** - works with your authentication system
20
+
21
+ ## Docs
22
+
23
+ - [Installation](#installation)
24
+ - [Queries](#queries)
25
+ - [Charts](#charts)
26
+ - [Dashboards](#dashboards)
27
+ - [Checks](#checks)
28
+
29
+ ## Installation
30
+
31
+ Add this line to your application’s Gemfile:
32
+
33
+ ```ruby
34
+ gem 'blazer'
35
+ ```
36
+
37
+ Run:
38
+
39
+ ```sh
40
+ rails g blazer:install
41
+ rake db:migrate
42
+ ```
43
+
44
+ And mount the dashboard in your `config/routes.rb`:
45
+
46
+ ```ruby
47
+ mount Blazer::Engine, at: "blazer"
48
+ ```
49
+
50
+ For production, specify your database:
51
+
52
+ ```ruby
53
+ ENV["BLAZER_DATABASE_URL"] = "postgres://user:password@hostname:5432/database"
54
+ ```
55
+
56
+ Blazer tries to protect against queries which modify data (by running each query in a transaction and rolling it back), but a safer approach is to use a read only user. [See how to create one](#permissions).
57
+
58
+ #### Checks (optional)
59
+
60
+ Be sure to set a host in `config/environments/production.rb` for emails to work.
61
+
62
+ ```ruby
63
+ config.action_mailer.default_url_options = {host: "blazerme.herokuapp.com"}
64
+ ```
65
+
66
+ Schedule checks to run (with cron, [Heroku Scheduler](https://elements.heroku.com/addons/scheduler), etc). The default options are every 5 minutes, 1 hour, or 1 day, which you can customize. For each of these options, set up a task to run.
67
+
68
+ ```sh
69
+ rake blazer:run_checks SCHEDULE="5 minutes"
70
+ rake blazer:run_checks SCHEDULE="1 hour"
71
+ rake blazer:run_checks SCHEDULE="1 day"
72
+ ```
73
+
74
+ You can also set up failing checks to be sent once a day (or whatever you prefer).
75
+
76
+ ```sh
77
+ rake blazer:send_failing_checks
78
+ ```
79
+
80
+ Here’s what it looks like with cron.
81
+
82
+ ```
83
+ */5 * * * * rake blazer:run_checks SCHEDULE="5 minutes"
84
+ 0 * * * * rake blazer:run_checks SCHEDULE="1 hour"
85
+ 30 7 * * * rake blazer:run_checks SCHEDULE="1 day"
86
+ 0 8 * * * rake blazer:send_failing_checks
87
+ ```
88
+
89
+ ## Permissions
90
+
91
+ ### PostgreSQL
92
+
93
+ Create a user with read only permissions:
94
+
95
+ ```sql
96
+ BEGIN;
97
+ CREATE ROLE blazer LOGIN PASSWORD 'secret123';
98
+ GRANT CONNECT ON DATABASE database_name TO blazer;
99
+ GRANT USAGE ON SCHEMA public TO blazer;
100
+ GRANT SELECT ON ALL TABLES IN SCHEMA public TO blazer;
101
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO blazer;
102
+ COMMIT;
103
+ ```
104
+
105
+ ### MySQL
106
+
107
+ Create a user with read only permissions:
108
+
109
+ ```sql
110
+ GRANT SELECT, SHOW VIEW ON database_name.* TO blazer@’127.0.0.1′ IDENTIFIED BY ‘secret123‘;
111
+ FLUSH PRIVILEGES;
112
+ ```
113
+
114
+ ### MongoDB
115
+
116
+ Create a user with read only permissions:
117
+
118
+ ```
119
+ db.createUser({user: "blazer", pwd: "password", roles: ["read"]})
120
+ ```
121
+
122
+ Also, make sure authorization is enabled when you start the server.
123
+
124
+ ### Sensitive Data
125
+
126
+ To protect sensitive info like password hashes and access tokens, use views. Documentation coming soon.
127
+
128
+ ## Authentication
129
+
130
+ Don’t forget to protect the dashboard in production.
131
+
132
+ ### Basic Authentication
133
+
134
+ Set the following variables in your environment or an initializer.
135
+
136
+ ```ruby
137
+ ENV["BLAZER_USERNAME"] = "andrew"
138
+ ENV["BLAZER_PASSWORD"] = "secret"
139
+ ```
140
+
141
+ ### Devise
142
+
143
+ ```ruby
144
+ authenticate :user, -> (user) { user.admin? } do
145
+ mount Blazer::Engine, at: "blazer"
146
+ end
147
+ ```
148
+
149
+ ### Other
150
+
151
+ Specify a `before_action` method to run in `blazer.yml`.
152
+
153
+ ```yml
154
+ before_action: require_admin
155
+ ```
156
+
157
+ Then define the custom authentication method in your `application_controller.rb`.
158
+
159
+ ```ruby
160
+ def require_admin
161
+ # depending on your auth, maybe something like...
162
+ current_user && current_user.admin?
163
+ end
164
+ ```
165
+
166
+ ## Queries
167
+
168
+ ### Variables
169
+
170
+ Create queries with variables.
171
+
172
+ ```sql
173
+ SELECT * FROM users WHERE gender = {gender}
174
+ ```
175
+
176
+ Use `{start_time}` and `{end_time}` for time ranges. [Example](https://blazerme.herokuapp.com/queries/9-time-range-selector?start_time=1997-10-03T05%3A00%3A00%2B00%3A00&end_time=1997-10-04T04%3A59%3A59%2B00%3A00)
177
+
178
+ ```sql
179
+ SELECT * FROM ratings WHERE rated_at >= {start_time} AND rated_at <= {end_time}
180
+ ```
181
+
182
+ ### Smart Variables
183
+
184
+ [Example](https://blazerme.herokuapp.com/queries/1-smart-variable)
185
+
186
+ Suppose you have the query:
187
+
188
+ ```sql
189
+ SELECT * FROM users WHERE occupation_id = {occupation_id}
190
+ ```
191
+
192
+ Instead of remembering each occupation’s id, users can select occupations by name.
193
+
194
+ Add a smart variable with:
195
+
196
+ ```yml
197
+ smart_variables:
198
+ occupation_id: "SELECT id, name FROM occupations ORDER BY name ASC"
199
+ ```
200
+
201
+ The first column is the value of the variable, and the second column is the label.
202
+
203
+ You can also use an array or hash for static data and enums.
204
+
205
+ ```yml
206
+ smart_variables:
207
+ period: ["day", "week", "month"]
208
+ status: {0: "Active", 1: "Archived"}
209
+ ```
210
+
211
+ ### Linked Columns
212
+
213
+ [Example](https://blazerme.herokuapp.com/queries/3-linked-column) - title column
214
+
215
+ Link results to other pages in your apps or around the web. Specify a column name and where it should link to. You can use the value of the result with `{value}`.
216
+
217
+ ```yml
218
+ linked_columns:
219
+ user_id: "/admin/users/{value}"
220
+ ip_address: "http://www.infosniper.net/index.php?ip_address={value}"
221
+ ```
222
+
223
+ ### Smart Columns
224
+
225
+ [Example](https://blazerme.herokuapp.com/queries/2-smart-column) - occupation_id column
226
+
227
+ Suppose you have the query:
228
+
229
+ ```sql
230
+ SELECT name, city_id FROM users
231
+ ```
232
+
233
+ See which city the user belongs to without a join.
234
+
235
+ ```yml
236
+ smart_columns:
237
+ city_id: "SELECT id, name FROM cities WHERE id IN {value}"
238
+ ```
239
+
240
+ You can also use a hash for static data and enums.
241
+
242
+ ```yml
243
+ smart_columns:
244
+ status: {0: "Active", 1: "Archived"}
245
+ ```
246
+
247
+ ### Caching
248
+
249
+ Blazer can automatically cache results to improve speed. It can cache slow queries:
250
+
251
+ ```yml
252
+ cache:
253
+ mode: slow
254
+ expires_in: 60 # min
255
+ slow_threshold: 15 # sec
256
+ ```
257
+
258
+ Or it can cache all queries:
259
+
260
+ ```yml
261
+ cache:
262
+ mode: all
263
+ expires_in: 60 # min
264
+ ```
265
+
266
+ Of course, you can force a refresh at any time.
267
+
268
+ ## Charts
269
+
270
+ Blazer will automatically generate charts based on the types of the columns returned in your query.
271
+
272
+ **Note:** The order of columns matters.
273
+
274
+ ### Line Chart
275
+
276
+ There are two ways to generate line charts.
277
+
278
+ 2+ columns - timestamp, numeric(s) - [Example](https://blazerme.herokuapp.com/queries/4-line-chart-format-1)
279
+
280
+ ```sql
281
+ SELECT date_trunc('week', created_at), COUNT(*) FROM users GROUP BY 1
282
+ ```
283
+
284
+ 3 columns - timestamp, string, numeric - [Example](https://blazerme.herokuapp.com/queries/5-line-chart-format-2)
285
+
286
+
287
+ ```sql
288
+ SELECT date_trunc('week', created_at), gender, COUNT(*) FROM users GROUP BY 1, 2
289
+ ```
290
+
291
+ ### Column Chart
292
+
293
+ There are also two ways to generate column charts.
294
+
295
+ 2+ columns - string, numeric(s) - [Example](https://blazerme.herokuapp.com/queries/6-column-chart-format-1)
296
+
297
+ ```sql
298
+ SELECT gender, COUNT(*) FROM users GROUP BY 1
299
+ ```
300
+
301
+ 3 columns - string, string, numeric - [Example](https://blazerme.herokuapp.com/queries/7-column-chart-format-2)
302
+
303
+ ```sql
304
+ SELECT gender, zip_code, COUNT(*) FROM users GROUP BY 1, 2
305
+ ```
306
+
307
+ ### Scatter Chart
308
+
309
+ 2 columns - both numeric
310
+
311
+ ```sql
312
+ SELECT x, y FROM table
313
+ ```
314
+
315
+ ### Maps
316
+
317
+ Columns named `latitude` and `longitude` or `lat` and `lon` or `lat` and `lng` - [Example](https://blazerme.herokuapp.com/queries/15-map)
318
+
319
+ ```sql
320
+ SELECT name, latitude, longitude FROM cities
321
+ ```
322
+
323
+ To enable, get an access token from [Mapbox](https://www.mapbox.com/) and set `ENV["MAPBOX_ACCESS_TOKEN"]`.
324
+
325
+ ### Targets
326
+
327
+ Use the column name `target` to draw a line for goals. [Example](https://blazerme.herokuapp.com/queries/8-target-line)
328
+
329
+ ```sql
330
+ SELECT date_trunc('week', created_at), COUNT(*) AS new_users, 100000 AS target FROM users GROUP BY 1
331
+ ```
332
+
333
+ ## Dashboards
334
+
335
+ Create a dashboard with multiple queries. [Example](https://blazerme.herokuapp.com/dashboards/1-dashboard-demo)
336
+
337
+ If the query has a chart, the chart is shown. Otherwise, you’ll see a table.
338
+
339
+ If any queries have variables, they will show up on the dashboard.
340
+
341
+ ## Checks
342
+
343
+ Checks give you a centralized place to see the health of your data. [Example](https://blazerme.herokuapp.com/checks)
344
+
345
+ Create a query to identify bad rows.
346
+
347
+ ```sql
348
+ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
349
+ ```
350
+
351
+ Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
352
+
353
+ ## Anomaly Detection
354
+
355
+ Anomaly detection is supported thanks to Twitter’s [AnomalyDetection](https://github.com/twitter/AnomalyDetection) library.
356
+
357
+ First, [install R](https://cloud.r-project.org/). Then, run:
358
+
359
+ ```R
360
+ install.packages("devtools")
361
+ devtools::install_github("twitter/AnomalyDetection")
362
+ ```
363
+
364
+ And add to `config/blazer.yml`:
365
+
366
+ ```yml
367
+ anomaly_checks: true
368
+ ```
369
+
370
+ If upgrading from version 1.4 or below, also follow the [upgrade instructions](#15).
371
+
372
+ If you’re on Heroku, follow [these additional instructions](#anomaly-detection-on-heroku).
373
+
374
+ ## Data Sources
375
+
376
+ Blazer supports multiple data sources :tada:
377
+
378
+ Add additional data sources in `config/blazer.yml`:
379
+
380
+ ```yml
381
+ data_sources:
382
+ main:
383
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
384
+ # timeout, smart_variables, linked_columns, smart_columns
385
+ catalog:
386
+ url: <%= ENV["CATALOG_DATABASE_URL"] %>
387
+ # ...
388
+ redshift:
389
+ url: <%= ENV["REDSHIFT_DATABASE_URL"] %>
390
+ # ...
391
+ ```
392
+
393
+ ### Full List
394
+
395
+ - [PostgreSQL](#postgresql-1)
396
+ - [MySQL](#mysql-1)
397
+ - [SQL Server](#sql-server)
398
+ - [Oracle](#oracle)
399
+ - [IBM DB2 and Informix](#ibm-db2-and-informix)
400
+ - [SQLite](#sqlite)
401
+ - [Amazon Redshift](#amazon-redshift)
402
+ - [Amazon Athena](#amazon-athena-master) [master]
403
+ - [Presto](#presto)
404
+ - [Apache Drill](#apache-drill)
405
+ - [Google BigQuery](#google-bigquery)
406
+ - [MongoDB](#mongodb-1)
407
+ - [Elasticsearch](#elasticsearch-beta) [beta]
408
+
409
+ You can also [create an adapter](#creating-an-adapter) for any other data store.
410
+
411
+ **Note:** In the examples below, we recommend using environment variables for urls.
412
+
413
+ ```yml
414
+ data_sources:
415
+ my_source:
416
+ url: <%= ENV["BLAZER_MY_SOURCE_URL"] %>
417
+ ```
418
+
419
+ ### PostgreSQL
420
+
421
+ Add [pg](https://bitbucket.org/ged/ruby-pg/wiki/Home) to your Gemfile (if it’s not there) and set:
422
+
423
+ ```yml
424
+ data_sources:
425
+ my_source:
426
+ url: postgres://user:password@hostname:5432/database
427
+ ```
428
+
429
+ ### MySQL
430
+
431
+ Add [mysql2](https://github.com/brianmario/mysql2) to your Gemfile (if it’s not there) and set:
432
+
433
+ ```yml
434
+ data_sources:
435
+ my_source:
436
+ url: mysql2://user:password@hostname:3306/database
437
+ ```
438
+
439
+ ### SQL Server
440
+
441
+ Add [tiny_tds](https://github.com/rails-sqlserver/tiny_tds) and [activerecord-sqlserver-adapter](https://github.com/rails-sqlserver/activerecord-sqlserver-adapter) to your Gemfile and set:
442
+
443
+ ```yml
444
+ data_sources:
445
+ my_source:
446
+ url: sqlserver://user:password@hostname:1433/database
447
+ ```
448
+
449
+ ### Oracle
450
+
451
+ Use [activerecord-oracle_enhanced-adapter](https://github.com/rsim/oracle-enhanced).
452
+
453
+ ### IBM DB2 and Informix
454
+
455
+ Use [ibm_db](https://github.com/ibmdb/ruby-ibmdb).
456
+
457
+ ### SQLite
458
+
459
+ Add [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) to your Gemfile and set:
460
+
461
+ ```yml
462
+ data_sources:
463
+ my_source:
464
+ url: sqlite3:path/to/database.sqlite3
465
+ ```
466
+
467
+ ### Amazon Redshift
468
+
469
+ Add [activerecord4-redshift-adapter](https://github.com/aamine/activerecord4-redshift-adapter) or [activerecord5-redshift-adapter](https://github.com/ConsultingMD/activerecord5-redshift-adapter) to your Gemfile and set:
470
+
471
+ ```yml
472
+ data_sources:
473
+ my_source:
474
+ url: redshift://user:password@hostname:5439/database
475
+ ```
476
+
477
+ ### Amazon Athena [master]
478
+
479
+ Add [aws-sdk](https://github.com/aws/aws-sdk-ruby) `~> 2` to your Gemfile and set:
480
+
481
+ ```yml
482
+ data_sources:
483
+ my_source:
484
+ adapter: athena
485
+ database: database
486
+ output_location: s3://some-bucket/
487
+ ```
488
+
489
+ ### Presto
490
+
491
+ Add [presto-client](https://github.com/treasure-data/presto-client-ruby) to your Gemfile and set:
492
+
493
+ ```yml
494
+ data_sources:
495
+ my_source:
496
+ url: presto://user@hostname:8080/catalog
497
+ ```
498
+
499
+ ### Apache Drill
500
+
501
+ Add [drill-sergeant](https://github.com/ankane/drill-sergeant) to your Gemfile and set:
502
+
503
+ ```yml
504
+ data_sources:
505
+ my_source:
506
+ adapter: drill
507
+ url: http://hostname:8047
508
+ ```
509
+
510
+ ### Google BigQuery
511
+
512
+ Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:
513
+
514
+ ```yml
515
+ data_sources:
516
+ my_source:
517
+ adapter: bigquery
518
+ project: your-project
519
+ keyfile: path/to/keyfile.json
520
+ ```
521
+
522
+ ### MongoDB
523
+
524
+ Add [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:
525
+
526
+ ```yml
527
+ data_sources:
528
+ my_source:
529
+ url: mongodb://user:password@hostname:27017/database
530
+ ```
531
+
532
+ ### Elasticsearch [beta]
533
+
534
+ Add [elasticsearch](https://github.com/elastic/elasticsearch-ruby) to your Gemfile and set:
535
+
536
+ ```yml
537
+ data_sources:
538
+ my_source:
539
+ adapter: elasticsearch
540
+ url: http://user:password@hostname:9200
541
+ ```
542
+
543
+ ## Creating an Adapter
544
+
545
+ Create an adapter for any data store with:
546
+
547
+ ```ruby
548
+ class FooAdapter < Blazer::Adapters::BaseAdapter
549
+ # code goes here
550
+ end
551
+
552
+ Blazer.register_adapter "foo", FooAdapter
553
+ ```
554
+
555
+ See the [Presto adapter](https://github.com/ankane/blazer/blob/master/lib/blazer/adapters/presto_adapter.rb) for a good example. Then use:
556
+
557
+ ```yml
558
+ data_sources:
559
+ my_source:
560
+ adapter: foo
561
+ url: http://user:password@hostname:9200/
562
+ ```
563
+
564
+ ## Query Permissions
565
+
566
+ Blazer supports a basic permissions model.
567
+
568
+ 1. Queries without a name are unlisted
569
+ 2. Queries whose name starts with `#` are only listed to the creator
570
+ 3. Queries whose name starts with `*` can only be edited by the creator
571
+
572
+ ## Learn SQL
573
+
574
+ Have team members who want to learn SQL? Here are a few great, free resources.
575
+
576
+ - [Khan Academy](https://www.khanacademy.org/computing/computer-programming/sql)
577
+ - [Codecademy](https://www.codecademy.com/learn/learn-sql)
578
+
579
+ ## Useful Tools
580
+
581
+ For an easy way to group by day, week, month, and more with correct time zones, check out [Groupdate](https://github.com/ankane/groupdate.sql).
582
+
583
+ ## Anomaly Detection on Heroku
584
+
585
+ Add the [R buildpack](https://github.com/virtualstaticvoid/heroku-buildpack-r) to your app.
586
+
587
+ ```sh
588
+ heroku buildpacks:add --index 1 https://github.com/virtualstaticvoid/heroku-buildpack-r.git\#cedar-14
589
+ ```
590
+
591
+ And create an `init.r` with:
592
+
593
+ ```sh
594
+ if (!"AnomalyDetection" %in% installed.packages()) {
595
+ install.packages("devtools")
596
+ devtools::install_github("twitter/AnomalyDetection")
597
+ }
598
+ ```
599
+
600
+ Commit and deploy away. The first deploy may take a few minutes.
601
+
602
+ ## Upgrading
603
+
604
+ ### 1.5
605
+
606
+ To take advantage of the anomaly detection, create a migration
607
+
608
+ ```sh
609
+ rails g migration upgrade_blazer_to_1_5
610
+ ```
611
+
612
+ with:
613
+
614
+ ```ruby
615
+ add_column(:blazer_checks, :check_type, :string)
616
+ add_column(:blazer_checks, :message, :text)
617
+ commit_db_transaction
618
+
619
+ Blazer::Check.reset_column_information
620
+
621
+ Blazer::Check.where(invert: true).update_all(check_type: "missing_data")
622
+ Blazer::Check.where(check_type: nil).update_all(check_type: "bad_data")
623
+ ```
624
+
625
+ ### 1.3
626
+
627
+ To take advantage of the latest features, create a migration
628
+
629
+ ```sh
630
+ rails g migration upgrade_blazer_to_1_3
631
+ ```
632
+
633
+ with:
634
+
635
+ ```ruby
636
+ add_column :blazer_dashboards, :creator_id, :integer
637
+ add_column :blazer_checks, :creator_id, :integer
638
+ add_column :blazer_checks, :invert, :boolean
639
+ add_column :blazer_checks, :schedule, :string
640
+ add_column :blazer_checks, :last_run_at, :timestamp
641
+ commit_db_transaction
642
+
643
+ Blazer::Check.update_all schedule: "1 hour"
644
+ ```
645
+
646
+ ### 1.0
647
+
648
+ Blazer 1.0 brings a number of new features:
649
+
650
+ - multiple data sources, including Redshift
651
+ - dashboards
652
+ - checks
653
+
654
+ To upgrade, run:
655
+
656
+ ```sh
657
+ bundle update blazer
658
+ ```
659
+
660
+ Create a migration
661
+
662
+ ```sh
663
+ rails g migration upgrade_blazer_to_1_0
664
+ ```
665
+
666
+ with:
667
+
668
+ ```ruby
669
+ add_column :blazer_queries, :data_source, :string
670
+ add_column :blazer_audits, :data_source, :string
671
+
672
+ create_table :blazer_dashboards do |t|
673
+ t.text :name
674
+ t.timestamps
675
+ end
676
+
677
+ create_table :blazer_dashboard_queries do |t|
678
+ t.references :dashboard
679
+ t.references :query
680
+ t.integer :position
681
+ t.timestamps
682
+ end
683
+
684
+ create_table :blazer_checks do |t|
685
+ t.references :query
686
+ t.string :state
687
+ t.text :emails
688
+ t.timestamps
689
+ end
690
+ ```
691
+
692
+ And run:
693
+
694
+ ```sh
695
+ rake db:migrate
696
+ ```
697
+
698
+ Update `config/blazer.yml` with:
699
+
700
+ ```yml
701
+ # see https://github.com/ankane/blazer for more info
702
+
703
+ data_sources:
704
+ main:
705
+ url: <%= ENV["BLAZER_DATABASE_URL"] %>
706
+
707
+ # statement timeout, in seconds
708
+ # applies to PostgreSQL only
709
+ # none by default
710
+ # timeout: 15
711
+
712
+ # time to cache results, in minutes
713
+ # can greatly improve speed
714
+ # none by default
715
+ # cache: 60
716
+
717
+ # wrap queries in a transaction for safety
718
+ # not necessary if you use a read-only user
719
+ # true by default
720
+ # use_transaction: false
721
+
722
+ smart_variables:
723
+ # zone_id: "SELECT id, name FROM zones ORDER BY name ASC"
724
+
725
+ linked_columns:
726
+ # user_id: "/admin/users/{value}"
727
+
728
+ smart_columns:
729
+ # user_id: "SELECT id, name FROM users WHERE id IN {value}"
730
+
731
+ # create audits
732
+ audit: true
733
+
734
+ # change the time zone
735
+ # time_zone: "Pacific Time (US & Canada)"
736
+
737
+ # class name of the user model
738
+ # user_class: User
739
+
740
+ # method name for the current user
741
+ # user_method: current_user
742
+
743
+ # method name for the display name
744
+ # user_name: name
745
+
746
+ # email to send checks from
747
+ # from_email: blazer@example.org
748
+ ```
749
+
750
+ ## TODO
751
+
752
+ - advanced permissions
753
+ - standalone version
754
+ - better navigation
755
+
756
+ ## History
757
+
758
+ View the [changelog](https://github.com/ankane/blazer/blob/master/CHANGELOG.md)
759
+
760
+ ## Thanks
761
+
762
+ Blazer uses a number of awesome open source projects, including [Rails](https://github.com/rails/rails/), [Vue.js](https://github.com/vuejs/vue), [jQuery](https://github.com/jquery/jquery), [Bootstrap](https://github.com/twbs/bootstrap), [Selectize](https://github.com/brianreavis/selectize.js), [StickyTableHeaders](https://github.com/jmosbech/StickyTableHeaders), [Stupid jQuery Table Sort](https://github.com/joequery/Stupid-Table-Plugin), and [Date Range Picker](https://github.com/dangrossman/bootstrap-daterangepicker).
763
+
764
+ Demo data from [MovieLens](http://grouplens.org/datasets/movielens/).
765
+
766
+ ## Want to Make Blazer Better?
767
+
768
+ That’s awesome! Here are a few ways you can help:
769
+
770
+ - [Report bugs](https://github.com/ankane/blazer/issues)
771
+ - Fix bugs and [submit pull requests](https://github.com/ankane/blazer/pulls)
772
+ - Write, clarify, or fix documentation
773
+ - Suggest or add new features
774
+
775
+ Check out the [dev app](https://github.com/ankane/blazer-dev) to get started.