finery 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +426 -0
- data/CONTRIBUTING.md +49 -0
- data/LICENSE.txt +25 -0
- data/README.md +1144 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
- data/app/assets/images/blazer/favicon.png +0 -0
- data/app/assets/javascripts/blazer/Sortable.js +3709 -0
- data/app/assets/javascripts/blazer/ace/ace.js +19630 -0
- data/app/assets/javascripts/blazer/ace/ext-language_tools.js +1981 -0
- data/app/assets/javascripts/blazer/ace/mode-sql.js +215 -0
- data/app/assets/javascripts/blazer/ace/snippets/sql.js +16 -0
- data/app/assets/javascripts/blazer/ace/snippets/text.js +9 -0
- data/app/assets/javascripts/blazer/ace/theme-twilight.js +18 -0
- data/app/assets/javascripts/blazer/ace.js +6 -0
- data/app/assets/javascripts/blazer/application.js +87 -0
- data/app/assets/javascripts/blazer/bootstrap.js +2580 -0
- data/app/assets/javascripts/blazer/chart.umd.js +13 -0
- data/app/assets/javascripts/blazer/chartjs-adapter-date-fns.bundle.js +6322 -0
- data/app/assets/javascripts/blazer/chartjs-plugin-annotation.min.js +7 -0
- data/app/assets/javascripts/blazer/chartkick.js +2570 -0
- data/app/assets/javascripts/blazer/daterangepicker.js +1578 -0
- data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
- data/app/assets/javascripts/blazer/highlight.min.js +466 -0
- data/app/assets/javascripts/blazer/jquery.js +10872 -0
- data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +325 -0
- data/app/assets/javascripts/blazer/mapkick.bundle.js +1029 -0
- data/app/assets/javascripts/blazer/moment-timezone-with-data.js +1548 -0
- data/app/assets/javascripts/blazer/moment.js +5685 -0
- data/app/assets/javascripts/blazer/queries.js +130 -0
- data/app/assets/javascripts/blazer/rails-ujs.js +746 -0
- data/app/assets/javascripts/blazer/routes.js +26 -0
- data/app/assets/javascripts/blazer/selectize.js +3891 -0
- data/app/assets/javascripts/blazer/stupidtable-custom-settings.js +13 -0
- data/app/assets/javascripts/blazer/stupidtable.js +281 -0
- data/app/assets/javascripts/blazer/vue.global.prod.js +1 -0
- data/app/assets/stylesheets/blazer/application.css +243 -0
- data/app/assets/stylesheets/blazer/bootstrap-propshaft.css +10 -0
- data/app/assets/stylesheets/blazer/bootstrap-sprockets.css.erb +10 -0
- data/app/assets/stylesheets/blazer/bootstrap.css +6828 -0
- data/app/assets/stylesheets/blazer/daterangepicker.css +410 -0
- data/app/assets/stylesheets/blazer/github.css +125 -0
- data/app/assets/stylesheets/blazer/selectize.css +403 -0
- data/app/controllers/blazer/base_controller.rb +133 -0
- data/app/controllers/blazer/checks_controller.rb +56 -0
- data/app/controllers/blazer/dashboards_controller.rb +99 -0
- data/app/controllers/blazer/queries_controller.rb +468 -0
- data/app/controllers/blazer/uploads_controller.rb +147 -0
- data/app/helpers/blazer/base_helper.rb +83 -0
- data/app/models/blazer/audit.rb +6 -0
- data/app/models/blazer/check.rb +104 -0
- data/app/models/blazer/connection.rb +5 -0
- data/app/models/blazer/dashboard.rb +17 -0
- data/app/models/blazer/dashboard_query.rb +9 -0
- data/app/models/blazer/query.rb +42 -0
- data/app/models/blazer/record.rb +5 -0
- data/app/models/blazer/upload.rb +11 -0
- data/app/models/blazer/uploads_connection.rb +7 -0
- data/app/views/blazer/_nav.html.erb +18 -0
- data/app/views/blazer/_variables.html.erb +127 -0
- data/app/views/blazer/check_mailer/failing_checks.html.erb +7 -0
- data/app/views/blazer/check_mailer/state_change.html.erb +48 -0
- data/app/views/blazer/checks/_form.html.erb +79 -0
- data/app/views/blazer/checks/edit.html.erb +3 -0
- data/app/views/blazer/checks/index.html.erb +72 -0
- data/app/views/blazer/checks/new.html.erb +3 -0
- data/app/views/blazer/dashboards/_form.html.erb +82 -0
- data/app/views/blazer/dashboards/edit.html.erb +3 -0
- data/app/views/blazer/dashboards/new.html.erb +3 -0
- data/app/views/blazer/dashboards/show.html.erb +53 -0
- data/app/views/blazer/queries/_caching.html.erb +16 -0
- data/app/views/blazer/queries/_cohorts.html.erb +48 -0
- data/app/views/blazer/queries/_form.html.erb +255 -0
- data/app/views/blazer/queries/docs.html.erb +147 -0
- data/app/views/blazer/queries/edit.html.erb +2 -0
- data/app/views/blazer/queries/home.html.erb +169 -0
- data/app/views/blazer/queries/new.html.erb +2 -0
- data/app/views/blazer/queries/run.html.erb +188 -0
- data/app/views/blazer/queries/schema.html.erb +55 -0
- data/app/views/blazer/queries/show.html.erb +72 -0
- data/app/views/blazer/uploads/_form.html.erb +27 -0
- data/app/views/blazer/uploads/edit.html.erb +3 -0
- data/app/views/blazer/uploads/index.html.erb +55 -0
- data/app/views/blazer/uploads/new.html.erb +3 -0
- data/app/views/layouts/blazer/application.html.erb +25 -0
- data/config/routes.rb +25 -0
- data/lib/blazer/adapters/athena_adapter.rb +182 -0
- data/lib/blazer/adapters/base_adapter.rb +76 -0
- data/lib/blazer/adapters/bigquery_adapter.rb +79 -0
- data/lib/blazer/adapters/cassandra_adapter.rb +70 -0
- data/lib/blazer/adapters/clickhouse_adapter.rb +84 -0
- data/lib/blazer/adapters/drill_adapter.rb +38 -0
- data/lib/blazer/adapters/druid_adapter.rb +102 -0
- data/lib/blazer/adapters/elasticsearch_adapter.rb +61 -0
- data/lib/blazer/adapters/hive_adapter.rb +55 -0
- data/lib/blazer/adapters/ignite_adapter.rb +64 -0
- data/lib/blazer/adapters/influxdb_adapter.rb +57 -0
- data/lib/blazer/adapters/neo4j_adapter.rb +62 -0
- data/lib/blazer/adapters/opensearch_adapter.rb +52 -0
- data/lib/blazer/adapters/presto_adapter.rb +54 -0
- data/lib/blazer/adapters/salesforce_adapter.rb +50 -0
- data/lib/blazer/adapters/snowflake_adapter.rb +82 -0
- data/lib/blazer/adapters/soda_adapter.rb +105 -0
- data/lib/blazer/adapters/spark_adapter.rb +14 -0
- data/lib/blazer/adapters/sql_adapter.rb +324 -0
- data/lib/blazer/adapters.rb +18 -0
- data/lib/blazer/annotations.rb +47 -0
- data/lib/blazer/anomaly_detectors.rb +22 -0
- data/lib/blazer/check_mailer.rb +27 -0
- data/lib/blazer/data_source.rb +270 -0
- data/lib/blazer/engine.rb +42 -0
- data/lib/blazer/forecasters.rb +7 -0
- data/lib/blazer/result.rb +178 -0
- data/lib/blazer/result_cache.rb +71 -0
- data/lib/blazer/run_statement.rb +44 -0
- data/lib/blazer/run_statement_job.rb +20 -0
- data/lib/blazer/slack_notifier.rb +94 -0
- data/lib/blazer/statement.rb +77 -0
- data/lib/blazer/version.rb +3 -0
- data/lib/blazer.rb +286 -0
- data/lib/finery.rb +3 -0
- data/lib/generators/blazer/install_generator.rb +22 -0
- data/lib/generators/blazer/templates/config.yml.tt +83 -0
- data/lib/generators/blazer/templates/install.rb.tt +47 -0
- data/lib/generators/blazer/templates/uploads.rb.tt +10 -0
- data/lib/generators/blazer/uploads_generator.rb +18 -0
- data/lib/tasks/blazer.rake +20 -0
- data/lib/tasks/finery.rake +20 -0
- data/licenses/LICENSE-ace.txt +24 -0
- data/licenses/LICENSE-bootstrap.txt +21 -0
- data/licenses/LICENSE-chart.js.txt +9 -0
- data/licenses/LICENSE-chartjs-adapter-date-fns.txt +9 -0
- data/licenses/LICENSE-chartkick.js.txt +22 -0
- data/licenses/LICENSE-date-fns.txt +21 -0
- data/licenses/LICENSE-daterangepicker.txt +21 -0
- data/licenses/LICENSE-fuzzysearch.txt +20 -0
- data/licenses/LICENSE-highlight.js.txt +29 -0
- data/licenses/LICENSE-jquery.txt +20 -0
- data/licenses/LICENSE-kurkle-color.txt +9 -0
- data/licenses/LICENSE-mapkick-bundle.txt +1029 -0
- data/licenses/LICENSE-moment-timezone.txt +20 -0
- data/licenses/LICENSE-moment.txt +22 -0
- data/licenses/LICENSE-rails-ujs.txt +20 -0
- data/licenses/LICENSE-selectize.txt +202 -0
- data/licenses/LICENSE-sortable.txt +21 -0
- data/licenses/LICENSE-stickytableheaders.txt +20 -0
- data/licenses/LICENSE-stupidtable.txt +19 -0
- data/licenses/LICENSE-vue.txt +21 -0
- metadata +250 -0
@@ -0,0 +1,270 @@
|
|
1
|
+
module Blazer
|
2
|
+
class DataSource
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
attr_reader :id, :settings
|
6
|
+
|
7
|
+
def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
|
8
|
+
|
9
|
+
def initialize(id, settings)
|
10
|
+
@id = id
|
11
|
+
@settings = settings
|
12
|
+
end
|
13
|
+
|
14
|
+
def adapter
|
15
|
+
settings["adapter"] || detect_adapter
|
16
|
+
end
|
17
|
+
|
18
|
+
def name
|
19
|
+
settings["name"] || @id
|
20
|
+
end
|
21
|
+
|
22
|
+
def linked_columns
|
23
|
+
settings["linked_columns"] || {}
|
24
|
+
end
|
25
|
+
|
26
|
+
def smart_columns
|
27
|
+
settings["smart_columns"] || {}
|
28
|
+
end
|
29
|
+
|
30
|
+
def smart_variables
|
31
|
+
settings["smart_variables"] || {}
|
32
|
+
end
|
33
|
+
|
34
|
+
def variable_defaults
|
35
|
+
settings["variable_defaults"] || {}
|
36
|
+
end
|
37
|
+
|
38
|
+
def annotations
|
39
|
+
settings["annotations"] || {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def timeout
|
43
|
+
settings["timeout"]
|
44
|
+
end
|
45
|
+
|
46
|
+
def cache
|
47
|
+
@cache ||= begin
|
48
|
+
if settings["cache"].is_a?(Hash)
|
49
|
+
settings["cache"]
|
50
|
+
elsif settings["cache"]
|
51
|
+
{
|
52
|
+
"mode" => "all",
|
53
|
+
"expires_in" => settings["cache"]
|
54
|
+
}
|
55
|
+
else
|
56
|
+
{
|
57
|
+
"mode" => "off"
|
58
|
+
}
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def cache_mode
|
64
|
+
cache["mode"]
|
65
|
+
end
|
66
|
+
|
67
|
+
def cache_expires_in
|
68
|
+
(cache["expires_in"] || 60).to_f
|
69
|
+
end
|
70
|
+
|
71
|
+
def cache_slow_threshold
|
72
|
+
(cache["slow_threshold"] || 15).to_f
|
73
|
+
end
|
74
|
+
|
75
|
+
def local_time_suffix
|
76
|
+
@local_time_suffix ||= Array(settings["local_time_suffix"])
|
77
|
+
end
|
78
|
+
|
79
|
+
def result_cache
|
80
|
+
@result_cache ||= Blazer::ResultCache.new(self)
|
81
|
+
end
|
82
|
+
|
83
|
+
def run_results(run_id)
|
84
|
+
result_cache.read_run(run_id)
|
85
|
+
end
|
86
|
+
|
87
|
+
def delete_results(run_id)
|
88
|
+
result_cache.delete_run(run_id)
|
89
|
+
end
|
90
|
+
|
91
|
+
def sub_variables(statement, vars)
|
92
|
+
statement = statement.dup
|
93
|
+
vars.each do |var, value|
|
94
|
+
# use block form to disable back-references
|
95
|
+
statement.gsub!("{#{var}}") { quote(value) }
|
96
|
+
end
|
97
|
+
statement
|
98
|
+
end
|
99
|
+
|
100
|
+
def run_statement(statement, options = {})
|
101
|
+
statement = Statement.new(statement, self) if statement.is_a?(String)
|
102
|
+
statement.bind unless statement.bind_statement
|
103
|
+
|
104
|
+
result = nil
|
105
|
+
if cache_mode != "off"
|
106
|
+
if options[:refresh_cache]
|
107
|
+
clear_cache(statement) # for checks
|
108
|
+
else
|
109
|
+
result = result_cache.read_statement(statement)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
unless result
|
114
|
+
comment = "blazer"
|
115
|
+
if options[:user].respond_to?(:id)
|
116
|
+
comment << ",user_id:#{options[:user].id}"
|
117
|
+
end
|
118
|
+
if options[:user].respond_to?(Blazer.user_name)
|
119
|
+
# only include letters, numbers, and spaces to prevent injection
|
120
|
+
comment << ",user_name:#{options[:user].send(Blazer.user_name).to_s.gsub(/[^a-zA-Z0-9 ]/, "")}"
|
121
|
+
end
|
122
|
+
if options[:query].respond_to?(:id)
|
123
|
+
comment << ",query_id:#{options[:query].id}"
|
124
|
+
end
|
125
|
+
if options[:check]
|
126
|
+
comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"
|
127
|
+
end
|
128
|
+
if options[:run_id]
|
129
|
+
comment << ",run_id:#{options[:run_id]}"
|
130
|
+
end
|
131
|
+
result = run_statement_helper(statement, comment, options)
|
132
|
+
end
|
133
|
+
|
134
|
+
if options[:async] && options[:run_id]
|
135
|
+
run_id = options[:run_id]
|
136
|
+
begin
|
137
|
+
result_cache.write_run(run_id, result)
|
138
|
+
rescue
|
139
|
+
result = Blazer::Result.new(self, [], [], "Error storing the results of this query :(", nil, false)
|
140
|
+
result_cache.write_run(run_id, result)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
result
|
145
|
+
end
|
146
|
+
|
147
|
+
def clear_cache(statement)
|
148
|
+
result_cache.delete_statement(statement)
|
149
|
+
end
|
150
|
+
|
151
|
+
def quote(value)
|
152
|
+
if quoting == :backslash_escape || quoting == :single_quote_escape
|
153
|
+
# only need to support types generated by process_vars
|
154
|
+
if value.is_a?(Integer) || value.is_a?(Float)
|
155
|
+
value.to_s
|
156
|
+
elsif value.nil?
|
157
|
+
"NULL"
|
158
|
+
else
|
159
|
+
value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
|
160
|
+
|
161
|
+
if quoting == :backslash_escape
|
162
|
+
"'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
|
163
|
+
else
|
164
|
+
"'#{value.gsub("'", "''")}'"
|
165
|
+
end
|
166
|
+
end
|
167
|
+
elsif quoting.respond_to?(:call)
|
168
|
+
quoting.call(value)
|
169
|
+
elsif quoting.nil?
|
170
|
+
raise Blazer::Error, "Quoting not specified"
|
171
|
+
else
|
172
|
+
raise Blazer::Error, "Unknown quoting"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def bind_params(statement, variables)
|
177
|
+
if parameter_binding == :positional
|
178
|
+
locations = []
|
179
|
+
variables.each do |k, v|
|
180
|
+
i = 0
|
181
|
+
while (idx = statement.index("{#{k}}", i))
|
182
|
+
locations << [v, idx]
|
183
|
+
i = idx + 1
|
184
|
+
end
|
185
|
+
end
|
186
|
+
variables.each do |k, v|
|
187
|
+
statement = statement.gsub("{#{k}}", "?")
|
188
|
+
end
|
189
|
+
[statement, locations.sort_by(&:last).map(&:first)]
|
190
|
+
elsif parameter_binding == :numeric
|
191
|
+
variables.each_with_index do |(k, v), i|
|
192
|
+
# add trailing space if followed by digit
|
193
|
+
# try to keep minimal to avoid fixing invalid queries like SELECT{var}
|
194
|
+
statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
|
195
|
+
end
|
196
|
+
[statement, variables.values]
|
197
|
+
elsif parameter_binding.respond_to?(:call)
|
198
|
+
parameter_binding.call(statement, variables)
|
199
|
+
elsif parameter_binding.nil?
|
200
|
+
[sub_variables(statement, variables), []]
|
201
|
+
else
|
202
|
+
raise Blazer::Error, "Unknown bind parameters"
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
protected
|
207
|
+
|
208
|
+
def adapter_instance
|
209
|
+
@adapter_instance ||= begin
|
210
|
+
# TODO add required settings to adapters
|
211
|
+
unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
|
212
|
+
raise Blazer::Error, "Empty url for data source: #{id}"
|
213
|
+
end
|
214
|
+
|
215
|
+
unless Blazer.adapters[adapter]
|
216
|
+
raise Blazer::Error, "Unknown adapter"
|
217
|
+
end
|
218
|
+
|
219
|
+
Blazer.adapters[adapter].new(self)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
def quoting
|
224
|
+
@quoting ||= adapter_instance.quoting
|
225
|
+
end
|
226
|
+
|
227
|
+
def parameter_binding
|
228
|
+
@parameter_binding ||= adapter_instance.parameter_binding
|
229
|
+
end
|
230
|
+
|
231
|
+
def run_statement_helper(statement, comment, options)
|
232
|
+
start_time = Blazer.monotonic_time
|
233
|
+
columns, rows, error =
|
234
|
+
if adapter_instance.parameter_binding
|
235
|
+
adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
|
236
|
+
else
|
237
|
+
adapter_instance.run_statement(statement.bind_statement, comment)
|
238
|
+
end
|
239
|
+
duration = Blazer.monotonic_time - start_time
|
240
|
+
|
241
|
+
cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
|
242
|
+
|
243
|
+
result = Blazer::Result.new(self, columns, rows, error, cache ? Time.now : nil, false)
|
244
|
+
|
245
|
+
if cache && adapter_instance.cachable?(statement.bind_statement)
|
246
|
+
begin
|
247
|
+
result_cache.write_statement(statement, result, expires_in: cache_expires_in.to_f * 60)
|
248
|
+
# set just_cached after caching
|
249
|
+
result.just_cached = true
|
250
|
+
rescue
|
251
|
+
# do nothing
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
result.cached_at = nil
|
256
|
+
result
|
257
|
+
end
|
258
|
+
|
259
|
+
# TODO check for adapter with same name, default to sql
|
260
|
+
def detect_adapter
|
261
|
+
scheme = settings["url"].to_s.split("://").first
|
262
|
+
case scheme
|
263
|
+
when "presto", "cassandra", "ignite"
|
264
|
+
scheme
|
265
|
+
else
|
266
|
+
"sql"
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Blazer
|
2
|
+
class Engine < ::Rails::Engine
|
3
|
+
isolate_namespace Blazer
|
4
|
+
|
5
|
+
initializer "blazer" do |app|
|
6
|
+
if defined?(Sprockets) && Sprockets::VERSION.to_i >= 4
|
7
|
+
app.config.assets.precompile += [
|
8
|
+
"blazer/application.js",
|
9
|
+
"blazer/application.css",
|
10
|
+
"blazer/glyphicons-halflings-regular.eot",
|
11
|
+
"blazer/glyphicons-halflings-regular.svg",
|
12
|
+
"blazer/glyphicons-halflings-regular.ttf",
|
13
|
+
"blazer/glyphicons-halflings-regular.woff",
|
14
|
+
"blazer/glyphicons-halflings-regular.woff2",
|
15
|
+
"blazer/favicon.png"
|
16
|
+
]
|
17
|
+
else
|
18
|
+
# use a proc instead of a string
|
19
|
+
app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
|
20
|
+
app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff|woff2)\z/ }
|
21
|
+
app.config.assets.precompile << proc { |path| path == "blazer/favicon.png" }
|
22
|
+
end
|
23
|
+
|
24
|
+
Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
|
25
|
+
Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
|
26
|
+
Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
|
27
|
+
Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
|
28
|
+
Blazer.before_action = Blazer.settings["before_action_method"] if Blazer.settings["before_action_method"]
|
29
|
+
Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
|
30
|
+
Blazer.cache ||= Rails.cache
|
31
|
+
|
32
|
+
Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
|
33
|
+
Blazer.forecasting = Blazer.settings["forecasting"] || false
|
34
|
+
Blazer.async = Blazer.settings["async"] || false
|
35
|
+
Blazer.images = Blazer.settings["images"] || false
|
36
|
+
Blazer.override_csp = Blazer.settings["override_csp"] || false
|
37
|
+
Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]
|
38
|
+
Blazer.slack_webhook_url = Blazer.settings["slack_webhook_url"] || ENV["BLAZER_SLACK_WEBHOOK_URL"]
|
39
|
+
Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"] || ENV["MAPBOX_ACCESS_TOKEN"]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
module Blazer
|
2
|
+
class Result
|
3
|
+
attr_reader :data_source, :columns, :rows, :error, :forecast_error
|
4
|
+
attr_accessor :cached_at, :just_cached
|
5
|
+
|
6
|
+
def initialize(data_source, columns, rows, error, cached_at, just_cached)
|
7
|
+
@data_source = data_source
|
8
|
+
@columns = columns
|
9
|
+
@rows = rows
|
10
|
+
@error = error
|
11
|
+
@cached_at = cached_at
|
12
|
+
@just_cached = just_cached
|
13
|
+
end
|
14
|
+
|
15
|
+
def timed_out?
|
16
|
+
error == Blazer::TIMEOUT_MESSAGE
|
17
|
+
end
|
18
|
+
|
19
|
+
def cached?
|
20
|
+
cached_at.present?
|
21
|
+
end
|
22
|
+
|
23
|
+
def smart_values
|
24
|
+
@smart_values ||= begin
|
25
|
+
smart_values = {}
|
26
|
+
columns.each_with_index do |key, i|
|
27
|
+
smart_columns_data_source =
|
28
|
+
([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_columns[key] }
|
29
|
+
|
30
|
+
if smart_columns_data_source
|
31
|
+
query = smart_columns_data_source.smart_columns[key]
|
32
|
+
res =
|
33
|
+
if query.is_a?(Hash)
|
34
|
+
query
|
35
|
+
else
|
36
|
+
values = rows.map { |r| r[i] }.compact.uniq
|
37
|
+
result = smart_columns_data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
|
38
|
+
result.rows
|
39
|
+
end
|
40
|
+
|
41
|
+
smart_values[key] = res.to_h { |k, v| [k.nil? ? k : k.to_s, v] }
|
42
|
+
end
|
43
|
+
end
|
44
|
+
smart_values
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def column_types
|
49
|
+
@column_types ||= begin
|
50
|
+
columns.each_with_index.map do |k, i|
|
51
|
+
v = (rows.find { |r| r[i] } || {})[i]
|
52
|
+
if smart_values[k]
|
53
|
+
"string"
|
54
|
+
elsif v.is_a?(Numeric)
|
55
|
+
"numeric"
|
56
|
+
elsif v.is_a?(Time) || v.is_a?(Date)
|
57
|
+
"time"
|
58
|
+
elsif v.nil?
|
59
|
+
nil
|
60
|
+
elsif v.is_a?(String) && v.encoding == Encoding::BINARY
|
61
|
+
"binary"
|
62
|
+
else
|
63
|
+
"string"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def chart_type
|
70
|
+
@chart_type ||= begin
|
71
|
+
if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }
|
72
|
+
"line"
|
73
|
+
elsif column_types == ["time", "string", "numeric"]
|
74
|
+
"line2"
|
75
|
+
elsif column_types == ["string", "numeric"] && @columns.last == "pie"
|
76
|
+
"pie"
|
77
|
+
elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
|
78
|
+
"bar"
|
79
|
+
elsif column_types == ["string", "string", "numeric"]
|
80
|
+
"bar2"
|
81
|
+
elsif column_types == ["numeric", "numeric"]
|
82
|
+
"scatter"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def forecastable?
|
88
|
+
@forecastable ||= Blazer.forecasting && column_types == ["time", "numeric"] && @rows.size >= 10
|
89
|
+
end
|
90
|
+
|
91
|
+
# TODO cache it?
|
92
|
+
# don't want to put result data (even hashed version)
|
93
|
+
# into cache without developer opt-in
|
94
|
+
def forecast
|
95
|
+
count = (@rows.size * 0.25).round.clamp(30, 365)
|
96
|
+
|
97
|
+
forecaster = Blazer.forecasters.fetch(Blazer.forecasting)
|
98
|
+
forecast = forecaster.call(@rows.to_h, count: count)
|
99
|
+
|
100
|
+
# round integers
|
101
|
+
if @rows[0][1].is_a?(Integer)
|
102
|
+
forecast = forecast.map { |k, v| [k, v.round] }.to_h
|
103
|
+
end
|
104
|
+
|
105
|
+
@rows.each do |row|
|
106
|
+
row[2] = nil
|
107
|
+
end
|
108
|
+
@rows.unshift(*forecast.map { |k, v| [k, nil, v] })
|
109
|
+
@columns << "forecast"
|
110
|
+
|
111
|
+
# reset cache
|
112
|
+
@column_types = nil
|
113
|
+
@chart_type = nil
|
114
|
+
|
115
|
+
forecast
|
116
|
+
rescue => e
|
117
|
+
@forecast_error = String.new("Error generating forecast")
|
118
|
+
@forecast_error << ": #{e.message.sub("Invalid parameter: ", "")}"
|
119
|
+
nil
|
120
|
+
end
|
121
|
+
|
122
|
+
def detect_anomaly
|
123
|
+
anomaly = nil
|
124
|
+
message = nil
|
125
|
+
|
126
|
+
if rows.empty?
|
127
|
+
message = "No data"
|
128
|
+
else
|
129
|
+
if chart_type == "line" || chart_type == "line2"
|
130
|
+
series = []
|
131
|
+
|
132
|
+
if chart_type == "line"
|
133
|
+
columns[1..-1].each_with_index.each do |k, i|
|
134
|
+
series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
|
135
|
+
end
|
136
|
+
else
|
137
|
+
rows.group_by { |r| v = r[1]; (smart_values[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
|
138
|
+
series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
current_series = nil
|
143
|
+
begin
|
144
|
+
anomalies = []
|
145
|
+
series.each do |s|
|
146
|
+
current_series = s[:name]
|
147
|
+
anomalies << s[:name] if anomaly?(s[:data])
|
148
|
+
end
|
149
|
+
anomaly = anomalies.any?
|
150
|
+
if anomaly
|
151
|
+
if anomalies.size == 1
|
152
|
+
message = "Anomaly detected in #{anomalies.first}"
|
153
|
+
else
|
154
|
+
message = "Anomalies detected in #{anomalies.to_sentence}"
|
155
|
+
end
|
156
|
+
else
|
157
|
+
message = "No anomalies detected"
|
158
|
+
end
|
159
|
+
rescue => e
|
160
|
+
message = "#{current_series}: #{e.message}"
|
161
|
+
raise e if Rails.env.development?
|
162
|
+
end
|
163
|
+
else
|
164
|
+
message = "Bad format"
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
[anomaly, message]
|
169
|
+
end
|
170
|
+
|
171
|
+
def anomaly?(series)
|
172
|
+
series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
|
173
|
+
|
174
|
+
anomaly_detector = Blazer.anomaly_detectors.fetch(Blazer.anomaly_checks)
|
175
|
+
anomaly_detector.call(series)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module Blazer
|
2
|
+
class ResultCache
|
3
|
+
def initialize(data_source)
|
4
|
+
@data_source = data_source
|
5
|
+
end
|
6
|
+
|
7
|
+
def write_run(run_id, result)
|
8
|
+
write(run_cache_key(run_id), result, expires_in: 30.seconds)
|
9
|
+
end
|
10
|
+
|
11
|
+
def read_run(run_id)
|
12
|
+
read(run_cache_key(run_id))
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete_run(run_id)
|
16
|
+
delete(run_cache_key(run_id))
|
17
|
+
end
|
18
|
+
|
19
|
+
def write_statement(statement, result, expires_in:)
|
20
|
+
write(statement_cache_key(statement), result, expires_in: expires_in) if caching?
|
21
|
+
end
|
22
|
+
|
23
|
+
def read_statement(statement)
|
24
|
+
read(statement_cache_key(statement)) if caching?
|
25
|
+
end
|
26
|
+
|
27
|
+
def delete_statement(statement)
|
28
|
+
delete(statement_cache_key(statement)) if caching?
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def write(key, result, expires_in:)
|
34
|
+
raise ArgumentError, "expected Blazer::Result" unless result.is_a?(Blazer::Result)
|
35
|
+
value = [result.columns, result.rows, result.error, result.cached_at, result.just_cached]
|
36
|
+
cache.write(key, value, expires_in: expires_in)
|
37
|
+
end
|
38
|
+
|
39
|
+
def read(key)
|
40
|
+
value = cache.read(key)
|
41
|
+
if value
|
42
|
+
columns, rows, error, cached_at, just_cached = value
|
43
|
+
Blazer::Result.new(@data_source, columns, rows, error, cached_at, just_cached)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def delete(key)
|
48
|
+
cache.delete(key)
|
49
|
+
end
|
50
|
+
|
51
|
+
def caching?
|
52
|
+
@data_source.cache_mode != "off"
|
53
|
+
end
|
54
|
+
|
55
|
+
def cache_key(key)
|
56
|
+
(["blazer", "v5", @data_source.id] + key).join("/")
|
57
|
+
end
|
58
|
+
|
59
|
+
def statement_cache_key(statement)
|
60
|
+
cache_key(["statement", Digest::SHA256.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
|
61
|
+
end
|
62
|
+
|
63
|
+
def run_cache_key(run_id)
|
64
|
+
cache_key(["run", run_id])
|
65
|
+
end
|
66
|
+
|
67
|
+
def cache
|
68
|
+
Blazer.cache
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Blazer
|
2
|
+
class RunStatement
|
3
|
+
def perform(statement, options = {})
|
4
|
+
query = options[:query]
|
5
|
+
|
6
|
+
data_source = statement.data_source
|
7
|
+
statement.bind
|
8
|
+
|
9
|
+
# audit
|
10
|
+
if Blazer.audit
|
11
|
+
audit_statement = statement.bind_statement
|
12
|
+
audit_statement += "\n\n#{statement.bind_values.to_json}" if statement.bind_values.any?
|
13
|
+
audit = Blazer::Audit.new(statement: audit_statement)
|
14
|
+
audit.query = query
|
15
|
+
audit.data_source = data_source.id
|
16
|
+
audit.user = options[:user]
|
17
|
+
audit.save!
|
18
|
+
end
|
19
|
+
|
20
|
+
start_time = Blazer.monotonic_time
|
21
|
+
result = data_source.run_statement(statement, options)
|
22
|
+
duration = Blazer.monotonic_time - start_time
|
23
|
+
|
24
|
+
if Blazer.audit
|
25
|
+
audit.duration = duration if audit.respond_to?(:duration=)
|
26
|
+
audit.error = result.error if audit.respond_to?(:error=)
|
27
|
+
audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
|
28
|
+
audit.cached = result.cached? if audit.respond_to?(:cached=)
|
29
|
+
if !result.cached? && duration >= 10
|
30
|
+
audit.cost = data_source.cost(statement) if audit.respond_to?(:cost=)
|
31
|
+
end
|
32
|
+
audit.save! if audit.changed?
|
33
|
+
end
|
34
|
+
|
35
|
+
if query && !result.timed_out? && !result.cached? && !query.variables.any?
|
36
|
+
query.checks.each do |check|
|
37
|
+
check.update_state(result)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
result
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Blazer
|
2
|
+
class RunStatementJob < ActiveJob::Base
|
3
|
+
self.queue_adapter = :async
|
4
|
+
|
5
|
+
def perform(data_source_id, statement, options)
|
6
|
+
statement = Blazer::Statement.new(statement, data_source_id)
|
7
|
+
statement.values = options.delete(:values)
|
8
|
+
data_source = statement.data_source
|
9
|
+
begin
|
10
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
11
|
+
Blazer::RunStatement.new.perform(statement, options)
|
12
|
+
end
|
13
|
+
rescue Exception => e
|
14
|
+
result = Blazer::Result.new(data_source, [], [], "Unknown error", nil, false)
|
15
|
+
data_source.result_cache.write_run(options[:run_id], result)
|
16
|
+
raise e
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|