blazer 1.3.5 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of blazer might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +22 -10
- data/app/assets/javascripts/blazer/Chart.js +8820 -8686
- data/app/assets/javascripts/blazer/application.js +12 -0
- data/app/assets/javascripts/blazer/chartkick.js +114 -33
- data/app/controllers/blazer/base_controller.rb +0 -6
- data/app/controllers/blazer/checks_controller.rb +3 -2
- data/app/controllers/blazer/dashboards_controller.rb +3 -3
- data/app/controllers/blazer/queries_controller.rb +63 -75
- data/app/helpers/blazer/base_helper.rb +2 -2
- data/app/models/blazer/check.rb +17 -4
- data/app/views/blazer/checks/index.html.erb +2 -2
- data/app/views/blazer/checks/run.html.erb +3 -1
- data/app/views/blazer/dashboards/show.html.erb +6 -10
- data/app/views/blazer/queries/_form.html.erb +4 -1
- data/app/views/blazer/queries/run.html.erb +25 -17
- data/app/views/blazer/queries/show.html.erb +14 -10
- data/app/views/layouts/blazer/application.html.erb +2 -2
- data/blazer.gemspec +1 -0
- data/lib/blazer/data_source.rb +139 -37
- data/lib/blazer/version.rb +1 -1
- data/lib/blazer.rb +43 -30
- data/lib/generators/blazer/templates/config.yml +6 -3
- metadata +16 -2
@@ -1,7 +1,9 @@
|
|
1
1
|
<p style="text-muted">Running check...</p>
|
2
2
|
|
3
3
|
<script>
|
4
|
-
|
4
|
+
var data = <%= blazer_json_escape({statement: @query.statement, query_id: @query.id, check: true}.to_json).html_safe %>;
|
5
|
+
|
6
|
+
runQuery(data, function (data) {
|
5
7
|
setTimeout( function () {
|
6
8
|
window.location.href = "<%= checks_path %>";
|
7
9
|
}, 200);
|
@@ -32,9 +32,9 @@
|
|
32
32
|
|
33
33
|
<div style="margin-bottom: 60px;"></div>
|
34
34
|
|
35
|
-
<% if @data_sources.any? { |ds| ds.
|
35
|
+
<% if @data_sources.any? { |ds| ds.cache_mode != "off" } %>
|
36
36
|
<p class="text-muted" style="float: right;">
|
37
|
-
Some queries
|
37
|
+
Some queries may be cached
|
38
38
|
<%= link_to "Refresh", refresh_dashboard_path(@dashboard, variable_params), method: :post %>
|
39
39
|
</p>
|
40
40
|
<% end %>
|
@@ -143,16 +143,12 @@
|
|
143
143
|
</div>
|
144
144
|
</div>
|
145
145
|
<script>
|
146
|
-
var
|
147
|
-
|
148
|
-
|
149
|
-
data: <%= blazer_json_escape({statement: query.statement, query_id: query.id, only_chart: true}.to_json).html_safe %>,
|
150
|
-
dataType: "html"
|
151
|
-
}).done(function(data) {
|
146
|
+
var data = <%= blazer_json_escape({statement: query.statement, query_id: query.id, only_chart: true}.to_json).html_safe %>;
|
147
|
+
|
148
|
+
runQuery(data, function (data) {
|
152
149
|
$("#chart-<%= i %>").html(data);
|
153
150
|
$("#chart-<%= i %> table").stupidtable();
|
154
|
-
}
|
155
|
-
var message = (typeof errorThrown === "string") ? errorThrown : errorThrown.message;
|
151
|
+
}, function (message) {
|
156
152
|
$("#chart-<%= i %>").css("color", "red").html(message);
|
157
153
|
});
|
158
154
|
</script>
|
@@ -124,7 +124,10 @@
|
|
124
124
|
if (xhr) {
|
125
125
|
xhr.abort();
|
126
126
|
}
|
127
|
-
|
127
|
+
|
128
|
+
var data = $.extend({}, params, {statement: editor.getValue().replace(/\n/g, "\r\n"), data_source: $("#query_data_source").val()});
|
129
|
+
|
130
|
+
xhr = runQuery(data, function (data) {
|
128
131
|
$("#results").html(data);
|
129
132
|
|
130
133
|
error_line = /LINE (\d+)/g.exec($("#results").find('.alert-danger').text());
|
@@ -8,12 +8,15 @@
|
|
8
8
|
<% end %>
|
9
9
|
<% else %>
|
10
10
|
<% unless @only_chart %>
|
11
|
-
<% if @cached_at || @
|
11
|
+
<% if @cached_at || @just_cached %>
|
12
12
|
<p class="text-muted" style="float: right;">
|
13
13
|
<% if @cached_at %>
|
14
14
|
Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
|
15
15
|
<% elsif !params[:data_source] %>
|
16
16
|
Cached just now
|
17
|
+
<% if @data_source.cache_mode == "slow" %>
|
18
|
+
(over <%= "%g" % @data_source.cache_slow_threshold %>s)
|
19
|
+
<% end %>
|
17
20
|
<% end %>
|
18
21
|
|
19
22
|
<% if @query && !params[:data_source] %>
|
@@ -24,13 +27,14 @@
|
|
24
27
|
<p class="text-muted"><%= pluralize(@rows.size, "row") %></p>
|
25
28
|
<% end %>
|
26
29
|
<% if @rows.any? %>
|
27
|
-
<% values = @rows.first
|
30
|
+
<% values = @rows.first %>
|
28
31
|
<% chart_id = SecureRandom.hex %>
|
29
32
|
<% column_types = blazer_column_types(@columns, @rows, @boom) %>
|
30
33
|
<% chart_options = {id: chart_id, min: nil} %>
|
31
34
|
<% series_library = {} %>
|
32
|
-
<% @columns.
|
33
|
-
|
35
|
+
<% target_index = @columns.index { |k| k.downcase == "target" } %>
|
36
|
+
<% if target_index %>
|
37
|
+
<% series_library[target_index - 1] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
|
34
38
|
<% end %>
|
35
39
|
<% if blazer_maps? && @markers.any? %>
|
36
40
|
<div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
|
@@ -62,32 +66,35 @@
|
|
62
66
|
map.fitBounds(featureLayer.getBounds());
|
63
67
|
</script>
|
64
68
|
<% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>
|
65
|
-
|
66
|
-
<%= line_chart @columns.keys[1..-1].map{ |k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }, library: series_library[k]} }, chart_options %>
|
69
|
+
<%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: k, data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
|
67
70
|
<% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>
|
68
|
-
|
69
|
-
<%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] }, library: series_library[name]} }, chart_options %>
|
71
|
+
<%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
|
70
72
|
<% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>
|
71
|
-
|
72
|
-
<%= column_chart (values.size - 1).times.map { |i| name = @columns.keys[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[keys[0]] || {})[r[keys[0]].to_s] || r[keys[0]], r[keys[i + 1]]] } } }, id: chart_id %>
|
73
|
+
<%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
|
73
74
|
<% elsif @only_chart %>
|
74
75
|
<% if @rows.size == 1 && @rows.first.size == 1 %>
|
75
|
-
|
76
|
+
<% v = @rows.first.first %>
|
77
|
+
<% if v.is_a?(String) && v == "" %>
|
78
|
+
<div class="text-muted">empty string</div>
|
79
|
+
<% else %>
|
80
|
+
<p style="font-size: 160px;"><%= blazer_format_value(@columns.first, v) %></p>
|
81
|
+
<% end %>
|
76
82
|
<% else %>
|
77
83
|
<% @no_chart = true %>
|
78
84
|
<% end %>
|
79
85
|
<% end %>
|
80
86
|
|
81
87
|
<% unless @only_chart && !@no_chart %>
|
82
|
-
<% header_width = 100 / @
|
88
|
+
<% header_width = 100 / @columns.size.to_f %>
|
83
89
|
<div class="results-container">
|
84
|
-
<% if @columns
|
85
|
-
<pre><code><%= @rows.map { |r| r[
|
90
|
+
<% if @columns == ["QUERY PLAN"] %>
|
91
|
+
<pre><code><%= @rows.map { |r| r[0] }.join("\n") %></code></pre>
|
86
92
|
<% else %>
|
87
93
|
<table class="table results-table" style="margin-bottom: 0;">
|
88
94
|
<thead>
|
89
95
|
<tr>
|
90
|
-
<% @columns.
|
96
|
+
<% @columns.each_with_index do |key, i| %>
|
97
|
+
<% type = @column_types[i] %>
|
91
98
|
<th style="width: <%= header_width %>%;" data-sort="<%= type %>">
|
92
99
|
<div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
|
93
100
|
<%= key %>
|
@@ -99,10 +106,11 @@
|
|
99
106
|
<tbody>
|
100
107
|
<% @rows.each do |row| %>
|
101
108
|
<tr>
|
102
|
-
<% row.
|
109
|
+
<% row.each_with_index do |v, i| %>
|
110
|
+
<% k = @columns[i] %>
|
103
111
|
<td>
|
104
112
|
<% if v.is_a?(Time) %>
|
105
|
-
<% v = blazer_time_value(k, v) %>
|
113
|
+
<% v = blazer_time_value(@data_source, k, v) %>
|
106
114
|
<% end %>
|
107
115
|
|
108
116
|
<% unless v.nil? %>
|
@@ -157,23 +157,27 @@
|
|
157
157
|
</div>
|
158
158
|
|
159
159
|
<script>
|
160
|
-
|
161
|
-
url: "<%= run_queries_path %>",
|
162
|
-
method: "POST",
|
163
|
-
data: <%= blazer_json_escape(variable_params.merge(statement: @statement, query_id: @query.id).to_json).html_safe %>,
|
164
|
-
dataType: "html"
|
165
|
-
}).done(function(data) {
|
160
|
+
function showRun(data) {
|
166
161
|
$("#results").html(data);
|
167
162
|
$("#results table").stupidtable().stickyTableHeaders({fixedOffset: 60});
|
168
|
-
}
|
169
|
-
|
163
|
+
}
|
164
|
+
|
165
|
+
function showError(message) {
|
170
166
|
$("#results").css("color", "red").html(message);
|
171
|
-
}
|
167
|
+
}
|
168
|
+
|
169
|
+
var data = <%= blazer_json_escape(variable_params.merge(statement: @statement, query_id: @query.id).to_json).html_safe %>;
|
170
|
+
|
171
|
+
runQuery(data, showRun, showError);
|
172
172
|
</script>
|
173
173
|
<% end %>
|
174
174
|
|
175
175
|
<script>
|
176
|
-
|
176
|
+
// do not highlight really long queries
|
177
|
+
// this can lead to performance issues
|
178
|
+
if ($("code").text().length < 10000) {
|
179
|
+
hljs.initHighlightingOnLoad();
|
180
|
+
}
|
177
181
|
|
178
182
|
$(".form-inline input, .form-inline select").change( function () {
|
179
183
|
submitIfCompleted($(this).closest("form"));
|
@@ -6,10 +6,10 @@
|
|
6
6
|
<meta charset="utf-8" />
|
7
7
|
|
8
8
|
<%= stylesheet_link_tag "blazer/application" %>
|
9
|
+
<%= javascript_include_tag "blazer/application" %>
|
9
10
|
<script>
|
10
|
-
var
|
11
|
+
var runQueriesPath = <%= run_queries_path.to_json.html_safe %>;
|
11
12
|
</script>
|
12
|
-
<%= javascript_include_tag "blazer/application" %>
|
13
13
|
<% if blazer_maps? %>
|
14
14
|
<%= stylesheet_link_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.css" %>
|
15
15
|
<%= javascript_include_tag "https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.js" %>
|
data/blazer.gemspec
CHANGED
@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency "rails"
|
22
22
|
spec.add_dependency "chartkick"
|
23
|
+
spec.add_dependency "safely_block", ">= 0.1.1"
|
23
24
|
|
24
25
|
spec.add_development_dependency "bundler", "~> 1.7"
|
25
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
data/lib/blazer/data_source.rb
CHANGED
@@ -7,6 +7,11 @@ module Blazer
|
|
7
7
|
def initialize(id, settings)
|
8
8
|
@id = id
|
9
9
|
@settings = settings
|
10
|
+
|
11
|
+
unless settings["url"] || Rails.env.development?
|
12
|
+
raise Blazer::Error, "Empty url"
|
13
|
+
end
|
14
|
+
|
10
15
|
@connection_model =
|
11
16
|
Class.new(Blazer::Connection) do
|
12
17
|
def self.name
|
@@ -41,7 +46,32 @@ module Blazer
|
|
41
46
|
end
|
42
47
|
|
43
48
|
def cache
|
44
|
-
|
49
|
+
@cache ||= begin
|
50
|
+
if settings["cache"].is_a?(Hash)
|
51
|
+
settings["cache"]
|
52
|
+
elsif settings["cache"]
|
53
|
+
{
|
54
|
+
"mode" => "all",
|
55
|
+
"expires_in" => settings["cache"]
|
56
|
+
}
|
57
|
+
else
|
58
|
+
{
|
59
|
+
"mode" => "off"
|
60
|
+
}
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def cache_mode
|
66
|
+
cache["mode"]
|
67
|
+
end
|
68
|
+
|
69
|
+
def cache_expires_in
|
70
|
+
(cache["expires_in"] || 60).to_f
|
71
|
+
end
|
72
|
+
|
73
|
+
def cache_slow_threshold
|
74
|
+
(cache["slow_threshold"] || 15).to_f
|
45
75
|
end
|
46
76
|
|
47
77
|
def local_time_suffix
|
@@ -52,19 +82,70 @@ module Blazer
|
|
52
82
|
settings.key?("use_transaction") ? settings["use_transaction"] : true
|
53
83
|
end
|
54
84
|
|
85
|
+
def cost(statement)
|
86
|
+
result = explain(statement)
|
87
|
+
match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
|
88
|
+
match[1] if match
|
89
|
+
end
|
90
|
+
|
91
|
+
def explain(statement)
|
92
|
+
if postgresql? || redshift?
|
93
|
+
connection_model.connection.select_all("EXPLAIN #{statement}").rows.first.first
|
94
|
+
end
|
95
|
+
rescue
|
96
|
+
nil
|
97
|
+
end
|
98
|
+
|
99
|
+
def run_main_statement(statement, options = {})
|
100
|
+
query = options[:query]
|
101
|
+
Blazer.transform_statement.call(self, statement) if Blazer.transform_statement
|
102
|
+
|
103
|
+
# audit
|
104
|
+
if Blazer.audit
|
105
|
+
audit = Blazer::Audit.new(statement: statement)
|
106
|
+
audit.query = query
|
107
|
+
audit.data_source = id
|
108
|
+
audit.user = options[:user]
|
109
|
+
audit.save!
|
110
|
+
end
|
111
|
+
|
112
|
+
start_time = Time.now
|
113
|
+
columns, rows, error, cached_at, just_cached = run_statement(statement, options.merge(with_just_cached: true))
|
114
|
+
duration = Time.now - start_time
|
115
|
+
|
116
|
+
if Blazer.audit
|
117
|
+
audit.duration = duration if audit.respond_to?(:duration=)
|
118
|
+
audit.error = error if audit.respond_to?(:error=)
|
119
|
+
audit.timed_out = error == Blazer::TIMEOUT_MESSAGE if audit.respond_to?(:timed_out=)
|
120
|
+
audit.cached = cached_at.present? if audit.respond_to?(:cached=)
|
121
|
+
if !cached_at && duration >= 10
|
122
|
+
audit.cost = cost(statement) if audit.respond_to?(:cost=)
|
123
|
+
end
|
124
|
+
audit.save! if audit.changed?
|
125
|
+
end
|
126
|
+
|
127
|
+
if query && error != Blazer::TIMEOUT_MESSAGE
|
128
|
+
query.checks.each do |check|
|
129
|
+
check.update_state(rows, error)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
[columns, rows, error, cached_at, just_cached]
|
134
|
+
end
|
135
|
+
|
55
136
|
def run_statement(statement, options = {})
|
137
|
+
columns = nil
|
56
138
|
rows = nil
|
57
139
|
error = nil
|
58
140
|
cached_at = nil
|
141
|
+
just_cached = false
|
59
142
|
cache_key = self.cache_key(statement) if cache
|
60
143
|
if cache && !options[:refresh_cache]
|
61
144
|
value = Blazer.cache.read(cache_key)
|
62
|
-
rows, cached_at = Marshal.load(value) if value
|
145
|
+
columns, rows, cached_at = Marshal.load(value) if value
|
63
146
|
end
|
64
147
|
|
65
148
|
unless rows
|
66
|
-
rows = []
|
67
|
-
|
68
149
|
comment = "blazer"
|
69
150
|
if options[:user].respond_to?(:id)
|
70
151
|
comment << ",user_id:#{options[:user].id}"
|
@@ -76,38 +157,12 @@ module Blazer
|
|
76
157
|
if options[:query].respond_to?(:id)
|
77
158
|
comment << ",query_id:#{options[:query].id}"
|
78
159
|
end
|
79
|
-
|
80
|
-
in_transaction do
|
81
|
-
begin
|
82
|
-
if timeout
|
83
|
-
if postgresql? || redshift?
|
84
|
-
connection_model.connection.execute("SET statement_timeout = #{timeout.to_i * 1000}")
|
85
|
-
elsif mysql?
|
86
|
-
connection_model.connection.execute("SET max_execution_time = #{timeout.to_i * 1000}")
|
87
|
-
else
|
88
|
-
raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
|
93
|
-
cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
|
94
|
-
result.each do |untyped_row|
|
95
|
-
row = {}
|
96
|
-
untyped_row.each do |k, v|
|
97
|
-
row[k] = result.column_types.empty? ? v : result.column_types[k].send(cast_method, v)
|
98
|
-
end
|
99
|
-
rows << row
|
100
|
-
end
|
101
|
-
rescue ActiveRecord::StatementInvalid => e
|
102
|
-
error = e.message.sub(/.+ERROR: /, "")
|
103
|
-
error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
Blazer.cache.write(cache_key, Marshal.dump([rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache
|
160
|
+
columns, rows, error, just_cached = run_statement_helper(statement, comment)
|
108
161
|
end
|
109
162
|
|
110
|
-
[rows, error, cached_at]
|
163
|
+
output = [columns, rows, error, cached_at]
|
164
|
+
output << just_cached if options[:with_just_cached]
|
165
|
+
output
|
111
166
|
end
|
112
167
|
|
113
168
|
def clear_cache(statement)
|
@@ -115,7 +170,7 @@ module Blazer
|
|
115
170
|
end
|
116
171
|
|
117
172
|
def cache_key(statement)
|
118
|
-
["blazer", "
|
173
|
+
["blazer", "v3", id, Digest::MD5.hexdigest(statement)].join("/")
|
119
174
|
end
|
120
175
|
|
121
176
|
def schemas
|
@@ -124,8 +179,8 @@ module Blazer
|
|
124
179
|
end
|
125
180
|
|
126
181
|
def tables
|
127
|
-
rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name
|
128
|
-
|
182
|
+
columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))
|
183
|
+
rows.map(&:first)
|
129
184
|
end
|
130
185
|
|
131
186
|
def postgresql?
|
@@ -146,6 +201,53 @@ module Blazer
|
|
146
201
|
|
147
202
|
protected
|
148
203
|
|
204
|
+
def run_statement_helper(statement, comment)
|
205
|
+
columns = []
|
206
|
+
rows = []
|
207
|
+
error = nil
|
208
|
+
start_time = Time.now
|
209
|
+
result = nil
|
210
|
+
|
211
|
+
begin
|
212
|
+
in_transaction do
|
213
|
+
if timeout
|
214
|
+
if postgresql? || redshift?
|
215
|
+
connection_model.connection.execute("SET statement_timeout = #{timeout.to_i * 1000}")
|
216
|
+
elsif mysql?
|
217
|
+
connection_model.connection.execute("SET max_execution_time = #{timeout.to_i * 1000}")
|
218
|
+
else
|
219
|
+
raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
|
224
|
+
end
|
225
|
+
rescue ActiveRecord::StatementInvalid => e
|
226
|
+
error = e.message.sub(/.+ERROR: /, "")
|
227
|
+
error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
|
228
|
+
end
|
229
|
+
|
230
|
+
duration = Time.now - start_time
|
231
|
+
|
232
|
+
if result
|
233
|
+
columns = result.columns
|
234
|
+
cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
|
235
|
+
result.rows.each do |untyped_row|
|
236
|
+
rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(cast_method, untyped_row[i]) : nil })
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
cache_data = nil
|
241
|
+
if !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
|
242
|
+
cache_data = Marshal.dump([columns, rows, Time.now]) rescue nil
|
243
|
+
if cache_data
|
244
|
+
Blazer.cache.write(cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
[columns, rows, error, !cache_data.nil?]
|
249
|
+
end
|
250
|
+
|
149
251
|
def adapter_name
|
150
252
|
connection_model.connection.adapter_name
|
151
253
|
end
|
data/lib/blazer/version.rb
CHANGED
data/lib/blazer.rb
CHANGED
@@ -4,9 +4,11 @@ require "chartkick"
|
|
4
4
|
require "blazer/version"
|
5
5
|
require "blazer/data_source"
|
6
6
|
require "blazer/engine"
|
7
|
+
require "safely/core"
|
7
8
|
|
8
9
|
module Blazer
|
9
|
-
class
|
10
|
+
class Error < StandardError; end
|
11
|
+
class TimeoutNotSupported < Error; end
|
10
12
|
|
11
13
|
class << self
|
12
14
|
attr_accessor :audit
|
@@ -28,7 +30,9 @@ module Blazer
|
|
28
30
|
TIMEOUT_ERRORS = [
|
29
31
|
"canceling statement due to statement timeout", # postgres
|
30
32
|
"cancelled on user's request", # redshift
|
31
|
-
"
|
33
|
+
"canceled on user's request", # redshift
|
34
|
+
"system requested abort", # redshift
|
35
|
+
"maximum statement execution time exceeded" # mysql
|
32
36
|
]
|
33
37
|
BELONGS_TO_OPTIONAL = {}
|
34
38
|
BELONGS_TO_OPTIONAL[:optional] = true if Rails::VERSION::MAJOR >= 5
|
@@ -64,43 +68,52 @@ module Blazer
|
|
64
68
|
checks = Blazer::Check.includes(:query)
|
65
69
|
checks = checks.where(schedule: schedule) if schedule
|
66
70
|
checks.find_each do |check|
|
67
|
-
|
68
|
-
|
69
|
-
|
71
|
+
Safely.safely { run_check(check) }
|
72
|
+
end
|
73
|
+
end
|
70
74
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
if error == Blazer::TIMEOUT_MESSAGE
|
76
|
-
Rails.logger.info "[blazer timeout] query=#{check.query.name}"
|
77
|
-
tries += 1
|
78
|
-
sleep(10)
|
79
|
-
elsif error.to_s.start_with?("PG::ConnectionBad")
|
80
|
-
data_sources[check.query.data_source].reconnect
|
81
|
-
Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
|
82
|
-
tries += 1
|
83
|
-
sleep(10)
|
84
|
-
else
|
85
|
-
break
|
86
|
-
end
|
87
|
-
end
|
88
|
-
check.update_state(rows, error)
|
89
|
-
# TODO use proper logfmt
|
90
|
-
Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"
|
75
|
+
def self.run_check(check)
|
76
|
+
rows = nil
|
77
|
+
error = nil
|
78
|
+
tries = 1
|
91
79
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
80
|
+
ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
|
81
|
+
# try 3 times on timeout errors
|
82
|
+
data_source = data_sources[check.query.data_source]
|
83
|
+
statement = check.query.statement
|
84
|
+
Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
|
85
|
+
|
86
|
+
while tries <= 3
|
87
|
+
columns, rows, error, cached_at = data_source.run_statement(statement, refresh_cache: true)
|
88
|
+
if error == Blazer::TIMEOUT_MESSAGE
|
89
|
+
Rails.logger.info "[blazer timeout] query=#{check.query.name}"
|
90
|
+
tries += 1
|
91
|
+
sleep(10)
|
92
|
+
elsif error.to_s.start_with?("PG::ConnectionBad")
|
93
|
+
data_sources[check.query.data_source].reconnect
|
94
|
+
Rails.logger.info "[blazer reconnect] query=#{check.query.name}"
|
95
|
+
tries += 1
|
96
|
+
sleep(10)
|
97
|
+
else
|
98
|
+
break
|
99
|
+
end
|
96
100
|
end
|
101
|
+
check.update_state(rows, error)
|
102
|
+
# TODO use proper logfmt
|
103
|
+
Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"
|
97
104
|
|
105
|
+
instrument[:statement] = statement
|
106
|
+
instrument[:data_source] = data_source
|
107
|
+
instrument[:state] = check.state
|
108
|
+
instrument[:rows] = rows.try(:size)
|
109
|
+
instrument[:error] = error
|
110
|
+
instrument[:tries] = tries
|
98
111
|
end
|
99
112
|
end
|
100
113
|
|
101
114
|
def self.send_failing_checks
|
102
115
|
emails = {}
|
103
|
-
Blazer::Check.includes(:query).where(state:
|
116
|
+
Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
|
104
117
|
check.split_emails.each do |email|
|
105
118
|
(emails[email] ||= []) << check
|
106
119
|
end
|
@@ -8,10 +8,13 @@ data_sources:
|
|
8
8
|
# none by default
|
9
9
|
# timeout: 15
|
10
10
|
|
11
|
-
#
|
11
|
+
# caching settings
|
12
12
|
# can greatly improve speed
|
13
|
-
#
|
14
|
-
# cache:
|
13
|
+
# off by default
|
14
|
+
# cache:
|
15
|
+
# mode: slow # or all
|
16
|
+
# expires_in: 60 # min
|
17
|
+
# slow_threshold: 15 # sec, only used in slow mode
|
15
18
|
|
16
19
|
# wrap queries in a transaction for safety
|
17
20
|
# not necessary if you use a read-only user
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: blazer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: safely_block
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.1.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.1.1
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bundler
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|