flare 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +60 -1
- data/app/controllers/flare/jobs_controller.rb +1 -1
- data/app/controllers/flare/requests_controller.rb +1 -1
- data/app/controllers/flare/spans_controller.rb +1 -1
- data/app/helpers/flare/application_helper.rb +23 -0
- data/app/views/flare/jobs/index.html.erb +2 -1
- data/app/views/flare/requests/index.html.erb +2 -1
- data/app/views/flare/requests/show.html.erb +16 -7
- data/app/views/flare/spans/index.html.erb +2 -1
- data/lib/flare/cli/doctor_command.rb +4 -11
- data/lib/flare/cli/local_spans.rb +110 -0
- data/lib/flare/cli/nplus1_command.rb +97 -0
- data/lib/flare/cli/project.rb +28 -0
- data/lib/flare/cli/request_command.rb +120 -0
- data/lib/flare/cli/requests_command.rb +45 -0
- data/lib/flare/cli/setup_command.rb +64 -41
- data/lib/flare/cli/status_command.rb +5 -2
- data/lib/flare/cli.rb +12 -0
- data/lib/flare/client_headers.rb +7 -3
- data/lib/flare/configuration.rb +20 -0
- data/lib/flare/export_filter.rb +273 -0
- data/lib/flare/filtering_span_processor.rb +14 -0
- data/lib/flare/http_transport.rb +4 -0
- data/lib/flare/metric_span_processor.rb +51 -35
- data/lib/flare/metric_submitter.rb +9 -1
- data/lib/flare/recording_batch_span_processor.rb +9 -1
- data/lib/flare/redis_statement_processor.rb +29 -0
- data/lib/flare/revision.rb +82 -0
- data/lib/flare/source_location.rb +21 -16
- data/lib/flare/sql_mask.rb +235 -0
- data/lib/flare/sqlite_exporter.rb +12 -8
- data/lib/flare/storage/sqlite.rb +195 -144
- data/lib/flare/storage.rb +1 -40
- data/lib/flare/store_statement.rb +136 -0
- data/lib/flare/trace_blob.rb +9 -5
- data/lib/flare/trace_exporter.rb +18 -3
- data/lib/flare/upload_url_pool.rb +2 -15
- data/lib/flare/utf8.rb +20 -0
- data/lib/flare/version.rb +1 -1
- data/lib/flare.rb +21 -2
- data/public/flare-assets/flare.css +10 -60
- metadata +45 -32
|
@@ -4,6 +4,7 @@ require "sqlite3"
|
|
|
4
4
|
require "json"
|
|
5
5
|
|
|
6
6
|
require_relative "deadline"
|
|
7
|
+
require_relative "utf8"
|
|
7
8
|
|
|
8
9
|
module Flare
|
|
9
10
|
class SQLiteExporter
|
|
@@ -128,8 +129,11 @@ module Flare
|
|
|
128
129
|
span_data.name&.start_with?("Flare::")
|
|
129
130
|
end
|
|
130
131
|
|
|
132
|
+
# Stored in UTC because readers compare and sort these as strings, which only
|
|
133
|
+
# works when every row has the same offset. Local time breaks that at a clock
|
|
134
|
+
# change: 1:05 after falling back sorts before the 1:50 that came earlier.
|
|
131
135
|
def create_span(span_data)
|
|
132
|
-
now = Time.now.iso8601(6)
|
|
136
|
+
now = Time.now.utc.iso8601(6)
|
|
133
137
|
|
|
134
138
|
sql = <<~SQL
|
|
135
139
|
INSERT INTO flare_spans (name, kind, span_id, trace_id, parent_span_id, start_timestamp, end_timestamp, total_recorded_links, total_recorded_events, total_recorded_properties, created_at, updated_at)
|
|
@@ -137,7 +141,7 @@ module Flare
|
|
|
137
141
|
SQL
|
|
138
142
|
|
|
139
143
|
values = [
|
|
140
|
-
span_data.name,
|
|
144
|
+
UTF8.scrub(span_data.name),
|
|
141
145
|
span_data.kind.to_s,
|
|
142
146
|
span_data.hex_span_id,
|
|
143
147
|
span_data.hex_trace_id,
|
|
@@ -162,15 +166,15 @@ module Flare
|
|
|
162
166
|
end
|
|
163
167
|
|
|
164
168
|
def create_event(span_record_id, span_event)
|
|
165
|
-
now = Time.now.iso8601(6)
|
|
166
|
-
timestamp = span_event.timestamp ? Time.at(span_event.timestamp / 1_000_000_000.0).iso8601(6) : now
|
|
169
|
+
now = Time.now.utc.iso8601(6)
|
|
170
|
+
timestamp = span_event.timestamp ? Time.at(span_event.timestamp / 1_000_000_000.0).utc.iso8601(6) : now
|
|
167
171
|
|
|
168
172
|
sql = <<~SQL
|
|
169
173
|
INSERT INTO flare_events (span_id, name, created_at, updated_at)
|
|
170
174
|
VALUES (?, ?, ?, ?)
|
|
171
175
|
SQL
|
|
172
176
|
|
|
173
|
-
connection.execute(sql, [span_record_id, span_event.name, timestamp, now])
|
|
177
|
+
connection.execute(sql, [span_record_id, UTF8.scrub(span_event.name), timestamp, now])
|
|
174
178
|
event_record_id = connection.last_insert_row_id
|
|
175
179
|
create_properties("Flare::Event", event_record_id, span_event.attributes)
|
|
176
180
|
end
|
|
@@ -178,7 +182,7 @@ module Flare
|
|
|
178
182
|
def create_properties(owner_type, owner_id, attributes)
|
|
179
183
|
return unless attributes
|
|
180
184
|
|
|
181
|
-
now = Time.now.iso8601(6)
|
|
185
|
+
now = Time.now.utc.iso8601(6)
|
|
182
186
|
|
|
183
187
|
sql = <<~SQL
|
|
184
188
|
INSERT INTO flare_properties (key, value, value_type, owner_type, owner_id, created_at, updated_at)
|
|
@@ -189,8 +193,8 @@ module Flare
|
|
|
189
193
|
next if value.nil?
|
|
190
194
|
|
|
191
195
|
value_type = determine_value_type(value)
|
|
192
|
-
serialized_value = JSON.generate(value)
|
|
193
|
-
connection.execute(sql, [key, serialized_value, value_type, owner_type, owner_id, now, now])
|
|
196
|
+
serialized_value = JSON.generate(UTF8.scrub(value))
|
|
197
|
+
connection.execute(sql, [UTF8.scrub(key), serialized_value, value_type, owner_type, owner_id, now, now])
|
|
194
198
|
end
|
|
195
199
|
end
|
|
196
200
|
|
data/lib/flare/storage/sqlite.rb
CHANGED
|
@@ -8,22 +8,26 @@ module Flare
|
|
|
8
8
|
class SQLite < Base
|
|
9
9
|
MISSING_PARENT_ID = "0000000000000000"
|
|
10
10
|
|
|
11
|
-
#
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
# Namespaces of the controllers that come with Rails and Flare, as stored
|
|
12
|
+
# in code.namespace (the controller's class name).
|
|
13
|
+
RAILS_CONTROLLER_NAMESPACES = %w[
|
|
14
|
+
ActiveStorage::
|
|
15
|
+
ActionMailbox::
|
|
16
|
+
Rails::
|
|
17
|
+
Flare::
|
|
17
18
|
].freeze
|
|
18
19
|
|
|
19
|
-
|
|
20
|
+
# readonly: true opens an existing database as is, without creating
|
|
21
|
+
# tables, for tools that only inspect spans (like `flare requests`).
|
|
22
|
+
def initialize(database_path, readonly: false)
|
|
20
23
|
@database_path = database_path
|
|
24
|
+
@readonly = readonly
|
|
21
25
|
@mutex = Mutex.new
|
|
22
|
-
@setup =
|
|
26
|
+
@setup = readonly
|
|
23
27
|
end
|
|
24
28
|
|
|
25
29
|
# List root spans that are HTTP requests (for the requests index)
|
|
26
|
-
def list_requests(status: nil, method: nil, name: nil, origin: nil, limit: 50, offset: 0)
|
|
30
|
+
def list_requests(status: nil, method: nil, name: nil, origin: nil, sort: nil, limit: 50, offset: 0)
|
|
27
31
|
# Find root spans with kind=server that have http.method property
|
|
28
32
|
conditions = ["s.parent_span_id = ?", "s.kind = ?"]
|
|
29
33
|
values = [MISSING_PARENT_ID, "server"]
|
|
@@ -61,17 +65,7 @@ module Flare
|
|
|
61
65
|
values << "%#{name}%"
|
|
62
66
|
end
|
|
63
67
|
|
|
64
|
-
|
|
65
|
-
if origin == "rails"
|
|
66
|
-
controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
|
|
67
|
-
conditions << "(#{controller_conditions.join(" OR ")})"
|
|
68
|
-
RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
|
|
69
|
-
elsif origin == "app"
|
|
70
|
-
controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
|
|
71
|
-
conditions << "(controller_prop.value IS NULL OR NOT (#{controller_conditions.join(" OR ")}))"
|
|
72
|
-
RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
|
|
73
|
-
end
|
|
74
|
-
end
|
|
68
|
+
add_origin_condition(origin, conditions, values)
|
|
75
69
|
|
|
76
70
|
where_clause = "WHERE #{conditions.join(" AND ")}"
|
|
77
71
|
values << limit
|
|
@@ -95,7 +89,7 @@ module Flare
|
|
|
95
89
|
LEFT JOIN flare_properties action_prop ON action_prop.owner_type = 'Flare::Span' AND action_prop.owner_id = s.id AND action_prop.key = 'code.function'
|
|
96
90
|
#{where_clause}
|
|
97
91
|
AND COALESCE(method_prop.value, method_prop_old.value) IS NOT NULL
|
|
98
|
-
ORDER BY
|
|
92
|
+
ORDER BY #{span_order(sort)}
|
|
99
93
|
LIMIT ? OFFSET ?
|
|
100
94
|
SQL
|
|
101
95
|
|
|
@@ -103,7 +97,7 @@ module Flare
|
|
|
103
97
|
end
|
|
104
98
|
|
|
105
99
|
# List root spans that are jobs (for the jobs index)
|
|
106
|
-
def list_jobs(status: nil, name: nil, limit: 50, offset: 0)
|
|
100
|
+
def list_jobs(status: nil, name: nil, sort: nil, limit: 50, offset: 0)
|
|
107
101
|
# Find root spans with kind=consumer (ActiveJob processing)
|
|
108
102
|
conditions = ["s.parent_span_id = ?", "s.kind = ?"]
|
|
109
103
|
values = [MISSING_PARENT_ID, "consumer"]
|
|
@@ -125,7 +119,7 @@ module Flare
|
|
|
125
119
|
LEFT JOIN flare_properties job_class_prop ON job_class_prop.owner_type = 'Flare::Span' AND job_class_prop.owner_id = s.id AND job_class_prop.key = 'code.namespace'
|
|
126
120
|
LEFT JOIN flare_properties queue_prop ON queue_prop.owner_type = 'Flare::Span' AND queue_prop.owner_id = s.id AND queue_prop.key = 'messaging.destination'
|
|
127
121
|
#{where_clause}
|
|
128
|
-
ORDER BY
|
|
122
|
+
ORDER BY #{span_order(sort)}
|
|
129
123
|
LIMIT ? OFFSET ?
|
|
130
124
|
SQL
|
|
131
125
|
|
|
@@ -146,38 +140,39 @@ module Flare
|
|
|
146
140
|
# Transaction statements to filter out from queries list
|
|
147
141
|
TRANSACTION_STATEMENTS = %w[BEGIN COMMIT ROLLBACK].freeze
|
|
148
142
|
|
|
143
|
+
# Active Record names its own statements on every adapter, including ones
|
|
144
|
+
# the list above can't match (SQLite's "BEGIN immediate TRANSACTION",
|
|
145
|
+
# savepoints, PRAGMA, SHOW and SET).
|
|
146
|
+
BOOKKEEPING_QUERY_NAMES = %w[TRANSACTION SCHEMA].freeze
|
|
147
|
+
|
|
148
|
+
# The queries filter's rule in Ruby, for one span's properties.
|
|
149
|
+
def self.bookkeeping_query?(properties)
|
|
150
|
+
BOOKKEEPING_QUERY_NAMES.include?(properties["name"]) ||
|
|
151
|
+
TRANSACTION_STATEMENTS.include?(properties["db.statement"].to_s.strip.upcase)
|
|
152
|
+
end
|
|
153
|
+
|
|
149
154
|
# List spans by category (for the spans listing pages)
|
|
150
|
-
def list_spans_by_category(category, name: nil, limit: 50, offset: 0)
|
|
155
|
+
def list_spans_by_category(category, name: nil, sort: nil, limit: 50, offset: 0)
|
|
151
156
|
patterns = SPAN_CATEGORIES[category] || []
|
|
152
157
|
return [] if patterns.empty? && category != "exceptions"
|
|
153
158
|
|
|
154
159
|
if category == "exceptions"
|
|
155
160
|
# Exceptions are stored as events on spans, not as spans themselves
|
|
156
|
-
return list_exception_spans(name: name, limit: limit, offset: offset)
|
|
161
|
+
return list_exception_spans(name: name, sort: sort, limit: limit, offset: offset)
|
|
157
162
|
end
|
|
158
163
|
|
|
159
164
|
conditions = ["s.parent_span_id != ?"]
|
|
160
165
|
values = [MISSING_PARENT_ID]
|
|
161
166
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
patterns.each { |p| values << "%#{p}%" }
|
|
167
|
+
transaction_join, category_conditions, category_values = category_filter(category)
|
|
168
|
+
conditions.concat(category_conditions)
|
|
169
|
+
values.concat(category_values)
|
|
166
170
|
|
|
167
171
|
if name
|
|
168
172
|
conditions << "s.name LIKE ?"
|
|
169
173
|
values << "%#{name}%"
|
|
170
174
|
end
|
|
171
175
|
|
|
172
|
-
# For queries, exclude transaction statements (BEGIN, COMMIT, ROLLBACK)
|
|
173
|
-
transaction_join = ""
|
|
174
|
-
if category == "queries"
|
|
175
|
-
transaction_join = "LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'"
|
|
176
|
-
exclusions = TRANSACTION_STATEMENTS.map { "?" }.join(", ")
|
|
177
|
-
conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{exclusions}))"
|
|
178
|
-
TRANSACTION_STATEMENTS.each { |stmt| values << "\"#{stmt}\"" }
|
|
179
|
-
end
|
|
180
|
-
|
|
181
176
|
where_clause = "WHERE #{conditions.join(" AND ")}"
|
|
182
177
|
values << limit
|
|
183
178
|
values << offset
|
|
@@ -195,7 +190,7 @@ module Flare
|
|
|
195
190
|
LEFT JOIN flare_properties root_action ON root_action.owner_type = 'Flare::Span' AND root_action.owner_id = root.id AND root_action.key = 'code.function'
|
|
196
191
|
#{transaction_join}
|
|
197
192
|
#{where_clause}
|
|
198
|
-
ORDER BY
|
|
193
|
+
ORDER BY #{span_order(sort)}
|
|
199
194
|
LIMIT ? OFFSET ?
|
|
200
195
|
SQL
|
|
201
196
|
|
|
@@ -213,24 +208,15 @@ module Flare
|
|
|
213
208
|
conditions = ["s.parent_span_id != ?"]
|
|
214
209
|
values = [MISSING_PARENT_ID]
|
|
215
210
|
|
|
216
|
-
|
|
217
|
-
conditions
|
|
218
|
-
|
|
211
|
+
transaction_join, category_conditions, category_values = category_filter(category)
|
|
212
|
+
conditions.concat(category_conditions)
|
|
213
|
+
values.concat(category_values)
|
|
219
214
|
|
|
220
215
|
if name
|
|
221
216
|
conditions << "s.name LIKE ?"
|
|
222
217
|
values << "%#{name}%"
|
|
223
218
|
end
|
|
224
219
|
|
|
225
|
-
# For queries, exclude transaction statements (BEGIN, COMMIT, ROLLBACK)
|
|
226
|
-
transaction_join = ""
|
|
227
|
-
if category == "queries"
|
|
228
|
-
transaction_join = "LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'"
|
|
229
|
-
exclusions = TRANSACTION_STATEMENTS.map { "?" }.join(", ")
|
|
230
|
-
conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{exclusions}))"
|
|
231
|
-
TRANSACTION_STATEMENTS.each { |stmt| values << "\"#{stmt}\"" }
|
|
232
|
-
end
|
|
233
|
-
|
|
234
220
|
where_clause = "WHERE #{conditions.join(" AND ")}"
|
|
235
221
|
|
|
236
222
|
row = query_one(<<~SQL, values)
|
|
@@ -243,6 +229,27 @@ module Flare
|
|
|
243
229
|
row ? row["count"] : 0
|
|
244
230
|
end
|
|
245
231
|
|
|
232
|
+
# Older SQLite builds allow 999 bind parameters per statement.
|
|
233
|
+
ID_BATCH_SIZE = 500
|
|
234
|
+
|
|
235
|
+
# Number of SQL query spans in each trace (transaction statements
|
|
236
|
+
# excluded, like the queries list). Returns { trace_id => count }.
|
|
237
|
+
def count_queries_by_trace(trace_ids)
|
|
238
|
+
transaction_join, filter_conditions, filter_values = category_filter("queries")
|
|
239
|
+
|
|
240
|
+
conditions = ["s.parent_span_id != ?", *filter_conditions].join(" AND ")
|
|
241
|
+
rows = query_all_in_batches(trace_ids, after: [MISSING_PARENT_ID, *filter_values]) do |placeholders|
|
|
242
|
+
<<~SQL
|
|
243
|
+
SELECT s.trace_id, COUNT(*) as count
|
|
244
|
+
FROM flare_spans s
|
|
245
|
+
#{transaction_join}
|
|
246
|
+
WHERE s.trace_id IN (#{placeholders}) AND #{conditions}
|
|
247
|
+
GROUP BY s.trace_id
|
|
248
|
+
SQL
|
|
249
|
+
end
|
|
250
|
+
rows.to_h { |row| [row["trace_id"], row["count"]] }
|
|
251
|
+
end
|
|
252
|
+
|
|
246
253
|
# Find a single span by its database ID
|
|
247
254
|
def find_span(id)
|
|
248
255
|
row = query_one(<<~SQL, [id])
|
|
@@ -268,7 +275,7 @@ module Flare
|
|
|
268
275
|
end
|
|
269
276
|
|
|
270
277
|
# List spans that have exception events
|
|
271
|
-
def list_exception_spans(name: nil, limit: 50, offset: 0)
|
|
278
|
+
def list_exception_spans(name: nil, sort: nil, limit: 50, offset: 0)
|
|
272
279
|
conditions = []
|
|
273
280
|
values = []
|
|
274
281
|
|
|
@@ -300,7 +307,7 @@ module Flare
|
|
|
300
307
|
LEFT JOIN flare_properties exc_message ON exc_message.owner_type = 'Flare::Event' AND exc_message.owner_id = e.id AND exc_message.key = 'exception.message'
|
|
301
308
|
LEFT JOIN flare_properties exc_stacktrace ON exc_stacktrace.owner_type = 'Flare::Event' AND exc_stacktrace.owner_id = e.id AND exc_stacktrace.key = 'exception.stacktrace'
|
|
302
309
|
#{where_clause}
|
|
303
|
-
ORDER BY e.created_at DESC
|
|
310
|
+
ORDER BY #{span_order(sort, "e.created_at DESC")}
|
|
304
311
|
LIMIT ? OFFSET ?
|
|
305
312
|
SQL
|
|
306
313
|
|
|
@@ -382,17 +389,7 @@ module Flare
|
|
|
382
389
|
values << "%#{name}%"
|
|
383
390
|
end
|
|
384
391
|
|
|
385
|
-
|
|
386
|
-
if origin == "rails"
|
|
387
|
-
controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
|
|
388
|
-
conditions << "(#{controller_conditions.join(" OR ")})"
|
|
389
|
-
RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
|
|
390
|
-
elsif origin == "app"
|
|
391
|
-
controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
|
|
392
|
-
conditions << "(controller_prop.value IS NULL OR NOT (#{controller_conditions.join(" OR ")}))"
|
|
393
|
-
RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
|
|
394
|
-
end
|
|
395
|
-
end
|
|
392
|
+
add_origin_condition(origin, conditions, values)
|
|
396
393
|
|
|
397
394
|
where_clause = "WHERE #{conditions.join(" AND ")}"
|
|
398
395
|
|
|
@@ -485,13 +482,12 @@ module Flare
|
|
|
485
482
|
|
|
486
483
|
# Load properties for multiple owners at once
|
|
487
484
|
def load_properties_for_ids(owner_type, owner_ids)
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
SQL
|
|
485
|
+
rows = query_all_in_batches(owner_ids, before: [owner_type]) do |placeholders|
|
|
486
|
+
<<~SQL
|
|
487
|
+
SELECT owner_id, key, value, value_type FROM flare_properties
|
|
488
|
+
WHERE owner_type = ? AND owner_id IN (#{placeholders})
|
|
489
|
+
SQL
|
|
490
|
+
end
|
|
495
491
|
|
|
496
492
|
result = Hash.new { |h, k| h[k] = {} }
|
|
497
493
|
rows.each do |row|
|
|
@@ -502,13 +498,9 @@ module Flare
|
|
|
502
498
|
|
|
503
499
|
# Load events for multiple spans at once
|
|
504
500
|
def load_events_for_spans(span_ids)
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
event_rows = query_all(<<~SQL, span_ids)
|
|
509
|
-
SELECT * FROM flare_events
|
|
510
|
-
WHERE span_id IN (#{placeholders})
|
|
511
|
-
SQL
|
|
501
|
+
event_rows = query_all_in_batches(span_ids) do |placeholders|
|
|
502
|
+
"SELECT * FROM flare_events WHERE span_id IN (#{placeholders})"
|
|
503
|
+
end
|
|
512
504
|
|
|
513
505
|
# Group events by span_id
|
|
514
506
|
events_by_span = Hash.new { |h, k| h[k] = [] }
|
|
@@ -539,69 +531,76 @@ module Flare
|
|
|
539
531
|
end
|
|
540
532
|
|
|
541
533
|
def prune(retention_hours:, max_spans:)
|
|
542
|
-
cutoff = (Time.now - (retention_hours * 3600)).iso8601(6)
|
|
543
|
-
|
|
544
|
-
# Delete old properties first (for old spans and events)
|
|
545
|
-
execute(<<~SQL, [cutoff])
|
|
546
|
-
DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
|
|
547
|
-
SELECT id FROM flare_spans WHERE created_at < ?
|
|
548
|
-
)
|
|
549
|
-
SQL
|
|
550
|
-
|
|
551
|
-
execute(<<~SQL, [cutoff])
|
|
552
|
-
DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
|
|
553
|
-
SELECT id FROM flare_events WHERE span_id IN (
|
|
554
|
-
SELECT id FROM flare_spans WHERE created_at < ?
|
|
555
|
-
)
|
|
556
|
-
)
|
|
557
|
-
SQL
|
|
558
|
-
|
|
559
|
-
# Delete old events
|
|
560
|
-
execute(<<~SQL, [cutoff])
|
|
561
|
-
DELETE FROM flare_events WHERE span_id IN (
|
|
562
|
-
SELECT id FROM flare_spans WHERE created_at < ?
|
|
563
|
-
)
|
|
564
|
-
SQL
|
|
534
|
+
cutoff = (Time.now - (retention_hours * 3600)).utc.iso8601(6)
|
|
565
535
|
|
|
566
|
-
#
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
536
|
+
# One transaction: a writer between statements could otherwise shift the
|
|
537
|
+
# newest-N window, leaving spans stripped of their properties or orphaned rows.
|
|
538
|
+
setup_database unless @setup
|
|
539
|
+
@mutex.synchronize do
|
|
540
|
+
connection.transaction(:immediate) do |db|
|
|
541
|
+
# Delete old properties first (for old spans and events)
|
|
542
|
+
db.execute(<<~SQL, [cutoff])
|
|
543
|
+
DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
|
|
544
|
+
SELECT id FROM flare_spans WHERE created_at < ?
|
|
545
|
+
)
|
|
546
|
+
SQL
|
|
547
|
+
|
|
548
|
+
db.execute(<<~SQL, [cutoff])
|
|
549
|
+
DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
|
|
550
|
+
SELECT id FROM flare_events WHERE span_id IN (
|
|
551
|
+
SELECT id FROM flare_spans WHERE created_at < ?
|
|
552
|
+
)
|
|
553
|
+
)
|
|
554
|
+
SQL
|
|
555
|
+
|
|
556
|
+
# Delete old events
|
|
557
|
+
db.execute(<<~SQL, [cutoff])
|
|
558
|
+
DELETE FROM flare_events WHERE span_id IN (
|
|
559
|
+
SELECT id FROM flare_spans WHERE created_at < ?
|
|
560
|
+
)
|
|
561
|
+
SQL
|
|
562
|
+
|
|
563
|
+
# Delete old spans
|
|
564
|
+
db.execute(<<~SQL, [cutoff])
|
|
565
|
+
DELETE FROM flare_spans WHERE created_at < ?
|
|
566
|
+
SQL
|
|
567
|
+
|
|
568
|
+
# Also prune if over max_spans (keep newest)
|
|
569
|
+
db.execute(<<~SQL, [max_spans])
|
|
570
|
+
DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
|
|
571
|
+
SELECT id FROM flare_spans
|
|
572
|
+
ORDER BY created_at DESC
|
|
573
|
+
LIMIT -1 OFFSET ?
|
|
574
|
+
)
|
|
575
|
+
SQL
|
|
576
|
+
|
|
577
|
+
db.execute(<<~SQL, [max_spans])
|
|
578
|
+
DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
|
|
579
|
+
SELECT id FROM flare_events WHERE span_id IN (
|
|
580
|
+
SELECT id FROM flare_spans
|
|
581
|
+
ORDER BY created_at DESC
|
|
582
|
+
LIMIT -1 OFFSET ?
|
|
583
|
+
)
|
|
584
|
+
)
|
|
585
|
+
SQL
|
|
586
|
+
|
|
587
|
+
db.execute(<<~SQL, [max_spans])
|
|
588
|
+
DELETE FROM flare_events WHERE span_id IN (
|
|
589
|
+
SELECT id FROM flare_spans
|
|
590
|
+
ORDER BY created_at DESC
|
|
591
|
+
LIMIT -1 OFFSET ?
|
|
592
|
+
)
|
|
593
|
+
SQL
|
|
594
|
+
|
|
595
|
+
db.execute(<<~SQL, [max_spans])
|
|
596
|
+
DELETE FROM flare_spans WHERE id IN (
|
|
597
|
+
SELECT id FROM flare_spans
|
|
598
|
+
ORDER BY created_at DESC
|
|
599
|
+
LIMIT -1 OFFSET ?
|
|
600
|
+
)
|
|
601
|
+
SQL
|
|
602
|
+
end
|
|
603
|
+
end
|
|
605
604
|
end
|
|
606
605
|
|
|
607
606
|
def clear_all
|
|
@@ -687,20 +686,56 @@ module Flare
|
|
|
687
686
|
db.execute("PRAGMA cache_size=2000")
|
|
688
687
|
end
|
|
689
688
|
|
|
689
|
+
# SQL join, conditions and bind values that select a category's spans by
|
|
690
|
+
# name. Queries also leave out transaction statements and what Active
|
|
691
|
+
# Record names its own bookkeeping.
|
|
692
|
+
def category_filter(category)
|
|
693
|
+
patterns = SPAN_CATEGORIES.fetch(category)
|
|
694
|
+
conditions = ["(#{patterns.map { "s.name LIKE ?" }.join(" OR ")})"]
|
|
695
|
+
values = patterns.map { |p| "%#{p}%" }
|
|
696
|
+
join = ""
|
|
697
|
+
|
|
698
|
+
if category == "queries"
|
|
699
|
+
join = <<~SQL
|
|
700
|
+
LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'
|
|
701
|
+
LEFT JOIN flare_properties query_name_prop ON query_name_prop.owner_type = 'Flare::Span' AND query_name_prop.owner_id = s.id AND query_name_prop.key = 'name'
|
|
702
|
+
SQL
|
|
703
|
+
conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{TRANSACTION_STATEMENTS.map { "?" }.join(", ")}))"
|
|
704
|
+
values.concat(TRANSACTION_STATEMENTS.map { |stmt| "\"#{stmt}\"" })
|
|
705
|
+
conditions << "(query_name_prop.value IS NULL OR query_name_prop.value NOT IN (#{BOOKKEEPING_QUERY_NAMES.map { "?" }.join(", ")}))"
|
|
706
|
+
values.concat(BOOKKEEPING_QUERY_NAMES.map { |query_name| "\"#{query_name}\"" })
|
|
707
|
+
end
|
|
708
|
+
|
|
709
|
+
[join, conditions, values]
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
# Ties on duration, like an unsorted list, read newest first.
|
|
713
|
+
def span_order(sort, newest = "s.created_at DESC")
|
|
714
|
+
sort == "slowest" ? "(s.end_timestamp - s.start_timestamp) DESC, #{newest}" : newest
|
|
715
|
+
end
|
|
716
|
+
|
|
717
|
+
# Keyed by path, like SQLiteExporter, so storages for different files on
|
|
718
|
+
# the same thread don't share a connection. A child must not use its
|
|
719
|
+
# parent's handle or close it while the parent may still be using it.
|
|
720
|
+
def connection_key
|
|
721
|
+
:"flare_storage_db_#{$$}_#{@database_path.hash}"
|
|
722
|
+
end
|
|
723
|
+
|
|
690
724
|
def connection
|
|
691
|
-
Thread.current[
|
|
725
|
+
Thread.current[connection_key] ||= begin
|
|
692
726
|
dir = File.dirname(@database_path)
|
|
693
727
|
FileUtils.mkdir_p(dir) unless File.directory?(dir)
|
|
694
|
-
db = ::SQLite3::Database.new(@database_path, results_as_hash: true)
|
|
728
|
+
db = ::SQLite3::Database.new(@database_path, results_as_hash: true, readonly: @readonly)
|
|
695
729
|
db.busy_timeout = 5000
|
|
696
730
|
db
|
|
697
731
|
end
|
|
698
732
|
end
|
|
699
733
|
|
|
700
734
|
def close_connection
|
|
701
|
-
|
|
735
|
+
key = connection_key
|
|
736
|
+
if db = Thread.current[key]
|
|
702
737
|
db.close rescue nil
|
|
703
|
-
Thread.current[
|
|
738
|
+
Thread.current[key] = nil
|
|
704
739
|
end
|
|
705
740
|
end
|
|
706
741
|
|
|
@@ -711,6 +746,13 @@ module Flare
|
|
|
711
746
|
end
|
|
712
747
|
end
|
|
713
748
|
|
|
749
|
+
# Runs the block's SQL once per batch of ids, staying under the bind limit.
|
|
750
|
+
def query_all_in_batches(ids, before: [], after: [])
|
|
751
|
+
ids.each_slice(ID_BATCH_SIZE).flat_map do |batch|
|
|
752
|
+
query_all(yield(batch.map { "?" }.join(", ")), [*before, *batch, *after])
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
|
|
714
756
|
def query_one(sql, values = [])
|
|
715
757
|
setup_database unless @setup
|
|
716
758
|
@mutex.synchronize do
|
|
@@ -742,6 +784,15 @@ module Flare
|
|
|
742
784
|
}
|
|
743
785
|
end
|
|
744
786
|
|
|
787
|
+
def add_origin_condition(origin, conditions, values)
|
|
788
|
+
return unless %w[app rails].include?(origin)
|
|
789
|
+
|
|
790
|
+
# Property values are stored as JSON, so a class name starts with a quote.
|
|
791
|
+
framework = "(#{RAILS_CONTROLLER_NAMESPACES.map { "controller_prop.value LIKE ?" }.join(" OR ")})"
|
|
792
|
+
conditions << (origin == "rails" ? framework : "(controller_prop.value IS NULL OR NOT #{framework})")
|
|
793
|
+
values.concat(RAILS_CONTROLLER_NAMESPACES.map { |namespace| "\"#{namespace}%" })
|
|
794
|
+
end
|
|
795
|
+
|
|
745
796
|
def row_to_request(row)
|
|
746
797
|
span = row_to_span(row)
|
|
747
798
|
|
data/lib/flare/storage.rb
CHANGED
|
@@ -3,52 +3,13 @@
|
|
|
3
3
|
module Flare
|
|
4
4
|
module Storage
|
|
5
5
|
class Base
|
|
6
|
-
def
|
|
7
|
-
raise NotImplementedError
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def save_clues(clues)
|
|
11
|
-
raise NotImplementedError
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def find_case(uuid)
|
|
15
|
-
raise NotImplementedError
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def list_cases(type: nil, status: nil, method: nil, name: nil, origin: nil, limit: 50, offset: 0)
|
|
19
|
-
raise NotImplementedError
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def clues_for_case(case_uuid)
|
|
23
|
-
raise NotImplementedError
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def list_clues(type: nil, search: nil, limit: 50, offset: 0)
|
|
27
|
-
raise NotImplementedError
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def find_clue(id)
|
|
31
|
-
raise NotImplementedError
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def prune(retention_hours:, max_cases:)
|
|
6
|
+
def prune(retention_hours:, max_spans:)
|
|
35
7
|
raise NotImplementedError
|
|
36
8
|
end
|
|
37
9
|
|
|
38
10
|
def clear_all
|
|
39
11
|
raise NotImplementedError
|
|
40
12
|
end
|
|
41
|
-
|
|
42
|
-
def count_cases(type: nil, status: nil, method: nil, name: nil, origin: nil)
|
|
43
|
-
raise NotImplementedError
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def count_clues(type: nil, search: nil)
|
|
47
|
-
raise NotImplementedError
|
|
48
|
-
end
|
|
49
13
|
end
|
|
50
14
|
end
|
|
51
15
|
end
|
|
52
|
-
|
|
53
|
-
# storage/sqlite is loaded on demand when spans are enabled
|
|
54
|
-
# to avoid requiring sqlite3 in production environments
|