flare 0.4.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +60 -1
  3. data/app/controllers/flare/jobs_controller.rb +1 -1
  4. data/app/controllers/flare/requests_controller.rb +1 -1
  5. data/app/controllers/flare/spans_controller.rb +1 -1
  6. data/app/helpers/flare/application_helper.rb +23 -0
  7. data/app/views/flare/jobs/index.html.erb +2 -1
  8. data/app/views/flare/requests/index.html.erb +2 -1
  9. data/app/views/flare/requests/show.html.erb +16 -7
  10. data/app/views/flare/spans/index.html.erb +2 -1
  11. data/lib/flare/cli/doctor_command.rb +4 -11
  12. data/lib/flare/cli/local_spans.rb +110 -0
  13. data/lib/flare/cli/nplus1_command.rb +97 -0
  14. data/lib/flare/cli/project.rb +28 -0
  15. data/lib/flare/cli/request_command.rb +120 -0
  16. data/lib/flare/cli/requests_command.rb +45 -0
  17. data/lib/flare/cli/setup_command.rb +64 -41
  18. data/lib/flare/cli/status_command.rb +5 -2
  19. data/lib/flare/cli.rb +12 -0
  20. data/lib/flare/client_headers.rb +7 -3
  21. data/lib/flare/configuration.rb +20 -0
  22. data/lib/flare/export_filter.rb +273 -0
  23. data/lib/flare/filtering_span_processor.rb +14 -0
  24. data/lib/flare/http_transport.rb +4 -0
  25. data/lib/flare/metric_span_processor.rb +51 -35
  26. data/lib/flare/metric_submitter.rb +9 -1
  27. data/lib/flare/recording_batch_span_processor.rb +9 -1
  28. data/lib/flare/redis_statement_processor.rb +29 -0
  29. data/lib/flare/revision.rb +82 -0
  30. data/lib/flare/source_location.rb +21 -16
  31. data/lib/flare/sql_mask.rb +235 -0
  32. data/lib/flare/sqlite_exporter.rb +12 -8
  33. data/lib/flare/storage/sqlite.rb +195 -144
  34. data/lib/flare/storage.rb +1 -40
  35. data/lib/flare/store_statement.rb +136 -0
  36. data/lib/flare/trace_blob.rb +9 -5
  37. data/lib/flare/trace_exporter.rb +18 -3
  38. data/lib/flare/upload_url_pool.rb +2 -15
  39. data/lib/flare/utf8.rb +20 -0
  40. data/lib/flare/version.rb +1 -1
  41. data/lib/flare.rb +21 -2
  42. data/public/flare-assets/flare.css +10 -60
  43. metadata +45 -32
@@ -4,6 +4,7 @@ require "sqlite3"
4
4
  require "json"
5
5
 
6
6
  require_relative "deadline"
7
+ require_relative "utf8"
7
8
 
8
9
  module Flare
9
10
  class SQLiteExporter
@@ -128,8 +129,11 @@ module Flare
128
129
  span_data.name&.start_with?("Flare::")
129
130
  end
130
131
 
132
+ # Stored in UTC because readers compare and sort these as strings, which only
133
+ # works when every row has the same offset. Local time breaks that at a clock
134
+ # change: 1:05 after falling back sorts before the 1:50 that came earlier.
131
135
  def create_span(span_data)
132
- now = Time.now.iso8601(6)
136
+ now = Time.now.utc.iso8601(6)
133
137
 
134
138
  sql = <<~SQL
135
139
  INSERT INTO flare_spans (name, kind, span_id, trace_id, parent_span_id, start_timestamp, end_timestamp, total_recorded_links, total_recorded_events, total_recorded_properties, created_at, updated_at)
@@ -137,7 +141,7 @@ module Flare
137
141
  SQL
138
142
 
139
143
  values = [
140
- span_data.name,
144
+ UTF8.scrub(span_data.name),
141
145
  span_data.kind.to_s,
142
146
  span_data.hex_span_id,
143
147
  span_data.hex_trace_id,
@@ -162,15 +166,15 @@ module Flare
162
166
  end
163
167
 
164
168
  def create_event(span_record_id, span_event)
165
- now = Time.now.iso8601(6)
166
- timestamp = span_event.timestamp ? Time.at(span_event.timestamp / 1_000_000_000.0).iso8601(6) : now
169
+ now = Time.now.utc.iso8601(6)
170
+ timestamp = span_event.timestamp ? Time.at(span_event.timestamp / 1_000_000_000.0).utc.iso8601(6) : now
167
171
 
168
172
  sql = <<~SQL
169
173
  INSERT INTO flare_events (span_id, name, created_at, updated_at)
170
174
  VALUES (?, ?, ?, ?)
171
175
  SQL
172
176
 
173
- connection.execute(sql, [span_record_id, span_event.name, timestamp, now])
177
+ connection.execute(sql, [span_record_id, UTF8.scrub(span_event.name), timestamp, now])
174
178
  event_record_id = connection.last_insert_row_id
175
179
  create_properties("Flare::Event", event_record_id, span_event.attributes)
176
180
  end
@@ -178,7 +182,7 @@ module Flare
178
182
  def create_properties(owner_type, owner_id, attributes)
179
183
  return unless attributes
180
184
 
181
- now = Time.now.iso8601(6)
185
+ now = Time.now.utc.iso8601(6)
182
186
 
183
187
  sql = <<~SQL
184
188
  INSERT INTO flare_properties (key, value, value_type, owner_type, owner_id, created_at, updated_at)
@@ -189,8 +193,8 @@ module Flare
189
193
  next if value.nil?
190
194
 
191
195
  value_type = determine_value_type(value)
192
- serialized_value = JSON.generate(value)
193
- connection.execute(sql, [key, serialized_value, value_type, owner_type, owner_id, now, now])
196
+ serialized_value = JSON.generate(UTF8.scrub(value))
197
+ connection.execute(sql, [UTF8.scrub(key), serialized_value, value_type, owner_type, owner_id, now, now])
194
198
  end
195
199
  end
196
200
 
@@ -8,22 +8,26 @@ module Flare
8
8
  class SQLite < Base
9
9
  MISSING_PARENT_ID = "0000000000000000"
10
10
 
11
- # Rails framework controller prefixes to filter (lowercase/underscored format)
12
- RAILS_CONTROLLER_PREFIXES = %w[
13
- active_storage/
14
- action_mailbox/
15
- rails/
16
- flare/
11
+ # Namespaces of the controllers that come with Rails and Flare, as stored
12
+ # in code.namespace (the controller's class name).
13
+ RAILS_CONTROLLER_NAMESPACES = %w[
14
+ ActiveStorage::
15
+ ActionMailbox::
16
+ Rails::
17
+ Flare::
17
18
  ].freeze
18
19
 
19
- def initialize(database_path)
20
+ # readonly: true opens an existing database as is, without creating
21
+ # tables, for tools that only inspect spans (like `flare requests`).
22
+ def initialize(database_path, readonly: false)
20
23
  @database_path = database_path
24
+ @readonly = readonly
21
25
  @mutex = Mutex.new
22
- @setup = false
26
+ @setup = readonly
23
27
  end
24
28
 
25
29
  # List root spans that are HTTP requests (for the requests index)
26
- def list_requests(status: nil, method: nil, name: nil, origin: nil, limit: 50, offset: 0)
30
+ def list_requests(status: nil, method: nil, name: nil, origin: nil, sort: nil, limit: 50, offset: 0)
27
31
  # Find root spans with kind=server that have http.method property
28
32
  conditions = ["s.parent_span_id = ?", "s.kind = ?"]
29
33
  values = [MISSING_PARENT_ID, "server"]
@@ -61,17 +65,7 @@ module Flare
61
65
  values << "%#{name}%"
62
66
  end
63
67
 
64
- if origin
65
- if origin == "rails"
66
- controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
67
- conditions << "(#{controller_conditions.join(" OR ")})"
68
- RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
69
- elsif origin == "app"
70
- controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
71
- conditions << "(controller_prop.value IS NULL OR NOT (#{controller_conditions.join(" OR ")}))"
72
- RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
73
- end
74
- end
68
+ add_origin_condition(origin, conditions, values)
75
69
 
76
70
  where_clause = "WHERE #{conditions.join(" AND ")}"
77
71
  values << limit
@@ -95,7 +89,7 @@ module Flare
95
89
  LEFT JOIN flare_properties action_prop ON action_prop.owner_type = 'Flare::Span' AND action_prop.owner_id = s.id AND action_prop.key = 'code.function'
96
90
  #{where_clause}
97
91
  AND COALESCE(method_prop.value, method_prop_old.value) IS NOT NULL
98
- ORDER BY s.created_at DESC
92
+ ORDER BY #{span_order(sort)}
99
93
  LIMIT ? OFFSET ?
100
94
  SQL
101
95
 
@@ -103,7 +97,7 @@ module Flare
103
97
  end
104
98
 
105
99
  # List root spans that are jobs (for the jobs index)
106
- def list_jobs(status: nil, name: nil, limit: 50, offset: 0)
100
+ def list_jobs(status: nil, name: nil, sort: nil, limit: 50, offset: 0)
107
101
  # Find root spans with kind=consumer (ActiveJob processing)
108
102
  conditions = ["s.parent_span_id = ?", "s.kind = ?"]
109
103
  values = [MISSING_PARENT_ID, "consumer"]
@@ -125,7 +119,7 @@ module Flare
125
119
  LEFT JOIN flare_properties job_class_prop ON job_class_prop.owner_type = 'Flare::Span' AND job_class_prop.owner_id = s.id AND job_class_prop.key = 'code.namespace'
126
120
  LEFT JOIN flare_properties queue_prop ON queue_prop.owner_type = 'Flare::Span' AND queue_prop.owner_id = s.id AND queue_prop.key = 'messaging.destination'
127
121
  #{where_clause}
128
- ORDER BY s.created_at DESC
122
+ ORDER BY #{span_order(sort)}
129
123
  LIMIT ? OFFSET ?
130
124
  SQL
131
125
 
@@ -146,38 +140,39 @@ module Flare
146
140
  # Transaction statements to filter out from queries list
147
141
  TRANSACTION_STATEMENTS = %w[BEGIN COMMIT ROLLBACK].freeze
148
142
 
143
+ # Active Record names its own statements on every adapter, including ones
144
+ # the list above can't match (SQLite's "BEGIN immediate TRANSACTION",
145
+ # savepoints, PRAGMA, SHOW and SET).
146
+ BOOKKEEPING_QUERY_NAMES = %w[TRANSACTION SCHEMA].freeze
147
+
148
+ # The queries filter's rule in Ruby, for one span's properties.
149
+ def self.bookkeeping_query?(properties)
150
+ BOOKKEEPING_QUERY_NAMES.include?(properties["name"]) ||
151
+ TRANSACTION_STATEMENTS.include?(properties["db.statement"].to_s.strip.upcase)
152
+ end
153
+
149
154
  # List spans by category (for the spans listing pages)
150
- def list_spans_by_category(category, name: nil, limit: 50, offset: 0)
155
+ def list_spans_by_category(category, name: nil, sort: nil, limit: 50, offset: 0)
151
156
  patterns = SPAN_CATEGORIES[category] || []
152
157
  return [] if patterns.empty? && category != "exceptions"
153
158
 
154
159
  if category == "exceptions"
155
160
  # Exceptions are stored as events on spans, not as spans themselves
156
- return list_exception_spans(name: name, limit: limit, offset: offset)
161
+ return list_exception_spans(name: name, sort: sort, limit: limit, offset: offset)
157
162
  end
158
163
 
159
164
  conditions = ["s.parent_span_id != ?"]
160
165
  values = [MISSING_PARENT_ID]
161
166
 
162
- # Build OR conditions for matching span names
163
- pattern_conditions = patterns.map { "s.name LIKE ?" }
164
- conditions << "(#{pattern_conditions.join(" OR ")})"
165
- patterns.each { |p| values << "%#{p}%" }
167
+ transaction_join, category_conditions, category_values = category_filter(category)
168
+ conditions.concat(category_conditions)
169
+ values.concat(category_values)
166
170
 
167
171
  if name
168
172
  conditions << "s.name LIKE ?"
169
173
  values << "%#{name}%"
170
174
  end
171
175
 
172
- # For queries, exclude transaction statements (BEGIN, COMMIT, ROLLBACK)
173
- transaction_join = ""
174
- if category == "queries"
175
- transaction_join = "LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'"
176
- exclusions = TRANSACTION_STATEMENTS.map { "?" }.join(", ")
177
- conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{exclusions}))"
178
- TRANSACTION_STATEMENTS.each { |stmt| values << "\"#{stmt}\"" }
179
- end
180
-
181
176
  where_clause = "WHERE #{conditions.join(" AND ")}"
182
177
  values << limit
183
178
  values << offset
@@ -195,7 +190,7 @@ module Flare
195
190
  LEFT JOIN flare_properties root_action ON root_action.owner_type = 'Flare::Span' AND root_action.owner_id = root.id AND root_action.key = 'code.function'
196
191
  #{transaction_join}
197
192
  #{where_clause}
198
- ORDER BY s.created_at DESC
193
+ ORDER BY #{span_order(sort)}
199
194
  LIMIT ? OFFSET ?
200
195
  SQL
201
196
 
@@ -213,24 +208,15 @@ module Flare
213
208
  conditions = ["s.parent_span_id != ?"]
214
209
  values = [MISSING_PARENT_ID]
215
210
 
216
- pattern_conditions = patterns.map { "s.name LIKE ?" }
217
- conditions << "(#{pattern_conditions.join(" OR ")})"
218
- patterns.each { |p| values << "%#{p}%" }
211
+ transaction_join, category_conditions, category_values = category_filter(category)
212
+ conditions.concat(category_conditions)
213
+ values.concat(category_values)
219
214
 
220
215
  if name
221
216
  conditions << "s.name LIKE ?"
222
217
  values << "%#{name}%"
223
218
  end
224
219
 
225
- # For queries, exclude transaction statements (BEGIN, COMMIT, ROLLBACK)
226
- transaction_join = ""
227
- if category == "queries"
228
- transaction_join = "LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'"
229
- exclusions = TRANSACTION_STATEMENTS.map { "?" }.join(", ")
230
- conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{exclusions}))"
231
- TRANSACTION_STATEMENTS.each { |stmt| values << "\"#{stmt}\"" }
232
- end
233
-
234
220
  where_clause = "WHERE #{conditions.join(" AND ")}"
235
221
 
236
222
  row = query_one(<<~SQL, values)
@@ -243,6 +229,27 @@ module Flare
243
229
  row ? row["count"] : 0
244
230
  end
245
231
 
232
+ # Older SQLite builds allow 999 bind parameters per statement.
233
+ ID_BATCH_SIZE = 500
234
+
235
+ # Number of SQL query spans in each trace (transaction statements
236
+ # excluded, like the queries list). Returns { trace_id => count }.
237
+ def count_queries_by_trace(trace_ids)
238
+ transaction_join, filter_conditions, filter_values = category_filter("queries")
239
+
240
+ conditions = ["s.parent_span_id != ?", *filter_conditions].join(" AND ")
241
+ rows = query_all_in_batches(trace_ids, after: [MISSING_PARENT_ID, *filter_values]) do |placeholders|
242
+ <<~SQL
243
+ SELECT s.trace_id, COUNT(*) as count
244
+ FROM flare_spans s
245
+ #{transaction_join}
246
+ WHERE s.trace_id IN (#{placeholders}) AND #{conditions}
247
+ GROUP BY s.trace_id
248
+ SQL
249
+ end
250
+ rows.to_h { |row| [row["trace_id"], row["count"]] }
251
+ end
252
+
246
253
  # Find a single span by its database ID
247
254
  def find_span(id)
248
255
  row = query_one(<<~SQL, [id])
@@ -268,7 +275,7 @@ module Flare
268
275
  end
269
276
 
270
277
  # List spans that have exception events
271
- def list_exception_spans(name: nil, limit: 50, offset: 0)
278
+ def list_exception_spans(name: nil, sort: nil, limit: 50, offset: 0)
272
279
  conditions = []
273
280
  values = []
274
281
 
@@ -300,7 +307,7 @@ module Flare
300
307
  LEFT JOIN flare_properties exc_message ON exc_message.owner_type = 'Flare::Event' AND exc_message.owner_id = e.id AND exc_message.key = 'exception.message'
301
308
  LEFT JOIN flare_properties exc_stacktrace ON exc_stacktrace.owner_type = 'Flare::Event' AND exc_stacktrace.owner_id = e.id AND exc_stacktrace.key = 'exception.stacktrace'
302
309
  #{where_clause}
303
- ORDER BY e.created_at DESC
310
+ ORDER BY #{span_order(sort, "e.created_at DESC")}
304
311
  LIMIT ? OFFSET ?
305
312
  SQL
306
313
 
@@ -382,17 +389,7 @@ module Flare
382
389
  values << "%#{name}%"
383
390
  end
384
391
 
385
- if origin
386
- if origin == "rails"
387
- controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
388
- conditions << "(#{controller_conditions.join(" OR ")})"
389
- RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
390
- elsif origin == "app"
391
- controller_conditions = RAILS_CONTROLLER_PREFIXES.map { "controller_prop.value LIKE ?" }
392
- conditions << "(controller_prop.value IS NULL OR NOT (#{controller_conditions.join(" OR ")}))"
393
- RAILS_CONTROLLER_PREFIXES.each { |prefix| values << "%#{prefix}%" }
394
- end
395
- end
392
+ add_origin_condition(origin, conditions, values)
396
393
 
397
394
  where_clause = "WHERE #{conditions.join(" AND ")}"
398
395
 
@@ -485,13 +482,12 @@ module Flare
485
482
 
486
483
  # Load properties for multiple owners at once
487
484
  def load_properties_for_ids(owner_type, owner_ids)
488
- return {} if owner_ids.empty?
489
-
490
- placeholders = owner_ids.map { "?" }.join(", ")
491
- rows = query_all(<<~SQL, [owner_type] + owner_ids)
492
- SELECT owner_id, key, value, value_type FROM flare_properties
493
- WHERE owner_type = ? AND owner_id IN (#{placeholders})
494
- SQL
485
+ rows = query_all_in_batches(owner_ids, before: [owner_type]) do |placeholders|
486
+ <<~SQL
487
+ SELECT owner_id, key, value, value_type FROM flare_properties
488
+ WHERE owner_type = ? AND owner_id IN (#{placeholders})
489
+ SQL
490
+ end
495
491
 
496
492
  result = Hash.new { |h, k| h[k] = {} }
497
493
  rows.each do |row|
@@ -502,13 +498,9 @@ module Flare
502
498
 
503
499
  # Load events for multiple spans at once
504
500
  def load_events_for_spans(span_ids)
505
- return {} if span_ids.empty?
506
-
507
- placeholders = span_ids.map { "?" }.join(", ")
508
- event_rows = query_all(<<~SQL, span_ids)
509
- SELECT * FROM flare_events
510
- WHERE span_id IN (#{placeholders})
511
- SQL
501
+ event_rows = query_all_in_batches(span_ids) do |placeholders|
502
+ "SELECT * FROM flare_events WHERE span_id IN (#{placeholders})"
503
+ end
512
504
 
513
505
  # Group events by span_id
514
506
  events_by_span = Hash.new { |h, k| h[k] = [] }
@@ -539,69 +531,76 @@ module Flare
539
531
  end
540
532
 
541
533
  def prune(retention_hours:, max_spans:)
542
- cutoff = (Time.now - (retention_hours * 3600)).iso8601(6)
543
-
544
- # Delete old properties first (for old spans and events)
545
- execute(<<~SQL, [cutoff])
546
- DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
547
- SELECT id FROM flare_spans WHERE created_at < ?
548
- )
549
- SQL
550
-
551
- execute(<<~SQL, [cutoff])
552
- DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
553
- SELECT id FROM flare_events WHERE span_id IN (
554
- SELECT id FROM flare_spans WHERE created_at < ?
555
- )
556
- )
557
- SQL
558
-
559
- # Delete old events
560
- execute(<<~SQL, [cutoff])
561
- DELETE FROM flare_events WHERE span_id IN (
562
- SELECT id FROM flare_spans WHERE created_at < ?
563
- )
564
- SQL
534
+ cutoff = (Time.now - (retention_hours * 3600)).utc.iso8601(6)
565
535
 
566
- # Delete old spans
567
- execute(<<~SQL, [cutoff])
568
- DELETE FROM flare_spans WHERE created_at < ?
569
- SQL
570
-
571
- # Also prune if over max_spans (keep newest)
572
- execute(<<~SQL, [max_spans])
573
- DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
574
- SELECT id FROM flare_spans
575
- ORDER BY created_at DESC
576
- LIMIT -1 OFFSET ?
577
- )
578
- SQL
579
-
580
- execute(<<~SQL, [max_spans])
581
- DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
582
- SELECT id FROM flare_events WHERE span_id IN (
583
- SELECT id FROM flare_spans
584
- ORDER BY created_at DESC
585
- LIMIT -1 OFFSET ?
586
- )
587
- )
588
- SQL
589
-
590
- execute(<<~SQL, [max_spans])
591
- DELETE FROM flare_events WHERE span_id IN (
592
- SELECT id FROM flare_spans
593
- ORDER BY created_at DESC
594
- LIMIT -1 OFFSET ?
595
- )
596
- SQL
597
-
598
- execute(<<~SQL, [max_spans])
599
- DELETE FROM flare_spans WHERE id IN (
600
- SELECT id FROM flare_spans
601
- ORDER BY created_at DESC
602
- LIMIT -1 OFFSET ?
603
- )
604
- SQL
536
+ # One transaction: a writer between statements could otherwise shift the
537
+ # newest-N window, leaving spans stripped of their properties or orphaned rows.
538
+ setup_database unless @setup
539
+ @mutex.synchronize do
540
+ connection.transaction(:immediate) do |db|
541
+ # Delete old properties first (for old spans and events)
542
+ db.execute(<<~SQL, [cutoff])
543
+ DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
544
+ SELECT id FROM flare_spans WHERE created_at < ?
545
+ )
546
+ SQL
547
+
548
+ db.execute(<<~SQL, [cutoff])
549
+ DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
550
+ SELECT id FROM flare_events WHERE span_id IN (
551
+ SELECT id FROM flare_spans WHERE created_at < ?
552
+ )
553
+ )
554
+ SQL
555
+
556
+ # Delete old events
557
+ db.execute(<<~SQL, [cutoff])
558
+ DELETE FROM flare_events WHERE span_id IN (
559
+ SELECT id FROM flare_spans WHERE created_at < ?
560
+ )
561
+ SQL
562
+
563
+ # Delete old spans
564
+ db.execute(<<~SQL, [cutoff])
565
+ DELETE FROM flare_spans WHERE created_at < ?
566
+ SQL
567
+
568
+ # Also prune if over max_spans (keep newest)
569
+ db.execute(<<~SQL, [max_spans])
570
+ DELETE FROM flare_properties WHERE owner_type = 'Flare::Span' AND owner_id IN (
571
+ SELECT id FROM flare_spans
572
+ ORDER BY created_at DESC
573
+ LIMIT -1 OFFSET ?
574
+ )
575
+ SQL
576
+
577
+ db.execute(<<~SQL, [max_spans])
578
+ DELETE FROM flare_properties WHERE owner_type = 'Flare::Event' AND owner_id IN (
579
+ SELECT id FROM flare_events WHERE span_id IN (
580
+ SELECT id FROM flare_spans
581
+ ORDER BY created_at DESC
582
+ LIMIT -1 OFFSET ?
583
+ )
584
+ )
585
+ SQL
586
+
587
+ db.execute(<<~SQL, [max_spans])
588
+ DELETE FROM flare_events WHERE span_id IN (
589
+ SELECT id FROM flare_spans
590
+ ORDER BY created_at DESC
591
+ LIMIT -1 OFFSET ?
592
+ )
593
+ SQL
594
+
595
+ db.execute(<<~SQL, [max_spans])
596
+ DELETE FROM flare_spans WHERE id IN (
597
+ SELECT id FROM flare_spans
598
+ ORDER BY created_at DESC
599
+ LIMIT -1 OFFSET ?
600
+ )
601
+ SQL
602
+ end
603
+ end
605
604
  end
606
605
 
607
606
  def clear_all
@@ -687,20 +686,56 @@ module Flare
687
686
  db.execute("PRAGMA cache_size=2000")
688
687
  end
689
688
 
689
+ # SQL join, conditions and bind values that select a category's spans by
690
+ # name. Queries also leave out transaction statements and what Active
691
+ # Record names its own bookkeeping.
692
+ def category_filter(category)
693
+ patterns = SPAN_CATEGORIES.fetch(category)
694
+ conditions = ["(#{patterns.map { "s.name LIKE ?" }.join(" OR ")})"]
695
+ values = patterns.map { |p| "%#{p}%" }
696
+ join = ""
697
+
698
+ if category == "queries"
699
+ join = <<~SQL
700
+ LEFT JOIN flare_properties stmt_prop ON stmt_prop.owner_type = 'Flare::Span' AND stmt_prop.owner_id = s.id AND stmt_prop.key = 'db.statement'
701
+ LEFT JOIN flare_properties query_name_prop ON query_name_prop.owner_type = 'Flare::Span' AND query_name_prop.owner_id = s.id AND query_name_prop.key = 'name'
702
+ SQL
703
+ conditions << "(stmt_prop.value IS NULL OR stmt_prop.value NOT IN (#{TRANSACTION_STATEMENTS.map { "?" }.join(", ")}))"
704
+ values.concat(TRANSACTION_STATEMENTS.map { |stmt| "\"#{stmt}\"" })
705
+ conditions << "(query_name_prop.value IS NULL OR query_name_prop.value NOT IN (#{BOOKKEEPING_QUERY_NAMES.map { "?" }.join(", ")}))"
706
+ values.concat(BOOKKEEPING_QUERY_NAMES.map { |query_name| "\"#{query_name}\"" })
707
+ end
708
+
709
+ [join, conditions, values]
710
+ end
711
+
712
+ # Ties on duration, like an unsorted list, read newest first.
713
+ def span_order(sort, newest = "s.created_at DESC")
714
+ sort == "slowest" ? "(s.end_timestamp - s.start_timestamp) DESC, #{newest}" : newest
715
+ end
716
+
717
+ # Keyed by path, like SQLiteExporter, so storages for different files on
718
+ # the same thread don't share a connection. A child must not use its
719
+ # parent's handle or close it while the parent may still be using it.
720
+ def connection_key
721
+ :"flare_storage_db_#{$$}_#{@database_path.hash}"
722
+ end
723
+
690
724
  def connection
691
- Thread.current[:flare_storage_db] ||= begin
725
+ Thread.current[connection_key] ||= begin
692
726
  dir = File.dirname(@database_path)
693
727
  FileUtils.mkdir_p(dir) unless File.directory?(dir)
694
- db = ::SQLite3::Database.new(@database_path, results_as_hash: true)
728
+ db = ::SQLite3::Database.new(@database_path, results_as_hash: true, readonly: @readonly)
695
729
  db.busy_timeout = 5000
696
730
  db
697
731
  end
698
732
  end
699
733
 
700
734
  def close_connection
701
- if db = Thread.current[:flare_storage_db]
735
+ key = connection_key
736
+ if db = Thread.current[key]
702
737
  db.close rescue nil
703
- Thread.current[:flare_storage_db] = nil
738
+ Thread.current[key] = nil
704
739
  end
705
740
  end
706
741
 
@@ -711,6 +746,13 @@ module Flare
711
746
  end
712
747
  end
713
748
 
749
+ # Runs the block's SQL once per batch of ids, staying under the bind limit.
750
+ def query_all_in_batches(ids, before: [], after: [])
751
+ ids.each_slice(ID_BATCH_SIZE).flat_map do |batch|
752
+ query_all(yield(batch.map { "?" }.join(", ")), [*before, *batch, *after])
753
+ end
754
+ end
755
+
714
756
  def query_one(sql, values = [])
715
757
  setup_database unless @setup
716
758
  @mutex.synchronize do
@@ -742,6 +784,15 @@ module Flare
742
784
  }
743
785
  end
744
786
 
787
+ def add_origin_condition(origin, conditions, values)
788
+ return unless %w[app rails].include?(origin)
789
+
790
+ # Property values are stored as JSON, so a class name starts with a quote.
791
+ framework = "(#{RAILS_CONTROLLER_NAMESPACES.map { "controller_prop.value LIKE ?" }.join(" OR ")})"
792
+ conditions << (origin == "rails" ? framework : "(controller_prop.value IS NULL OR NOT #{framework})")
793
+ values.concat(RAILS_CONTROLLER_NAMESPACES.map { |namespace| "\"#{namespace}%" })
794
+ end
795
+
745
796
  def row_to_request(row)
746
797
  span = row_to_span(row)
747
798
 
data/lib/flare/storage.rb CHANGED
@@ -3,52 +3,13 @@
3
3
  module Flare
4
4
  module Storage
5
5
  class Base
6
- def save_case(attributes)
7
- raise NotImplementedError
8
- end
9
-
10
- def save_clues(clues)
11
- raise NotImplementedError
12
- end
13
-
14
- def find_case(uuid)
15
- raise NotImplementedError
16
- end
17
-
18
- def list_cases(type: nil, status: nil, method: nil, name: nil, origin: nil, limit: 50, offset: 0)
19
- raise NotImplementedError
20
- end
21
-
22
- def clues_for_case(case_uuid)
23
- raise NotImplementedError
24
- end
25
-
26
- def list_clues(type: nil, search: nil, limit: 50, offset: 0)
27
- raise NotImplementedError
28
- end
29
-
30
- def find_clue(id)
31
- raise NotImplementedError
32
- end
33
-
34
- def prune(retention_hours:, max_cases:)
6
+ def prune(retention_hours:, max_spans:)
35
7
  raise NotImplementedError
36
8
  end
37
9
 
38
10
  def clear_all
39
11
  raise NotImplementedError
40
12
  end
41
-
42
- def count_cases(type: nil, status: nil, method: nil, name: nil, origin: nil)
43
- raise NotImplementedError
44
- end
45
-
46
- def count_clues(type: nil, search: nil)
47
- raise NotImplementedError
48
- end
49
13
  end
50
14
  end
51
15
  end
52
-
53
- # storage/sqlite is loaded on demand when spans are enabled
54
- # to avoid requiring sqlite3 in production environments