prosopite 2.1.2 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/prosopite.rb CHANGED
@@ -1,3 +1,8 @@
1
+ require 'fileutils'
2
+ require 'set'
3
+ require 'yaml'
4
+ require 'prosopite/version'
5
+ require 'prosopite/fixes'
1
6
 
2
7
  module Prosopite
3
8
  DEFAULT_ALLOW_LIST = [
@@ -5,7 +10,30 @@ module Prosopite
5
10
  'active_record/validations/uniqueness'
6
11
  ].freeze
7
12
 
13
+ # Ruby 3.2+ can pass values on to the fibers spawned by the current one
14
+ FIBER_STORAGE = Fiber.respond_to?(:[])
15
+
16
+ # The call stacks of queries that ran once, kept to show where records were loaded, take
17
+ # memory, so only this many are kept per scan
18
+ MAX_SINGLE_CALLERS = 500
19
+
20
+ TODO_HEADER = <<~YAML
21
+ # Known N+1 queries, which Prosopite ignores until they're fixed. Remove the entries of
22
+ # the ones you fix, or delete this file and record it again with Prosopite.record_todo.
23
+ YAML
24
+
8
25
  class NPlusOneQueriesError < StandardError; end
26
+
27
+ # An Active Support subscriber that pauses scans during its event, see Prosopite.pause_during
28
+ class EventPause
29
+ def start(_name, _id, _payload)
30
+ (Thread.current[:prosopite_event_pauses] ||= []) << Prosopite.send(:begin_pause)
31
+ end
32
+
33
+ def finish(_name, _id, _payload)
34
+ Prosopite.send(:end_pause, Thread.current[:prosopite_event_pauses].pop)
35
+ end
36
+ end
9
37
  class << self
10
38
  attr_writer :raise,
11
39
  :stderr_logger,
@@ -14,49 +42,80 @@ module Prosopite
14
42
  :custom_logger,
15
43
  :ignore_pauses,
16
44
  :backtrace_cleaner,
17
- :enabled
45
+ :enabled,
46
+ :suggest_fixes
18
47
 
19
48
  attr_accessor :allow_stack_paths,
49
+ :focus_stack_paths,
20
50
  :ignore_queries,
21
- :min_n_queries
51
+ :min_n_queries,
52
+ :display_max_n_queries,
53
+ :display_query_count,
54
+ :add_footer,
55
+ :browser_console,
56
+ :record_todo,
57
+ :github_annotations,
58
+ :json_report,
59
+ :min_duration_ms
60
+
61
+ attr_reader :todo_file
62
+
63
+ attr_reader :ignore_location_paths
64
+
65
+ def ignore_location_paths=(paths)
66
+ @ignore_location_paths = paths
67
+ # One regexp for all, as String#match? would compile one for every frame of every query
68
+ @ignore_location_paths_regexp = paths.blank? ? nil : patterns_regexp(paths)
69
+ # The frames of a file have the same path string in every query, so match it once
70
+ @ignored_paths = {}.compare_by_identity
71
+ end
22
72
 
23
73
  def allow_list=(value)
24
- puts "Prosopite.allow_list= is deprecated. Use Prosopite.allow_stack_paths= instead."
74
+ warn "Prosopite.allow_list= is deprecated and will be removed in 3.0. Use Prosopite.allow_stack_paths= instead."
25
75
 
26
76
  self.allow_stack_paths = value
27
77
  end
28
78
 
79
+ attr_reader :sample_rate
80
+
81
+ def sample_rate=(rate)
82
+ share = rate.nil? ? nil : Float(rate)
83
+ raise ArgumentError, "Prosopite.sample_rate must be from 0 to 1, not #{rate}" unless share.nil? || share.between?(0, 1)
84
+
85
+ @sample_rate = share
86
+ end
87
+
88
+ def todo_file=(path)
89
+ @todo_file = path&.to_s
90
+ @todo_entries = nil
91
+ @todo_recorded = Set.new
92
+ end
93
+
29
94
  def backtrace_cleaner
30
95
  @backtrace_cleaner ||= Rails.backtrace_cleaner
31
96
  end
32
97
 
33
98
  def enabled?
34
- @enabled = true if @enabled.nil?
35
-
36
- @enabled
99
+ @enabled != false
37
100
  end
38
101
 
39
102
  def disabled?
40
103
  !enabled?
41
104
  end
42
105
 
43
- def scan
44
- tc[:prosopite_scan] ||= false
45
- if scan? || disabled?
106
+ def suggest_fixes?
107
+ @suggest_fixes != false
108
+ end
109
+
110
+ def scan(context = nil)
111
+ # A scan of the current fiber, even a paused one, makes this one nested, while a
112
+ # fiber spawned during a scan can still start a scan of its own
113
+ own_state = Thread.current[:prosopite_state]
114
+ if (own_state && own_state[:prosopite_scan]) || disabled? || (@sample_rate && rand >= @sample_rate)
46
115
  return block_given? ? yield : nil
47
116
  end
48
117
 
49
- subscribe
50
-
51
- tc[:prosopite_query_counter] = Hash.new(0)
52
- tc[:prosopite_query_holder] = Hash.new { |h, k| h[k] = [] }
53
- tc[:prosopite_query_caller] = {}
54
-
55
- @allow_stack_paths ||= []
56
- @ignore_pauses ||= false
57
- @min_n_queries ||= 2
58
-
59
- tc[:prosopite_scan] = true
118
+ state = start_scan(context)
60
119
 
61
120
  if block_given?
62
121
  begin
@@ -64,53 +123,86 @@ module Prosopite
64
123
  finish
65
124
  block_result
66
125
  ensure
67
- tc[:prosopite_scan] = false
126
+ discard(state)
68
127
  end
69
128
  end
70
129
  end
71
130
 
131
+ # Runs the block in a scan of its own and returns its N+1 queries, without sending
132
+ # them anywhere or raising, e.g. for test assertions
133
+ def n_plus_one_queries
134
+ outer_state = Thread.current[:prosopite_state]
135
+ outer_inherited_state = Fiber[:prosopite_state] if FIBER_STORAGE
136
+ outer_notifications = tc[:prosopite_notifications]
137
+ state = start_scan(nil)
138
+ # The scans of code in the block, e.g. of the Rack middleware, can't finish it
139
+ state[:prosopite_assertion] = true
140
+
141
+ begin
142
+ yield
143
+ state[:prosopite_scan] = false
144
+ create_notifications
145
+ build_notifications
146
+ ensure
147
+ release(state)
148
+ Thread.current[:prosopite_state] = outer_state
149
+ Fiber[:prosopite_state] = outer_inherited_state if FIBER_STORAGE
150
+ tc[:prosopite_notifications] = outer_notifications
151
+ end
152
+ end
153
+
72
154
  def tc
73
155
  Thread.current
74
156
  end
75
157
 
158
+ # A pause applies to the whole scan, in every fiber and thread taking part in it
76
159
  def pause
77
160
  if @ignore_pauses
78
161
  return block_given? ? yield : nil
79
162
  end
80
163
 
81
164
  if block_given?
165
+ state = begin_pause
82
166
  begin
83
- previous = tc[:prosopite_scan]
84
- tc[:prosopite_scan] = false
85
167
  yield
86
168
  ensure
87
- tc[:prosopite_scan] = previous
169
+ end_pause(state)
88
170
  end
89
- else
90
- tc[:prosopite_scan] = false
171
+ elsif (state = scan_state)
172
+ state[:prosopite_paused] = true
91
173
  end
92
174
  end
93
175
 
176
+ # Pauses scans while the code instrumented as this Active Support event runs,
177
+ # e.g. 'factory_bot.run_factory' for the factories of FactoryBot
178
+ def pause_during(event)
179
+ ActiveSupport::Notifications.subscribe(event, EventPause.new)
180
+ end
181
+
94
182
  def resume
95
- tc[:prosopite_scan] = true
183
+ state = scan_state
184
+ state[:prosopite_paused] = false if state
96
185
  end
97
186
 
98
187
  def scan?
99
- !!(tc[:prosopite_scan] && tc[:prosopite_query_counter] &&
100
- tc[:prosopite_query_holder] && tc[:prosopite_query_caller])
188
+ !!scanning_state
101
189
  end
102
190
 
103
191
  def finish
104
- return unless scan?
105
-
106
- tc[:prosopite_scan] = false
107
-
108
- create_notifications
109
- send_notifications if tc[:prosopite_notifications].present?
110
-
111
- tc[:prosopite_query_counter] = nil
112
- tc[:prosopite_query_holder] = nil
113
- tc[:prosopite_query_caller] = nil
192
+ # Only the fiber that started the scan, or the thread of an ActionController::Live
193
+ # action, can finish it, even while it's paused, e.g. by another fiber or thread
194
+ state = Thread.current[:prosopite_state]
195
+ return unless state && state[:prosopite_scan] && !state[:prosopite_assertion]
196
+
197
+ state[:prosopite_scan] = false
198
+
199
+ begin
200
+ create_notifications
201
+ filter_notifications
202
+ tc[:prosopite_notifications].present? ? send_notifications : []
203
+ ensure
204
+ discard(state)
205
+ end
114
206
  end
115
207
 
116
208
  def start_raise
@@ -130,52 +222,62 @@ module Prosopite
130
222
  end
131
223
 
132
224
  def create_notifications
133
- tc[:prosopite_notifications] = {}
225
+ state = Thread.current[:prosopite_state]
226
+ # By identity, as the queries of N+1s in different places can be the same strings
227
+ tc[:prosopite_notifications] = {}.compare_by_identity
228
+ allow_list = nil
229
+ # The queries of an N+1 are often the same string, with bind parameters
230
+ fingerprints_by_adapter = Hash.new do |by_adapter, adapter|
231
+ by_adapter[adapter] = Hash.new { |by_query, query| by_query[query] = fingerprint_query(query, adapter) }
232
+ end
134
233
 
135
- tc[:prosopite_query_counter].each do |location_key, count|
234
+ # A copy, as an ActionController::Live action may still be running queries
235
+ state[:prosopite_query_counter].dup.each do |location_key, count|
136
236
  if count >= @min_n_queries
137
- fingerprints = tc[:prosopite_query_holder][location_key].group_by do |q|
237
+ db_adapter = state[:prosopite_query_adapter][location_key]
238
+ fingerprints = state[:prosopite_query_holder][location_key].group_by do |q|
138
239
  begin
139
- fingerprint(q)
240
+ fingerprints_by_adapter[db_adapter][q]
140
241
  rescue
141
242
  raise q
142
243
  end
143
244
  end
144
245
 
145
- queries = fingerprints.values.select { |q| q.size >= @min_n_queries }
246
+ queries = fingerprints.select { |_, q| q.size >= @min_n_queries }
146
247
 
147
248
  next unless queries.any?
148
249
 
149
- kaller = tc[:prosopite_query_caller][location_key].map(&:to_s)
150
- allow_list = (@allow_stack_paths + DEFAULT_ALLOW_LIST)
151
- is_allowed = kaller.any? { |f| allow_list.any? { |s| f.match?(s) } }
152
-
153
- unless is_allowed
154
- queries.each do |q|
155
- tc[:prosopite_notifications][q] = kaller
250
+ kaller = state[:prosopite_query_caller][location_key].map(&:to_s)
251
+ allow_list ||= patterns_regexp(@allow_stack_paths + DEFAULT_ALLOW_LIST)
252
+ is_allowed = kaller.any? { |f| f.match?(allow_list) }
253
+ is_focused = @focus_stack_paths.empty? ||
254
+ kaller.any? { |f| @focus_stack_paths.any? { |s| s.is_a?(Regexp) ? f.match?(s) : f.include?(s) } }
255
+
256
+ if is_focused && !is_allowed
257
+ duration_ms = state[:prosopite_query_duration][location_key]
258
+ queries.each do |fingerprint, q|
259
+ tc[:prosopite_notifications][q] = {
260
+ kaller: kaller, duration_ms: duration_ms, fingerprint: fingerprint, location_key: location_key
261
+ }
156
262
  end
157
263
  end
158
264
  end
159
265
  end
160
266
  end
161
267
 
162
- def fingerprint(query)
163
- db_adapter = ActiveRecord::Base.connection_db_config.adapter
164
- if db_adapter.include?('mysql') || db_adapter.include?('trilogy')
268
+ def fingerprint(query, db_adapter = ActiveRecord::Base.connection_db_config.adapter)
269
+ if db_adapter.match?(/mysql|trilogy/i)
165
270
  mysql_fingerprint(query)
166
- else
167
- begin
168
- require 'pg_query'
169
- rescue LoadError => e
170
- msg = "Could not load the 'pg_query' gem. Add `gem 'pg_query'` to your Gemfile"
171
- raise LoadError, msg, e.backtrace
172
- end
271
+ elsif db_adapter.match?(/postg/i) && pg_query?
173
272
  PgQuery.fingerprint(query)
273
+ else
274
+ # SQLite, Oracle, PostgreSQL without pg_query etc. use double quotes for identifiers, not strings
275
+ mysql_fingerprint(query, ansi_quotes: true)
174
276
  end
175
277
  end
176
278
 
177
279
  # Many thanks to https://github.com/genkami/fluent-plugin-query-fingerprint/
178
- def mysql_fingerprint(query)
280
+ def mysql_fingerprint(query, ansi_quotes: false)
179
281
  query = query.dup
180
282
 
181
283
  return "mysqldump" if query =~ %r#\ASELECT /\*!40001 SQL_NO_CACHE \*/ \* FROM `#
@@ -194,8 +296,10 @@ module Prosopite
194
296
  return query if query.gsub!(/\Ause \S+\Z/i, "use ?")
195
297
 
196
298
  query.gsub!(/\\["']/, "")
197
- query.gsub!(/".*?"/m, "?")
299
+ query.gsub!(/".*?"/m, "?") unless ansi_quotes
198
300
  query.gsub!(/'.*?'/m, "?")
301
+ # The placeholders of PostgreSQL, before the numbers, so that IN lists of them collapse
302
+ query.gsub!(/\$\d+/, "?") if ansi_quotes
199
303
 
200
304
  query.gsub!(/\btrue\b|\bfalse\b/i, "?")
201
305
 
@@ -224,39 +328,73 @@ module Prosopite
224
328
  end
225
329
 
226
330
  def send_notifications
227
- @custom_logger ||= false
228
- @rails_logger ||= false
229
- @stderr_logger ||= false
230
- @prosopite_logger ||= false
231
-
232
- notifications_str = String.new('')
331
+ notifications = build_notifications
233
332
 
234
- tc[:prosopite_notifications].each do |queries, kaller|
235
- notifications_str << "N+1 queries detected:\n"
236
-
237
- queries.each { |q| notifications_str << " #{q}\n" }
333
+ notifications.each do |notification|
334
+ ActiveSupport::Notifications.instrument('n_plus_one_queries.prosopite', notification)
335
+ end
238
336
 
239
- notifications_str << "Call stack:\n"
240
- kaller = backtrace_cleaner.clean(kaller)
241
- kaller.each do |f|
242
- notifications_str << " #{f}\n"
243
- end
337
+ notifications_str = notifications.map { |notification| notification_message(notification) }.join
244
338
 
245
- notifications_str << "\n"
246
- end
339
+ annotate_github(notifications) if @github_annotations
340
+ write_json_report(notifications) if @json_report
247
341
 
248
342
  @custom_logger.warn(notifications_str) if @custom_logger
249
343
 
250
- Rails.logger.warn(red(notifications_str)) if @rails_logger
344
+ Rails.logger&.warn(red(notifications_str)) if rails_logger?
251
345
  $stderr.puts(red(notifications_str)) if @stderr_logger
346
+ log_to_file(notifications_str) if @prosopite_logger
347
+
348
+ raise NPlusOneQueriesError.new(notifications_str) if raise?
349
+
350
+ notifications
351
+ end
352
+
353
+ def notification_message(notification)
354
+ queries = notification[:queries]
355
+ duration_ms = notification[:duration_ms]
356
+
357
+ details = []
358
+ details << "#{queries.count} queries" if @display_query_count
359
+ details << "#{duration_ms.round(1)}ms"
360
+
361
+ message = String.new("N+1 queries detected (#{details.join(', ')}):\n")
252
362
 
253
- if @prosopite_logger
254
- File.open(File.join(Rails.root, 'log', 'prosopite.log'), 'a') do |f|
255
- f.puts(notifications_str)
363
+ queries_to_display =
364
+ if @display_max_n_queries
365
+ queries.take(@display_max_n_queries)
366
+ else
367
+ queries
368
+ end
369
+
370
+ queries_to_display.each { |q| message << " #{q}\n" }
371
+
372
+ if notification[:fixes].present?
373
+ message << (notification[:fixes].size == 1 ? "Possible fix:\n" : "Possible fixes:\n")
374
+ notification[:fixes].each do |fix|
375
+ message << " #{fix}\n"
256
376
  end
257
377
  end
258
378
 
259
- raise NPlusOneQueriesError.new(notifications_str) if raise?
379
+ if notification[:context]
380
+ message << "Context:\n"
381
+ notification[:context].to_s.each_line do |line|
382
+ message << " #{line.chomp}\n"
383
+ end
384
+ end
385
+
386
+ # Last, as extensions like dial read the frames after it
387
+ message << "Call stack:\n"
388
+ notification[:call_stack].each do |f|
389
+ message << " #{f}\n"
390
+ end
391
+
392
+ message << "\n"
393
+ end
394
+
395
+ # Unless it's set, the Rails log is where N+1 queries go when no other logger is set
396
+ def rails_logger?
397
+ @rails_logger.nil? ? !(@custom_logger || @stderr_logger || @prosopite_logger) : @rails_logger
260
398
  end
261
399
 
262
400
  def red(str)
@@ -264,7 +402,8 @@ module Prosopite
264
402
  end
265
403
 
266
404
  def ignore_query?(sql)
267
- @ignore_queries ||= []
405
+ return false if @ignore_queries.blank?
406
+
268
407
  @ignore_queries.any? { |q| q === sql }
269
408
  end
270
409
 
@@ -272,29 +411,273 @@ module Prosopite
272
411
  @subscribed ||= false
273
412
  return if @subscribed
274
413
 
275
- ActiveSupport::Notifications.subscribe 'sql.active_record' do |_, _, _, _, data|
414
+ ActiveSupport::Notifications.subscribe 'sql.active_record' do |_, start, finish, _, data|
276
415
  sql, name = data[:sql], data[:name]
416
+ state = scanning_state
277
417
 
278
- if scan? && name != "SCHEMA" && sql.include?('SELECT') && data[:cached].nil? && !ignore_query?(sql)
418
+ if state && name != "SCHEMA" && data[:cached].nil? && sql.include?('SELECT') && !ignore_query?(sql)
279
419
  query_caller = caller_locations
420
+ connection = data[:connection]
421
+ pool = connection&.pool
422
+ ignore_regexp = @ignore_location_paths_regexp
423
+ ignored_paths = @ignored_paths
424
+ # Queries on different databases (e.g. shards) can't be merged into one,
425
+ # so they are kept under different location keys.
280
426
  # Calculate the location key with as few allocations as possible
281
- location_key = [].tap do |array|
427
+ location_key = [(pool.db_config if pool.respond_to?(:db_config))].tap do |array|
282
428
  query_caller.each do |loc|
283
- array << loc.path
429
+ path = loc.path
430
+ next if ignore_regexp && ignored_paths.fetch(path) { ignored_path!(ignored_paths, ignore_regexp, path) }
431
+
432
+ array << path
284
433
  array << loc.lineno
285
434
  end
286
435
  end.hash
287
436
 
288
- tc[:prosopite_query_counter][location_key] += 1
289
- tc[:prosopite_query_holder][location_key] << sql
290
-
291
- if tc[:prosopite_query_counter][location_key] > 1
292
- tc[:prosopite_query_caller][location_key] = query_caller
437
+ counter = state[:prosopite_query_counter]
438
+ state[:prosopite_query_holder][location_key] << sql
439
+ state[:prosopite_query_duration][location_key] += (finish - start) * 1000
440
+ state[:prosopite_query_adapter][location_key] ||= connection&.adapter_name
441
+
442
+ # Store the call stack before the count reaches 2, since an ActionController::Live
443
+ # action can still run queries while the request thread finishes the scan. Fix
444
+ # suggestions show where records were loaded from the ones of single queries too.
445
+ callers = state[:prosopite_query_caller]
446
+ if counter[location_key] > 0 || callers.size < MAX_SINGLE_CALLERS
447
+ callers[location_key] ||= query_caller
293
448
  end
449
+ counter[location_key] += 1
294
450
  end
295
451
  end
296
452
 
297
453
  @subscribed = true
298
454
  end
455
+
456
+ private
457
+
458
+ # pg_query is optional, as its native extension can be hard to install
459
+ def pg_query?
460
+ return @pg_query if defined?(@pg_query)
461
+
462
+ @pg_query =
463
+ begin
464
+ require 'pg_query'
465
+ true
466
+ rescue LoadError
467
+ false
468
+ end
469
+ end
470
+
471
+ # true for log/prosopite.log, or a path, or a logger
472
+ def log_to_file(notifications_str)
473
+ return @prosopite_logger.warn(notifications_str) if @prosopite_logger.respond_to?(:warn)
474
+
475
+ path = @prosopite_logger == true ? File.join(Rails.root, 'log', 'prosopite.log') : @prosopite_logger.to_s
476
+ FileUtils.mkdir_p(File.dirname(path))
477
+ File.open(path, 'a') { |file| file.puts(notifications_str) }
478
+ end
479
+
480
+ def begin_pause
481
+ return if @ignore_pauses
482
+
483
+ state = scan_state
484
+ state[:prosopite_pauses] += 1 if state
485
+ state
486
+ end
487
+
488
+ def end_pause(state)
489
+ state[:prosopite_pauses] -= 1 if state
490
+ end
491
+
492
+ def start_scan(context)
493
+ subscribe
494
+
495
+ @allow_stack_paths ||= []
496
+ @focus_stack_paths ||= []
497
+ @ignore_pauses ||= false
498
+ @min_n_queries ||= 2
499
+
500
+ # A new state for every scan, so that fibers or threads still holding the
501
+ # previous one can't add to this scan
502
+ state = {
503
+ prosopite_scan: true,
504
+ prosopite_paused: false,
505
+ prosopite_pauses: 0,
506
+ prosopite_query_counter: Hash.new(0),
507
+ prosopite_query_holder: Hash.new { |h, k| h[k] = [] },
508
+ prosopite_query_caller: {},
509
+ prosopite_query_duration: Hash.new(0.0),
510
+ prosopite_query_adapter: {},
511
+ prosopite_context: context,
512
+ thread: Thread.current
513
+ }
514
+ Thread.current[:prosopite_state] = state
515
+ Fiber[:prosopite_state] = state if FIBER_STORAGE
516
+ state
517
+ end
518
+
519
+ def build_notifications
520
+ state = Thread.current[:prosopite_state]
521
+ loads = Fixes::Loads.new(state, backtrace_cleaner) if suggest_fixes?
522
+ tc[:prosopite_notifications].map do |queries, info|
523
+ {
524
+ queries: queries,
525
+ call_stack: backtrace_cleaner.clean(info[:kaller]),
526
+ duration_ms: info[:duration_ms],
527
+ fixes: fixes(queries.first, info[:location_key], loads),
528
+ fingerprint: info[:fingerprint],
529
+ context: state[:prosopite_context]
530
+ }
531
+ end
532
+ end
533
+
534
+ # Computed for the N+1 queries left after the filters, as they take some time
535
+ def fixes(query, location_key, loads)
536
+ return [] unless loads && location_key
537
+
538
+ Fixes.for(query, loop_table: loads.loop_table(location_key)) { |model| loads.at(location_key, model.table_name) }
539
+ rescue StandardError
540
+ # A suggestion must never keep an N+1 from being reported
541
+ []
542
+ end
543
+
544
+ # The state of the scan of the current fiber: set by the fiber that started it,
545
+ # copied by ActionController::Live to the thread of its action, and passed on
546
+ # to the fibers spawned in the same thread
547
+ def scan_state
548
+ state = Thread.current[:prosopite_state]
549
+ return state if state || !FIBER_STORAGE
550
+
551
+ state = Fiber[:prosopite_state]
552
+ state if state && state[:thread].equal?(Thread.current)
553
+ end
554
+
555
+ # Workflow commands that show the N+1 queries on the code in GitHub Actions, once each
556
+ def annotate_github(notifications)
557
+ @github_annotated ||= Set.new
558
+
559
+ notifications.each do |notification|
560
+ file, line = notification[:call_stack].first.to_s.match(/\A(.+?):(\d+)/)&.captures
561
+ next unless @github_annotated.add?([file, line, notification[:fingerprint]])
562
+
563
+ properties = { file: file, line: line, title: 'N+1 queries detected' }.compact
564
+ properties = properties.map { |name, value| "#{name}=#{escape_github(value).gsub(':', '%3A').gsub(',', '%2C')}" }
565
+ $stdout.puts("::warning #{properties.join(',')}::#{escape_github(notification_message(notification).chomp)}")
566
+ end
567
+ end
568
+
569
+ def escape_github(value)
570
+ value.gsub('%', '%25').gsub("\r", '%0D').gsub("\n", '%0A')
571
+ end
572
+
573
+ # A line of JSON for each N+1, under a lock, as parallel test processes write together
574
+ def write_json_report(notifications)
575
+ File.open(@json_report, 'a') do |file|
576
+ file.flock(File::LOCK_EX)
577
+ notifications.each { |notification| file.puts(notification.to_json) }
578
+ end
579
+ end
580
+
581
+ # Filters for the N+1 queries to report, which assertions don't use
582
+ def filter_notifications
583
+ if @min_duration_ms
584
+ tc[:prosopite_notifications].reject! { |_, info| info[:duration_ms].to_f < @min_duration_ms }
585
+ end
586
+ apply_todo if @todo_file
587
+ end
588
+
589
+ # Ignores the N+1 queries in the todo file, or records them to it
590
+ def apply_todo
591
+ tc[:prosopite_notifications].reject! do |queries, info|
592
+ entry = { 'fingerprint' => info[:fingerprint], 'location' => todo_location(info[:kaller]) }
593
+
594
+ if @record_todo
595
+ # Once per process, as each entry rewrites the file
596
+ add_todo_entry(entry.merge('query' => queries.first)) if @todo_recorded.add?(entry.values)
597
+ true
598
+ else
599
+ todo_entries.include?([entry['fingerprint'], entry['location']])
600
+ end
601
+ end
602
+ end
603
+
604
+ # The file of the N+1, without its line, so that unrelated changes don't affect it
605
+ def todo_location(kaller)
606
+ backtrace_cleaner.clean(kaller).first&.sub(/:\d+.*\z/, '')
607
+ end
608
+
609
+ def todo_entries
610
+ @todo_entries ||= read_todo_entries(@todo_file).map { |entry| [entry['fingerprint'], entry['location']] }.to_set
611
+ end
612
+
613
+ def read_todo_entries(path)
614
+ File.exist?(path) ? YAML.safe_load(File.read(path)) || [] : []
615
+ end
616
+
617
+ # Merges the entry into the file, under a lock, as parallel test processes record together
618
+ def add_todo_entry(entry)
619
+ File.open(@todo_file, File::RDWR | File::CREAT, 0o644) do |file|
620
+ file.flock(File::LOCK_EX)
621
+ entries = YAML.safe_load(file.read) || []
622
+ next if entries.any? { |e| e.values_at('fingerprint', 'location') == entry.values_at('fingerprint', 'location') }
623
+
624
+ entries << entry
625
+ entries.sort_by! { |e| [e['location'].to_s, e['fingerprint'].to_s] }
626
+ file.rewind
627
+ file.write(TODO_HEADER + entries.to_yaml)
628
+ file.truncate(file.pos)
629
+ end
630
+ end
631
+
632
+ def scanning_state
633
+ state = scan_state
634
+ state if state && scanning?(state)
635
+ end
636
+
637
+ def scanning?(state)
638
+ state[:prosopite_scan] && !state[:prosopite_paused] && state[:prosopite_pauses].zero?
639
+ end
640
+
641
+ def ignored_path!(ignored_paths, ignore_regexp, path)
642
+ # Reloading in development makes new path strings
643
+ ignored_paths.clear if ignored_paths.size >= 10_000
644
+ ignored_paths[path] = ignore_regexp.match?(path)
645
+ end
646
+
647
+ # Strings are patterns too, like for String#match?
648
+ def patterns_regexp(patterns)
649
+ Regexp.union(patterns.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) })
650
+ end
651
+
652
+ # Ends the scan of the state without reporting it
653
+ def discard(state)
654
+ state[:prosopite_scan] = false
655
+ if Thread.current[:prosopite_state].equal?(state)
656
+ Thread.current[:prosopite_state] = nil
657
+ Fiber[:prosopite_state] = nil if FIBER_STORAGE
658
+ end
659
+ release(state)
660
+ end
661
+
662
+ # Extensions may override fingerprint(query) without the adapter, as before 2.3.0
663
+ def fingerprint_query(query, db_adapter)
664
+ if db_adapter && method(:fingerprint).arity != 1
665
+ fingerprint(query, db_adapter)
666
+ else
667
+ fingerprint(query)
668
+ end
669
+ end
670
+
671
+ # Fibers and threads that inherited the state, like threads created during
672
+ # the scan, can keep it for their lifetime, so empty it
673
+ def release(state)
674
+ state[:thread] = nil
675
+ state[:prosopite_context] = nil
676
+ state[:prosopite_query_counter].clear
677
+ state[:prosopite_query_holder].clear
678
+ state[:prosopite_query_caller].clear
679
+ state[:prosopite_query_duration].clear
680
+ state[:prosopite_query_adapter].clear
681
+ end
299
682
  end
300
683
  end