prosopite 2.1.2 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +169 -0
- data/README.md +406 -183
- data/lib/generators/prosopite/install_generator.rb +53 -0
- data/lib/generators/prosopite/templates/prosopite.rb.tt +18 -0
- data/lib/prosopite/fixes.rb +181 -0
- data/lib/prosopite/middleware/rack.rb +212 -2
- data/lib/prosopite/middleware/sidekiq.rb +2 -2
- data/lib/prosopite/minitest.rb +23 -0
- data/lib/prosopite/rspec.rb +20 -0
- data/lib/prosopite/version.rb +1 -1
- data/lib/prosopite.rb +478 -95
- metadata +22 -24
- data/.github/workflows/ci.yml +0 -15
- data/.gitignore +0 -8
- data/Gemfile +0 -8
- data/Gemfile.lock +0 -152
- data/Rakefile +0 -9
- data/prosopite.gemspec +0 -34
data/lib/prosopite.rb
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
require 'fileutils'
|
|
2
|
+
require 'set'
|
|
3
|
+
require 'yaml'
|
|
4
|
+
require 'prosopite/version'
|
|
5
|
+
require 'prosopite/fixes'
|
|
1
6
|
|
|
2
7
|
module Prosopite
|
|
3
8
|
DEFAULT_ALLOW_LIST = [
|
|
@@ -5,7 +10,30 @@ module Prosopite
|
|
|
5
10
|
'active_record/validations/uniqueness'
|
|
6
11
|
].freeze
|
|
7
12
|
|
|
13
|
+
# Ruby 3.2+ can pass values on to the fibers spawned by the current one
|
|
14
|
+
FIBER_STORAGE = Fiber.respond_to?(:[])
|
|
15
|
+
|
|
16
|
+
# The call stacks of queries that ran once, kept to show where records were loaded, take
|
|
17
|
+
# memory, so only this many are kept per scan
|
|
18
|
+
MAX_SINGLE_CALLERS = 500
|
|
19
|
+
|
|
20
|
+
TODO_HEADER = <<~YAML
|
|
21
|
+
# Known N+1 queries, which Prosopite ignores until they're fixed. Remove the entries of
|
|
22
|
+
# the ones you fix, or delete this file and record it again with Prosopite.record_todo.
|
|
23
|
+
YAML
|
|
24
|
+
|
|
8
25
|
class NPlusOneQueriesError < StandardError; end
|
|
26
|
+
|
|
27
|
+
# An Active Support subscriber that pauses scans during its event, see Prosopite.pause_during
|
|
28
|
+
class EventPause
|
|
29
|
+
def start(_name, _id, _payload)
|
|
30
|
+
(Thread.current[:prosopite_event_pauses] ||= []) << Prosopite.send(:begin_pause)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def finish(_name, _id, _payload)
|
|
34
|
+
Prosopite.send(:end_pause, Thread.current[:prosopite_event_pauses].pop)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
9
37
|
class << self
|
|
10
38
|
attr_writer :raise,
|
|
11
39
|
:stderr_logger,
|
|
@@ -14,49 +42,80 @@ module Prosopite
|
|
|
14
42
|
:custom_logger,
|
|
15
43
|
:ignore_pauses,
|
|
16
44
|
:backtrace_cleaner,
|
|
17
|
-
:enabled
|
|
45
|
+
:enabled,
|
|
46
|
+
:suggest_fixes
|
|
18
47
|
|
|
19
48
|
attr_accessor :allow_stack_paths,
|
|
49
|
+
:focus_stack_paths,
|
|
20
50
|
:ignore_queries,
|
|
21
|
-
:min_n_queries
|
|
51
|
+
:min_n_queries,
|
|
52
|
+
:display_max_n_queries,
|
|
53
|
+
:display_query_count,
|
|
54
|
+
:add_footer,
|
|
55
|
+
:browser_console,
|
|
56
|
+
:record_todo,
|
|
57
|
+
:github_annotations,
|
|
58
|
+
:json_report,
|
|
59
|
+
:min_duration_ms
|
|
60
|
+
|
|
61
|
+
attr_reader :todo_file
|
|
62
|
+
|
|
63
|
+
attr_reader :ignore_location_paths
|
|
64
|
+
|
|
65
|
+
def ignore_location_paths=(paths)
|
|
66
|
+
@ignore_location_paths = paths
|
|
67
|
+
# One regexp for all, as String#match? would compile one for every frame of every query
|
|
68
|
+
@ignore_location_paths_regexp = paths.blank? ? nil : patterns_regexp(paths)
|
|
69
|
+
# The frames of a file have the same path string in every query, so match it once
|
|
70
|
+
@ignored_paths = {}.compare_by_identity
|
|
71
|
+
end
|
|
22
72
|
|
|
23
73
|
def allow_list=(value)
|
|
24
|
-
|
|
74
|
+
warn "Prosopite.allow_list= is deprecated and will be removed in 3.0. Use Prosopite.allow_stack_paths= instead."
|
|
25
75
|
|
|
26
76
|
self.allow_stack_paths = value
|
|
27
77
|
end
|
|
28
78
|
|
|
79
|
+
attr_reader :sample_rate
|
|
80
|
+
|
|
81
|
+
def sample_rate=(rate)
|
|
82
|
+
share = rate.nil? ? nil : Float(rate)
|
|
83
|
+
raise ArgumentError, "Prosopite.sample_rate must be from 0 to 1, not #{rate}" unless share.nil? || share.between?(0, 1)
|
|
84
|
+
|
|
85
|
+
@sample_rate = share
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def todo_file=(path)
|
|
89
|
+
@todo_file = path&.to_s
|
|
90
|
+
@todo_entries = nil
|
|
91
|
+
@todo_recorded = Set.new
|
|
92
|
+
end
|
|
93
|
+
|
|
29
94
|
def backtrace_cleaner
|
|
30
95
|
@backtrace_cleaner ||= Rails.backtrace_cleaner
|
|
31
96
|
end
|
|
32
97
|
|
|
33
98
|
def enabled?
|
|
34
|
-
@enabled
|
|
35
|
-
|
|
36
|
-
@enabled
|
|
99
|
+
@enabled != false
|
|
37
100
|
end
|
|
38
101
|
|
|
39
102
|
def disabled?
|
|
40
103
|
!enabled?
|
|
41
104
|
end
|
|
42
105
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
|
|
106
|
+
def suggest_fixes?
|
|
107
|
+
@suggest_fixes != false
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def scan(context = nil)
|
|
111
|
+
# A scan of the current fiber, even a paused one, makes this one nested, while a
|
|
112
|
+
# fiber spawned during a scan can still start a scan of its own
|
|
113
|
+
own_state = Thread.current[:prosopite_state]
|
|
114
|
+
if (own_state && own_state[:prosopite_scan]) || disabled? || (@sample_rate && rand >= @sample_rate)
|
|
46
115
|
return block_given? ? yield : nil
|
|
47
116
|
end
|
|
48
117
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
tc[:prosopite_query_counter] = Hash.new(0)
|
|
52
|
-
tc[:prosopite_query_holder] = Hash.new { |h, k| h[k] = [] }
|
|
53
|
-
tc[:prosopite_query_caller] = {}
|
|
54
|
-
|
|
55
|
-
@allow_stack_paths ||= []
|
|
56
|
-
@ignore_pauses ||= false
|
|
57
|
-
@min_n_queries ||= 2
|
|
58
|
-
|
|
59
|
-
tc[:prosopite_scan] = true
|
|
118
|
+
state = start_scan(context)
|
|
60
119
|
|
|
61
120
|
if block_given?
|
|
62
121
|
begin
|
|
@@ -64,53 +123,86 @@ module Prosopite
|
|
|
64
123
|
finish
|
|
65
124
|
block_result
|
|
66
125
|
ensure
|
|
67
|
-
|
|
126
|
+
discard(state)
|
|
68
127
|
end
|
|
69
128
|
end
|
|
70
129
|
end
|
|
71
130
|
|
|
131
|
+
# Runs the block in a scan of its own and returns its N+1 queries, without sending
|
|
132
|
+
# them anywhere or raising, e.g. for test assertions
|
|
133
|
+
def n_plus_one_queries
|
|
134
|
+
outer_state = Thread.current[:prosopite_state]
|
|
135
|
+
outer_inherited_state = Fiber[:prosopite_state] if FIBER_STORAGE
|
|
136
|
+
outer_notifications = tc[:prosopite_notifications]
|
|
137
|
+
state = start_scan(nil)
|
|
138
|
+
# The scans of code in the block, e.g. of the Rack middleware, can't finish it
|
|
139
|
+
state[:prosopite_assertion] = true
|
|
140
|
+
|
|
141
|
+
begin
|
|
142
|
+
yield
|
|
143
|
+
state[:prosopite_scan] = false
|
|
144
|
+
create_notifications
|
|
145
|
+
build_notifications
|
|
146
|
+
ensure
|
|
147
|
+
release(state)
|
|
148
|
+
Thread.current[:prosopite_state] = outer_state
|
|
149
|
+
Fiber[:prosopite_state] = outer_inherited_state if FIBER_STORAGE
|
|
150
|
+
tc[:prosopite_notifications] = outer_notifications
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
72
154
|
def tc
|
|
73
155
|
Thread.current
|
|
74
156
|
end
|
|
75
157
|
|
|
158
|
+
# A pause applies to the whole scan, in every fiber and thread taking part in it
|
|
76
159
|
def pause
|
|
77
160
|
if @ignore_pauses
|
|
78
161
|
return block_given? ? yield : nil
|
|
79
162
|
end
|
|
80
163
|
|
|
81
164
|
if block_given?
|
|
165
|
+
state = begin_pause
|
|
82
166
|
begin
|
|
83
|
-
previous = tc[:prosopite_scan]
|
|
84
|
-
tc[:prosopite_scan] = false
|
|
85
167
|
yield
|
|
86
168
|
ensure
|
|
87
|
-
|
|
169
|
+
end_pause(state)
|
|
88
170
|
end
|
|
89
|
-
|
|
90
|
-
|
|
171
|
+
elsif (state = scan_state)
|
|
172
|
+
state[:prosopite_paused] = true
|
|
91
173
|
end
|
|
92
174
|
end
|
|
93
175
|
|
|
176
|
+
# Pauses scans while the code instrumented as this Active Support event runs,
|
|
177
|
+
# e.g. 'factory_bot.run_factory' for the factories of FactoryBot
|
|
178
|
+
def pause_during(event)
|
|
179
|
+
ActiveSupport::Notifications.subscribe(event, EventPause.new)
|
|
180
|
+
end
|
|
181
|
+
|
|
94
182
|
def resume
|
|
95
|
-
|
|
183
|
+
state = scan_state
|
|
184
|
+
state[:prosopite_paused] = false if state
|
|
96
185
|
end
|
|
97
186
|
|
|
98
187
|
def scan?
|
|
99
|
-
!!
|
|
100
|
-
tc[:prosopite_query_holder] && tc[:prosopite_query_caller])
|
|
188
|
+
!!scanning_state
|
|
101
189
|
end
|
|
102
190
|
|
|
103
191
|
def finish
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
192
|
+
# Only the fiber that started the scan, or the thread of an ActionController::Live
|
|
193
|
+
# action, can finish it, even while it's paused, e.g. by another fiber or thread
|
|
194
|
+
state = Thread.current[:prosopite_state]
|
|
195
|
+
return unless state && state[:prosopite_scan] && !state[:prosopite_assertion]
|
|
196
|
+
|
|
197
|
+
state[:prosopite_scan] = false
|
|
198
|
+
|
|
199
|
+
begin
|
|
200
|
+
create_notifications
|
|
201
|
+
filter_notifications
|
|
202
|
+
tc[:prosopite_notifications].present? ? send_notifications : []
|
|
203
|
+
ensure
|
|
204
|
+
discard(state)
|
|
205
|
+
end
|
|
114
206
|
end
|
|
115
207
|
|
|
116
208
|
def start_raise
|
|
@@ -130,52 +222,62 @@ module Prosopite
|
|
|
130
222
|
end
|
|
131
223
|
|
|
132
224
|
def create_notifications
|
|
133
|
-
|
|
225
|
+
state = Thread.current[:prosopite_state]
|
|
226
|
+
# By identity, as the queries of N+1s in different places can be the same strings
|
|
227
|
+
tc[:prosopite_notifications] = {}.compare_by_identity
|
|
228
|
+
allow_list = nil
|
|
229
|
+
# The queries of an N+1 are often the same string, with bind parameters
|
|
230
|
+
fingerprints_by_adapter = Hash.new do |by_adapter, adapter|
|
|
231
|
+
by_adapter[adapter] = Hash.new { |by_query, query| by_query[query] = fingerprint_query(query, adapter) }
|
|
232
|
+
end
|
|
134
233
|
|
|
135
|
-
|
|
234
|
+
# A copy, as an ActionController::Live action may still be running queries
|
|
235
|
+
state[:prosopite_query_counter].dup.each do |location_key, count|
|
|
136
236
|
if count >= @min_n_queries
|
|
137
|
-
|
|
237
|
+
db_adapter = state[:prosopite_query_adapter][location_key]
|
|
238
|
+
fingerprints = state[:prosopite_query_holder][location_key].group_by do |q|
|
|
138
239
|
begin
|
|
139
|
-
|
|
240
|
+
fingerprints_by_adapter[db_adapter][q]
|
|
140
241
|
rescue
|
|
141
242
|
raise q
|
|
142
243
|
end
|
|
143
244
|
end
|
|
144
245
|
|
|
145
|
-
queries = fingerprints.
|
|
246
|
+
queries = fingerprints.select { |_, q| q.size >= @min_n_queries }
|
|
146
247
|
|
|
147
248
|
next unless queries.any?
|
|
148
249
|
|
|
149
|
-
kaller =
|
|
150
|
-
allow_list
|
|
151
|
-
is_allowed = kaller.any? { |f|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
250
|
+
kaller = state[:prosopite_query_caller][location_key].map(&:to_s)
|
|
251
|
+
allow_list ||= patterns_regexp(@allow_stack_paths + DEFAULT_ALLOW_LIST)
|
|
252
|
+
is_allowed = kaller.any? { |f| f.match?(allow_list) }
|
|
253
|
+
is_focused = @focus_stack_paths.empty? ||
|
|
254
|
+
kaller.any? { |f| @focus_stack_paths.any? { |s| s.is_a?(Regexp) ? f.match?(s) : f.include?(s) } }
|
|
255
|
+
|
|
256
|
+
if is_focused && !is_allowed
|
|
257
|
+
duration_ms = state[:prosopite_query_duration][location_key]
|
|
258
|
+
queries.each do |fingerprint, q|
|
|
259
|
+
tc[:prosopite_notifications][q] = {
|
|
260
|
+
kaller: kaller, duration_ms: duration_ms, fingerprint: fingerprint, location_key: location_key
|
|
261
|
+
}
|
|
156
262
|
end
|
|
157
263
|
end
|
|
158
264
|
end
|
|
159
265
|
end
|
|
160
266
|
end
|
|
161
267
|
|
|
162
|
-
def fingerprint(query)
|
|
163
|
-
db_adapter
|
|
164
|
-
if db_adapter.include?('mysql') || db_adapter.include?('trilogy')
|
|
268
|
+
def fingerprint(query, db_adapter = ActiveRecord::Base.connection_db_config.adapter)
|
|
269
|
+
if db_adapter.match?(/mysql|trilogy/i)
|
|
165
270
|
mysql_fingerprint(query)
|
|
166
|
-
|
|
167
|
-
begin
|
|
168
|
-
require 'pg_query'
|
|
169
|
-
rescue LoadError => e
|
|
170
|
-
msg = "Could not load the 'pg_query' gem. Add `gem 'pg_query'` to your Gemfile"
|
|
171
|
-
raise LoadError, msg, e.backtrace
|
|
172
|
-
end
|
|
271
|
+
elsif db_adapter.match?(/postg/i) && pg_query?
|
|
173
272
|
PgQuery.fingerprint(query)
|
|
273
|
+
else
|
|
274
|
+
# SQLite, Oracle, PostgreSQL without pg_query etc. use double quotes for identifiers, not strings
|
|
275
|
+
mysql_fingerprint(query, ansi_quotes: true)
|
|
174
276
|
end
|
|
175
277
|
end
|
|
176
278
|
|
|
177
279
|
# Many thanks to https://github.com/genkami/fluent-plugin-query-fingerprint/
|
|
178
|
-
def mysql_fingerprint(query)
|
|
280
|
+
def mysql_fingerprint(query, ansi_quotes: false)
|
|
179
281
|
query = query.dup
|
|
180
282
|
|
|
181
283
|
return "mysqldump" if query =~ %r#\ASELECT /\*!40001 SQL_NO_CACHE \*/ \* FROM `#
|
|
@@ -194,8 +296,10 @@ module Prosopite
|
|
|
194
296
|
return query if query.gsub!(/\Ause \S+\Z/i, "use ?")
|
|
195
297
|
|
|
196
298
|
query.gsub!(/\\["']/, "")
|
|
197
|
-
query.gsub!(/".*?"/m, "?")
|
|
299
|
+
query.gsub!(/".*?"/m, "?") unless ansi_quotes
|
|
198
300
|
query.gsub!(/'.*?'/m, "?")
|
|
301
|
+
# The placeholders of PostgreSQL, before the numbers, so that IN lists of them collapse
|
|
302
|
+
query.gsub!(/\$\d+/, "?") if ansi_quotes
|
|
199
303
|
|
|
200
304
|
query.gsub!(/\btrue\b|\bfalse\b/i, "?")
|
|
201
305
|
|
|
@@ -224,39 +328,73 @@ module Prosopite
|
|
|
224
328
|
end
|
|
225
329
|
|
|
226
330
|
def send_notifications
|
|
227
|
-
|
|
228
|
-
@rails_logger ||= false
|
|
229
|
-
@stderr_logger ||= false
|
|
230
|
-
@prosopite_logger ||= false
|
|
231
|
-
|
|
232
|
-
notifications_str = String.new('')
|
|
331
|
+
notifications = build_notifications
|
|
233
332
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
queries.each { |q| notifications_str << " #{q}\n" }
|
|
333
|
+
notifications.each do |notification|
|
|
334
|
+
ActiveSupport::Notifications.instrument('n_plus_one_queries.prosopite', notification)
|
|
335
|
+
end
|
|
238
336
|
|
|
239
|
-
|
|
240
|
-
kaller = backtrace_cleaner.clean(kaller)
|
|
241
|
-
kaller.each do |f|
|
|
242
|
-
notifications_str << " #{f}\n"
|
|
243
|
-
end
|
|
337
|
+
notifications_str = notifications.map { |notification| notification_message(notification) }.join
|
|
244
338
|
|
|
245
|
-
|
|
246
|
-
|
|
339
|
+
annotate_github(notifications) if @github_annotations
|
|
340
|
+
write_json_report(notifications) if @json_report
|
|
247
341
|
|
|
248
342
|
@custom_logger.warn(notifications_str) if @custom_logger
|
|
249
343
|
|
|
250
|
-
Rails.logger
|
|
344
|
+
Rails.logger&.warn(red(notifications_str)) if rails_logger?
|
|
251
345
|
$stderr.puts(red(notifications_str)) if @stderr_logger
|
|
346
|
+
log_to_file(notifications_str) if @prosopite_logger
|
|
347
|
+
|
|
348
|
+
raise NPlusOneQueriesError.new(notifications_str) if raise?
|
|
349
|
+
|
|
350
|
+
notifications
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
def notification_message(notification)
|
|
354
|
+
queries = notification[:queries]
|
|
355
|
+
duration_ms = notification[:duration_ms]
|
|
356
|
+
|
|
357
|
+
details = []
|
|
358
|
+
details << "#{queries.count} queries" if @display_query_count
|
|
359
|
+
details << "#{duration_ms.round(1)}ms"
|
|
360
|
+
|
|
361
|
+
message = String.new("N+1 queries detected (#{details.join(', ')}):\n")
|
|
252
362
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
363
|
+
queries_to_display =
|
|
364
|
+
if @display_max_n_queries
|
|
365
|
+
queries.take(@display_max_n_queries)
|
|
366
|
+
else
|
|
367
|
+
queries
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
queries_to_display.each { |q| message << " #{q}\n" }
|
|
371
|
+
|
|
372
|
+
if notification[:fixes].present?
|
|
373
|
+
message << (notification[:fixes].size == 1 ? "Possible fix:\n" : "Possible fixes:\n")
|
|
374
|
+
notification[:fixes].each do |fix|
|
|
375
|
+
message << " #{fix}\n"
|
|
256
376
|
end
|
|
257
377
|
end
|
|
258
378
|
|
|
259
|
-
|
|
379
|
+
if notification[:context]
|
|
380
|
+
message << "Context:\n"
|
|
381
|
+
notification[:context].to_s.each_line do |line|
|
|
382
|
+
message << " #{line.chomp}\n"
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# Last, as extensions like dial read the frames after it
|
|
387
|
+
message << "Call stack:\n"
|
|
388
|
+
notification[:call_stack].each do |f|
|
|
389
|
+
message << " #{f}\n"
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
message << "\n"
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# Unless it's set, the Rails log is where N+1 queries go when no other logger is set
|
|
396
|
+
def rails_logger?
|
|
397
|
+
@rails_logger.nil? ? !(@custom_logger || @stderr_logger || @prosopite_logger) : @rails_logger
|
|
260
398
|
end
|
|
261
399
|
|
|
262
400
|
def red(str)
|
|
@@ -264,7 +402,8 @@ module Prosopite
|
|
|
264
402
|
end
|
|
265
403
|
|
|
266
404
|
def ignore_query?(sql)
|
|
267
|
-
@ignore_queries
|
|
405
|
+
return false if @ignore_queries.blank?
|
|
406
|
+
|
|
268
407
|
@ignore_queries.any? { |q| q === sql }
|
|
269
408
|
end
|
|
270
409
|
|
|
@@ -272,29 +411,273 @@ module Prosopite
|
|
|
272
411
|
@subscribed ||= false
|
|
273
412
|
return if @subscribed
|
|
274
413
|
|
|
275
|
-
ActiveSupport::Notifications.subscribe 'sql.active_record' do |_,
|
|
414
|
+
ActiveSupport::Notifications.subscribe 'sql.active_record' do |_, start, finish, _, data|
|
|
276
415
|
sql, name = data[:sql], data[:name]
|
|
416
|
+
state = scanning_state
|
|
277
417
|
|
|
278
|
-
if
|
|
418
|
+
if state && name != "SCHEMA" && data[:cached].nil? && sql.include?('SELECT') && !ignore_query?(sql)
|
|
279
419
|
query_caller = caller_locations
|
|
420
|
+
connection = data[:connection]
|
|
421
|
+
pool = connection&.pool
|
|
422
|
+
ignore_regexp = @ignore_location_paths_regexp
|
|
423
|
+
ignored_paths = @ignored_paths
|
|
424
|
+
# Queries on different databases (e.g. shards) can't be merged into one,
|
|
425
|
+
# so they are kept under different location keys.
|
|
280
426
|
# Calculate the location key with as few allocations as possible
|
|
281
|
-
location_key = [].tap do |array|
|
|
427
|
+
location_key = [(pool.db_config if pool.respond_to?(:db_config))].tap do |array|
|
|
282
428
|
query_caller.each do |loc|
|
|
283
|
-
|
|
429
|
+
path = loc.path
|
|
430
|
+
next if ignore_regexp && ignored_paths.fetch(path) { ignored_path!(ignored_paths, ignore_regexp, path) }
|
|
431
|
+
|
|
432
|
+
array << path
|
|
284
433
|
array << loc.lineno
|
|
285
434
|
end
|
|
286
435
|
end.hash
|
|
287
436
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
437
|
+
counter = state[:prosopite_query_counter]
|
|
438
|
+
state[:prosopite_query_holder][location_key] << sql
|
|
439
|
+
state[:prosopite_query_duration][location_key] += (finish - start) * 1000
|
|
440
|
+
state[:prosopite_query_adapter][location_key] ||= connection&.adapter_name
|
|
441
|
+
|
|
442
|
+
# Store the call stack before the count reaches 2, since an ActionController::Live
|
|
443
|
+
# action can still run queries while the request thread finishes the scan. Fix
|
|
444
|
+
# suggestions show where records were loaded from the ones of single queries too.
|
|
445
|
+
callers = state[:prosopite_query_caller]
|
|
446
|
+
if counter[location_key] > 0 || callers.size < MAX_SINGLE_CALLERS
|
|
447
|
+
callers[location_key] ||= query_caller
|
|
293
448
|
end
|
|
449
|
+
counter[location_key] += 1
|
|
294
450
|
end
|
|
295
451
|
end
|
|
296
452
|
|
|
297
453
|
@subscribed = true
|
|
298
454
|
end
|
|
455
|
+
|
|
456
|
+
private
|
|
457
|
+
|
|
458
|
+
# pg_query is optional, as its native extension can be hard to install
|
|
459
|
+
def pg_query?
|
|
460
|
+
return @pg_query if defined?(@pg_query)
|
|
461
|
+
|
|
462
|
+
@pg_query =
|
|
463
|
+
begin
|
|
464
|
+
require 'pg_query'
|
|
465
|
+
true
|
|
466
|
+
rescue LoadError
|
|
467
|
+
false
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
# true for log/prosopite.log, or a path, or a logger
|
|
472
|
+
def log_to_file(notifications_str)
|
|
473
|
+
return @prosopite_logger.warn(notifications_str) if @prosopite_logger.respond_to?(:warn)
|
|
474
|
+
|
|
475
|
+
path = @prosopite_logger == true ? File.join(Rails.root, 'log', 'prosopite.log') : @prosopite_logger.to_s
|
|
476
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
477
|
+
File.open(path, 'a') { |file| file.puts(notifications_str) }
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
def begin_pause
|
|
481
|
+
return if @ignore_pauses
|
|
482
|
+
|
|
483
|
+
state = scan_state
|
|
484
|
+
state[:prosopite_pauses] += 1 if state
|
|
485
|
+
state
|
|
486
|
+
end
|
|
487
|
+
|
|
488
|
+
def end_pause(state)
|
|
489
|
+
state[:prosopite_pauses] -= 1 if state
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def start_scan(context)
|
|
493
|
+
subscribe
|
|
494
|
+
|
|
495
|
+
@allow_stack_paths ||= []
|
|
496
|
+
@focus_stack_paths ||= []
|
|
497
|
+
@ignore_pauses ||= false
|
|
498
|
+
@min_n_queries ||= 2
|
|
499
|
+
|
|
500
|
+
# A new state for every scan, so that fibers or threads still holding the
|
|
501
|
+
# previous one can't add to this scan
|
|
502
|
+
state = {
|
|
503
|
+
prosopite_scan: true,
|
|
504
|
+
prosopite_paused: false,
|
|
505
|
+
prosopite_pauses: 0,
|
|
506
|
+
prosopite_query_counter: Hash.new(0),
|
|
507
|
+
prosopite_query_holder: Hash.new { |h, k| h[k] = [] },
|
|
508
|
+
prosopite_query_caller: {},
|
|
509
|
+
prosopite_query_duration: Hash.new(0.0),
|
|
510
|
+
prosopite_query_adapter: {},
|
|
511
|
+
prosopite_context: context,
|
|
512
|
+
thread: Thread.current
|
|
513
|
+
}
|
|
514
|
+
Thread.current[:prosopite_state] = state
|
|
515
|
+
Fiber[:prosopite_state] = state if FIBER_STORAGE
|
|
516
|
+
state
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
def build_notifications
|
|
520
|
+
state = Thread.current[:prosopite_state]
|
|
521
|
+
loads = Fixes::Loads.new(state, backtrace_cleaner) if suggest_fixes?
|
|
522
|
+
tc[:prosopite_notifications].map do |queries, info|
|
|
523
|
+
{
|
|
524
|
+
queries: queries,
|
|
525
|
+
call_stack: backtrace_cleaner.clean(info[:kaller]),
|
|
526
|
+
duration_ms: info[:duration_ms],
|
|
527
|
+
fixes: fixes(queries.first, info[:location_key], loads),
|
|
528
|
+
fingerprint: info[:fingerprint],
|
|
529
|
+
context: state[:prosopite_context]
|
|
530
|
+
}
|
|
531
|
+
end
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
# Computed for the N+1 queries left after the filters, as they take some time
|
|
535
|
+
def fixes(query, location_key, loads)
|
|
536
|
+
return [] unless loads && location_key
|
|
537
|
+
|
|
538
|
+
Fixes.for(query, loop_table: loads.loop_table(location_key)) { |model| loads.at(location_key, model.table_name) }
|
|
539
|
+
rescue StandardError
|
|
540
|
+
# A suggestion must never keep an N+1 from being reported
|
|
541
|
+
[]
|
|
542
|
+
end
|
|
543
|
+
|
|
544
|
+
# The state of the scan of the current fiber: set by the fiber that started it,
|
|
545
|
+
# copied by ActionController::Live to the thread of its action, and passed on
|
|
546
|
+
# to the fibers spawned in the same thread
|
|
547
|
+
def scan_state
|
|
548
|
+
state = Thread.current[:prosopite_state]
|
|
549
|
+
return state if state || !FIBER_STORAGE
|
|
550
|
+
|
|
551
|
+
state = Fiber[:prosopite_state]
|
|
552
|
+
state if state && state[:thread].equal?(Thread.current)
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
# Workflow commands that show the N+1 queries on the code in GitHub Actions, once each
|
|
556
|
+
def annotate_github(notifications)
|
|
557
|
+
@github_annotated ||= Set.new
|
|
558
|
+
|
|
559
|
+
notifications.each do |notification|
|
|
560
|
+
file, line = notification[:call_stack].first.to_s.match(/\A(.+?):(\d+)/)&.captures
|
|
561
|
+
next unless @github_annotated.add?([file, line, notification[:fingerprint]])
|
|
562
|
+
|
|
563
|
+
properties = { file: file, line: line, title: 'N+1 queries detected' }.compact
|
|
564
|
+
properties = properties.map { |name, value| "#{name}=#{escape_github(value).gsub(':', '%3A').gsub(',', '%2C')}" }
|
|
565
|
+
$stdout.puts("::warning #{properties.join(',')}::#{escape_github(notification_message(notification).chomp)}")
|
|
566
|
+
end
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
def escape_github(value)
|
|
570
|
+
value.gsub('%', '%25').gsub("\r", '%0D').gsub("\n", '%0A')
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
# A line of JSON for each N+1, under a lock, as parallel test processes write together
|
|
574
|
+
def write_json_report(notifications)
|
|
575
|
+
File.open(@json_report, 'a') do |file|
|
|
576
|
+
file.flock(File::LOCK_EX)
|
|
577
|
+
notifications.each { |notification| file.puts(notification.to_json) }
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
# Filters for the N+1 queries to report, which assertions don't use
|
|
582
|
+
def filter_notifications
|
|
583
|
+
if @min_duration_ms
|
|
584
|
+
tc[:prosopite_notifications].reject! { |_, info| info[:duration_ms].to_f < @min_duration_ms }
|
|
585
|
+
end
|
|
586
|
+
apply_todo if @todo_file
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
# Ignores the N+1 queries in the todo file, or records them to it
|
|
590
|
+
def apply_todo
|
|
591
|
+
tc[:prosopite_notifications].reject! do |queries, info|
|
|
592
|
+
entry = { 'fingerprint' => info[:fingerprint], 'location' => todo_location(info[:kaller]) }
|
|
593
|
+
|
|
594
|
+
if @record_todo
|
|
595
|
+
# Once per process, as each entry rewrites the file
|
|
596
|
+
add_todo_entry(entry.merge('query' => queries.first)) if @todo_recorded.add?(entry.values)
|
|
597
|
+
true
|
|
598
|
+
else
|
|
599
|
+
todo_entries.include?([entry['fingerprint'], entry['location']])
|
|
600
|
+
end
|
|
601
|
+
end
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
# The file of the N+1, without its line, so that unrelated changes don't affect it
|
|
605
|
+
def todo_location(kaller)
|
|
606
|
+
backtrace_cleaner.clean(kaller).first&.sub(/:\d+.*\z/, '')
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
def todo_entries
|
|
610
|
+
@todo_entries ||= read_todo_entries(@todo_file).map { |entry| [entry['fingerprint'], entry['location']] }.to_set
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
def read_todo_entries(path)
|
|
614
|
+
File.exist?(path) ? YAML.safe_load(File.read(path)) || [] : []
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
# Merges the entry into the file, under a lock, as parallel test processes record together
|
|
618
|
+
def add_todo_entry(entry)
|
|
619
|
+
File.open(@todo_file, File::RDWR | File::CREAT, 0o644) do |file|
|
|
620
|
+
file.flock(File::LOCK_EX)
|
|
621
|
+
entries = YAML.safe_load(file.read) || []
|
|
622
|
+
next if entries.any? { |e| e.values_at('fingerprint', 'location') == entry.values_at('fingerprint', 'location') }
|
|
623
|
+
|
|
624
|
+
entries << entry
|
|
625
|
+
entries.sort_by! { |e| [e['location'].to_s, e['fingerprint'].to_s] }
|
|
626
|
+
file.rewind
|
|
627
|
+
file.write(TODO_HEADER + entries.to_yaml)
|
|
628
|
+
file.truncate(file.pos)
|
|
629
|
+
end
|
|
630
|
+
end
|
|
631
|
+
|
|
632
|
+
def scanning_state
|
|
633
|
+
state = scan_state
|
|
634
|
+
state if state && scanning?(state)
|
|
635
|
+
end
|
|
636
|
+
|
|
637
|
+
def scanning?(state)
|
|
638
|
+
state[:prosopite_scan] && !state[:prosopite_paused] && state[:prosopite_pauses].zero?
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def ignored_path!(ignored_paths, ignore_regexp, path)
|
|
642
|
+
# Reloading in development makes new path strings
|
|
643
|
+
ignored_paths.clear if ignored_paths.size >= 10_000
|
|
644
|
+
ignored_paths[path] = ignore_regexp.match?(path)
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
# Strings are patterns too, like for String#match?
|
|
648
|
+
def patterns_regexp(patterns)
|
|
649
|
+
Regexp.union(patterns.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) })
|
|
650
|
+
end
|
|
651
|
+
|
|
652
|
+
# Ends the scan of the state without reporting it
|
|
653
|
+
def discard(state)
|
|
654
|
+
state[:prosopite_scan] = false
|
|
655
|
+
if Thread.current[:prosopite_state].equal?(state)
|
|
656
|
+
Thread.current[:prosopite_state] = nil
|
|
657
|
+
Fiber[:prosopite_state] = nil if FIBER_STORAGE
|
|
658
|
+
end
|
|
659
|
+
release(state)
|
|
660
|
+
end
|
|
661
|
+
|
|
662
|
+
# Extensions may override fingerprint(query) without the adapter, as before 2.3.0
|
|
663
|
+
def fingerprint_query(query, db_adapter)
|
|
664
|
+
if db_adapter && method(:fingerprint).arity != 1
|
|
665
|
+
fingerprint(query, db_adapter)
|
|
666
|
+
else
|
|
667
|
+
fingerprint(query)
|
|
668
|
+
end
|
|
669
|
+
end
|
|
670
|
+
|
|
671
|
+
# Fibers and threads that inherited the state, like threads created during
|
|
672
|
+
# the scan, can keep it for their lifetime, so empty it
|
|
673
|
+
def release(state)
|
|
674
|
+
state[:thread] = nil
|
|
675
|
+
state[:prosopite_context] = nil
|
|
676
|
+
state[:prosopite_query_counter].clear
|
|
677
|
+
state[:prosopite_query_holder].clear
|
|
678
|
+
state[:prosopite_query_caller].clear
|
|
679
|
+
state[:prosopite_query_duration].clear
|
|
680
|
+
state[:prosopite_query_adapter].clear
|
|
681
|
+
end
|
|
299
682
|
end
|
|
300
683
|
end
|