prosopite 1.0.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/Gemfile.lock +52 -2
- data/README.md +89 -7
- data/lib/prosopite/version.rb +1 -1
- data/lib/prosopite.rb +86 -14
- data/prosopite.gemspec +2 -0
- metadata +31 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71358661958f67bb09075fa89db0f0c09868c0a53780afa3780a8cc7d4ded49a
|
4
|
+
data.tar.gz: 1de6a0ab099524457cf49c60d1f5ae0041dffc1b9e281202ccea77cb2cd89ba9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8478248b48b55a59eedfde1c472382eccc543a0f2ce03a891174c326d03a17db19cd3c9a393a2f1a89568a2c275e04ae8ca71d13f30cf67fef64870ed5531494
|
7
|
+
data.tar.gz: de26d196d0a767511b7a9e1f6cf5c87b00c838e17dc2e630f3689a0d43b7a47542c5c0e28f70f3b49a9dca7b577349c6dc66c97b14535cd9e010f83be9f2c58c
|
data/.github/workflows/ci.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,11 +1,24 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
prosopite (
|
4
|
+
prosopite (1.3.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
8
8
|
specs:
|
9
|
+
actionpack (6.1.3)
|
10
|
+
actionview (= 6.1.3)
|
11
|
+
activesupport (= 6.1.3)
|
12
|
+
rack (~> 2.0, >= 2.0.9)
|
13
|
+
rack-test (>= 0.6.3)
|
14
|
+
rails-dom-testing (~> 2.0)
|
15
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
16
|
+
actionview (6.1.3)
|
17
|
+
activesupport (= 6.1.3)
|
18
|
+
builder (~> 3.1)
|
19
|
+
erubi (~> 1.4)
|
20
|
+
rails-dom-testing (~> 2.0)
|
21
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
9
22
|
activemodel (6.1.3)
|
10
23
|
activesupport (= 6.1.3)
|
11
24
|
activerecord (6.1.3)
|
@@ -17,20 +30,55 @@ GEM
|
|
17
30
|
minitest (>= 5.1)
|
18
31
|
tzinfo (~> 2.0)
|
19
32
|
zeitwerk (~> 2.3)
|
33
|
+
ansi (1.5.0)
|
34
|
+
builder (3.2.4)
|
20
35
|
coderay (1.1.3)
|
21
36
|
concurrent-ruby (1.1.8)
|
37
|
+
crass (1.0.6)
|
38
|
+
erubi (1.12.0)
|
22
39
|
factory_bot (6.1.0)
|
23
40
|
activesupport (>= 5.0.0)
|
24
41
|
i18n (1.8.9)
|
25
42
|
concurrent-ruby (~> 1.0)
|
43
|
+
loofah (2.19.1)
|
44
|
+
crass (~> 1.0.2)
|
45
|
+
nokogiri (>= 1.5.9)
|
26
46
|
method_source (1.0.0)
|
47
|
+
mini_portile2 (2.8.1)
|
27
48
|
minitest (5.14.3)
|
49
|
+
minitest-reporters (1.5.0)
|
50
|
+
ansi
|
51
|
+
builder
|
52
|
+
minitest (>= 5.0)
|
53
|
+
ruby-progressbar
|
54
|
+
nokogiri (1.14.1)
|
55
|
+
mini_portile2 (~> 2.8.0)
|
56
|
+
racc (~> 1.4)
|
57
|
+
nokogiri (1.14.1-x86_64-linux)
|
58
|
+
racc (~> 1.4)
|
28
59
|
pg_query (1.3.0)
|
29
60
|
pry (0.14.0)
|
30
61
|
coderay (~> 1.1)
|
31
62
|
method_source (~> 1.0)
|
32
|
-
|
63
|
+
racc (1.6.2)
|
64
|
+
rack (2.2.6.2)
|
65
|
+
rack-test (2.0.2)
|
66
|
+
rack (>= 1.3)
|
67
|
+
rails-dom-testing (2.0.3)
|
68
|
+
activesupport (>= 4.2.0)
|
69
|
+
nokogiri (>= 1.6)
|
70
|
+
rails-html-sanitizer (1.5.0)
|
71
|
+
loofah (~> 2.19, >= 2.19.1)
|
72
|
+
railties (6.1.3)
|
73
|
+
actionpack (= 6.1.3)
|
74
|
+
activesupport (= 6.1.3)
|
75
|
+
method_source
|
76
|
+
rake (>= 0.8.7)
|
77
|
+
thor (~> 1.0)
|
78
|
+
rake (13.0.6)
|
79
|
+
ruby-progressbar (1.11.0)
|
33
80
|
sqlite3 (1.4.2)
|
81
|
+
thor (1.2.1)
|
34
82
|
tzinfo (2.0.4)
|
35
83
|
concurrent-ruby (~> 1.0)
|
36
84
|
zeitwerk (2.4.2)
|
@@ -43,9 +91,11 @@ DEPENDENCIES
|
|
43
91
|
activerecord
|
44
92
|
factory_bot
|
45
93
|
minitest
|
94
|
+
minitest-reporters
|
46
95
|
pg_query
|
47
96
|
prosopite!
|
48
97
|
pry
|
98
|
+
railties
|
49
99
|
rake (~> 13.0)
|
50
100
|
sqlite3
|
51
101
|
|
data/README.md
CHANGED
@@ -115,10 +115,36 @@ Or install it yourself as:
|
|
115
115
|
|
116
116
|
The preferred type of notifications can be configured with:
|
117
117
|
|
118
|
+
* `Prosopite.min_n_queries`: Minimum number of N queries to report per N+1 case. Defaults to 2.
|
119
|
+
* `Prosopite.raise = true`: Raise warnings as exceptions
|
118
120
|
* `Prosopite.rails_logger = true`: Send warnings to the Rails log
|
119
121
|
* `Prosopite.prosopite_logger = true`: Send warnings to `log/prosopite.log`
|
120
122
|
* `Prosopite.stderr_logger = true`: Send warnings to STDERR
|
121
|
-
* `Prosopite.
|
123
|
+
* `Prosopite.custom_logger = my_custom_logger`:
|
124
|
+
* `Prosopite.backtrace_cleaner = my_custom_backtrace_cleaner`: use a different [ActiveSupport::BacktraceCleaner](https://api.rubyonrails.org/classes/ActiveSupport/BacktraceCleaner.html). Default to `Rails.backtrace_cleaner` if present.
|
125
|
+
|
126
|
+
### Custom Logging Configuration
|
127
|
+
|
128
|
+
You can supply a custom logger with the `Prosopite.custom_logger` setting.
|
129
|
+
|
130
|
+
This is useful for circumstances where you don't want your logs to be
|
131
|
+
highlighted with red, or you want logs sent to a custom location.
|
132
|
+
|
133
|
+
One common scenario is that you may be generating json logs and sending them to
|
134
|
+
Datadog, ELK stack, or similar, and don't want to have to remove the default red
|
135
|
+
escaping data from messages sent to the Rails logger, or want to tag them
|
136
|
+
differently with your own custom logger.
|
137
|
+
|
138
|
+
```ruby
|
139
|
+
# Turns off logging with red highlights, but still sends them to the Rails logger
|
140
|
+
Prosopite.custom_logger = Rails.logger
|
141
|
+
```
|
142
|
+
|
143
|
+
```ruby
|
144
|
+
# Use a completely custom logging instance
|
145
|
+
Prosopite.custom_logger = MyLoggerClass.new
|
146
|
+
|
147
|
+
```
|
122
148
|
|
123
149
|
## Development Environment Usage
|
124
150
|
|
@@ -127,11 +153,12 @@ Prosopite auto-detection can be enabled on all controllers:
|
|
127
153
|
```ruby
|
128
154
|
class ApplicationController < ActionController::Base
|
129
155
|
unless Rails.env.production?
|
130
|
-
|
131
|
-
Prosopite.scan
|
132
|
-
end
|
156
|
+
around_action :n_plus_one_detection
|
133
157
|
|
134
|
-
|
158
|
+
def n_plus_one_detection
|
159
|
+
Prosopite.scan
|
160
|
+
yield
|
161
|
+
ensure
|
135
162
|
Prosopite.finish
|
136
163
|
end
|
137
164
|
end
|
@@ -178,10 +205,16 @@ WARNING: scan/finish should run before/after **each** test and NOT before/after
|
|
178
205
|
|
179
206
|
## Allow list
|
180
207
|
|
181
|
-
Ignore notifications for call stacks containing one or more substrings:
|
208
|
+
Ignore notifications for call stacks containing one or more substrings / regex:
|
209
|
+
|
210
|
+
```ruby
|
211
|
+
Prosopite.allow_stack_paths = ['substring_in_call_stack', /regex/]
|
212
|
+
```
|
213
|
+
|
214
|
+
Ignore notifications matching a specific SQL query:
|
182
215
|
|
183
216
|
```ruby
|
184
|
-
Prosopite.
|
217
|
+
Prosopite.ignore_queries = [/regex_match/, "SELECT * from EXACT_STRING_MATCH"]
|
185
218
|
```
|
186
219
|
|
187
220
|
## Scanning code outside controllers or tests
|
@@ -194,6 +227,55 @@ Prosopite.scan
|
|
194
227
|
Prosopite.finish
|
195
228
|
```
|
196
229
|
|
230
|
+
In block form the `Prosopite.finish` is called automatically for you at the end of the block:
|
231
|
+
|
232
|
+
```ruby
|
233
|
+
Prosopite.scan do
|
234
|
+
<code to scan>
|
235
|
+
end
|
236
|
+
```
|
237
|
+
|
238
|
+
The result of the code block is also returned by `Prosopite.scan`, so you can wrap calls as follows:
|
239
|
+
|
240
|
+
```ruby
|
241
|
+
my_object = Prosopite.scan do
|
242
|
+
MyObjectFactory.create(params)
|
243
|
+
end
|
244
|
+
```
|
245
|
+
|
246
|
+
## Pausing and resuming scans
|
247
|
+
|
248
|
+
Scans can be paused:
|
249
|
+
|
250
|
+
```ruby
|
251
|
+
Prosopite.scan
|
252
|
+
# <code to scan>
|
253
|
+
Prosopite.pause
|
254
|
+
# <code that has n+1s>
|
255
|
+
Prosopite.resume
|
256
|
+
# <code to scan>
|
257
|
+
Prosopite.finish
|
258
|
+
```
|
259
|
+
|
260
|
+
You can also pause items in a block, and the `Prosopite.resume` will be done
|
261
|
+
for you automatically:
|
262
|
+
|
263
|
+
```ruby
|
264
|
+
Prosopite.scan
|
265
|
+
# <code to scan>
|
266
|
+
|
267
|
+
result = Prosopite.pause do
|
268
|
+
# <code that has n+1s>
|
269
|
+
end
|
270
|
+
|
271
|
+
Prosopite.finish
|
272
|
+
```
|
273
|
+
|
274
|
+
Pauses can be ignored with `Prosopite.ignore_pauses = true` in case you want to remember their N+1 queries.
|
275
|
+
|
276
|
+
An example of when you might use this is if you are [testing Active Jobs inline](https://guides.rubyonrails.org/testing.html#testing-jobs),
|
277
|
+
and don't want to run Prosopite on background job code, just foreground app code. In that case you could write an [Active Job callback](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks) that pauses the scan while the job is running.
|
278
|
+
|
197
279
|
## Contributing
|
198
280
|
|
199
281
|
Bug reports and pull requests are welcome on GitHub at https://github.com/charkost/prosopite.
|
data/lib/prosopite/version.rb
CHANGED
data/lib/prosopite.rb
CHANGED
@@ -1,12 +1,29 @@
|
|
1
1
|
|
2
2
|
module Prosopite
|
3
|
+
DEFAULT_ALLOW_LIST = %w(active_record/associations/preloader active_record/validations/uniqueness)
|
4
|
+
|
3
5
|
class NPlusOneQueriesError < StandardError; end
|
4
6
|
class << self
|
5
7
|
attr_writer :raise,
|
6
8
|
:stderr_logger,
|
7
9
|
:rails_logger,
|
8
10
|
:prosopite_logger,
|
9
|
-
:
|
11
|
+
:custom_logger,
|
12
|
+
:allow_stack_paths,
|
13
|
+
:ignore_queries,
|
14
|
+
:ignore_pauses,
|
15
|
+
:min_n_queries,
|
16
|
+
:backtrace_cleaner
|
17
|
+
|
18
|
+
def allow_list=(value)
|
19
|
+
puts "Prosopite.allow_list= is deprecated. Use Prosopite.allow_stack_paths= instead."
|
20
|
+
|
21
|
+
self.allow_stack_paths = value
|
22
|
+
end
|
23
|
+
|
24
|
+
def backtrace_cleaner
|
25
|
+
@backtrace_cleaner ||= Rails.backtrace_cleaner
|
26
|
+
end
|
10
27
|
|
11
28
|
def scan
|
12
29
|
tc[:prosopite_scan] ||= false
|
@@ -18,17 +35,52 @@ module Prosopite
|
|
18
35
|
tc[:prosopite_query_holder] = Hash.new { |h, k| h[k] = [] }
|
19
36
|
tc[:prosopite_query_caller] = {}
|
20
37
|
|
21
|
-
@
|
38
|
+
@allow_stack_paths ||= []
|
39
|
+
@ignore_pauses ||= false
|
40
|
+
@min_n_queries ||= 2
|
22
41
|
|
23
42
|
tc[:prosopite_scan] = true
|
43
|
+
|
44
|
+
if block_given?
|
45
|
+
begin
|
46
|
+
block_result = yield
|
47
|
+
finish
|
48
|
+
block_result
|
49
|
+
ensure
|
50
|
+
tc[:prosopite_scan] = false
|
51
|
+
end
|
52
|
+
end
|
24
53
|
end
|
25
54
|
|
26
55
|
def tc
|
27
56
|
Thread.current
|
28
57
|
end
|
29
58
|
|
59
|
+
def pause
|
60
|
+
if @ignore_pauses
|
61
|
+
return block_given? ? yield : nil
|
62
|
+
end
|
63
|
+
|
64
|
+
if block_given?
|
65
|
+
begin
|
66
|
+
previous = tc[:prosopite_scan]
|
67
|
+
tc[:prosopite_scan] = false
|
68
|
+
yield
|
69
|
+
ensure
|
70
|
+
tc[:prosopite_scan] = previous
|
71
|
+
end
|
72
|
+
else
|
73
|
+
tc[:prosopite_scan] = false
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def resume
|
78
|
+
tc[:prosopite_scan] = true
|
79
|
+
end
|
80
|
+
|
30
81
|
def scan?
|
31
|
-
tc[:prosopite_scan]
|
82
|
+
!!(tc[:prosopite_scan] && tc[:prosopite_query_counter] &&
|
83
|
+
tc[:prosopite_query_holder] && tc[:prosopite_query_caller])
|
32
84
|
end
|
33
85
|
|
34
86
|
def finish
|
@@ -38,14 +90,18 @@ module Prosopite
|
|
38
90
|
|
39
91
|
create_notifications
|
40
92
|
send_notifications if tc[:prosopite_notifications].present?
|
93
|
+
|
94
|
+
tc[:prosopite_query_counter] = nil
|
95
|
+
tc[:prosopite_query_holder] = nil
|
96
|
+
tc[:prosopite_query_caller] = nil
|
41
97
|
end
|
42
98
|
|
43
99
|
def create_notifications
|
44
100
|
tc[:prosopite_notifications] = {}
|
45
101
|
|
46
102
|
tc[:prosopite_query_counter].each do |location_key, count|
|
47
|
-
if count
|
48
|
-
fingerprints = tc[:prosopite_query_holder][location_key].
|
103
|
+
if count >= @min_n_queries
|
104
|
+
fingerprints = tc[:prosopite_query_holder][location_key].group_by do |q|
|
49
105
|
begin
|
50
106
|
fingerprint(q)
|
51
107
|
rescue
|
@@ -53,13 +109,17 @@ module Prosopite
|
|
53
109
|
end
|
54
110
|
end
|
55
111
|
|
56
|
-
|
112
|
+
queries = fingerprints.values.select { |q| q.size >= @min_n_queries }
|
57
113
|
|
58
|
-
|
59
|
-
queries = tc[:prosopite_query_holder][location_key]
|
114
|
+
next unless queries.any?
|
60
115
|
|
61
|
-
|
62
|
-
|
116
|
+
kaller = tc[:prosopite_query_caller][location_key]
|
117
|
+
allow_list = (@allow_stack_paths + DEFAULT_ALLOW_LIST)
|
118
|
+
is_allowed = kaller.any? { |f| allow_list.any? { |s| f.match?(s) } }
|
119
|
+
|
120
|
+
unless is_allowed
|
121
|
+
queries.each do |q|
|
122
|
+
tc[:prosopite_notifications][q] = kaller
|
63
123
|
end
|
64
124
|
end
|
65
125
|
end
|
@@ -105,7 +165,7 @@ module Prosopite
|
|
105
165
|
|
106
166
|
query.gsub!(/\btrue\b|\bfalse\b/i, "?")
|
107
167
|
|
108
|
-
query.gsub!(/[0-9+-][0-9a-f.
|
168
|
+
query.gsub!(/[0-9+-][0-9a-f.x+-]*/, "?")
|
109
169
|
query.gsub!(/[xb.+-]\?/, "?")
|
110
170
|
|
111
171
|
query.strip!
|
@@ -128,6 +188,7 @@ module Prosopite
|
|
128
188
|
end
|
129
189
|
|
130
190
|
def send_notifications
|
191
|
+
@custom_logger ||= false
|
131
192
|
@rails_logger ||= false
|
132
193
|
@stderr_logger ||= false
|
133
194
|
@prosopite_logger ||= false
|
@@ -137,14 +198,20 @@ module Prosopite
|
|
137
198
|
|
138
199
|
tc[:prosopite_notifications].each do |queries, kaller|
|
139
200
|
notifications_str << "N+1 queries detected:\n"
|
201
|
+
|
140
202
|
queries.each { |q| notifications_str << " #{q}\n" }
|
203
|
+
|
141
204
|
notifications_str << "Call stack:\n"
|
205
|
+
kaller = backtrace_cleaner.clean(kaller)
|
142
206
|
kaller.each do |f|
|
143
|
-
notifications_str << " #{f}\n"
|
207
|
+
notifications_str << " #{f}\n"
|
144
208
|
end
|
209
|
+
|
145
210
|
notifications_str << "\n"
|
146
211
|
end
|
147
212
|
|
213
|
+
@custom_logger.warn(notifications_str) if @custom_logger
|
214
|
+
|
148
215
|
Rails.logger.warn(red(notifications_str)) if @rails_logger
|
149
216
|
$stderr.puts(red(notifications_str)) if @stderr_logger
|
150
217
|
|
@@ -161,14 +228,19 @@ module Prosopite
|
|
161
228
|
str.split("\n").map { |line| "\e[91m#{line}\e[0m" }.join("\n")
|
162
229
|
end
|
163
230
|
|
231
|
+
def ignore_query?(sql)
|
232
|
+
@ignore_queries ||= []
|
233
|
+
@ignore_queries.any? { |q| q === sql }
|
234
|
+
end
|
235
|
+
|
164
236
|
def subscribe
|
165
237
|
@subscribed ||= false
|
166
238
|
return if @subscribed
|
167
239
|
|
168
240
|
ActiveSupport::Notifications.subscribe 'sql.active_record' do |_, _, _, _, data|
|
169
|
-
sql = data[:sql]
|
241
|
+
sql, name = data[:sql], data[:name]
|
170
242
|
|
171
|
-
if scan? && sql.include?('SELECT') && data[:cached].nil?
|
243
|
+
if scan? && name != "SCHEMA" && sql.include?('SELECT') && data[:cached].nil? && !ignore_query?(sql)
|
172
244
|
location_key = Digest::SHA1.hexdigest(caller.join)
|
173
245
|
|
174
246
|
tc[:prosopite_query_counter][location_key] += 1
|
data/prosopite.gemspec
CHANGED
@@ -28,5 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.add_development_dependency "minitest"
|
29
29
|
spec.add_development_dependency "factory_bot"
|
30
30
|
spec.add_development_dependency "activerecord"
|
31
|
+
spec.add_development_dependency "railties"
|
31
32
|
spec.add_development_dependency "sqlite3"
|
33
|
+
spec.add_development_dependency "minitest-reporters"
|
32
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prosopite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mpampis Kostas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-02-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: railties
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: sqlite3
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +94,20 @@ dependencies:
|
|
80
94
|
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: minitest-reporters
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
83
111
|
description: N+1 auto-detection for Rails with zero false positives / false negatives
|
84
112
|
email:
|
85
113
|
- charkost.rb@gmail.com
|
@@ -118,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
146
|
- !ruby/object:Gem::Version
|
119
147
|
version: '0'
|
120
148
|
requirements: []
|
121
|
-
|
122
|
-
rubygems_version: 2.7.6.2
|
149
|
+
rubygems_version: 3.1.6
|
123
150
|
signing_key:
|
124
151
|
specification_version: 4
|
125
152
|
summary: N+1 auto-detection for Rails with zero false positives / false negatives
|