rails-profiler 0.31.3 → 0.31.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +129 -0
- data/app/assets/builds/profiler.js +9 -1
- data/app/controllers/profiler/api/console_controller.rb +1 -11
- data/app/controllers/profiler/api/jobs_controller.rb +1 -11
- data/app/controllers/profiler/api/profiles_controller.rb +18 -28
- data/app/controllers/profiler/api/tests_controller.rb +1 -11
- data/app/controllers/profiler/api/toolbar_controller.rb +0 -20
- data/app/controllers/profiler/application_controller.rb +55 -15
- data/app/controllers/profiler/profiles_controller.rb +2 -18
- data/lib/profiler/ajax_data.rb +40 -0
- data/lib/profiler/cluster/master_client.rb +7 -15
- data/lib/profiler/cluster/security.rb +3 -3
- data/lib/profiler/cluster/slave_proxy.rb +14 -9
- data/lib/profiler/collectors/lifecycle.rb +2 -2
- data/lib/profiler/collectors/log_collector.rb +54 -5
- data/lib/profiler/collectors/mailer_collector.rb +8 -4
- data/lib/profiler/collectors/request_collector.rb +6 -1
- data/lib/profiler/collectors/scoped_notifications.rb +1 -1
- data/lib/profiler/configuration.rb +30 -8
- data/lib/profiler/console_profiler.rb +1 -1
- data/lib/profiler/engine.rb +2 -1
- data/lib/profiler/env_override_store.rb +4 -4
- data/lib/profiler/explain_runner.rb +5 -1
- data/lib/profiler/instrumentation/net_http_instrumentation.rb +138 -40
- data/lib/profiler/job_profiler.rb +1 -1
- data/lib/profiler/local_request.rb +2 -6
- data/lib/profiler/mcp/file_cache.rb +2 -1
- data/lib/profiler/mcp/resources/failing_tests.rb +1 -1
- data/lib/profiler/mcp/resources/recent_console.rb +2 -2
- data/lib/profiler/mcp/resources/recent_jobs.rb +2 -2
- data/lib/profiler/mcp/resources/slow_tests.rb +1 -1
- data/lib/profiler/mcp/server.rb +9 -3
- data/lib/profiler/mcp/tools/get_profile_ajax.rb +2 -0
- data/lib/profiler/mcp/tools/get_profile_http.rb +6 -1
- data/lib/profiler/mcp/tools/get_test_profile_detail.rb +1 -1
- data/lib/profiler/mcp/tools/query_console_profiles.rb +6 -2
- data/lib/profiler/mcp/tools/query_jobs.rb +6 -2
- data/lib/profiler/mcp/tools/query_mailers.rb +5 -1
- data/lib/profiler/mcp/tools/query_profiles.rb +6 -2
- data/lib/profiler/mcp/tools/query_test_profiles.rb +6 -2
- data/lib/profiler/mcp/tools/run_tests.rb +5 -2
- data/lib/profiler/middleware/profiler_middleware.rb +22 -16
- data/lib/profiler/middleware/streamed_profile.rb +1 -1
- data/lib/profiler/middleware/toolbar_injector.rb +21 -2
- data/lib/profiler/models/profile.rb +60 -2
- data/lib/profiler/process_snapshot.rb +8 -3
- data/lib/profiler/railtie.rb +4 -1
- data/lib/profiler/redaction.rb +1 -3
- data/lib/profiler/sse/redis_event_bus.rb +2 -1
- data/lib/profiler/storage/base_store.rb +4 -2
- data/lib/profiler/storage/blob_store.rb +1 -1
- data/lib/profiler/storage/file_store.rb +337 -58
- data/lib/profiler/storage/memory_store.rb +60 -31
- data/lib/profiler/storage/private_files.rb +16 -3
- data/lib/profiler/storage/redis_store.rb +252 -33
- data/lib/profiler/storage/sqlite_store.rb +59 -11
- data/lib/profiler/storage/summary.rb +50 -0
- data/lib/profiler/storage/unavailable.rb +57 -0
- data/lib/profiler/test_helpers/reporter.rb +2 -2
- data/lib/profiler/test_profiler.rb +1 -1
- data/lib/profiler/test_runner/runner.rb +3 -7
- data/lib/profiler/version.rb +1 -1
- data/lib/profiler.rb +138 -4
- metadata +4 -1
|
@@ -2,13 +2,38 @@
|
|
|
2
2
|
|
|
3
3
|
require "redis"
|
|
4
4
|
require "json"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
require "time"
|
|
5
7
|
require_relative "base_store"
|
|
8
|
+
require_relative "summary"
|
|
6
9
|
require_relative "../models/profile"
|
|
7
10
|
|
|
8
11
|
module Profiler
|
|
9
12
|
module Storage
|
|
13
|
+
# Profiles as JSON strings with a TTL, listed by a sorted set scored by start time. Beside
|
|
14
|
+
# them: a sorted set per profile type, a set of children per parent and the summary each list
|
|
15
|
+
# shows, so that listing one type, a page of summaries or the children of a page reads only
|
|
16
|
+
# what it returns; and at most max_profiles profiles, the oldest evicted first. Data written by
|
|
17
|
+
# a version without these indexes is indexed once, on first use.
|
|
10
18
|
class RedisStore < BaseStore
|
|
11
19
|
DEFAULT_TTL = 24 * 60 * 60 # 24 hours
|
|
20
|
+
INDEX_VERSION = "1"
|
|
21
|
+
# Past max_profiles, the first saved profiles are evicted down to this share of it.
|
|
22
|
+
LOW_WATER = 0.8
|
|
23
|
+
# Indexing the data of an earlier version: tokens read per batch, and how long one process
|
|
24
|
+
# holds the right to do it.
|
|
25
|
+
INDEX_BATCH = 500
|
|
26
|
+
INDEX_LOCK_SECONDS = 60
|
|
27
|
+
# Past the cap, a save removes at most this many profiles itself; a larger backlog (the data
|
|
28
|
+
# of an earlier version) is removed by one process, under a lock, by batches of INDEX_BATCH.
|
|
29
|
+
EVICT_BATCH = 100
|
|
30
|
+
EVICT_LOCK_SECONDS = 60
|
|
31
|
+
# Deletes a lock only while it still holds the value its owner set: one that expired and was
|
|
32
|
+
# taken by another process is left to it.
|
|
33
|
+
RELEASE_LOCK = <<~LUA
|
|
34
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) end
|
|
35
|
+
return 0
|
|
36
|
+
LUA
|
|
12
37
|
|
|
13
38
|
attr_reader :redis
|
|
14
39
|
|
|
@@ -16,15 +41,17 @@ module Profiler
|
|
|
16
41
|
@redis = options[:redis] || build_redis_client(options)
|
|
17
42
|
@ttl = options[:ttl] || DEFAULT_TTL
|
|
18
43
|
@key_prefix = options[:key_prefix] || "profiler"
|
|
44
|
+
@max_profiles = options.key?(:max_profiles) ? options[:max_profiles] : Profiler.configuration.max_profiles
|
|
45
|
+
@indexed = false
|
|
19
46
|
end
|
|
20
47
|
|
|
21
48
|
def do_save(token, profile)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
49
|
+
ensure_index
|
|
50
|
+
data = profile.to_h
|
|
51
|
+
unindex(token, read_summary(token))
|
|
52
|
+
@redis.setex(profile_key(token), @ttl, data.to_json)
|
|
53
|
+
index(token, profile.started_at.to_f, Summary.build(data))
|
|
54
|
+
evict_first_saved(token)
|
|
28
55
|
token
|
|
29
56
|
end
|
|
30
57
|
|
|
@@ -37,54 +64,51 @@ module Profiler
|
|
|
37
64
|
|
|
38
65
|
Models::Profile.from_json(json_data)
|
|
39
66
|
rescue => e
|
|
40
|
-
|
|
67
|
+
Profiler.log_error("RedisStore: could not load profile #{token}", e)
|
|
41
68
|
nil
|
|
42
69
|
end
|
|
43
70
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
71
|
+
# Newest first. summary: true reads the summaries only.
|
|
72
|
+
def list(limit: 50, offset: 0, type: nil, summary: false)
|
|
73
|
+
ensure_index
|
|
74
|
+
tokens = @redis.zrevrange(type ? type_list_key(type) : list_key, offset, offset + limit - 1)
|
|
75
|
+
return tokens.filter_map { |token| load(token) } unless summary
|
|
76
|
+
|
|
77
|
+
summaries = tokens.empty? ? [] : @redis.mget(*tokens.map { |token| summary_key(token) })
|
|
78
|
+
tokens.zip(summaries).filter_map do |token, json|
|
|
79
|
+
next Summary.to_profile(JSON.parse(json)) if json
|
|
80
|
+
|
|
81
|
+
profile = load(token)
|
|
82
|
+
profile && Summary.to_profile(Summary.build(profile))
|
|
83
|
+
end
|
|
47
84
|
end
|
|
48
85
|
|
|
49
86
|
def cleanup(older_than: 24 * 60 * 60)
|
|
87
|
+
ensure_index
|
|
50
88
|
cutoff_time = Time.now.to_f - older_than
|
|
51
|
-
@redis.
|
|
89
|
+
@redis.zrangebyscore(list_key, "-inf", cutoff_time).each_slice(INDEX_BATCH) { |tokens| remove_tokens(tokens) }
|
|
52
90
|
end
|
|
53
91
|
|
|
54
92
|
def find_by_parent(parent_token)
|
|
55
93
|
return [] unless Token.valid?(parent_token)
|
|
56
94
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
# Load each profile and filter by parent_token
|
|
61
|
-
tokens.map { |token| load(token) }
|
|
62
|
-
.compact
|
|
95
|
+
ensure_index
|
|
96
|
+
@redis.smembers(children_key(parent_token))
|
|
97
|
+
.filter_map { |token| load(token) if Token.valid?(token) }
|
|
63
98
|
.select { |profile| profile.parent_token == parent_token }
|
|
64
|
-
.sort_by
|
|
99
|
+
.sort_by(&:started_at)
|
|
65
100
|
end
|
|
66
101
|
|
|
67
102
|
def delete(token)
|
|
68
103
|
return unless Token.valid?(token)
|
|
69
104
|
|
|
70
|
-
|
|
71
|
-
|
|
105
|
+
ensure_index
|
|
106
|
+
remove(token)
|
|
72
107
|
end
|
|
73
108
|
|
|
74
109
|
def clear(type: nil)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if type.nil?
|
|
78
|
-
@redis.del(profile_key(token))
|
|
79
|
-
@redis.zrem(list_key, token)
|
|
80
|
-
else
|
|
81
|
-
profile = load(token)
|
|
82
|
-
if profile&.profile_type == type.to_s
|
|
83
|
-
@redis.del(profile_key(token))
|
|
84
|
-
@redis.zrem(list_key, token)
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
end
|
|
110
|
+
ensure_index
|
|
111
|
+
@redis.zrange(type ? type_list_key(type) : list_key, 0, -1).each_slice(INDEX_BATCH) { |tokens| remove_tokens(tokens) }
|
|
88
112
|
end
|
|
89
113
|
|
|
90
114
|
private
|
|
@@ -94,6 +118,169 @@ module Profiler
|
|
|
94
118
|
Redis.new(url: url)
|
|
95
119
|
end
|
|
96
120
|
|
|
121
|
+
# score: the start time, the order of the lists. saved_at: the order of the saves, which the
|
|
122
|
+
# eviction follows.
|
|
123
|
+
def index(token, score, summary, saved_at: Time.now.to_f, client: @redis)
|
|
124
|
+
type = (summary[:profile_type] || summary["profile_type"] || "http").to_s
|
|
125
|
+
parent = summary[:parent_token] || summary["parent_token"]
|
|
126
|
+
client.setex(summary_key(token), @ttl, JSON.generate(summary))
|
|
127
|
+
client.zadd(list_key, score, token)
|
|
128
|
+
client.zadd(type_list_key(type), score, token)
|
|
129
|
+
client.zadd(saved_key, saved_at, token)
|
|
130
|
+
client.sadd(types_key, type)
|
|
131
|
+
return unless Token.valid?(parent)
|
|
132
|
+
|
|
133
|
+
client.sadd(children_key(parent), token)
|
|
134
|
+
client.expire(children_key(parent), @ttl)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Takes token out of the type list and the children set its previous save put it in.
|
|
138
|
+
def unindex(token, summary)
|
|
139
|
+
return unless summary
|
|
140
|
+
|
|
141
|
+
@redis.zrem(type_list_key(summary["profile_type"] || "http"), token)
|
|
142
|
+
parent = summary["parent_token"]
|
|
143
|
+
@redis.srem(children_key(parent), token) if Token.valid?(parent)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def remove(token)
|
|
147
|
+
remove_tokens([token])
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Removes tokens and their index entries in three round trips, whatever their number: their
|
|
151
|
+
# summaries (type and parent), the types when a summary is missing, then one pipeline.
|
|
152
|
+
def remove_tokens(tokens)
|
|
153
|
+
tokens = tokens.select { |token| Token.valid?(token) }
|
|
154
|
+
return if tokens.empty?
|
|
155
|
+
|
|
156
|
+
summaries = @redis.mget(*tokens.map { |token| summary_key(token) }).map { |json| parse_summary(json) }
|
|
157
|
+
types = summaries.all? ? [] : @redis.smembers(types_key)
|
|
158
|
+
@redis.pipelined do |pipe|
|
|
159
|
+
tokens.zip(summaries).each do |token, summary|
|
|
160
|
+
if summary
|
|
161
|
+
pipe.zrem(type_list_key(summary["profile_type"] || "http"), token)
|
|
162
|
+
parent = summary["parent_token"]
|
|
163
|
+
pipe.srem(children_key(parent), token) if Token.valid?(parent)
|
|
164
|
+
else
|
|
165
|
+
types.each { |type| pipe.zrem(type_list_key(type), token) }
|
|
166
|
+
end
|
|
167
|
+
pipe.del(profile_key(token), summary_key(token))
|
|
168
|
+
end
|
|
169
|
+
pipe.zrem(list_key, tokens)
|
|
170
|
+
pipe.zrem(saved_key, tokens)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def parse_summary(json)
|
|
175
|
+
json && JSON.parse(json)
|
|
176
|
+
rescue JSON::ParserError
|
|
177
|
+
nil
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def read_summary(token)
|
|
181
|
+
parse_summary(@redis.get(summary_key(token)))
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# In the order of the saves, not of the starts: a job saved when it ends is a new profile,
|
|
185
|
+
# and the one just saved (keep) is never evicted. ZCARD is O(1): the eviction costs nothing
|
|
186
|
+
# until the cap is passed. A backlog larger than EVICT_BATCH is removed by one process, under
|
|
187
|
+
# a lock, by batches; meanwhile the other saves remove EVICT_BATCH profiles each at most.
|
|
188
|
+
def evict_first_saved(keep)
|
|
189
|
+
return unless @max_profiles
|
|
190
|
+
|
|
191
|
+
target = [(@max_profiles * LOW_WATER).floor, 1].max
|
|
192
|
+
excess = @redis.zcard(saved_key) - target
|
|
193
|
+
return if excess <= @max_profiles - target
|
|
194
|
+
|
|
195
|
+
if excess <= EVICT_BATCH
|
|
196
|
+
remove_tokens(@redis.zrange(saved_key, 0, excess - 1) - [keep])
|
|
197
|
+
else
|
|
198
|
+
evicted_all = with_lock(evict_lock_key, EVICT_LOCK_SECONDS) do
|
|
199
|
+
loop do
|
|
200
|
+
excess = @redis.zcard(saved_key) - target
|
|
201
|
+
break if excess <= 0
|
|
202
|
+
|
|
203
|
+
batch = @redis.zrange(saved_key, 0, [excess, INDEX_BATCH].min - 1) - [keep]
|
|
204
|
+
break if batch.empty?
|
|
205
|
+
|
|
206
|
+
remove_tokens(batch)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
remove_tokens(@redis.zrange(saved_key, 0, EVICT_BATCH - 1) - [keep]) unless evicted_all
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Runs the block holding key, set to a value of this call's own; false, without running it,
|
|
214
|
+
# when another process holds it.
|
|
215
|
+
def with_lock(key, seconds)
|
|
216
|
+
owner = SecureRandom.hex(16)
|
|
217
|
+
return false unless @redis.set(key, owner, nx: true, ex: seconds)
|
|
218
|
+
|
|
219
|
+
begin
|
|
220
|
+
yield
|
|
221
|
+
ensure
|
|
222
|
+
@redis.eval(RELEASE_LOCK, keys: [key], argv: [owner])
|
|
223
|
+
end
|
|
224
|
+
true
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Indexes, once, the profiles saved by a version that kept only the list, which it never
|
|
228
|
+
# pruned: the tokens started long before the TTL first, then the others read by batches.
|
|
229
|
+
# One process does it, under a lock; the others go on meanwhile, with the list alone.
|
|
230
|
+
def ensure_index
|
|
231
|
+
return if @indexed
|
|
232
|
+
return @indexed = true if @redis.get(index_version_key) == INDEX_VERSION
|
|
233
|
+
|
|
234
|
+
with_lock(index_lock_key, INDEX_LOCK_SECONDS) do
|
|
235
|
+
# An hour of margin for a profile saved long after it started.
|
|
236
|
+
@redis.zremrangebyscore(list_key, "-inf", Time.now.to_f - @ttl - 3600)
|
|
237
|
+
tokens = @redis.zrange(list_key, 0, -1)
|
|
238
|
+
# Past the cap, the first saved would be evicted right after: removed without indexing.
|
|
239
|
+
if @max_profiles && tokens.size > @max_profiles
|
|
240
|
+
target = [(@max_profiles * LOW_WATER).floor, 1].max
|
|
241
|
+
tokens.first(tokens.size - target).each_slice(INDEX_BATCH) { |batch| drop_unindexed(batch) }
|
|
242
|
+
tokens = tokens.last(target)
|
|
243
|
+
end
|
|
244
|
+
tokens.each_slice(INDEX_BATCH) { |batch| index_batch(batch) }
|
|
245
|
+
@redis.set(index_version_key, INDEX_VERSION)
|
|
246
|
+
@indexed = true
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Profiles of an earlier version, which only the list and their own key know.
|
|
251
|
+
def drop_unindexed(tokens)
|
|
252
|
+
@redis.pipelined do |pipe|
|
|
253
|
+
tokens.each { |token| pipe.del(profile_key(token)) if Token.valid?(token) }
|
|
254
|
+
pipe.zrem(list_key, tokens)
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def index_batch(tokens)
|
|
259
|
+
tokens = tokens.select { |token| Token.valid?(token) }
|
|
260
|
+
return if tokens.empty?
|
|
261
|
+
|
|
262
|
+
expired = []
|
|
263
|
+
profiles = tokens.zip(@redis.mget(*tokens.map { |token| profile_key(token) })).filter_map do |token, json|
|
|
264
|
+
if json.nil?
|
|
265
|
+
expired << token
|
|
266
|
+
next
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# The summary from the parsed JSON, without building the Profile (and inflating its bodies).
|
|
270
|
+
data = JSON.parse(json, symbolize_names: true)
|
|
271
|
+
[token, data[:started_at] ? Time.parse(data[:started_at]).to_f : 0.0, Summary.build(data)]
|
|
272
|
+
rescue StandardError => e
|
|
273
|
+
Profiler.log_error("RedisStore: could not index profile #{token}", e)
|
|
274
|
+
nil
|
|
275
|
+
end
|
|
276
|
+
@redis.pipelined do |pipe|
|
|
277
|
+
profiles.each do |token, started_at, summary|
|
|
278
|
+
index(token, started_at, summary, saved_at: started_at, client: pipe)
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
@redis.zrem(list_key, expired) unless expired.empty?
|
|
282
|
+
end
|
|
283
|
+
|
|
97
284
|
def profile_key(token)
|
|
98
285
|
"#{@key_prefix}:#{token}"
|
|
99
286
|
end
|
|
@@ -101,6 +288,38 @@ module Profiler
|
|
|
101
288
|
def list_key
|
|
102
289
|
"#{@key_prefix}:list"
|
|
103
290
|
end
|
|
291
|
+
|
|
292
|
+
def type_list_key(type)
|
|
293
|
+
"#{@key_prefix}:list:#{type}"
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def types_key
|
|
297
|
+
"#{@key_prefix}:types"
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def summary_key(token)
|
|
301
|
+
"#{@key_prefix}:summary:#{token}"
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def children_key(parent_token)
|
|
305
|
+
"#{@key_prefix}:children:#{parent_token}"
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def index_version_key
|
|
309
|
+
"#{@key_prefix}:index_version"
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def index_lock_key
|
|
313
|
+
"#{@key_prefix}:index_lock"
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def evict_lock_key
|
|
317
|
+
"#{@key_prefix}:evict_lock"
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def saved_key
|
|
321
|
+
"#{@key_prefix}:saved"
|
|
322
|
+
end
|
|
104
323
|
end
|
|
105
324
|
end
|
|
106
325
|
end
|
|
@@ -5,15 +5,21 @@ require "fileutils"
|
|
|
5
5
|
require_relative "base_store"
|
|
6
6
|
require_relative "blob_store"
|
|
7
7
|
require_relative "private_files"
|
|
8
|
+
require_relative "summary"
|
|
8
9
|
require_relative "token"
|
|
9
10
|
require_relative "../models/profile"
|
|
10
11
|
|
|
11
12
|
module Profiler
|
|
12
13
|
module Storage
|
|
13
14
|
class SqliteStore < BaseStore
|
|
15
|
+
# Past max_profiles, the oldest profiles are evicted down to this share of it.
|
|
16
|
+
LOW_WATER = 0.8
|
|
17
|
+
|
|
14
18
|
def initialize(options = {})
|
|
15
19
|
require "sqlite3"
|
|
16
20
|
|
|
21
|
+
@max_profiles = options.key?(:max_profiles) ? options[:max_profiles] : Profiler.configuration.max_profiles
|
|
22
|
+
|
|
17
23
|
db_path = (options[:database] || default_db_path).to_s
|
|
18
24
|
blob_path = options[:blob_path] || PrivateFiles.tmp_dir("blobs")
|
|
19
25
|
|
|
@@ -55,11 +61,11 @@ module Profiler
|
|
|
55
61
|
INSERT OR REPLACE INTO profiler_profiles (
|
|
56
62
|
token, profile_type, gem_version, path, method, status, duration, memory,
|
|
57
63
|
started_at, finished_at, parent_token, is_ajax,
|
|
58
|
-
tabs, params, headers, response_headers, collectors_meta
|
|
64
|
+
tabs, params, headers, response_headers, collectors_meta, summary
|
|
59
65
|
) VALUES (
|
|
60
66
|
:token, :profile_type, :gem_version, :path, :method, :status, :duration, :memory,
|
|
61
67
|
:started_at, :finished_at, :parent_token, :is_ajax,
|
|
62
|
-
:tabs, :params, :headers, :response_headers, :collectors_meta
|
|
68
|
+
:tabs, :params, :headers, :response_headers, :collectors_meta, :summary
|
|
63
69
|
)
|
|
64
70
|
SQL
|
|
65
71
|
token: token,
|
|
@@ -78,8 +84,10 @@ module Profiler
|
|
|
78
84
|
params: JSON.generate(data[:params] || {}),
|
|
79
85
|
headers: JSON.generate(data[:headers] || {}),
|
|
80
86
|
response_headers: JSON.generate(data[:response_headers] || {}),
|
|
81
|
-
collectors_meta: JSON.generate(collectors_meta)
|
|
87
|
+
collectors_meta: JSON.generate(collectors_meta),
|
|
88
|
+
summary: JSON.generate(Summary.build(data))
|
|
82
89
|
)
|
|
90
|
+
evict_oldest
|
|
83
91
|
|
|
84
92
|
token
|
|
85
93
|
end
|
|
@@ -94,16 +102,22 @@ module Profiler
|
|
|
94
102
|
|
|
95
103
|
row_to_profile(row)
|
|
96
104
|
rescue => e
|
|
97
|
-
|
|
105
|
+
Profiler.log_error("SqliteStore: could not load profile #{token}", e)
|
|
98
106
|
nil
|
|
99
107
|
end
|
|
100
108
|
|
|
101
|
-
|
|
109
|
+
# Newest first. summary: true reads the summary column only, for the rows that have it.
|
|
110
|
+
def list(limit: 50, offset: 0, type: nil, summary: false)
|
|
111
|
+
where = type ? "WHERE profile_type = :type" : ""
|
|
112
|
+
# The summaries need neither the params, headers and tabs nor the collector data.
|
|
113
|
+
columns = summary ? "token, summary" : "*"
|
|
102
114
|
rows = @db.execute(
|
|
103
|
-
"SELECT
|
|
104
|
-
limit: limit, offset: offset
|
|
115
|
+
"SELECT #{columns} FROM profiler_profiles #{where} ORDER BY started_at DESC, rowid DESC LIMIT :limit OFFSET :offset",
|
|
116
|
+
{ limit: limit, offset: offset }.merge(type ? { type: type.to_s } : {})
|
|
105
117
|
)
|
|
106
|
-
rows.
|
|
118
|
+
return rows.filter_map { |row| row_to_profile(row, load_blobs: false) } unless summary
|
|
119
|
+
|
|
120
|
+
rows.filter_map { |row| row_to_summary(row) }
|
|
107
121
|
end
|
|
108
122
|
|
|
109
123
|
def find_by_parent(parent_token)
|
|
@@ -147,6 +161,40 @@ module Profiler
|
|
|
147
161
|
|
|
148
162
|
private
|
|
149
163
|
|
|
164
|
+
# In the order the profiles were written (rowid: INSERT OR REPLACE gives a saved again
|
|
165
|
+
# profile a new one), not started: a job saved when it ends is a new profile, and the one
|
|
166
|
+
# just written is never evicted. One query on the rowid tells whether the table holds more
|
|
167
|
+
# than max_profiles.
|
|
168
|
+
def evict_oldest
|
|
169
|
+
return unless @max_profiles
|
|
170
|
+
return unless @db.get_first_value(
|
|
171
|
+
"SELECT 1 FROM profiler_profiles ORDER BY rowid DESC LIMIT 1 OFFSET :max", max: @max_profiles
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
keep = [(@max_profiles * LOW_WATER).floor, 1].max
|
|
175
|
+
tokens = @db.execute(
|
|
176
|
+
"SELECT token FROM profiler_profiles ORDER BY rowid DESC LIMIT -1 OFFSET :keep", keep: keep
|
|
177
|
+
).map { |r| r["token"] }
|
|
178
|
+
tokens.each_slice(500) do |slice|
|
|
179
|
+
@db.execute("DELETE FROM profiler_profiles WHERE token IN (#{(["?"] * slice.size).join(",")})", slice)
|
|
180
|
+
end
|
|
181
|
+
tokens.each { |token| @blob_store.delete(token) }
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# A row written before the summary column falls back to the whole row, summarized.
|
|
185
|
+
def row_to_summary(row)
|
|
186
|
+
if row["summary"].nil? || row["summary"].empty?
|
|
187
|
+
full = @db.get_first_row("SELECT * FROM profiler_profiles WHERE token = :token", token: row["token"])
|
|
188
|
+
profile = full && row_to_profile(full, load_blobs: false)
|
|
189
|
+
return profile && Summary.to_profile(Summary.build(profile))
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
Summary.to_profile(JSON.parse(row["summary"]))
|
|
193
|
+
rescue JSON::ParserError => e
|
|
194
|
+
Profiler.log_error("SqliteStore: could not read the summary of #{row["token"]}", e)
|
|
195
|
+
nil
|
|
196
|
+
end
|
|
197
|
+
|
|
150
198
|
def save_http_response_bodies(token, requests)
|
|
151
199
|
bodies = requests.map do |req|
|
|
152
200
|
{ "response_body" => req["response_body"], "response_body_encoding" => req["response_body_encoding"] }
|
|
@@ -213,8 +261,8 @@ module Profiler
|
|
|
213
261
|
ON profiler_profiles(profile_type);
|
|
214
262
|
SQL
|
|
215
263
|
|
|
216
|
-
|
|
217
|
-
@db.execute("ALTER TABLE profiler_profiles ADD COLUMN
|
|
264
|
+
%w[gem_version summary].each do |column|
|
|
265
|
+
@db.execute("ALTER TABLE profiler_profiles ADD COLUMN #{column} TEXT")
|
|
218
266
|
rescue SQLite3::Exception
|
|
219
267
|
# column already exists
|
|
220
268
|
end
|
|
@@ -260,7 +308,7 @@ module Profiler
|
|
|
260
308
|
collectors_data: collectors_data
|
|
261
309
|
)
|
|
262
310
|
rescue => e
|
|
263
|
-
|
|
311
|
+
Profiler.log_error("SqliteStore: could not deserialize profile #{row["token"]}", e)
|
|
264
312
|
nil
|
|
265
313
|
end
|
|
266
314
|
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../models/profile"
|
|
4
|
+
|
|
5
|
+
module Profiler
|
|
6
|
+
module Storage
|
|
7
|
+
# What the profile lists show of a profile, for the stores to keep apart from the profile
|
|
8
|
+
# itself: the top-level fields without the bodies, params, headers and tabs, and for each
|
|
9
|
+
# collector its scalar values only (database.total_queries, job.queue, test.test_name...),
|
|
10
|
+
# nested lists and hashes left out, strings cut at MAX_SCALAR_LENGTH (a console return value,
|
|
11
|
+
# an exception message). Built from Profile#to_h, after the collectors and the
|
|
12
|
+
# redaction have run: a summary is a subset of the stored profile (a long string cut to its
|
|
13
|
+
# start) and holds no value the profile does not.
|
|
14
|
+
module Summary
|
|
15
|
+
MAX_SCALAR_LENGTH = 500
|
|
16
|
+
|
|
17
|
+
DROPPED = %i[params headers response_headers request_body request_body_encoding
|
|
18
|
+
response_body response_body_encoding tabs].freeze
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
# From a Profile, or the Hash its #to_h returns.
|
|
23
|
+
def build(profile)
|
|
24
|
+
data = profile.is_a?(Hash) ? profile : profile.to_h
|
|
25
|
+
summary = data.transform_keys(&:to_sym).except(*DROPPED)
|
|
26
|
+
summary[:collectors_data] = (summary[:collectors_data] || {}).to_h do |name, values|
|
|
27
|
+
[name.to_s, values.is_a?(Hash) ? scalars(values) : {}]
|
|
28
|
+
end
|
|
29
|
+
summary
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# The Profile a list shows, from a summary with string or symbol keys.
|
|
33
|
+
def to_profile(summary)
|
|
34
|
+
Models::Profile.from_hash(summary.transform_keys(&:to_sym))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def scalars(values)
|
|
38
|
+
values.each_with_object({}) do |(key, value), kept|
|
|
39
|
+
next unless scalar?(value)
|
|
40
|
+
|
|
41
|
+
kept[key.to_s] = value.is_a?(String) && value.length > MAX_SCALAR_LENGTH ? value[0, MAX_SCALAR_LENGTH] : value
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def scalar?(value)
|
|
46
|
+
value.nil? || value.is_a?(String) || value.is_a?(Numeric) || value == true || value == false
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Profiler
|
|
4
|
+
module Storage
|
|
5
|
+
# Stands in for a store that could not be created (an index or lock replaced by a symbolic
|
|
6
|
+
# link, a directory that cannot be written, a database that cannot be opened): the cause is
|
|
7
|
+
# said once per process, without a backtrace, rather than with one on every request. Saves are
|
|
8
|
+
# dropped; reads, which a person asked for from the dashboard or an MCP tool, raise the cause.
|
|
9
|
+
# Profiler.storage tries to create the store again on its next call, so fixing the cause is
|
|
10
|
+
# enough.
|
|
11
|
+
class Unavailable
|
|
12
|
+
# What the reads raise: the cause in one line, without a path (the profiler's routes answer
|
|
13
|
+
# it as a 503, the MCP tools as an error).
|
|
14
|
+
class Error < Profiler::Error; end
|
|
15
|
+
|
|
16
|
+
@said = {}
|
|
17
|
+
@mutex = Mutex.new
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
def for(error)
|
|
21
|
+
key = [Process.pid, error.class.name, error.message]
|
|
22
|
+
first = @mutex.synchronize { @said[key] ? false : (@said[key] = true) }
|
|
23
|
+
if first
|
|
24
|
+
Profiler.log_warn("storage unavailable, profiles are not saved until it is fixed", error)
|
|
25
|
+
end
|
|
26
|
+
new(error)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def reset!
|
|
30
|
+
@mutex.synchronize { @said = {} }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def initialize(error)
|
|
35
|
+
@error = Error.new("The profiler storage is unavailable: #{self.class.short_cause(error)}")
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# First line of the message, absolute paths replaced, credentials masked, at most 300
|
|
39
|
+
# characters.
|
|
40
|
+
def self.short_cause(error)
|
|
41
|
+
line = error.message.to_s.lines.first.to_s.strip
|
|
42
|
+
line = line.gsub(%r{(?<=\A|[\s'"(=])(?:[A-Za-z]:)?[/\\][^\s'"]*[^\s'":,.;)]}, "[path]")
|
|
43
|
+
line = Redaction.hide_credentials(line) if defined?(Redaction)
|
|
44
|
+
line = line[0, 300]
|
|
45
|
+
line.empty? ? error.class.name : line
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def save(token, _profile)
|
|
49
|
+
token
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
%i[load list cleanup exists? find_by_parent delete clear].each do |name|
|
|
53
|
+
define_method(name) { |*, **| raise @error }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -8,7 +8,7 @@ module Profiler
|
|
|
8
8
|
def self.print
|
|
9
9
|
return unless Profiler.enabled?
|
|
10
10
|
|
|
11
|
-
profiles = Profiler.storage.list(limit: 1000).select { |p| p.profile_type == "test" }
|
|
11
|
+
profiles = Profiler.storage.list(limit: 1000, type: "test").select { |p| p.profile_type == "test" }
|
|
12
12
|
return if profiles.empty?
|
|
13
13
|
|
|
14
14
|
passed = profiles.count { |p| test_status(p) == "passed" }
|
|
@@ -75,7 +75,7 @@ module Profiler
|
|
|
75
75
|
|
|
76
76
|
$stdout.puts lines.join("\n")
|
|
77
77
|
rescue => e
|
|
78
|
-
|
|
78
|
+
Profiler.log_error("Reporter: could not generate the report", e)
|
|
79
79
|
end
|
|
80
80
|
|
|
81
81
|
def self.has_n1?(profile)
|
|
@@ -114,7 +114,7 @@ module Profiler
|
|
|
114
114
|
collector.collect if collector.respond_to?(:collect)
|
|
115
115
|
profile.add_collector_metadata(collector)
|
|
116
116
|
rescue => e
|
|
117
|
-
|
|
117
|
+
Profiler.log_error("TestProfiler: collector #{collector.class} failed", e)
|
|
118
118
|
end
|
|
119
119
|
end
|
|
120
120
|
|
|
@@ -51,7 +51,7 @@ module Profiler
|
|
|
51
51
|
root = project_root
|
|
52
52
|
allowed = if Profiler.configuration.test_runner_allow_undiscovered_files
|
|
53
53
|
warn_once(:undiscovered,
|
|
54
|
-
"
|
|
54
|
+
"config.test_runner_allow_undiscovered_files is enabled: the test runner " \
|
|
55
55
|
"runs any file under the Rails root, not only the discovered tests.")
|
|
56
56
|
nil
|
|
57
57
|
else
|
|
@@ -101,11 +101,7 @@ module Profiler
|
|
|
101
101
|
@warned[key] = true
|
|
102
102
|
end
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
Rails.logger.warn(message)
|
|
106
|
-
else
|
|
107
|
-
warn(message)
|
|
108
|
-
end
|
|
104
|
+
Profiler.log_warn(message)
|
|
109
105
|
end
|
|
110
106
|
|
|
111
107
|
def self.reset_warnings!
|
|
@@ -227,7 +223,7 @@ module Profiler
|
|
|
227
223
|
|
|
228
224
|
unless left_out.empty?
|
|
229
225
|
warn_once([:env, left_out.sort],
|
|
230
|
-
"
|
|
226
|
+
"Environment overrides not passed to the test runner, because they make it " \
|
|
231
227
|
"load other code: #{left_out.sort.join(", ")}.")
|
|
232
228
|
end
|
|
233
229
|
|
data/lib/profiler/version.rb
CHANGED