rails-profiler 0.45.1 → 0.45.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: af2f64f9bd55d0aa4a85949ef9b1acd256be278cff05a4659541f270f8a6f7fa
4
- data.tar.gz: 36ca8a5422e3fef0333d10b7d57a9c15cf9a66ac11fbe2daadc3a18685d95f8d
3
+ metadata.gz: 84a3ca793017ac61edb9131e4471d02fdacdc9324f569a7d53a81d63f0ceb07c
4
+ data.tar.gz: 29856f7b8101ed1a112fc2f80e722fbcd4996633bca3fca4f82d03fccb020901
5
5
  SHA512:
6
- metadata.gz: 9cb61932d90337b7428435ad21ebceb9ac29cb8c634c2ef61bfe2fbde6ea9330201286f54d3d0ddffe034158f8113650c52a1d3ea2b5ccc367c147859cea0be0
7
- data.tar.gz: 327222d2579665d70af75527d59459aac46fbd454f5c94476cbda59d516fa314790a7575405c8145ea12023c6635a74bc59cafcdccc8e3e7f02ae2b4b82d029c
6
+ metadata.gz: fd9398c466b98aec376ba14770d67f21bff834799d1ec7d55c9fe9c02cdb6033b6b39b3d242172c709a25ea5c1b230d76e24b4ab2b26706f8ccbd433c8ba5be0
7
+ data.tar.gz: 974d4dc48e021535125bf3c1f8ae67baf35c15b08d914f4ae92535dea2de33084598d2b7d200a65d21e891fad13274c08a336dac85f4a31455eb01502aed6fc5
data/CHANGELOG.md CHANGED
@@ -19,6 +19,22 @@ every commit of every tag interval is accounted for one way or the other.
19
19
 
20
20
  ## [Unreleased]
21
21
 
22
+ ## [0.45.3] - 2026-10-27
23
+
24
+ <!-- stamped -->
25
+
26
+ ### Fixed
27
+
28
+ - **Outbound HTTP:** The body of a request sent through a `body_stream` that cannot be rewound (RestClient's payload, a pipe) is now captured, redacted and cut at `max_captured_body_bytes` like any other, instead of showing "Request body not captured". The profiler copies the bytes Net::HTTP reads and leaves the stream and what the server receives untouched.
29
+
30
+ ## [0.45.2] - 2026-10-27
31
+
32
+ <!-- stamped -->
33
+
34
+ ### Fixed
35
+
36
+ - **HTTP capture:** A JSON, NDJSON or form body longer than `max_captured_body_bytes` is now redacted whole and the result cut to the limit, so a large response or request shows its first bytes with sensitive values masked instead of a single `[FILTERED: unparseable ... body]` mask. A body whose whole text is not available (inflation stopped at the limit, a partly read `body_stream`) or that exceeds 8 MiB is masked with a `[FILTERED: truncated ... body, kept N of M bytes, ...]` label that says it was truncated, and only a genuinely invalid JSON keeps the `unparseable` label.
37
+
22
38
  ## [0.45.1] - 2026-10-27
23
39
 
24
40
  <!-- stamped -->
@@ -91,6 +91,12 @@ module Profiler
91
91
  save_if_collected
92
92
  end
93
93
 
94
+ # Called from NetHttpInstrumentation to add what is only known once the request is sent.
95
+ def update_request(entry, **data)
96
+ @mutex.synchronize { entry.merge!(data) }
97
+ save_if_collected
98
+ end
99
+
94
100
  # Called from NetHttpInstrumentation when the HTTP call raises.
95
101
  def fail_request(entry, error:, duration:)
96
102
  @mutex.synchronize { entry.merge!(in_flight: false, status: 0, duration: duration, error: error) }
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Profiler
4
+ module Instrumentation
5
+ # Only the methods the original has are defined: IO.copy_stream picks readpartial over
6
+ # read when it finds it, and Net::HTTP rewinds a stream only if it can.
7
+ module Partial
8
+ def readpartial(*args)
9
+ result = @original.readpartial(*args)
10
+ record(result, false)
11
+ result
12
+ rescue EOFError
13
+ @finished = true
14
+ raise
15
+ end
16
+
17
+ def read_nonblock(*args, **opts)
18
+ result = @original.read_nonblock(*args, **opts)
19
+ record(result, false) if result.is_a?(String)
20
+ result
21
+ rescue EOFError
22
+ @finished = true
23
+ raise
24
+ end
25
+ end
26
+
27
+ module Rewindable
28
+ # Net::HTTP rewinds a stream before it retries an idempotent request: what was copied
29
+ # belongs to the attempt that failed.
30
+ def rewind(*args)
31
+ result = @original.rewind(*args)
32
+ @buffer.clear
33
+ @total_read = 0
34
+ @finished = false
35
+ result
36
+ end
37
+ end
38
+
39
+ # Stands in for a request body_stream that cannot be rewound (a pipe, a socket, a payload
40
+ # object such as RestClient's) while Net::HTTP sends it. It hands every read to the original
41
+ # stream untouched and copies what came out, up to one byte past +limit+, into a buffer; nothing is
42
+ # read ahead, so the server gets exactly the bytes it would have got.
43
+ #
44
+ # to_io and to_path are deliberately not forwarded: IO.copy_stream would use them to go around
45
+ # this object and read the original directly.
46
+ class CapturingStream
47
+ attr_reader :original, :total_read
48
+
49
+ def initialize(original, limit)
50
+ @original = original
51
+ @limit = limit
52
+ @buffer = String.new(encoding: Encoding::BINARY)
53
+ @total_read = 0
54
+ @finished = false
55
+ extend Partial if original.respond_to?(:readpartial)
56
+ extend Rewindable if original.respond_to?(:rewind)
57
+ end
58
+
59
+ # The bytes seen so far, at most one past +limit+ (so that the caller can tell the body was
60
+ # cut, and cut it with Redaction.cut_bytes, which holds back what could start a credential).
61
+ def captured
62
+ @buffer.dup
63
+ end
64
+
65
+ # True once a read has reported the end of the stream.
66
+ def finished?
67
+ @finished
68
+ end
69
+
70
+ def read(*args)
71
+ result = @original.read(*args)
72
+ record(result, args.first.nil?)
73
+ result
74
+ end
75
+
76
+ def respond_to_missing?(name, include_private = false)
77
+ return false if %i[to_io to_path].include?(name)
78
+
79
+ @original.respond_to?(name, include_private) || super
80
+ end
81
+
82
+ def method_missing(name, *args, **opts, &block)
83
+ if %i[to_io to_path].include?(name) || !@original.respond_to?(name)
84
+ return super
85
+ end
86
+
87
+ opts.empty? ? @original.public_send(name, *args, &block) : @original.public_send(name, *args, **opts, &block)
88
+ end
89
+
90
+ private
91
+
92
+ # +whole+: a read without a length, which returns the rest and so ends the stream.
93
+ def record(chunk, whole)
94
+ @finished = true if chunk.nil? || whole
95
+ return if chunk.nil? || chunk.empty?
96
+
97
+ @total_read += chunk.bytesize
98
+ room = @limit ? @limit + 1 - @buffer.bytesize : chunk.bytesize
99
+ @buffer << chunk.byteslice(0, room).b if room.positive?
100
+ end
101
+ end
102
+ end
103
+ end
@@ -10,6 +10,7 @@ require_relative "../body_content"
10
10
  require_relative "../configuration"
11
11
  require_relative "../current_context"
12
12
  require_relative "../trace_context"
13
+ require_relative "capturing_stream"
13
14
 
14
15
  module Profiler
15
16
  module Instrumentation
@@ -34,10 +35,19 @@ module Profiler
34
35
  url = Redaction.filter_url(build_url(host, port, req.path, use_ssl?))
35
36
  limit = Profiler.configuration.max_captured_body_bytes
36
37
  captured = NetHttpInstrumentation.capture_request_body(req, body, limit)
38
+ tee = captured[:tee]
37
39
  req_headers = Redaction.filter_headers(req.to_hash.transform_values { |v| v.join(", ") })
38
40
  req_content_type = req["content-type"].to_s
39
41
  req_body = captured[:content]
40
- processed_req = req_body.nil? || req_body.empty? ? { body: nil, encoding: "text" } : NetHttpInstrumentation.process_body(req_body, req_content_type)
42
+ processed_req = if req_body.nil? || req_body.empty?
43
+ { body: nil, encoding: "text" }
44
+ else
45
+ NetHttpInstrumentation.process_body(
46
+ req_body, req_content_type,
47
+ limit: captured[:truncated] ? limit : nil,
48
+ size: captured[:size_is_minimum] ? nil : captured[:size]
49
+ )
50
+ end
41
51
 
42
52
  request_id = SecureRandom.hex(8)
43
53
  started_at = Time.now.iso8601(3)
@@ -75,9 +85,13 @@ module Profiler
75
85
  resp_content_encoding = response["content-encoding"].to_s.strip.downcase
76
86
  resp_body = NetHttpInstrumentation.decompress_body(resp_body_raw, resp_content_encoding, limit)
77
87
  resp_truncated = limit ? resp_body.bytesize > limit : false
78
- resp_body = Redaction.cut_bytes(resp_body, limit) if resp_truncated
79
88
  resp_content_type = response["content-type"].to_s
80
- processed_resp = NetHttpInstrumentation.process_body(resp_body, resp_content_type)
89
+ # Cut after the redaction, not before: a cut JSON text cannot be parsed. Only an
90
+ # inflation stopped at the limit leaves a beginning with no known whole size.
91
+ resp_whole_size = !resp_truncated || resp_body.equal?(resp_body_raw) ? resp_body.bytesize : nil
92
+ processed_resp = NetHttpInstrumentation.process_body(
93
+ resp_body, resp_content_type, limit: resp_truncated ? limit : nil, size: resp_whole_size
94
+ )
81
95
 
82
96
  t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
83
97
 
@@ -106,6 +120,7 @@ module Profiler
106
120
  raise
107
121
  ensure
108
122
  Thread.current[:profiler_http_recording] = false
123
+ NetHttpInstrumentation.finish_stream_capture(req, tee, entry, collector) if defined?(tee) && tee
109
124
  end
110
125
 
111
126
  private
@@ -200,8 +215,8 @@ module Profiler
200
215
  unless content.nil? || content.to_s.empty?
201
216
  content = content.to_s
202
217
  truncated = limit ? content.bytesize > limit : false
203
- return { content: truncated ? Redaction.cut_bytes(content, limit) : content,
204
- size: content.bytesize, size_is_minimum: false, truncated: truncated, not_captured: false }
218
+ # Kept whole: process_body redacts it, then cuts the result.
219
+ return { content: content, size: content.bytesize, size_is_minimum: false, truncated: truncated, not_captured: false }
205
220
  end
206
221
 
207
222
  stream = req.body_stream
@@ -210,7 +225,9 @@ module Profiler
210
225
  declared = req["content-length"].to_s
211
226
  declared = declared.match?(/\A\d+\z/) ? declared.to_i : nil
212
227
  unless rewindable?(stream)
213
- return { content: nil, size: declared, size_is_minimum: false, truncated: false, not_captured: true }
228
+ # Read as Net::HTTP sends it, by a stand-in that copies what passes (see finish_stream_capture).
229
+ tee = tee_stream(req, stream, limit)
230
+ return { content: nil, size: declared, size_is_minimum: false, truncated: false, not_captured: true, tee: tee }
214
231
  end
215
232
 
216
233
  position = stream.pos
@@ -234,6 +251,48 @@ module Profiler
234
251
  { content: nil, size: declared, size_is_minimum: false, truncated: false, not_captured: true }
235
252
  end
236
253
 
254
+ # Puts a CapturingStream in place of +stream+ on the request; nil (the request untouched) when
255
+ # the stream cannot be read or the stand-in cannot be installed.
256
+ def self.tee_stream(req, stream, limit)
257
+ return nil unless stream.respond_to?(:read)
258
+
259
+ # A JSON, NDJSON or form body is redacted whole before it is cut, so more of it is copied.
260
+ mime = req["content-type"].to_s.split(";").first.to_s.strip.downcase
261
+ copy = limit && mime.match?(%r{json|x-www-form-urlencoded}) ? Redaction::MAX_REDACTABLE_BODY_BYTES : limit
262
+ tee = CapturingStream.new(stream, copy)
263
+ req.body_stream = tee
264
+ tee
265
+ rescue StandardError => e
266
+ Profiler.log_error_once(:net_http_body_stream_tee, "NetHttpInstrumentation: could not capture a request body_stream", e)
267
+ nil
268
+ end
269
+
270
+ # Once the request has been sent (or has failed): the original stream goes back on the
271
+ # request, and the entry gets the body that passed, redacted like any other. An entry whose
272
+ # stream was never read stays "not captured".
273
+ def self.finish_stream_capture(req, tee, entry, collector)
274
+ req.body_stream = tee.original if req.body_stream.equal?(tee)
275
+ return unless entry && tee.total_read.positive?
276
+
277
+ limit = Profiler.configuration.max_captured_body_bytes
278
+ declared = req["content-length"].to_s
279
+ declared = declared.match?(/\A\d+\z/) ? declared.to_i : nil
280
+ raw = tee.captured
281
+ size = declared || tee.total_read
282
+ cut = limit && raw.bytesize > limit
283
+ processed = process_body(raw.force_encoding(Encoding::UTF_8), req["content-type"].to_s,
284
+ limit: (limit if cut), size: size)
285
+ kept = cut ? limit : raw.bytesize
286
+ collector.update_request(
287
+ entry,
288
+ request_body: processed[:body], request_body_encoding: processed[:encoding],
289
+ request_size: size, request_size_is_minimum: declared.nil? && !tee.finished?,
290
+ request_body_truncated: kept < size, request_body_not_captured: false
291
+ )
292
+ rescue StandardError => e
293
+ Profiler.log_error_once(:net_http_body_stream_tee, "NetHttpInstrumentation: could not record a request body_stream", e)
294
+ end
295
+
237
296
  # What is left to read of a stream that knows its size (a StringIO, a file), else nil.
238
297
  def self.remaining_size(stream, position)
239
298
  size = stream.respond_to?(:size) ? stream.size : nil
@@ -295,18 +354,24 @@ module Profiler
295
354
  defined?(Net::ReadAdapter) && response.body.is_a?(Net::ReadAdapter)
296
355
  end
297
356
 
298
- def self.process_body(body, content_type)
357
+ # +limit+ is set when the body is kept only in part: it is then redacted whole and the result
358
+ # cut to +limit+ bytes (Redaction.filter_truncated_body). +size+ is the length of the whole
359
+ # body, nil when it is not known.
360
+ def self.process_body(body, content_type, limit: nil, size: body&.bytesize)
299
361
  return { body: nil, encoding: "text" } if body.nil? || body.empty?
300
362
 
301
363
  if BodyContent.binary?(body, content_type)
302
- # Masked on the raw bytes, before the encoding hides them from any later search.
303
- { body: Base64.strict_encode64(Redaction.hide_credentials(body.b)), encoding: "base64" }
364
+ # Masked on the raw bytes, before the encoding hides them from any later search, and
365
+ # cut after the masking: a credential cut in two would escape it.
366
+ masked = Redaction.hide_credentials(body.b)
367
+ masked = Redaction.cut_bytes(masked, limit) if limit
368
+ { body: Base64.strict_encode64(masked), encoding: "base64" }
304
369
  else
305
370
  # Bytes are read as UTF-8, as a text body on the wire is; what is not valid UTF-8 shows
306
371
  # as "?".
307
372
  body = body.dup.force_encoding(Encoding::UTF_8) if body.encoding == Encoding::BINARY
308
- text = Redaction.filter_body(body, content_type)
309
- .encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
373
+ filtered = limit ? Redaction.filter_truncated_body(body, content_type, limit, size) : Redaction.filter_body(body, content_type)
374
+ text = filtered.encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
310
375
  text = text.scrub("?") unless text.valid_encoding?
311
376
  { body: text, encoding: "text" }
312
377
  end
@@ -36,6 +36,11 @@ module Profiler
36
36
  FORM_TYPE = "application/x-www-form-urlencoded"
37
37
  MULTIPART_TYPE = %r{\Amultipart/}i
38
38
 
39
+ # The largest body filter_truncated_body parses to redact it whole. A bigger JSON, NDJSON or
40
+ # form body is not parsed (a parse costs several times its size in memory) and is masked with
41
+ # a label instead, never passed through unredacted.
42
+ MAX_REDACTABLE_BODY_BYTES = 8 * 1024 * 1024
43
+
39
44
  # The characters outside ASCII whose case folding yields ASCII letters
40
45
  # (Unicode CaseFolding.txt): with any of them, a case-insensitive regexp
41
46
  # can match an ASCII filter where a plain ASCII search cannot.
@@ -52,7 +57,7 @@ module Profiler
52
57
  # profile can be read on any node. Wraps every public filter, so that each
53
58
  # capture point gets it.
54
59
  module CredentialMasking
55
- %i[filter_hash filter_value filter_named filter_headers filter_body filter_query filter_url
60
+ %i[filter_hash filter_value filter_named filter_headers filter_body filter_truncated_body filter_query filter_url
56
61
  env_snapshot env_value env_overrides].each do |name|
57
62
  define_method(name) { |*args| hide_credentials(super(*args)) }
58
63
  end
@@ -221,6 +226,35 @@ module Profiler
221
226
  "[FILTERED: #{mime} body that could not be filtered, #{raw.bytesize} bytes]"
222
227
  end
223
228
 
229
+ # The part of a body a profile keeps when it is longer than +limit+ bytes, redacted. The body
230
+ # is redacted whole first and the result is cut after, so the part kept is redaction output
231
+ # and never shows a value the filter masks. +size+ is the length of the whole body when
232
+ # known; the body is whole when +raw+ has that length, and only a beginning otherwise
233
+ # (an inflation stopped at the limit, a body_stream read in part).
234
+ #
235
+ # A JSON, NDJSON or form body that cannot be redacted whole (only its beginning is known, or
236
+ # it is over MAX_REDACTABLE_BODY_BYTES) is masked entirely with a label that says it was
237
+ # truncated: a cut JSON text cannot be parsed and a text search would miss keys. Other types
238
+ # are cut first, as filter_body leaves them as they are or masks them whole anyway. A whole
239
+ # body that is not valid JSON keeps the "unparseable" label of filter_body.
240
+ def filter_truncated_body(raw, content_type, limit, size = raw.is_a?(String) ? raw.bytesize : nil)
241
+ return raw unless raw.is_a?(String) && !raw.empty?
242
+ return filter_body(raw, content_type) if limit.nil? || (size == raw.bytesize && raw.bytesize <= limit)
243
+ return cut_bytes(raw, limit) unless enabled?
244
+
245
+ mime = content_type.to_s.split(";").first.to_s.strip.downcase
246
+ return filter_body(cut_bytes(raw, limit), content_type) unless structured_type?(mime)
247
+
248
+ whole = size == raw.bytesize
249
+ return cut_bytes(filter_body(raw, content_type), limit) if whole && raw.bytesize <= MAX_REDACTABLE_BODY_BYTES
250
+
251
+ reason = whole ? "too large to redact" : "only its beginning is known, it cannot be redacted"
252
+ "[FILTERED: truncated #{mime} body, kept #{limit} of #{size || "more than #{limit}"} bytes, #{reason}]"
253
+ rescue StandardError => e
254
+ report(e, "filtering a truncated body")
255
+ "[FILTERED: truncated body that could not be filtered, #{raw.bytesize} bytes]"
256
+ end
257
+
224
258
  # Masks the filtered values of a query string, pair by pair, leaving the
225
259
  # rest of the string exactly as it was.
226
260
  def filter_query(query)
@@ -481,6 +515,10 @@ module Profiler
481
515
  utf8.valid_encoding? ? utf8 : string.b
482
516
  end
483
517
 
518
+ def structured_type?(mime)
519
+ mime.match?(JSON_TYPES) || mime.match?(NDJSON_TYPES) || mime == FORM_TYPE
520
+ end
521
+
484
522
  def filter_json(text, mime)
485
523
  return text unless json_needs_parse?(text)
486
524
 
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Profiler
4
- VERSION = "0.45.1"
4
+ VERSION = "0.45.3"
5
5
 
6
6
  GEM_ROOT = File.expand_path("../..", __dir__)
7
7
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rails-profiler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.45.1
4
+ version: 0.45.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sébastien Duplessy
@@ -194,6 +194,7 @@ files:
194
194
  - lib/profiler/env_override_store.rb
195
195
  - lib/profiler/explain_runner.rb
196
196
  - lib/profiler/instrumentation/active_job_instrumentation.rb
197
+ - lib/profiler/instrumentation/capturing_stream.rb
197
198
  - lib/profiler/instrumentation/executor_context_propagation.rb
198
199
  - lib/profiler/instrumentation/irb_instrumentation.rb
199
200
  - lib/profiler/instrumentation/net_http_instrumentation.rb