dead_bro 0.2.33 → 0.2.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1569a9a3be524d19ba616313570f01cb251647a76384347993655a8ffbaef9e
-  data.tar.gz: 3a091b0d844b12cfb9b61cf7b0d2eec04514bb5ef988fba7203d45b4ab237e67
+  metadata.gz: 1e87bf1af27a0f9c5e26679757060bbef1d1a84e3a9ff597e997ee6665041f06
+  data.tar.gz: 9872d696d014494ed15f7ac83e809530a1d4f54f5d5db23d783e8c6af99f2fd2
 SHA512:
-  metadata.gz: ebc44f95253369e7c8ab6f3873ed9143e0d72382c477bae39ca09491c22c93b236b2bc1dcc07599dd755797564f4a0f25f34104cf1f95246ea8d91fa9b4c1935
-  data.tar.gz: 6dd40034800398ceac4e82895d427e484ee7fd1b321a22882c1138c0e73514bfb4675a14437abd88cb6f4e5327c01ad85f2c7e75269956c4dd9d4e240a946714
+  metadata.gz: ae026a67741f9561d7b2423cd9c35a6bd80b3c1b1b0365279b665caea0cd2d45f5e7519bae715890114458be8dbe284bab766a7437a41f8c841557fbfbb00ee2
+  data.tar.gz: 87c6441aa1789ca958c5980edadef4d5e1d5a73e03573d8684a02dbb62c26a31f8ae13300487901b63e4ba1fcca62fc4ba0676badd059d44783613dc21fb09af

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.2.34] - 2026-09-29
+
+### Fixed
+- **A request carrying bytes that aren't valid UTF-8 (e.g. a bot's user agent with a stray `\xA1`) got a 500 from the host app and wasn't recorded.** Rack hands header values over as raw bytes, and the subscriber only stripped NUL bytes, so `JSON.dump` raised `JSON::GeneratorError` inside the `process_action.action_controller` listener. Rails re-raises a listener's exception into the request. `ErrorMiddleware` then reported that error with the same raw bytes, failed the same way and swallowed it. The user agent, path, host, params, exception messages and backtraces (and every `ErrorMiddleware` field) are now converted to valid UTF-8 before sending, with invalid bytes replaced by U+FFFD, and the request is recorded with the scrubbed value.
+- **A DeadBro listener or middleware can no longer raise into the host app.** Every `ActiveSupport::Notifications` callback (requests, jobs, SQL, cache, Redis, Elasticsearch, views, AR instantiation), the outgoing HTTP and Redis hooks, and `SqlTrackingMiddleware`'s setup and teardown now rescue the same set of errors and drop the metric instead. Teardown also finishes when one of its steps fails, so allocation tracing is still stopped. Before, a failing background job could have its own exception replaced by DeadBro's, and a job whose payload couldn't be built left its tracking state behind for the next job on the same worker thread. This is deliberately not `rescue Exception`: `Rack::Timeout`, `Sidekiq::Shutdown`, `Interrupt` and a plain `Timeout.timeout(sec)` that fire while DeadBro code runs still reach the host. `Timeout.timeout(sec, SomeStandardError)` can still be swallowed if it fires mid-callback, as it can by any `rescue => e`.
+- **SQL containing invalid UTF-8 (e.g. a binary literal inlined by an adapter that doesn't use prepared statements) failed the host's query.** The SQL sanitizer's regexes raised `ArgumentError`; the SQL is now scrubbed first and the query is tracked.
+- **A param key that isn't valid UTF-8 made DeadBro drop the request's params.** The sensitive-key check raised on it; keys are now scrubbed first, so the params are kept and still redacted. (Only reachable from a JSON body in an app with no `filter_parameters`; otherwise Rails fails the request on such a key first.)
+
+### Changed
+- Cache keys, Redis keys and commands, job arguments and log messages are scrubbed where they're captured, so a bad byte in one of them (e.g. a cache key built from the user agent) no longer costs the request a full copy of its payload.
+- If a payload still can't be JSON-encoded because some other string isn't valid UTF-8, the client retries once with every string in the payload scrubbed rather than losing it. Payloads that can't be serialized even then are dropped and counted in `DeadBro.client.serialization_failures`.
+
 ## [0.2.32] - 2026-09-22
 
 ### Fixed

data/lib/dead_bro/ar_object_tracker.rb

@@ -13,6 +13,7 @@ module DeadBro
         count = Thread.current[THREAD_KEY]
         next unless count
         Thread.current[THREAD_KEY] = count + (data[:record_count] || 1).to_i
+      rescue *DeadBro::CONTAINED_ERRORS
       end
     rescue StandardError
       # Never raise from instrumentation install

data/lib/dead_bro/cache_subscriber.rb

@@ -30,6 +30,7 @@ module DeadBro
           if event && should_continue_tracking?
             Thread.current[THREAD_LOCAL_KEY] << event
           end
+        rescue *DeadBro::CONTAINED_ERRORS
         end
       rescue
       end
@@ -85,7 +86,9 @@ module DeadBro
 
     def self.safe_key(key)
       return nil if key.nil?
-      s = key.to_s
+      # Keys often embed request data (e.g. a throttle keyed by user agent);
+      # scrubbing here keeps one bad byte off the client's slow retry path.
+      s = DeadBro::Sanitizer.string(key)
       (s.length > 200) ? s[0, 200] + "…" : s
     rescue
       nil

data/lib/dead_bro/client.rb

@@ -10,6 +10,13 @@ module DeadBro
     def initialize(configuration = DeadBro.configuration)
       @configuration = configuration
       @circuit_breaker = create_circuit_breaker
+      @failures_mutex = Mutex.new
+      @serialization_failures = 0
+    end
+
+    # Payloads dropped because they couldn't be JSON-encoded even after scrubbing.
+    def serialization_failures
+      @failures_mutex.synchronize { @serialization_failures }
     end
 
     def post_metric(event_name:, payload:, force: false)
@@ -41,6 +48,9 @@ module DeadBro
       if sync
         # Called from the monitor thread on startup — run inline so settings are
         # applied before the first collection tick.
+        json = serialize(body)
+        return unless json
+
         uri = URI.parse(metrics_endpoint_url)
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = (uri.scheme == "https")
@@ -49,7 +59,7 @@ module DeadBro
         request = Net::HTTP::Post.new(uri.request_uri)
         request["Content-Type"] = "application/json"
         request["Authorization"] = "Bearer #{@configuration.api_key}"
-        request.body = JSON.dump(body)
+        request.body = json
         perform_request(http, request, event_name: "heartbeat", apply_settings: true)
       else
         dispatch_request(
@@ -119,6 +129,9 @@ module DeadBro
     end
 
     def dispatch_request(url:, body:, event_name:, apply_settings: false)
+      json = serialize(body)
+      return unless json
+
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = (uri.scheme == "https")
@@ -131,13 +144,38 @@ module DeadBro
       if @configuration.settings_received_at
         request["X-Settings-Received-At"] = @configuration.settings_received_at.utc.iso8601
       end
-      request.body = JSON.dump(body)
+      request.body = json
 
       DeadBro::Dispatcher.instance.dispatch do
         perform_request(http, request, event_name: event_name, apply_settings: apply_settings)
       end
     end
 
+    # Must not raise: for request and job payloads this runs on the host's own
+    # thread. JSON.dump raises on any String that isn't valid UTF-8; the payload
+    # builders scrub the fields known to carry raw request bytes, and this
+    # catches whatever else slips through (a log line, a cache key, a job
+    # argument) at the cost of one retry instead of the whole payload. Returns
+    # nil when the payload can't be sent.
+    def serialize(body)
+      JSON.dump(body)
+    rescue JSON::GeneratorError, EncodingError
+      begin
+        JSON.dump(DeadBro::Sanitizer.deep(body))
+      rescue *DeadBro::CONTAINED_ERRORS
+        record_serialization_failure
+      end
+    rescue *DeadBro::CONTAINED_ERRORS
+      # Not an encoding problem (e.g. an object whose to_json raises), so
+      # scrubbing strings wouldn't help.
+      record_serialization_failure
+    end
+
+    def record_serialization_failure
+      @failures_mutex.synchronize { @serialization_failures += 1 }
+      nil
+    end
+
     def perform_request(http, request, event_name:, apply_settings: false)
       response = http.request(request)
 

data/lib/dead_bro/elasticsearch_subscriber.rb

@@ -123,7 +123,7 @@ module DeadBro
             tracking_start = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
             start_offset_ms = tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil
             events << build_event(method, path, payload[:status], duration_ms, start_offset_ms)
-          rescue
+          rescue *DeadBro::CONTAINED_ERRORS
           end
         end
       rescue

data/lib/dead_bro/error_middleware.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "digest"
 require "rack"
 
 module DeadBro
@@ -21,8 +20,9 @@ module DeadBro
         event_name = exception.class.name.to_s
         event_name = EVENT_NAME if event_name.empty?
         @client.post_metric(event_name: event_name, payload: payload, force: true)
-      rescue
-        # Never let APM reporting interfere with the host app
+      rescue *DeadBro::CONTAINED_ERRORS
+        # Never let APM reporting interfere with the host app — in particular,
+        # never replace the host's exception with one of ours.
       end
       raise
     end
@@ -32,12 +32,12 @@ module DeadBro
     def build_payload(exception, env)
       req = rack_request(env)
 
-      {
+      payload = {
         exception_class: exception.class.name,
         message: truncate(exception.message.to_s, 1000),
         backtrace: safe_backtrace(exception),
-        fingerprint: compute_fingerprint(exception),
-        cause_chain: build_cause_chain(exception),
+        fingerprint: DeadBro::Subscriber.compute_error_fingerprint(exception),
+        cause_chain: DeadBro::Subscriber.build_cause_chain(exception),
         occurred_at: Time.now.utc.to_i,
         rack:
           {
@@ -57,6 +57,10 @@ module DeadBro
         process_kind: DeadBro.process_kind,
         logs: DeadBro.logger.logs
       }
+      # Every field above can carry raw client bytes (headers are binary strings),
+      # so scrub the whole payload rather than field by field. This path only
+      # runs for uncaught exceptions, so the extra walk is cheap.
+      DeadBro::Sanitizer.deep(payload)
     end
 
     def rack_request(env)
@@ -71,61 +75,23 @@ module DeadBro
       []
     end
 
-    def normalize_message(msg)
-      msg.to_s
-        .gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i, "UUID")
-        .gsub(/\b\d+\b/, "N")
-        .gsub(/"[^"]*"/, '"?"')
-        .gsub(/'[^']*'/, "'?'")
-        .strip
-    end
-
-    def compute_fingerprint(exception)
-      top_frame = Array(exception.backtrace).first.to_s.gsub(/:\d+:in /, ":N:in ")
-      input = "#{exception.class.name}|#{normalize_message(exception.message)}|#{top_frame}"
-      Digest::SHA256.hexdigest(input)[0, 16]
-    rescue
-      nil
-    end
-
-    def build_cause_chain(exception)
-      return [] unless exception
-      chain = []
-      cause = exception.cause
-      depth = 0
-      while cause && depth < 5
-        chain << {
-          exception_class: cause.class.name,
-          message: truncate(cause.message.to_s, 500),
-          backtrace_top: Array(cause.backtrace).first(3)
-        }
-        cause = cause.cause
-        depth += 1
-      end
-      chain
-    rescue
-      []
-    end
-
     def safe_params(req)
       return {} unless req
 
       params = req.params || {}
       # Redact at every nesting level (e.g. user[password]) before serializing.
-      JSON.parse(JSON.dump(redact_sensitive(params)))
+      JSON.parse(JSON.dump(DeadBro::Sanitizer.deep(redact_sensitive(params))))
     rescue
       {}
     end
 
-    # Matches a key segment so nested/prefixed/suffixed sensitive keys are caught
-    # without redacting innocent keys like passenger_count.
-    SENSITIVE_SEGMENT_RE = /(?:\A|[_\-\[])(password|passwd|secret|token|api_?key|access_?key|auth|authorization|credential|ssn|credit_?card|card_?number|cvv|cvc)(?:\z|[_\-\]])/i
-
     def redact_sensitive(value)
       case value
       when Hash
         value.each_with_object({}) do |(k, v), memo|
-          memo[k] = SENSITIVE_SEGMENT_RE.match?(k.to_s) ? "[FILTERED]" : redact_sensitive(v)
+          # Scrubbed first: the sensitive-key regex raises on a key that isn't valid UTF-8.
+          k = DeadBro::Sanitizer.string(k) if k.is_a?(String)
+          memo[k] = DeadBro::Subscriber.sensitive_key?(k) ? "[FILTERED]" : redact_sensitive(v)
         end
       when Array
         value.map { |v| redact_sensitive(v) }

data/lib/dead_bro/http_instrumentation.rb

@@ -79,7 +79,7 @@ module DeadBro
                   Thread.current[THREAD_LOCAL_KEY] << payload
                 end
               end
-            rescue
+            rescue *DeadBro::CONTAINED_ERRORS
             end
           end
         end
@@ -120,7 +120,7 @@ module DeadBro
                   Thread.current[THREAD_LOCAL_KEY] << payload
                 end
               end
-            rescue
+            rescue *DeadBro::CONTAINED_ERRORS
             end
           end
         end
@@ -177,7 +177,7 @@ module DeadBro
                     Thread.current[key] << payload
                   end
                 end
-              rescue
+              rescue *DeadBro::CONTAINED_ERRORS
               end
             end
           end

data/lib/dead_bro/job_sql_tracking_middleware.rb

@@ -42,6 +42,7 @@ module DeadBro
         end
 
         DeadBro::WatchTracker.start_request_tracking if defined?(DeadBro::WatchTracker)
+      rescue *DeadBro::CONTAINED_ERRORS
       end
     rescue
       # Never raise from instrumentation install

data/lib/dead_bro/job_subscriber.rb

@@ -15,7 +15,7 @@ module DeadBro
       ActiveSupport::Notifications.subscribe("perform_start.active_job") do |_name, _started, _finished, _unique_id, _data|
         DeadBro::GcTracker.start_request_tracking if defined?(DeadBro::GcTracker)
         DeadBro::ArObjectTracker.start_request_tracking if defined?(DeadBro::ArObjectTracker)
-      rescue
+      rescue *DeadBro::CONTAINED_ERRORS
       end
 
       # Track job execution — success AND failure both land here. ActiveJob wraps
@@ -156,8 +156,8 @@ module DeadBro
 
         if has_error
           payload[:exception_class] = exception.class.name
-          payload[:message] = exception.message.to_s[0, 1000]
-          payload[:backtrace] = Array(exception.backtrace).first(50)
+          payload[:message] = DeadBro::Sanitizer.string(exception.message.to_s[0, 1000])
+          payload[:backtrace] = Array(exception.backtrace).first(50).map { |line| DeadBro::Sanitizer.string(line) }
           payload[:fingerprint] = DeadBro::Subscriber.compute_error_fingerprint(exception)
           payload[:cause_chain] = DeadBro::Subscriber.build_cause_chain(exception)
           payload[:error] = true
@@ -167,6 +167,11 @@ module DeadBro
         # completions the sampling decision above already accounted for any
         # per-job-type override, so client#post_metric must not re-roll it globally.
         client.post_metric(event_name: name, payload: payload, force: true)
+      rescue *DeadBro::CONTAINED_ERRORS
+        # Anything raised here would fail the host's job (see
+        # DeadBro::CONTAINED_ERRORS) — drop this job's metrics instead. Jobs have
+        # no Rack middleware to reset thread-locals afterwards, so drain here.
+        drain_job_tracking
       end
     rescue
       # Never raise from instrumentation install
@@ -259,17 +264,19 @@ module DeadBro
     def self.safe_arguments(arguments)
       return [] unless arguments.is_a?(Array)
 
-      # Limit and sanitize job arguments
+      # Limit and sanitize job arguments. Strings are scrubbed to valid UTF-8 here
+      # so one binary argument doesn't send the payload down the client's slow retry path.
       arguments.first(10).map do |arg|
         case arg
         when String
-          (arg.length > 200) ? arg[0, 200] + "..." : arg
+          str = DeadBro::Sanitizer.string(arg)
+          (str.length > 200) ? str[0, 200] + "..." : str
         when Hash
           # Filter sensitive keys and limit size
           filtered = arg.reject { |k, _| %w[password token secret key].include?(k.to_s) }
-          (filtered.keys.size > 20) ? filtered.first(20).to_h : filtered
+          DeadBro::Sanitizer.deep((filtered.keys.size > 20) ? filtered.first(20).to_h : filtered)
         when Array
-          arg.first(5)
+          DeadBro::Sanitizer.deep(arg.first(5))
         when ActiveRecord::Base
           # Handle ActiveRecord objects safely
           "#{arg.class.name}##{begin
@@ -279,7 +286,8 @@ module DeadBro
           end}"
         else
           # Convert to string and truncate, but avoid object inspection
-          (arg.to_s.length > 200) ? arg.to_s[0, 200] + "..." : arg.to_s
+          str = DeadBro::Sanitizer.string(arg)
+          (str.length > 200) ? str[0, 200] + "..." : str
         end
       end
     rescue

data/lib/dead_bro/logger.rb

@@ -72,6 +72,9 @@ module DeadBro
 
     def log(severity, message)
       timestamp = Time.now.utc
+      # Messages often interpolate request data; scrubbed here so one bad byte
+      # doesn't send the payload down the client's slow retry path.
+      message = DeadBro::Sanitizer.string(message)
 
       buffer = (Thread.current[@thread_logs_key] ||= [])
       if buffer.length >= MAX_LOG_ENTRIES
@@ -80,7 +83,7 @@ module DeadBro
       else
         buffer << {
           sev: severity.to_s,
-          msg: message.to_s,
+          msg: message,
           time: timestamp.iso8601(3) # Include milliseconds for better precision
         }
       end

data/lib/dead_bro/memory_phase_tracker.rb

@@ -91,7 +91,7 @@ module DeadBro
         state[:buckets][parent[:phase]] += now - parent[:checkpoint]
       end
       stack << {phase: phase, checkpoint: now}
-    rescue StandardError
+    rescue *DeadBro::CONTAINED_ERRORS
       # Best-effort only.
     end
 
@@ -110,7 +110,7 @@ module DeadBro
       if (parent = stack.last)
         parent[:checkpoint] = now
       end
-    rescue StandardError
+    rescue *DeadBro::CONTAINED_ERRORS
       # Best-effort only.
     end
 

data/lib/dead_bro/memory_tracking_subscriber.rb

@@ -39,6 +39,7 @@ module DeadBro
               allocations: allocations,
               allocated_bytes: allocated_bytes
             )
+          rescue *DeadBro::CONTAINED_ERRORS
           end
         rescue
           # Allocation tracking might not be available in all Ruby versions

data/lib/dead_bro/redis_subscriber.rb

@@ -77,7 +77,7 @@ module DeadBro
             duration_ms = ((finish_time - start_time) * 1000.0).round(2)
 
             begin
-              cmd_info = extract_command_info(command)
+              cmd_info = RedisSubscriber.extract_command(command: command)
               tracking_start = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
               start_offset_ms = tracking_start ? ((wall_start - tracking_start) * 1000.0).round(2) : nil
               event = {
@@ -87,14 +87,14 @@ module DeadBro
                 args_count: cmd_info[:args_count],
                 duration_ms: duration_ms,
                 start_offset_ms: start_offset_ms,
-                db: safe_db(@db),
+                db: RedisSubscriber.safe_db(@db),
                 error: error ? error.class.name : nil
               }
 
               if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
                 Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
               end
-            rescue
+            rescue *DeadBro::CONTAINED_ERRORS
             end
           end
         end
@@ -120,13 +120,13 @@ module DeadBro
                 commands_count: commands_count,
                 duration_ms: duration_ms,
                 start_offset_ms: start_offset_ms,
-                db: safe_db(@db)
+                db: RedisSubscriber.safe_db(@db)
               }
 
               if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
                 Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
               end
-            rescue
+            rescue *DeadBro::CONTAINED_ERRORS
             end
           end
         end
@@ -152,52 +152,16 @@ module DeadBro
                 commands_count: commands_count,
                 duration_ms: duration_ms,
                 start_offset_ms: start_offset_ms,
-                db: safe_db(@db)
+                db: RedisSubscriber.safe_db(@db)
               }
 
               if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
                 Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
               end
-            rescue
+            rescue *DeadBro::CONTAINED_ERRORS
             end
           end
         end
-
-        def extract_command_info(command)
-          parts = Array(command).map(&:to_s)
-          command_name = parts.first&.upcase
-          key = parts[1]
-          args_count = (parts.length > 1) ? parts.length - 1 : 0
-
-          {
-            command: safe_command(command_name),
-            key: safe_key(key),
-            args_count: args_count
-          }
-        rescue
-          {command: nil, key: nil, args_count: nil}
-        end
-
-        def safe_command(cmd)
-          return nil if cmd.nil?
-          cmd.to_s[0, 20]
-        rescue
-          nil
-        end
-
-        def safe_key(key)
-          return nil if key.nil?
-          s = key.to_s
-          (s.length > 200) ? s[0, 200] + "…" : s
-        rescue
-          nil
-        end
-
-        def safe_db(db)
-          Integer(db)
-        rescue
-          nil
-        end
       end
 
       ::Redis::Client.prepend(mod) unless ::Redis::Client.ancestors.include?(mod)
@@ -219,6 +183,7 @@ module DeadBro
             if event && should_continue_tracking?
               Thread.current[THREAD_LOCAL_KEY] << event
             end
+          rescue *DeadBro::CONTAINED_ERRORS
           end
         rescue
         end
@@ -296,14 +261,16 @@ module DeadBro
 
     def self.safe_command(cmd)
       return nil if cmd.nil?
-      cmd.to_s[0, 20]
+      DeadBro::Sanitizer.string(cmd)[0, 20]
     rescue
       nil
     end
 
+    # Redis keys can be binary (digests, packed ids) or embed request data;
+    # scrubbing here keeps one bad byte off the client's slow retry path.
     def self.safe_key(key)
       return nil if key.nil?
-      s = key.to_s
+      s = DeadBro::Sanitizer.string(key)
       (s.length > 200) ? s[0, 200] + "…" : s
     rescue
       nil

data/lib/dead_bro/sanitizer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module DeadBro
+  # Makes values safe to JSON-encode and to store on the backend: every String
+  # comes out as valid UTF-8 without NUL bytes.
+  #
+  # Rack hands header values over as binary strings holding whatever bytes the
+  # client sent, and JSON.dump raises on any that aren't valid UTF-8 — a bot's
+  # user agent carrying a stray "\xA1" was enough to turn a host app's 200 into
+  # a 500. NUL bytes are valid JSON, but PostgreSQL rejects them in text columns.
+  module Sanitizer
+    REPLACEMENT = "�"
+    NUL = "\u0000"
+
+    module_function
+
+    # Returns value.to_s as valid UTF-8 without NUL bytes. A string that is
+    # already clean is returned as is, so the common case allocates nothing.
+    def string(value)
+      str = value.to_s
+      str = to_utf8(str) unless str.ascii_only? || (str.encoding == Encoding::UTF_8 && str.valid_encoding?)
+      str.include?(NUL) ? str.delete(NUL) : str
+    end
+
+    # Copies Hashes and Arrays, running every String (keys included) through
+    # .string. Anything else is returned untouched — Symbols too, since they come
+    # from code rather than from clients, and payload code looks keys up by Symbol.
+    def deep(value)
+      case value
+      when String
+        string(value)
+      when Hash
+        value.each_with_object({}) { |(k, v), out| out[deep(k)] = deep(v) }
+      when Array
+        value.map { |v| deep(v) }
+      else
+        value
+      end
+    end
+
+    def to_utf8(str)
+      case str.encoding
+      when Encoding::BINARY, Encoding::US_ASCII, Encoding::UTF_8
+        # Raw bytes of unknown origin, or a broken UTF-8 string: read the bytes
+        # as UTF-8 and replace only the sequences that aren't.
+        utf8 = str.dup.force_encoding(Encoding::UTF_8)
+        utf8.valid_encoding? ? utf8 : utf8.scrub(REPLACEMENT)
+      else
+        str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
+      end
+    rescue EncodingError
+      # No converter from this encoding — keep whatever reads as UTF-8.
+      str.dup.force_encoding(Encoding::UTF_8).scrub(REPLACEMENT)
+    end
+  end
+end

data/lib/dead_bro/sql_subscriber.rb

@@ -271,6 +271,7 @@ module DeadBro
         if should_continue_tracking?(current, MAX_TRACKED_QUERIES)
           current << query_info
         end
+      rescue *DeadBro::CONTAINED_ERRORS
       end
     end
 
@@ -379,6 +380,9 @@ module DeadBro
       # are big UPDATE/INSERT with long literal blobs; don't burn regex time on
       # those when we're going to truncate anyway.
       sql = sql[0..SANITIZE_MAX_LENGTH] + "..." if sql.length > SANITIZE_MAX_LENGTH
+      # The regexes below raise on invalid UTF-8 (e.g. a binary literal inlined
+      # by an adapter without prepared statements).
+      sql = DeadBro::Sanitizer.string(sql)
 
       # Only scan for sensitive KV pairs if one of the keywords is actually
       # present — saves two regex passes on the vast majority of queries.
@@ -419,6 +423,10 @@ module DeadBro
       return false if stripped.match?(/\AWITH\b/i) && stripped.match?(EXPLAIN_DML_KEYWORD_RE)
 
       true
+    rescue
+      # e.g. invalid UTF-8 in the SQL, which strip and match? raise on — when in
+      # doubt, don't run anything against the customer's database.
+      false
     end
 
     def self.start_explain_background(sql, connection_id, query_info, binds = nil)
@@ -494,7 +502,7 @@ module DeadBro
       timestamp = Time.now.utc
       log_entry = {
         sev: severity.to_s,
-        msg: message.to_s,
+        msg: DeadBro::Sanitizer.string(message),
         time: timestamp.iso8601(3)
       }
 
@@ -778,7 +786,7 @@ module DeadBro
       # dominating CPU on N+1-heavy requests (100s of full Thread#backtrace
       # allocations). The main subscriber now captures a trimmed backtrace
       # lazily — and only when a query exceeds slow_query_threshold_ms.
-    rescue
+    rescue *DeadBro::CONTAINED_ERRORS
     end
 
     def finish(name, id, payload)
@@ -796,7 +804,7 @@ module DeadBro
       delta = end_count - start_count
       results = (Thread.current[DeadBro::SqlSubscriber::THREAD_LOCAL_ALLOC_RESULTS_KEY] ||= {})
       results[id] = delta
-    rescue
+    rescue *DeadBro::CONTAINED_ERRORS
     end
   end
 end

data/lib/dead_bro/sql_tracking_middleware.rb

@@ -9,6 +9,18 @@ module DeadBro
     def call(env)
       return @app.call(env) if DeadBro.configuration.skip_tracking?
 
+      start_tracking(env)
+      @app.call(env)
+    ensure
+      stop_tracking
+    end
+
+    private
+
+    # Setup and teardown both run inside the host's request: if either raised,
+    # the host would get a 500 (or, from teardown, have its own response or
+    # exception replaced by ours). Tracking is best-effort, the request is not.
+    def start_tracking(env)
       # Capture rack entry time before any setup so middleware overhead is accurately measured.
       rack_entry = Time.now
       Thread.current[DeadBro::TRACKING_START_TIME_KEY] = rack_entry
@@ -87,9 +99,10 @@ module DeadBro
 
       # Start outgoing HTTP accumulation for this request
       Thread.current[:dead_bro_http_events] = []
+    rescue *DeadBro::CONTAINED_ERRORS
+    end
 
-      @app.call(env)
-    ensure
+    def stop_tracking
       # Clean up thread-local storage
       if defined?(DeadBro::SqlSubscriber)
         Thread.current[:dead_bro_sql_queries]
@@ -117,11 +130,16 @@ module DeadBro
         Thread.current[:dead_bro_lightweight_memory] = nil
       end
 
-      # Clean up HTTP events, ES events, DB connection tracking, and tracking start time
+      # Clean up HTTP events, ES events, DB connection tracking, and tracking start time.
+      # The calls below rescue on their own: teardown has to finish, so one
+      # failing step must not skip the ones after it.
       Thread.current[:dead_bro_elasticsearch_events] = nil
       Thread.current[:dead_bro_http_events] = nil
       Thread.current[:dead_bro_queue_duration_ms] = nil
-      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
+      begin
+        DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
+      rescue *DeadBro::CONTAINED_ERRORS
+      end
       Thread.current[DeadBro::GcTracker::THREAD_KEY] = nil if defined?(DeadBro::GcTracker)
       # Bypass stop_request_tracking intentionally — cleanup only, no return value needed here.
       Thread.current[DeadBro::ArObjectTracker::THREAD_KEY] = nil if defined?(DeadBro::ArObjectTracker)
@@ -130,14 +148,18 @@ module DeadBro
       # Safety net: ensure allocation tracing is never left running across
       # requests (Subscriber normally stops it after analyzing).
       if Thread.current[:dead_bro_alloc_active]
-        DeadBro::AllocationSourceSampler.stop if defined?(DeadBro::AllocationSourceSampler)
+        begin
+          DeadBro::AllocationSourceSampler.stop if defined?(DeadBro::AllocationSourceSampler)
+        rescue *DeadBro::CONTAINED_ERRORS
+        end
       end
+    rescue *DeadBro::CONTAINED_ERRORS
+      # Backstop for anything unforeseen above.
+    ensure
       Thread.current[:dead_bro_alloc_active] = nil
       Thread.current[DeadBro::TRACKING_START_TIME_KEY] = nil
     end
 
-    private
-
     def parse_queue_start(env, rack_entry)
       raw = env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"]
       return nil if raw.nil? || raw.empty?

data/lib/dead_bro/subscriber.rb

@@ -189,7 +189,7 @@ module DeadBro
           begin
             exception_class, exception_message = data[:exception] if data[:exception]
             exception_obj = data[:exception_object]
-            backtrace = Array(exception_obj&.backtrace).first(50)
+            backtrace = Array(exception_obj&.backtrace).first(50).map { |line| sanitize_string(line) }
 
             error_payload = detail_fields.merge(
               controller: data[:controller],
@@ -207,7 +207,7 @@ module DeadBro
               user_agent: safe_user_agent(data),
               user_id: extract_user_id(data),
               exception_class: exception_class || exception_obj&.class&.name,
-              message: (exception_message || exception_obj&.message).to_s[0, 1000],
+              message: sanitize_string((exception_message || exception_obj&.message).to_s[0, 1000]),
               backtrace: backtrace,
               fingerprint: compute_error_fingerprint(exception_obj),
               cause_chain: build_cause_chain(exception_obj),
@@ -243,6 +243,10 @@ module DeadBro
         # already made above; client#post_metric must not re-roll it with the
         # global-only rate, which would silently override a per-type sample rate.
         client.post_metric(event_name: name, payload: payload, force: true)
+      rescue *DeadBro::CONTAINED_ERRORS
+        # Anything raised here would fail the host's request (see
+        # DeadBro::CONTAINED_ERRORS) — drop this request's metrics instead.
+        drain_request_tracking
       end
     end
 
@@ -309,7 +313,7 @@ module DeadBro
     end
 
     def self.sanitize_string(str)
-      str.to_s.gsub("\x00", "")
+      DeadBro::Sanitizer.string(str)
     end
 
     # Matched against a key segment (delimited by start/end, underscore, dash, or
@@ -336,6 +340,9 @@ module DeadBro
       when Hash
         entries = value.to_a[0, max_hash_keys]
         entries.each_with_object({}) do |(k, v), memo|
+          # Scrubbed first: the sensitive-key regex raises on a key that isn't
+          # valid UTF-8, which would drop the whole params hash.
+          k = sanitize_string(k) if k.is_a?(String)
           memo[k] = sensitive_key?(k) ? "[FILTERED]" : truncate_value(v, max_str: max_str, max_array: max_array, max_hash_keys: max_hash_keys)
         end
       else
@@ -497,7 +504,7 @@ module DeadBro
     end
 
     def self.normalize_error_message(msg)
-      msg.to_s
+      sanitize_string(msg)
         .gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i, "UUID")
         .gsub(/\b\d+\b/, "N")
         .gsub(/"[^"]*"/, '"?"')
@@ -507,7 +514,7 @@ module DeadBro
 
     def self.compute_error_fingerprint(exception)
       return nil unless exception
-      top_frame = Array(exception.backtrace).first.to_s.gsub(/:\d+:in /, ":N:in ")
+      top_frame = sanitize_string(Array(exception.backtrace).first).gsub(/:\d+:in /, ":N:in ")
       input = "#{exception.class.name}|#{normalize_error_message(exception.message)}|#{top_frame}"
       Digest::SHA256.hexdigest(input)[0, 16]
     rescue
@@ -522,8 +529,8 @@ module DeadBro
       while cause && depth < 5
         chain << {
           exception_class: cause.class.name,
-          message: cause.message.to_s[0, 500],
-          backtrace_top: Array(cause.backtrace).first(3)
+          message: sanitize_string(cause.message.to_s[0, 500]),
+          backtrace_top: Array(cause.backtrace).first(3).map { |line| sanitize_string(line) }
         }
         cause = cause.cause
         depth += 1

data/lib/dead_bro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadBro
-  VERSION = "0.2.33"
+  VERSION = "0.2.34"
 end

data/lib/dead_bro/view_rendering_subscriber.rb

@@ -19,6 +19,7 @@ module DeadBro
                        duration_ms: ((finished - started) * 1000.0).round(2),
                        rendered_at: Time.now.utc.to_i,
                        start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
+      rescue *DeadBro::CONTAINED_ERRORS
       end
 
       ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_EVENT) do |_name, started, finished, _uid, data|
@@ -28,6 +29,7 @@ module DeadBro
                        cache_key: data[:cache_key],
                        rendered_at: Time.now.utc.to_i,
                        start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
+      rescue *DeadBro::CONTAINED_ERRORS
       end
 
       ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_EVENT) do |_name, started, finished, _uid, data|
@@ -38,6 +40,7 @@ module DeadBro
                        collection_cached_count: (data[:cached_count] || 0).to_i,
                        rendered_at: Time.now.utc.to_i,
                        start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
+      rescue *DeadBro::CONTAINED_ERRORS
       end
     rescue
     end

data/lib/dead_bro.rb

@@ -30,6 +30,7 @@ module DeadBro
   autoload :MemoryDetails, "dead_bro/memory_details"
   autoload :Logger, "dead_bro/logger"
   autoload :WatchTracker, "dead_bro/watch_tracker"
+  autoload :Sanitizer, "dead_bro/sanitizer"
   begin
     require "dead_bro/railtie"
   rescue LoadError
@@ -37,6 +38,23 @@ module DeadBro
 
   class Error < StandardError; end
 
+  # What DeadBro code running inside the host's request or job — notification
+  # callbacks, middleware — rescues so it can never fail the host. Rails re-raises
+  # a notification listener's exception into the instrumented block, so a raise
+  # in a process_action callback turns the host's 200 into a 500.
+  #
+  # Deliberately not Exception: an asynchronous interrupt that happens to land
+  # while our code runs (Timeout, Rack::Timeout, Sidekiq::Shutdown, Interrupt)
+  # must still reach the host, or its timeout/shutdown handling silently stops
+  # working. These are the errors our own code can raise synchronously.
+  #
+  # One gap remains: Timeout.timeout(sec, SomeStandardError) delivers a
+  # StandardError, so it is swallowed if it fires mid-callback — as it would be
+  # by any `rescue => e` in the host's own code, on every Ruby. Plain
+  # Timeout.timeout(sec) is not affected. Rescuing Exception wouldn't close that
+  # gap; it would only open the ones above.
+  CONTAINED_ERRORS = [StandardError, ScriptError, SystemStackError].freeze
+
   # Returned by DeadBro.analyze. sql_queries is intentionally omitted from
   # inspect/to_s/pretty_print to avoid bloating console output.
   # Access it explicitly via .sql_queries.
@@ -338,6 +356,7 @@ module DeadBro
             end
 
             local_sql_queries << {duration_ms: duration_ms, sql: normalized_sql, query_type: query_type}
+          rescue *DeadBro::CONTAINED_ERRORS
           end
       end
     rescue

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dead_bro
 version: !ruby/object:Gem::Version
-  version: 0.2.33
+  version: 0.2.34
 platform: ruby
 authors:
 - Emanuel Comsa
@@ -54,6 +54,7 @@ files:
 - lib/dead_bro/monitor.rb
 - lib/dead_bro/railtie.rb
 - lib/dead_bro/redis_subscriber.rb
+- lib/dead_bro/sanitizer.rb
 - lib/dead_bro/sql_subscriber.rb
 - lib/dead_bro/sql_tracking_middleware.rb
 - lib/dead_bro/subscriber.rb
@@ -81,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.22
 specification_version: 4
 summary: Minimal APM for Rails apps.
 test_files: []