dead_bro 0.2.13 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c308968cd84cfa8df47974178b741ab2847809754e86dc49cbe716d07a7a0bb
-  data.tar.gz: 6cdfb10f1ac9b2ecd9994bf6f35d8468022dc024e16e9ab31c1664fa91db7041
+  metadata.gz: 2251b88befeee4d2b7668bc504cc99d94cd154dd4539258545c1084e7a388709
+  data.tar.gz: 10141477e9697b91d95290ccfc695abb304e8dfbdfb526f95646269be03be0ef
 SHA512:
-  metadata.gz: 2a8f31655b9512739fa991ee94675c1f948c5ae2229dbb9de62a4e3c6ea0b5a8883ea880f986d911044e200d0c8ae6e696f6d8a58e5c338da6c8c3756c08e543
-  data.tar.gz: ddf1ed72e8310eb1f42db17f5124b704c5214447706ed68c1f9bae9f47f669fbf56006733f430a064af0fc8965be161dbb85bcaca2aee6b6c14a68b4865216f9
+  metadata.gz: 39ec9243a8792016f7cf9bf5f8d81856c121365b5cebef91ef7d318f642e7480a66b32a92b63ec944ee3aebb9e92ffdfc27c9e284e41da7daa081661e893b38a
+  data.tar.gz: 519f6df31fb8dc567a6974052e58787098021078514eaa2877a2f582651c5a5fec316f4da58a625a4b6d00eed7fae614fe5dd79f1da17ae2d4d7227ce8f7061b

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.2.14] - 2026-05-23
+
+### Added
+- `DbConnectionSubscriber`: measures the time threads spend waiting to acquire a connection from the ActiveRecord connection pool and counts checkouts per request. Uses `prepend` on `ActiveRecord::ConnectionAdapters::ConnectionPool#checkout` so the overhead is minimal and the instrumentation is invisible to application code. `db_connection_wait_ms` and `db_connection_checkouts` are included in both request and job payloads.
+- Queue duration tracking for web requests via `X-Request-Start` / `X-Queue-Start` headers. Parses both the Heroku microsecond format (`t=<µs>`) and the nginx seconds format (`t=<s.ms>`), and applies a 60 s clock-skew cap so a misconfigured proxy timestamp cannot produce a nonsensical value.
+- Queue duration tracking for background jobs: time from `job.enqueued_at` to when `perform` begins is reported as `queue_duration_ms` in every job payload.
+- `RedisSubscriber`: prepend-based instrumentation on `Redis::Client` that records every individual command, pipeline, and `MULTI`/`EXEC` block with command name, key, duration in ms, and database index. Falls back to an `ActiveSupport::Notifications` subscription for `redis.*` events emitted by other libraries. Capped at 1 000 events per request to bound memory growth.
+- DB connection tracking wired into background jobs: `DbConnectionSubscriber` is started and stopped around job execution in both the normal completion path and the exception handler fallback path, with cleanup in `drain_job_tracking`.
+
 ## [0.1.0] - 2025-08-28
 
 - Initial release

data/lib/dead_bro/db_connection_subscriber.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module DeadBro
+  module DbConnectionSubscriber
+    WAIT_KEY  = :dead_bro_db_connection_wait_ms
+    COUNT_KEY = :dead_bro_db_connection_checkouts
+
+    # Prepended onto ConnectionPool so every checkout is timed.
+    # Only accumulates when a request is being tracked (thread-local is a Numeric).
+    module CheckoutInstrumentation
+      def checkout(*args)
+        return super unless Thread.current[DbConnectionSubscriber::WAIT_KEY].is_a?(Numeric)
+
+        # Initialize conn before calling super so the rescue block can tell whether
+        # checkout succeeded before timing code raised (avoids double-checkout).
+        conn = nil
+        t0   = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        conn = super
+        Thread.current[DbConnectionSubscriber::WAIT_KEY]  += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000.0
+        Thread.current[DbConnectionSubscriber::COUNT_KEY] += 1
+        conn
+      rescue
+        conn || super
+      end
+    end
+
+    def self.install!
+      return unless defined?(ActiveRecord::ConnectionAdapters::ConnectionPool)
+      return if ActiveRecord::ConnectionAdapters::ConnectionPool.ancestors.include?(CheckoutInstrumentation)
+
+      ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(CheckoutInstrumentation)
+    rescue StandardError => e
+      warn "[DeadBro] DbConnectionSubscriber install failed: #{e.class}: #{e.message}"
+    end
+
+    def self.start_request_tracking
+      Thread.current[WAIT_KEY]  = 0.0
+      Thread.current[COUNT_KEY] = 0
+    end
+
+    def self.stop_request_tracking
+      wait_ms   = Thread.current[WAIT_KEY]
+      checkouts = Thread.current[COUNT_KEY]
+      Thread.current[WAIT_KEY]  = nil
+      Thread.current[COUNT_KEY] = nil
+      { wait_ms: wait_ms&.round(2), checkouts: checkouts }
+    end
+  end
+end

data/lib/dead_bro/gc_tracker.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module DeadBro
+  module GcTracker
+    THREAD_KEY = :dead_bro_gc_start
+
+    def self.start_request_tracking
+      Thread.current[THREAD_KEY] = snapshot
+    end
+
+    def self.stop_request_tracking
+      before = Thread.current[THREAD_KEY]
+      return {} if before.nil? || before.empty?
+      diff(before, snapshot)
+    ensure
+      Thread.current[THREAD_KEY] = nil
+    end
+
+    def self.snapshot
+      return {} unless defined?(GC) && GC.respond_to?(:stat)
+      stat = GC.stat
+      {
+        minor_gc_count: stat[:minor_gc_count] || 0,
+        major_gc_count: stat[:major_gc_count] || 0,
+        total_allocated_objects: stat[:total_allocated_objects] || 0,
+        gc_time_ns: GC.respond_to?(:total_time) ? GC.total_time : nil
+      }
+    rescue
+      {}
+    end
+
+    def self.diff(before, after)
+      return {} if before.empty? || after.empty?
+      gc_time_ms = if before[:gc_time_ns] && after[:gc_time_ns]
+        ((after[:gc_time_ns] - before[:gc_time_ns]) / 1_000_000.0).round(3)
+      end
+      {
+        minor_gc_runs: (after[:minor_gc_count] || 0) - (before[:minor_gc_count] || 0),
+        major_gc_runs: (after[:major_gc_count] || 0) - (before[:major_gc_count] || 0),
+        allocated_objects: (after[:total_allocated_objects] || 0) - (before[:total_allocated_objects] || 0),
+        gc_time_ms: gc_time_ms
+      }
+    rescue
+      {}
+    end
+  end
+end

data/lib/dead_bro/job_sql_tracking_middleware.rb

@@ -24,6 +24,11 @@ module DeadBro
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
           DeadBro::MemoryTrackingSubscriber.start_request_tracking
         end
+
+        # Start DB connection pool wait tracking
+        if defined?(DeadBro::DbConnectionSubscriber)
+          DeadBro::DbConnectionSubscriber.start_request_tracking
+        end
       end
     rescue
       # Never raise from instrumentation install

data/lib/dead_bro/job_subscriber.rb

@@ -12,6 +12,12 @@ module DeadBro
     JOB_EXCEPTION_EVENT_NAME = "exception.active_job"
 
     def self.subscribe!(client: Client.new)
+      # Snap GC state before the job runs so stop_request_tracking gets a valid diff
+      ActiveSupport::Notifications.subscribe("perform_start.active_job") do |_name, _started, _finished, _unique_id, _data|
+        DeadBro::GcTracker.start_request_tracking if defined?(DeadBro::GcTracker)
+      rescue
+      end
+
       # Track job execution
       ActiveSupport::Notifications.subscribe(JOB_EVENT_NAME) do |name, started, finished, _unique_id, data|
         begin
@@ -43,6 +49,7 @@ module DeadBro
         end
 
         duration_ms = ((finished - started) * 1000.0).round(2)
+        queue_duration_ms = job_queue_duration_ms(data[:job], started)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
         # This handles job backends that don't emit perform_start events
@@ -50,6 +57,7 @@ module DeadBro
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          DeadBro::DbConnectionSubscriber.start_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
           if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
             DeadBro::MemoryTrackingSubscriber.start_request_tracking
           else
@@ -59,6 +67,8 @@ module DeadBro
 
         # Get SQL queries executed during this job
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
+        gc_pressure = defined?(DeadBro::GcTracker) ? DeadBro::GcTracker.stop_request_tracking : {}
 
         # Stop memory tracking and get collected memory data
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
@@ -94,6 +104,10 @@ module DeadBro
           queue_name: data[:job].queue_name,
           arguments: safe_arguments(data[:job].arguments),
           duration_ms: duration_ms,
+          queue_duration_ms: queue_duration_ms,
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
+          gc_pressure: gc_pressure,
           status: "completed",
           sql_queries: sql_queries,
           rails_env: DeadBro.env,
@@ -129,13 +143,14 @@ module DeadBro
 
         duration_ms = ((finished - started) * 1000.0).round(2)
         exception = data[:exception_object]
-        data[:job].class.name
+        queue_duration_ms = job_queue_duration_ms(data[:job], started)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
         unless DeadBro::SqlSubscriber.tracking_active?
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          DeadBro::DbConnectionSubscriber.start_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
           if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
             DeadBro::MemoryTrackingSubscriber.start_request_tracking
           else
@@ -145,6 +160,8 @@ module DeadBro
 
         # Get SQL queries executed during this job
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
+        gc_pressure = defined?(DeadBro::GcTracker) ? DeadBro::GcTracker.stop_request_tracking : {}
 
         # Stop memory tracking and get collected memory data
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
@@ -180,6 +197,10 @@ module DeadBro
           queue_name: data[:job].queue_name,
           arguments: safe_arguments(data[:job].arguments),
           duration_ms: duration_ms,
+          queue_duration_ms: queue_duration_ms,
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
+          gc_pressure: gc_pressure,
           status: "failed",
           sql_queries: sql_queries,
           exception_class: exception&.class&.name,
@@ -205,6 +226,8 @@ module DeadBro
     # build a payload (excluded job / sampled out). Matches Subscriber.drain_request_tracking.
     def self.drain_job_tracking
       DeadBro::SqlSubscriber.stop_request_tracking if defined?(DeadBro::SqlSubscriber)
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
+      DeadBro::GcTracker.stop_request_tracking if defined?(DeadBro::GcTracker)
       DeadBro::LightweightMemoryTracker.stop_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
       if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
         DeadBro::MemoryTrackingSubscriber.stop_request_tracking
@@ -215,6 +238,17 @@ module DeadBro
 
     private
 
+    def self.job_queue_duration_ms(job, perform_started)
+      enqueued_at = job.enqueued_at
+      return nil if enqueued_at.nil?
+
+      enqueued_time = enqueued_at.is_a?(Time) ? enqueued_at : Time.parse(enqueued_at.to_s)
+      diff_ms = ((perform_started - enqueued_time) * 1000.0).round(2)
+      diff_ms >= 0 ? diff_ms : nil
+    rescue
+      nil
+    end
+
     def self.safe_arguments(arguments)
       return [] unless arguments.is_a?(Array)
 

data/lib/dead_bro/railtie.rb

@@ -36,6 +36,10 @@ if defined?(Rails) && defined?(Rails::Railtie)
           require "dead_bro/elasticsearch_subscriber"
           DeadBro::ElasticsearchSubscriber.subscribe!
 
+          # Install DB connection pool wait tracking
+          require "dead_bro/db_connection_subscriber"
+          DeadBro::DbConnectionSubscriber.install!
+
           # Install view rendering tracking
           require "dead_bro/view_rendering_subscriber"
           DeadBro::ViewRenderingSubscriber.subscribe!(client: shared_client)

data/lib/dead_bro/sql_tracking_middleware.rb

@@ -9,6 +9,15 @@ module DeadBro
     def call(env)
       return @app.call(env) if DeadBro.configuration.skip_tracking?
 
+      # Capture rack entry time before any setup so middleware overhead is accurately measured.
+      rack_entry = Time.now
+      Thread.current[DeadBro::TRACKING_START_TIME_KEY] = rack_entry
+
+      # Queue time: gap between when the upstream proxy accepted the connection and when a Rack
+      # worker picked it up. Heroku sets X-Request-Start as "t=<microseconds>"; nginx typically
+      # uses "t=<seconds.ms>". Both are parsed below.
+      Thread.current[:dead_bro_queue_duration_ms] = parse_queue_start(env, rack_entry)
+
       # Clear logs for this request
       DeadBro.logger.clear
 
@@ -47,12 +56,17 @@ module DeadBro
         DeadBro::ElasticsearchSubscriber.start_request_tracking
       end
 
+      # Start DB connection pool wait tracking
+      if defined?(DeadBro::DbConnectionSubscriber)
+        DeadBro::DbConnectionSubscriber.start_request_tracking
+      end
+
+      # Start GC pressure tracking — snapshot before any app code runs
+      DeadBro::GcTracker.start_request_tracking if defined?(DeadBro::GcTracker)
+
       # Start outgoing HTTP accumulation for this request
       Thread.current[:dead_bro_http_events] = []
 
-      # Set tracking start time once for all subscribers (before starting any tracking)
-      Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
-
       @app.call(env)
     ensure
       # Clean up thread-local storage
@@ -81,10 +95,41 @@ module DeadBro
         Thread.current[:dead_bro_lightweight_memory] = nil
       end
 
-      # Clean up HTTP events, ES events, and tracking start time
+      # Clean up HTTP events, ES events, DB connection tracking, and tracking start time
       Thread.current[:dead_bro_elasticsearch_events] = nil
       Thread.current[:dead_bro_http_events] = nil
+      Thread.current[:dead_bro_queue_duration_ms] = nil
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
+      Thread.current[DeadBro::GcTracker::THREAD_KEY] = nil if defined?(DeadBro::GcTracker)
       Thread.current[DeadBro::TRACKING_START_TIME_KEY] = nil
     end
+
+    private
+
+    def parse_queue_start(env, rack_entry)
+      raw = env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"]
+      return nil if raw.nil? || raw.empty?
+
+      # Strip "t=" prefix used by Heroku and nginx
+      raw = raw.sub(/\At=/, "")
+      num = raw.to_f
+      return nil if num <= 0
+
+      request_start =
+        if num > 1_000_000_000_000_000 # microseconds (Heroku)
+          Time.at(num / 1_000_000.0)
+        elsif num > 1_000_000_000_000 # milliseconds
+          Time.at(num / 1_000.0)
+        else # seconds (nginx)
+          Time.at(num)
+        end
+
+      # Guard against clocks being out of sync or wildly misconfigured proxy timestamps.
+      # Cap at 60 s — anything larger almost certainly means the header value is wrong.
+      diff_ms = ((rack_entry - request_start) * 1000.0).round(2)
+      diff_ms >= 0 && diff_ms <= 60_000 ? diff_ms : nil
+    rescue
+      nil
+    end
   end
 end

data/lib/dead_bro/subscriber.rb

@@ -49,6 +49,11 @@ module DeadBro
         end
 
         duration_ms = ((finished - started) * 1000.0).round(2)
+
+        # Time spent in Rack middleware before ActionController took over (routing, session, auth, etc.)
+        rack_start = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
+        rack_duration_ms = rack_start ? ([((started - rack_start) * 1000.0), 0].max).round(2) : nil
+
         # Stop SQL tracking and get collected queries (this was started by the request)
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
 
@@ -57,6 +62,12 @@ module DeadBro
         redis_events = defined?(DeadBro::RedisSubscriber) ? DeadBro::RedisSubscriber.stop_request_tracking : []
         elasticsearch_events = defined?(DeadBro::ElasticsearchSubscriber) ? DeadBro::ElasticsearchSubscriber.stop_request_tracking : []
 
+        # Stop DB connection pool wait tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
+
+        # Stop GC pressure tracking — diff minor/major runs, objects allocated, GC time
+        gc_pressure = defined?(DeadBro::GcTracker) ? DeadBro::GcTracker.stop_request_tracking : {}
+
         # Stop view rendering tracking and get collected view events
         view_events = DeadBro::ViewRenderingSubscriber.stop_request_tracking
         view_performance = DeadBro::ViewRenderingSubscriber.analyze_view_performance(view_events)
@@ -166,6 +177,11 @@ module DeadBro
           view_performance: view_performance,
           memory_events: memory_events,
           memory_performance: memory_performance,
+          rack_duration_ms: rack_duration_ms,
+          queue_duration_ms: Thread.current[:dead_bro_queue_duration_ms],
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
+          gc_pressure: gc_pressure,
           logs: DeadBro.logger.logs
         }
         client.post_metric(event_name: name, payload: payload)
@@ -186,6 +202,8 @@ module DeadBro
         DeadBro::MemoryTrackingSubscriber.stop_request_tracking
       end
       Thread.current[:dead_bro_http_events] = nil
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
+      DeadBro::GcTracker.stop_request_tracking if defined?(DeadBro::GcTracker)
     rescue
       # Best effort — draining must never raise from the notifications callback.
     end

data/lib/dead_bro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadBro
-  VERSION = "0.2.13"
+  VERSION = "0.2.15"
 end

data/lib/dead_bro.rb

@@ -18,6 +18,7 @@ module DeadBro
   autoload :MemoryTrackingSubscriber, "dead_bro/memory_tracking_subscriber"
   autoload :MemoryLeakDetector, "dead_bro/memory_leak_detector"
   autoload :LightweightMemoryTracker, "dead_bro/lightweight_memory_tracker"
+  autoload :GcTracker, "dead_bro/gc_tracker"
   autoload :MemoryHelpers, "dead_bro/memory_helpers"
   autoload :JobSubscriber, "dead_bro/job_subscriber"
   autoload :JobSqlTrackingMiddleware, "dead_bro/job_sql_tracking_middleware"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dead_bro
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.15
 platform: ruby
 authors:
 - Emanuel Comsa
@@ -33,9 +33,11 @@ files:
 - lib/dead_bro/collectors/sample_store.rb
 - lib/dead_bro/collectors/system.rb
 - lib/dead_bro/configuration.rb
+- lib/dead_bro/db_connection_subscriber.rb
 - lib/dead_bro/dispatcher.rb
 - lib/dead_bro/elasticsearch_subscriber.rb
 - lib/dead_bro/error_middleware.rb
+- lib/dead_bro/gc_tracker.rb
 - lib/dead_bro/http_instrumentation.rb
 - lib/dead_bro/job_sql_tracking_middleware.rb
 - lib/dead_bro/job_subscriber.rb