dead_bro 0.2.12 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56f6a72ccfa1a5e362c74eddace39a1ffeba97d06bab551f38d9091311609770
-  data.tar.gz: 20ee34708c5eed8ac00af8d462953422ba7a24e3b9701e56e31eea8ba35752f0
+  metadata.gz: 3f7618eda689dd41bbb5c71cdff40619e5857a71b31a182f234064d89197d79c
+  data.tar.gz: fbcac3f8c189834690a5a75e013f07cec20044f4e74285139aa22f66ae0cf65d
 SHA512:
-  metadata.gz: 932ce6601969ac9f95ba0ce327b93a5412c167a4c7fe367d0c14bd05c5c751b69bb3da48eaf829e73bcf6ca10c66fd7678ff0caca4eba7e0ec42a3b4bae72e54
-  data.tar.gz: 032ee4230c5b043a7ef8cf176963500eb60157927ad18f18c8e657f2439b31ab8b3526e7dd90036f79ec5fc7e978b55de540fcad14925f1fb33ffb6308379964
+  metadata.gz: 68864135e9fd51e65293ee369f1002e6f37341627fd674d570781fa163dd5b2345a4e7421356f49bc2bc00c1b8f37946b7fe9d31ff0370a6781f892392c7ec5d
+  data.tar.gz: 5e72422ba6297af9ef9a521e675d832dd94a0f81783ae964be53ecdd16eaa96dd4bc55448baceb697ea1b0b4bfd024090009d43dd87a3893f197df607cc08fd4

data/lib/dead_bro/client.rb

@@ -15,11 +15,12 @@ module DeadBro
     def post_metric(event_name:, payload:, force: false)
       return if @configuration.api_key.nil?
       return unless @configuration.enabled
+      return if @configuration.skip_tracking?
       return if !force && !@configuration.should_sample?
       return if circuit_open?
 
       payload = truncate_payload_for_request(payload)
-      body = {event: event_name, payload: payload, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id}
+      body = {event: event_name, payload: payload, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id, gem_version: DeadBro::VERSION}
 
       dispatch_request(
         url: metrics_endpoint_url,
@@ -35,7 +36,7 @@ module DeadBro
       return if @configuration.api_key.nil?
 
       @configuration.last_heartbeat_attempt_at = Time.now.utc
-      body = {event: "heartbeat", payload: {}, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id}
+      body = {event: "heartbeat", payload: {}, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id, gem_version: DeadBro::VERSION}
 
       dispatch_request(
         url: metrics_endpoint_url,
@@ -50,10 +51,11 @@ module DeadBro
     def post_monitor_stats(payload)
       return if @configuration.api_key.nil?
       return unless @configuration.enabled
+      return if @configuration.skip_tracking?
       return unless @configuration.job_queue_monitoring_enabled
       return if circuit_open?
 
-      body = {payload: payload, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id}
+      body = {payload: payload, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id, gem_version: DeadBro::VERSION}
 
       dispatch_request(
         url: monitor_endpoint_url,
@@ -140,6 +142,12 @@ module DeadBro
             @configuration.last_heartbeat_at = Time.now.utc
           end
         end
+
+        if response.is_a?(Net::HTTPSuccess)
+          @configuration.skip_until = nil
+        elsif response.is_a?(Net::HTTPInsufficientStorage)
+          @configuration.skip_until = Time.now.utc + DeadBro::Configuration::METRICS_BACKEND_SKIP_AFTER_507_SECONDS
+        end
       elsif @circuit_breaker && @configuration.circuit_breaker_enabled
         @circuit_breaker.record_failure
       end

data/lib/dead_bro/configuration.rb

@@ -22,6 +22,10 @@ module DeadBro
     # Tracks when we last received settings from the backend (in-memory only)
     attr_accessor :settings_received_at
 
+    # After HTTP 507 Insufficient Storage from the API, skip all tracking until this
+    # UTC time (in-memory only). Cleared on the next successful API response.
+    attr_accessor :skip_until
+
     # Last successful heartbeat HTTP response time while disabled (in-memory only)
     attr_accessor :last_heartbeat_at
 
@@ -30,6 +34,8 @@ module DeadBro
 
     HEARTBEAT_INTERVAL = 60 # seconds
 
+    METRICS_BACKEND_SKIP_AFTER_507_SECONDS = 600 # 10 minutes
+
     # First non-empty ENV value wins for release/revision payloads and deploy grouping on the server.
     # Order is roughly: DeadBro-native → common CI/hosting → observability tooling.
     DEPLOY_REVISION_ENV_KEYS = %w[
@@ -87,6 +93,7 @@ module DeadBro
       @enable_system_stats = false
 
       @settings_received_at = nil
+      @skip_until = nil
       @last_heartbeat_at = nil
       @last_heartbeat_attempt_at = nil
       @settings_mutex = Mutex.new
@@ -152,6 +159,13 @@ module DeadBro
       last_heartbeat_attempt_at.nil? || (Time.now.utc - last_heartbeat_attempt_at) >= HEARTBEAT_INTERVAL
     end
 
+    def skip_tracking?
+      t = skip_until
+      return false unless t
+
+      Time.now.utc < t
+    end
+
     def resolve_deploy_id
       explicit = @explicit_deploy_revision&.to_s&.strip
       return explicit unless explicit.nil? || explicit.empty?

data/lib/dead_bro/db_connection_subscriber.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module DeadBro
+  module DbConnectionSubscriber
+    WAIT_KEY  = :dead_bro_db_connection_wait_ms
+    COUNT_KEY = :dead_bro_db_connection_checkouts
+
+    # Prepended onto ConnectionPool so every checkout is timed.
+    # Only accumulates when a request is being tracked (thread-local is a Numeric).
+    module CheckoutInstrumentation
+      def checkout(*args)
+        return super unless Thread.current[DbConnectionSubscriber::WAIT_KEY].is_a?(Numeric)
+
+        # Initialize conn before calling super so the rescue block can tell whether
+        # checkout succeeded before timing code raised (avoids double-checkout).
+        conn = nil
+        t0   = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        conn = super
+        Thread.current[DbConnectionSubscriber::WAIT_KEY]  += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000.0
+        Thread.current[DbConnectionSubscriber::COUNT_KEY] += 1
+        conn
+      rescue
+        conn || super
+      end
+    end
+
+    def self.install!
+      return unless defined?(ActiveRecord::ConnectionAdapters::ConnectionPool)
+      return if ActiveRecord::ConnectionAdapters::ConnectionPool.ancestors.include?(CheckoutInstrumentation)
+
+      ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(CheckoutInstrumentation)
+    rescue StandardError => e
+      warn "[DeadBro] DbConnectionSubscriber install failed: #{e.class}: #{e.message}"
+    end
+
+    def self.start_request_tracking
+      Thread.current[WAIT_KEY]  = 0.0
+      Thread.current[COUNT_KEY] = 0
+    end
+
+    def self.stop_request_tracking
+      wait_ms   = Thread.current[WAIT_KEY]
+      checkouts = Thread.current[COUNT_KEY]
+      Thread.current[WAIT_KEY]  = nil
+      Thread.current[COUNT_KEY] = nil
+      { wait_ms: wait_ms&.round(2), checkouts: checkouts }
+    end
+  end
+end

data/lib/dead_bro/job_sql_tracking_middleware.rb

@@ -5,6 +5,8 @@ module DeadBro
     def self.subscribe!
       # Start SQL tracking when a job begins - use the start event, not the complete event
       ActiveSupport::Notifications.subscribe("perform_start.active_job") do |name, started, finished, _unique_id, data|
+        next if DeadBro.configuration.skip_tracking?
+
         # Clear logs for this job
         DeadBro.logger.clear
 
@@ -22,6 +24,11 @@ module DeadBro
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
           DeadBro::MemoryTrackingSubscriber.start_request_tracking
         end
+
+        # Start DB connection pool wait tracking
+        if defined?(DeadBro::DbConnectionSubscriber)
+          DeadBro::DbConnectionSubscriber.start_request_tracking
+        end
       end
     rescue
       # Never raise from instrumentation install

data/lib/dead_bro/job_subscriber.rb

@@ -15,6 +15,11 @@ module DeadBro
       # Track job execution
       ActiveSupport::Notifications.subscribe(JOB_EVENT_NAME) do |name, started, finished, _unique_id, data|
         begin
+          if DeadBro.configuration.skip_tracking?
+            drain_job_tracking
+            next
+          end
+
           job_class_name = data[:job].class.name
           if DeadBro.configuration.excluded_job?(job_class_name)
             drain_job_tracking
@@ -38,6 +43,7 @@ module DeadBro
         end
 
         duration_ms = ((finished - started) * 1000.0).round(2)
+        queue_duration_ms = job_queue_duration_ms(data[:job], started)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
         # This handles job backends that don't emit perform_start events
@@ -45,6 +51,7 @@ module DeadBro
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          DeadBro::DbConnectionSubscriber.start_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
           if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
             DeadBro::MemoryTrackingSubscriber.start_request_tracking
           else
@@ -54,6 +61,7 @@ module DeadBro
 
         # Get SQL queries executed during this job
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
 
         # Stop memory tracking and get collected memory data
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
@@ -89,6 +97,9 @@ module DeadBro
           queue_name: data[:job].queue_name,
           arguments: safe_arguments(data[:job].arguments),
           duration_ms: duration_ms,
+          queue_duration_ms: queue_duration_ms,
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
           status: "completed",
           sql_queries: sql_queries,
           rails_env: DeadBro.env,
@@ -106,6 +117,11 @@ module DeadBro
       # Track job exceptions
       ActiveSupport::Notifications.subscribe(JOB_EXCEPTION_EVENT_NAME) do |name, started, finished, _unique_id, data|
         begin
+          if DeadBro.configuration.skip_tracking?
+            drain_job_tracking
+            next
+          end
+
           job_class_name = data[:job].class.name
           if DeadBro.configuration.excluded_job?(job_class_name)
             next
@@ -119,13 +135,14 @@ module DeadBro
 
         duration_ms = ((finished - started) * 1000.0).round(2)
         exception = data[:exception_object]
-        data[:job].class.name
+        queue_duration_ms = job_queue_duration_ms(data[:job], started)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
         unless DeadBro::SqlSubscriber.tracking_active?
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          DeadBro::DbConnectionSubscriber.start_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
           if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
             DeadBro::MemoryTrackingSubscriber.start_request_tracking
           else
@@ -135,6 +152,7 @@ module DeadBro
 
         # Get SQL queries executed during this job
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
 
         # Stop memory tracking and get collected memory data
         if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
@@ -170,6 +188,9 @@ module DeadBro
           queue_name: data[:job].queue_name,
           arguments: safe_arguments(data[:job].arguments),
           duration_ms: duration_ms,
+          queue_duration_ms: queue_duration_ms,
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
           status: "failed",
           sql_queries: sql_queries,
           exception_class: exception&.class&.name,
@@ -195,6 +216,7 @@ module DeadBro
     # build a payload (excluded job / sampled out). Matches Subscriber.drain_request_tracking.
     def self.drain_job_tracking
       DeadBro::SqlSubscriber.stop_request_tracking if defined?(DeadBro::SqlSubscriber)
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
       DeadBro::LightweightMemoryTracker.stop_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
       if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
         DeadBro::MemoryTrackingSubscriber.stop_request_tracking
@@ -205,6 +227,17 @@ module DeadBro
 
     private
 
+    def self.job_queue_duration_ms(job, perform_started)
+      enqueued_at = job.enqueued_at
+      return nil if enqueued_at.nil?
+
+      enqueued_time = enqueued_at.is_a?(Time) ? enqueued_at : Time.parse(enqueued_at.to_s)
+      diff_ms = ((perform_started - enqueued_time) * 1000.0).round(2)
+      diff_ms >= 0 ? diff_ms : nil
+    rescue
+      nil
+    end
+
     def self.safe_arguments(arguments)
       return [] unless arguments.is_a?(Array)
 

data/lib/dead_bro/railtie.rb

@@ -36,6 +36,10 @@ if defined?(Rails) && defined?(Rails::Railtie)
           require "dead_bro/elasticsearch_subscriber"
           DeadBro::ElasticsearchSubscriber.subscribe!
 
+          # Install DB connection pool wait tracking
+          require "dead_bro/db_connection_subscriber"
+          DeadBro::DbConnectionSubscriber.install!
+
           # Install view rendering tracking
           require "dead_bro/view_rendering_subscriber"
           DeadBro::ViewRenderingSubscriber.subscribe!(client: shared_client)

data/lib/dead_bro/sql_tracking_middleware.rb

@@ -7,6 +7,17 @@ module DeadBro
     end
 
     def call(env)
+      return @app.call(env) if DeadBro.configuration.skip_tracking?
+
+      # Capture rack entry time before any setup so middleware overhead is accurately measured.
+      rack_entry = Time.now
+      Thread.current[DeadBro::TRACKING_START_TIME_KEY] = rack_entry
+
+      # Queue time: gap between when the upstream proxy accepted the connection and when a Rack
+      # worker picked it up. Heroku sets X-Request-Start as "t=<microseconds>"; nginx typically
+      # uses "t=<seconds.ms>". Both are parsed below.
+      Thread.current[:dead_bro_queue_duration_ms] = parse_queue_start(env, rack_entry)
+
       # Clear logs for this request
       DeadBro.logger.clear
 
@@ -45,12 +56,14 @@ module DeadBro
         DeadBro::ElasticsearchSubscriber.start_request_tracking
       end
 
+      # Start DB connection pool wait tracking
+      if defined?(DeadBro::DbConnectionSubscriber)
+        DeadBro::DbConnectionSubscriber.start_request_tracking
+      end
+
       # Start outgoing HTTP accumulation for this request
       Thread.current[:dead_bro_http_events] = []
 
-      # Set tracking start time once for all subscribers (before starting any tracking)
-      Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
-
       @app.call(env)
     ensure
       # Clean up thread-local storage
@@ -79,10 +92,40 @@ module DeadBro
         Thread.current[:dead_bro_lightweight_memory] = nil
       end
 
-      # Clean up HTTP events, ES events, and tracking start time
+      # Clean up HTTP events, ES events, DB connection tracking, and tracking start time
       Thread.current[:dead_bro_elasticsearch_events] = nil
       Thread.current[:dead_bro_http_events] = nil
+      Thread.current[:dead_bro_queue_duration_ms] = nil
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
       Thread.current[DeadBro::TRACKING_START_TIME_KEY] = nil
     end
+
+    private
+
+    def parse_queue_start(env, rack_entry)
+      raw = env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"]
+      return nil if raw.nil? || raw.empty?
+
+      # Strip "t=" prefix used by Heroku and nginx
+      raw = raw.sub(/\At=/, "")
+      num = raw.to_f
+      return nil if num <= 0
+
+      request_start =
+        if num > 1_000_000_000_000_000 # microseconds (Heroku)
+          Time.at(num / 1_000_000.0)
+        elsif num > 1_000_000_000_000 # milliseconds
+          Time.at(num / 1_000.0)
+        else # seconds (nginx)
+          Time.at(num)
+        end
+
+      # Guard against clocks being out of sync or wildly misconfigured proxy timestamps.
+      # Cap at 60 s — anything larger almost certainly means the header value is wrong.
+      diff_ms = ((rack_entry - request_start) * 1000.0).round(2)
+      diff_ms >= 0 && diff_ms <= 60_000 ? diff_ms : nil
+    rescue
+      nil
+    end
   end
 end

data/lib/dead_bro/subscriber.rb

@@ -16,6 +16,12 @@ module DeadBro
           next
         end
 
+        if DeadBro.configuration.skip_tracking?
+          client.post_heartbeat if DeadBro.configuration.heartbeat_due?
+          drain_request_tracking
+          next
+        end
+
         # Skip excluded controllers or controller#action pairs
         # Also check exclusive_controller_actions - if defined, only track those
         notification = data.is_a?(Hash) ? data : {}
@@ -43,6 +49,11 @@ module DeadBro
         end
 
         duration_ms = ((finished - started) * 1000.0).round(2)
+
+        # Time spent in Rack middleware before ActionController took over (routing, session, auth, etc.)
+        rack_start = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
+        rack_duration_ms = rack_start ? ([((started - rack_start) * 1000.0), 0].max).round(2) : nil
+
         # Stop SQL tracking and get collected queries (this was started by the request)
         sql_queries = DeadBro::SqlSubscriber.stop_request_tracking
 
@@ -51,6 +62,9 @@ module DeadBro
         redis_events = defined?(DeadBro::RedisSubscriber) ? DeadBro::RedisSubscriber.stop_request_tracking : []
         elasticsearch_events = defined?(DeadBro::ElasticsearchSubscriber) ? DeadBro::ElasticsearchSubscriber.stop_request_tracking : []
 
+        # Stop DB connection pool wait tracking
+        db_connection_stats = defined?(DeadBro::DbConnectionSubscriber) ? DeadBro::DbConnectionSubscriber.stop_request_tracking : {}
+
         # Stop view rendering tracking and get collected view events
         view_events = DeadBro::ViewRenderingSubscriber.stop_request_tracking
         view_performance = DeadBro::ViewRenderingSubscriber.analyze_view_performance(view_events)
@@ -160,6 +174,10 @@ module DeadBro
           view_performance: view_performance,
           memory_events: memory_events,
           memory_performance: memory_performance,
+          rack_duration_ms: rack_duration_ms,
+          queue_duration_ms: Thread.current[:dead_bro_queue_duration_ms],
+          db_connection_wait_ms: db_connection_stats[:wait_ms],
+          db_connection_checkouts: db_connection_stats[:checkouts],
           logs: DeadBro.logger.logs
         }
         client.post_metric(event_name: name, payload: payload)
@@ -180,6 +198,7 @@ module DeadBro
         DeadBro::MemoryTrackingSubscriber.stop_request_tracking
       end
       Thread.current[:dead_bro_http_events] = nil
+      DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
     rescue
       # Best effort — draining must never raise from the notifications callback.
     end

data/lib/dead_bro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadBro
-  VERSION = "0.2.12"
+  VERSION = "0.2.14"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dead_bro
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.14
 platform: ruby
 authors:
 - Emanuel Comsa
@@ -33,6 +33,7 @@ files:
 - lib/dead_bro/collectors/sample_store.rb
 - lib/dead_bro/collectors/system.rb
 - lib/dead_bro/configuration.rb
+- lib/dead_bro/db_connection_subscriber.rb
 - lib/dead_bro/dispatcher.rb
 - lib/dead_bro/elasticsearch_subscriber.rb
 - lib/dead_bro/error_middleware.rb