forge_ops_tracker 0.7.0 → 0.10.1

data/lib/forge_ops_tracker/integrations/net_http.rb

@@ -11,19 +11,101 @@ module ForgeOpsTracker
       # transaction_name is "<HTTP method> <host>", not the full URL: a request's own path or
       # query string could carry an id or a token, the same low-cardinality/no-secrets-in-a-label
       # reasoning every other transaction_name in this system already follows.
+      #
+      # Known, honest gap, confirmed directly against Net::HTTP's own source and real behavior, not
+      # assumed: Net::HTTP.get_response and the Net::HTTP.start { |http| ... } block form both open
+      # the TCP connection inside #start, *before* the block (and so before #request, the only
+      # method this wraps) ever runs, so a connection-refused/DNS-failure error raised during that
+      # setup is structurally invisible here, for timing, breadcrumbs, and the failure detection
+      # below alike (a pre-existing limitation this round's own test suite is what actually
+      # surfaced, not something newly introduced). Net::HTTP.new(...).request(...), the other
+      # common calling convention (no explicit #start at all), is different and unaffected:
+      # confirmed directly that Net::HTTP connects lazily, inside #request itself, the first time
+      # it's called on a not-yet-started instance, so that path's own connection failures are
+      # caught exactly like any other #request failure.
       module Timing
         # The guard (return super unless ...) skips timing, but the ensure block below still runs
         # on every exit path regardless, same as any ensure; start staying nil is what actually
         # skips recording there, not a second guard duplicating this one.
         def request(req, body = nil, &block)
-          return super unless ForgeOpsTracker.configuration.track_performance && ForgeOpsTracker.configuration.enabled?
+          configuration = ForgeOpsTracker.configuration
+          return super unless (configuration.track_performance || configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
 
           start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-          super
+          response = super
+        rescue StandardError => e
+          # Only track_failures below needs the actual exception object (for error_class/
+          # error_message); everything else keys off response staying nil, already true on this
+          # path without capturing anything. re-raised unchanged: this gem must never swallow or
+          # alter what the host app's own call to Net::HTTP#request would have raised.
+          raised_error = e
+          raise
         ensure
           if start
             duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
-            Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
+            # Computed once, unconditionally, not separately inside each flag-gated block below:
+            # both the breadcrumb and track_failures sections need it, and a value only ever
+            # assigned inside one conditional branch but read from another relies on Ruby's own
+            # "a local is declared the moment its assignment is lexically parsed, regardless of
+            # whether that branch actually ran" scoping, a real but easy-to-break-on-reorder
+            # subtlety not worth the risk here.
+            status = response&.code&.to_i
+            if configuration.track_performance
+              Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
+            end
+            # The same "method + host, never the full path/query string" reasoning
+            # transaction_name above already follows: a URL's own path or query could carry an id
+            # or a token, and this is a breadcrumb message, not a value this gem controls the
+            # cardinality of. Mirrors sdks/typescript's own fetch/xhr breadcrumbs: category
+            # "http", status + duration_ms in data, level "warning" for a non-2xx/3xx response.
+            if configuration.track_breadcrumbs
+              # No status at all means the request itself raised (a timeout, a connection
+              # refused, ...) before a response ever came back: worth "error", not the misleading
+              # "info" a bare status-based check would default a nil status to, the same "the
+              # request truly failed" case sdks/typescript's own fetch breadcrumb catch block
+              # marks "error" for.
+              level = if status.nil?
+                "error"
+              elsif status >= 400
+                "warning"
+              else
+                "info"
+              end
+              ForgeOpsTracker.add_breadcrumb(
+                "#{req.method} #{address}",
+                category: "http",
+                level: level,
+                data: { status: status, duration_ms: duration_ms.round(1) }.compact
+              )
+            end
+            # Independent of track_performance/track_breadcrumbs above, same "several genuinely
+            # independent mechanisms recorded from the same instrumentation point" pattern this
+            # exact call site already established for breadcrumbs. A failure is the request
+            # raising, or a 5xx: not a 4xx, the identical distinction the breadcrumb level just
+            # above already draws between "the dependency is unhealthy" (worth "error"/failure)
+            # and "the dependency responded, just not with success" (only "warning").
+            if configuration.track_failures && (status.nil? || status >= 500)
+              Railtie.failure_event_buffer.record(
+                kind: "dependency", transaction_name: "#{req.method} #{address}",
+                error_class: raised_error&.class&.name, error_message: raised_error&.message
+              )
+            end
+            # A leaf span, the same "parented onto whatever's currently open, never touches the
+            # stack itself" shape the sql.active_record subscription's own span recording already
+            # takes: an outbound HTTP call has no children of its own to nest anything under
+            # either.
+            if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+              # duration_ms itself is exact (computed from the monotonic clock above); started_at
+              # here is only derived from it (now minus duration), an approximation off the wall
+              # clock rather than a second real timestamp captured at the top of this method - a
+              # few milliseconds of drift on where the bar starts in a waterfall, never on how long
+              # it actually took.
+              start_time = ::Time.now.utc - (duration_ms / 1000.0)
+              buffer.record(
+                span_id: SecureRandom.hex(8), name: "#{req.method} #{address}", kind: "http",
+                started_at: start_time, duration_ms: duration_ms, data: { status: status }.compact
+              )
+            end
           end
         end
       end

data/lib/forge_ops_tracker/integrations/redis_client.rb

@@ -0,0 +1,62 @@
+module ForgeOpsTracker
+  module Integrations
+    module RedisClientTiming
+      # Times every Redis command, applied once via RedisClient.prepend(Timing) from Railtie, the
+      # same Module#prepend + super shape Integrations::NetHTTP::Timing already uses for the
+      # identical reason: neither Ruby's Redis ecosystem nor RedisClient itself fires an
+      # ActiveSupport::Notifications event of its own, so wrapping #call directly is the only hook
+      # available. Targets RedisClient specifically (the modern, actively-maintained low-level
+      # client both the current redis gem (5.x+) and standalone RedisClient users go through), not
+      # the older redis-rb 4.x Redis::Client - confirmed directly (not assumed) that redis 5.x no
+      # longer defines Redis::Client at all, so a hook there would silently do nothing on any
+      # currently-installed version of the gem.
+      #
+      # Unlike controller/query timing, this is span-only: there's no "redis" PerformanceSample
+      # kind for this to feed into the aggregate Performance page (that page has no Redis panel at
+      # all yet), so this only ever contributes to a captured trace's own waterfall, plus a
+      # breadcrumb and a dependency failure record, the same two other independent mechanisms
+      # NetHTTP::Timing's own call site already establishes.
+      #
+      # name is "Redis <COMMAND>" (the command verb alone, upcased: GET, SET, LPUSH, ...), never
+      # the key or any argument - the same low-cardinality, no-secrets-in-a-label reasoning every
+      # other transaction_name/span name in this gem already follows; a Redis key can easily carry
+      # a customer's own id or other sensitive value.
+      def call(*command, **kwargs)
+        configuration = ForgeOpsTracker.configuration
+        return super unless (configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
+
+        name = "Redis #{command.first.to_s.upcase}"
+        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        result = super
+      rescue StandardError => e
+        raised_error = e
+        raise
+      ensure
+        if start
+          duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
+
+          if configuration.track_breadcrumbs
+            ForgeOpsTracker.add_breadcrumb(
+              name, category: "redis", level: raised_error ? "error" : "info",
+              data: { duration_ms: duration_ms.round(1) }
+            )
+          end
+
+          if configuration.track_failures && raised_error
+            Railtie.failure_event_buffer.record(
+              kind: "dependency", transaction_name: name,
+              error_class: raised_error.class.name, error_message: raised_error.message
+            )
+          end
+
+          if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+            buffer.record(
+              span_id: SecureRandom.hex(8), name: name, kind: "redis",
+              started_at: ::Time.now.utc - (duration_ms / 1000.0), duration_ms: duration_ms
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/forge_ops_tracker/metric_buffer.rb

@@ -2,7 +2,7 @@ require "thread"
 
 module ForgeOpsTracker
   # Collects individual ForgeOpsTracker.capture_metric calls in-process and periodically flushes
-  # them as one batch, rather than one network call per capture -- the same "lazily start a
+  # them as one batch, rather than one network call per capture: the same "lazily start a
   # background thread on first use" pattern SessionFlusher/PerformanceFlusher already use, so each
   # forked Puma/Passenger worker gets its own fresh thread instead of inheriting a dead one across
   # fork. Unlike those two, this collects a *list* of individually-meaningful entries rather than
@@ -10,6 +10,12 @@ module ForgeOpsTracker
   # they'll want a genuinely accurate count/sum of later, not something to pre-aggregate away
   # client-side, so CustomMetricsController stores one row per entry as-is.
   class MetricBuffer
+    # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
+    # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
+    # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
+    # is delivering at the front of the array, which is what makes removing exactly them exact.
+    MAX_ENTRIES = 1000
+
     def initialize(configuration, client: Client.new(configuration))
       @configuration = configuration
       @client = client
@@ -23,9 +29,13 @@ module ForgeOpsTracker
     end
 
     def record(metric_name:, value:)
+      return false unless keepable?(value)
+
       ensure_worker_started
 
       @mutex.synchronize do
+        return false if @entries.size >= MAX_ENTRIES
+
         @entries << {
           metric_name: metric_name, value: value,
           environment: configuration.environment.to_s, release: configuration.release,
@@ -46,17 +56,24 @@ module ForgeOpsTracker
 
       @mutex.synchronize do
         return if @entries.empty?
-        snapshot = @entries
+        snapshot = @entries.dup
       end
 
       return unless client.deliver_metrics(snapshot)
 
-      @mutex.synchronize { @entries = [] }
+      # Exactly the entries just delivered: anything recorded while the request was in flight sits
+      # after them and stays for the next flush (resetting the whole array here lost it).
+      @mutex.synchronize { @entries.shift(snapshot.size) }
     end
 
     private
       attr_reader :configuration, :client
 
+      # A NaN or infinite value is not valid JSON and would make the server reject the whole batch.
+      def keepable?(value)
+        value.is_a?(Numeric) && value.to_f.finite?
+      end
+
       def ensure_worker_started
         return if @thread&.alive?
 

data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb

@@ -0,0 +1,35 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request a fresh, empty BreadcrumbBuffer to accumulate into (via
+    # ForgeOpsTracker.add_breadcrumb, or the automatic sql.active_record/process_action.
+    # action_controller/Net::HTTP subscriptions Railtie installs), so one request's trail never
+    # bleeds into another's, then hands ErrorSubscriber#report a way to read it back the exact same
+    # way ForgeOpsTracker::Middleware::UserContext's own thread-local does for the current user
+    # (Rails.error.subscribe's own #report interface never receives the Rack env, so this
+    # thread-local is the only channel available). A sibling to SessionTracking/UserContext, not
+    # folded into either: distinct concern, same "own file, own class" shape every middleware in
+    # this gem already takes.
+    #
+    # The ensure-clear is load-bearing, not optional, same reasoning UserContext's own doc
+    # comment gives: Puma reuses threads across requests, so leaving a buffer set would leak one
+    # request's breadcrumbs into a later, unrelated request handled on the same thread.
+    class BreadcrumbContext
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_breadcrumbs && configuration.enabled?
+
+        Thread.current[:forge_ops_tracker_breadcrumbs] = BreadcrumbBuffer.new(configuration)
+        app.call(env)
+      ensure
+        Thread.current[:forge_ops_tracker_breadcrumbs] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/middleware/span_tracing.rb

@@ -0,0 +1,38 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request its own fresh SpanBuffer (see that class for the stack/nesting
+    # mechanics), the same "Thread.current, cleared in an ensure" shape BreadcrumbContext already
+    # uses and for the identical reason: Puma reuses threads across requests, so a buffer left set
+    # would leak one request's spans into a later, unrelated one handled on the same thread.
+    #
+    # The actual send-or-don't decision lives here, not in SpanBuffer or SpanQueue: only once
+    # app.call returns does this middleware know the root span's own final duration (recorded by
+    # Railtie's process_action.action_controller subscription, which fires from *inside* that
+    # call, after every nested query/HTTP/Redis call has already run), so this is the first point
+    # that can compare it against configuration.trace_capture_threshold_ms and decide the trace was
+    # never worth sending in the first place - the entire reason a normal, fast request never costs
+    # a single byte over the wire, unlike PerformanceSample's own always-on aggregate reporting.
+    class SpanTracing
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_tracing && configuration.enabled?
+
+        buffer = SpanBuffer.new
+        Thread.current[:forge_ops_tracker_spans] = buffer
+        app.call(env)
+      ensure
+        if buffer&.slow?(configuration.trace_capture_threshold_ms)
+          Railtie.span_queue.push(trace_id: buffer.trace_id, spans: buffer.spans)
+        end
+        Thread.current[:forge_ops_tracker_spans] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/performance_flusher.rb

@@ -32,10 +32,14 @@ module ForgeOpsTracker
       ensure_worker_started
 
       @mutex.synchronize do
-        bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0 })
+        bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0, histogram: Hash.new(0) })
         bucket[:count] += 1
         bucket[:duration_sum_ms] += duration_ms
         bucket[:max_duration_ms] = duration_ms if duration_ms > bucket[:max_duration_ms]
+        # The distribution count/sum/max above can't reconstruct: see HistogramBucketer's own
+        # comment for why an approximate percentile from these bucket counts, not a true one from
+        # the raw values this gem deliberately never stores, is what the server computes from this.
+        bucket[:histogram][HistogramBucketer.bucket_for(duration_ms)] += 1
       end
     end
 
@@ -44,14 +48,26 @@ module ForgeOpsTracker
     # just grows instead of losing what was already tallied; same reasoning SessionFlusher#flush
     # already documents, and the same reason PerformanceSamplesController accepts a batch rather
     # than a single row: a failed flush shouldn't have to re-deliver by transaction one at a time.
+    #
+    # Only exactly what this snapshot delivered is removed afterward (subtracted from whatever is in
+    # each bucket by then), never the whole hash reset: #record can run on another thread while the
+    # HTTP request is in flight, so a call for a bucket already in the snapshot, or a brand-new one,
+    # can land between the snapshot and delivery succeeding, and resetting afterward would silently
+    # discard it. The SDKs ported from this gem all fixed that; this class had the same bug.
+    # max_duration_ms is left as whatever is currently on the bucket, sent or not: a max can't be
+    # "subtracted" back out, and leaving it never overstates the next period's own max.
     def flush
       snapshot = nil
+      sent = nil
       period_started_at = nil
+      period_ended_at = nil
 
       @mutex.synchronize do
         return if @buckets.empty?
 
         period_started_at = @period_started_at
+        period_ended_at = Time.now.utc
+        sent = @buckets.transform_values { |bucket| { count: bucket[:count], duration_sum_ms: bucket[:duration_sum_ms], histogram: bucket[:histogram].dup } }
         snapshot = @buckets.map do |(transaction_name, kind), bucket|
           {
             transaction_name: transaction_name,
@@ -59,10 +75,11 @@ module ForgeOpsTracker
             environment: configuration.environment.to_s,
             release: configuration.release,
             period_started_at: period_started_at.iso8601,
-            period_ended_at: Time.now.utc.iso8601,
+            period_ended_at: period_ended_at.iso8601,
             request_count: bucket[:count],
             duration_sum_ms: bucket[:duration_sum_ms],
-            max_duration_ms: bucket[:max_duration_ms]
+            max_duration_ms: bucket[:max_duration_ms],
+            histogram: bucket[:histogram].dup
           }
         end
       end
@@ -70,8 +87,19 @@ module ForgeOpsTracker
       return unless snapshot && client.deliver_performance_samples(snapshot)
 
       @mutex.synchronize do
-        @buckets = {}
-        @period_started_at = Time.now.utc
+        sent.each do |key, delivered|
+          current = @buckets[key]
+          next unless current
+
+          current[:count] -= delivered[:count]
+          current[:duration_sum_ms] = [ current[:duration_sum_ms] - delivered[:duration_sum_ms], 0.0 ].max
+          delivered[:histogram].each do |bucket_key, count|
+            current[:histogram][bucket_key] -= count
+            current[:histogram].delete(bucket_key) if current[:histogram][bucket_key] <= 0
+          end
+          @buckets.delete(key) if current[:count] <= 0
+        end
+        @period_started_at = period_ended_at
       end
     end
 

data/lib/forge_ops_tracker/performance_instrumentation.rb

@@ -43,6 +43,38 @@ module ForgeOpsTracker
       payload[:job].class.name
     end
 
+    # nil means "don't record this one": a job run via perform_now (directly, or as ActiveJob's
+    # own synchronous fallback) never goes through enqueue/serialize at all, so its enqueued_at
+    # stays nil (confirmed directly against ActiveJob::Core/Enqueuing's own source, not assumed),
+    # and a job that was never actually enqueued has no queue wait time to report; a bogus
+    # zero-or-negative duration would be actively misleading in a "how backed up is this queue"
+    # view, not just uninteresting.
+    #
+    # Takes at: explicitly (the perform.active_job event's own start time) rather than calling
+    # Time.now itself: keeps this unit-testable with a plain fixed Time, the same reason every
+    # other method in this module takes a plain payload Hash instead of reaching for real
+    # Rails/ActiveJob state (see this file's own top comment).
+    def job_queue_wait_ms(payload, at:)
+      enqueued_at = payload[:job].enqueued_at
+      return nil unless enqueued_at
+
+      (at - enqueued_at) * 1000.0
+    end
+
+    # nil means "this job didn't fail": ActiveSupport::Notifications.instrument (which
+    # perform.active_job already goes through) automatically populates payload[:exception_object]
+    # with whatever the instrumented block raised, confirmed directly against the installed
+    # activesupport gem's own documented behavior, not assumed; a job that completed normally
+    # never gets this key at all. Returns the raised exception's own class name/message, not the
+    # exception object itself, matching every other detail this module already extracts as plain
+    # strings rather than passing real framework objects back out to Railtie.
+    def job_failure_details(payload)
+      exception = payload[:exception_object]
+      return nil unless exception
+
+      { error_class: exception.class.name, error_message: exception.message.to_s }
+    end
+
     # True for a job dispatched through ActiveJob but actually run on Sidekiq: Sidekiq's own
     # server middleware chain wraps every job it runs, ActiveJob-dispatched or not, so without
     # this check a job on the Sidekiq/ActiveJob combination would be recorded twice: once here

data/lib/forge_ops_tracker/railtie.rb

@@ -14,6 +14,20 @@ module ForgeOpsTracker
       def performance_flusher
         @performance_flusher ||= ForgeOpsTracker::PerformanceFlusher.new(ForgeOpsTracker.configuration)
       end
+
+      # Same reasoning as performance_flusher above, exposed the identical way: shared across the
+      # two failure-detecting call sites below (the perform.active_job subscription, Net::HTTP's
+      # own Timing#request), one buffer/thread rather than each starting a redundant one.
+      def failure_event_buffer
+        @failure_event_buffer ||= ForgeOpsTracker::FailureEventBuffer.new(ForgeOpsTracker.configuration)
+      end
+
+      # Same reasoning again, exposed the identical way: shared across every call site that might
+      # finish a captured trace (Middleware::SpanTracing today; nothing else yet, but the same
+      # "one instance, not one per call site" reasoning applies regardless).
+      def span_queue
+        @span_queue ||= ForgeOpsTracker::SpanQueue.new(ForgeOpsTracker.configuration)
+      end
     end
 
     initializer "forge_ops_tracker.subscribe_error_reporter" do |app|
@@ -39,6 +53,26 @@ module ForgeOpsTracker
       app.middleware.use ForgeOpsTracker::Middleware::UserContext, configuration: ForgeOpsTracker.configuration
     end
 
+    # Gives every request its own fresh BreadcrumbBuffer to accumulate into (see
+    # ForgeOpsTracker::Middleware::BreadcrumbContext for why this has to be a middleware, the same
+    # "Rails.error.subscribe never receives the Rack env" reasoning UserContext's own comment
+    # gives): must run before the sql.active_record/process_action.action_controller subscriptions
+    # below can have anywhere to record into, but Rack middleware ordering guarantees that already,
+    # since this wraps app.call(env), and those notifications only ever fire from *inside* that
+    # call.
+    initializer "forge_ops_tracker.track_breadcrumbs" do |app|
+      app.middleware.use ForgeOpsTracker::Middleware::BreadcrumbContext, configuration: ForgeOpsTracker.configuration
+    end
+
+    # Same "must run before the notifications below have anywhere to record into" reasoning as
+    # track_breadcrumbs's own comment just above, for the exact same structural reason (Rack
+    # middleware always wraps the actual controller dispatch those notifications fire from inside
+    # of, regardless of this initializer's own position relative to track_breadcrumbs/
+    # track_current_user/track_sessions).
+    initializer "forge_ops_tracker.track_tracing" do |app|
+      app.middleware.use ForgeOpsTracker::Middleware::SpanTracing, configuration: ForgeOpsTracker.configuration
+    end
+
     # Rails already fires "process_action.action_controller" after every request with the
     # controller/action and how long it took (event.duration, in ms) built in, so unlike session
     # tracking above (which needed a Rack middleware because there's no single existing hook for
@@ -54,34 +88,123 @@ module ForgeOpsTracker
     # kept out of these blocks specifically so that decision logic is unit-testable without a real
     # Rails/ActiveJob boot (this gem's own test suite doesn't require Rails at all; see that
     # file's own comment).
+    #
+    # The same two subscriptions also record a breadcrumb, gated on track_breadcrumbs
+    # independently of track_performance: an app could want the trail without the timing data, or
+    # vice versa, and each already has to check its own flag regardless, so there's no real cost
+    # to keeping them independent rather than tying breadcrumbs to whether performance monitoring
+    # happens to be on.
+    #
+    # Every event.time below is wrapped in Time.at(...) before being handed to SpanBuffer#record or
+    # job_queue_wait_ms, both of which need a real Time. ActiveSupport::Notifications::Event#time
+    # returns a plain Float (seconds since the epoch) on the Rails version this gem currently
+    # targets, not a Time; Time.at(...) also passes a real Time straight through unchanged, so this
+    # stays correct if a future Rails version goes back to returning one.
     initializer "forge_ops_tracker.track_performance" do
       configuration = ForgeOpsTracker.configuration
       flusher = Railtie.performance_flusher
 
       ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*args|
-        next unless configuration.track_performance && configuration.enabled?
+        next unless configuration.enabled?
 
         event = ActiveSupport::Notifications::Event.new(*args)
         transaction_name = PerformanceInstrumentation.controller_transaction_name(event.payload)
-        flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "controller")
+
+        if configuration.track_performance
+          flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "controller")
+        end
+
+        if configuration.track_breadcrumbs
+          status = event.payload[:status]
+          ForgeOpsTracker.add_breadcrumb(
+            "#{event.payload[:method]} #{transaction_name}",
+            category: "controller",
+            level: status && status >= 500 ? "error" : "info",
+            data: { status: status, path: event.payload[:path] }.compact
+          )
+        end
+
+        # Records this request's own root span, not gated on track_performance/track_breadcrumbs
+        # above (its own independent configuration.track_tracing flag instead, the same "several
+        # genuinely independent mechanisms recorded from the same instrumentation point" pattern
+        # this exact call site already established for breadcrumbs): this is the one call in the
+        # whole gem that ever passes root: true, since process_action.action_controller is the only
+        # instrumentation point that wraps a genuinely whole request. See SpanBuffer#record's own
+        # comment for why root: true has to force parent_span_id nil explicitly here rather than
+        # reading it off the (already-popped-by-now) stack.
+        if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+          buffer.pop
+          buffer.record(
+            span_id: buffer.root_span_id, name: transaction_name, kind: "controller",
+            started_at: Time.at(event.time), duration_ms: event.duration, root: true
+          )
+        end
       end
 
       ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
-        next unless configuration.track_performance && configuration.enabled?
+        next unless configuration.enabled?
 
         event = ActiveSupport::Notifications::Event.new(*args)
         transaction_name = PerformanceInstrumentation.query_transaction_name(event.payload)
         next if transaction_name.nil? # SCHEMA/cached, see PerformanceInstrumentation's own comment
 
-        flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "query")
+        if configuration.track_performance
+          flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "query")
+        end
+
+        if configuration.track_breadcrumbs
+          ForgeOpsTracker.add_breadcrumb(transaction_name, category: "query", data: { duration_ms: event.duration.round(1) })
+        end
+
+        # A leaf span, parented onto whatever's currently open (the controller root, or a
+        # manually-wrapped ForgeOpsTracker.span block around it): a query has no children of its
+        # own to nest anything under, so unlike the controller root above, this never touches the
+        # stack itself, just records against whatever's already on top of it.
+        if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+          buffer.record(
+            span_id: SecureRandom.hex(8), name: transaction_name, kind: "database",
+            started_at: Time.at(event.time), duration_ms: event.duration
+          )
+        end
       end
 
       ActiveSupport::Notifications.subscribe("perform.active_job") do |*args|
-        next unless configuration.track_performance && configuration.enabled?
+        # Permissive enough for either concern below to run on its own: track_performance off but
+        # track_failures on (or vice versa) still needs this subscription to fire at all.
+        next unless (configuration.track_performance || configuration.track_failures) && configuration.enabled?
 
         event = ActiveSupport::Notifications::Event.new(*args)
-        transaction_name = PerformanceInstrumentation.job_transaction_name(event.payload)
-        flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "job")
+
+        if configuration.track_performance
+          transaction_name = PerformanceInstrumentation.job_transaction_name(event.payload)
+          flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "job")
+
+          # A second, independent sample from the same notification: how long this job actually
+          # waited in its queue before this perform even started, not how long perform itself took
+          # (the "job" kind above). Grouped by queue name, not job class, since queue-level backlog
+          # is the actionable unit here (a slow queue affects every job class routed through it),
+          # the same reason Solid Queue's own StatsReporter already reports queue_depth per queue
+          # name rather than per job class. See PerformanceInstrumentation.job_queue_wait_ms's own
+          # comment for why this is nil, and skipped, for a job that was never actually enqueued.
+          wait_ms = PerformanceInstrumentation.job_queue_wait_ms(event.payload, at: Time.at(event.time))
+          if wait_ms
+            flusher.record(transaction_name: event.payload[:job].queue_name, duration_ms: wait_ms, kind: "job_latency")
+          end
+        end
+
+        # Independent of track_performance above, same reasoning breadcrumbs already established
+        # at this exact call site: a job whose perform raised, via
+        # PerformanceInstrumentation.job_failure_details' own use of ActiveSupport::Notifications'
+        # automatic payload[:exception_object] (see that method's own comment).
+        if configuration.track_failures
+          failure = PerformanceInstrumentation.job_failure_details(event.payload)
+          if failure
+            Railtie.failure_event_buffer.record(
+              kind: "job", transaction_name: PerformanceInstrumentation.job_transaction_name(event.payload),
+              error_class: failure[:error_class], error_message: failure[:error_message]
+            )
+          end
+        end
       end
     end
 
@@ -124,6 +247,19 @@ module ForgeOpsTracker
       ForgeOpsTracker::Integrations::Puma::StatsReporter.start(configuration: ForgeOpsTracker.configuration)
     end
 
+    # Same defined? guard as track_sidekiq/track_solid_queue/track_puma above, not
+    # track_net_http's unconditional one: ActiveRecord is neither Ruby stdlib nor a declared
+    # dependency of this gem itself (this gem has none at all beyond Rails::Railtie being
+    # defined; see the gemspec's own comment on why even Sidekiq/Puma are dev-only dependencies),
+    # so an API-only Rails app with ActiveRecord genuinely removed is a real, if uncommon, case to
+    # guard against rather than assume away.
+    initializer "forge_ops_tracker.track_active_record_pool" do
+      next unless defined?(::ActiveRecord::Base)
+
+      require "forge_ops_tracker/integrations/active_record_pool"
+      ForgeOpsTracker::Integrations::ActiveRecordPool::StatsReporter.start(configuration: ForgeOpsTracker.configuration)
+    end
+
     # Net::HTTP is always present (Ruby's own standard library), so this always prepends; the
     # actual on/off decision happens at call time inside Timing#request itself (checking
     # configuration.track_performance/enabled? the same way the subscriptions above do), so
@@ -133,5 +269,22 @@ module ForgeOpsTracker
       require "forge_ops_tracker/integrations/net_http"
       Net::HTTP.prepend(ForgeOpsTracker::Integrations::NetHTTP::Timing)
     end
+
+    # defined?(::RedisClient) guarded, the same way track_sidekiq/track_solid_queue/track_puma
+    # above are: RedisClient is neither Ruby stdlib nor a declared dependency of this gem (see the
+    # gemspec's own comment on why even Sidekiq/Puma are dev-only dependencies), so a host app with
+    # no Redis at all is a real case to guard against, not assume away. Runs after Rails itself has
+    # finished booting (config.after_initialize, not a plain initializer like track_net_http
+    # above): unlike Net::HTTP (Ruby stdlib, always loaded), whether RedisClient is defined yet at
+    # plain initializer time depends on load order relative to whatever gem requires it (redis,
+    # sidekiq, ActiveJob's redis queue adapter, ...), which this gem has no control over; deferring
+    # the defined? check to after every railtie (including this app's own dependencies) has
+    # finished initializing avoids a load-order-dependent false negative.
+    config.after_initialize do
+      next unless defined?(::RedisClient)
+
+      require "forge_ops_tracker/integrations/redis_client"
+      ::RedisClient.prepend(ForgeOpsTracker::Integrations::RedisClientTiming)
+    end
   end
 end