forge_ops_tracker 0.7.0 → 0.10.2

data/lib/forge_ops_tracker/integrations/active_record_pool.rb

@@ -0,0 +1,41 @@
+module ForgeOpsTracker
+  module Integrations
+    module ActiveRecordPool
+      # Reports this process's own ActiveRecord connection pool utilization as infrastructure
+      # metrics on a periodic timer, the same shape as Sidekiq::StatsReporter/SolidQueue::StatsReporter
+      # (see either's own comment), and, like both of those, only ever loaded when ActiveRecord is
+      # actually present (checked with defined? in Railtie, not assumed).
+      #
+      # Scoped to the current process's own primary connection pool only, via the bare
+      # ActiveRecord::Base.connection_pool this process is actually using: a known v1 limitation,
+      # not an oversight, for an app with multiple databases (ActiveRecord::Base.connection_pool is
+      # only ever one of potentially several pools such an app has configured). Reporting every
+      # configured pool would need ActiveRecord::Base.connection_handler's own registry, a genuinely
+      # separate piece of work; this integration reports what a single-database app (the common
+      # case, and the only case this gem's own test suite/README examples ever set up) actually has.
+      class StatsReporter
+        def self.start(configuration: ForgeOpsTracker.configuration)
+          return unless configuration.track_performance && configuration.enabled?
+
+          PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
+        end
+
+        def report
+          stat = ::ActiveRecord::Base.connection_pool.stat
+          size = stat[:size].to_i
+          busy = stat[:busy].to_i
+
+          # 0/0 (a pool nobody's ever checked a connection out of yet, or, in practice, never) reads
+          # as 0% utilized, not NaN/an error: a pool that's never been used is exactly as "not under
+          # pressure" as a pool with plenty of headroom, not a value worth surfacing as broken.
+          utilization_pct = size.zero? ? 0.0 : (100.0 * busy / size).round(1)
+
+          ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.utilization_pct", value: utilization_pct)
+          ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.size", value: size)
+          ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.busy", value: busy)
+          ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.waiting", value: stat[:waiting].to_i)
+        end
+      end
+    end
+  end
+end

data/lib/forge_ops_tracker/integrations/net_http.rb

@@ -11,19 +11,101 @@ module ForgeOpsTracker
       # transaction_name is "<HTTP method> <host>", not the full URL: a request's own path or
       # query string could carry an id or a token, the same low-cardinality/no-secrets-in-a-label
       # reasoning every other transaction_name in this system already follows.
+      #
+      # Known, honest gap, confirmed directly against Net::HTTP's own source and real behavior, not
+      # assumed: Net::HTTP.get_response and the Net::HTTP.start { |http| ... } block form both open
+      # the TCP connection inside #start, *before* the block (and so before #request, the only
+      # method this wraps) ever runs, so a connection-refused/DNS-failure error raised during that
+      # setup is structurally invisible here, for timing, breadcrumbs, and the failure detection
+      # below alike (a pre-existing limitation this round's own test suite is what actually
+      # surfaced, not something newly introduced). Net::HTTP.new(...).request(...), the other
+      # common calling convention (no explicit #start at all), is different and unaffected:
+      # confirmed directly that Net::HTTP connects lazily, inside #request itself, the first time
+      # it's called on a not-yet-started instance, so that path's own connection failures are
+      # caught exactly like any other #request failure.
       module Timing
         # The guard (return super unless ...) skips timing, but the ensure block below still runs
         # on every exit path regardless, same as any ensure; start staying nil is what actually
         # skips recording there, not a second guard duplicating this one.
         def request(req, body = nil, &block)
-          return super unless ForgeOpsTracker.configuration.track_performance && ForgeOpsTracker.configuration.enabled?
+          configuration = ForgeOpsTracker.configuration
+          return super unless (configuration.track_performance || configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
 
           start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-          super
+          response = super
+        rescue StandardError => e
+          # Only track_failures below needs the actual exception object (for error_class/
+          # error_message); everything else keys off response staying nil, already true on this
+          # path without capturing anything. re-raised unchanged: this gem must never swallow or
+          # alter what the host app's own call to Net::HTTP#request would have raised.
+          raised_error = e
+          raise
         ensure
           if start
             duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
-            Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
+            # Computed once, unconditionally, not separately inside each flag-gated block below:
+            # both the breadcrumb and track_failures sections need it, and a value only ever
+            # assigned inside one conditional branch but read from another relies on Ruby's own
+            # "a local is declared the moment its assignment is lexically parsed, regardless of
+            # whether that branch actually ran" scoping, a real but easy-to-break-on-reorder
+            # subtlety not worth the risk here.
+            status = response&.code&.to_i
+            if configuration.track_performance
+              Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
+            end
+            # The same "method + host, never the full path/query string" reasoning
+            # transaction_name above already follows: a URL's own path or query could carry an id
+            # or a token, and this is a breadcrumb message, not a value this gem controls the
+            # cardinality of. Mirrors sdks/typescript's own fetch/xhr breadcrumbs: category
+            # "http", status + duration_ms in data, level "warning" for a non-2xx/3xx response.
+            if configuration.track_breadcrumbs
+              # No status at all means the request itself raised (a timeout, a connection
+              # refused, ...) before a response ever came back: worth "error", not the misleading
+              # "info" a bare status-based check would default a nil status to, the same "the
+              # request truly failed" case sdks/typescript's own fetch breadcrumb catch block
+              # marks "error" for.
+              level = if status.nil?
+                "error"
+              elsif status >= 400
+                "warning"
+              else
+                "info"
+              end
+              ForgeOpsTracker.add_breadcrumb(
+                "#{req.method} #{address}",
+                category: "http",
+                level: level,
+                data: { status: status, duration_ms: duration_ms.round(1) }.compact
+              )
+            end
+            # Independent of track_performance/track_breadcrumbs above, same "several genuinely
+            # independent mechanisms recorded from the same instrumentation point" pattern this
+            # exact call site already established for breadcrumbs. A failure is the request
+            # raising, or a 5xx: not a 4xx, the identical distinction the breadcrumb level just
+            # above already draws between "the dependency is unhealthy" (worth "error"/failure)
+            # and "the dependency responded, just not with success" (only "warning").
+            if configuration.track_failures && (status.nil? || status >= 500)
+              Railtie.failure_event_buffer.record(
+                kind: "dependency", transaction_name: "#{req.method} #{address}",
+                error_class: raised_error&.class&.name, error_message: raised_error&.message
+              )
+            end
+            # A leaf span, the same "parented onto whatever's currently open, never touches the
+            # stack itself" shape the sql.active_record subscription's own span recording already
+            # takes: an outbound HTTP call has no children of its own to nest anything under
+            # either.
+            if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+              # duration_ms itself is exact (computed from the monotonic clock above); started_at
+              # here is only derived from it (now minus duration), an approximation off the wall
+              # clock rather than a second real timestamp captured at the top of this method - a
+              # few milliseconds of drift on where the bar starts in a waterfall, never on how long
+              # it actually took.
+              start_time = ::Time.now.utc - (duration_ms / 1000.0)
+              buffer.record(
+                span_id: SecureRandom.hex(8), name: "#{req.method} #{address}", kind: "http",
+                started_at: start_time, duration_ms: duration_ms, data: { status: status }.compact
+              )
+            end
           end
         end
       end

data/lib/forge_ops_tracker/integrations/redis_client.rb

@@ -0,0 +1,62 @@
+module ForgeOpsTracker
+  module Integrations
+    module RedisClientTiming
+      # Times every Redis command, applied once via RedisClient.prepend(Timing) from Railtie, the
+      # same Module#prepend + super shape Integrations::NetHTTP::Timing already uses for the
+      # identical reason: neither Ruby's Redis ecosystem nor RedisClient itself fires an
+      # ActiveSupport::Notifications event of its own, so wrapping #call directly is the only hook
+      # available. Targets RedisClient specifically (the modern, actively-maintained low-level
+      # client both the current redis gem (5.x+) and standalone RedisClient users go through), not
+      # the older redis-rb 4.x Redis::Client - confirmed directly (not assumed) that redis 5.x no
+      # longer defines Redis::Client at all, so a hook there would silently do nothing on any
+      # currently-installed version of the gem.
+      #
+      # Unlike controller/query timing, this is span-only: there's no "redis" PerformanceSample
+      # kind for this to feed into the aggregate Performance page (that page has no Redis panel at
+      # all yet), so this only ever contributes to a captured trace's own waterfall, plus a
+      # breadcrumb and a dependency failure record, the same two other independent mechanisms
+      # NetHTTP::Timing's own call site already establishes.
+      #
+      # name is "Redis <COMMAND>" (the command verb alone, upcased: GET, SET, LPUSH, ...), never
+      # the key or any argument - the same low-cardinality, no-secrets-in-a-label reasoning every
+      # other transaction_name/span name in this gem already follows; a Redis key can easily carry
+      # a customer's own id or other sensitive value.
+      def call(*command, **kwargs)
+        configuration = ForgeOpsTracker.configuration
+        return super unless (configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
+
+        name = "Redis #{command.first.to_s.upcase}"
+        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        result = super
+      rescue StandardError => e
+        raised_error = e
+        raise
+      ensure
+        if start
+          duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
+
+          if configuration.track_breadcrumbs
+            ForgeOpsTracker.add_breadcrumb(
+              name, category: "redis", level: raised_error ? "error" : "info",
+              data: { duration_ms: duration_ms.round(1) }
+            )
+          end
+
+          if configuration.track_failures && raised_error
+            Railtie.failure_event_buffer.record(
+              kind: "dependency", transaction_name: name,
+              error_class: raised_error.class.name, error_message: raised_error.message
+            )
+          end
+
+          if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
+            buffer.record(
+              span_id: SecureRandom.hex(8), name: name, kind: "redis",
+              started_at: ::Time.now.utc - (duration_ms / 1000.0), duration_ms: duration_ms
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/forge_ops_tracker/metric_buffer.rb

@@ -2,7 +2,7 @@ require "thread"
 
 module ForgeOpsTracker
   # Collects individual ForgeOpsTracker.capture_metric calls in-process and periodically flushes
-  # them as one batch, rather than one network call per capture -- the same "lazily start a
+  # them as one batch, rather than one network call per capture: the same "lazily start a
   # background thread on first use" pattern SessionFlusher/PerformanceFlusher already use, so each
   # forked Puma/Passenger worker gets its own fresh thread instead of inheriting a dead one across
   # fork. Unlike those two, this collects a *list* of individually-meaningful entries rather than
@@ -10,6 +10,12 @@ module ForgeOpsTracker
   # they'll want a genuinely accurate count/sum of later, not something to pre-aggregate away
   # client-side, so CustomMetricsController stores one row per entry as-is.
   class MetricBuffer
+    # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
+    # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
+    # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
+    # is delivering at the front of the array, which is what makes removing exactly them exact.
+    MAX_ENTRIES = 1000
+
     def initialize(configuration, client: Client.new(configuration))
       @configuration = configuration
       @client = client
@@ -23,9 +29,13 @@ module ForgeOpsTracker
     end
 
     def record(metric_name:, value:)
+      return false unless keepable?(value)
+
       ensure_worker_started
 
       @mutex.synchronize do
+        return false if @entries.size >= MAX_ENTRIES
+
         @entries << {
           metric_name: metric_name, value: value,
           environment: configuration.environment.to_s, release: configuration.release,
@@ -46,17 +56,24 @@ module ForgeOpsTracker
 
       @mutex.synchronize do
         return if @entries.empty?
-        snapshot = @entries
+        snapshot = @entries.dup
       end
 
       return unless client.deliver_metrics(snapshot)
 
-      @mutex.synchronize { @entries = [] }
+      # Exactly the entries just delivered: anything recorded while the request was in flight sits
+      # after them and stays for the next flush (resetting the whole array here lost it).
+      @mutex.synchronize { @entries.shift(snapshot.size) }
     end
 
     private
       attr_reader :configuration, :client
 
+      # A NaN or infinite value is not valid JSON and would make the server reject the whole batch.
+      def keepable?(value)
+        value.is_a?(Numeric) && value.to_f.finite?
+      end
+
       def ensure_worker_started
         return if @thread&.alive?
 

data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb

@@ -0,0 +1,35 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request a fresh, empty BreadcrumbBuffer to accumulate into (via
+    # ForgeOpsTracker.add_breadcrumb, or the automatic sql.active_record/process_action.
+    # action_controller/Net::HTTP subscriptions Railtie installs), so one request's trail never
+    # bleeds into another's, then hands ErrorSubscriber#report a way to read it back the exact same
+    # way ForgeOpsTracker::Middleware::UserContext's own thread-local does for the current user
+    # (Rails.error.subscribe's own #report interface never receives the Rack env, so this
+    # thread-local is the only channel available). A sibling to SessionTracking/UserContext, not
+    # folded into either: distinct concern, same "own file, own class" shape every middleware in
+    # this gem already takes.
+    #
+    # The ensure-clear is load-bearing, not optional, same reasoning UserContext's own doc
+    # comment gives: Puma reuses threads across requests, so leaving a buffer set would leak one
+    # request's breadcrumbs into a later, unrelated request handled on the same thread.
+    class BreadcrumbContext
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_breadcrumbs && configuration.enabled?
+
+        Thread.current[:forge_ops_tracker_breadcrumbs] = BreadcrumbBuffer.new(configuration)
+        app.call(env)
+      ensure
+        Thread.current[:forge_ops_tracker_breadcrumbs] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/middleware/span_tracing.rb

@@ -0,0 +1,38 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request its own fresh SpanBuffer (see that class for the stack/nesting
+    # mechanics), the same "Thread.current, cleared in an ensure" shape BreadcrumbContext already
+    # uses and for the identical reason: Puma reuses threads across requests, so a buffer left set
+    # would leak one request's spans into a later, unrelated one handled on the same thread.
+    #
+    # The actual send-or-don't decision lives here, not in SpanBuffer or SpanQueue: only once
+    # app.call returns does this middleware know the root span's own final duration (recorded by
+    # Railtie's process_action.action_controller subscription, which fires from *inside* that
+    # call, after every nested query/HTTP/Redis call has already run), so this is the first point
+    # that can compare it against configuration.trace_capture_threshold_ms and decide the trace was
+    # never worth sending in the first place - the entire reason a normal, fast request never costs
+    # a single byte over the wire, unlike PerformanceSample's own always-on aggregate reporting.
+    class SpanTracing
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_tracing && configuration.enabled?
+
+        buffer = SpanBuffer.new
+        Thread.current[:forge_ops_tracker_spans] = buffer
+        app.call(env)
+      ensure
+        if buffer&.slow?(configuration.trace_capture_threshold_ms)
+          Railtie.span_queue.push(trace_id: buffer.trace_id, spans: buffer.spans)
+        end
+        Thread.current[:forge_ops_tracker_spans] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/performance_flusher.rb

@@ -32,10 +32,14 @@ module ForgeOpsTracker
       ensure_worker_started
 
       @mutex.synchronize do
-        bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0 })
+        bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0, histogram: Hash.new(0) })
         bucket[:count] += 1
         bucket[:duration_sum_ms] += duration_ms
         bucket[:max_duration_ms] = duration_ms if duration_ms > bucket[:max_duration_ms]
+        # The distribution count/sum/max above can't reconstruct: see HistogramBucketer's own
+        # comment for why an approximate percentile from these bucket counts, not a true one from
+        # the raw values this gem deliberately never stores, is what the server computes from this.
+        bucket[:histogram][HistogramBucketer.bucket_for(duration_ms)] += 1
       end
     end
 
@@ -44,14 +48,26 @@ module ForgeOpsTracker
     # just grows instead of losing what was already tallied; same reasoning SessionFlusher#flush
     # already documents, and the same reason PerformanceSamplesController accepts a batch rather
     # than a single row: a failed flush shouldn't have to re-deliver by transaction one at a time.
+    #
+    # Only exactly what this snapshot delivered is removed afterward (subtracted from whatever is in
+    # each bucket by then), never the whole hash reset: #record can run on another thread while the
+    # HTTP request is in flight, so a call for a bucket already in the snapshot, or a brand-new one,
+    # can land between the snapshot and delivery succeeding, and resetting afterward would silently
+    # discard it. The SDKs ported from this gem all fixed that; this class had the same bug.
+    # max_duration_ms is left as whatever is currently on the bucket, sent or not: a max can't be
+    # "subtracted" back out, and leaving it never overstates the next period's own max.
     def flush
       snapshot = nil
+      sent = nil
       period_started_at = nil
+      period_ended_at = nil
 
       @mutex.synchronize do
         return if @buckets.empty?
 
         period_started_at = @period_started_at
+        period_ended_at = Time.now.utc
+        sent = @buckets.transform_values { |bucket| { count: bucket[:count], duration_sum_ms: bucket[:duration_sum_ms], histogram: bucket[:histogram].dup } }
         snapshot = @buckets.map do |(transaction_name, kind), bucket|
           {
             transaction_name: transaction_name,
@@ -59,10 +75,11 @@ module ForgeOpsTracker
             environment: configuration.environment.to_s,
             release: configuration.release,
             period_started_at: period_started_at.iso8601,
-            period_ended_at: Time.now.utc.iso8601,
+            period_ended_at: period_ended_at.iso8601,
             request_count: bucket[:count],
             duration_sum_ms: bucket[:duration_sum_ms],
-            max_duration_ms: bucket[:max_duration_ms]
+            max_duration_ms: bucket[:max_duration_ms],
+            histogram: bucket[:histogram].dup
           }
         end
       end
@@ -70,8 +87,19 @@ module ForgeOpsTracker
       return unless snapshot && client.deliver_performance_samples(snapshot)
 
       @mutex.synchronize do
-        @buckets = {}
-        @period_started_at = Time.now.utc
+        sent.each do |key, delivered|
+          current = @buckets[key]
+          next unless current
+
+          current[:count] -= delivered[:count]
+          current[:duration_sum_ms] = [ current[:duration_sum_ms] - delivered[:duration_sum_ms], 0.0 ].max
+          delivered[:histogram].each do |bucket_key, count|
+            current[:histogram][bucket_key] -= count
+            current[:histogram].delete(bucket_key) if current[:histogram][bucket_key] <= 0
+          end
+          @buckets.delete(key) if current[:count] <= 0
+        end
+        @period_started_at = period_ended_at
       end
     end
 

data/lib/forge_ops_tracker/performance_instrumentation.rb

@@ -43,6 +43,38 @@ module ForgeOpsTracker
       payload[:job].class.name
     end
 
+    # nil means "don't record this one": a job run via perform_now (directly, or as ActiveJob's
+    # own synchronous fallback) never goes through enqueue/serialize at all, so its enqueued_at
+    # stays nil (confirmed directly against ActiveJob::Core/Enqueuing's own source, not assumed),
+    # and a job that was never actually enqueued has no queue wait time to report; a bogus
+    # zero-or-negative duration would be actively misleading in a "how backed up is this queue"
+    # view, not just uninteresting.
+    #
+    # Takes at: explicitly (the perform.active_job event's own start time) rather than calling
+    # Time.now itself: keeps this unit-testable with a plain fixed Time, the same reason every
+    # other method in this module takes a plain payload Hash instead of reaching for real
+    # Rails/ActiveJob state (see this file's own top comment).
+    def job_queue_wait_ms(payload, at:)
+      enqueued_at = payload[:job].enqueued_at
+      return nil unless enqueued_at
+
+      (at - enqueued_at) * 1000.0
+    end
+
+    # nil means "this job didn't fail": ActiveSupport::Notifications.instrument (which
+    # perform.active_job already goes through) automatically populates payload[:exception_object]
+    # with whatever the instrumented block raised, confirmed directly against the installed
+    # activesupport gem's own documented behavior, not assumed; a job that completed normally
+    # never gets this key at all. Returns the raised exception's own class name/message, not the
+    # exception object itself, matching every other detail this module already extracts as plain
+    # strings rather than passing real framework objects back out to Railtie.
+    def job_failure_details(payload)
+      exception = payload[:exception_object]
+      return nil unless exception
+
+      { error_class: exception.class.name, error_message: exception.message.to_s }
+    end
+
     # True for a job dispatched through ActiveJob but actually run on Sidekiq: Sidekiq's own
     # server middleware chain wraps every job it runs, ActiveJob-dispatched or not, so without
     # this check a job on the Sidekiq/ActiveJob combination would be recorded twice: once here