forge_ops_tracker 0.5.0 → 0.10.1

data/lib/forge_ops_tracker/metric_buffer.rb

@@ -0,0 +1,97 @@
+require "thread"
+
+module ForgeOpsTracker
+  # Collects individual ForgeOpsTracker.capture_metric calls in-process and periodically flushes
+  # them as one batch, rather than one network call per capture: the same "lazily start a
+  # background thread on first use" pattern SessionFlusher/PerformanceFlusher already use, so each
+  # forked Puma/Passenger worker gets its own fresh thread instead of inheriting a dead one across
+  # fork. Unlike those two, this collects a *list* of individually-meaningful entries rather than
+  # summing them into buckets: a customer's own signup or payment is exactly the kind of thing
+  # they'll want a genuinely accurate count/sum of later, not something to pre-aggregate away
+  # client-side, so CustomMetricsController stores one row per entry as-is.
+  class MetricBuffer
+    # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
+    # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
+    # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
+    # is delivering at the front of the array, which is what makes removing exactly them exact.
+    MAX_ENTRIES = 1000
+
+    def initialize(configuration, client: Client.new(configuration))
+      @configuration = configuration
+      @client = client
+      @mutex = Mutex.new
+      @entries = []
+      @thread = nil
+
+      # Flushes whatever's already buffered on a normal process exit, so the last partial window
+      # isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
+      at_exit { flush }
+    end
+
+    def record(metric_name:, value:)
+      return false unless keepable?(value)
+
+      ensure_worker_started
+
+      @mutex.synchronize do
+        return false if @entries.size >= MAX_ENTRIES
+
+        @entries << {
+          metric_name: metric_name, value: value,
+          environment: configuration.environment.to_s, release: configuration.release,
+          recorded_at: Time.now.utc.iso8601
+        }
+      end
+    end
+
+    # Snapshots and resets the buffered entries, then delivers them as one batch. A failed
+    # delivery keeps every entry where it is rather than resetting, so the next flush's batch
+    # just grows instead of losing what was already buffered; same reasoning SessionFlusher#flush
+    # already documents, including the same small, accepted race (an entry recorded during the
+    # in-flight HTTP request gets folded into this snapshot's own reset rather than kept for the
+    # next one) that class already lives with, for the same reason: simple and consistent beats a
+    # cleverer dedup that would only trade one rare edge case for a subtler one.
+    def flush
+      snapshot = nil
+
+      @mutex.synchronize do
+        return if @entries.empty?
+        snapshot = @entries.dup
+      end
+
+      return unless client.deliver_metrics(snapshot)
+
+      # Exactly the entries just delivered: anything recorded while the request was in flight sits
+      # after them and stays for the next flush (resetting the whole array here lost it).
+      @mutex.synchronize { @entries.shift(snapshot.size) }
+    end
+
+    private
+      attr_reader :configuration, :client
+
+      # A NaN or infinite value is not valid JSON and would make the server reject the whole batch.
+      def keepable?(value)
+        value.is_a?(Numeric) && value.to_f.finite?
+      end
+
+      def ensure_worker_started
+        return if @thread&.alive?
+
+        @mutex.synchronize do
+          return if @thread&.alive?
+
+          @thread = Thread.new { run }
+          @thread.abort_on_exception = false
+        end
+      end
+
+      def run
+        loop do
+          sleep configuration.metric_flush_interval
+          flush
+        rescue StandardError => e
+          configuration.logger&.debug { "[ForgeOpsTracker] metric flush thread error: #{e.class}: #{e.message}" }
+        end
+      end
+  end
+end

data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb

@@ -0,0 +1,35 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request a fresh, empty BreadcrumbBuffer to accumulate into (via
+    # ForgeOpsTracker.add_breadcrumb, or the automatic sql.active_record/process_action.
+    # action_controller/Net::HTTP subscriptions Railtie installs), so one request's trail never
+    # bleeds into another's, then hands ErrorSubscriber#report a way to read it back the exact same
+    # way ForgeOpsTracker::Middleware::UserContext's own thread-local does for the current user
+    # (Rails.error.subscribe's own #report interface never receives the Rack env, so this
+    # thread-local is the only channel available). A sibling to SessionTracking/UserContext, not
+    # folded into either: distinct concern, same "own file, own class" shape every middleware in
+    # this gem already takes.
+    #
+    # The ensure-clear is load-bearing, not optional, same reasoning UserContext's own doc
+    # comment gives: Puma reuses threads across requests, so leaving a buffer set would leak one
+    # request's breadcrumbs into a later, unrelated request handled on the same thread.
+    class BreadcrumbContext
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_breadcrumbs && configuration.enabled?
+
+        Thread.current[:forge_ops_tracker_breadcrumbs] = BreadcrumbBuffer.new(configuration)
+        app.call(env)
+      ensure
+        Thread.current[:forge_ops_tracker_breadcrumbs] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/middleware/span_tracing.rb

@@ -0,0 +1,38 @@
+module ForgeOpsTracker
+  module Middleware
+    # Gives every request its own fresh SpanBuffer (see that class for the stack/nesting
+    # mechanics), the same "Thread.current, cleared in an ensure" shape BreadcrumbContext already
+    # uses and for the identical reason: Puma reuses threads across requests, so a buffer left set
+    # would leak one request's spans into a later, unrelated one handled on the same thread.
+    #
+    # The actual send-or-don't decision lives here, not in SpanBuffer or SpanQueue: only once
+    # app.call returns does this middleware know the root span's own final duration (recorded by
+    # Railtie's process_action.action_controller subscription, which fires from *inside* that
+    # call, after every nested query/HTTP/Redis call has already run), so this is the first point
+    # that can compare it against configuration.trace_capture_threshold_ms and decide the trace was
+    # never worth sending in the first place - the entire reason a normal, fast request never costs
+    # a single byte over the wire, unlike PerformanceSample's own always-on aggregate reporting.
+    class SpanTracing
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_tracing && configuration.enabled?
+
+        buffer = SpanBuffer.new
+        Thread.current[:forge_ops_tracker_spans] = buffer
+        app.call(env)
+      ensure
+        if buffer&.slow?(configuration.trace_capture_threshold_ms)
+          Railtie.span_queue.push(trace_id: buffer.trace_id, spans: buffer.spans)
+        end
+        Thread.current[:forge_ops_tracker_spans] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+    end
+  end
+end

data/lib/forge_ops_tracker/middleware/user_context.rb

@@ -0,0 +1,52 @@
+module ForgeOpsTracker
+  module Middleware
+    # Wraps every request: if Warden is mounted (env["warden"] present, whether or not the app
+    # uses Devise specifically, since Devise just mounts Warden automatically, but any Warden-
+    # based auth setup works here, confirmed directly against the installed warden gem's own
+    # Warden::Manager/Warden::Proxy source) and it has a signed-in user, stashes a serialized
+    # version of that user in a thread-local for the duration of this request so
+    # ErrorSubscriber#report (which never receives the Rack env at all: Rails.error.subscribe's
+    # own interface hands it error/context/severity, nothing request-shaped) can read it back and
+    # attach it to whatever gets reported. A sibling to SessionTracking, not folded into it:
+    # distinct concern, same "own file, own class" shape every middleware in this gem already
+    # takes.
+    #
+    # The ensure-clear is load-bearing, not optional: Puma reuses threads across requests, so
+    # leaving this set would leak one request's user into a later, unrelated request handled on
+    # the same thread.
+    class UserContext
+      def initialize(app, configuration: ForgeOpsTracker.configuration)
+        @app = app
+        @configuration = configuration
+      end
+
+      def call(env)
+        return app.call(env) unless configuration.track_current_user && configuration.enabled?
+
+        if env.key?("warden") && (user = env["warden"].user)
+          serialized = serialize(user)
+          Thread.current[:forge_ops_tracker_current_user] = serialized unless serialized.empty?
+        end
+
+        app.call(env)
+      ensure
+        Thread.current[:forge_ops_tracker_current_user] = nil
+      end
+
+      private
+        attr_reader :app, :configuration
+
+        # Duck-types rather than assuming a Devise-shaped model: id if the object responds to it,
+        # email if it responds to it, username else name if either responds to it. A custom
+        # Warden setup with a differently-shaped user object just gets whatever subset applies
+        # here; ForgeOpsTracker.set_user covers anything this can't infer.
+        def serialize(user)
+          {
+            id: (user.id if user.respond_to?(:id)),
+            email: (user.email if user.respond_to?(:email)),
+            username: (user.username if user.respond_to?(:username)) || (user.name if user.respond_to?(:name))
+          }.compact
+        end
+    end
+  end
+end

data/lib/forge_ops_tracker/performance_flusher.rb

@@ -0,0 +1,129 @@
+require "thread"
+
+module ForgeOpsTracker
+  # Times every request/job/query/outbound call in-process, bucketed by [transaction_name, kind]
+  # (see Railtie's several ActiveSupport::Notifications subscriptions, each a different kind), and
+  # periodically flushes each distinct bucket as one small aggregate report, rather than one
+  # network call per event. One shared flusher for every kind, not one per kind: the aggregation
+  # shape (count + duration sum + max over a period) is identical regardless of what's actually
+  # being timed. Same "lazily start a background thread on first use, not at load time" pattern
+  # SessionFlusher already uses, so each forked Puma/Passenger worker gets its own fresh thread
+  # instead of inheriting a dead one across fork.
+  class PerformanceFlusher
+    def initialize(configuration, client: Client.new(configuration))
+      @configuration = configuration
+      @client = client
+      @mutex = Mutex.new
+      @buckets = {}
+      @period_started_at = Time.now.utc
+      @thread = nil
+
+      # Flushes whatever's already tallied on a normal process exit, so the last partial window
+      # (anything shorter than a full performance_flush_interval) isn't silently dropped; same
+      # reasoning SessionFlusher's own at_exit hook documents.
+      at_exit { flush }
+    end
+
+    # kind ("controller"/"job"/"query"/"http", see Railtie's own subscriptions) is part of the
+    # bucket key alongside transaction_name, not folded into transaction_name itself: a job and a
+    # controller action (or, less likely but still real, a query and a job) can share a name
+    # without colliding into the same bucket.
+    def record(transaction_name:, duration_ms:, kind: "controller")
+      ensure_worker_started
+
+      @mutex.synchronize do
+        bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0, histogram: Hash.new(0) })
+        bucket[:count] += 1
+        bucket[:duration_sum_ms] += duration_ms
+        bucket[:max_duration_ms] = duration_ms if duration_ms > bucket[:max_duration_ms]
+        # The distribution count/sum/max above can't reconstruct: see HistogramBucketer's own
+        # comment for why an approximate percentile from these bucket counts, not a true one from
+        # the raw values this gem deliberately never stores, is what the server computes from this.
+        bucket[:histogram][HistogramBucketer.bucket_for(duration_ms)] += 1
+      end
+    end
+
+    # Snapshots and resets the in-process buckets, then delivers them as one batch. A failed
+    # delivery keeps every bucket where it is rather than resetting, so the next flush's window
+    # just grows instead of losing what was already tallied; same reasoning SessionFlusher#flush
+    # already documents, and the same reason PerformanceSamplesController accepts a batch rather
+    # than a single row: a failed flush shouldn't have to re-deliver by transaction one at a time.
+    #
+    # Only exactly what this snapshot delivered is removed afterward (subtracted from whatever is in
+    # each bucket by then), never the whole hash reset: #record can run on another thread while the
+    # HTTP request is in flight, so a call for a bucket already in the snapshot, or a brand-new one,
+    # can land between the snapshot and delivery succeeding, and resetting afterward would silently
+    # discard it. The SDKs ported from this gem all fixed that; this class had the same bug.
+    # max_duration_ms is left as whatever is currently on the bucket, sent or not: a max can't be
+    # "subtracted" back out, and leaving it never overstates the next period's own max.
+    def flush
+      snapshot = nil
+      sent = nil
+      period_started_at = nil
+      period_ended_at = nil
+
+      @mutex.synchronize do
+        return if @buckets.empty?
+
+        period_started_at = @period_started_at
+        period_ended_at = Time.now.utc
+        sent = @buckets.transform_values { |bucket| { count: bucket[:count], duration_sum_ms: bucket[:duration_sum_ms], histogram: bucket[:histogram].dup } }
+        snapshot = @buckets.map do |(transaction_name, kind), bucket|
+          {
+            transaction_name: transaction_name,
+            kind: kind,
+            environment: configuration.environment.to_s,
+            release: configuration.release,
+            period_started_at: period_started_at.iso8601,
+            period_ended_at: period_ended_at.iso8601,
+            request_count: bucket[:count],
+            duration_sum_ms: bucket[:duration_sum_ms],
+            max_duration_ms: bucket[:max_duration_ms],
+            histogram: bucket[:histogram].dup
+          }
+        end
+      end
+
+      return unless snapshot && client.deliver_performance_samples(snapshot)
+
+      @mutex.synchronize do
+        sent.each do |key, delivered|
+          current = @buckets[key]
+          next unless current
+
+          current[:count] -= delivered[:count]
+          current[:duration_sum_ms] = [ current[:duration_sum_ms] - delivered[:duration_sum_ms], 0.0 ].max
+          delivered[:histogram].each do |bucket_key, count|
+            current[:histogram][bucket_key] -= count
+            current[:histogram].delete(bucket_key) if current[:histogram][bucket_key] <= 0
+          end
+          @buckets.delete(key) if current[:count] <= 0
+        end
+        @period_started_at = period_ended_at
+      end
+    end
+
+    private
+      attr_reader :configuration, :client
+
+      def ensure_worker_started
+        return if @thread&.alive?
+
+        @mutex.synchronize do
+          return if @thread&.alive?
+
+          @thread = Thread.new { run }
+          @thread.abort_on_exception = false
+        end
+      end
+
+      def run
+        loop do
+          sleep configuration.performance_flush_interval
+          flush
+        rescue StandardError => e
+          configuration.logger&.debug { "[ForgeOpsTracker] performance flush thread error: #{e.class}: #{e.message}" }
+        end
+      end
+  end
+end

data/lib/forge_ops_tracker/performance_instrumentation.rb

@@ -0,0 +1,92 @@
+module ForgeOpsTracker
+  # The actual "what do we call this, and should we even record it" decisions Railtie's
+  # ActiveSupport::Notifications subscriptions make, pulled out into plain module-function methods
+  # rather than left inline in each subscribe block. Railtie itself only ever loads when
+  # Rails::Railtie is already defined (see lib/forge_ops_tracker.rb's own final line), so nothing
+  # in that file can be exercised by this gem's own test suite without adding Rails itself as a
+  # development dependency just to fire a notification event by hand; every method here takes a
+  # plain payload Hash instead of a real ActiveSupport::Notifications::Event, so the decision logic
+  # itself (the SCHEMA/cached skip, the ActiveJob-wrapper dedup) is fully unit-testable with no
+  # Rails/Sidekiq/ActiveJob dependency at all. Railtie's own subscribe blocks stay thin, calling
+  # straight into these.
+  module PerformanceInstrumentation
+    module_function
+
+    def controller_transaction_name(payload)
+      "#{payload[:controller]}##{payload[:action]}"
+    end
+
+    # nil means "don't record this one": Rails' own sql.active_record fires for two kinds of
+    # noise that would drown out real query timing otherwise. "SCHEMA" is Rails' own internal
+    # introspection (column lookups, etc.), never something a customer's own code triggered;
+    # payload[:cached] is a genuinely instant hash lookup off Rails' own query cache, not a real
+    # round trip to the database, so timing it would understate every cached query's own real
+    # cost the next time it isn't cached.
+    #
+    # payload[:name] (Rails' own auto-generated label, e.g. "User Load", "Order Create"), not
+    # payload[:sql] (the literal query text): a customer's own query text could carry a literal
+    # value in some edge cases despite ActiveRecord's own parameterization, and even where it
+    # doesn't, raw SQL text is far higher cardinality than this system's transaction_name is
+    # designed for everywhere else. Falls back to the literal string "SQL" for the rare query
+    # with no name at all, rather than skipping it outright.
+    def query_transaction_name(payload)
+      return nil if payload[:name] == "SCHEMA" || payload[:cached]
+
+      # A plain nil-or-empty check, not ActiveSupport's #presence: this module is unit-tested
+      # without Rails/ActiveSupport loaded at all (see this file's own top comment), so it can't
+      # rely on a method ActiveSupport adds to String/NilClass.
+      name = payload[:name]
+      name.nil? || name.empty? ? "SQL" : name
+    end
+
+    def job_transaction_name(payload)
+      payload[:job].class.name
+    end
+
+    # nil means "don't record this one": a job run via perform_now (directly, or as ActiveJob's
+    # own synchronous fallback) never goes through enqueue/serialize at all, so its enqueued_at
+    # stays nil (confirmed directly against ActiveJob::Core/Enqueuing's own source, not assumed),
+    # and a job that was never actually enqueued has no queue wait time to report; a bogus
+    # zero-or-negative duration would be actively misleading in a "how backed up is this queue"
+    # view, not just uninteresting.
+    #
+    # Takes at: explicitly (the perform.active_job event's own start time) rather than calling
+    # Time.now itself: keeps this unit-testable with a plain fixed Time, the same reason every
+    # other method in this module takes a plain payload Hash instead of reaching for real
+    # Rails/ActiveJob state (see this file's own top comment).
+    def job_queue_wait_ms(payload, at:)
+      enqueued_at = payload[:job].enqueued_at
+      return nil unless enqueued_at
+
+      (at - enqueued_at) * 1000.0
+    end
+
+    # nil means "this job didn't fail": ActiveSupport::Notifications.instrument (which
+    # perform.active_job already goes through) automatically populates payload[:exception_object]
+    # with whatever the instrumented block raised, confirmed directly against the installed
+    # activesupport gem's own documented behavior, not assumed; a job that completed normally
+    # never gets this key at all. Returns the raised exception's own class name/message, not the
+    # exception object itself, matching every other detail this module already extracts as plain
+    # strings rather than passing real framework objects back out to Railtie.
+    def job_failure_details(payload)
+      exception = payload[:exception_object]
+      return nil unless exception
+
+      { error_class: exception.class.name, error_message: exception.message.to_s }
+    end
+
+    # True for a job dispatched through ActiveJob but actually run on Sidekiq: Sidekiq's own
+    # server middleware chain wraps every job it runs, ActiveJob-dispatched or not, so without
+    # this check a job on the Sidekiq/ActiveJob combination would be recorded twice: once here
+    # (kind: "job", via the raw Sidekiq middleware) and once already by the perform.active_job
+    # subscription that already covers every ActiveJob job regardless of which queue backend
+    # actually runs it (Solid Queue, Sidekiq, or anything else). The class name itself is
+    # ActiveJob's own, not this gem's: confirmed directly against how ActiveJob wraps a job for
+    # Sidekiq, not assumed from either project's docs alone.
+    def sidekiq_worker_is_active_job_wrapper?(worker)
+      return false unless defined?(::ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
+
+      worker.is_a?(::ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
+    end
+  end
+end

data/lib/forge_ops_tracker/periodic_poller.rb

@@ -0,0 +1,50 @@
+module ForgeOpsTracker
+  # A small shared background-thread loop for the Sidekiq/Solid Queue/Puma stats reporters (see
+  # integrations/): unlike PerformanceFlusher and the two metric buffers, which genuinely differ
+  # in what they buffer and are kept as separate sibling classes on purpose (see
+  # InfrastructureMetricBuffer's own comment), these three reporters only differ in *what they
+  # poll*, not *how they poll it*: lazily start a thread on first use (so a forked worker gets
+  # its own live thread regardless of when it forked relative to load, same reasoning
+  # PerformanceFlusher's own lazy start documents), sleep the interval, call the block, rescue and
+  # log so one bad tick never kills the loop, forever. Genuinely mechanical enough across all
+  # three to share, unlike those.
+  #
+  # No at_exit flush: unlike the flushers/buffers, which hold real accumulated data that would
+  # otherwise be lost, a gauge reading skipped on process exit is simply the same as one skipped
+  # because the interval hadn't ticked yet: nothing to lose.
+  class PeriodicPoller
+    def self.start(interval, logger: nil, &block)
+      new(interval, logger: logger, &block).tap(&:start)
+    end
+
+    def initialize(interval, logger: nil, &block)
+      @interval = interval
+      @logger = logger
+      @block = block
+      @mutex = Mutex.new
+      @thread = nil
+    end
+
+    def start
+      return if @thread&.alive?
+
+      @mutex.synchronize do
+        return if @thread&.alive?
+
+        @thread = Thread.new { run }
+        @thread.abort_on_exception = false
+      end
+      self
+    end
+
+    private
+      def run
+        loop do
+          sleep @interval
+          @block.call
+        rescue StandardError => e
+          @logger&.debug { "[ForgeOpsTracker] periodic poller error: #{e.class}: #{e.message}" }
+        end
+      end
+  end
+end