forge_ops_tracker 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a535fb7cb193fd614ee40259a9b20e62832149c7a2dbeb7d1d40253c0630f65a
-  data.tar.gz: 559a19cd52a56addacc87c5533bc0954c7ca2de6eac81a6afb18769f280c06c5
+  metadata.gz: 304c9a15155ecdf53fe8e181dd7f43b56a08d7397e7d63555e428185ece06560
+  data.tar.gz: d4d2cb88ffec650e579c7279c590e4eaee8f288bc6ef7041d2c2f5785767fa2f
 SHA512:
-  metadata.gz: 158342795c1dbb5e7575da43ddcd239c25b52111c268ede3fdfdbb9b5621cf1a10cd120e602f75fe2e57bd46faed9483aaa1a61f8c9f91770474210414400e8f
-  data.tar.gz: 5b4f11f1758833c58f8925f1354f8a7da49b80d04344cc2c340a2662a186fcfc0a8451709faef4c78a60d618c9f0ea936c460c97b3af8bad2687fbc8f6fbfdae
+  metadata.gz: dfd3a51d60e0decd113e2a8bd50db101ebd4ec57ebb468a522413b4f9ffa0c5c15d5440939c7a6f772561ce6ffe5c7078322d2903f0d3ec87006f1dd368c1718
+  data.tar.gz: 1646c0b506c357122c513604d031974318df0086a0c2a962c99e80143624be459755fab5dfff3df7718f24bb6caebe699ba5325fafb79b35a92f76ffa0783b63

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.17.0 (2026-10-13)
+
+- Fixed: a batch the server refused was re-sent at every flush for as long as the process lived,
+  with everything recorded since added to it. Performance samples, custom and infrastructure
+  metrics, session counts and failure events are now dropped when the server answers with a 4xx
+  other than 408 or 429 (a 400, 401, 403, 413 or 422, for example), since sending the same batch
+  again can only fail the same way. Once a metric buffer had filled up behind such a batch, every
+  new metric was dropped too; that no longer happens.
+- A batch that could not be delivered for now (a network error, a timeout, a 5xx, a 408 or a 429)
+  is still kept for the next flush, but only for 5 flushes in a row. After the fifth it is dropped
+  and the buffer starts clean, so a long outage no longer holds the same data in memory forever.
+- The failure event buffer now holds at most 1000 entries; further ones are dropped until a flush
+  lets go of them, the same way the metric buffers already behave.
+- With a `config.logger`, each drop is logged once at debug with what was dropped and why, and a
+  non-2xx answer is logged with its status code.
+- Internal: `ForgeOpsTracker::Client#deliver` and its sibling `deliver_*` methods now return
+  `:delivered`, `:rejected` or `:failed` instead of `true`/`false`. All three are truthy, so code
+  calling `Client` directly should compare against `:delivered`. Nothing in the documented API
+  changed.
+
 ## 0.16.0 (2026-10-02)
 
 - Changed: outside Rails, the environment now defaults to `production` when nothing sets it (it

data/lib/forge_ops_tracker/change_snapshot.rb

@@ -66,7 +66,7 @@ module ForgeOpsTracker
     end
 
     def deliver
-      client.deliver_change_snapshot(payload)
+      DeliveryStatus.from(client.deliver_change_snapshot(payload)) == DeliveryStatus::DELIVERED
     rescue StandardError => e
       configuration.logger&.debug { "[ForgeOpsTracker] change snapshot failed: #{e.class}: #{e.message}" }
       false

data/lib/forge_ops_tracker/client.rb

@@ -5,8 +5,10 @@ require "uri"
 module ForgeOpsTracker
   # Delivers one payload over HTTP. Every failure mode; DNS, connection,
   # timeout, TLS, a non-2xx response; is caught here and turned into a
-  # `false` return rather than a raised exception, since a broken or
-  # unreachable tracker must never be able to break the host app.
+  # DeliveryStatus rather than a raised exception, since a broken or
+  # unreachable tracker must never be able to break the host app. Every
+  # deliver_* method answers one of DeliveryStatus::DELIVERED, REJECTED or
+  # FAILED (all three are truthy, so compare, never test for truth).
   class Client
     def initialize(configuration)
       @configuration = configuration
@@ -16,8 +18,8 @@ module ForgeOpsTracker
       post(configuration.ingestion_uri, payload)
     end
 
-    # Same delivery contract as deliver above (never raises, returns a bare
-    # true/false), just a different endpoint under the same DSN's host and
+    # Same delivery contract as deliver above (never raises, returns a
+    # DeliveryStatus), just a different endpoint under the same DSN's host and
     # api_key; see Configuration#session_checkins_uri.
     def deliver_session_checkin(payload)
       post(configuration.session_checkins_uri, payload)
@@ -61,7 +63,7 @@ module ForgeOpsTracker
     end
 
     # Same delivery contract again; see Configuration#changes_uri and ForgeOpsTracker.record_change.
-    # One change per call. A plan without change tracking answers 403, which is just a false here.
+    # One change per call. A plan without change tracking answers 403, which is a REJECTED here.
     def deliver_change(payload)
       post(configuration.changes_uri, payload)
     end
@@ -72,7 +74,7 @@ module ForgeOpsTracker
     end
 
     # Same again; see Configuration#query_plans_uri and QueryExplainer. One plan per call. A plan
-    # without performance monitoring answers 403, which is just a false here.
+    # without performance monitoring answers 403, which is a REJECTED here.
     def deliver_query_plan(payload)
       post(configuration.query_plans_uri, payload)
     end
@@ -81,13 +83,15 @@ module ForgeOpsTracker
       attr_reader :configuration
 
       def post(uri, payload)
-        return false unless uri
+        return DeliveryStatus::FAILED unless uri
 
         response = http_for(uri).request(build_request(uri, payload))
-        response.is_a?(Net::HTTPSuccess)
+        status = DeliveryStatus.for_code(response.code)
+        log { "delivery to #{uri.path} answered HTTP #{response.code}" } unless status == DeliveryStatus::DELIVERED
+        status
       rescue StandardError => e
         log { "delivery failed: #{e.class}: #{e.message}" }
-        false
+        DeliveryStatus::FAILED
       end
 
       def http_for(uri)

data/lib/forge_ops_tracker/delivery_queue.rb

@@ -39,7 +39,7 @@ module ForgeOpsTracker
         rescue ThreadError
           break
         end
-        client.public_send(@deliver_with, payload)
+        deliver(payload)
       end
       sleep 0.01 while @in_flight.positive? && monotonic_now < deadline
     rescue StandardError => e
@@ -78,7 +78,7 @@ module ForgeOpsTracker
           payload = @queue.pop
           @in_flight += 1
           begin
-            client.public_send(@deliver_with, payload)
+            deliver(payload)
           ensure
             @in_flight -= 1
           end
@@ -87,6 +87,17 @@ module ForgeOpsTracker
         end
       end
 
+      # One attempt per payload, as it has always been: a queued event is never held back for a
+      # retry, whatever the outcome, so all that is left to do with a refusal is to say so once.
+      def deliver(payload)
+        case DeliveryStatus.from(client.public_send(@deliver_with, payload))
+        when DeliveryStatus::REJECTED
+          log { "dropping event: the server rejected it and will not accept a retry" }
+        when DeliveryStatus::FAILED
+          log { "dropping event: delivery failed" }
+        end
+      end
+
       # Registered on the first push, not at load, so a process that never reports anything never
       # waits on exit at all.
       def register_drain

data/lib/forge_ops_tracker/delivery_status.rb

@@ -0,0 +1,74 @@
+module ForgeOpsTracker
+  # What became of one delivery attempt, as every Client#deliver_* method reports it. A bare
+  # true/false could not tell "the server will never take this payload" apart from "try again
+  # later", so a batch a plan or a validation rule refused was re-sent at every flush for as long
+  # as the process lived.
+  module DeliveryStatus
+    # Any 2xx.
+    DELIVERED = :delivered
+    # A 4xx other than 408 and 429: sending the same payload again can only fail the same way.
+    REJECTED = :rejected
+    # A network error, a timeout, a 5xx, a 408 or a 429: worth another attempt later.
+    FAILED = :failed
+
+    RETRYABLE_CLIENT_ERRORS = [ 408, 429 ].freeze
+
+    def self.for_code(code)
+      code = code.to_i
+      return DELIVERED if code.between?(200, 299)
+      return REJECTED if code.between?(400, 499) && !RETRYABLE_CLIENT_ERRORS.include?(code)
+
+      FAILED
+    end
+
+    # Reads whatever a client handed back. A status passes through; anything else is read the way
+    # the old boolean contract was (truthy delivered, falsy failed for now), so a client injected
+    # through a `client:` keyword that still answers true/false keeps working.
+    def self.from(result)
+      return result if [ DELIVERED, REJECTED, FAILED ].include?(result)
+
+      result ? DELIVERED : FAILED
+    end
+  end
+
+  # Decides, after each flush of one buffer/flusher, whether the batch it just sent is done with.
+  # Delivered and rejected batches are let go of at once; one that failed for now is kept, but only
+  # for MAX_FAILED_FLUSHES flushes in a row, so an outage (or a server answering 500 to one
+  # particular batch) can never pin the same data in memory for the life of the process.
+  class FlushOutcome
+    MAX_FAILED_FLUSHES = 5
+
+    # what names the batch in the debug log ("metrics", "performance samples").
+    def initialize(configuration, what)
+      @configuration = configuration
+      @what = what
+      @failed_flushes = 0
+    end
+
+    # True when the caller should remove what it just sent (delivered, or dropped), false when it
+    # should keep it for the next flush.
+    def release?(result)
+      case DeliveryStatus.from(result)
+      when DeliveryStatus::DELIVERED
+        @failed_flushes = 0
+        true
+      when DeliveryStatus::REJECTED
+        @failed_flushes = 0
+        log { "dropping buffered #{@what}: the server rejected them and will not accept a retry" }
+        true
+      else
+        @failed_flushes += 1
+        return false if @failed_flushes < MAX_FAILED_FLUSHES
+
+        @failed_flushes = 0
+        log { "dropping buffered #{@what}: #{MAX_FAILED_FLUSHES} flushes in a row failed" }
+        true
+      end
+    end
+
+    private
+      def log
+        @configuration.logger&.debug { "[ForgeOpsTracker] #{yield}" }
+      end
+  end
+end

data/lib/forge_ops_tracker/failure_event_buffer.rb

@@ -8,12 +8,17 @@ module ForgeOpsTracker
   # kind of thing worth an accurate individual record of, not something to pre-aggregate away the
   # way PerformanceFlusher's own count/sum/max buckets duration.
   class FailureEventBuffer
+    # Same cap, and the same "drop the newest, not the oldest" reasoning, as MetricBuffer's own
+    # MAX_ENTRIES: a burst of failures during an outage must not grow this without bound.
+    MAX_ENTRIES = 1000
+
     def initialize(configuration, client: Client.new(configuration))
       @configuration = configuration
       @client = client
       @mutex = Mutex.new
       @entries = []
       @thread = nil
+      @outcome = FlushOutcome.new(configuration, "failure events")
 
       # Flushes whatever's already buffered on a normal process exit, so the last partial window
       # isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
@@ -24,6 +29,8 @@ module ForgeOpsTracker
       ensure_worker_started
 
       @mutex.synchronize do
+        return false if @entries.size >= MAX_ENTRIES
+
         @entries << {
           kind: kind, transaction_name: transaction_name, error_class: error_class, error_message: error_message,
           environment: configuration.environment.to_s, release: configuration.release,
@@ -33,18 +40,22 @@ module ForgeOpsTracker
     end
 
     # Snapshots and resets the buffered entries, then delivers them as one batch. Same "a failed
-    # delivery keeps every entry for the next attempt" contract as MetricBuffer#flush.
+    # delivery keeps every entry for the next attempt" contract as MetricBuffer#flush, with the
+    # same two exceptions: a rejected batch, and one that failed FlushOutcome::MAX_FAILED_FLUSHES
+    # flushes in a row, are dropped.
     def flush
       snapshot = nil
 
       @mutex.synchronize do
         return if @entries.empty?
-        snapshot = @entries
+        snapshot = @entries.dup
       end
 
-      return unless client.deliver_failure_events(snapshot)
+      return unless @outcome.release?(client.deliver_failure_events(snapshot))
 
-      @mutex.synchronize { @entries = [] }
+      # Exactly the entries just sent: anything recorded while the request was in flight sits after
+      # them and stays for the next flush.
+      @mutex.synchronize { @entries.shift(snapshot.size) }
     end
 
     private

data/lib/forge_ops_tracker/infrastructure_metric_buffer.rb

@@ -3,16 +3,16 @@ require "thread"
 module ForgeOpsTracker
   # Collects individual ForgeOpsTracker.capture_infrastructure_metric readings in-process and
   # periodically flushes them as one batch, same shape as MetricBuffer (a sibling class, not a
-  # shared base -- this gem keeps SessionFlusher/PerformanceFlusher as separate, independently-
+  # shared base; this gem keeps SessionFlusher/PerformanceFlusher as separate, independently-
   # readable classes despite their own similar shape too, and these two follow that precedent).
   # Its main real caller is a short-lived cron/script process that exits right after a handful of
   # capture calls, so the at_exit flush below is what actually matters in practice; the periodic
   # background-thread timer still runs for a long-lived process that chooses to call this
   # continuously instead, but rarely gets the chance to fire in the cron case.
   class InfrastructureMetricBuffer
-    # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
-    # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
-    # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
+    # Once this many entries are buffered, further ones are dropped until a flush lets go of them:
+    # an unreachable server fails every flush, and an uncapped buffer would keep growing until
+    # FlushOutcome gives up on the batch. Dropping the newest rather than the oldest keeps the entries a flush
     # is delivering at the front of the array, which is what makes removing exactly them exact.
     MAX_ENTRIES = 1000
 
@@ -22,6 +22,7 @@ module ForgeOpsTracker
       @mutex = Mutex.new
       @entries = []
       @thread = nil
+      @outcome = FlushOutcome.new(configuration, "infrastructure metrics")
 
       at_exit { flush }
     end
@@ -38,7 +39,7 @@ module ForgeOpsTracker
       end
     end
 
-    # Same reset-on-success/keep-on-failure contract as MetricBuffer#flush; see that class's own
+    # Same reset-on-success/keep-on-failure/drop-on-rejection contract as MetricBuffer#flush; see that class's own
     # comment for the accepted race this shares with it.
     def flush
       snapshot = nil
@@ -48,9 +49,9 @@ module ForgeOpsTracker
         snapshot = @entries.dup
       end
 
-      return unless client.deliver_infrastructure_metrics(snapshot)
+      return unless @outcome.release?(client.deliver_infrastructure_metrics(snapshot))
 
-      # Exactly the entries just delivered: anything recorded while the request was in flight sits
+      # Exactly the entries just sent: anything recorded while the request was in flight sits
       # after them and stays for the next flush (resetting the whole array here lost it).
       @mutex.synchronize { @entries.shift(snapshot.size) }
     end

data/lib/forge_ops_tracker/metric_buffer.rb

@@ -10,9 +10,9 @@ module ForgeOpsTracker
   # they'll want a genuinely accurate count/sum of later, not something to pre-aggregate away
   # client-side, so CustomMetricsController stores one row per entry as-is.
   class MetricBuffer
-    # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
-    # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
-    # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
+    # Once this many entries are buffered, further ones are dropped until a flush lets go of them:
+    # an unreachable server fails every flush, and an uncapped buffer would keep growing until
+    # FlushOutcome gives up on the batch. Dropping the newest rather than the oldest keeps the entries a flush
     # is delivering at the front of the array, which is what makes removing exactly them exact.
     MAX_ENTRIES = 1000
 
@@ -22,6 +22,7 @@ module ForgeOpsTracker
       @mutex = Mutex.new
       @entries = []
       @thread = nil
+      @outcome = FlushOutcome.new(configuration, "metrics")
 
       # Flushes whatever's already buffered on a normal process exit, so the last partial window
       # isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
@@ -44,10 +45,11 @@ module ForgeOpsTracker
       end
     end
 
-    # Snapshots and resets the buffered entries, then delivers them as one batch. A failed
-    # delivery keeps every entry where it is rather than resetting, so the next flush's batch
-    # just grows instead of losing what was already buffered; same reasoning SessionFlusher#flush
-    # already documents, including the same small, accepted race (an entry recorded during the
+    # Snapshots and resets the buffered entries, then delivers them as one batch. A delivery that
+    # failed for now keeps every entry where it is rather than resetting, so the next flush's batch
+    # just grows instead of losing what was already buffered (a rejected batch, or one that failed
+    # FlushOutcome::MAX_FAILED_FLUSHES flushes in a row, is dropped instead); same reasoning
+    # SessionFlusher#flush already documents, including the same small, accepted race (an entry recorded during the
     # in-flight HTTP request gets folded into this snapshot's own reset rather than kept for the
     # next one) that class already lives with, for the same reason: simple and consistent beats a
     # cleverer dedup that would only trade one rare edge case for a subtler one.
@@ -59,9 +61,9 @@ module ForgeOpsTracker
         snapshot = @entries.dup
       end
 
-      return unless client.deliver_metrics(snapshot)
+      return unless @outcome.release?(client.deliver_metrics(snapshot))
 
-      # Exactly the entries just delivered: anything recorded while the request was in flight sits
+      # Exactly the entries just sent: anything recorded while the request was in flight sits
       # after them and stays for the next flush (resetting the whole array here lost it).
       @mutex.synchronize { @entries.shift(snapshot.size) }
     end

data/lib/forge_ops_tracker/performance_flusher.rb

@@ -17,6 +17,7 @@ module ForgeOpsTracker
       @buckets = {}
       @period_started_at = Time.now.utc
       @thread = nil
+      @outcome = FlushOutcome.new(configuration, "performance samples")
 
       # Flushes whatever's already tallied on a normal process exit, so the last partial window
       # (anything shorter than a full performance_flush_interval) isn't silently dropped; same
@@ -48,6 +49,9 @@ module ForgeOpsTracker
     # just grows instead of losing what was already tallied; same reasoning SessionFlusher#flush
     # already documents, and the same reason PerformanceSamplesController accepts a batch rather
     # than a single row: a failed flush shouldn't have to re-deliver by transaction one at a time.
+    # That only holds for a delivery that failed for now, and only FlushOutcome::MAX_FAILED_FLUSHES
+    # times in a row: a batch the server rejected, or one that keeps failing, is removed exactly
+    # the way a delivered one is, since re-sending it could never succeed.
     #
     # Only exactly what this snapshot delivered is removed afterward (subtracted from whatever is in
     # each bucket by then), never the whole hash reset: #record can run on another thread while the
@@ -84,7 +88,7 @@ module ForgeOpsTracker
         end
       end
 
-      return unless snapshot && client.deliver_performance_samples(snapshot)
+      return unless snapshot && @outcome.release?(client.deliver_performance_samples(snapshot))
 
       @mutex.synchronize do
         sent.each do |key, delivered|

data/lib/forge_ops_tracker/query_explainer.rb

@@ -135,10 +135,11 @@ module ForgeOpsTracker
         return false
       end
 
-      client.deliver_query_plan(
+      result = client.deliver_query_plan(
         statement: job.statement, db_system: "postgresql", plan: masked_plan,
         duration_ms: job.duration_ms, environment: configuration.environment, captured_at: job.captured_at
       )
+      DeliveryStatus.from(result) == DeliveryStatus::DELIVERED
     rescue StandardError => e
       log { "EXPLAIN failed: #{e.class}: #{e.message}" }
       false

data/lib/forge_ops_tracker/session_flusher.rb

@@ -16,6 +16,7 @@ module ForgeOpsTracker
       @crashed_sessions_count = 0
       @period_started_at = Time.now.utc
       @thread = nil
+      @outcome = FlushOutcome.new(configuration, "session counts")
 
       # Flushes whatever's already tallied on a normal process exit, so the last partial window
       # (anything shorter than a full session_flush_interval) isn't silently dropped.
@@ -34,6 +35,8 @@ module ForgeOpsTracker
     # Snapshots and resets the in-process counters, then delivers them. A failed delivery keeps
     # the counts where they are rather than resetting, so the next flush's window just grows
     # instead of losing what was already tallied; there's no other copy of this data anywhere.
+    # Counts the server rejected, or that failed FlushOutcome::MAX_FAILED_FLUSHES flushes in a
+    # row, are reset all the same: sending them again could never succeed.
     def flush
       snapshot = nil
 
@@ -50,7 +53,7 @@ module ForgeOpsTracker
         }
       end
 
-      return unless snapshot && client.deliver_session_checkin(snapshot)
+      return unless snapshot && @outcome.release?(client.deliver_session_checkin(snapshot))
 
       @mutex.synchronize do
         @sessions_count = 0

data/lib/forge_ops_tracker/version.rb

@@ -1,3 +1,3 @@
 module ForgeOpsTracker
-  VERSION = "0.16.0"
+  VERSION = "0.17.0"
 end

data/lib/forge_ops_tracker.rb

@@ -4,6 +4,7 @@ require "forge_ops_tracker/configuration"
 require "forge_ops_tracker/pii_scrubber"
 require "forge_ops_tracker/sql_statement"
 require "forge_ops_tracker/event_builder"
+require "forge_ops_tracker/delivery_status"
 require "forge_ops_tracker/client"
 require "forge_ops_tracker/delivery_queue"
 require "forge_ops_tracker/change"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forge_ops_tracker
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - ForgeOps
@@ -109,6 +109,7 @@ files:
 - lib/forge_ops_tracker/client.rb
 - lib/forge_ops_tracker/configuration.rb
 - lib/forge_ops_tracker/delivery_queue.rb
+- lib/forge_ops_tracker/delivery_status.rb
 - lib/forge_ops_tracker/error_subscriber.rb
 - lib/forge_ops_tracker/event_builder.rb
 - lib/forge_ops_tracker/failure_event_buffer.rb