bitfab 0.36.5 → 0.36.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49b714ee6d468fc34c0c6009802c5736d33f03275f06d7d7148635b5d70b7caa
-  data.tar.gz: 3816843ddea8722c0222cfc0bed5c65ea05f5abb5ff764233e5ab302d609f2d2
+  metadata.gz: dd73e07889e4f74cd94c6ca534e423a7aa95e3ce6f1e1ba306e9fe33df8de8df
+  data.tar.gz: fb2eb3cd99066d8338f6ad03d18bc3a8de9f3955659aa66835c83066a95d3595
 SHA512:
-  metadata.gz: 1f5504e7d7622ccf55145b9124ee0d8ca42f774bdaea720a92382af42148bbf16afb72708d673fb359b37a9840364a770e4a1a87003f016aff2ebd1a59a52b38
-  data.tar.gz: db5fbc39adf358e640ae315348eb15ceb5798c67747d092eaaf37651cf9a89542ae9c4d8899a44ebfd4425ba623598e9e5af382dec11f119a917d52cf9ccb0dd
+  metadata.gz: b7155a3d0426c00f57668975485f2ba722bd3f0555dd897024f586290b36265f5851d49cc88242eb552421758b9d624b31e21edfb8290e68f693110e8b93cd32
+  data.tar.gz: 87ae26a27f8864294f431b80d87f0a7f768fdb2a2f09786929dad90d61316ce7c1106e75fe9271c63260cebe3545a2e73676e94295aa281f4d24c0d101184b0f

data/lib/bitfab/http_client.rb

@@ -12,7 +12,15 @@ require_relative "version"
 require_relative "warn_once"
 
 module Bitfab
+  # What a caller learns about one tracked trace once it takes it back.
+  DeliveryReport = Struct.new(:span_count, :closed, :delivered, keyword_init: true)
+
+  TraceDelivery = Struct.new(:submitted_span_ids, :acked_span_ids, :closed, :closing_acked,
+    keyword_init: true)
+
   class HttpClient
+    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
+
     REPLAY_DB_BRANCH_REQUEST_TIMEOUT_SECONDS = 300
     private_constant :REPLAY_DB_BRANCH_REQUEST_TIMEOUT_SECONDS
 
@@ -23,6 +31,13 @@ module Bitfab
       @service_url = (service_url || DEFAULT_SERVICE_URL).chomp("/")
       @timeout = timeout
       @transport_mutex = Mutex.new
+      # Eager for the same reason the transport's mutex is: `||=` racing between
+      # the export threads recording acks would hand each a different mutex.
+      @delivery_mutex = Mutex.new
+      @trace_deliveries = {}
+      @delivery_pid = Process.pid
+      @carrier_seq_mutex = Mutex.new
+      @carrier_seq = 0
       @transport = nil
       @closed = false
     end
@@ -108,7 +123,11 @@ module Bitfab
 
     # Queue an external span on this client's trace transport (fire-and-forget).
     def send_external_span(payload)
-      trace_transport&.submit("external_span", payload.merge("sdkVersion" => VERSION))
+      trace_transport&.submit(
+        "external_span",
+        payload.merge("sdkVersion" => VERSION),
+        recorded_meta("external_span", payload, carrier_ref(payload))
+      )
     end
 
     # Make a GET request to the Bitfab API.
@@ -254,7 +273,15 @@ module Bitfab
 
     # Queue an external trace on this client's trace transport (fire-and-forget).
     def send_external_trace(payload)
-      trace_transport&.submit("external_trace", payload.merge("sdkVersion" => VERSION))
+      trace_transport&.submit(
+        "external_trace",
+        payload.merge("sdkVersion" => VERSION),
+        recorded_meta(
+          "external_trace",
+          payload,
+          (payload["completed"] == true) ? carrier_ref(payload) : nil
+        )
+      )
     end
 
     # Read the replay traces the server has fully persisted so far.
@@ -268,6 +295,75 @@ module Bitfab
 
     private
 
+    # The delivery identity of a carrier, read from the payload here because
+    # this is where the payload shape is owned. The transport is handed the
+    # result and never looks inside a payload itself.
+    # Everything the transport needs to know about a carrier, derived here
+    # because this is where the payload shape is owned. The transport applies
+    # these and never looks inside a payload itself.
+    def carrier_meta(operation, payload, ref)
+      Otel::CarrierMeta.new(
+        ref:,
+        name: carrier_name(operation, payload),
+        started_at: payload_timestamp(payload, "started_at"),
+        ended_at: payload_timestamp(payload, "ended_at"),
+        errored: payload_error?(payload)
+      )
+    end
+
+    def carrier_name(operation, payload)
+      raw_span = payload["rawSpan"]
+      if operation == "external_span" && raw_span.is_a?(Hash)
+        name = raw_span.dig("span_data", "name")
+        return name if name.is_a?(String)
+      end
+      return payload["traceFunctionKey"] if payload["traceFunctionKey"].is_a?(String)
+
+      "bitfab.#{operation}"
+    end
+
+    def payload_timestamp(payload, field)
+      raw = payload.dig("rawSpan", field) if payload["rawSpan"].is_a?(Hash)
+      if raw.nil?
+        raw_trace = payload["externalTrace"] || payload["rawTrace"]
+        raw = raw_trace[field] if raw_trace.is_a?(Hash)
+      end
+      return nil unless raw.is_a?(String)
+
+      begin
+        Time.iso8601(raw)
+      rescue ArgumentError
+        nil
+      end
+    end
+
+    def payload_error?(payload)
+      raw_span = payload["rawSpan"]
+      return true if raw_span.is_a?(Hash) && !raw_span.dig("span_data", "error").nil?
+
+      errors = payload["errors"]
+      errors.is_a?(Array) && !errors.empty?
+    end
+
+    def carrier_ref(payload)
+      trace_id = payload["sourceTraceId"]
+      unless trace_id.is_a?(String)
+        raw_trace = payload["externalTrace"] || payload["rawTrace"]
+        trace_id = raw_trace["id"] if raw_trace.is_a?(Hash)
+      end
+      return nil unless trace_id.is_a?(String)
+
+      raw_span = payload["rawSpan"]
+      return Otel::CarrierRef.new(trace_id:) if raw_span.nil?
+
+      span_id = raw_span["id"] if raw_span.is_a?(Hash)
+      # A counter, not a clock: CLOCK_MONOTONIC repeats across calls close
+      # together, and two carriers sharing an id collapse into one ledger entry,
+      # so a single ack would mark both delivered.
+      span_id = next_submission_id unless span_id.is_a?(String)
+      Otel::CarrierRef.new(trace_id:, span_id:)
+    end
+
     # Returns nil once the client is closed. A stray traced call after close
     # must not crash the host app, and raising here would be swallowed by the
     # span finalize path, dropping the span with no explanation.
@@ -283,13 +379,184 @@ module Bitfab
         end
 
         @transport ||= Transport.create_trace_transport(
-          direct_sender: method(:send_transport_request)
+          direct_sender: method(:deliver_carriers),
+          on_delivered: method(:record_delivered_carriers)
         )
       end
     end
 
-    def send_transport_request(endpoint, request, timeout)
-      send_prepared(endpoint, request, timeout:, max_retries: 1)
+    # Post one encoded batch and decide what the server's answer means, so the
+    # transport never reads a response. Rejections and permanent statuses come
+    # back as a non-retryable DeliveryError; anything the server might still
+    # accept on a second try comes back retryable.
+    def deliver_carriers(request, timeout)
+      response = begin
+        # send_prepared, not send_encoded: the exporter already encoded this
+        # batch to size the request, and re-encoding would do it twice.
+        send_prepared(OTLP_TRACES_ENDPOINT, request, timeout:, max_retries: 1)
+      rescue => e
+        status = http_status(e)
+        raise Otel::DeliveryError.new(
+          "OTLP ingestion failed: #{e.message}",
+          # OTLP's retryable set, plus 500. Every other 4xx is the server's
+          # verdict on the payload and will be the same next time.
+          #
+          # 500 is a deliberate deviation: OTLP treats it as the app being
+          # broken, which assumes a collector that fails deterministically.
+          # Bitfab ingestion answers every unhandled error with 500, so a
+          # connection blip or a cold start arrives here indistinguishable from
+          # a real fault, and giving up on the first one drops spans a second
+          # attempt would deliver.
+          retryable: status.nil? || [429, 500, 502, 503, 504].include?(status),
+          oversized: status == 413,
+          retry_after_ms: retry_after_ms(e)
+        )
+      end
+
+      partial_success = response.is_a?(Hash) ? response["partialSuccess"] : nil
+      return unless partial_success.is_a?(Hash)
+
+      rejected = partial_success["rejectedSpans"]
+      return if rejected.nil? || rejected.to_s == "0"
+
+      # The server's verdict on the payload, not a transient fault.
+      raise Otel::DeliveryError.new(
+        "OTLP ingestion rejected #{rejected} span(s): " \
+        "#{partial_success["errorMessage"] || "no reason provided"}"
+      )
+    end
+
+    # Retry-After as milliseconds. The header is either a delay in seconds or an
+    # HTTP date; both forms appear in the wild, so both are read. Anything else,
+    # or a date already past, yields nil so the caller backs off on its own.
+    def retry_after_ms(error)
+      return nil unless error.respond_to?(:response)
+
+      response = error.response
+      header = response.respond_to?(:[]) ? response["Retry-After"] : nil
+      return nil if header.nil? || header.to_s.empty?
+
+      seconds = Float(header, exception: false)
+      return (seconds >= 0) ? seconds * 1000 : nil if seconds
+
+      begin
+        [0, (Time.httpdate(header.to_s) - Time.now) * 1000].max
+      rescue ArgumentError
+        nil
+      end
+    end
+
+    def http_status(error)
+      return nil unless error.respond_to?(:response)
+
+      response = error.response
+      response.respond_to?(:code) ? response.code.to_i : nil
+    end
+
+    public
+
+    # Start tracking delivery for trace_ids. Nothing is recorded for a trace
+    # that was never tracked, so ordinary tracing costs no bookkeeping at all.
+    def track_trace_deliveries(trace_ids)
+      @delivery_mutex.synchronize do
+        trace_ids.each do |trace_id|
+          trace_deliveries[trace_id] ||= TraceDelivery.new(
+            submitted_span_ids: Set.new, acked_span_ids: Set.new,
+            closed: false, closing_acked: false
+          )
+        end
+      end
+    end
+
+    # Whether any tracked trace has had its closing carrier submitted.
+    def closed_deliveries?(trace_ids)
+      @delivery_mutex.synchronize do
+        trace_ids.any? { |trace_id| trace_deliveries[trace_id]&.closed }
+      end
+    end
+
+    # Report what each tracked trace submitted and whether the server confirmed
+    # it, and stop tracking them. Every id passed is freed, so a caller cannot
+    # leak a record for a trace that never closed.
+    #
+    # delivered is only meaningful once a flush has settled: acks land before an
+    # export returns, so a flush that reported success has already collected
+    # every ack it is going to collect.
+    def take_trace_deliveries(trace_ids)
+      @delivery_mutex.synchronize do
+        trace_ids.each_with_object({}) do |trace_id, acc|
+          delivery = trace_deliveries.delete(trace_id)
+          next if delivery.nil?
+
+          acc[trace_id] = DeliveryReport.new(
+            span_count: delivery.submitted_span_ids.size,
+            closed: delivery.closed,
+            delivered: delivery.closing_acked &&
+              delivery.submitted_span_ids.subset?(delivery.acked_span_ids)
+          )
+        end
+      end
+    end
+
+    private
+
+    def record_submitted_carrier(ref)
+      return if ref.nil?
+
+      @delivery_mutex.synchronize do
+        delivery = trace_deliveries[ref.trace_id]
+        next if delivery.nil?
+
+        if ref.span_id.nil?
+          delivery.closed = true
+        else
+          delivery.submitted_span_ids << ref.span_id
+        end
+      end
+    end
+
+    # Ingestion commits every carrier in a request before it answers, so a
+    # delivered ref is proof its row exists: the same fact the replay status
+    # endpoint would report, already in hand.
+    #
+    # Called on exporter threads, hence the mutex.
+    def record_delivered_carriers(refs)
+      @delivery_mutex.synchronize do
+        refs.each do |ref|
+          delivery = trace_deliveries[ref.trace_id]
+          next if delivery.nil?
+
+          if ref.span_id.nil?
+            delivery.closing_acked = true
+          else
+            delivery.acked_span_ids << ref.span_id
+          end
+        end
+      end
+    end
+
+    # Build a carrier's meta and record what it adds to its expected set.
+    def recorded_meta(operation, payload, ref)
+      record_submitted_carrier(ref)
+      carrier_meta(operation, payload, ref)
+    end
+
+    # The ledger records what the PARENT submitted. The child's transport is
+    # rebuilt empty and its carrier refs are dropped, so nothing inherited here
+    # will ever be exported or acked: a barrier reading it would wait on acks
+    # that cannot arrive, or count the parent's deliveries as its own and finish
+    # having sent nothing. Ruby has no after-fork hook, so this checks the pid
+    # the same way Otel.ensure_process_state does.
+    def next_submission_id
+      @carrier_seq_mutex.synchronize { "submission-#{@carrier_seq += 1}" }
+    end
+
+    def trace_deliveries
+      if @delivery_pid != Process.pid
+        @delivery_pid = Process.pid
+        @trace_deliveries = {}
+      end
+      @trace_deliveries
     end
 
     # Normalize each entry to a hash with stable string keys, accepting either

data/lib/bitfab/otel.rb

@@ -19,7 +19,6 @@ module Bitfab
   module Otel
     OPERATION_ATTRIBUTE = "bitfab.operation"
     PAYLOAD_ATTRIBUTE = "bitfab.payload"
-    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES"
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
     MAX_EXPORT_REQUEST_BYTES = 3_000_000
@@ -32,7 +31,12 @@ module Bitfab
     SCHEDULE_DELAY_MILLIS = 5_000
     EXPORT_TIMEOUT_MILLIS = 30_000
     EXPORT_RETRIES = 3
-    RETRY_DELAY_SECONDS = 0.1
+    RETRY_BASE_DELAY_MILLIS = 100
+    # Ceiling on the exponential growth of our OWN backoff. It does not bound a
+    # wait the server asked for: OTLP says to honor Retry-After, and calls data
+    # dropped while throttled the outcome to avoid. What bounds an honored wait
+    # is the export budget, since the processor kills a longer export.
+    RETRY_BACKOFF_CEILING_MILLIS = 5_000
 
     # The comma that joins adjacent spans in the request's span list.
     SPAN_SEPARATOR_BYTES = 1
@@ -44,7 +48,22 @@ module Bitfab
     # Encoding once and remembering the size is what keeps request packing
     # linear: sizing a candidate batch by re-encoding the whole request
     # re-escapes every carrier's bitfab.payload string on every span considered.
-    EncodedSpan = Struct.new(:encoded, :size)
+    EncodedSpan = Struct.new(:encoded, :size, :ref)
+
+    # Which carrier a payload is, for delivery accounting only. Supplied by the
+    # caller that built the payload: the transport never reads inside one.
+    #
+    # span_id is nil for the carrier that closes a trace, which is what tells
+    # the transport the trace's expected set has stopped growing.
+    CarrierRef = Struct.new(:trace_id, :span_id, keyword_init: true)
+
+    # Everything the transport needs to know ABOUT a payload without reading
+    # one. Supplied by the caller that built it; each field falls back to
+    # something the transport can decide without looking inside.
+    #
+    # ref is nil for carriers nobody accounts for, name defaults to
+    # "bitfab.<operation>", nil timestamps let OTel stamp the carrier itself.
+    CarrierMeta = Struct.new(:ref, :name, :started_at, :ended_at, :errored, keyword_init: true)
 
     # The invariant head and tail of an OTLP request for one export window. Key
     # order matches what JSON.generate emits for the equivalent Hash, so a body
@@ -63,8 +82,21 @@ module Bitfab
 
     INVALID_SPAN_ID = ("\0" * 8).b
 
-    PayloadTooLargeError = Class.new(StandardError)
-    PartialSuccessError = Class.new(StandardError)
+    # Why a delivery failed, in the only two terms the transport acts on. A
+    # sender classifies everything it can see, including a network fault
+    # carrying no verdict; anything else reaching the transport is a fault in
+    # the sender and is not retried.
+    class DeliveryError < StandardError
+      attr_reader :retryable, :oversized, :retry_after_ms
+
+      def initialize(message, retryable: false, oversized: false, retry_after_ms: nil)
+        super(message)
+        @retryable = retryable
+        @oversized = oversized
+        # How long the server asked us to wait, when it said so.
+        @retry_after_ms = retry_after_ms
+      end
+    end
 
     class << self
       def max_request_bytes_from_env
@@ -97,9 +129,10 @@ module Bitfab
         DEFAULT_EXPORT_CONCURRENCY
       end
 
-      def create_transport(direct_sender:)
+      def create_transport(direct_sender:, on_delivered: nil)
         BatchTransport.new(
           direct_sender:,
+          on_delivered:,
           export_concurrency: export_concurrency_from_env,
           max_request_bytes: max_request_bytes_from_env
         )
@@ -131,38 +164,22 @@ module Bitfab
         end
       end
 
-      # Count the spans submitted for each replay trace and forget them, so the
-      # replay barrier knows how many spans the server must have persisted
-      # before the run can be finalized.
-      def take_replay_span_counts(trace_ids)
+      # Bounded, because the only thing that removes an entry is the encode
+      # that consumes it: a span the processor drops under backpressure never
+      # reaches encode, and its ref would otherwise sit here for the life of the
+      # process. The queue is the ceiling on how many spans can be waiting at
+      # once, so anything older than that is already unreachable. Evicting a
+      # live one costs an ack, and the barrier falls back to asking the server.
+      def record_carrier_ref(span_id, ref)
         ensure_process_state
         submission_mutex.synchronize do
-          counts = trace_ids.each_with_object({}) do |trace_id, acc|
-            next unless replay_trace_submissions.include?(trace_id)
-
-            acc[trace_id] = (trace_submission_span_ids.delete(trace_id) || Set.new).size
-          end
-          replay_trace_submissions.subtract(trace_ids)
-          counts
+          carrier_refs[span_id] = ref
+          carrier_refs.shift while carrier_refs.size > MAX_QUEUE_SIZE
         end
       end
 
-      def record_submission(operation, payload)
-        ensure_process_state
-        source_trace_id = payload["sourceTraceId"]
-        unless source_trace_id.is_a?(String)
-          raw_trace = payload["externalTrace"] || payload["rawTrace"]
-          source_trace_id = raw_trace["id"] if raw_trace.is_a?(Hash)
-        end
-        return unless source_trace_id.is_a?(String)
-
-        submission_mutex.synchronize do
-          if operation == "external_span"
-            record_span_submission(source_trace_id, payload)
-          elsif payload["completed"] == true
-            record_trace_completion(source_trace_id, payload)
-          end
-        end
+      def take_carrier_ref(span_id)
+        submission_mutex.synchronize { carrier_refs.delete(span_id) }
       end
 
       def monotonic_now
@@ -181,31 +198,14 @@ module Bitfab
         reset_state_after_fork
       end
 
-      # Replay submission counts belong to the parent's run, never the child's.
+      # Carrier refs belong to the parent's run, never the child's.
       def reset_state_after_fork
         @state_pid = Process.pid
-        @trace_submission_span_ids = {}
-        @replay_trace_submissions = Set.new
+        @carrier_refs = {}
       end
 
       private
 
-      def record_span_submission(source_trace_id, payload)
-        raw_span = payload["rawSpan"]
-        span_id = raw_span["id"] if raw_span.is_a?(Hash)
-        span_id = "submission-#{monotonic_now}" unless span_id.is_a?(String)
-        (trace_submission_span_ids[source_trace_id] ||= Set.new) << span_id
-      end
-
-      def record_trace_completion(source_trace_id, payload)
-        if payload["testRunId"].is_a?(String)
-          replay_trace_submissions << source_trace_id
-          trace_submission_span_ids[source_trace_id] ||= Set.new
-        else
-          trace_submission_span_ids.delete(source_trace_id)
-        end
-      end
-
       def current_process_transports
         live_transports_mutex.synchronize { live_transports.select { |t| t.owner_pid == Process.pid } }
       end
@@ -222,22 +222,24 @@ module Bitfab
         @submission_mutex ||= Mutex.new
       end
 
-      def trace_submission_span_ids
-        @trace_submission_span_ids ||= {}
-      end
-
-      def replay_trace_submissions
-        @replay_trace_submissions ||= Set.new
+      def carrier_refs
+        @carrier_refs ||= {}
       end
     end
 
     # Encodes carrier spans as OTLP/JSON and posts them straight to Bitfab.
     class DirectExporter
-      def initialize(direct_sender:, max_request_bytes:, max_request_batch_size:, export_concurrency:)
+      def initialize(direct_sender:, max_request_bytes:, max_request_batch_size:, export_concurrency:, on_delivered: nil)
         @direct_sender = direct_sender
+        @on_delivered = on_delivered
         @max_request_bytes = max_request_bytes
         @max_request_batch_size = max_request_batch_size
         @export_concurrency = export_concurrency
+        # Eager, unlike the module-level mutexes: this one is first touched by
+        # the export threads fanning out a batch, and `||=` racing between them
+        # would hand each a different mutex and no mutual exclusion at all.
+        @throttle_mutex = Mutex.new
+        @throttled_until = 0.0
       end
 
       def export(spans, timeout: nil)
@@ -324,6 +326,10 @@ module Bitfab
             prepared = Compress.prepare_request_body(Encoder.encode_request(envelope, request_spans))
             if prepared.wire_bytes <= @max_request_bytes
               send_with_retries(prepared)
+              # Refs come from the original spans: trimming rebuilds a span
+              # without one, and a trimmed carrier still reached the server
+              # under its own identity.
+              report_delivered(batch.spans)
               return true
             end
           end
@@ -345,72 +351,132 @@ module Bitfab
           request_raw_bytes = envelope.size + trimmed.size
           already_trimmed = true
         end
-      rescue PayloadTooLargeError
-        warn_oversized(batch.spans.size)
-        false
-      rescue PartialSuccessError
+      rescue DeliveryError => e
+        # Returned, never re-raised: a raise here escapes the export worker and
+        # aborts the whole batch fan-out at join, where TypeScript and Python
+        # record a failed export and let sibling batches finish.
+        if e.oversized
+          warn_oversized(batch.spans.size)
+        else
+          Bitfab.warn_always("failed to export an OpenTelemetry span batch: #{e.message}")
+        end
         false
       rescue => e
-        Bitfab.warn_once(
-          "otel-export-failed",
-          "failed to export an OpenTelemetry span batch (further occurrences " \
-          "suppressed): #{e.message}"
-        )
+        Bitfab.warn_always("failed to export an OpenTelemetry span batch: #{e.message}")
         false
       end
 
       def warn_oversized(span_count)
         subject = (span_count == 1) ? "a single OpenTelemetry span" : "an OpenTelemetry span batch"
-        Bitfab.warn_once(
-          "otel-payload-too-large",
-          "#{subject} exceeded the ingestion request limit and could not be exported"
-        )
+        Bitfab.warn_always("#{subject} exceeded the ingestion request limit and could not be exported")
       end
 
       def send_with_retries(request)
         attempt = 0
+        # One budget for the whole exchange, waits included: the processor kills
+        # the export at this deadline, so a wait past it cannot be served.
+        deadline = Otel.monotonic_now + EXPORT_TIMEOUT_MILLIS / 1000.0
         begin
           attempt += 1
-          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, request, EXPORT_TIMEOUT_MILLIS / 1000.0)
-          check_partial_success(response)
-        rescue PartialSuccessError
-          raise
+          await_throttle(deadline)
+          @direct_sender.call(request, [0, deadline - Otel.monotonic_now].max)
         rescue => e
-          raise PayloadTooLargeError if response_status(e) == 413
+          record_throttle(e)
+          raise if oversized?(e)
           raise if attempt >= EXPORT_RETRIES || !retryable?(e)
 
-          sleep(RETRY_DELAY_SECONDS)
+          wait = retry_wait_seconds(e, attempt - 1, deadline - Otel.monotonic_now)
+          raise if wait.nil?
+
+          sleep(wait)
           retry
         end
       end
 
-      def check_partial_success(response)
-        partial_success = response.is_a?(Hash) ? response["partialSuccess"] : nil
-        return unless partial_success.is_a?(Hash)
+      # Announce the carriers a request delivered. Wrapped because a listener
+      # that raises must never turn a delivered batch into a failed export.
+      def report_delivered(spans)
+        return if @on_delivered.nil?
 
-        rejected = partial_success["rejectedSpans"]
-        return if rejected.nil? || rejected.to_s == "0"
+        refs = spans.filter_map(&:ref)
+        return if refs.empty?
 
-        Bitfab.warn_once(
-          "otel-partial-success",
-          "OTLP ingestion rejected #{rejected} span(s): " \
-          "#{partial_success["errorMessage"] || "no reason provided"}"
-        )
-        raise PartialSuccessError
+        begin
+          @on_delivered.call(refs)
+        rescue => e
+          Bitfab.warn_once("otel-delivery-listener-failed", "a delivery listener raised: #{e.message}")
+        end
+      end
+
+      # Only what the sender classified. Anything else reaching here is a fault
+      # in the sender itself, and retrying a deterministic bug just delays it.
+      def retryable?(error)
+        error.is_a?(DeliveryError) && error.retryable
+      end
+
+      def oversized?(error)
+        error.is_a?(DeliveryError) && error.oversized
+      end
+
+      # How long to wait before the next send attempt, or nil to stop trying.
+      #
+      # A server that sent Retry-After has told us when it wants us back, so
+      # that wait is honored exactly. Clamping it would return early, which is
+      # the single thing the server asked us not to do; when the wait is longer
+      # than we are willing to hold a batch, the honest answer is to give up
+      # rather than come back sooner and add load to something already
+      # struggling.
+      #
+      # Absent an instruction, back off exponentially so a struggling server is
+      # not hit on a fixed cadence, and jitter it so every client in a fleet
+      # does not return in lockstep.
+      # Half the budget, not all of it: a wait is only worth taking if what is
+      # left afterwards can still carry the request. Spending the whole budget
+      # waiting means being killed mid-wait, losing the batch anyway.
+      def retry_wait_seconds(error, attempt, remaining_seconds)
+        affordable = remaining_seconds / 2.0
+        requested = error.is_a?(DeliveryError) ? error.retry_after_ms : nil
+        if requested
+          return nil unless requested / 1000.0 < affordable
+
+          return requested / 1000.0
+        end
+
+        backoff = [RETRY_BASE_DELAY_MILLIS * (2**attempt), RETRY_BACKOFF_CEILING_MILLIS].min
+        jittered = (backoff / 2.0 + rand * (backoff / 2.0)) / 1000.0
+        (jittered < affordable) ? jittered : nil
       end
 
-      def response_status(error)
-        return nil unless error.respond_to?(:response)
+      # Remember a throttle the server asked for, so the requests fanned out
+      # alongside this one respect it too. Delaying only the request that was
+      # refused leaves the others in the window hitting a server that just asked
+      # for room.
+      def record_throttle(error)
+        requested = error.is_a?(DeliveryError) ? error.retry_after_ms : nil
+        return unless requested
 
-        response = error.response
-        response.respond_to?(:code) ? response.code.to_i : nil
+        @throttle_mutex.synchronize do
+          @throttled_until = [@throttled_until.to_f, Otel.monotonic_now + requested / 1000.0].max
+        end
       end
 
-      def retryable?(error)
-        status = response_status(error)
-        return true if status.nil?
+      # Waits out an active throttle, or reports the batch undeliverable when
+      # the throttle outlasts what we are willing to hold it for. Either way
+      # nothing is sent while the server has asked us to stay away.
+      def await_throttle(deadline)
+        remaining = @throttle_mutex.synchronize { @throttled_until.to_f - Otel.monotonic_now }
+        return if remaining <= 0
+
+        # Waited out, not refused: OTLP asks the client to hold off until the
+        # window passes. Only a throttle outliving what the budget can serve is
+        # refused, because the processor would kill the wait before it sent.
+        if remaining >= (deadline - Otel.monotonic_now) / 2.0
+          raise DeliveryError.new(
+            "OTLP ingestion is throttled for another #{remaining.round(1)}s, longer than the export budget"
+          )
+        end
 
-        status >= 500 || [408, 425, 429].include?(status)
+        sleep(remaining)
       end
     end
 
@@ -458,10 +524,11 @@ module Bitfab
 
       def initialize(direct_sender:, max_export_batch_size: nil,
         max_request_batch_size: DIRECT_MAX_REQUEST_BATCH_SIZE, max_queue_size: MAX_QUEUE_SIZE,
-        export_concurrency: DEFAULT_EXPORT_CONCURRENCY, max_request_bytes: nil)
+        export_concurrency: DEFAULT_EXPORT_CONCURRENCY, max_request_bytes: nil, on_delivered: nil)
         raise ArgumentError, "max_request_batch_size must be a positive integer" unless max_request_batch_size.positive?
 
         @direct_sender = direct_sender
+        @on_delivered = on_delivered
         @max_export_batch_size = max_export_batch_size || DIRECT_MAX_EXPORT_BATCH_SIZE
         @max_request_batch_size = max_request_batch_size
         @max_queue_size = max_queue_size
@@ -475,18 +542,18 @@ module Bitfab
         Otel.register_transport(self)
       end
 
-      def submit(operation, payload)
-        Otel.record_submission(operation, payload)
+      def submit(operation, payload, meta = nil)
+        meta ||= CarrierMeta.new
         # Encoding is the expensive part of a submit and needs no mutual
         # exclusion, so it stays outside the lock.
-        name = Encoder.span_name(operation, payload)
+        name = meta.name || "bitfab.#{operation}"
         encoded_payload = Serialize.safe_generate(
           payload,
           max_carrier_bytes: PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
         )
-        started_at = Encoder.timestamp(payload, "started_at")
-        ended_at = Encoder.timestamp(payload, "ended_at")
-        errored = Encoder.error?(payload)
+        started_at = meta.started_at || Time.now
+        ended_at = meta.ended_at || Time.now
+        errored = meta.errored
 
         # Pipeline construction and the closed check are serialized: concurrent
         # first submits in a forked child would otherwise each build a pipeline,
@@ -506,6 +573,7 @@ module Bitfab
             },
             start_timestamp: started_at
           )
+          Otel.record_carrier_ref(span.context.hex_span_id, meta.ref) if meta.ref
           span.status = OpenTelemetry::Trace::Status.error if errored
           span.finish(end_timestamp: ended_at)
         end
@@ -630,7 +698,8 @@ module Bitfab
           direct_sender: @direct_sender,
           max_request_bytes: @max_request_bytes,
           max_request_batch_size: @max_request_batch_size,
-          export_concurrency: @export_concurrency
+          export_concurrency: @export_concurrency,
+          on_delivered: @on_delivered
         )
       end
     end
@@ -640,7 +709,7 @@ module Bitfab
       class << self
         def encode_span(span)
           encoded = JSON.generate(span_to_otlp(span))
-          EncodedSpan.new(encoded, encoded.bytesize)
+          EncodedSpan.new(encoded, encoded.bytesize, Otel.take_carrier_ref(span.hex_span_id))
         end
 
         def trim_span(span)
@@ -691,40 +760,6 @@ module Bitfab
           result
         end
 
-        def span_name(operation, payload)
-          raw_span = payload["rawSpan"]
-          if operation == "external_span" && raw_span.is_a?(Hash)
-            name = raw_span.dig("span_data", "name")
-            return name if name.is_a?(String)
-          end
-          return payload["traceFunctionKey"] if payload["traceFunctionKey"].is_a?(String)
-
-          "bitfab.#{operation}"
-        end
-
-        def timestamp(payload, field)
-          raw = payload.dig("rawSpan", field) if payload["rawSpan"].is_a?(Hash)
-          if raw.nil?
-            raw_trace = payload["externalTrace"] || payload["rawTrace"]
-            raw = raw_trace[field] if raw_trace.is_a?(Hash)
-          end
-          return Time.now unless raw.is_a?(String)
-
-          begin
-            Time.iso8601(raw)
-          rescue ArgumentError
-            Time.now
-          end
-        end
-
-        def error?(payload)
-          raw_span = payload["rawSpan"]
-          return true if raw_span.is_a?(Hash) && !raw_span.dig("span_data", "error").nil?
-
-          errors = payload["errors"]
-          errors.is_a?(Array) && !errors.empty?
-        end
-
         private
 
         def status(span_status)

data/lib/bitfab/replay.rb

@@ -865,7 +865,8 @@ module Bitfab
         adapt_ctx:,
         db_branch_lease: lease,
         source_bitfab_trace_id: original_trace_id,
-        db_snapshot_ref:
+        db_snapshot_ref:,
+        http_client:
       )
     rescue => e
       warn "Bitfab: replay item for span #{original_span_id} failed before execution: #{e.message}"
@@ -1009,19 +1010,33 @@ module Bitfab
 
     # Block until the server has persisted every replay trace this run queued.
     #
-    # Flushing only proves the batches left the process. The server writes a
-    # trace's spans and completion independently, so the barrier also polls
-    # replay status with the span count each trace owes: a trace is ready only
-    # once its completion and all of its spans have landed.
+    # In the normal case the verdict is local: ingestion commits a request's
+    # carriers before it answers, so a flush that delivered every carrier has
+    # already proven the traces are whole and polling only asked the server to
+    # repeat itself. Acks cannot settle a request that timed out client-side
+    # after the server committed, so anything short of fully delivered falls
+    # through to the server poll, which stays the authority there.
     def wait_for_replay_persistence(http_client, test_run_id, sdk_trace_ids)
-      expected_span_counts = Transport.take_replay_span_counts(sdk_trace_ids.compact)
-      return if expected_span_counts.empty?
+      trace_ids = sdk_trace_ids.compact
 
-      unless Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS)
-        raise "Replay traces could not be flushed before the delivery deadline " \
-          "(test_run_id #{test_run_id})."
+      # Checked before flushing: a run whose traces never closed submitted
+      # nothing to wait on, and must not emit an export request to discover it.
+      unless http_client.closed_deliveries?(trace_ids)
+        http_client.take_trace_deliveries(trace_ids)
+        return
       end
 
+      # A failed flush is a hint, not a verdict. It means delivery was not
+      # CONFIRMED within the deadline, which is not the same as lost: the export
+      # timeout can fire while requests are still in flight and the server goes
+      # on to persist every one of them. Failing here failed runs whose data had
+      # fully landed.
+      flushed = Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS)
+
+      deliveries = http_client.take_trace_deliveries(trace_ids).select { |_id, report| report.closed }
+      expected_span_counts = deliveries.transform_values(&:span_count)
+      return if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
+
       deadline = Otel.monotonic_now + PERSISTENCE_TIMEOUT_SECONDS
       missing = expected_span_counts.keys
 
@@ -1036,9 +1051,15 @@ module Bitfab
         sleep((deadline - Otel.monotonic_now).clamp(0, PERSISTENCE_POLL_SECONDS))
       end
 
+      cause = if flushed
+        ""
+      else
+        " Delivery was also not confirmed before the flush deadline, so the " \
+          "spans likely never reached the server."
+      end
       raise "Replay traces were not fully persisted before the delivery deadline " \
         "(test_run_id #{test_run_id}, missing #{missing.length} of " \
-        "#{expected_span_counts.length} trace(s))."
+        "#{expected_span_counts.length} trace(s)).#{cause}"
     end
 
     # Normalize a complete-replay tokens hash (string-keyed JSON) into the
@@ -1059,7 +1080,7 @@ module Bitfab
     def execute_item(item, receiver, method_name, test_run_id, input_source_span_id = nil, metrics = {},
       input_source_trace_id: nil, mock_strategy: "marked", mock_tree: nil, mock_overrides: nil,
       fetch_span_output: nil, adapt_inputs: nil, adapt_ctx: nil, db_branch_lease: nil, source_bitfab_trace_id: nil,
-      db_snapshot_ref: nil)
+      db_snapshot_ref: nil, http_client: nil)
       args, kwargs = Serialize.deserialize_inputs(item)
 
       fn_result = nil
@@ -1070,6 +1091,10 @@ module Bitfab
       # complete-replay loop). Carried on the item under :_sdk_trace_id, never
       # surfaced as the public :trace_id.
       sdk_trace_id = SecureRandom.uuid
+      # Declared before this item's first span is submitted: the transport
+      # records delivery only for traces someone asked about, so anything
+      # submitted before this would go untracked.
+      http_client&.track_trace_deliveries([sdk_trace_id])
 
       begin
         ReplayContext.with_context(

data/lib/bitfab/transport.rb

@@ -7,8 +7,8 @@ module Bitfab
   # mechanism, so callers never depend on OpenTelemetry directly.
   module Transport
     class << self
-      def create_trace_transport(direct_sender:)
-        Otel.create_transport(direct_sender:)
+      def create_trace_transport(direct_sender:, on_delivered: nil)
+        Otel.create_transport(direct_sender:, on_delivered:)
       end
 
       def flush_trace_transports(timeout = 30.0)
@@ -18,10 +18,6 @@ module Bitfab
       def shutdown_trace_transports(timeout = 30.0)
         Otel.shutdown_transports(timeout)
       end
-
-      def take_replay_span_counts(trace_ids)
-        Otel.take_replay_span_counts(trace_ids)
-      end
     end
   end
 end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.36.5"
+  VERSION = "0.36.6"
 end

data/lib/bitfab/warn_once.rb

@@ -27,6 +27,16 @@ module Bitfab
       end
     end
 
+    # Every occurrence, not one per key. Reserved for the export path, where
+    # the count IS the signal: suppressing repeats makes a sustained outage
+    # look like a single blip, and TypeScript, Python and Go all report each
+    # dropped batch.
+    def warn_always(message)
+      warn "Bitfab: #{message}"
+    rescue Exception # rubocop:disable Lint/RescueException
+      # Logging must never crash the host app (e.g. a closed $stderr).
+    end
+
     # Test-only: clear the dedup set so a warning can fire again.
     def _reset_warn_once
       @warn_once_mutex.synchronize { @warn_once_seen.clear }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.36.5
+  version: 0.36.6
 platform: ruby
 authors:
 - Harvest Team