bitfab 0.36.6 → 0.36.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd73e07889e4f74cd94c6ca534e423a7aa95e3ce6f1e1ba306e9fe33df8de8df
-  data.tar.gz: fb2eb3cd99066d8338f6ad03d18bc3a8de9f3955659aa66835c83066a95d3595
+  metadata.gz: 87c160ba40f51aca6fded283a601f2e917b948403436879e7dcca1793673a4aa
+  data.tar.gz: 730c4a41a9f26ee51e2e25490d8dd89aa1205a0acb091f0ce9e2560b40583622
 SHA512:
-  metadata.gz: b7155a3d0426c00f57668975485f2ba722bd3f0555dd897024f586290b36265f5851d49cc88242eb552421758b9d624b31e21edfb8290e68f693110e8b93cd32
-  data.tar.gz: 87ae26a27f8864294f431b80d87f0a7f768fdb2a2f09786929dad90d61316ce7c1106e75fe9271c63260cebe3545a2e73676e94295aa281f4d24c0d101184b0f
+  metadata.gz: 58ca8e468386578b6792e8aef86a74ab7f601b432e7333391931a028b8dd0a7e1006a317be2aeccc58fcdef5a89ab550fe950c89527ecf0babe888ed5cb32d1e
+  data.tar.gz: 56aa1281a7583d0c106b64044def3494221e33e9e1b45f25b2ee0113c43a72f6fc76c55ca2c0230b0430f92c2e55e7a0a7174ca980ac53c972b72a632d912155

data/lib/bitfab/http_client.rb

@@ -13,10 +13,12 @@ require_relative "warn_once"
 
 module Bitfab
   # What a caller learns about one tracked trace once it takes it back.
-  DeliveryReport = Struct.new(:span_count, :closed, :delivered, keyword_init: true)
+  # server_trace_id is the server's assigned traces.id, read back off the OTLP
+  # ingest response; nil when the server predates the field or nothing was acked.
+  DeliveryReport = Struct.new(:span_count, :closed, :delivered, :server_trace_id, keyword_init: true)
 
   TraceDelivery = Struct.new(:submitted_span_ids, :acked_span_ids, :closed, :closing_acked,
-    keyword_init: true)
+    :server_trace_id, keyword_init: true)
 
   class HttpClient
     OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
@@ -413,6 +415,9 @@ module Bitfab
         )
       end
 
+      server_trace_ids = response.is_a?(Hash) ? response["traceIds"] : nil
+      record_server_trace_ids(server_trace_ids) if server_trace_ids.is_a?(Hash)
+
       partial_success = response.is_a?(Hash) ? response["partialSuccess"] : nil
       return unless partial_success.is_a?(Hash)
 
@@ -492,14 +497,40 @@ module Bitfab
             span_count: delivery.submitted_span_ids.size,
             closed: delivery.closed,
             delivered: delivery.closing_acked &&
-              delivery.submitted_span_ids.subset?(delivery.acked_span_ids)
+              delivery.submitted_span_ids.subset?(delivery.acked_span_ids),
+            server_trace_id: delivery.server_trace_id
           )
         end
       end
     end
 
+    # The server's assigned traces.id for a tracked trace if it has already been
+    # read back off an ingest response, without stopping tracking. Lets a replay
+    # surface the id mid-run for items whose spans already landed.
+    def peek_server_trace_id(trace_id)
+      @delivery_mutex.synchronize { trace_deliveries[trace_id]&.server_trace_id }
+    end
+
     private
 
+    # Record the server's assigned traces.id for each tracked source trace, read
+    # back off the OTLP ingest response. Keyed by source trace id, the same key
+    # the delivery ledger uses. Untracked ids are ignored.
+    #
+    # Called on exporter threads, hence the mutex.
+    def record_server_trace_ids(mapping)
+      @delivery_mutex.synchronize do
+        mapping.each do |source_trace_id, server_trace_id|
+          next unless server_trace_id.is_a?(String)
+
+          delivery = trace_deliveries[source_trace_id]
+          next if delivery.nil?
+
+          delivery.server_trace_id = server_trace_id
+        end
+      end
+    end
+
     def record_submitted_carrier(ref)
       return if ref.nil?
 

data/lib/bitfab/otel.rb

@@ -19,6 +19,7 @@ module Bitfab
   module Otel
     OPERATION_ATTRIBUTE = "bitfab.operation"
     PAYLOAD_ATTRIBUTE = "bitfab.payload"
+    CARRIER_REF_IVAR = :@bitfab_carrier_ref
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES"
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
     MAX_EXPORT_REQUEST_BYTES = 3_000_000
@@ -164,24 +165,6 @@ module Bitfab
         end
       end
 
-      # Bounded, because the only thing that removes an entry is the encode
-      # that consumes it: a span the processor drops under backpressure never
-      # reaches encode, and its ref would otherwise sit here for the life of the
-      # process. The queue is the ceiling on how many spans can be waiting at
-      # once, so anything older than that is already unreachable. Evicting a
-      # live one costs an ack, and the barrier falls back to asking the server.
-      def record_carrier_ref(span_id, ref)
-        ensure_process_state
-        submission_mutex.synchronize do
-          carrier_refs[span_id] = ref
-          carrier_refs.shift while carrier_refs.size > MAX_QUEUE_SIZE
-        end
-      end
-
-      def take_carrier_ref(span_id)
-        submission_mutex.synchronize { carrier_refs.delete(span_id) }
-      end
-
       def monotonic_now
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
@@ -198,10 +181,8 @@ module Bitfab
         reset_state_after_fork
       end
 
-      # Carrier refs belong to the parent's run, never the child's.
       def reset_state_after_fork
         @state_pid = Process.pid
-        @carrier_refs = {}
       end
 
       private
@@ -217,14 +198,6 @@ module Bitfab
       def live_transports_mutex
         @live_transports_mutex ||= Mutex.new
       end
-
-      def submission_mutex
-        @submission_mutex ||= Mutex.new
-      end
-
-      def carrier_refs
-        @carrier_refs ||= {}
-      end
     end
 
     # Encodes carrier spans as OTLP/JSON and posts them straight to Bitfab.
@@ -551,6 +524,7 @@ module Bitfab
           payload,
           max_carrier_bytes: PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
         )
+        encoded_payload.instance_variable_set(CARRIER_REF_IVAR, meta.ref) if meta.ref
         started_at = meta.started_at || Time.now
         ended_at = meta.ended_at || Time.now
         errored = meta.errored
@@ -573,7 +547,6 @@ module Bitfab
             },
             start_timestamp: started_at
           )
-          Otel.record_carrier_ref(span.context.hex_span_id, meta.ref) if meta.ref
           span.status = OpenTelemetry::Trace::Status.error if errored
           span.finish(end_timestamp: ended_at)
         end
@@ -709,7 +682,9 @@ module Bitfab
       class << self
         def encode_span(span)
           encoded = JSON.generate(span_to_otlp(span))
-          EncodedSpan.new(encoded, encoded.bytesize, Otel.take_carrier_ref(span.hex_span_id))
+          payload = span.attributes&.fetch(PAYLOAD_ATTRIBUTE, nil)
+          ref = payload&.instance_variable_get(CARRIER_REF_IVAR)
+          EncodedSpan.new(encoded, encoded.bytesize, ref)
         end
 
         def trim_span(span)

data/lib/bitfab/replay.rb

@@ -315,10 +315,21 @@ module Bitfab
       # to finalize once the server confirms every replay trace it queued: the
       # trace-ID mapping complete_replay builds would otherwise race the
       # in-flight batches and hand back nil for every item.
-      preserve_replay_failure(result_items, test_run_id, full_test_run_url) do
+      delivered_trace_ids = preserve_replay_failure(result_items, test_run_id, full_test_run_url) do
         wait_for_replay_persistence(http_client, test_run_id, result_items.map { |item| item[:_sdk_trace_id] })
       end
 
+      # Primary source for item[:trace_id]: the server's assigned traces.id, read
+      # back per trace off the ingest response during the flush above (server-
+      # sourced, no polling, already in hand before complete_replay). The
+      # complete_replay map below remains the fallback for servers that don't
+      # return ids on ingest.
+      result_items.each do |item|
+        local_id = item[:_sdk_trace_id]
+        read_back = local_id && delivered_trace_ids[local_id]
+        item[:trace_id] = read_back if read_back
+      end
+
       # complete_replay finalizes the run and returns the token/diagnostic
       # mapping; its failures propagate loudly because a run that never
       # completed can't be finalized.
@@ -345,9 +356,10 @@ module Bitfab
         result_items.each do |item|
           local_id = item[:_sdk_trace_id]
           mapped = local_id && trace_id_map[local_id]
-          # Write the real server replay trace id in as it comes back; the item
-          # held nil until now (the client correlation id is never surfaced).
-          item[:trace_id] = mapped
+          # Fallback fill for servers that did not return ids on the ingest
+          # response; the read-back loop above already set it when they did, so
+          # never overwrite a resolved id with nil here.
+          item[:trace_id] ||= mapped
           if item[:error].nil?
             completed_count += 1
             missing << local_id if mapped.nil?
@@ -653,11 +665,13 @@ module Bitfab
       errored = 0
       # Each event carries the single item that just finished so a progress UI
       # can render per-trace pass/fail as the run streams. The item's :trace_id
-      # is nil at this stage: the server replay trace id isn't known until run()
-      # writes it in after complete_replay, and the client correlation id is
-      # never surfaced. original_trace_id (the historical trace being replayed,
-      # taken from the server item) is what a UI keys on to identify what just
-      # finished. source_trace_id/source_span_id are kept as deprecated aliases.
+      # is the server's assigned traces.id, read back off the ingest response and
+      # surfaced as this item finishes (its trace is flushed on finish); nil only
+      # if that flush could not confirm delivery in time (the end-of-run barrier
+      # then fills the returned item), and the client correlation id is never
+      # surfaced. original_trace_id (the historical trace being replayed, taken
+      # from the server item) is what a UI keys on to identify what just finished.
+      # source_trace_id/source_span_id are kept as deprecated aliases.
       report_start = lambda do |original_trace_id, original_span_id|
         progress_mutex.synchronize do
           started += 1
@@ -679,7 +693,27 @@ module Bitfab
         end
       end
 
+      # Flush an item's trace the moment it finishes, so its server traces.id is
+      # read back off the ingest response and surfaced on the item as it settles,
+      # instead of waiting for the end-of-run barrier. Waiting until the end is
+      # worst at low concurrency: each trace is closed and ready when its item
+      # finishes, but nothing fills a batch to force an export, so ready ids sit
+      # in the queue for the whole run. Serializing on a mutex keeps concurrent
+      # finishers from storming the server: a flush drains the whole queue, so
+      # the first finisher exports the batch and the rest find nothing to send.
+      flush_mutex = Mutex.new
+      flush_finished_item_trace = lambda do
+        flush_mutex.synchronize { Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS) }
+      rescue => e
+        warn "Bitfab: replay per-item flush failed: #{e.message}"
+      end
+
       report = lambda do |result, original_trace_id, original_span_id, test_run_id|
+        # Deliver this item's trace now, then read its server id back off the
+        # ingest response. A flush failure never crashes the run: the id stays
+        # nil and the end-of-run barrier remains the authority on persistence.
+        flush_finished_item_trace.call
+        result[:trace_id] = http_client.peek_server_trace_id(result[:_sdk_trace_id]) || result[:trace_id]
         progress_mutex.synchronize do
           completed += 1
           error = result[:error]
@@ -1023,7 +1057,7 @@ module Bitfab
       # nothing to wait on, and must not emit an export request to discover it.
       unless http_client.closed_deliveries?(trace_ids)
         http_client.take_trace_deliveries(trace_ids)
-        return
+        return {}
       end
 
       # A failed flush is a hint, not a verdict. It means delivery was not
@@ -1033,9 +1067,15 @@ module Bitfab
       # fully landed.
       flushed = Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS)
 
-      deliveries = http_client.take_trace_deliveries(trace_ids).select { |_id, report| report.closed }
+      raw_deliveries = http_client.take_trace_deliveries(trace_ids)
+      # Server-assigned traces.id per replay trace, read back off the ingest
+      # response during the flush and keyed by the client-side replay id.
+      read_back_trace_ids = raw_deliveries.each_with_object({}) do |(id, report), acc|
+        acc[id] = report.server_trace_id unless report.server_trace_id.nil?
+      end
+      deliveries = raw_deliveries.select { |_id, report| report.closed }
       expected_span_counts = deliveries.transform_values(&:span_count)
-      return if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
+      return read_back_trace_ids if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
 
       deadline = Otel.monotonic_now + PERSISTENCE_TIMEOUT_SECONDS
       missing = expected_span_counts.keys
@@ -1045,7 +1085,7 @@ module Bitfab
         ready = status["traceIds"]
         ready = {} unless ready.is_a?(Hash)
         missing = expected_span_counts.keys - ready.keys
-        return if missing.empty?
+        return read_back_trace_ids if missing.empty?
         break if Otel.monotonic_now >= deadline
 
         sleep((deadline - Otel.monotonic_now).clamp(0, PERSISTENCE_POLL_SECONDS))

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.36.6"
+  VERSION = "0.36.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.36.6
+  version: 0.36.8
 platform: ruby
 authors:
 - Harvest Team