bitfab 0.36.7 → 0.36.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f18189ba0b9b7a2c8cd23ee6672f927af7f9f6400457d71d86b90c13a61698e9
-  data.tar.gz: 61f38c9eb03166eeef759aca2914a015e92fe89247c163541c786331ed8fa26d
+  metadata.gz: 23d0d3e96160502ecf10da7fe952d08fec34439595e4c61e50ce9ea7c086a11c
+  data.tar.gz: 7bb58eab2f4749bd073593ec4c359d8460d1a1012eab60727999ad765a3e6d1e
 SHA512:
-  metadata.gz: 8940e1f95426c3fa3343d48a340b3b4b2d2adec10c714b9e204ac7fdcfa348fba28b6bf4ed1820c1acd67536883a21701ccf401bc547cf2c3bd4c9b929169f6f
-  data.tar.gz: 88935462030c7997ab153444e327666cc0040a0baed397c19322bfd1cedff2c9ef6624b64ce227aba02cad471d6ad05c639b1cdfbdb6a69f3a6e2f1346dca395
+  metadata.gz: 7360217eac4dfaa7ae66c645415598a9b794e2433434b425de54a1922243a93012270d9146309aca8d3ee65e9084da047cd6c6a5751cade83e1eb7b7d7458ab7
+  data.tar.gz: ee2575e01da6352effba7a1943621aa44280aabd5ead4aed76ce85561db6ccd8251c28c369de846fce1879e8ff4a08da9a6211b924d40f0a78430bec98302e76

data/lib/bitfab/http_client.rb

@@ -13,10 +13,12 @@ require_relative "warn_once"
 
 module Bitfab
   # What a caller learns about one tracked trace once it takes it back.
-  DeliveryReport = Struct.new(:span_count, :closed, :delivered, keyword_init: true)
+  # server_trace_id is the server's assigned traces.id, read back off the OTLP
+  # ingest response; nil when the server predates the field or nothing was acked.
+  DeliveryReport = Struct.new(:span_count, :closed, :delivered, :server_trace_id, keyword_init: true)
 
   TraceDelivery = Struct.new(:submitted_span_ids, :acked_span_ids, :closed, :closing_acked,
-    keyword_init: true)
+    :server_trace_id, keyword_init: true)
 
   class HttpClient
     OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
@@ -413,6 +415,9 @@ module Bitfab
         )
       end
 
+      server_trace_ids = response.is_a?(Hash) ? response["traceIds"] : nil
+      record_server_trace_ids(server_trace_ids) if server_trace_ids.is_a?(Hash)
+
       partial_success = response.is_a?(Hash) ? response["partialSuccess"] : nil
       return unless partial_success.is_a?(Hash)
 
@@ -492,14 +497,40 @@ module Bitfab
             span_count: delivery.submitted_span_ids.size,
             closed: delivery.closed,
             delivered: delivery.closing_acked &&
-              delivery.submitted_span_ids.subset?(delivery.acked_span_ids)
+              delivery.submitted_span_ids.subset?(delivery.acked_span_ids),
+            server_trace_id: delivery.server_trace_id
           )
         end
       end
     end
 
+    # The server's assigned traces.id for a tracked trace if it has already been
+    # read back off an ingest response, without stopping tracking. Lets a replay
+    # surface the id mid-run for items whose spans already landed.
+    def peek_server_trace_id(trace_id)
+      @delivery_mutex.synchronize { trace_deliveries[trace_id]&.server_trace_id }
+    end
+
     private
 
+    # Record the server's assigned traces.id for each tracked source trace, read
+    # back off the OTLP ingest response. Keyed by source trace id, the same key
+    # the delivery ledger uses. Untracked ids are ignored.
+    #
+    # Called on exporter threads, hence the mutex.
+    def record_server_trace_ids(mapping)
+      @delivery_mutex.synchronize do
+        mapping.each do |source_trace_id, server_trace_id|
+          next unless server_trace_id.is_a?(String)
+
+          delivery = trace_deliveries[source_trace_id]
+          next if delivery.nil?
+
+          delivery.server_trace_id = server_trace_id
+        end
+      end
+    end
+
     def record_submitted_carrier(ref)
       return if ref.nil?
 

data/lib/bitfab/replay.rb

@@ -315,10 +315,21 @@ module Bitfab
       # to finalize once the server confirms every replay trace it queued: the
       # trace-ID mapping complete_replay builds would otherwise race the
       # in-flight batches and hand back nil for every item.
-      preserve_replay_failure(result_items, test_run_id, full_test_run_url) do
+      delivered_trace_ids = preserve_replay_failure(result_items, test_run_id, full_test_run_url) do
         wait_for_replay_persistence(http_client, test_run_id, result_items.map { |item| item[:_sdk_trace_id] })
       end
 
+      # Primary source for item[:trace_id]: the server's assigned traces.id, read
+      # back per trace off the ingest response during the flush above (server-
+      # sourced, no polling, already in hand before complete_replay). The
+      # complete_replay map below remains the fallback for servers that don't
+      # return ids on ingest.
+      result_items.each do |item|
+        local_id = item[:_sdk_trace_id]
+        read_back = local_id && delivered_trace_ids[local_id]
+        item[:trace_id] = read_back if read_back
+      end
+
       # complete_replay finalizes the run and returns the token/diagnostic
       # mapping; its failures propagate loudly because a run that never
       # completed can't be finalized.
@@ -345,9 +356,10 @@ module Bitfab
         result_items.each do |item|
           local_id = item[:_sdk_trace_id]
           mapped = local_id && trace_id_map[local_id]
-          # Write the real server replay trace id in as it comes back; the item
-          # held nil until now (the client correlation id is never surfaced).
-          item[:trace_id] = mapped
+          # Fallback fill for servers that did not return ids on the ingest
+          # response; the read-back loop above already set it when they did, so
+          # never overwrite a resolved id with nil here.
+          item[:trace_id] ||= mapped
           if item[:error].nil?
             completed_count += 1
             missing << local_id if mapped.nil?
@@ -558,28 +570,28 @@ module Bitfab
 
       base, from_trunk = resolved
 
-      tracked = git(root, ["diff", "--name-status", "--no-renames", "-z", base, "--", ":!.bitfab"]) || ""
+      tracked = git(root, ["diff", "--name-status", "--find-renames", "-z", base, "--", ":!.bitfab"]) || ""
       untracked = git(root, ["ls-files", "--others", "--exclude-standard", "-z", "--", ":!.bitfab"]) || ""
 
       entries = cc_parse_name_status_z(tracked)
-      untracked.split("\0").reject(&:empty?).each { |p| entries << ["A", p] }
+      untracked.split("\0").reject(&:empty?).each { |p| entries << ["A", p, p] }
       return nil if entries.empty?
 
       files = []
       total = 0
-      entries.each do |status, path|
+      entries.each do |status, before_path, path|
         break if files.length >= CC_MAX_FILES
 
         # Skip oversized files by size BEFORE reading their full contents, so a
         # huge changed file can't OOM/stall replay just to be discarded.
-        before_bytes = (status == "A") ? 0 : cc_blob_bytes(root, base, path)
+        before_bytes = (status == "A") ? 0 : cc_blob_bytes(root, base, before_path)
         after_bytes = (status == "D") ? 0 : cc_working_bytes(root, path)
         next if before_bytes > CC_MAX_FILE_BYTES || after_bytes > CC_MAX_FILE_BYTES
 
         # Normalize CRLF -> LF on both sides: git's blob is already LF-normalized
         # (autocrlf clean), so a raw CRLF working file would otherwise skew every
         # line as changed. Comparing/storing LF keeps the diff meaningful.
-        before = (status == "A") ? "" : (git(root, ["show", "#{base}:#{path}"]) || "").gsub("\r\n", "\n")
+        before = (status == "A") ? "" : (git(root, ["show", "#{base}:#{before_path}"]) || "").gsub("\r\n", "\n")
         after = (status == "D") ? "" : cc_read_working_file(root, path).gsub("\r\n", "\n")
         next if before == after
 
@@ -607,8 +619,15 @@ module Bitfab
       out = []
       i = 0
       while i + 1 < parts.length
-        out << [parts[i][0], parts[i + 1]]
+        status = parts[i][0]
+        before_path = parts[i + 1]
         i += 2
+        if ["R", "C"].include?(status) && i < parts.length
+          out << [status, before_path, parts[i]]
+          i += 1
+        else
+          out << [status, before_path, before_path]
+        end
       end
       out
     end
@@ -653,11 +672,13 @@ module Bitfab
       errored = 0
       # Each event carries the single item that just finished so a progress UI
       # can render per-trace pass/fail as the run streams. The item's :trace_id
-      # is nil at this stage: the server replay trace id isn't known until run()
-      # writes it in after complete_replay, and the client correlation id is
-      # never surfaced. original_trace_id (the historical trace being replayed,
-      # taken from the server item) is what a UI keys on to identify what just
-      # finished. source_trace_id/source_span_id are kept as deprecated aliases.
+      # is the server's assigned traces.id, read back off the ingest response and
+      # surfaced as this item finishes (its trace is flushed on finish); nil only
+      # if that flush could not confirm delivery in time (the end-of-run barrier
+      # then fills the returned item), and the client correlation id is never
+      # surfaced. original_trace_id (the historical trace being replayed, taken
+      # from the server item) is what a UI keys on to identify what just finished.
+      # source_trace_id/source_span_id are kept as deprecated aliases.
       report_start = lambda do |original_trace_id, original_span_id|
         progress_mutex.synchronize do
           started += 1
@@ -679,7 +700,27 @@ module Bitfab
         end
       end
 
+      # Flush an item's trace the moment it finishes, so its server traces.id is
+      # read back off the ingest response and surfaced on the item as it settles,
+      # instead of waiting for the end-of-run barrier. Waiting until the end is
+      # worst at low concurrency: each trace is closed and ready when its item
+      # finishes, but nothing fills a batch to force an export, so ready ids sit
+      # in the queue for the whole run. Serializing on a mutex keeps concurrent
+      # finishers from storming the server: a flush drains the whole queue, so
+      # the first finisher exports the batch and the rest find nothing to send.
+      flush_mutex = Mutex.new
+      flush_finished_item_trace = lambda do
+        flush_mutex.synchronize { Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS) }
+      rescue => e
+        warn "Bitfab: replay per-item flush failed: #{e.message}"
+      end
+
       report = lambda do |result, original_trace_id, original_span_id, test_run_id|
+        # Deliver this item's trace now, then read its server id back off the
+        # ingest response. A flush failure never crashes the run: the id stays
+        # nil and the end-of-run barrier remains the authority on persistence.
+        flush_finished_item_trace.call
+        result[:trace_id] = http_client.peek_server_trace_id(result[:_sdk_trace_id]) || result[:trace_id]
         progress_mutex.synchronize do
           completed += 1
           error = result[:error]
@@ -1023,7 +1064,7 @@ module Bitfab
       # nothing to wait on, and must not emit an export request to discover it.
       unless http_client.closed_deliveries?(trace_ids)
         http_client.take_trace_deliveries(trace_ids)
-        return
+        return {}
       end
 
       # A failed flush is a hint, not a verdict. It means delivery was not
@@ -1033,9 +1074,15 @@ module Bitfab
       # fully landed.
       flushed = Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS)
 
-      deliveries = http_client.take_trace_deliveries(trace_ids).select { |_id, report| report.closed }
+      raw_deliveries = http_client.take_trace_deliveries(trace_ids)
+      # Server-assigned traces.id per replay trace, read back off the ingest
+      # response during the flush and keyed by the client-side replay id.
+      read_back_trace_ids = raw_deliveries.each_with_object({}) do |(id, report), acc|
+        acc[id] = report.server_trace_id unless report.server_trace_id.nil?
+      end
+      deliveries = raw_deliveries.select { |_id, report| report.closed }
       expected_span_counts = deliveries.transform_values(&:span_count)
-      return if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
+      return read_back_trace_ids if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
 
       deadline = Otel.monotonic_now + PERSISTENCE_TIMEOUT_SECONDS
       missing = expected_span_counts.keys
@@ -1045,7 +1092,7 @@ module Bitfab
         ready = status["traceIds"]
         ready = {} unless ready.is_a?(Hash)
         missing = expected_span_counts.keys - ready.keys
-        return if missing.empty?
+        return read_back_trace_ids if missing.empty?
         break if Otel.monotonic_now >= deadline
 
         sleep((deadline - Otel.monotonic_now).clamp(0, PERSISTENCE_POLL_SECONDS))

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.36.7"
+  VERSION = "0.36.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.36.7
+  version: 0.36.9
 platform: ruby
 authors:
 - Harvest Team