bitfab 0.36.7 → 0.36.9

package/dist/node.cjs

@@ -97,7 +97,7 @@ var __version__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.36.7";
+    __version__ = "0.36.9";
   }
 });
 
@@ -223,10 +223,11 @@ var init_errors = __esm({
   "src/errors.ts"() {
     "use strict";
     BitfabError = class extends Error {
-      constructor(message, url, status) {
+      constructor(message, url, status, retryAfterMs) {
         super(message);
         this.url = url;
         this.status = status;
+        this.retryAfterMs = retryAfterMs;
         this.name = "BitfabError";
       }
     };
@@ -542,6 +543,23 @@ var init_serializePayload = __esm({
   }
 });
 
+// src/transportTypes.ts
+var DeliveryError;
+var init_transportTypes = __esm({
+  "src/transportTypes.ts"() {
+    "use strict";
+    DeliveryError = class extends Error {
+      constructor(message, options = {}) {
+        super(message);
+        this.name = "DeliveryError";
+        this.retryable = options.retryable ?? false;
+        this.oversized = options.oversized ?? false;
+        this.retryAfterMs = options.retryAfterMs;
+      }
+    };
+  }
+});
+
 // src/unrefTimer.ts
 function unrefTimer(timer) {
   const handle = timer;
@@ -581,59 +599,6 @@ function logError(message, error) {
   } catch {
   }
 }
-function recordTraceSubmission(operation, payload) {
-  const sourceTraceId = resolveSourceTraceId(payload);
-  if (sourceTraceId === void 0) {
-    return;
-  }
-  if (operation === "external_span") {
-    const rawSpan = asRecord2(payload.rawSpan);
-    if (typeof rawSpan?.id !== "string") {
-      submissionCounter += 1;
-    }
-    const sourceSpanId = typeof rawSpan?.id === "string" ? rawSpan.id : `submission-${submissionCounter}`;
-    const existing = traceSubmissionSpanIds.get(sourceTraceId);
-    if (existing) {
-      existing.add(sourceSpanId);
-    } else {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set([sourceSpanId]));
-    }
-    return;
-  }
-  if (payload.completed !== true) {
-    return;
-  }
-  if (typeof payload.testRunId === "string") {
-    replayTraceSubmissions.add(sourceTraceId);
-    if (!traceSubmissionSpanIds.has(sourceTraceId)) {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set());
-    }
-  } else {
-    traceSubmissionSpanIds.delete(sourceTraceId);
-  }
-}
-function takeReplaySpanCounts(traceIds) {
-  const counts = {};
-  for (const traceId of traceIds) {
-    if (!replayTraceSubmissions.has(traceId)) {
-      continue;
-    }
-    counts[traceId] = traceSubmissionSpanIds.get(traceId)?.size ?? 0;
-    traceSubmissionSpanIds.delete(traceId);
-    replayTraceSubmissions.delete(traceId);
-  }
-  return counts;
-}
-function asRecord2(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
-}
-function resolveSourceTraceId(payload) {
-  if (typeof payload.sourceTraceId === "string") {
-    return payload.sourceTraceId;
-  }
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  return typeof rawTrace?.id === "string" ? rawTrace.id : void 0;
-}
 function otlpValue(value) {
   if (typeof value === "boolean") {
     return { boolValue: value };
@@ -691,7 +656,11 @@ function spanToOtlp(span) {
 }
 function encodeSpan(span) {
   const json = JSON.stringify(spanToOtlp(span));
-  return { json, size: byteLength(json) };
+  return {
+    json,
+    size: byteLength(json),
+    ref: carrierRefs.get(span)
+  };
 }
 function trimEncodedSpan(span) {
   try {
@@ -774,48 +743,27 @@ async function mapWithConcurrency(items, limit, task) {
   await Promise.all(workers);
   return results;
 }
-function responseStatus(error) {
-  return error instanceof BitfabError ? error.status : void 0;
-}
 function isRetryable(error) {
-  const status = responseStatus(error);
-  if (status === void 0) {
-    return true;
-  }
-  return RETRYABLE_STATUSES.has(status) || status >= 500;
+  return error instanceof DeliveryError && error.retryable;
 }
-function endSpan(span, endTime) {
-  span.end(endTime);
+function isOversized(error) {
+  return error instanceof DeliveryError && error.oversized;
 }
-function spanName(operation, payload) {
-  if (operation === "external_span") {
-    const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-    if (typeof spanData?.name === "string") {
-      return spanData.name;
-    }
+function retryWaitMillis(error, attempt, remainingMillis) {
+  const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+  const affordable = remainingMillis / 2;
+  if (requested !== void 0) {
+    return requested < affordable ? requested : null;
   }
-  if (typeof payload.traceFunctionKey === "string") {
-    return payload.traceFunctionKey;
-  }
-  return `bitfab.${operation}`;
-}
-function payloadTimestamp(payload, field) {
-  const rawSpan = asRecord2(payload.rawSpan);
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  const raw = rawSpan?.[field] ?? rawTrace?.[field];
-  if (typeof raw !== "string") {
-    return void 0;
-  }
-  const parsed = Date.parse(raw);
-  return Number.isNaN(parsed) ? void 0 : parsed;
+  const backoff = Math.min(
+    RETRY_BASE_DELAY_MILLIS * 2 ** attempt,
+    RETRY_BACKOFF_CEILING_MILLIS
+  );
+  const jittered = backoff / 2 + Math.random() * (backoff / 2);
+  return jittered < affordable ? jittered : null;
 }
-function hasError(payload) {
-  const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-  if (spanData?.error != null) {
-    return true;
-  }
-  const errors = payload.errors;
-  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
+function endSpan(span, endTime) {
+  span.end(endTime);
 }
 function createOtelTransport(options) {
   return new OtelBatchTransport({
@@ -854,7 +802,7 @@ function shutdownOtelTransports(timeoutMs = DEFAULT_LIFECYCLE_TIMEOUT_MS) {
     (transport, remaining) => transport.shutdown(remaining)
   );
 }
-var import_api, import_core, import_resources, import_sdk_trace_base, OPERATION_ATTRIBUTE, PAYLOAD_ATTRIBUTE, OTLP_TRACES_ENDPOINT, MAX_EXPORT_REQUEST_BYTES, MAX_DECOMPRESSED_REQUEST_BYTES, MAX_REQUEST_BYTES_ENV, EXPORT_CONCURRENCY_ENV, MAX_QUEUE_SIZE, DIRECT_MAX_EXPORT_BATCH_SIZE, DIRECT_MAX_REQUEST_BATCH_SIZE, DEFAULT_EXPORT_CONCURRENCY, MAX_EXPORT_CONCURRENCY, SCHEDULE_DELAY_MILLIS, EXPORT_TIMEOUT_MILLIS, RETRY_DELAY_MILLIS, MAX_SEND_ATTEMPTS, DEFAULT_LIFECYCLE_TIMEOUT_MS, RETRYABLE_STATUSES, liveTransports, traceSubmissionSpanIds, replayTraceSubmissions, submissionCounter, SPAN_SEPARATOR_BYTES, OtlpPayloadTooLargeError, OtlpPartialSuccessError, BitfabSpanExporter, DeliveryTrackingExporter, OtelBatchTransport;
+var import_api, import_core, import_resources, import_sdk_trace_base, OPERATION_ATTRIBUTE, PAYLOAD_ATTRIBUTE, MAX_EXPORT_REQUEST_BYTES, MAX_DECOMPRESSED_REQUEST_BYTES, MAX_REQUEST_BYTES_ENV, EXPORT_CONCURRENCY_ENV, MAX_QUEUE_SIZE, DIRECT_MAX_EXPORT_BATCH_SIZE, DIRECT_MAX_REQUEST_BATCH_SIZE, DEFAULT_EXPORT_CONCURRENCY, MAX_EXPORT_CONCURRENCY, SCHEDULE_DELAY_MILLIS, EXPORT_TIMEOUT_MILLIS, RETRY_BASE_DELAY_MILLIS, RETRY_BACKOFF_CEILING_MILLIS, MAX_SEND_ATTEMPTS, DEFAULT_LIFECYCLE_TIMEOUT_MS, liveTransports, carrierRefs, SPAN_SEPARATOR_BYTES, BitfabSpanExporter, DeliveryTrackingExporter, OtelBatchTransport;
 var init_otel = __esm({
   "src/otel.ts"() {
     "use strict";
@@ -868,11 +816,11 @@ var init_otel = __esm({
     init_payloadBudget();
     init_readEnv();
     init_serializePayload();
+    init_transportTypes();
     init_unrefTimer();
     init_warnOnce();
     OPERATION_ATTRIBUTE = "bitfab.operation";
     PAYLOAD_ATTRIBUTE = "bitfab.payload";
-    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
     MAX_EXPORT_REQUEST_BYTES = 3e6;
     MAX_DECOMPRESSED_REQUEST_BYTES = 8e6;
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES";
@@ -884,25 +832,23 @@ var init_otel = __esm({
     MAX_EXPORT_CONCURRENCY = 64;
     SCHEDULE_DELAY_MILLIS = 5e3;
     EXPORT_TIMEOUT_MILLIS = 3e4;
-    RETRY_DELAY_MILLIS = 100;
+    RETRY_BASE_DELAY_MILLIS = 100;
+    RETRY_BACKOFF_CEILING_MILLIS = 5e3;
     MAX_SEND_ATTEMPTS = 3;
     DEFAULT_LIFECYCLE_TIMEOUT_MS = 3e4;
-    RETRYABLE_STATUSES = /* @__PURE__ */ new Set([408, 425, 429]);
     liveTransports = /* @__PURE__ */ new Set();
-    traceSubmissionSpanIds = /* @__PURE__ */ new Map();
-    replayTraceSubmissions = /* @__PURE__ */ new Set();
-    submissionCounter = 0;
+    carrierRefs = /* @__PURE__ */ new WeakMap();
     SPAN_SEPARATOR_BYTES = 1;
-    OtlpPayloadTooLargeError = class extends Error {
-    };
-    OtlpPartialSuccessError = class extends Error {
-    };
     BitfabSpanExporter = class {
-      constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency) {
+      constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency, onDelivered, exportTimeoutMillis = EXPORT_TIMEOUT_MILLIS) {
         this.directSender = directSender;
         this.maxRequestBytes = maxRequestBytes;
         this.maxRequestBatchSize = maxRequestBatchSize;
         this.exportConcurrency = exportConcurrency;
+        this.onDelivered = onDelivered;
+        this.exportTimeoutMillis = exportTimeoutMillis;
+        /** Epoch ms until which the server has asked this exporter to stay away. */
+        this.throttledUntil = 0;
       }
       export(spans, resultCallback) {
         void this.exportAsync(spans).then(
@@ -968,6 +914,7 @@ var init_otel = __esm({
               );
               if (prepared.wireBytes <= this.maxRequestBytes) {
                 await this.sendWithRetries(prepared);
+                this.reportDelivered(batch.spans);
                 return true;
               }
             }
@@ -995,15 +942,12 @@ var init_otel = __esm({
             alreadyTrimmed = true;
           }
         } catch (error) {
-          if (error instanceof OtlpPayloadTooLargeError) {
+          if (isOversized(error)) {
             logError(
               batch.spans.length === 1 ? "a single OpenTelemetry span exceeded the ingestion request limit and could not be exported" : "an OpenTelemetry span batch exceeded the ingestion request limit and could not be exported"
             );
             return false;
           }
-          if (error instanceof OtlpPartialSuccessError) {
-            return false;
-          }
           logError("failed to export an OpenTelemetry span batch", error);
           return false;
         }
@@ -1021,37 +965,79 @@ var init_otel = __esm({
        * the server does not yet understand. The fix is a client-supplied
        * idempotency key that ingestion dedupes on.
        */
+      /**
+       * Remember a throttle the server asked for, so the requests fanned out
+       * alongside this one respect it too. Delaying only the request that was
+       * refused leaves the other seven in the window hitting a server that just
+       * asked for room.
+       */
+      recordThrottle(error) {
+        const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+        if (requested !== void 0) {
+          this.throttledUntil = Math.max(
+            this.throttledUntil,
+            Date.now() + requested
+          );
+        }
+      }
+      /**
+       * Waits out an active throttle, or reports the batch undeliverable when the
+       * throttle outlasts what we are willing to hold it for. Either way nothing is
+       * sent while the server has asked us to stay away.
+       */
+      async awaitThrottle(deadline) {
+        const remaining = this.throttledUntil - Date.now();
+        if (remaining <= 0) {
+          return;
+        }
+        if (remaining >= (deadline - Date.now()) / 2) {
+          throw new DeliveryError(
+            `OTLP ingestion is throttled for another ${remaining}ms, longer than the export budget`
+          );
+        }
+        await delay(remaining);
+      }
       async sendWithRetries(request) {
+        const deadline = Date.now() + this.exportTimeoutMillis;
         for (let attempt = 0; attempt < MAX_SEND_ATTEMPTS; attempt += 1) {
           try {
-            const response = await this.directSender(
-              OTLP_TRACES_ENDPOINT,
-              request,
-              EXPORT_TIMEOUT_MILLIS
-            );
-            const partialSuccess = asRecord2(response?.partialSuccess);
-            const rejected = partialSuccess?.rejectedSpans;
-            if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
-              logError(
-                `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
-              );
-              throw new OtlpPartialSuccessError();
-            }
+            await this.awaitThrottle(deadline);
+            await this.directSender(request, Math.max(0, deadline - Date.now()));
             return;
           } catch (error) {
-            if (error instanceof OtlpPartialSuccessError) {
+            if (isOversized(error)) {
               throw error;
             }
-            if (responseStatus(error) === 413) {
-              throw new OtlpPayloadTooLargeError();
-            }
+            this.recordThrottle(error);
             if (attempt === MAX_SEND_ATTEMPTS - 1 || !isRetryable(error)) {
               throw error;
             }
-            await delay(RETRY_DELAY_MILLIS);
+            const wait = retryWaitMillis(error, attempt, deadline - Date.now());
+            if (wait === null) {
+              throw error;
+            }
+            await delay(wait);
           }
         }
       }
+      /**
+       * Announce the carriers a request delivered. Wrapped because a listener that
+       * throws must never turn a delivered batch into a failed export.
+       */
+      reportDelivered(spans) {
+        if (this.onDelivered === void 0) {
+          return;
+        }
+        const refs = spans.map((span) => span.ref).filter((ref) => ref !== void 0);
+        if (refs.length === 0) {
+          return;
+        }
+        try {
+          this.onDelivered(refs);
+        } catch (error) {
+          logError("a delivery listener threw", error);
+        }
+      }
       async shutdown() {
       }
       async forceFlush() {
@@ -1109,7 +1095,9 @@ var init_otel = __esm({
             options.directSender,
             maxRequestBytes,
             maxRequestBatchSize,
-            options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY
+            options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY,
+            options.onDelivered,
+            options.exportTimeoutMillis ?? EXPORT_TIMEOUT_MILLIS
           )
         );
         this.processor = new import_sdk_trace_base.BatchSpanProcessor(this.deliveryTracker, {
@@ -1133,8 +1121,7 @@ var init_otel = __esm({
         this.tracer = this.provider.getTracer("bitfab", __version__);
         liveTransports.add(this);
       }
-      submit(operation, payload) {
-        recordTraceSubmission(operation, payload);
+      submit(operation, payload, meta = {}) {
         if (this.closed) {
           warnOnce(
             "otel-submit-after-shutdown",
@@ -1155,17 +1142,20 @@ var init_otel = __esm({
               ].join(", ")}); they were stubbed so the span still ships, but the trace may be incomplete or not replayable.`
             );
           }
-          const span = this.tracer.startSpan(spanName(operation, payload), {
+          const span = this.tracer.startSpan(meta.name ?? `bitfab.${operation}`, {
             attributes: {
               [OPERATION_ATTRIBUTE]: operation,
               [PAYLOAD_ATTRIBUTE]: body
             },
-            startTime: payloadTimestamp(payload, "started_at")
+            startTime: meta.startTime
           });
-          if (hasError(payload)) {
+          if (meta.ref !== void 0) {
+            carrierRefs.set(span, meta.ref);
+          }
+          if (meta.errored === true) {
             span.setStatus({ code: import_api.SpanStatusCode.ERROR });
           }
-          endSpan(span, payloadTimestamp(payload, "ended_at"));
+          endSpan(span, meta.endTime);
         } catch (error) {
           logError("failed to queue an OpenTelemetry span", error);
         }
@@ -1215,9 +1205,6 @@ function flushTraceTransports(timeoutMs) {
 function shutdownTraceTransports(timeoutMs) {
   return shutdownOtelTransports(timeoutMs);
 }
-function takeReplaySpanCounts2(traceIds) {
-  return takeReplaySpanCounts(traceIds);
-}
 var init_transport = __esm({
   "src/transport.ts"() {
     "use strict";
@@ -1266,7 +1253,96 @@ async function waitForPromises(promises, timeoutMs) {
     }
   }
 }
-var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, HttpClient;
+function readHeader(response, name) {
+  try {
+    return response.headers?.get(name) ?? null;
+  } catch {
+    return null;
+  }
+}
+function parseRetryAfterMs(header) {
+  const value = header?.trim();
+  if (!value) {
+    return void 0;
+  }
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) {
+    return seconds >= 0 ? seconds * 1e3 : void 0;
+  }
+  const at = Date.parse(value);
+  if (Number.isNaN(at)) {
+    return void 0;
+  }
+  return Math.max(0, at - Date.now());
+}
+function carrierMeta(operation, payload, ref) {
+  return {
+    ref,
+    name: carrierName(operation, payload),
+    startTime: payloadTimestamp(payload, "started_at"),
+    endTime: payloadTimestamp(payload, "ended_at"),
+    errored: payloadHasError(payload)
+  };
+}
+function carrierName(operation, payload) {
+  if (operation === "external_span") {
+    const spanData = asPayloadRecord(
+      asPayloadRecord(payload.rawSpan)?.span_data
+    );
+    if (typeof spanData?.name === "string") {
+      return spanData.name;
+    }
+  }
+  if (typeof payload.traceFunctionKey === "string") {
+    return payload.traceFunctionKey;
+  }
+  return `bitfab.${operation}`;
+}
+function payloadTimestamp(payload, field) {
+  const rawSpan = asPayloadRecord(payload.rawSpan);
+  const rawTrace = asPayloadRecord(payload.externalTrace) ?? asPayloadRecord(payload.rawTrace);
+  const raw = rawSpan?.[field] ?? rawTrace?.[field];
+  if (typeof raw !== "string") {
+    return void 0;
+  }
+  const parsed = Date.parse(raw);
+  return Number.isNaN(parsed) ? void 0 : parsed;
+}
+function payloadHasError(payload) {
+  const spanData = asPayloadRecord(asPayloadRecord(payload.rawSpan)?.span_data);
+  if (spanData?.error != null) {
+    return true;
+  }
+  const errors = payload.errors;
+  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
+}
+function asPayloadRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+function carrierRef(payload) {
+  const traceId = sourceTraceIdOf(payload);
+  if (traceId === void 0) {
+    return void 0;
+  }
+  const rawSpan = payload.rawSpan;
+  if (rawSpan === void 0) {
+    return { traceId };
+  }
+  const spanId = rawSpan?.id;
+  return {
+    traceId,
+    spanId: typeof spanId === "string" ? spanId : `submission-${++carrierSeq}`
+  };
+}
+function sourceTraceIdOf(payload) {
+  if (typeof payload.sourceTraceId === "string") {
+    return payload.sourceTraceId;
+  }
+  const rawTrace = payload.externalTrace ?? payload.rawTrace;
+  const id = rawTrace?.id;
+  return typeof id === "string" ? id : void 0;
+}
+var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, OTLP_TRACES_ENDPOINT, RETRYABLE_STATUSES, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, carrierSeq, HttpClient;
 var init_http = __esm({
   "src/http.ts"() {
     "use strict";
@@ -1276,9 +1352,12 @@ var init_http = __esm({
     init_replayContext();
     init_serializePayload();
     init_transport();
+    init_transportTypes();
     init_unrefTimer();
     init_warnOnce();
     REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS = 3e5;
+    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
+    RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
     EXIT_FLUSH_TIMEOUT_MS = 5e3;
     DEFAULT_LIFECYCLE_TIMEOUT_MS2 = 3e4;
     pendingTracePromises = /* @__PURE__ */ new Set();
@@ -1298,8 +1377,12 @@ var init_http = __esm({
         });
       });
     }
+    carrierSeq = 0;
     HttpClient = class {
       constructor(config) {
+        // Only traces a caller asked about are tracked, so ordinary tracing stores
+        // nothing here.
+        this.traceDeliveries = /* @__PURE__ */ new Map();
         // Deferred span work owned by THIS client. The module-global set backs the
         // process-wide `flushTraces()` and the exit hook, but per-client lifecycle
         // must not wait on another client's slow finalize: a false `close()` failure
@@ -1335,13 +1418,131 @@ var init_http = __esm({
         }
         if (!this.traceTransport) {
           this.traceTransport = createTraceTransport({
-            directSender: (endpoint, request, timeoutMs) => this.sendPrepared(endpoint, request, {
-              timeout: timeoutMs
-            })
+            directSender: (request, timeoutMs) => this.deliverCarriers(request, timeoutMs),
+            onDelivered: (refs) => this.recordDeliveredCarriers(refs)
           });
         }
         return this.traceTransport;
       }
+      /**
+       * Post one encoded batch and decide what the server's answer means, so the
+       * transport never reads a response. Rejections and permanent statuses come
+       * back as a non-retryable {@link DeliveryError}; anything the server might
+       * still accept on a second try comes back retryable.
+       */
+      async deliverCarriers(request, timeoutMs) {
+        let response;
+        try {
+          response = await this.sendPrepared(
+            OTLP_TRACES_ENDPOINT,
+            request,
+            { timeout: timeoutMs }
+          );
+        } catch (error) {
+          const status = error instanceof BitfabError ? error.status : void 0;
+          if (status === void 0) {
+            throw new DeliveryError(`OTLP ingestion failed: ${String(error)}`, {
+              retryable: true
+            });
+          }
+          throw new DeliveryError(`OTLP ingestion failed with HTTP ${status}`, {
+            retryable: RETRYABLE_STATUSES.has(status),
+            oversized: status === 413,
+            ...error instanceof BitfabError && error.retryAfterMs !== void 0 ? { retryAfterMs: error.retryAfterMs } : {}
+          });
+        }
+        const partialSuccess = asPayloadRecord(response?.partialSuccess);
+        const rejected = partialSuccess?.rejectedSpans;
+        if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
+          throw new DeliveryError(
+            `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
+          );
+        }
+      }
+      /**
+       * Start tracking delivery for `traceIds`. Nothing is recorded for a trace
+       * that was never tracked, so ordinary tracing costs no bookkeeping at all.
+       */
+      trackTraceDeliveries(traceIds) {
+        for (const traceId of traceIds) {
+          if (!this.traceDeliveries.has(traceId)) {
+            this.traceDeliveries.set(traceId, {
+              submittedSpanIds: /* @__PURE__ */ new Set(),
+              ackedSpanIds: /* @__PURE__ */ new Set(),
+              closed: false,
+              closingAcked: false
+            });
+          }
+        }
+      }
+      /** Whether any tracked trace has had its closing carrier submitted. */
+      hasClosedDeliveries(traceIds) {
+        return traceIds.some((traceId) => this.traceDeliveries.get(traceId)?.closed);
+      }
+      /**
+       * Report what each tracked trace submitted and whether the server confirmed
+       * it, and stop tracking them. Every id passed is freed, so a caller cannot
+       * leak a record for a trace that never closed.
+       *
+       * `delivered` is only meaningful once a flush has settled: acks land before
+       * an export resolves, so a flush that reported success has already collected
+       * every ack it is going to collect.
+       */
+      takeTraceDeliveries(traceIds) {
+        const reports = {};
+        for (const traceId of traceIds) {
+          const delivery = this.traceDeliveries.get(traceId);
+          if (delivery === void 0) {
+            continue;
+          }
+          this.traceDeliveries.delete(traceId);
+          reports[traceId] = {
+            spanCount: delivery.submittedSpanIds.size,
+            closed: delivery.closed,
+            delivered: delivery.closingAcked && [...delivery.submittedSpanIds].every(
+              (spanId) => delivery.ackedSpanIds.has(spanId)
+            )
+          };
+        }
+        return reports;
+      }
+      /** Build a carrier's meta and record what it adds to its trace's expected set. */
+      recordedMeta(operation, payload, ref) {
+        this.recordSubmittedCarrier(ref);
+        return carrierMeta(operation, payload, ref);
+      }
+      recordSubmittedCarrier(ref) {
+        if (ref === void 0) {
+          return;
+        }
+        const delivery = this.traceDeliveries.get(ref.traceId);
+        if (delivery === void 0) {
+          return;
+        }
+        if (ref.spanId === void 0) {
+          delivery.closed = true;
+        } else {
+          delivery.submittedSpanIds.add(ref.spanId);
+        }
+      }
+      /**
+       * Ingestion commits every carrier in a request before it answers, so a
+       * delivered ref is proof its row exists: the same fact the replay status
+       * endpoint would report, already in hand.
+       */
+      recordDeliveredCarriers(refs) {
+        for (const ref of refs) {
+          const delivery = this.traceDeliveries.get(ref.traceId);
+          if (delivery === void 0) {
+            continue;
+          }
+          if (ref.spanId === void 0) {
+            delivery.closingAcked = true;
+          } else {
+            delivery.ackedSpanIds.add(ref.spanId);
+          }
+        }
+      }
       /**
        * Track deferred span work so this client's own lifecycle waits for it, and
        * so the process-wide flush and exit hook do too.
@@ -1451,7 +1652,8 @@ var init_http = __esm({
             throw new BitfabError(
               `HTTP ${response.status}: ${errorText.slice(0, 500)}`,
               void 0,
-              response.status
+              response.status,
+              parseRetryAfterMs(readHeader(response, "retry-after"))
             );
           }
           const result = await response.json();
@@ -1537,11 +1739,12 @@ var init_http = __esm({
        * the OTLP carrier has no path to carry it.
        */
       sendInternalTrace(functionId, payload) {
-        this.getTraceTransport()?.submit("internal_trace", {
-          ...payload,
-          functionId,
-          sdkVersion: __version__
-        });
+        const body = { ...payload, functionId, sdkVersion: __version__ };
+        this.getTraceTransport()?.submit(
+          "internal_trace",
+          body,
+          carrierMeta("internal_trace", body, void 0)
+        );
       }
       /**
        * Queue an external span (from withSpan wrapper or OpenAI tracing) onto this
@@ -1550,10 +1753,11 @@ var init_http = __esm({
        * promise.
        */
       sendExternalSpan(payload) {
-        this.getTraceTransport()?.submit("external_span", {
-          ...payload,
-          sdkVersion: __version__
-        });
+        this.getTraceTransport()?.submit(
+          "external_span",
+          { ...payload, sdkVersion: __version__ },
+          this.recordedMeta("external_span", payload, carrierRef(payload))
+        );
       }
       /**
        * Queue an external trace completion (from OpenAI tracing) onto this
@@ -1562,10 +1766,15 @@ var init_http = __esm({
        * server-authoritative barrier in `replay.ts`, not by awaiting this call.
        */
       sendExternalTrace(payload) {
-        this.getTraceTransport()?.submit("external_trace", {
-          ...payload,
-          sdkVersion: __version__
-        });
+        this.getTraceTransport()?.submit(
+          "external_trace",
+          { ...payload, sdkVersion: __version__ },
+          this.recordedMeta(
+            "external_trace",
+            payload,
+            payload.completed === true ? carrierRef(payload) : void 0
+          )
+        );
       }
       /**
        * Partial update of an existing trace identified by its Bitfab trace ID.
@@ -2161,7 +2370,9 @@ __export(replay_exports, {
   ReplayError: () => ReplayError,
   replay: () => replay,
   reportReplayProgress: () => reportReplayProgress,
-  serializeReplayResult: () => serializeReplayResult
+  serializeReplayResult: () => serializeReplayResult,
+  sleepForReplayPersistence: () => sleepForReplayPersistence,
+  waitForReplayPersistence: () => waitForReplayPersistence
 });
 function dbBranchEnabled(dbBranch) {
   return dbBranch !== void 0 && dbBranch !== false;
@@ -2449,15 +2660,29 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
     REPLAY_PERSISTENCE_TIMEOUT_MS
   );
   if (!deferredSettled) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     throw new BitfabError(
       `Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (testRunId ${testRunId}).`
     );
   }
-  const expectedSpanCounts = takeReplaySpanCounts2(replayedTraceIds);
-  if (Object.keys(expectedSpanCounts).length === 0) {
+  if (!httpClient.hasClosedDeliveries(replayedTraceIds)) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     return;
   }
   const flushed = await flushTraces(REPLAY_PERSISTENCE_TIMEOUT_MS);
+  const deliveries = httpClient.takeTraceDeliveries(replayedTraceIds);
+  const expectedSpanCounts = {};
+  let allDelivered = true;
+  for (const [traceId, delivery] of Object.entries(deliveries)) {
+    if (!delivery.closed) {
+      continue;
+    }
+    expectedSpanCounts[traceId] = delivery.spanCount;
+    allDelivered = allDelivered && delivery.delivered;
+  }
+  if (allDelivered) {
+    return;
+  }
   const deadline = Date.now() + REPLAY_PERSISTENCE_TIMEOUT_MS;
   let missing = Object.keys(expectedSpanCounts).length;
   while (true) {
@@ -2475,17 +2700,18 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
     if (Date.now() >= deadline) {
       break;
     }
-    await sleep(Math.min(100, Math.max(0, deadline - Date.now())));
+    await sleepForReplayPersistence(
+      Math.min(100, Math.max(0, deadline - Date.now()))
+    );
   }
   const cause = flushed ? "" : " Delivery was also not confirmed before the flush deadline, so the spans likely never reached the server.";
   throw new BitfabError(
     `Replay traces were not fully persisted before the delivery deadline (testRunId ${testRunId}, missing ${missing} of ${Object.keys(expectedSpanCounts).length} trace(s)).${cause}`
   );
 }
-function sleep(ms) {
+function sleepForReplayPersistence(ms) {
   return new Promise((resolve) => {
-    const timer = setTimeout(resolve, ms);
-    unrefTimer(timer);
+    setTimeout(resolve, ms);
   });
 }
 async function mapWithConcurrency2(tasks, maxConcurrency, onSettled, onStarted) {
@@ -2566,6 +2792,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     ...registeredOverrides
   ];
   const replayedTraceIds = serverItems.map(() => randomUuid());
+  httpClient.trackTraceDeliveries(replayedTraceIds);
   const tasks = serverItems.map(
     (serverItem, index) => () => processItem(
       httpClient,
@@ -2763,8 +2990,6 @@ var init_replay = __esm({
     init_randomUuid();
     init_replayContext();
     init_serialize();
-    init_transport();
-    init_unrefTimer();
     REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
     BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
     ReplayError = class extends BitfabError {