@bitfab/sdk 0.36.7 → 0.36.9

package/dist/{chunk-CFECTUDU.js → chunk-MBKY42QC.js}

@@ -199,10 +199,11 @@ function looksBinary(s) {
 
 // src/errors.ts
 var BitfabError = class extends Error {
-  constructor(message, url, status) {
+  constructor(message, url, status, retryAfterMs) {
     super(message);
     this.url = url;
     this.status = status;
+    this.retryAfterMs = retryAfterMs;
     this.name = "BitfabError";
   }
 };
@@ -303,7 +304,7 @@ function encodeRequestBody(body) {
 }
 
 // src/version.generated.ts
-var __version__ = "0.36.7";
+var __version__ = "0.36.9";
 
 // src/constants.ts
 var DEFAULT_SERVICE_URL = "https://bitfab.ai";
@@ -643,6 +644,17 @@ import {
   BatchSpanProcessor
 } from "@opentelemetry/sdk-trace-base";
 
+// src/transportTypes.ts
+var DeliveryError = class extends Error {
+  constructor(message, options = {}) {
+    super(message);
+    this.name = "DeliveryError";
+    this.retryable = options.retryable ?? false;
+    this.oversized = options.oversized ?? false;
+    this.retryAfterMs = options.retryAfterMs;
+  }
+};
+
 // src/unrefTimer.ts
 function unrefTimer(timer) {
   const handle = timer;
@@ -654,7 +666,6 @@ function unrefTimer(timer) {
 // src/otel.ts
 var OPERATION_ATTRIBUTE = "bitfab.operation";
 var PAYLOAD_ATTRIBUTE = "bitfab.payload";
-var OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
 var MAX_EXPORT_REQUEST_BYTES = 3e6;
 var MAX_DECOMPRESSED_REQUEST_BYTES = 8e6;
 var MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES";
@@ -666,14 +677,12 @@ var DEFAULT_EXPORT_CONCURRENCY = 32;
 var MAX_EXPORT_CONCURRENCY = 64;
 var SCHEDULE_DELAY_MILLIS = 5e3;
 var EXPORT_TIMEOUT_MILLIS = 3e4;
-var RETRY_DELAY_MILLIS = 100;
+var RETRY_BASE_DELAY_MILLIS = 100;
+var RETRY_BACKOFF_CEILING_MILLIS = 5e3;
 var MAX_SEND_ATTEMPTS = 3;
 var DEFAULT_LIFECYCLE_TIMEOUT_MS = 3e4;
-var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([408, 425, 429]);
 var liveTransports = /* @__PURE__ */ new Set();
-var traceSubmissionSpanIds = /* @__PURE__ */ new Map();
-var replayTraceSubmissions = /* @__PURE__ */ new Set();
-var submissionCounter = 0;
+var carrierRefs = /* @__PURE__ */ new WeakMap();
 function readBoundedIntEnv(name, max, fallback, warnKey) {
   const raw = readEnv(name);
   if (raw === void 0) {
@@ -699,59 +708,6 @@ function logError(message, error) {
   } catch {
   }
 }
-function recordTraceSubmission(operation, payload) {
-  const sourceTraceId = resolveSourceTraceId(payload);
-  if (sourceTraceId === void 0) {
-    return;
-  }
-  if (operation === "external_span") {
-    const rawSpan = asRecord2(payload.rawSpan);
-    if (typeof rawSpan?.id !== "string") {
-      submissionCounter += 1;
-    }
-    const sourceSpanId = typeof rawSpan?.id === "string" ? rawSpan.id : `submission-${submissionCounter}`;
-    const existing = traceSubmissionSpanIds.get(sourceTraceId);
-    if (existing) {
-      existing.add(sourceSpanId);
-    } else {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set([sourceSpanId]));
-    }
-    return;
-  }
-  if (payload.completed !== true) {
-    return;
-  }
-  if (typeof payload.testRunId === "string") {
-    replayTraceSubmissions.add(sourceTraceId);
-    if (!traceSubmissionSpanIds.has(sourceTraceId)) {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set());
-    }
-  } else {
-    traceSubmissionSpanIds.delete(sourceTraceId);
-  }
-}
-function takeReplaySpanCounts(traceIds) {
-  const counts = {};
-  for (const traceId of traceIds) {
-    if (!replayTraceSubmissions.has(traceId)) {
-      continue;
-    }
-    counts[traceId] = traceSubmissionSpanIds.get(traceId)?.size ?? 0;
-    traceSubmissionSpanIds.delete(traceId);
-    replayTraceSubmissions.delete(traceId);
-  }
-  return counts;
-}
-function asRecord2(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
-}
-function resolveSourceTraceId(payload) {
-  if (typeof payload.sourceTraceId === "string") {
-    return payload.sourceTraceId;
-  }
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  return typeof rawTrace?.id === "string" ? rawTrace.id : void 0;
-}
 function otlpValue(value) {
   if (typeof value === "boolean") {
     return { boolValue: value };
@@ -810,7 +766,11 @@ function spanToOtlp(span) {
 var SPAN_SEPARATOR_BYTES = 1;
 function encodeSpan(span) {
   const json = JSON.stringify(spanToOtlp(span));
-  return { json, size: byteLength(json) };
+  return {
+    json,
+    size: byteLength(json),
+    ref: carrierRefs.get(span)
+  };
 }
 function trimEncodedSpan(span) {
   try {
@@ -893,26 +853,35 @@ async function mapWithConcurrency(items, limit, task) {
   await Promise.all(workers);
   return results;
 }
-var OtlpPayloadTooLargeError = class extends Error {
-};
-var OtlpPartialSuccessError = class extends Error {
-};
-function responseStatus(error) {
-  return error instanceof BitfabError ? error.status : void 0;
-}
 function isRetryable(error) {
-  const status = responseStatus(error);
-  if (status === void 0) {
-    return true;
+  return error instanceof DeliveryError && error.retryable;
+}
+function isOversized(error) {
+  return error instanceof DeliveryError && error.oversized;
+}
+function retryWaitMillis(error, attempt, remainingMillis) {
+  const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+  const affordable = remainingMillis / 2;
+  if (requested !== void 0) {
+    return requested < affordable ? requested : null;
   }
-  return RETRYABLE_STATUSES.has(status) || status >= 500;
+  const backoff = Math.min(
+    RETRY_BASE_DELAY_MILLIS * 2 ** attempt,
+    RETRY_BACKOFF_CEILING_MILLIS
+  );
+  const jittered = backoff / 2 + Math.random() * (backoff / 2);
+  return jittered < affordable ? jittered : null;
 }
 var BitfabSpanExporter = class {
-  constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency) {
+  constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency, onDelivered, exportTimeoutMillis = EXPORT_TIMEOUT_MILLIS) {
     this.directSender = directSender;
     this.maxRequestBytes = maxRequestBytes;
     this.maxRequestBatchSize = maxRequestBatchSize;
     this.exportConcurrency = exportConcurrency;
+    this.onDelivered = onDelivered;
+    this.exportTimeoutMillis = exportTimeoutMillis;
+    /** Epoch ms until which the server has asked this exporter to stay away. */
+    this.throttledUntil = 0;
   }
   export(spans, resultCallback) {
     void this.exportAsync(spans).then(
@@ -978,6 +947,7 @@ var BitfabSpanExporter = class {
           );
           if (prepared.wireBytes <= this.maxRequestBytes) {
             await this.sendWithRetries(prepared);
+            this.reportDelivered(batch.spans);
             return true;
           }
         }
@@ -1005,15 +975,12 @@ var BitfabSpanExporter = class {
         alreadyTrimmed = true;
       }
     } catch (error) {
-      if (error instanceof OtlpPayloadTooLargeError) {
+      if (isOversized(error)) {
         logError(
           batch.spans.length === 1 ? "a single OpenTelemetry span exceeded the ingestion request limit and could not be exported" : "an OpenTelemetry span batch exceeded the ingestion request limit and could not be exported"
         );
         return false;
       }
-      if (error instanceof OtlpPartialSuccessError) {
-        return false;
-      }
       logError("failed to export an OpenTelemetry span batch", error);
       return false;
     }
@@ -1031,37 +998,79 @@ var BitfabSpanExporter = class {
    * the server does not yet understand. The fix is a client-supplied
    * idempotency key that ingestion dedupes on.
    */
+  /**
+   * Remember a throttle the server asked for, so the requests fanned out
+   * alongside this one respect it too. Delaying only the request that was
+   * refused leaves the other seven in the window hitting a server that just
+   * asked for room.
+   */
+  recordThrottle(error) {
+    const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+    if (requested !== void 0) {
+      this.throttledUntil = Math.max(
+        this.throttledUntil,
+        Date.now() + requested
+      );
+    }
+  }
+  /**
+   * Waits out an active throttle, or reports the batch undeliverable when the
+   * throttle outlasts what we are willing to hold it for. Either way nothing is
+   * sent while the server has asked us to stay away.
+   */
+  async awaitThrottle(deadline) {
+    const remaining = this.throttledUntil - Date.now();
+    if (remaining <= 0) {
+      return;
+    }
+    if (remaining >= (deadline - Date.now()) / 2) {
+      throw new DeliveryError(
+        `OTLP ingestion is throttled for another ${remaining}ms, longer than the export budget`
+      );
+    }
+    await delay(remaining);
+  }
   async sendWithRetries(request) {
+    const deadline = Date.now() + this.exportTimeoutMillis;
     for (let attempt = 0; attempt < MAX_SEND_ATTEMPTS; attempt += 1) {
       try {
-        const response = await this.directSender(
-          OTLP_TRACES_ENDPOINT,
-          request,
-          EXPORT_TIMEOUT_MILLIS
-        );
-        const partialSuccess = asRecord2(response?.partialSuccess);
-        const rejected = partialSuccess?.rejectedSpans;
-        if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
-          logError(
-            `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
-          );
-          throw new OtlpPartialSuccessError();
-        }
+        await this.awaitThrottle(deadline);
+        await this.directSender(request, Math.max(0, deadline - Date.now()));
         return;
       } catch (error) {
-        if (error instanceof OtlpPartialSuccessError) {
+        if (isOversized(error)) {
           throw error;
         }
-        if (responseStatus(error) === 413) {
-          throw new OtlpPayloadTooLargeError();
-        }
+        this.recordThrottle(error);
         if (attempt === MAX_SEND_ATTEMPTS - 1 || !isRetryable(error)) {
           throw error;
         }
-        await delay(RETRY_DELAY_MILLIS);
+        const wait = retryWaitMillis(error, attempt, deadline - Date.now());
+        if (wait === null) {
+          throw error;
+        }
+        await delay(wait);
       }
     }
   }
+  /**
+   * Announce the carriers a request delivered. Wrapped because a listener that
+   * throws must never turn a delivered batch into a failed export.
+   */
+  reportDelivered(spans) {
+    if (this.onDelivered === void 0) {
+      return;
+    }
+    const refs = spans.map((span) => span.ref).filter((ref) => ref !== void 0);
+    if (refs.length === 0) {
+      return;
+    }
+    try {
+      this.onDelivered(refs);
+    } catch (error) {
+      logError("a delivery listener threw", error);
+    }
+  }
   async shutdown() {
   }
   async forceFlush() {
@@ -1119,7 +1128,9 @@ var OtelBatchTransport = class {
         options.directSender,
         maxRequestBytes,
         maxRequestBatchSize,
-        options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY
+        options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY,
+        options.onDelivered,
+        options.exportTimeoutMillis ?? EXPORT_TIMEOUT_MILLIS
       )
     );
     this.processor = new BatchSpanProcessor(this.deliveryTracker, {
@@ -1143,8 +1154,7 @@ var OtelBatchTransport = class {
     this.tracer = this.provider.getTracer("bitfab", __version__);
     liveTransports.add(this);
   }
-  submit(operation, payload) {
-    recordTraceSubmission(operation, payload);
+  submit(operation, payload, meta = {}) {
     if (this.closed) {
       warnOnce(
         "otel-submit-after-shutdown",
@@ -1165,17 +1175,20 @@ var OtelBatchTransport = class {
           ].join(", ")}); they were stubbed so the span still ships, but the trace may be incomplete or not replayable.`
         );
       }
-      const span = this.tracer.startSpan(spanName(operation, payload), {
+      const span = this.tracer.startSpan(meta.name ?? `bitfab.${operation}`, {
         attributes: {
           [OPERATION_ATTRIBUTE]: operation,
           [PAYLOAD_ATTRIBUTE]: body
         },
-        startTime: payloadTimestamp(payload, "started_at")
+        startTime: meta.startTime
       });
-      if (hasError(payload)) {
+      if (meta.ref !== void 0) {
+        carrierRefs.set(span, meta.ref);
+      }
+      if (meta.errored === true) {
         span.setStatus({ code: SpanStatusCode.ERROR });
       }
-      endSpan(span, payloadTimestamp(payload, "ended_at"));
+      endSpan(span, meta.endTime);
     } catch (error) {
       logError("failed to queue an OpenTelemetry span", error);
     }
@@ -1215,36 +1228,6 @@ var OtelBatchTransport = class {
 function endSpan(span, endTime) {
   span.end(endTime);
 }
-function spanName(operation, payload) {
-  if (operation === "external_span") {
-    const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-    if (typeof spanData?.name === "string") {
-      return spanData.name;
-    }
-  }
-  if (typeof payload.traceFunctionKey === "string") {
-    return payload.traceFunctionKey;
-  }
-  return `bitfab.${operation}`;
-}
-function payloadTimestamp(payload, field) {
-  const rawSpan = asRecord2(payload.rawSpan);
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  const raw = rawSpan?.[field] ?? rawTrace?.[field];
-  if (typeof raw !== "string") {
-    return void 0;
-  }
-  const parsed = Date.parse(raw);
-  return Number.isNaN(parsed) ? void 0 : parsed;
-}
-function hasError(payload) {
-  const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-  if (spanData?.error != null) {
-    return true;
-  }
-  const errors = payload.errors;
-  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
-}
 function createOtelTransport(options) {
   return new OtelBatchTransport({
     ...options,
@@ -1293,12 +1276,11 @@ function flushTraceTransports(timeoutMs) {
 function shutdownTraceTransports(timeoutMs) {
   return shutdownOtelTransports(timeoutMs);
 }
-function takeReplaySpanCounts2(traceIds) {
-  return takeReplaySpanCounts(traceIds);
-}
 
 // src/http.ts
 var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS = 3e5;
+var OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
+var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
 var EXIT_FLUSH_TIMEOUT_MS = 5e3;
 var DEFAULT_LIFECYCLE_TIMEOUT_MS2 = 3e4;
 var pendingTracePromises = /* @__PURE__ */ new Set();
@@ -1358,8 +1340,101 @@ if (typeof process !== "undefined" && process.versions != null && process.versio
     });
   });
 }
+function readHeader(response, name) {
+  try {
+    return response.headers?.get(name) ?? null;
+  } catch {
+    return null;
+  }
+}
+function parseRetryAfterMs(header) {
+  const value = header?.trim();
+  if (!value) {
+    return void 0;
+  }
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) {
+    return seconds >= 0 ? seconds * 1e3 : void 0;
+  }
+  const at = Date.parse(value);
+  if (Number.isNaN(at)) {
+    return void 0;
+  }
+  return Math.max(0, at - Date.now());
+}
+function carrierMeta(operation, payload, ref) {
+  return {
+    ref,
+    name: carrierName(operation, payload),
+    startTime: payloadTimestamp(payload, "started_at"),
+    endTime: payloadTimestamp(payload, "ended_at"),
+    errored: payloadHasError(payload)
+  };
+}
+function carrierName(operation, payload) {
+  if (operation === "external_span") {
+    const spanData = asPayloadRecord(
+      asPayloadRecord(payload.rawSpan)?.span_data
+    );
+    if (typeof spanData?.name === "string") {
+      return spanData.name;
+    }
+  }
+  if (typeof payload.traceFunctionKey === "string") {
+    return payload.traceFunctionKey;
+  }
+  return `bitfab.${operation}`;
+}
+function payloadTimestamp(payload, field) {
+  const rawSpan = asPayloadRecord(payload.rawSpan);
+  const rawTrace = asPayloadRecord(payload.externalTrace) ?? asPayloadRecord(payload.rawTrace);
+  const raw = rawSpan?.[field] ?? rawTrace?.[field];
+  if (typeof raw !== "string") {
+    return void 0;
+  }
+  const parsed = Date.parse(raw);
+  return Number.isNaN(parsed) ? void 0 : parsed;
+}
+function payloadHasError(payload) {
+  const spanData = asPayloadRecord(asPayloadRecord(payload.rawSpan)?.span_data);
+  if (spanData?.error != null) {
+    return true;
+  }
+  const errors = payload.errors;
+  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
+}
+function asPayloadRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+function carrierRef(payload) {
+  const traceId = sourceTraceIdOf(payload);
+  if (traceId === void 0) {
+    return void 0;
+  }
+  const rawSpan = payload.rawSpan;
+  if (rawSpan === void 0) {
+    return { traceId };
+  }
+  const spanId = rawSpan?.id;
+  return {
+    traceId,
+    spanId: typeof spanId === "string" ? spanId : `submission-${++carrierSeq}`
+  };
+}
+function sourceTraceIdOf(payload) {
+  if (typeof payload.sourceTraceId === "string") {
+    return payload.sourceTraceId;
+  }
+  const rawTrace = payload.externalTrace ?? payload.rawTrace;
+  const id = rawTrace?.id;
+  return typeof id === "string" ? id : void 0;
+}
+var carrierSeq = 0;
 var HttpClient = class {
   constructor(config) {
+    // Only traces a caller asked about are tracked, so ordinary tracing stores
+    // nothing here.
+    this.traceDeliveries = /* @__PURE__ */ new Map();
     // Deferred span work owned by THIS client. The module-global set backs the
     // process-wide `flushTraces()` and the exit hook, but per-client lifecycle
     // must not wait on another client's slow finalize: a false `close()` failure
@@ -1395,13 +1470,131 @@ var HttpClient = class {
     }
     if (!this.traceTransport) {
       this.traceTransport = createTraceTransport({
-        directSender: (endpoint, request, timeoutMs) => this.sendPrepared(endpoint, request, {
-          timeout: timeoutMs
-        })
+        directSender: (request, timeoutMs) => this.deliverCarriers(request, timeoutMs),
+        onDelivered: (refs) => this.recordDeliveredCarriers(refs)
       });
     }
     return this.traceTransport;
   }
+  /**
+   * Post one encoded batch and decide what the server's answer means, so the
+   * transport never reads a response. Rejections and permanent statuses come
+   * back as a non-retryable {@link DeliveryError}; anything the server might
+   * still accept on a second try comes back retryable.
+   */
+  async deliverCarriers(request, timeoutMs) {
+    let response;
+    try {
+      response = await this.sendPrepared(
+        OTLP_TRACES_ENDPOINT,
+        request,
+        { timeout: timeoutMs }
+      );
+    } catch (error) {
+      const status = error instanceof BitfabError ? error.status : void 0;
+      if (status === void 0) {
+        throw new DeliveryError(`OTLP ingestion failed: ${String(error)}`, {
+          retryable: true
+        });
+      }
+      throw new DeliveryError(`OTLP ingestion failed with HTTP ${status}`, {
+        retryable: RETRYABLE_STATUSES.has(status),
+        oversized: status === 413,
+        ...error instanceof BitfabError && error.retryAfterMs !== void 0 ? { retryAfterMs: error.retryAfterMs } : {}
+      });
+    }
+    const partialSuccess = asPayloadRecord(response?.partialSuccess);
+    const rejected = partialSuccess?.rejectedSpans;
+    if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
+      throw new DeliveryError(
+        `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
+      );
+    }
+  }
+  /**
+   * Start tracking delivery for `traceIds`. Nothing is recorded for a trace
+   * that was never tracked, so ordinary tracing costs no bookkeeping at all.
+   */
+  trackTraceDeliveries(traceIds) {
+    for (const traceId of traceIds) {
+      if (!this.traceDeliveries.has(traceId)) {
+        this.traceDeliveries.set(traceId, {
+          submittedSpanIds: /* @__PURE__ */ new Set(),
+          ackedSpanIds: /* @__PURE__ */ new Set(),
+          closed: false,
+          closingAcked: false
+        });
+      }
+    }
+  }
+  /** Whether any tracked trace has had its closing carrier submitted. */
+  hasClosedDeliveries(traceIds) {
+    return traceIds.some((traceId) => this.traceDeliveries.get(traceId)?.closed);
+  }
+  /**
+   * Report what each tracked trace submitted and whether the server confirmed
+   * it, and stop tracking them. Every id passed is freed, so a caller cannot
+   * leak a record for a trace that never closed.
+   *
+   * `delivered` is only meaningful once a flush has settled: acks land before
+   * an export resolves, so a flush that reported success has already collected
+   * every ack it is going to collect.
+   */
+  takeTraceDeliveries(traceIds) {
+    const reports = {};
+    for (const traceId of traceIds) {
+      const delivery = this.traceDeliveries.get(traceId);
+      if (delivery === void 0) {
+        continue;
+      }
+      this.traceDeliveries.delete(traceId);
+      reports[traceId] = {
+        spanCount: delivery.submittedSpanIds.size,
+        closed: delivery.closed,
+        delivered: delivery.closingAcked && [...delivery.submittedSpanIds].every(
+          (spanId) => delivery.ackedSpanIds.has(spanId)
+        )
+      };
+    }
+    return reports;
+  }
+  /** Build a carrier's meta and record what it adds to its trace's expected set. */
+  recordedMeta(operation, payload, ref) {
+    this.recordSubmittedCarrier(ref);
+    return carrierMeta(operation, payload, ref);
+  }
+  recordSubmittedCarrier(ref) {
+    if (ref === void 0) {
+      return;
+    }
+    const delivery = this.traceDeliveries.get(ref.traceId);
+    if (delivery === void 0) {
+      return;
+    }
+    if (ref.spanId === void 0) {
+      delivery.closed = true;
+    } else {
+      delivery.submittedSpanIds.add(ref.spanId);
+    }
+  }
+  /**
+   * Ingestion commits every carrier in a request before it answers, so a
+   * delivered ref is proof its row exists: the same fact the replay status
+   * endpoint would report, already in hand.
+   */
+  recordDeliveredCarriers(refs) {
+    for (const ref of refs) {
+      const delivery = this.traceDeliveries.get(ref.traceId);
+      if (delivery === void 0) {
+        continue;
+      }
+      if (ref.spanId === void 0) {
+        delivery.closingAcked = true;
+      } else {
+        delivery.ackedSpanIds.add(ref.spanId);
+      }
+    }
+  }
   /**
    * Track deferred span work so this client's own lifecycle waits for it, and
    * so the process-wide flush and exit hook do too.
@@ -1511,7 +1704,8 @@ var HttpClient = class {
         throw new BitfabError(
           `HTTP ${response.status}: ${errorText.slice(0, 500)}`,
           void 0,
-          response.status
+          response.status,
+          parseRetryAfterMs(readHeader(response, "retry-after"))
         );
       }
       const result = await response.json();
@@ -1597,11 +1791,12 @@ var HttpClient = class {
    * the OTLP carrier has no path to carry it.
    */
   sendInternalTrace(functionId, payload) {
-    this.getTraceTransport()?.submit("internal_trace", {
-      ...payload,
-      functionId,
-      sdkVersion: __version__
-    });
+    const body = { ...payload, functionId, sdkVersion: __version__ };
+    this.getTraceTransport()?.submit(
+      "internal_trace",
+      body,
+      carrierMeta("internal_trace", body, void 0)
+    );
   }
   /**
    * Queue an external span (from withSpan wrapper or OpenAI tracing) onto this
@@ -1610,10 +1805,11 @@ var HttpClient = class {
    * promise.
    */
   sendExternalSpan(payload) {
-    this.getTraceTransport()?.submit("external_span", {
-      ...payload,
-      sdkVersion: __version__
-    });
+    this.getTraceTransport()?.submit(
+      "external_span",
+      { ...payload, sdkVersion: __version__ },
+      this.recordedMeta("external_span", payload, carrierRef(payload))
+    );
   }
   /**
    * Queue an external trace completion (from OpenAI tracing) onto this
@@ -1622,10 +1818,15 @@ var HttpClient = class {
    * server-authoritative barrier in `replay.ts`, not by awaiting this call.
    */
   sendExternalTrace(payload) {
-    this.getTraceTransport()?.submit("external_trace", {
-      ...payload,
-      sdkVersion: __version__
-    });
+    this.getTraceTransport()?.submit(
+      "external_trace",
+      { ...payload, sdkVersion: __version__ },
+      this.recordedMeta(
+        "external_trace",
+        payload,
+        payload.completed === true ? carrierRef(payload) : void 0
+      )
+    );
   }
   /**
    * Partial update of an existing trace identified by its Bitfab trace ID.
@@ -2303,15 +2504,29 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
     REPLAY_PERSISTENCE_TIMEOUT_MS
   );
   if (!deferredSettled) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     throw new BitfabError(
       `Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (testRunId ${testRunId}).`
     );
   }
-  const expectedSpanCounts = takeReplaySpanCounts2(replayedTraceIds);
-  if (Object.keys(expectedSpanCounts).length === 0) {
+  if (!httpClient.hasClosedDeliveries(replayedTraceIds)) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     return;
   }
   const flushed = await flushTraces(REPLAY_PERSISTENCE_TIMEOUT_MS);
+  const deliveries = httpClient.takeTraceDeliveries(replayedTraceIds);
+  const expectedSpanCounts = {};
+  let allDelivered = true;
+  for (const [traceId, delivery] of Object.entries(deliveries)) {
+    if (!delivery.closed) {
+      continue;
+    }
+    expectedSpanCounts[traceId] = delivery.spanCount;
+    allDelivered = allDelivered && delivery.delivered;
+  }
+  if (allDelivered) {
+    return;
+  }
   const deadline = Date.now() + REPLAY_PERSISTENCE_TIMEOUT_MS;
   let missing = Object.keys(expectedSpanCounts).length;
   while (true) {
@@ -2329,17 +2544,18 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
     if (Date.now() >= deadline) {
       break;
     }
-    await sleep(Math.min(100, Math.max(0, deadline - Date.now())));
+    await sleepForReplayPersistence(
+      Math.min(100, Math.max(0, deadline - Date.now()))
+    );
   }
   const cause = flushed ? "" : " Delivery was also not confirmed before the flush deadline, so the spans likely never reached the server.";
   throw new BitfabError(
     `Replay traces were not fully persisted before the delivery deadline (testRunId ${testRunId}, missing ${missing} of ${Object.keys(expectedSpanCounts).length} trace(s)).${cause}`
   );
 }
-function sleep(ms) {
+function sleepForReplayPersistence(ms) {
   return new Promise((resolve) => {
-    const timer = setTimeout(resolve, ms);
-    unrefTimer(timer);
+    setTimeout(resolve, ms);
   });
 }
 async function mapWithConcurrency2(tasks, maxConcurrency, onSettled, onStarted) {
@@ -2420,6 +2636,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     ...registeredOverrides
   ];
   const replayedTraceIds = serverItems.map(() => randomUuid());
+  httpClient.trackTraceDeliveries(replayedTraceIds);
   const tasks = serverItems.map(
     (serverItem, index) => () => processItem(
       httpClient,
@@ -2634,6 +2851,8 @@ export {
   ReplayError,
   DbBranchReplayError,
   serializeReplayResult,
+  waitForReplayPersistence,
+  sleepForReplayPersistence,
   replay
 };
-//# sourceMappingURL=chunk-CFECTUDU.js.map
+//# sourceMappingURL=chunk-MBKY42QC.js.map