bitfab 0.36.6 → 0.36.8

package/dist/node.cjs

@@ -97,7 +97,7 @@ var __version__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.36.6";
+    __version__ = "0.36.8";
   }
 });
 
@@ -131,33 +131,58 @@ function toArrayBuffer(view) {
     view.byteOffset + view.byteLength
   );
 }
+function compressedRequest(body, rawBytes, compressed) {
+  if (compressed.byteLength >= rawBytes) {
+    return { body, rawBytes, wireBytes: rawBytes };
+  }
+  return {
+    body: compressed instanceof Uint8Array ? toArrayBuffer(compressed) : compressed,
+    contentEncoding: "gzip",
+    rawBytes,
+    wireBytes: compressed.byteLength
+  };
+}
 async function gzipViaStream(bytes) {
   const stream = new Blob([bytes]).stream().pipeThrough(new CompressionStream("gzip"));
   return await new Response(stream).arrayBuffer();
 }
 function encodeRequestBody(body) {
   if (readEnv(DISABLE_COMPRESSION_ENV)) {
-    return { body };
+    const rawBytes = new TextEncoder().encode(body).byteLength;
+    return { body, rawBytes, wireBytes: rawBytes };
   }
   const bytes = new TextEncoder().encode(body);
   if (bytes.byteLength < MIN_COMPRESSED_BYTES) {
-    return { body };
+    return {
+      body,
+      rawBytes: bytes.byteLength,
+      wireBytes: bytes.byteLength
+    };
   }
   if (gzipNode) {
     return gzipNode(bytes).then(
-      (compressed) => ({
-        body: toArrayBuffer(compressed),
-        contentEncoding: "gzip"
-      }),
-      () => ({ body })
+      (compressed) => compressedRequest(body, bytes.byteLength, compressed),
+      () => ({
+        body,
+        rawBytes: bytes.byteLength,
+        wireBytes: bytes.byteLength
+      })
     );
   }
   if (typeof CompressionStream === "undefined") {
-    return { body };
+    return {
+      body,
+      rawBytes: bytes.byteLength,
+      wireBytes: bytes.byteLength
+    };
   }
   return gzipViaStream(bytes).then(
-    (compressed) => ({ body: compressed, contentEncoding: "gzip" }),
-    () => ({ body })
+    (compressed) => compressedRequest(body, bytes.byteLength, compressed),
+    () => ({
+      body,
+      rawBytes: bytes.byteLength,
+      wireBytes: bytes.byteLength
+    })
   );
 }
 var DISABLE_COMPRESSION_ENV, MIN_COMPRESSED_BYTES, gzipNode, _nodeGzipReady;
@@ -198,10 +223,11 @@ var init_errors = __esm({
   "src/errors.ts"() {
     "use strict";
     BitfabError = class extends Error {
-      constructor(message, url, status) {
+      constructor(message, url, status, retryAfterMs) {
         super(message);
         this.url = url;
         this.status = status;
+        this.retryAfterMs = retryAfterMs;
         this.name = "BitfabError";
       }
     };
@@ -263,15 +289,15 @@ function carrierBytesOf(encoded, body) {
   }
   return encoded.length + extra;
 }
-function fitsCarrierBudget(body) {
+function fitsCarrierBudget(body, maxBytes = MAX_SPAN_CARRIER_BYTES) {
   const units = body.length;
-  if (units * MAX_BYTES_PER_UNIT + 2 <= MAX_SPAN_CARRIER_BYTES) {
+  if (units * MAX_BYTES_PER_UNIT + 2 <= maxBytes) {
     return true;
   }
-  if (units + 2 > MAX_SPAN_CARRIER_BYTES) {
+  if (units + 2 > maxBytes) {
     return false;
   }
-  return carrierByteLength(body) <= MAX_SPAN_CARRIER_BYTES;
+  return carrierByteLength(body) <= maxBytes;
 }
 function asRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
@@ -315,7 +341,7 @@ function collectCandidates(containers) {
   }
   return candidates.sort((a, b) => b.size - a.size);
 }
-function trimPayloadToBudget(payload, encode) {
+function trimPayloadToBudget(payload, encode, maxBytes = MAX_SPAN_CARRIER_BYTES) {
   const { copy, containers } = cloneTrimmable(payload);
   const candidates = collectCandidates(containers);
   if (candidates.length === 0) {
@@ -331,30 +357,31 @@ function trimPayloadToBudget(payload, encode) {
     } catch {
       return void 0;
     }
-    if (fitsCarrierBudget(body)) {
+    if (fitsCarrierBudget(body, maxBytes)) {
       return { value: copy, trimmed };
     }
   }
   return void 0;
 }
-function markPayloadTrimmed(value, trimmed) {
+function markPayloadTrimmed(value, trimmed, maxBytes = MAX_SPAN_CARRIER_BYTES) {
   const existing = Array.isArray(value.errors) ? value.errors : [];
   value.errors = [
     ...existing,
     {
       source: "sdk",
       step: "payload_budget",
-      error: `trimmed oversized field(s) to fit the ${MAX_SPAN_CARRIER_BYTES}-byte span carrier budget: ${[
+      error: `trimmed oversized field(s) to fit the ${maxBytes}-byte span carrier budget: ${[
         ...new Set(trimmed)
       ].join(", ")}`
     }
   ];
 }
-var MAX_SPAN_CARRIER_BYTES, textEncoder, MAX_BYTES_PER_UNIT, STRUCTURAL_SPAN_KEYS;
+var MAX_SPAN_CARRIER_BYTES, MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES, textEncoder, MAX_BYTES_PER_UNIT, STRUCTURAL_SPAN_KEYS;
 var init_payloadBudget = __esm({
   "src/payloadBudget.ts"() {
     "use strict";
     MAX_SPAN_CARRIER_BYTES = 28e5;
+    MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES = 78e5;
     textEncoder = typeof TextEncoder !== "undefined" ? new TextEncoder() : null;
     MAX_BYTES_PER_UNIT = 3;
     STRUCTURAL_SPAN_KEYS = /* @__PURE__ */ new Set([
@@ -386,30 +413,31 @@ var init_warnOnce = __esm({
 });
 
 // src/serializePayload.ts
-function serializePayloadBody(payload) {
+function serializePayloadBody(payload, maxCarrierBytes = MAX_SPAN_CARRIER_BYTES) {
   const encoded = encodePayloadBody(payload);
-  if (fitsCarrierBudget(encoded.body)) {
+  if (fitsCarrierBudget(encoded.body, maxCarrierBytes)) {
     return { body: encoded.body, dropped: encoded.dropped };
   }
-  return applyPayloadBudget(encoded);
+  return applyPayloadBudget(encoded, maxCarrierBytes);
 }
-function applyPayloadBudget(encoded) {
+function applyPayloadBudget(encoded, maxCarrierBytes) {
   const result = encoded.value ? trimPayloadToBudget(
     encoded.value,
-    (value) => encodePayloadBody(value).body
+    (value) => encodePayloadBody(value).body,
+    maxCarrierBytes
   ) : void 0;
   if (!result) {
     return { body: encoded.body, dropped: encoded.dropped };
   }
   warnOnce(
     "payload:over-budget",
-    `a span payload exceeded the ${MAX_SPAN_CARRIER_BYTES}-byte carrier budget; its largest field(s) (${[
+    `a span payload exceeded the ${maxCarrierBytes}-byte carrier budget; its largest field(s) (${[
       ...new Set(result.trimmed)
     ].join(
       ", "
     )}) were replaced with placeholders so the span still ships. The span is incomplete and may not be replayable.`
   );
-  markPayloadTrimmed(result.value, result.trimmed);
+  markPayloadTrimmed(result.value, result.trimmed, maxCarrierBytes);
   return {
     body: encodePayloadBody(result.value).body,
     dropped: encoded.dropped
@@ -515,6 +543,23 @@ var init_serializePayload = __esm({
   }
 });
 
+// src/transportTypes.ts
+var DeliveryError;
+var init_transportTypes = __esm({
+  "src/transportTypes.ts"() {
+    "use strict";
+    DeliveryError = class extends Error {
+      constructor(message, options = {}) {
+        super(message);
+        this.name = "DeliveryError";
+        this.retryable = options.retryable ?? false;
+        this.oversized = options.oversized ?? false;
+        this.retryAfterMs = options.retryAfterMs;
+      }
+    };
+  }
+});
+
 // src/unrefTimer.ts
 function unrefTimer(timer) {
   const handle = timer;
@@ -554,59 +599,6 @@ function logError(message, error) {
   } catch {
   }
 }
-function recordTraceSubmission(operation, payload) {
-  const sourceTraceId = resolveSourceTraceId(payload);
-  if (sourceTraceId === void 0) {
-    return;
-  }
-  if (operation === "external_span") {
-    const rawSpan = asRecord2(payload.rawSpan);
-    if (typeof rawSpan?.id !== "string") {
-      submissionCounter += 1;
-    }
-    const sourceSpanId = typeof rawSpan?.id === "string" ? rawSpan.id : `submission-${submissionCounter}`;
-    const existing = traceSubmissionSpanIds.get(sourceTraceId);
-    if (existing) {
-      existing.add(sourceSpanId);
-    } else {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set([sourceSpanId]));
-    }
-    return;
-  }
-  if (payload.completed !== true) {
-    return;
-  }
-  if (typeof payload.testRunId === "string") {
-    replayTraceSubmissions.add(sourceTraceId);
-    if (!traceSubmissionSpanIds.has(sourceTraceId)) {
-      traceSubmissionSpanIds.set(sourceTraceId, /* @__PURE__ */ new Set());
-    }
-  } else {
-    traceSubmissionSpanIds.delete(sourceTraceId);
-  }
-}
-function takeReplaySpanCounts(traceIds) {
-  const counts = {};
-  for (const traceId of traceIds) {
-    if (!replayTraceSubmissions.has(traceId)) {
-      continue;
-    }
-    counts[traceId] = traceSubmissionSpanIds.get(traceId)?.size ?? 0;
-    traceSubmissionSpanIds.delete(traceId);
-    replayTraceSubmissions.delete(traceId);
-  }
-  return counts;
-}
-function asRecord2(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
-}
-function resolveSourceTraceId(payload) {
-  if (typeof payload.sourceTraceId === "string") {
-    return payload.sourceTraceId;
-  }
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  return typeof rawTrace?.id === "string" ? rawTrace.id : void 0;
-}
 function otlpValue(value) {
   if (typeof value === "boolean") {
     return { boolValue: value };
@@ -664,7 +656,36 @@ function spanToOtlp(span) {
 }
 function encodeSpan(span) {
   const json = JSON.stringify(spanToOtlp(span));
-  return { json, size: byteLength(json) };
+  return {
+    json,
+    size: byteLength(json),
+    ref: carrierRefs.get(span)
+  };
+}
+function trimEncodedSpan(span) {
+  try {
+    const carrier = JSON.parse(span.json);
+    const attribute = carrier.attributes?.find(
+      (entry) => entry.key === PAYLOAD_ATTRIBUTE
+    );
+    const payloadBody = attribute?.value?.stringValue;
+    if (!attribute?.value || payloadBody === void 0) {
+      return void 0;
+    }
+    const payload = JSON.parse(payloadBody);
+    attribute.value.stringValue = serializePayloadBody(
+      payload,
+      MAX_SPAN_CARRIER_BYTES
+    ).body;
+    const json = JSON.stringify(carrier);
+    return { json, size: byteLength(json) };
+  } catch {
+    return void 0;
+  }
+}
+async function prepareRequest(body) {
+  const prepared = encodeRequestBody(body);
+  return prepared instanceof Promise ? await prepared : prepared;
 }
 function requestEnvelope(first) {
   const scope = first.instrumentationScope;
@@ -722,48 +743,27 @@ async function mapWithConcurrency(items, limit, task) {
   await Promise.all(workers);
   return results;
 }
-function responseStatus(error) {
-  return error instanceof BitfabError ? error.status : void 0;
-}
 function isRetryable(error) {
-  const status = responseStatus(error);
-  if (status === void 0) {
-    return true;
-  }
-  return RETRYABLE_STATUSES.has(status) || status >= 500;
+  return error instanceof DeliveryError && error.retryable;
 }
-function endSpan(span, endTime) {
-  span.end(endTime);
+function isOversized(error) {
+  return error instanceof DeliveryError && error.oversized;
 }
-function spanName(operation, payload) {
-  if (operation === "external_span") {
-    const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-    if (typeof spanData?.name === "string") {
-      return spanData.name;
-    }
+function retryWaitMillis(error, attempt, remainingMillis) {
+  const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+  const affordable = remainingMillis / 2;
+  if (requested !== void 0) {
+    return requested < affordable ? requested : null;
   }
-  if (typeof payload.traceFunctionKey === "string") {
-    return payload.traceFunctionKey;
-  }
-  return `bitfab.${operation}`;
-}
-function payloadTimestamp(payload, field) {
-  const rawSpan = asRecord2(payload.rawSpan);
-  const rawTrace = asRecord2(payload.externalTrace) ?? asRecord2(payload.rawTrace);
-  const raw = rawSpan?.[field] ?? rawTrace?.[field];
-  if (typeof raw !== "string") {
-    return void 0;
-  }
-  const parsed = Date.parse(raw);
-  return Number.isNaN(parsed) ? void 0 : parsed;
+  const backoff = Math.min(
+    RETRY_BASE_DELAY_MILLIS * 2 ** attempt,
+    RETRY_BACKOFF_CEILING_MILLIS
+  );
+  const jittered = backoff / 2 + Math.random() * (backoff / 2);
+  return jittered < affordable ? jittered : null;
 }
-function hasError(payload) {
-  const spanData = asRecord2(asRecord2(payload.rawSpan)?.span_data);
-  if (spanData?.error != null) {
-    return true;
-  }
-  const errors = payload.errors;
-  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
+function endSpan(span, endTime) {
+  span.end(endTime);
 }
 function createOtelTransport(options) {
   return new OtelBatchTransport({
@@ -802,7 +802,7 @@ function shutdownOtelTransports(timeoutMs = DEFAULT_LIFECYCLE_TIMEOUT_MS) {
     (transport, remaining) => transport.shutdown(remaining)
   );
 }
-var import_api, import_core, import_resources, import_sdk_trace_base, OPERATION_ATTRIBUTE, PAYLOAD_ATTRIBUTE, OTLP_TRACES_ENDPOINT, MAX_EXPORT_REQUEST_BYTES, MAX_REQUEST_BYTES_ENV, EXPORT_CONCURRENCY_ENV, MAX_QUEUE_SIZE, DIRECT_MAX_EXPORT_BATCH_SIZE, DIRECT_MAX_REQUEST_BATCH_SIZE, DEFAULT_EXPORT_CONCURRENCY, MAX_EXPORT_CONCURRENCY, SCHEDULE_DELAY_MILLIS, EXPORT_TIMEOUT_MILLIS, RETRY_DELAY_MILLIS, MAX_SEND_ATTEMPTS, DEFAULT_LIFECYCLE_TIMEOUT_MS, RETRYABLE_STATUSES, liveTransports, traceSubmissionSpanIds, replayTraceSubmissions, submissionCounter, SPAN_SEPARATOR_BYTES, OtlpPayloadTooLargeError, OtlpPartialSuccessError, BitfabSpanExporter, DeliveryTrackingExporter, OtelBatchTransport;
+var import_api, import_core, import_resources, import_sdk_trace_base, OPERATION_ATTRIBUTE, PAYLOAD_ATTRIBUTE, MAX_EXPORT_REQUEST_BYTES, MAX_DECOMPRESSED_REQUEST_BYTES, MAX_REQUEST_BYTES_ENV, EXPORT_CONCURRENCY_ENV, MAX_QUEUE_SIZE, DIRECT_MAX_EXPORT_BATCH_SIZE, DIRECT_MAX_REQUEST_BATCH_SIZE, DEFAULT_EXPORT_CONCURRENCY, MAX_EXPORT_CONCURRENCY, SCHEDULE_DELAY_MILLIS, EXPORT_TIMEOUT_MILLIS, RETRY_BASE_DELAY_MILLIS, RETRY_BACKOFF_CEILING_MILLIS, MAX_SEND_ATTEMPTS, DEFAULT_LIFECYCLE_TIMEOUT_MS, liveTransports, carrierRefs, SPAN_SEPARATOR_BYTES, BitfabSpanExporter, DeliveryTrackingExporter, OtelBatchTransport;
 var init_otel = __esm({
   "src/otel.ts"() {
     "use strict";
@@ -810,17 +810,19 @@ var init_otel = __esm({
     import_core = require("@opentelemetry/core");
     import_resources = require("@opentelemetry/resources");
     import_sdk_trace_base = require("@opentelemetry/sdk-trace-base");
+    init_compress();
     init_constants();
     init_errors();
     init_payloadBudget();
     init_readEnv();
     init_serializePayload();
+    init_transportTypes();
     init_unrefTimer();
     init_warnOnce();
     OPERATION_ATTRIBUTE = "bitfab.operation";
     PAYLOAD_ATTRIBUTE = "bitfab.payload";
-    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
     MAX_EXPORT_REQUEST_BYTES = 3e6;
+    MAX_DECOMPRESSED_REQUEST_BYTES = 8e6;
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES";
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY";
     MAX_QUEUE_SIZE = 8192;
@@ -830,25 +832,23 @@ var init_otel = __esm({
     MAX_EXPORT_CONCURRENCY = 64;
     SCHEDULE_DELAY_MILLIS = 5e3;
     EXPORT_TIMEOUT_MILLIS = 3e4;
-    RETRY_DELAY_MILLIS = 100;
+    RETRY_BASE_DELAY_MILLIS = 100;
+    RETRY_BACKOFF_CEILING_MILLIS = 5e3;
     MAX_SEND_ATTEMPTS = 3;
     DEFAULT_LIFECYCLE_TIMEOUT_MS = 3e4;
-    RETRYABLE_STATUSES = /* @__PURE__ */ new Set([408, 425, 429]);
     liveTransports = /* @__PURE__ */ new Set();
-    traceSubmissionSpanIds = /* @__PURE__ */ new Map();
-    replayTraceSubmissions = /* @__PURE__ */ new Set();
-    submissionCounter = 0;
+    carrierRefs = /* @__PURE__ */ new WeakMap();
     SPAN_SEPARATOR_BYTES = 1;
-    OtlpPayloadTooLargeError = class extends Error {
-    };
-    OtlpPartialSuccessError = class extends Error {
-    };
     BitfabSpanExporter = class {
-      constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency) {
+      constructor(directSender, maxRequestBytes, maxRequestBatchSize, exportConcurrency, onDelivered, exportTimeoutMillis = EXPORT_TIMEOUT_MILLIS) {
         this.directSender = directSender;
         this.maxRequestBytes = maxRequestBytes;
         this.maxRequestBatchSize = maxRequestBatchSize;
         this.exportConcurrency = exportConcurrency;
+        this.onDelivered = onDelivered;
+        this.exportTimeoutMillis = exportTimeoutMillis;
+        /** Epoch ms until which the server has asked this exporter to stay away. */
+        this.throttledUntil = 0;
       }
       export(spans, resultCallback) {
         void this.exportAsync(spans).then(
@@ -903,25 +903,51 @@ var init_otel = __esm({
         return batches;
       }
       async send(envelope, batch) {
-        if (batch.size > this.maxRequestBytes) {
-          logError(
-            "a single OpenTelemetry span exceeded the configured request-size target and could not be exported"
-          );
-          return false;
-        }
         try {
-          await this.sendWithRetries(encodeRequest(envelope, batch.spans));
-          return true;
+          let requestSpans = batch.spans;
+          let requestRawBytes = batch.size;
+          let alreadyTrimmed = false;
+          while (true) {
+            if (requestRawBytes <= MAX_DECOMPRESSED_REQUEST_BYTES) {
+              const prepared = await prepareRequest(
+                encodeRequest(envelope, requestSpans)
+              );
+              if (prepared.wireBytes <= this.maxRequestBytes) {
+                await this.sendWithRetries(prepared);
+                this.reportDelivered(batch.spans);
+                return true;
+              }
+            }
+            if (batch.spans.length !== 1) {
+              logError(
+                "an OpenTelemetry span batch exceeded the configured request-size target and could not be exported"
+              );
+              return false;
+            }
+            if (alreadyTrimmed) {
+              logError(
+                "a single OpenTelemetry span exceeded the configured request-size target after trimming"
+              );
+              return false;
+            }
+            const trimmed = trimEncodedSpan(batch.spans[0]);
+            if (!trimmed) {
+              logError(
+                "a single OpenTelemetry span exceeded the configured request-size target and could not be trimmed"
+              );
+              return false;
+            }
+            requestSpans = [trimmed];
+            requestRawBytes = envelope.size + trimmed.size;
+            alreadyTrimmed = true;
+          }
         } catch (error) {
-          if (error instanceof OtlpPayloadTooLargeError) {
+          if (isOversized(error)) {
             logError(
               batch.spans.length === 1 ? "a single OpenTelemetry span exceeded the ingestion request limit and could not be exported" : "an OpenTelemetry span batch exceeded the ingestion request limit and could not be exported"
             );
             return false;
           }
-          if (error instanceof OtlpPartialSuccessError) {
-            return false;
-          }
           logError("failed to export an OpenTelemetry span batch", error);
           return false;
         }
@@ -939,37 +965,79 @@ var init_otel = __esm({
        * the server does not yet understand. The fix is a client-supplied
        * idempotency key that ingestion dedupes on.
        */
-      async sendWithRetries(body) {
+      /**
+       * Remember a throttle the server asked for, so the requests fanned out
+       * alongside this one respect it too. Delaying only the request that was
+       * refused leaves the other seven in the window hitting a server that just
+       * asked for room.
+       */
+      recordThrottle(error) {
+        const requested = error instanceof DeliveryError ? error.retryAfterMs : void 0;
+        if (requested !== void 0) {
+          this.throttledUntil = Math.max(
+            this.throttledUntil,
+            Date.now() + requested
+          );
+        }
+      }
+      /**
+       * Waits out an active throttle, or reports the batch undeliverable when the
+       * throttle outlasts what we are willing to hold it for. Either way nothing is
+       * sent while the server has asked us to stay away.
+       */
+      async awaitThrottle(deadline) {
+        const remaining = this.throttledUntil - Date.now();
+        if (remaining <= 0) {
+          return;
+        }
+        if (remaining >= (deadline - Date.now()) / 2) {
+          throw new DeliveryError(
+            `OTLP ingestion is throttled for another ${remaining}ms, longer than the export budget`
+          );
+        }
+        await delay(remaining);
+      }
+      async sendWithRetries(request) {
+        const deadline = Date.now() + this.exportTimeoutMillis;
         for (let attempt = 0; attempt < MAX_SEND_ATTEMPTS; attempt += 1) {
           try {
-            const response = await this.directSender(
-              OTLP_TRACES_ENDPOINT,
-              body,
-              EXPORT_TIMEOUT_MILLIS
-            );
-            const partialSuccess = asRecord2(response?.partialSuccess);
-            const rejected = partialSuccess?.rejectedSpans;
-            if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
-              logError(
-                `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
-              );
-              throw new OtlpPartialSuccessError();
-            }
+            await this.awaitThrottle(deadline);
+            await this.directSender(request, Math.max(0, deadline - Date.now()));
             return;
           } catch (error) {
-            if (error instanceof OtlpPartialSuccessError) {
+            if (isOversized(error)) {
               throw error;
             }
-            if (responseStatus(error) === 413) {
-              throw new OtlpPayloadTooLargeError();
-            }
+            this.recordThrottle(error);
             if (attempt === MAX_SEND_ATTEMPTS - 1 || !isRetryable(error)) {
               throw error;
             }
-            await delay(RETRY_DELAY_MILLIS);
+            const wait = retryWaitMillis(error, attempt, deadline - Date.now());
+            if (wait === null) {
+              throw error;
+            }
+            await delay(wait);
           }
         }
       }
+      /**
+       * Announce the carriers a request delivered. Wrapped because a listener that
+       * throws must never turn a delivered batch into a failed export.
+       */
+      reportDelivered(spans) {
+        if (this.onDelivered === void 0) {
+          return;
+        }
+        const refs = spans.map((span) => span.ref).filter((ref) => ref !== void 0);
+        if (refs.length === 0) {
+          return;
+        }
+        try {
+          this.onDelivered(refs);
+        } catch (error) {
+          logError("a delivery listener threw", error);
+        }
+      }
       async shutdown() {
       }
       async forceFlush() {
@@ -1027,7 +1095,9 @@ var init_otel = __esm({
             options.directSender,
             maxRequestBytes,
             maxRequestBatchSize,
-            options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY
+            options.exportConcurrency ?? DEFAULT_EXPORT_CONCURRENCY,
+            options.onDelivered,
+            options.exportTimeoutMillis ?? EXPORT_TIMEOUT_MILLIS
           )
         );
         this.processor = new import_sdk_trace_base.BatchSpanProcessor(this.deliveryTracker, {
@@ -1051,8 +1121,7 @@ var init_otel = __esm({
         this.tracer = this.provider.getTracer("bitfab", __version__);
         liveTransports.add(this);
       }
-      submit(operation, payload) {
-        recordTraceSubmission(operation, payload);
+      submit(operation, payload, meta = {}) {
         if (this.closed) {
           warnOnce(
             "otel-submit-after-shutdown",
@@ -1061,7 +1130,10 @@ var init_otel = __esm({
           return;
         }
         try {
-          const { body, dropped } = serializePayloadBody(payload);
+          const { body, dropped } = serializePayloadBody(
+            payload,
+            MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
+          );
           if (dropped.length > 0) {
             warnOnce(
               "otel-carrier-payload-stubbed",
@@ -1070,17 +1142,20 @@ var init_otel = __esm({
               ].join(", ")}); they were stubbed so the span still ships, but the trace may be incomplete or not replayable.`
             );
           }
-          const span = this.tracer.startSpan(spanName(operation, payload), {
+          const span = this.tracer.startSpan(meta.name ?? `bitfab.${operation}`, {
             attributes: {
               [OPERATION_ATTRIBUTE]: operation,
               [PAYLOAD_ATTRIBUTE]: body
             },
-            startTime: payloadTimestamp(payload, "started_at")
+            startTime: meta.startTime
           });
-          if (hasError(payload)) {
+          if (meta.ref !== void 0) {
+            carrierRefs.set(span, meta.ref);
+          }
+          if (meta.errored === true) {
             span.setStatus({ code: import_api.SpanStatusCode.ERROR });
           }
-          endSpan(span, payloadTimestamp(payload, "ended_at"));
+          endSpan(span, meta.endTime);
         } catch (error) {
           logError("failed to queue an OpenTelemetry span", error);
         }
@@ -1130,9 +1205,6 @@ function flushTraceTransports(timeoutMs) {
 function shutdownTraceTransports(timeoutMs) {
   return shutdownOtelTransports(timeoutMs);
 }
-function takeReplaySpanCounts2(traceIds) {
-  return takeReplaySpanCounts(traceIds);
-}
 var init_transport = __esm({
   "src/transport.ts"() {
     "use strict";
@@ -1181,7 +1253,96 @@ async function waitForPromises(promises, timeoutMs) {
     }
   }
 }
-var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, HttpClient;
+function readHeader(response, name) {
+  try {
+    return response.headers?.get(name) ?? null;
+  } catch {
+    return null;
+  }
+}
+function parseRetryAfterMs(header) {
+  const value = header?.trim();
+  if (!value) {
+    return void 0;
+  }
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) {
+    return seconds >= 0 ? seconds * 1e3 : void 0;
+  }
+  const at = Date.parse(value);
+  if (Number.isNaN(at)) {
+    return void 0;
+  }
+  return Math.max(0, at - Date.now());
+}
+function carrierMeta(operation, payload, ref) {
+  return {
+    ref,
+    name: carrierName(operation, payload),
+    startTime: payloadTimestamp(payload, "started_at"),
+    endTime: payloadTimestamp(payload, "ended_at"),
+    errored: payloadHasError(payload)
+  };
+}
+function carrierName(operation, payload) {
+  if (operation === "external_span") {
+    const spanData = asPayloadRecord(
+      asPayloadRecord(payload.rawSpan)?.span_data
+    );
+    if (typeof spanData?.name === "string") {
+      return spanData.name;
+    }
+  }
+  if (typeof payload.traceFunctionKey === "string") {
+    return payload.traceFunctionKey;
+  }
+  return `bitfab.${operation}`;
+}
+function payloadTimestamp(payload, field) {
+  const rawSpan = asPayloadRecord(payload.rawSpan);
+  const rawTrace = asPayloadRecord(payload.externalTrace) ?? asPayloadRecord(payload.rawTrace);
+  const raw = rawSpan?.[field] ?? rawTrace?.[field];
+  if (typeof raw !== "string") {
+    return void 0;
+  }
+  const parsed = Date.parse(raw);
+  return Number.isNaN(parsed) ? void 0 : parsed;
+}
+function payloadHasError(payload) {
+  const spanData = asPayloadRecord(asPayloadRecord(payload.rawSpan)?.span_data);
+  if (spanData?.error != null) {
+    return true;
+  }
+  const errors = payload.errors;
+  return Array.isArray(errors) ? errors.length > 0 : Boolean(errors);
+}
+function asPayloadRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+function carrierRef(payload) {
+  const traceId = sourceTraceIdOf(payload);
+  if (traceId === void 0) {
+    return void 0;
+  }
+  const rawSpan = payload.rawSpan;
+  if (rawSpan === void 0) {
+    return { traceId };
+  }
+  const spanId = rawSpan?.id;
+  return {
+    traceId,
+    spanId: typeof spanId === "string" ? spanId : `submission-${++carrierSeq}`
+  };
+}
+function sourceTraceIdOf(payload) {
+  if (typeof payload.sourceTraceId === "string") {
+    return payload.sourceTraceId;
+  }
+  const rawTrace = payload.externalTrace ?? payload.rawTrace;
+  const id = rawTrace?.id;
+  return typeof id === "string" ? id : void 0;
+}
+var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, OTLP_TRACES_ENDPOINT, RETRYABLE_STATUSES, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, carrierSeq, HttpClient;
 var init_http = __esm({
   "src/http.ts"() {
     "use strict";
@@ -1191,9 +1352,12 @@ var init_http = __esm({
     init_replayContext();
     init_serializePayload();
     init_transport();
+    init_transportTypes();
     init_unrefTimer();
     init_warnOnce();
     REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS = 3e5;
+    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
+    RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
     EXIT_FLUSH_TIMEOUT_MS = 5e3;
     DEFAULT_LIFECYCLE_TIMEOUT_MS2 = 3e4;
     pendingTracePromises = /* @__PURE__ */ new Set();
@@ -1213,8 +1377,12 @@ var init_http = __esm({
         });
       });
     }
+    carrierSeq = 0;
     HttpClient = class {
       constructor(config) {
+        // Only traces a caller asked about are tracked, so ordinary tracing stores
+        // nothing here.
+        this.traceDeliveries = /* @__PURE__ */ new Map();
         // Deferred span work owned by THIS client. The module-global set backs the
         // process-wide `flushTraces()` and the exit hook, but per-client lifecycle
         // must not wait on another client's slow finalize: a false `close()` failure
@@ -1250,13 +1418,131 @@ var init_http = __esm({
         }
         if (!this.traceTransport) {
           this.traceTransport = createTraceTransport({
-            directSender: (endpoint, body, timeoutMs) => this.sendEncoded(endpoint, body, {
-              timeout: timeoutMs
-            })
+            directSender: (request, timeoutMs) => this.deliverCarriers(request, timeoutMs),
+            onDelivered: (refs) => this.recordDeliveredCarriers(refs)
           });
         }
         return this.traceTransport;
       }
+      /**
+       * Post one encoded batch and decide what the server's answer means, so the
+       * transport never reads a response. Rejections and permanent statuses come
+       * back as a non-retryable {@link DeliveryError}; anything the server might
+       * still accept on a second try comes back retryable.
+       */
+      async deliverCarriers(request, timeoutMs) {
+        let response;
+        try {
+          response = await this.sendPrepared(
+            OTLP_TRACES_ENDPOINT,
+            request,
+            { timeout: timeoutMs }
+          );
+        } catch (error) {
+          const status = error instanceof BitfabError ? error.status : void 0;
+          if (status === void 0) {
+            throw new DeliveryError(`OTLP ingestion failed: ${String(error)}`, {
+              retryable: true
+            });
+          }
+          throw new DeliveryError(`OTLP ingestion failed with HTTP ${status}`, {
+            retryable: RETRYABLE_STATUSES.has(status),
+            oversized: status === 413,
+            ...error instanceof BitfabError && error.retryAfterMs !== void 0 ? { retryAfterMs: error.retryAfterMs } : {}
+          });
+        }
+        const partialSuccess = asPayloadRecord(response?.partialSuccess);
+        const rejected = partialSuccess?.rejectedSpans;
+        if (rejected !== void 0 && rejected !== "0" && rejected !== 0) {
+          throw new DeliveryError(
+            `OTLP ingestion rejected ${rejected} span(s): ${partialSuccess?.errorMessage ?? "no reason provided"}`
+          );
+        }
+      }
+      /**
+       * Start tracking delivery for `traceIds`. Nothing is recorded for a trace
+       * that was never tracked, so ordinary tracing costs no bookkeeping at all.
+       */
+      trackTraceDeliveries(traceIds) {
+        for (const traceId of traceIds) {
+          if (!this.traceDeliveries.has(traceId)) {
+            this.traceDeliveries.set(traceId, {
+              submittedSpanIds: /* @__PURE__ */ new Set(),
+              ackedSpanIds: /* @__PURE__ */ new Set(),
+              closed: false,
+              closingAcked: false
+            });
+          }
+        }
+      }
+      /** Whether any tracked trace has had its closing carrier submitted. */
+      hasClosedDeliveries(traceIds) {
+        return traceIds.some((traceId) => this.traceDeliveries.get(traceId)?.closed);
+      }
+      /**
+       * Report what each tracked trace submitted and whether the server confirmed
+       * it, and stop tracking them. Every id passed is freed, so a caller cannot
+       * leak a record for a trace that never closed.
+       *
+       * `delivered` is only meaningful once a flush has settled: acks land before
+       * an export resolves, so a flush that reported success has already collected
+       * every ack it is going to collect.
+       */
+      takeTraceDeliveries(traceIds) {
+        const reports = {};
+        for (const traceId of traceIds) {
+          const delivery = this.traceDeliveries.get(traceId);
+          if (delivery === void 0) {
+            continue;
+          }
+          this.traceDeliveries.delete(traceId);
+          reports[traceId] = {
+            spanCount: delivery.submittedSpanIds.size,
+            closed: delivery.closed,
+            delivered: delivery.closingAcked && [...delivery.submittedSpanIds].every(
+              (spanId) => delivery.ackedSpanIds.has(spanId)
+            )
+          };
+        }
+        return reports;
+      }
+      /** Build a carrier's meta and record what it adds to its trace's expected set. */
+      recordedMeta(operation, payload, ref) {
+        this.recordSubmittedCarrier(ref);
+        return carrierMeta(operation, payload, ref);
+      }
+      recordSubmittedCarrier(ref) {
+        if (ref === void 0) {
+          return;
+        }
+        const delivery = this.traceDeliveries.get(ref.traceId);
+        if (delivery === void 0) {
+          return;
+        }
+        if (ref.spanId === void 0) {
+          delivery.closed = true;
+        } else {
+          delivery.submittedSpanIds.add(ref.spanId);
+        }
+      }
+      /**
+       * Ingestion commits every carrier in a request before it answers, so a
+       * delivered ref is proof its row exists: the same fact the replay status
+       * endpoint would report, already in hand.
+       */
+      recordDeliveredCarriers(refs) {
+        for (const ref of refs) {
+          const delivery = this.traceDeliveries.get(ref.traceId);
+          if (delivery === void 0) {
+            continue;
+          }
+          if (ref.spanId === void 0) {
+            delivery.closingAcked = true;
+          } else {
+            delivery.ackedSpanIds.add(ref.spanId);
+          }
+        }
+      }
       /**
        * Track deferred span work so this client's own lifecycle waits for it, and
        * so the process-wide flush and exit hook do too.
@@ -1337,13 +1623,16 @@ var init_http = __esm({
        * same data twice.
        */
       async sendEncoded(endpoint, body, options) {
+        const prepared = encodeRequestBody(body);
+        const encoded = prepared instanceof Promise ? await prepared : prepared;
+        return this.sendPrepared(endpoint, encoded, options);
+      }
+      async sendPrepared(endpoint, encoded, options) {
         const url = `${this.serviceUrl}${endpoint}`;
         const timeout = options?.timeout ?? this.timeout;
         const method = options?.method ?? "POST";
         const controller = new AbortController();
         const timeoutId = setTimeout(() => controller.abort(), timeout);
-        const prepared = encodeRequestBody(body);
-        const encoded = prepared instanceof Promise ? await prepared : prepared;
         const headers = {
           "Content-Type": "application/json",
           Authorization: `Bearer ${this.resolveApiKey() ?? ""}`
@@ -1363,7 +1652,8 @@ var init_http = __esm({
             throw new BitfabError(
               `HTTP ${response.status}: ${errorText.slice(0, 500)}`,
               void 0,
-              response.status
+              response.status,
+              parseRetryAfterMs(readHeader(response, "retry-after"))
             );
           }
           const result = await response.json();
@@ -1449,11 +1739,12 @@ var init_http = __esm({
        * the OTLP carrier has no path to carry it.
        */
       sendInternalTrace(functionId, payload) {
-        this.getTraceTransport()?.submit("internal_trace", {
-          ...payload,
-          functionId,
-          sdkVersion: __version__
-        });
+        const body = { ...payload, functionId, sdkVersion: __version__ };
+        this.getTraceTransport()?.submit(
+          "internal_trace",
+          body,
+          carrierMeta("internal_trace", body, void 0)
+        );
       }
       /**
        * Queue an external span (from withSpan wrapper or OpenAI tracing) onto this
@@ -1462,10 +1753,11 @@ var init_http = __esm({
        * promise.
        */
       sendExternalSpan(payload) {
-        this.getTraceTransport()?.submit("external_span", {
-          ...payload,
-          sdkVersion: __version__
-        });
+        this.getTraceTransport()?.submit(
+          "external_span",
+          { ...payload, sdkVersion: __version__ },
+          this.recordedMeta("external_span", payload, carrierRef(payload))
+        );
       }
       /**
        * Queue an external trace completion (from OpenAI tracing) onto this
@@ -1474,10 +1766,15 @@ var init_http = __esm({
        * server-authoritative barrier in `replay.ts`, not by awaiting this call.
        */
       sendExternalTrace(payload) {
-        this.getTraceTransport()?.submit("external_trace", {
-          ...payload,
-          sdkVersion: __version__
-        });
+        this.getTraceTransport()?.submit(
+          "external_trace",
+          { ...payload, sdkVersion: __version__ },
+          this.recordedMeta(
+            "external_trace",
+            payload,
+            payload.completed === true ? carrierRef(payload) : void 0
+          )
+        );
       }
       /**
        * Partial update of an existing trace identified by its Bitfab trace ID.
@@ -1817,8 +2114,8 @@ var init_serialize = __esm({
     import_superjson = __toESM(require("superjson"), 1);
     init_payloadBudget();
     init_warnOnce();
-    MAX_SERIALIZED_BYTES = MAX_SPAN_CARRIER_BYTES;
-    MAX_FRAMEWORK_SERIALIZED_BYTES = MAX_SPAN_CARRIER_BYTES;
+    MAX_SERIALIZED_BYTES = MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES;
+    MAX_FRAMEWORK_SERIALIZED_BYTES = MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES;
     MAX_SAFE_DEPTH = 6;
   }
 });
@@ -2073,7 +2370,8 @@ __export(replay_exports, {
   ReplayError: () => ReplayError,
   replay: () => replay,
   reportReplayProgress: () => reportReplayProgress,
-  serializeReplayResult: () => serializeReplayResult
+  serializeReplayResult: () => serializeReplayResult,
+  waitForReplayPersistence: () => waitForReplayPersistence
 });
 function dbBranchEnabled(dbBranch) {
   return dbBranch !== void 0 && dbBranch !== false;
@@ -2361,15 +2659,29 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
     REPLAY_PERSISTENCE_TIMEOUT_MS
   );
   if (!deferredSettled) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     throw new BitfabError(
       `Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (testRunId ${testRunId}).`
     );
   }
-  const expectedSpanCounts = takeReplaySpanCounts2(replayedTraceIds);
-  if (Object.keys(expectedSpanCounts).length === 0) {
+  if (!httpClient.hasClosedDeliveries(replayedTraceIds)) {
+    httpClient.takeTraceDeliveries(replayedTraceIds);
     return;
   }
   const flushed = await flushTraces(REPLAY_PERSISTENCE_TIMEOUT_MS);
+  const deliveries = httpClient.takeTraceDeliveries(replayedTraceIds);
+  const expectedSpanCounts = {};
+  let allDelivered = true;
+  for (const [traceId, delivery] of Object.entries(deliveries)) {
+    if (!delivery.closed) {
+      continue;
+    }
+    expectedSpanCounts[traceId] = delivery.spanCount;
+    allDelivered = allDelivered && delivery.delivered;
+  }
+  if (allDelivered) {
+    return;
+  }
   const deadline = Date.now() + REPLAY_PERSISTENCE_TIMEOUT_MS;
   let missing = Object.keys(expectedSpanCounts).length;
   while (true) {
@@ -2478,6 +2790,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     ...registeredOverrides
   ];
   const replayedTraceIds = serverItems.map(() => randomUuid());
+  httpClient.trackTraceDeliveries(replayedTraceIds);
   const tasks = serverItems.map(
     (serverItem, index) => () => processItem(
       httpClient,
@@ -2675,7 +2988,6 @@ var init_replay = __esm({
     init_randomUuid();
     init_replayContext();
     init_serialize();
-    init_transport();
     init_unrefTimer();
     REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
     BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";