bitfab 0.59.0 → 0.60.0

package/dist/index.cjs

@@ -51,7 +51,7 @@ var __version__, __packageName__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.59.0";
+    __version__ = "0.60.0";
     __packageName__ = "bitfab";
   }
 });
@@ -2113,6 +2113,7 @@ __export(http_exports, {
   HttpClient: () => HttpClient,
   awaitOnExit: () => awaitOnExit,
   awaitPendingRequests: () => awaitPendingRequests,
+  experimentIdWireFields: () => experimentIdWireFields,
   flushTraces: () => flushTraces,
   parseRetryAfterMs: () => parseRetryAfterMs,
   serializePayloadBody: () => serializePayloadBody
@@ -2281,6 +2282,9 @@ function warnDroppedExternalTrace(error) {
     `a trace was dropped because the send step failed: ${describeError(error)}`
   );
 }
+function experimentIdWireFields(experimentId) {
+  return { experimentId, testRunId: experimentId };
+}
 var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, REPLAY_COMPLETE_REQUEST_TIMEOUT_MS, OTLP_TRACES_ENDPOINT, RETRYABLE_STATUSES, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, liveHttpClients, carrierSeq, HttpClient;
 var init_http = __esm({
   "src/http.ts"() {
@@ -2885,7 +2889,7 @@ var init_http = __esm({
       }
       /**
        * Start a replay session by fetching historical traces.
-       * Blocking call - creates a test run and returns lightweight item references.
+       * Blocking call - creates an experiment and returns lightweight item references.
        */
       async startReplay(traceFunctionKey, limit, traceIds, name, codeChangeDescription, codeChangeFiles, includeDbBranchLease, experimentGroupId, datasetIds, graderIds, dbBranchSettings, attempts, includeOriginalMetadata, onlyWithAssertions, judgeAssertions) {
         const payload = { traceFunctionKey };
@@ -2941,9 +2945,20 @@ var init_http = __esm({
           payload.git = git;
         }
         const timeout = includeDbBranchLease ? REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS : 3e4;
-        return this.request("/api/sdk/replay/start", payload, {
-          timeout
-        });
+        const response = await this.request(
+          "/api/sdk/replay/start",
+          payload,
+          { timeout }
+        );
+        const { testRunId, testRunUrl, ...rest } = response;
+        const experimentId = response.experimentId ?? testRunId;
+        const experimentUrl = response.experimentUrl ?? testRunUrl;
+        if (experimentId === void 0 || experimentUrl === void 0) {
+          throw new BitfabError(
+            "Replay start response is missing the experiment ID or URL."
+          );
+        }
+        return { ...rest, experimentId, experimentUrl };
       }
       /**
        * Fetch an external span by ID.
@@ -3040,21 +3055,21 @@ var init_http = __esm({
        * has a final status AND at least that many persisted spans, which is what
        * makes this a real barrier rather than a "the row exists" check.
        */
-      async getReplayStatus(testRunId, expectedSpanCounts) {
+      async getReplayStatus(experimentId, expectedSpanCounts) {
         return this.request(
           "/api/sdk/replay/status",
-          { testRunId, expectedSpanCounts },
+          { ...experimentIdWireFields(experimentId), expectedSpanCounts },
           { timeout: 3e4 }
         );
       }
       /**
-       * Mark a replay test run as completed.
+       * Mark a replay experiment as completed.
        * Blocking call.
        */
-      async completeReplay(testRunId) {
+      async completeReplay(experimentId) {
         return this.request(
           "/api/sdk/replay/complete",
-          { testRunId },
+          experimentIdWireFields(experimentId),
           { timeout: REPLAY_COMPLETE_REQUEST_TIMEOUT_MS }
         );
       }
@@ -3076,11 +3091,11 @@ var init_http = __esm({
        * snapshot + preview branch and polls operations to readiness, which
        * can take seconds.
        */
-      async resolveDbBranchLease(testRunId, traceId, dbBranchSettings, attempt) {
+      async resolveDbBranchLease(experimentId, traceId, dbBranchSettings, attempt) {
         return this.request(
           "/api/sdk/replay/resolveDbBranchLease",
           {
-            testRunId,
+            ...experimentIdWireFields(experimentId),
             traceId,
             dbBranchSettings,
             ...attempt !== void 0 && attempt > 0 ? { attempt } : {}
@@ -3669,7 +3684,7 @@ function replayJsonReplacer(_key, value) {
 function serializeReplayResult(result) {
   return JSON.stringify(result, replayJsonReplacer, 2);
 }
-async function preserveReplayFailure(operation, items, testRunId, testRunUrl) {
+async function preserveReplayFailure(operation, items, experimentId, experimentUrl) {
   try {
     return await operation();
   } catch (cause) {
@@ -3679,8 +3694,8 @@ async function preserveReplayFailure(operation, items, testRunId, testRunUrl) {
     throw new ReplayError(
       errorMessage(cause),
       items,
-      testRunId,
-      testRunUrl,
+      experimentId,
+      experimentUrl,
       cause
     );
   }
@@ -3734,7 +3749,7 @@ function buildMockTree(rootNode) {
   }
   return { spans };
 }
-async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
+async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
   let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
   let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
   let dbSnapshotRef = serverItem.dbSnapshotRef;
@@ -3754,7 +3769,7 @@ async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy,
       let resolved2;
       try {
         resolved2 = await httpClient.resolveDbBranchLease(
-          testRunId,
+          experimentId,
           originalTraceId,
           dbBranchSettings,
           attempt
@@ -3866,7 +3881,7 @@ async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy,
       replayStarted = performance.now();
       const maybePromise = runWithReplayContext(
         {
-          testRunId,
+          experimentId,
           traceId: replayedTraceId,
           inputSourceSpanId: span.id,
           inputSourceTraceId: span.externalTraceId,
@@ -3949,14 +3964,14 @@ async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy,
     originalTraceOutline: null
   };
 }
-async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds) {
+async function waitForReplayPersistence(httpClient, experimentId, replayedTraceIds) {
   const deferredSettled = await httpClient.settleDeferredWork(
     REPLAY_PERSISTENCE_TIMEOUT_MS
   );
   if (!deferredSettled) {
     httpClient.takeTraceDeliveries(replayedTraceIds);
     throw new BitfabError(
-      `Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (testRunId ${testRunId}).`
+      `Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (experiment ${experimentId}).`
     );
   }
   if (!httpClient.hasClosedDeliveries(replayedTraceIds)) {
@@ -3985,7 +4000,7 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
   let missing = Object.keys(expectedSpanCounts).length;
   while (true) {
     const status = await httpClient.getReplayStatus(
-      testRunId,
+      experimentId,
       expectedSpanCounts
     );
     const ready = status.traceIds ?? {};
@@ -4004,7 +4019,7 @@ async function waitForReplayPersistence(httpClient, testRunId, replayedTraceIds)
   }
   const cause = flushed ? "" : " Delivery was also not confirmed before the flush deadline, so the spans likely never reached the server.";
   throw new BitfabError(
-    `Replay traces were not fully persisted before the delivery deadline (testRunId ${testRunId}, missing ${missing} of ${Object.keys(expectedSpanCounts).length} trace(s)).${cause}`
+    `Replay traces were not fully persisted before the delivery deadline (experiment ${experimentId}, missing ${missing} of ${Object.keys(expectedSpanCounts).length} trace(s)).${cause}`
   );
 }
 function sleepForReplayPersistence(ms) {
@@ -4071,8 +4086,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     }
   }
   const {
-    testRunId,
-    testRunUrl,
+    experimentId,
+    experimentUrl,
     items: serverItems
   } = await httpClient.startReplay(
     traceFunctionKey,
@@ -4105,7 +4120,14 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     }
   }
   const mockStrategy = options?.mock ?? "marked";
-  const fullTestRunUrl = `${serviceUrl}${testRunUrl}`;
+  const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
+  try {
+    options?.onExperimentStart?.({
+      experimentId,
+      experimentUrl: fullExperimentUrl
+    });
+  } catch {
+  }
   const resolvedOverrides = [
     ...normalizeMockOverrides(options?.mockOverride),
     ...registeredOverrides
@@ -4118,7 +4140,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       httpClient,
       serverItem,
       fn,
-      testRunId,
+      experimentId,
       mockStrategy,
       resolvedOverrides,
       replayedTraceIds[index],
@@ -4173,7 +4195,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
       try {
         onItemFinish?.({
-          testRunId,
+          experimentId,
+          testRunId: experimentId,
           completed,
           total,
           succeeded,
@@ -4221,7 +4244,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       try {
         options.onItemStart?.({
           type: "started",
-          testRunId,
+          experimentId,
+          testRunId: experimentId,
           started,
           completed,
           total,
@@ -4240,21 +4264,23 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     } : void 0
   );
   if (options?.dryRun === true) {
-    await httpClient.completeReplay(testRunId).catch(() => void 0);
+    await httpClient.completeReplay(experimentId).catch(() => void 0);
     const dryResult = {
       items: resultItems,
-      testRunId,
-      testRunUrl: fullTestRunUrl,
+      experimentId,
+      experimentUrl: fullExperimentUrl,
+      testRunId: experimentId,
+      testRunUrl: fullExperimentUrl,
       attempts
     };
     await writeReplayResultFile(dryResult);
     return dryResult;
   }
   const deliveredTraceIds = await preserveReplayFailure(
-    () => waitForReplayPersistence(httpClient, testRunId, replayedTraceIds),
+    () => waitForReplayPersistence(httpClient, experimentId, replayedTraceIds),
     resultItems,
-    testRunId,
-    fullTestRunUrl
+    experimentId,
+    fullExperimentUrl
   );
   for (let index = 0; index < resultItems.length; index += 1) {
     const localId = replayedTraceIds[index];
@@ -4264,10 +4290,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     }
   }
   const completeResult = await preserveReplayFailure(
-    () => httpClient.completeReplay(testRunId),
+    () => httpClient.completeReplay(experimentId),
     resultItems,
-    testRunId,
-    fullTestRunUrl
+    experimentId,
+    fullExperimentUrl
   );
   const serverTraceIds = completeResult.traceIds;
   const replayTokens = completeResult.tokens;
@@ -4294,20 +4320,20 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     if (completedCount > 0 && missing.length === completedCount) {
       const serverCount = completeResult.traceCount !== void 0 ? ` The server persisted ${completeResult.traceCount} trace(s) for this run.` : "";
       const cause = new BitfabError(
-        `Replay completed but the server has no persisted trace for any of the ${completedCount} completed item(s) (testRunId ${testRunId}).${serverCount} Trace uploads were awaited, so either the uploads failed (check for "Bitfab: Failed to create" errors above) or the replayed function is not wrapped with withSpan.`
+        `Replay completed but the server has no persisted trace for any of the ${completedCount} completed item(s) (experiment ${experimentId}).${serverCount} Trace uploads were awaited, so either the uploads failed (check for "Bitfab: Failed to create" errors above) or the replayed function is not wrapped with withSpan.`
       );
       throw new ReplayError(
         cause.message,
         resultItems,
-        testRunId,
-        fullTestRunUrl,
+        experimentId,
+        fullExperimentUrl,
         cause
       );
     }
     if (missing.length > 0) {
       try {
         console.error(
-          `Bitfab: server has no persisted trace for ${missing.length} of ${completedCount} completed replay item(s) (testRunId ${testRunId}). Their replay token usage is unavailable and they cannot be labeled.`
+          `Bitfab: server has no persisted trace for ${missing.length} of ${completedCount} completed replay item(s) (experiment ${experimentId}). Their replay token usage is unavailable and they cannot be labeled.`
         );
       } catch {
       }
@@ -4315,8 +4341,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   }
   const result = {
     items: resultItems,
-    testRunId,
-    testRunUrl: fullTestRunUrl,
+    experimentId,
+    experimentUrl: fullExperimentUrl,
+    testRunId: experimentId,
+    testRunUrl: fullExperimentUrl,
     attempts
   };
   await writeReplayResultFile(result);
@@ -4324,7 +4352,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     try {
       options?.onProgress?.({
         type: "complete",
-        testRunId,
+        experimentId,
+        testRunId: experimentId,
         completed: total,
         total,
         succeeded,
@@ -4374,13 +4403,15 @@ var init_replay = __esm({
     MAX_REPLAY_ATTEMPTS = 100;
     BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
     ReplayError = class extends BitfabError {
-      constructor(message, items, testRunId, testRunUrl, cause) {
-        super(message, testRunUrl);
+      constructor(message, items, experimentId, experimentUrl, cause) {
+        super(message, experimentUrl);
         this.items = items;
-        this.testRunId = testRunId;
-        this.testRunUrl = testRunUrl;
+        this.experimentId = experimentId;
+        this.experimentUrl = experimentUrl;
         this.cause = cause;
         this.name = "ReplayError";
+        this.testRunId = experimentId;
+        this.testRunUrl = experimentUrl;
       }
     };
     DbBranchReplayError = class extends BitfabError {
@@ -6066,6 +6097,7 @@ var GradersClient = class {
 init_http();
 
 // src/labels.ts
+init_http();
 var LABELS_PATH = "/api/sdk/traces/labels";
 var HUMAN_LABELS_PATH = "/api/sdk/traces/labels/human";
 function labelEvidencePath(traceId, assertionId) {
@@ -6075,27 +6107,27 @@ var LabelsClient = class {
   constructor(httpClient) {
     this.httpClient = httpClient;
   }
-  async save(update, testRunId) {
-    const [outcome] = await this.saveAll([update], testRunId);
+  async save(update, experimentId) {
+    const [outcome] = await this.saveAll([update], experimentId);
     return outcome;
   }
-  async saveAll(updates, testRunId) {
+  async saveAll(updates, experimentId) {
     const response = await this.httpClient.request(
       LABELS_PATH,
       {
         labels: updates,
-        ...testRunId === void 0 ? {} : { testRunId }
+        ...experimentId === void 0 ? {} : experimentIdWireFields(experimentId)
       }
     );
     return response.labels;
   }
   /** Withhold a verdict for a trace or replay attempt whose check did not run. */
-  async skip(target, testRunId) {
-    return this.save({ ...target, skip: true }, testRunId);
+  async skip(target, experimentId) {
+    return this.save({ ...target, skip: true }, experimentId);
   }
   /** Clear a previous verdict for a trace, replay attempt, or assertion. */
-  async archive(target, testRunId) {
-    return this.save({ ...target, archive: true }, testRunId);
+  async archive(target, experimentId) {
+    return this.save({ ...target, archive: true }, experimentId);
   }
   async saveHuman(update) {
     const [outcome] = await this.saveHumanAll([update]);
@@ -8453,6 +8485,15 @@ var BitfabVercelAiHandler = class {
 
 // src/client.ts
 init_warnOnce();
+function resolveSpanExperimentId(options) {
+  const { experimentId, testRunId } = options;
+  if (experimentId !== void 0 && testRunId !== void 0 && experimentId !== testRunId) {
+    throw new BitfabError(
+      `experimentId ("${experimentId}") and its deprecated alias testRunId ("${testRunId}") were both passed with different values. Pass only experimentId.`
+    );
+  }
+  return experimentId ?? testRunId;
+}
 var TRUNCATION_LIMIT_ORDER = [
   "max_depth",
   "max_captured_subtree_spans"
@@ -9042,13 +9083,12 @@ var Bitfab = class {
         "bitfab.node({ capture: false }) cannot use mockOnReplay: true because an uncaptured node has no recorded output."
       );
     }
+    const experimentId = resolveSpanExperimentId(options);
     return {
       capture,
       type: options.type ?? "custom",
       ...options.name !== void 0 && { name: options.name },
-      ...options.testRunId !== void 0 && {
-        testRunId: options.testRunId
-      },
+      ...experimentId !== void 0 && { experimentId },
       ...options.mockOnReplay !== void 0 && {
         mockOnReplay: options.mockOnReplay
       },
@@ -9183,6 +9223,7 @@ var Bitfab = class {
               return invokeWithoutNode();
             }
             const mockOnReplay = nodeConfiguration?.mockOnReplay ?? options.mockOnReplayDefault;
+            const nodeExperimentId = nodeConfiguration === void 0 ? void 0 : resolveSpanExperimentId(nodeConfiguration);
             const childOptions = {
               name: spanName,
               type: nodeConfiguration?.type ?? "function",
@@ -9193,8 +9234,8 @@ var Bitfab = class {
               captureContent: true,
               autoTraceDefinition: definition,
               ...declaredNode && { declaredNode },
-              ...nodeConfiguration?.testRunId !== void 0 && {
-                testRunId: nodeConfiguration.testRunId
+              ...nodeExperimentId !== void 0 && {
+                experimentId: nodeExperimentId
               },
               ...mockOnReplay !== void 0 && {
                 mockOnReplay
@@ -9855,6 +9896,7 @@ var Bitfab = class {
     const options = typeof optionsOrFn === "function" ? {} : optionsOrFn;
     const fn = typeof optionsOrFn === "function" ? optionsOrFn : maybeFn;
     const self = this;
+    const optionExperimentId = resolveSpanExperimentId(options);
     const fnIsAsyncFunction = fn.constructor.name === "AsyncFunction";
     const fnReturnsPromise = fnIsAsyncFunction || (() => {
       try {
@@ -9957,7 +9999,7 @@ var Bitfab = class {
         const inputs = args;
         const startedAt = nowIsoTimestamp();
         const replayCtxAtStart = getReplayContext();
-        const testRunId = replayCtxAtStart?.testRunId ?? options.testRunId;
+        const experimentId = replayCtxAtStart?.experimentId ?? optionExperimentId;
         if (isRootSpan && !activeTraceStates.has(traceId)) {
           const dbSnapshotRef = buildSnapshotRef(self.dbSnapshot, startedAt);
           self.simulationPlan.refresh();
@@ -9966,7 +10008,7 @@ var Bitfab = class {
             traceFunctionKey,
             startedAt,
             contexts: [],
-            ...testRunId !== void 0 && { testRunId },
+            ...experimentId !== void 0 && { experimentId },
             ...replayCtxAtStart?.inputSourceTraceId && {
               inputSourceTraceId: replayCtxAtStart.inputSourceTraceId
             },
@@ -10012,7 +10054,7 @@ var Bitfab = class {
                 contexts: newContext.contexts,
                 prompt: newContext.prompt,
                 endedAt,
-                ...testRunId !== void 0 && { testRunId },
+                ...experimentId !== void 0 && { experimentId },
                 ...replayCtx?.inputSourceSpanId && {
                   inputSourceSpanId: replayCtx.inputSourceSpanId
                 }
@@ -10033,7 +10075,7 @@ var Bitfab = class {
                 name: traceState?.name,
                 metadata: callerTraceMetadata(traceId),
                 contexts: traceState?.contexts ?? [],
-                testRunId: traceState?.testRunId,
+                experimentId: traceState?.experimentId,
                 inputSourceTraceId: traceState?.inputSourceTraceId,
                 replayAttempt: traceState?.replayAttempt,
                 dbSnapshotRef: traceState?.dbSnapshotRef,
@@ -10347,9 +10389,6 @@ var Bitfab = class {
     return {
       traceId,
       addContext: (context) => {
-        if (!this.shouldRecord()) {
-          return Promise.resolve();
-        }
         if (typeof context !== "object" || context === null) {
           return Promise.resolve();
         }
@@ -10358,27 +10397,18 @@ var Bitfab = class {
         });
       },
       setMetadata: (metadata) => {
-        if (!this.shouldRecord()) {
-          return Promise.resolve();
-        }
         if (typeof metadata !== "object" || metadata === null) {
           return Promise.resolve();
         }
         return this.httpClient.patchTrace(traceId, { mergeMetadata: metadata });
       },
       setSessionId: (sessionId) => {
-        if (!this.shouldRecord()) {
-          return Promise.resolve();
-        }
         if (typeof sessionId !== "string" || sessionId.length === 0) {
           return Promise.resolve();
         }
         return this.httpClient.patchTrace(traceId, { setSessionId: sessionId });
       },
       setName: (name) => {
-        if (!this.shouldRecord()) {
-          return Promise.resolve();
-        }
         if (typeof name !== "string" || name.length === 0) {
           return Promise.resolve();
         }
@@ -10504,7 +10534,7 @@ var Bitfab = class {
       completed: true,
       ...params.dropped && { dropped: true },
       ...params.sessionId && { sessionId: params.sessionId },
-      ...params.testRunId && { testRunId: params.testRunId }
+      ...params.experimentId !== void 0 && experimentIdWireFields(params.experimentId)
     });
   }
   sendsSpanOnCompletion(context) {
@@ -10616,7 +10646,7 @@ var Bitfab = class {
       },
       ...params.declaredNode === true && { [DECLARED_NODE_FIELD]: true },
       rawSpan: externalSpan,
-      ...params.testRunId && { testRunId: params.testRunId },
+      ...params.experimentId !== void 0 && experimentIdWireFields(params.experimentId),
       ...params.mocked && { mocked: true },
       ...params.mockTarget && { mockTarget: params.mockTarget },
       ...params.mockSource && { mockSource: params.mockSource }