bitfab 0.58.5 → 0.59.1

package/dist/{chunk-UYAQRSCB.js → chunk-NVOQ7YMI.js}

@@ -6,7 +6,7 @@ import {
   runWithAutoTraceEntry,
   runWithAutoTraceNodeConfiguration,
   runWithAutoTraceRootContext
-} from "./chunk-ETZUSSF6.js";
+} from "./chunk-5DNB7EN7.js";
 import {
   NO_MOCK_OVERRIDE,
   createIntegrationFixtureContext,
@@ -17,7 +17,7 @@ import {
   serializeValue,
   toJsonSafe,
   toJsonSafeReport
-} from "./chunk-TAEWAXPA.js";
+} from "./chunk-3D5EEY47.js";
 import {
   BitfabError,
   CONTENT_OFF_KEY,
@@ -28,6 +28,7 @@ import {
   ROOT_TRACE_FUNCTION_KEY_FIELD,
   SimulationPlan,
   callerTraceMetadata,
+  experimentIdWireFields,
   getReplayContext,
   gitRunner,
   isFrameworkInstrumentation,
@@ -35,7 +36,7 @@ import {
   readEnv,
   recordCallerTraceMetadata,
   retireTraceMetadata
-} from "./chunk-7SPNHC33.js";
+} from "./chunk-GJCZABUC.js";
 import {
   __privateAdd,
   __privateGet,
@@ -1262,11 +1263,39 @@ function currentCommitRef() {
 
 // src/datasets.ts
 var DEFAULT_RERUN_TIMEOUT_MS = 9e4;
-var DEFAULT_RERUN_POLL_INTERVAL_MS = 1e3;
+var RERUN_POLL_INTERVAL_MS = 1e3;
 var TERMINAL_RERUN_STATUSES = /* @__PURE__ */ new Set([
   "completed",
   "errored"
 ]);
+var GraderRerunError = class extends Error {
+  constructor(message, run) {
+    super(message);
+    this.run = run;
+    this.name = "GraderRerunError";
+  }
+};
+var GraderRerunTimeoutError = class extends GraderRerunError {
+  constructor(datasetId, run, timeoutMs) {
+    super(
+      `Grader re-run ${run.id} on dataset ${datasetId} was still ${run.status} after ${timeoutMs}ms. It keeps running on Bitfab; read it later with getGraderRerun.`,
+      run
+    );
+    this.name = "GraderRerunTimeoutError";
+  }
+};
+function finishedRerun(datasetId, run, joinedExisting) {
+  if (!TERMINAL_RERUN_STATUSES.has(run.status)) {
+    return null;
+  }
+  if (run.status === "errored") {
+    throw new GraderRerunError(
+      `Grader re-run ${run.id} on dataset ${datasetId} errored: ${run.error ?? "no reason recorded"}`,
+      run
+    );
+  }
+  return { run, joinedExisting };
+}
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1366,28 +1395,25 @@ var DatasetsClient = class {
       { graderIds }
     );
   }
-  /**
-   * Re-run graders over every trace in the dataset. Defaults to every assigned
-   * grader; an unassigned id is rejected. Waits for the run to finish (up to
-   * `timeoutMs`, default 90s) unless `wait` is `false`, and returns the last
-   * run state seen either way. A request matching an in-flight run joins it.
-   */
   async rerunGraders(datasetId, options = {}) {
     const started = await this.httpClient.request(
       datasetPath(datasetId, "/rerunGraders"),
       options.graderIds === void 0 ? {} : { graderIds: options.graderIds }
     );
-    if (options.wait === false) {
-      return started;
-    }
-    const deadline = Date.now() + (options.timeoutMs ?? DEFAULT_RERUN_TIMEOUT_MS);
-    const interval = options.pollIntervalMs ?? DEFAULT_RERUN_POLL_INTERVAL_MS;
+    const timeoutMs = options.timeoutMs ?? DEFAULT_RERUN_TIMEOUT_MS;
+    const deadline = Date.now() + timeoutMs;
     let run = started.run;
-    while (!TERMINAL_RERUN_STATUSES.has(run.status) && Date.now() < deadline) {
-      await sleep(interval);
+    while (true) {
+      const finished = finishedRerun(datasetId, run, started.joinedExisting);
+      if (finished !== null) {
+        return finished;
+      }
+      if (Date.now() >= deadline) {
+        throw new GraderRerunTimeoutError(datasetId, run, timeoutMs);
+      }
+      await sleep(RERUN_POLL_INTERVAL_MS);
       run = await this.getGraderRerun(datasetId, run.id) ?? run;
     }
-    return { run, joinedExisting: started.joinedExisting };
   }
   /**
    * The dataset's active grader re-run, or the run named by `runId`. Returns
@@ -1459,27 +1485,27 @@ var LabelsClient = class {
   constructor(httpClient) {
     this.httpClient = httpClient;
   }
-  async save(update, testRunId) {
-    const [outcome] = await this.saveAll([update], testRunId);
+  async save(update, experimentId) {
+    const [outcome] = await this.saveAll([update], experimentId);
     return outcome;
   }
-  async saveAll(updates, testRunId) {
+  async saveAll(updates, experimentId) {
     const response = await this.httpClient.request(
       LABELS_PATH,
       {
         labels: updates,
-        ...testRunId === void 0 ? {} : { testRunId }
+        ...experimentId === void 0 ? {} : experimentIdWireFields(experimentId)
       }
     );
     return response.labels;
   }
   /** Withhold a verdict for a trace or replay attempt whose check did not run. */
-  async skip(target, testRunId) {
-    return this.save({ ...target, skip: true }, testRunId);
+  async skip(target, experimentId) {
+    return this.save({ ...target, skip: true }, experimentId);
   }
   /** Clear a previous verdict for a trace, replay attempt, or assertion. */
-  async archive(target, testRunId) {
-    return this.save({ ...target, archive: true }, testRunId);
+  async archive(target, experimentId) {
+    return this.save({ ...target, archive: true }, experimentId);
   }
   async saveHuman(update) {
     const [outcome] = await this.saveHumanAll([update]);
@@ -3205,6 +3231,46 @@ function runWithSeedContext(ctx, fn) {
 // src/traces.ts
 var ASSERTIONS_PATH = "/api/sdk/traces/assertions";
 var SEARCH_PATH = "/api/sdk/traces/search";
+var DEFAULT_GENERATION_TIMEOUT_MS = 6e5;
+var GENERATION_POLL_INTERVAL_MS = 2e3;
+var AssertionGenerationError = class extends Error {
+  constructor(message, traceId) {
+    super(message);
+    this.traceId = traceId;
+    this.name = "AssertionGenerationError";
+  }
+};
+var AssertionGenerationTimeoutError = class extends AssertionGenerationError {
+  constructor(traceId, timeoutMs) {
+    super(
+      `Assertion generation for trace ${traceId} did not finish within ${timeoutMs}ms. Generation keeps running on Bitfab. Call getAssertions and read its generation field to check its status.`,
+      traceId
+    );
+    this.name = "AssertionGenerationTimeoutError";
+  }
+};
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isGenerationFinished(traceId, generation) {
+  if (generation === null || generation.status === "running") {
+    return false;
+  }
+  if (generation.status === "failed") {
+    throw new AssertionGenerationError(
+      `Assertion generation for trace ${traceId} failed: ${generation.error ?? "no reason recorded"}. Pass regenerate: true to try again.`,
+      traceId
+    );
+  }
+  return true;
+}
+function draftsInGenerationOrder(assertions, assertionIds) {
+  const byId = new Map(assertions.map((assertion) => [assertion.id, assertion]));
+  return assertionIds.flatMap((id) => {
+    const assertion = byId.get(id);
+    return assertion === void 0 ? [] : [assertion];
+  });
+}
 function traceAssertionsPath(traceId, suffix = "") {
   return `/api/sdk/traces/${encodeURIComponent(traceId)}/assertions${suffix}`;
 }
@@ -3236,6 +3302,34 @@ var TracesClient = class {
     });
     return response.assertions;
   }
+  async generateAssertions(traceId, options = {}) {
+    const started = await this.httpClient.request(
+      traceAssertionsPath(traceId, "/generate"),
+      {
+        ...options.guidance === void 0 ? {} : { guidance: options.guidance },
+        ...options.categoryIds === void 0 ? {} : { categoryIds: options.categoryIds },
+        ...options.regenerate === void 0 ? {} : { regenerate: options.regenerate }
+      }
+    );
+    if (isGenerationFinished(traceId, started.generation)) {
+      return started.assertions;
+    }
+    const timeoutMs = options.timeoutMs ?? DEFAULT_GENERATION_TIMEOUT_MS;
+    const deadline = Date.now() + timeoutMs;
+    while (true) {
+      if (Date.now() >= deadline) {
+        throw new AssertionGenerationTimeoutError(traceId, timeoutMs);
+      }
+      await sleep2(GENERATION_POLL_INTERVAL_MS);
+      const read2 = await this.getAssertions(traceId);
+      if (isGenerationFinished(traceId, read2.generation)) {
+        return draftsInGenerationOrder(
+          read2.assertions,
+          read2.generation.assertionIds
+        );
+      }
+    }
+  }
   async archiveAssertions(params) {
     const response = await this.httpClient.request(
       traceAssertionsPath(params.traceId, "/archive"),
@@ -3734,6 +3828,15 @@ var BitfabVercelAiHandler = class {
 };
 
 // src/client.ts
+function resolveSpanExperimentId(options) {
+  const { experimentId, testRunId } = options;
+  if (experimentId !== void 0 && testRunId !== void 0 && experimentId !== testRunId) {
+    throw new BitfabError(
+      `experimentId ("${experimentId}") and its deprecated alias testRunId ("${testRunId}") were both passed with different values. Pass only experimentId.`
+    );
+  }
+  return experimentId ?? testRunId;
+}
 var TRUNCATION_LIMIT_ORDER = [
   "max_depth",
   "max_captured_subtree_spans"
@@ -4323,13 +4426,12 @@ var Bitfab = class {
         "bitfab.node({ capture: false }) cannot use mockOnReplay: true because an uncaptured node has no recorded output."
       );
     }
+    const experimentId = resolveSpanExperimentId(options);
     return {
       capture,
       type: options.type ?? "custom",
       ...options.name !== void 0 && { name: options.name },
-      ...options.testRunId !== void 0 && {
-        testRunId: options.testRunId
-      },
+      ...experimentId !== void 0 && { experimentId },
       ...options.mockOnReplay !== void 0 && {
         mockOnReplay: options.mockOnReplay
       },
@@ -4464,6 +4566,7 @@ var Bitfab = class {
               return invokeWithoutNode();
             }
             const mockOnReplay = nodeConfiguration?.mockOnReplay ?? options.mockOnReplayDefault;
+            const nodeExperimentId = nodeConfiguration === void 0 ? void 0 : resolveSpanExperimentId(nodeConfiguration);
             const childOptions = {
               name: spanName,
               type: nodeConfiguration?.type ?? "function",
@@ -4474,8 +4577,8 @@ var Bitfab = class {
               captureContent: true,
               autoTraceDefinition: definition,
               ...declaredNode && { declaredNode },
-              ...nodeConfiguration?.testRunId !== void 0 && {
-                testRunId: nodeConfiguration.testRunId
+              ...nodeExperimentId !== void 0 && {
+                experimentId: nodeExperimentId
               },
               ...mockOnReplay !== void 0 && {
                 mockOnReplay
@@ -5136,6 +5239,7 @@ var Bitfab = class {
     const options = typeof optionsOrFn === "function" ? {} : optionsOrFn;
     const fn = typeof optionsOrFn === "function" ? optionsOrFn : maybeFn;
     const self = this;
+    const optionExperimentId = resolveSpanExperimentId(options);
     const fnIsAsyncFunction = fn.constructor.name === "AsyncFunction";
     const fnReturnsPromise = fnIsAsyncFunction || (() => {
       try {
@@ -5238,7 +5342,7 @@ var Bitfab = class {
         const inputs = args;
         const startedAt = nowIsoTimestamp();
         const replayCtxAtStart = getReplayContext();
-        const testRunId = replayCtxAtStart?.testRunId ?? options.testRunId;
+        const experimentId = replayCtxAtStart?.experimentId ?? optionExperimentId;
         if (isRootSpan && !activeTraceStates.has(traceId)) {
           const dbSnapshotRef = buildSnapshotRef(self.dbSnapshot, startedAt);
           self.simulationPlan.refresh();
@@ -5247,7 +5351,7 @@ var Bitfab = class {
             traceFunctionKey,
             startedAt,
             contexts: [],
-            ...testRunId !== void 0 && { testRunId },
+            ...experimentId !== void 0 && { experimentId },
             ...replayCtxAtStart?.inputSourceTraceId && {
               inputSourceTraceId: replayCtxAtStart.inputSourceTraceId
             },
@@ -5293,7 +5397,7 @@ var Bitfab = class {
                 contexts: newContext.contexts,
                 prompt: newContext.prompt,
                 endedAt,
-                ...testRunId !== void 0 && { testRunId },
+                ...experimentId !== void 0 && { experimentId },
                 ...replayCtx?.inputSourceSpanId && {
                   inputSourceSpanId: replayCtx.inputSourceSpanId
                 }
@@ -5314,7 +5418,7 @@ var Bitfab = class {
                 name: traceState?.name,
                 metadata: callerTraceMetadata(traceId),
                 contexts: traceState?.contexts ?? [],
-                testRunId: traceState?.testRunId,
+                experimentId: traceState?.experimentId,
                 inputSourceTraceId: traceState?.inputSourceTraceId,
                 replayAttempt: traceState?.replayAttempt,
                 dbSnapshotRef: traceState?.dbSnapshotRef,
@@ -5785,7 +5889,7 @@ var Bitfab = class {
       completed: true,
       ...params.dropped && { dropped: true },
       ...params.sessionId && { sessionId: params.sessionId },
-      ...params.testRunId && { testRunId: params.testRunId }
+      ...params.experimentId !== void 0 && experimentIdWireFields(params.experimentId)
     });
   }
   sendsSpanOnCompletion(context) {
@@ -5897,7 +6001,7 @@ var Bitfab = class {
       },
       ...params.declaredNode === true && { [DECLARED_NODE_FIELD]: true },
       rawSpan: externalSpan,
-      ...params.testRunId && { testRunId: params.testRunId },
+      ...params.experimentId !== void 0 && experimentIdWireFields(params.experimentId),
       ...params.mocked && { mocked: true },
       ...params.mockTarget && { mockTarget: params.mockTarget },
       ...params.mockSource && { mockSource: params.mockSource }
@@ -6043,7 +6147,7 @@ var Bitfab = class {
       await runWithSeedContext({ traceId }, async () => target(...args));
     } finally {
       try {
-        const { flushTraces: flushTraces2 } = await import("./http-F6QV3YLY.js");
+        const { flushTraces: flushTraces2 } = await import("./http-2QEMP27E.js");
         await flushTraces2(3e4);
       } finally {
         unrecorded = activeTraceStates.delete(traceId);
@@ -6102,7 +6206,7 @@ var Bitfab = class {
         `Function is wrapped with trace function key '${wrappedKey}' but replay was called with '${traceFunctionKey}'. Pass matching keys, or pass the unwrapped function to replay it under the explicit key.`
       );
     }
-    const { replay: doReplay } = await import("./replay-WLJFWS6T.js");
+    const { replay: doReplay } = await import("./replay-WYPFFXSU.js");
     return doReplay(
       this.httpClient,
       this.serviceUrl,
@@ -6422,7 +6526,7 @@ async function seedFromRegistry(registry, pipeline, cases, options = {}) {
       sessionId: seedCase.sessionId
     })
   );
-  const { flushTraces: flushTraces2 } = await import("./http-F6QV3YLY.js");
+  const { flushTraces: flushTraces2 } = await import("./http-2QEMP27E.js");
   await flushTraces2(3e4);
   return { pipeline, traceFunctionKey, traceIds };
 }
@@ -6440,6 +6544,8 @@ function resolveTraceFunctionKey(registration) {
 export {
   AssertionCategoriesClient,
   BitfabClaudeAgentHandler,
+  GraderRerunError,
+  GraderRerunTimeoutError,
   DatasetsClient,
   SUPPORTED_PROVIDERS,
   GradersClient,
@@ -6449,6 +6555,8 @@ export {
   BitfabOpenAIAgentHandler,
   OrganizationMembersClient,
   BitfabRivetIntegration,
+  AssertionGenerationError,
+  AssertionGenerationTimeoutError,
   TracesClient,
   BitfabOpenAITracingProcessor,
   BitfabVercelAiHandler,
@@ -6462,4 +6570,4 @@ export {
   reseedFromRegistry,
   seedFromRegistry
 };
-//# sourceMappingURL=chunk-UYAQRSCB.js.map
+//# sourceMappingURL=chunk-NVOQ7YMI.js.map