bitfab 0.61.0 → 0.61.2

package/dist/index.cjs

@@ -51,7 +51,7 @@ var __version__, __packageName__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.61.0";
+    __version__ = "0.61.2";
     __packageName__ = "bitfab";
   }
 });
@@ -2960,6 +2960,46 @@ var init_http = __esm({
         }
         return { ...rest, experimentId, experimentUrl };
       }
+      async resumeReplay(experimentId, traceFunctionKey, codeChangeFiles, force) {
+        const payload = {
+          experimentId,
+          traceFunctionKey,
+          lazyDbBranchLease: true
+        };
+        if (force === true) {
+          payload.force = true;
+        }
+        if (codeChangeFiles !== void 0) {
+          payload.codeChangeFiles = codeChangeFiles;
+        }
+        const git = await resolvedGitState();
+        if (git && !isEmptyGitState(git)) {
+          payload.git = git;
+        }
+        const response = await this.request(
+          "/api/sdk/replay/resume",
+          payload,
+          { timeout: REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS }
+        );
+        const { testRunId, testRunUrl, ...rest } = response;
+        const resumedExperimentId = response.experimentId ?? testRunId;
+        const experimentUrl = response.experimentUrl ?? testRunUrl;
+        if (resumedExperimentId === void 0 || experimentUrl === void 0) {
+          throw new BitfabError(
+            "Replay resume response is missing the experiment ID or URL."
+          );
+        }
+        return {
+          ...rest,
+          experimentId: resumedExperimentId,
+          experimentUrl,
+          work: rest.work ?? [],
+          finished: rest.finished ?? [],
+          skippedOriginalTraceIds: rest.skippedOriginalTraceIds ?? [],
+          includeDbBranchLease: rest.includeDbBranchLease ?? false,
+          dbBranchSettings: rest.dbBranchSettings ?? null
+        };
+      }
       /**
        * Fetch an external span by ID.
        * Blocking GET request.
@@ -3595,6 +3635,88 @@ function expandReplayWork(serverItems, attempts) {
     (_, attempt) => serverItems.map((serverItem) => ({ serverItem, attempt }))
   ).flat();
 }
+function validateResume(resume, options) {
+  if (resume.trim().length === 0) {
+    throw new BitfabError("resume must be the ID of the experiment to resume.");
+  }
+  const conflicting = RESUME_CONFLICTING_OPTIONS.filter(
+    (name) => options[name] !== void 0 && options[name] !== false
+  );
+  if (conflicting.length > 0) {
+    throw new BitfabError(
+      `resume continues experiment ${resume} with the traces, attempts, and graders it started with, so it cannot be combined with ${conflicting.join(", ")}. Drop those options, or start a new replay without resume.`
+    );
+  }
+}
+async function resumeReplaySession(httpClient, experimentId, traceFunctionKey, codeChangeFiles, force, requestedDbBranch) {
+  const response = await httpClient.resumeReplay(
+    experimentId,
+    traceFunctionKey,
+    codeChangeFiles,
+    force
+  );
+  const work = response.work.map(({ itemIndex, attempt }) => {
+    const serverItem = response.items[itemIndex];
+    if (serverItem === void 0) {
+      throw new BitfabError(
+        `Replay resume for experiment ${response.experimentId} named item ${itemIndex}, but the server returned only ${response.items.length} item(s).`
+      );
+    }
+    return { serverItem, attempt };
+  });
+  if (response.skippedOriginalTraceIds.length > 0) {
+    try {
+      console.warn(
+        `Bitfab: ${response.skippedOriginalTraceIds.length} trace(s) selected when experiment ${response.experimentId} started no longer exist, so resume skips them: ${response.skippedOriginalTraceIds.join(", ")}`
+      );
+    } catch {
+    }
+  }
+  if (requestedDbBranch !== void 0 && dbBranchEnabled(requestedDbBranch) !== response.includeDbBranchLease) {
+    try {
+      console.warn(
+        `Bitfab: experiment ${response.experimentId} started with database branching ${response.includeDbBranchLease ? "on" : "off"}, so resume keeps it ${response.includeDbBranchLease ? "on" : "off"} and ignores the dbBranch option passed here.`
+      );
+    } catch {
+    }
+  }
+  return {
+    experimentId: response.experimentId,
+    experimentUrl: response.experimentUrl,
+    attempts: response.attempts,
+    work,
+    finished: response.finished,
+    includeDbBranchLease: response.includeDbBranchLease,
+    dbBranchSettings: response.dbBranchSettings ?? void 0
+  };
+}
+function carriedOverReplayItem(finished) {
+  return {
+    traceId: finished.traceId,
+    originalTraceId: finished.originalTraceId,
+    originalSpanId: "",
+    sourceTraceId: finished.originalTraceId,
+    sourceSpanId: "",
+    attempt: finished.attempt,
+    input: [],
+    result: void 0,
+    originalOutput: void 0,
+    error: finished.hasError ? CARRIED_OVER_ERROR : null,
+    traceError: null,
+    replayError: null,
+    durationMs: null,
+    originalDurationMs: null,
+    originalTokens: null,
+    originalModel: null,
+    tokens: null,
+    model: null,
+    dbSnapshotRef: null,
+    dbBranchTimings: null,
+    traceOutline: null,
+    originalTraceOutline: null,
+    carriedOver: true
+  };
+}
 function resolveAttempts(attempts) {
   if (attempts === void 0) {
     return 1;
@@ -4047,6 +4169,13 @@ async function mapWithConcurrency2(tasks, maxConcurrency, onSettled, onStarted)
   return results;
 }
 async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, registeredOverrides = []) {
+  if (options?.resume !== void 0) {
+    validateResume(options.resume, options);
+  } else if (options?.force === true) {
+    throw new BitfabError(
+      "force only applies when resuming an experiment. Pass it together with resume."
+    );
+  }
   if (options?.traceIds !== void 0) {
     if (options.traceIds.length === 0) {
       throw new BitfabError("traceIds must contain at least one trace ID.");
@@ -4066,7 +4195,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     } catch {
     }
   }
-  const attempts = resolveAttempts(options?.attempts);
+  const requestedAttempts = resolveAttempts(options?.attempts);
   const maxConcurrency = options?.maxConcurrency ?? 10;
   if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
     throw new BitfabError(
@@ -4085,40 +4214,59 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
     }
   }
-  const {
-    experimentId,
-    experimentUrl,
-    items: serverItems
-  } = await httpClient.startReplay(
-    traceFunctionKey,
-    // limit is meaningless with explicit traceIds (the ID list determines
-    // the count), so it's omitted from the request entirely.
-    options?.traceIds ? void 0 : options?.limit ?? 5,
-    options?.traceIds,
-    options?.name,
-    codeChangeDescription,
-    codeChangeFiles,
-    // A dry run executes nothing, so a branch would be provisioned (and billed)
-    // for code that never runs, and a seeded source's refusal would fail the item
-    // before it could report its resolved inputs.
-    dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true,
-    options?.experimentGroupId,
-    datasetIds,
-    options?.graderIds,
-    resolveDbBranchSettings(options?.dbBranch),
-    attempts,
-    options?.adaptInputs !== void 0,
-    options?.onlyWithAssertions,
-    options?.judgeAssertions
-  );
-  if (serverItems.length === 0) {
-    try {
-      console.warn(
-        `Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
-      );
-    } catch {
+  const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
+  let session;
+  if (options?.resume === void 0) {
+    const started2 = await httpClient.startReplay(
+      traceFunctionKey,
+      // limit is meaningless with explicit traceIds (the ID list determines
+      // the count), so it's omitted from the request entirely.
+      options?.traceIds ? void 0 : options?.limit ?? 5,
+      options?.traceIds,
+      options?.name,
+      codeChangeDescription,
+      codeChangeFiles,
+      // A dry run executes nothing, so a branch would be provisioned (and billed)
+      // for code that never runs, and a seeded source's refusal would fail the item
+      // before it could report its resolved inputs.
+      includeDbBranchLease,
+      options?.experimentGroupId,
+      datasetIds,
+      options?.graderIds,
+      resolveDbBranchSettings(options?.dbBranch),
+      requestedAttempts,
+      options?.adaptInputs !== void 0,
+      options?.onlyWithAssertions,
+      options?.judgeAssertions
+    );
+    if (started2.items.length === 0) {
+      try {
+        console.warn(
+          `Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
+        );
+      } catch {
+      }
     }
+    session = {
+      experimentId: started2.experimentId,
+      experimentUrl: started2.experimentUrl,
+      attempts: requestedAttempts,
+      work: expandReplayWork(started2.items, requestedAttempts),
+      finished: [],
+      includeDbBranchLease,
+      dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
+    };
+  } else {
+    session = await resumeReplaySession(
+      httpClient,
+      options.resume,
+      traceFunctionKey,
+      codeChangeFiles,
+      options.force,
+      options.dbBranch
+    );
   }
+  const { experimentId, experimentUrl, attempts } = session;
   const mockStrategy = options?.mock ?? "marked";
   const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
   try {
@@ -4132,7 +4280,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     ...normalizeMockOverrides(options?.mockOverride),
     ...registeredOverrides
   ];
-  const workItems = expandReplayWork(serverItems, attempts);
+  const workItems = session.work;
+  const carriedOverItems = session.finished.map(
+    (finished) => carriedOverReplayItem(finished)
+  );
   const replayedTraceIds = workItems.map(() => randomUuid());
   httpClient.trackTraceDeliveries(replayedTraceIds);
   const tasks = workItems.map(
@@ -4145,18 +4296,18 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       resolvedOverrides,
       replayedTraceIds[index],
       attempt,
-      dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true,
-      resolveDbBranchSettings(options?.dbBranch),
+      session.includeDbBranchLease,
+      session.dbBranchSettings,
       options?.adaptInputs,
       options?.dryRun === true
     )
   );
-  const total = tasks.length;
+  const total = tasks.length + carriedOverItems.length;
   const onItemFinish = options?.onItemFinish ?? options?.onProgress;
-  let completed = 0;
-  let started = 0;
-  let succeeded = 0;
-  let errored = 0;
+  let completed = carriedOverItems.length;
+  let started = carriedOverItems.length;
+  let errored = carriedOverItems.filter((item) => item.error !== null).length;
+  let succeeded = completed - errored;
   let flushInFlight = null;
   let flushPending = false;
   const flushFinishedItemTrace = () => {
@@ -4176,7 +4327,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     }
     return flushInFlight;
   };
-  const resultItems = await mapWithConcurrency2(
+  const ranItems = await mapWithConcurrency2(
     tasks,
     maxConcurrency,
     async (item, index) => {
@@ -4263,6 +4414,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
     } : void 0
   );
+  const resultItems = [...carriedOverItems, ...ranItems];
   if (options?.dryRun === true) {
     await httpClient.completeReplay(experimentId).catch(() => void 0);
     const dryResult = {
@@ -4282,11 +4434,11 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     experimentId,
     fullExperimentUrl
   );
-  for (let index = 0; index < resultItems.length; index += 1) {
+  for (let index = 0; index < ranItems.length; index += 1) {
     const localId = replayedTraceIds[index];
     const readBack = localId ? deliveredTraceIds[localId] : void 0;
     if (readBack !== void 0) {
-      resultItems[index].traceId = readBack;
+      ranItems[index].traceId = readBack;
     }
   }
   const completeResult = await preserveReplayFailure(
@@ -4300,8 +4452,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   if (serverTraceIds !== void 0) {
     const missing = [];
     let completedCount = 0;
-    for (let index = 0; index < resultItems.length; index += 1) {
-      const item = resultItems[index];
+    for (let index = 0; index < ranItems.length; index += 1) {
+      const item = ranItems[index];
       const localId = replayedTraceIds[index];
       const mapped = localId ? serverTraceIds[localId] : void 0;
       item.traceId = item.traceId ?? mapped ?? null;
@@ -4387,7 +4539,7 @@ async function writeReplayResultFile(result) {
     }
   }
 }
-var REPLAY_PERSISTENCE_TIMEOUT_MS, MAX_REPLAY_ATTEMPTS, BITFAB_PROGRESS_PREFIX, ReplayError, DbBranchReplayError;
+var REPLAY_PERSISTENCE_TIMEOUT_MS, MAX_REPLAY_ATTEMPTS, RESUME_CONFLICTING_OPTIONS, CARRIED_OVER_ERROR, BITFAB_PROGRESS_PREFIX, ReplayError, DbBranchReplayError;
 var init_replay = __esm({
   "src/replay.ts"() {
     "use strict";
@@ -4401,6 +4553,17 @@ var init_replay = __esm({
     init_serialize();
     REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
     MAX_REPLAY_ATTEMPTS = 100;
+    RESUME_CONFLICTING_OPTIONS = [
+      "traceIds",
+      "limit",
+      "datasetId",
+      "datasetIds",
+      "attempts",
+      "graderIds",
+      "onlyWithAssertions",
+      "dryRun"
+    ];
+    CARRIED_OVER_ERROR = "This item errored in an earlier run of this experiment, before it was resumed.";
     BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
     ReplayError = class extends BitfabError {
       constructor(message, items, experimentId, experimentUrl, cause) {
@@ -5226,7 +5389,7 @@ function settleAutoTraceRoot(context) {
   }
   warnOnce(
     "auto-trace-transform-missing",
-    `trace root "${context.traceFunctionKey}" recorded only its own span. No module in this build was compiled by @bitfab/transform, so calls beneath the root were not discovered. Install @bitfab/transform and add its adapter to the server build: ${SUBTREE_TRACING_DOCS_URL}`
+    `trace root "${context.traceFunctionKey}" recorded only its own span. No module in this build was compiled by @bitfab/sdk/transform, so calls beneath the root were not discovered. Add the SDK's transform adapter to the server build: ${SUBTREE_TRACING_DOCS_URL}`
   );
 }
 function runWithAutoTraceRootContext(context, spanStack, fn) {