bitfab 0.61.3 → 0.61.4

package/dist/{chunk-IZUZ5NBB.js → chunk-PSAEQ4OW.js}

@@ -4,8 +4,9 @@ import {
   flushTraces,
   gitRunner,
   replayContextReady,
-  runWithReplayContext
-} from "./chunk-6IMUONDN.js";
+  runWithReplayContext,
+  unrefTimer
+} from "./chunk-54EGXUZ3.js";
 import {
   runWithCaptureSuppressed,
   warnOnce
@@ -437,6 +438,101 @@ function fallbackUuidV4() {
   });
 }
 
+// src/replayHeartbeat.ts
+var REPLAY_HEARTBEAT_INTERVAL_MS = 6e4;
+function startReplayHeartbeat(send) {
+  let stopped = false;
+  let inFlight;
+  const timer = setInterval(() => {
+    if (stopped || inFlight !== void 0) {
+      return;
+    }
+    inFlight = send().catch(() => void 0).finally(() => {
+      inFlight = void 0;
+    });
+  }, REPLAY_HEARTBEAT_INTERVAL_MS);
+  unrefTimer(timer);
+  return {
+    stop: async () => {
+      stopped = true;
+      clearInterval(timer);
+      await inFlight;
+    }
+  };
+}
+
+// src/replayInterrupt.ts
+var REPLAY_INTERRUPT_SIGNALS = [
+  "SIGINT",
+  "SIGTERM"
+];
+function signalCapableProcess() {
+  if (typeof process === "undefined" || typeof process.on !== "function") {
+    return void 0;
+  }
+  return process;
+}
+function describeReplayInterrupt(experimentId, markedInterrupted) {
+  if (markedInterrupted) {
+    return `[replay] Experiment ${experimentId} interrupted.`;
+  }
+  return `[replay] Experiment ${experimentId} stopped, but Bitfab could not mark it interrupted. It will be marked interrupted within 15 minutes. A resume in the next two minutes may be refused as still running.`;
+}
+function reportInterrupted(experimentId, markedInterrupted) {
+  try {
+    console.error(describeReplayInterrupt(experimentId, markedInterrupted));
+  } catch {
+  }
+}
+function installReplayInterruptHandlers(target) {
+  const host = signalCapableProcess();
+  if (host === void 0) {
+    return { remove: () => {
+    }, interrupted: () => false };
+  }
+  const handlers = /* @__PURE__ */ new Map();
+  const removeHandlers = () => {
+    for (const [signal, handler] of handlers) {
+      host.removeListener(signal, handler);
+    }
+    handlers.clear();
+  };
+  let interrupting = false;
+  const handleSignal = async (signal) => {
+    const hostHandlesSignal = host.listenerCount(signal) > 1;
+    const markedInterrupted = await target.interrupt();
+    removeHandlers();
+    if (target.onInterrupt !== void 0) {
+      try {
+        target.onInterrupt({
+          experimentId: target.experimentId,
+          experimentUrl: target.experimentUrl,
+          signal,
+          markedInterrupted
+        });
+      } catch {
+      }
+      return;
+    }
+    reportInterrupted(target.experimentId, markedInterrupted);
+    if (!hostHandlesSignal) {
+      host.kill(host.pid, signal);
+    }
+  };
+  for (const signal of REPLAY_INTERRUPT_SIGNALS) {
+    const handler = () => {
+      if (interrupting) {
+        return;
+      }
+      interrupting = true;
+      void handleSignal(signal);
+    };
+    handlers.set(signal, handler);
+    host.on(signal, handler);
+  }
+  return { remove: removeHandlers, interrupted: () => interrupting };
+}
+
 // src/selectiveReplay.ts
 function replayNodeKey(node) {
   return JSON.stringify([node.traceFunctionKey, node.spanName]);
@@ -1357,7 +1453,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
   let session;
   if (options?.resume === void 0) {
-    const started2 = await httpClient.startReplay(
+    const started = await httpClient.startReplay(
       traceFunctionKey,
       // limit is meaningless with explicit traceIds (the ID list determines
       // the count), so it's omitted from the request entirely.
@@ -1380,7 +1476,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       options?.judgeAssertions,
       options?.experimentalSelectiveReplay
     );
-    if (started2.items.length === 0) {
+    if (started.items.length === 0) {
       try {
         console.warn(
           `Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
@@ -1389,10 +1485,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
     }
     session = {
-      experimentId: started2.experimentId,
-      experimentUrl: started2.experimentUrl,
+      experimentId: started.experimentId,
+      experimentUrl: started.experimentUrl,
       attempts: requestedAttempts,
-      work: expandReplayWork(started2.items, requestedAttempts),
+      work: expandReplayWork(started.items, requestedAttempts),
       finished: [],
       includeDbBranchLease,
       dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
@@ -1407,9 +1503,68 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       options.dbBranch
     );
   }
-  const { experimentId, experimentUrl, attempts } = session;
+  const { experimentId } = session;
+  const fullExperimentUrl = `${serviceUrl}${session.experimentUrl}`;
+  const heartbeat = startReplayHeartbeat(
+    () => httpClient.heartbeatReplay(experimentId)
+  );
+  const interruptReplay = async () => {
+    await heartbeat.stop();
+    return httpClient.interruptReplay(experimentId);
+  };
+  const interruptHandlers = installReplayInterruptHandlers({
+    experimentId,
+    experimentUrl: fullExperimentUrl,
+    interrupt: interruptReplay,
+    onInterrupt: options?.onInterrupt
+  });
+  let completeReturned = false;
+  const markCompleteReturned = () => {
+    completeReturned = true;
+    void heartbeat.stop();
+  };
+  try {
+    return await runReplaySession(
+      httpClient,
+      fn,
+      session,
+      fullExperimentUrl,
+      maxConcurrency,
+      interruptHandlers.remove,
+      markCompleteReturned,
+      interruptHandlers.interrupted,
+      options,
+      registeredOverrides
+    );
+  } catch (error) {
+    if (!completeReturned && !interruptHandlers.interrupted()) {
+      await interruptReplay();
+    }
+    throw error;
+  } finally {
+    interruptHandlers.remove();
+    await heartbeat.stop();
+  }
+}
+function throwIfInterrupted(interrupted, items, experimentId, experimentUrl) {
+  if (!interrupted()) {
+    return;
+  }
+  const cause = new BitfabError(
+    `Experiment ${experimentId} was interrupted, so it was not completed or graded. Resume it by passing { resume: "${experimentId}" } to replay, or with bitfab-replay --resume ${experimentId}.`,
+    experimentUrl
+  );
+  throw new ReplayError(
+    cause.message,
+    items,
+    experimentId,
+    experimentUrl,
+    cause
+  );
+}
+async function runReplaySession(httpClient, fn, session, fullExperimentUrl, maxConcurrency, stopInterruptingOnSignal, markCompleteReturned, interrupted, options, registeredOverrides) {
+  const { experimentId, attempts } = session;
   const mockStrategy = options?.mock ?? "marked";
-  const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
   try {
     options?.onExperimentStart?.({
       experimentId,
@@ -1559,7 +1714,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   );
   const resultItems = [...carriedOverItems, ...ranItems];
   if (options?.dryRun === true) {
+    throwIfInterrupted(
+      interrupted,
+      resultItems,
+      experimentId,
+      fullExperimentUrl
+    );
+    stopInterruptingOnSignal();
     await httpClient.completeReplay(experimentId).catch(() => void 0);
+    markCompleteReturned();
     const dryResult = {
       items: resultItems,
       experimentId,
@@ -1584,12 +1747,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       ranItems[index].traceId = readBack;
     }
   }
+  throwIfInterrupted(interrupted, resultItems, experimentId, fullExperimentUrl);
+  stopInterruptingOnSignal();
   const completeResult = await preserveReplayFailure(
     () => httpClient.completeReplay(experimentId),
     resultItems,
     experimentId,
     fullExperimentUrl
   );
+  markCompleteReturned();
   const serverTraceIds = completeResult.traceIds;
   const replayTokens = completeResult.tokens;
   if (serverTraceIds !== void 0) {
@@ -1702,4 +1868,4 @@ export {
   sleepForReplayPersistence,
   replay
 };
-//# sourceMappingURL=chunk-IZUZ5NBB.js.map
+//# sourceMappingURL=chunk-PSAEQ4OW.js.map