@bitfab/sdk 0.61.3 → 0.61.4

package/dist/index.cjs

@@ -42,7 +42,7 @@ var __version__, __packageName__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.61.3";
+    __version__ = "0.61.4";
     __packageName__ = "@bitfab/sdk";
   }
 });
@@ -2277,7 +2277,7 @@ function warnDroppedExternalTrace(error) {
 function experimentIdWireFields(experimentId) {
   return { experimentId, testRunId: experimentId };
 }
-var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, REPLAY_COMPLETE_REQUEST_TIMEOUT_MS, OTLP_TRACES_ENDPOINT, RETRYABLE_STATUSES, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, liveHttpClients, carrierSeq, HttpClient;
+var REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS, REPLAY_COMPLETE_REQUEST_TIMEOUT_MS, REPLAY_INTERRUPT_REQUEST_TIMEOUT_MS, REPLAY_HEARTBEAT_REQUEST_TIMEOUT_MS, OTLP_TRACES_ENDPOINT, RETRYABLE_STATUSES, EXIT_FLUSH_TIMEOUT_MS, DEFAULT_LIFECYCLE_TIMEOUT_MS2, pendingTracePromises, liveHttpClients, carrierSeq, HttpClient;
 var init_http = __esm({
   "src/http.ts"() {
     "use strict";
@@ -2296,6 +2296,8 @@ var init_http = __esm({
     init_warnOnce();
     REPLAY_DB_BRANCH_REQUEST_TIMEOUT_MS = 3e5;
     REPLAY_COMPLETE_REQUEST_TIMEOUT_MS = 12e4;
+    REPLAY_INTERRUPT_REQUEST_TIMEOUT_MS = 5e3;
+    REPLAY_HEARTBEAT_REQUEST_TIMEOUT_MS = 5e3;
     OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces";
     RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
     EXIT_FLUSH_TIMEOUT_MS = 5e3;
@@ -3108,6 +3110,28 @@ var init_http = __esm({
           { timeout: REPLAY_COMPLETE_REQUEST_TIMEOUT_MS }
         );
       }
+      async interruptReplay(experimentId) {
+        try {
+          await this.request(
+            "/api/sdk/replay/interrupt",
+            experimentIdWireFields(experimentId),
+            { timeout: REPLAY_INTERRUPT_REQUEST_TIMEOUT_MS }
+          );
+          return true;
+        } catch {
+          return false;
+        }
+      }
+      async heartbeatReplay(experimentId) {
+        try {
+          await this.request(
+            "/api/sdk/replay/heartbeat",
+            experimentIdWireFields(experimentId),
+            { timeout: REPLAY_HEARTBEAT_REQUEST_TIMEOUT_MS }
+          );
+        } catch {
+        }
+      }
       async getReseedSource(traceId) {
         return this.get(
           `/api/sdk/traces/${encodeURIComponent(traceId)}/reseedSource`
@@ -3611,6 +3635,114 @@ var init_integrationFixture = __esm({
   }
 });
 
+// src/replayHeartbeat.ts
+function startReplayHeartbeat(send) {
+  let stopped = false;
+  let inFlight;
+  const timer = setInterval(() => {
+    if (stopped || inFlight !== void 0) {
+      return;
+    }
+    inFlight = send().catch(() => void 0).finally(() => {
+      inFlight = void 0;
+    });
+  }, REPLAY_HEARTBEAT_INTERVAL_MS);
+  unrefTimer(timer);
+  return {
+    stop: async () => {
+      stopped = true;
+      clearInterval(timer);
+      await inFlight;
+    }
+  };
+}
+var REPLAY_HEARTBEAT_INTERVAL_MS;
+var init_replayHeartbeat = __esm({
+  "src/replayHeartbeat.ts"() {
+    "use strict";
+    init_unrefTimer();
+    REPLAY_HEARTBEAT_INTERVAL_MS = 6e4;
+  }
+});
+
+// src/replayInterrupt.ts
+function signalCapableProcess() {
+  if (typeof process === "undefined" || typeof process.on !== "function") {
+    return void 0;
+  }
+  return process;
+}
+function describeReplayInterrupt(experimentId, markedInterrupted) {
+  if (markedInterrupted) {
+    return `[replay] Experiment ${experimentId} interrupted.`;
+  }
+  return `[replay] Experiment ${experimentId} stopped, but Bitfab could not mark it interrupted. It will be marked interrupted within 15 minutes. A resume in the next two minutes may be refused as still running.`;
+}
+function reportInterrupted(experimentId, markedInterrupted) {
+  try {
+    console.error(describeReplayInterrupt(experimentId, markedInterrupted));
+  } catch {
+  }
+}
+function installReplayInterruptHandlers(target) {
+  const host = signalCapableProcess();
+  if (host === void 0) {
+    return { remove: () => {
+    }, interrupted: () => false };
+  }
+  const handlers = /* @__PURE__ */ new Map();
+  const removeHandlers = () => {
+    for (const [signal, handler] of handlers) {
+      host.removeListener(signal, handler);
+    }
+    handlers.clear();
+  };
+  let interrupting = false;
+  const handleSignal = async (signal) => {
+    const hostHandlesSignal = host.listenerCount(signal) > 1;
+    const markedInterrupted = await target.interrupt();
+    removeHandlers();
+    if (target.onInterrupt !== void 0) {
+      try {
+        target.onInterrupt({
+          experimentId: target.experimentId,
+          experimentUrl: target.experimentUrl,
+          signal,
+          markedInterrupted
+        });
+      } catch {
+      }
+      return;
+    }
+    reportInterrupted(target.experimentId, markedInterrupted);
+    if (!hostHandlesSignal) {
+      host.kill(host.pid, signal);
+    }
+  };
+  for (const signal of REPLAY_INTERRUPT_SIGNALS) {
+    const handler = () => {
+      if (interrupting) {
+        return;
+      }
+      interrupting = true;
+      void handleSignal(signal);
+    };
+    handlers.set(signal, handler);
+    host.on(signal, handler);
+  }
+  return { remove: removeHandlers, interrupted: () => interrupting };
+}
+var REPLAY_INTERRUPT_SIGNALS;
+var init_replayInterrupt = __esm({
+  "src/replayInterrupt.ts"() {
+    "use strict";
+    REPLAY_INTERRUPT_SIGNALS = [
+      "SIGINT",
+      "SIGTERM"
+    ];
+  }
+});
+
 // src/selectiveReplay.ts
 function replayNodeKey(node) {
   return JSON.stringify([node.traceFunctionKey, node.spanName]);
@@ -4520,7 +4652,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
   let session;
   if (options?.resume === void 0) {
-    const started2 = await httpClient.startReplay(
+    const started = await httpClient.startReplay(
       traceFunctionKey,
       // limit is meaningless with explicit traceIds (the ID list determines
       // the count), so it's omitted from the request entirely.
@@ -4543,7 +4675,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       options?.judgeAssertions,
       options?.experimentalSelectiveReplay
     );
-    if (started2.items.length === 0) {
+    if (started.items.length === 0) {
       try {
         console.warn(
           `Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
@@ -4552,10 +4684,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
     }
     session = {
-      experimentId: started2.experimentId,
-      experimentUrl: started2.experimentUrl,
+      experimentId: started.experimentId,
+      experimentUrl: started.experimentUrl,
       attempts: requestedAttempts,
-      work: expandReplayWork(started2.items, requestedAttempts),
+      work: expandReplayWork(started.items, requestedAttempts),
       finished: [],
       includeDbBranchLease,
       dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
@@ -4570,9 +4702,68 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       options.dbBranch
     );
   }
-  const { experimentId, experimentUrl, attempts } = session;
+  const { experimentId } = session;
+  const fullExperimentUrl = `${serviceUrl}${session.experimentUrl}`;
+  const heartbeat = startReplayHeartbeat(
+    () => httpClient.heartbeatReplay(experimentId)
+  );
+  const interruptReplay = async () => {
+    await heartbeat.stop();
+    return httpClient.interruptReplay(experimentId);
+  };
+  const interruptHandlers = installReplayInterruptHandlers({
+    experimentId,
+    experimentUrl: fullExperimentUrl,
+    interrupt: interruptReplay,
+    onInterrupt: options?.onInterrupt
+  });
+  let completeReturned = false;
+  const markCompleteReturned = () => {
+    completeReturned = true;
+    void heartbeat.stop();
+  };
+  try {
+    return await runReplaySession(
+      httpClient,
+      fn,
+      session,
+      fullExperimentUrl,
+      maxConcurrency,
+      interruptHandlers.remove,
+      markCompleteReturned,
+      interruptHandlers.interrupted,
+      options,
+      registeredOverrides
+    );
+  } catch (error) {
+    if (!completeReturned && !interruptHandlers.interrupted()) {
+      await interruptReplay();
+    }
+    throw error;
+  } finally {
+    interruptHandlers.remove();
+    await heartbeat.stop();
+  }
+}
+function throwIfInterrupted(interrupted, items, experimentId, experimentUrl) {
+  if (!interrupted()) {
+    return;
+  }
+  const cause = new BitfabError(
+    `Experiment ${experimentId} was interrupted, so it was not completed or graded. Resume it by passing { resume: "${experimentId}" } to replay, or with bitfab-replay --resume ${experimentId}.`,
+    experimentUrl
+  );
+  throw new ReplayError(
+    cause.message,
+    items,
+    experimentId,
+    experimentUrl,
+    cause
+  );
+}
+async function runReplaySession(httpClient, fn, session, fullExperimentUrl, maxConcurrency, stopInterruptingOnSignal, markCompleteReturned, interrupted, options, registeredOverrides) {
+  const { experimentId, attempts } = session;
   const mockStrategy = options?.mock ?? "marked";
-  const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
   try {
     options?.onExperimentStart?.({
       experimentId,
@@ -4722,7 +4913,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
   );
   const resultItems = [...carriedOverItems, ...ranItems];
   if (options?.dryRun === true) {
+    throwIfInterrupted(
+      interrupted,
+      resultItems,
+      experimentId,
+      fullExperimentUrl
+    );
+    stopInterruptingOnSignal();
     await httpClient.completeReplay(experimentId).catch(() => void 0);
+    markCompleteReturned();
     const dryResult = {
       items: resultItems,
       experimentId,
@@ -4747,12 +4946,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       ranItems[index].traceId = readBack;
     }
   }
+  throwIfInterrupted(interrupted, resultItems, experimentId, fullExperimentUrl);
+  stopInterruptingOnSignal();
   const completeResult = await preserveReplayFailure(
     () => httpClient.completeReplay(experimentId),
     resultItems,
     experimentId,
     fullExperimentUrl
   );
+  markCompleteReturned();
   const serverTraceIds = completeResult.traceIds;
   const replayTokens = completeResult.tokens;
   if (serverTraceIds !== void 0) {
@@ -4856,6 +5058,8 @@ var init_replay = __esm({
     init_mockOverride();
     init_randomUuid();
     init_replayContext();
+    init_replayHeartbeat();
+    init_replayInterrupt();
     init_selectiveReplayRuntime();
     init_serialize();
     REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
@@ -11632,6 +11836,7 @@ init_replay();
 // src/replayRegistry.ts
 init_errors();
 init_replay();
+init_replayInterrupt();
 function defineReplayRegistry(registry) {
   return registry;
 }