@noetaris/harness 0.12.0 → 0.13.0

package/dist/index.js

@@ -564,6 +564,9 @@ function composeObservers(observers, onObserverError) {
     onStepSettled: (ctx, event) => {
       for (const o of observers) safeInvoke("onStepSettled", () => o.onStepSettled?.(ctx, event), onObserverError);
     },
+    onCheckpoint: (ctx, event) => {
+      for (const o of observers) safeInvoke("onCheckpoint", () => o.onCheckpoint?.(ctx, event), onObserverError);
+    },
     // Each observer isolated on its own; later keys win.
     carry: (ctx) => {
       let merged;
@@ -766,8 +769,9 @@ async function runLoop(graph, state, ctx, schema, shouldStop, startCursor, callb
   ctx["within"] = createWithinFn(obs, () => stepCtxRef.current, onObserverError);
   const telemetryObserver = obs !== void 0 ? composeObservers([obs], onObserverError) : NOOP_OBSERVER;
   let cursor = startCursor ?? graph.entryStep;
+  let first = true;
   try {
-    while (true) {
+    for (; ; first = false) {
       if (shouldStop?.()) {
         endRun("$stopped");
         return withCarrier({ state, signal: null, cursor, paused: true });
@@ -790,6 +794,14 @@ async function runLoop(graph, state, ctx, schema, shouldStop, startCursor, callb
           return withCarrier({ state, signal: null, cursor, paused: true });
         }
       }
+      if (!first && callbacks?.checkpoint !== void 0 && typeof cursor === "string") {
+        const started = Date.now();
+        const saved = await callbacks.checkpoint(cursor, state, carrier);
+        if (saved !== void 0) {
+          const event = { step: cursor, version: saved.version, durationMs: Date.now() - started };
+          safeInvoke("onCheckpoint", () => obs?.onCheckpoint?.(runCtx, event), onObserverError);
+        }
+      }
       if (isForkCursor(cursor)) {
         const forkCursor = cursor;
         const forkNode = graph.steps.find((s) => s.name === forkCursor.fork);
@@ -976,7 +988,7 @@ async function dispatchFork(forkNode, priorBranches, outerState, outerCtx, schem
   const branchSignals = forkNode.branches.map(
     (_, i) => outerSignal !== void 0 ? AbortSignal.any([outerSignal, localAborts[i].signal]) : localAborts[i].signal
   );
-  const { onBeforeStep: _outerOnBeforeStep, onAfterStep: _outerOnAfterStep, onComplete: _outerOnComplete, ...branchCallbacksRest } = callbacks ?? {};
+  const { onBeforeStep: _outerOnBeforeStep, onAfterStep: _outerOnAfterStep, onComplete: _outerOnComplete, checkpoint: _outerCheckpoint, ...branchCallbacksRest } = callbacks ?? {};
   const branchCallbacksBase = callbacks !== void 0 ? branchCallbacksRest : void 0;
   const settled = await Promise.allSettled(
     forkNode.branches.map(async (branch, i) => {
@@ -1109,6 +1121,13 @@ function defaultClaimTtlMs() {
   const ttlMs = Number(process.env["HARNESS_CLAIM_TTL_MS"]);
   return isValidTtlMs(ttlMs) ? ttlMs : 3e4;
 }
+function isCheckpointMode(value) {
+  return value === "off" || value === "step";
+}
+async function saveRun(store, agentId, sessionId, run, lease) {
+  if (lease !== null && lease !== void 0) await store.save(agentId, sessionId, run, { lease });
+  else await store.save(agentId, sessionId, run);
+}
 function collectPendingInterrupts(cursor, state) {
   if (cursor !== void 0 && isForkCursor(cursor)) {
     const result = [];
@@ -1130,14 +1149,23 @@ function storedSessionToPhase(loaded) {
   if (loaded === null) {
     return { phase: "fresh" };
   }
+  const step = loaded.step === void 0 ? "" : isForkCursor(loaded.step) ? loaded.step.fork : loaded.step;
   if (loaded.phase === "paused") {
-    const step = loaded.step === void 0 ? "" : isForkCursor(loaded.step) ? loaded.step.fork : loaded.step;
     const pendingInterrupts = collectPendingInterrupts(loaded.step, loaded.finalState);
     if (loaded.signal !== void 0) {
       return { phase: "paused", step, signal: loaded.signal, pendingInterrupts };
     }
     return { phase: "paused", step, pendingInterrupts };
   }
+  if (loaded.phase === "running") {
+    const instanceId = loaded.metadata?.instanceId;
+    return {
+      phase: "running",
+      step,
+      ...instanceId !== void 0 ? { instanceId } : {},
+      checkpointedAt: loaded.settledAt
+    };
+  }
   if (loaded.signal !== void 0) {
     return { phase: "completed", signal: loaded.signal };
   }
@@ -1238,12 +1266,19 @@ var SessionBusyError = class extends Error {
 var LeaseExpiredError = class extends Error {
   /** The session whose lease expired. */
   sessionId;
-  constructor(sessionId) {
+  constructor(sessionId, cause) {
     super(`lease for session "${sessionId}" has expired \u2014 run stopped without saving`);
     this.name = "LeaseExpiredError";
     this.sessionId = sessionId;
+    if (cause !== void 0) this.cause = cause;
   }
 };
+function isLeaseLost(error) {
+  return error instanceof Error && error.name === "LeaseLostError";
+}
+function isConflict(error) {
+  return error instanceof Error && error.name === "ConcurrentModificationError";
+}
 
 // src/agent/ctx-keep-alive.ts
 function createKeepAliveFn(leaseRef, store, originalTtlMs) {
@@ -1353,7 +1388,7 @@ async function runWithSession(store, agentId, sessionId, runId, graph, initialSt
     }
     return false;
   };
-  const { mode = "run", requestId: callerRequestId, ...loopOptions } = options ?? {};
+  const { mode = "run", requestId: callerRequestId, checkpoint: checkpointMode = "off", ...loopOptions } = options ?? {};
   if (store === void 0) {
     const state2 = initializeState(null, initialStateArg, schema);
     const result2 = await runLoop(graph, state2, ctx, schema, composedShouldStop, void 0, {
@@ -1394,7 +1429,7 @@ async function runWithSession(store, agentId, sessionId, runId, graph, initialSt
       state: row.finalState,
       signal: row.signal ?? null,
       cursor: row.step ?? null,
-      paused: row.phase === "paused"
+      paused: row.phase !== "completed"
     };
   }
   const segmentRequestId = mode === "run" ? requestId : loaded?.requestId;
@@ -1403,8 +1438,47 @@ async function runWithSession(store, agentId, sessionId, runId, graph, initialSt
   const state = initializeState(base, input, schema);
   const initialStateSnapshot = { ...state };
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const metadata = options?.instanceId !== void 0 ? { instanceId: options.instanceId } : void 0;
+  let nextVersion = (loaded?.version ?? -1) + 1;
+  const baseRow = (carrier) => ({
+    agentId,
+    runId,
+    sessionId,
+    version: nextVersion,
+    startedAt,
+    settledAt: (/* @__PURE__ */ new Date()).toISOString(),
+    initialState: initialStateSnapshot,
+    ...metadata !== void 0 ? { metadata } : {},
+    ...carrier !== void 0 ? { carrier } : {},
+    ...segmentRequestId !== void 0 ? { requestId: segmentRequestId } : {}
+  });
+  const checkpoint = async (next, live, carrier) => {
+    let finalState;
+    try {
+      finalState = structuredClone(live);
+    } catch (error) {
+      options?.onStoreError?.(error, "checkpoint");
+      return void 0;
+    }
+    const row = { ...baseRow(carrier), phase: "running", step: next, finalState };
+    try {
+      await saveRun(store, agentId, sessionId, row, leaseRef.current);
+    } catch (error) {
+      if (isLeaseLost(error) || isConflict(error)) {
+        const expiredError = new LeaseExpiredError(sessionId, error);
+        options?.onStoreError?.(expiredError, "claim");
+        throw expiredError;
+      }
+      options?.onStoreError?.(error, "checkpoint");
+      return void 0;
+    }
+    nextVersion++;
+    await extendLease(store, leaseRef, options?.claimTtlMs ?? defaultClaimTtlMs());
+    return { version: row.version };
+  };
   const result = await runLoop(graph, state, ctx, schema, composedShouldStop, loaded?.step, {
     ...loopOptions,
+    ...checkpointMode === "step" ? { checkpoint } : {},
     runId,
     ...options?.parentRunId !== void 0 ? { parentRunId: options.parentRunId } : {},
     ...segmentRequestId !== void 0 ? { requestId: segmentRequestId } : {},
@@ -1415,53 +1489,39 @@ async function runWithSession(store, agentId, sessionId, runId, graph, initialSt
     options?.onStoreError?.(expiredError, "claim");
     throw expiredError;
   }
-  const settledAt = (/* @__PURE__ */ new Date()).toISOString();
-  const metadata = options?.instanceId !== void 0 ? { instanceId: options.instanceId } : void 0;
+  const saved = {
+    ...baseRow(result.carrier),
+    phase: result.paused ? "paused" : "completed",
+    finalState: result.state,
+    ...result.paused ? { step: result.cursor } : {},
+    ...result.signal !== null ? { signal: result.signal } : {}
+  };
   try {
-    const version = (loaded?.version ?? -1) + 1;
-    if (result.paused) {
-      const saved = {
-        agentId,
-        runId,
-        sessionId,
-        version,
-        startedAt,
-        settledAt,
-        phase: "paused",
-        initialState: initialStateSnapshot,
-        finalState: result.state,
-        step: result.cursor,
-        ...result.signal !== null ? { signal: result.signal } : {},
-        ...metadata !== void 0 ? { metadata } : {},
-        ...result.carrier !== void 0 ? { carrier: result.carrier } : {},
-        ...segmentRequestId !== void 0 ? { requestId: segmentRequestId } : {}
-      };
-      await store.save(agentId, sessionId, saved);
-    } else {
-      const saved = {
-        agentId,
-        runId,
-        sessionId,
-        version,
-        startedAt,
-        settledAt,
-        phase: "completed",
-        initialState: initialStateSnapshot,
-        finalState: result.state,
-        ...result.signal !== null ? { signal: result.signal } : {},
-        ...metadata !== void 0 ? { metadata } : {},
-        ...result.carrier !== void 0 ? { carrier: result.carrier } : {},
-        ...segmentRequestId !== void 0 ? { requestId: segmentRequestId } : {}
-      };
-      await store.save(agentId, sessionId, saved);
-    }
+    await saveRun(store, agentId, sessionId, saved, leaseRef.current);
   } catch (error) {
+    if (isLeaseLost(error)) {
+      const expiredError = new LeaseExpiredError(sessionId, error);
+      options?.onStoreError?.(expiredError, "claim");
+      throw expiredError;
+    }
     options?.onStoreError?.(error, "persist");
   }
   return result;
 }
+async function extendLease(store, leaseRef, ttlMs) {
+  const held = leaseRef.current;
+  if (held === null || typeof store.extendClaim !== "function") return;
+  try {
+    leaseRef.current = await store.extendClaim(held, { ttlMs });
+  } catch {
+  }
+}
 
 // src/agent/claim-scope.ts
+function heldLease(options) {
+  const lease = options.leaseRef?.current ?? null;
+  return lease !== null ? { lease } : {};
+}
 function failOutcome(failState, error) {
   const state = failState();
   state["$error"] = error;
@@ -1555,7 +1615,7 @@ function injectInterruptResponseIntoState(cursor, state, interruptId, response)
     found: true
   };
 }
-async function injectInterruptResponse(store, agentId, sessionId, interruptId, response) {
+async function injectInterruptResponse(store, agentId, sessionId, interruptId, response, lease) {
   const loaded = await store.load(agentId, sessionId);
   if (loaded === null) throw new NoInterruptError();
   if (loaded.phase !== "paused") throw new NoInterruptError();
@@ -1578,14 +1638,19 @@ async function injectInterruptResponse(store, agentId, sessionId, interruptId, r
     ...loaded.branchedFrom !== void 0 ? { branchedFrom: loaded.branchedFrom } : {},
     ...loaded.requestId !== void 0 ? { requestId: loaded.requestId } : {}
   };
-  await store.save(agentId, sessionId, updated);
+  await saveRun(store, agentId, sessionId, updated, lease);
 }
 async function injectThenRun(args) {
-  const { store, agentId, sessionId, interruptId, response, onStoreError, failState, run } = args;
+  const { store, agentId, sessionId, interruptId, response, lease, onStoreError, failState, run } = args;
   try {
-    await injectInterruptResponse(store, agentId, sessionId, interruptId, response);
+    await injectInterruptResponse(store, agentId, sessionId, interruptId, response, lease);
   } catch (injectError) {
     if (injectError instanceof NoInterruptError) throw injectError;
+    if (isLeaseLost(injectError)) {
+      const expiredError = new LeaseExpiredError(sessionId, injectError);
+      onStoreError?.(expiredError, "claim");
+      return failOutcome(failState, expiredError);
+    }
     const storeError = new StoreLoadError(injectError);
     onStoreError?.(storeError, "claim");
     return failOutcome(failState, storeError);
@@ -2329,13 +2394,14 @@ var InvalidClaimOptionsError = class extends Error {
     this.ttlMs = ttlMs;
   }
 };
+var AGENT_OPTION_EXPECTED = { observer: "an object", onObserverError: "a function", checkpoint: "'off' or 'step'" };
 var InvalidAgentOptionError = class extends Error {
   /** The option that was rejected. */
   key;
   /** The rejected value, as given. */
   value;
   constructor(key, value) {
-    super(`${key} must be ${key === "observer" ? "an object" : "a function"} \u2014 got ${describeValue(value)} in createAgent options`);
+    super(`${key} must be ${AGENT_OPTION_EXPECTED[key]} \u2014 got ${describeValue(value)} in createAgent options`);
     this.name = "InvalidAgentOptionError";
     this.key = key;
     this.value = value;
@@ -2347,7 +2413,7 @@ var InvalidRunResourceError = class extends Error {
   /** The rejected value, as given. */
   value;
   constructor(key, value) {
-    super(`${key} must be a non-empty string \u2014 got ${describeValue(value)} in agent.run() resources`);
+    super(key === "checkpoint" ? `checkpoint must be 'off' or 'step' \u2014 got ${describeValue(value)} in the call's resources` : `${key} must be a non-empty string \u2014 got ${describeValue(value)} in agent.run() resources`);
     this.name = "InvalidRunResourceError";
     this.key = key;
     this.value = value;
@@ -2359,6 +2425,12 @@ function readIdResource(resources, key) {
   if (typeof value !== "string" || value.length === 0) throw new InvalidRunResourceError(key, value);
   return value;
 }
+function readCheckpointResource(resources) {
+  const value = resources["checkpoint"];
+  if (value === void 0) return void 0;
+  if (!isCheckpointMode(value)) throw new InvalidRunResourceError("checkpoint", value);
+  return value;
+}
 var _agentInternals = /* @__PURE__ */ Symbol("_agentInternals");
 function createAgent(id, h, slots, agentOptions) {
   const internals = getInternals(h);
@@ -2413,6 +2485,10 @@ function createAgent(id, h, slots, agentOptions) {
       ...onObserverError !== void 0 ? { onObserverError } : {}
     };
   };
+  const agentCheckpoint = agentOptions?.checkpoint;
+  if (agentCheckpoint !== void 0 && !isCheckpointMode(agentCheckpoint)) {
+    throw new InvalidAgentOptionError("checkpoint", agentCheckpoint);
+  }
   const resolvedProviders = /* @__PURE__ */ new Map();
   const storeEntries = [];
   const runtimeKeys = /* @__PURE__ */ new Set();
@@ -2462,13 +2538,13 @@ function createAgent(id, h, slots, agentOptions) {
     runtimeKeys,
     requiredKeys
   };
-  const reservedRunKeys = /* @__PURE__ */ new Set(["sessionId", "requestId", "signal", "events", "listeners", "observer", "onObserverError", "claimOptions", "parentRunId"]);
+  const reservedRunKeys = /* @__PURE__ */ new Set(["sessionId", "requestId", "signal", "events", "listeners", "observer", "onObserverError", "claimOptions", "parentRunId", "checkpoint"]);
   const inFlightSessions = /* @__PURE__ */ new Set();
   const debugSessions = /* @__PURE__ */ new Set();
   const interruptPendingSessions = /* @__PURE__ */ new Set();
   const resumeFailState = () => initializeState(null, {}, agentInternals.stateSchema);
   const makeAgentResumeHandle = (resp, sId, iId, resumeResources) => {
-    const { onStoreError, observer, onObserverError, telemetrySlots, parentRunId, claimOptions } = resumeResources;
+    const { onStoreError, observer, onObserverError, telemetrySlots, parentRunId, claimOptions, checkpoint } = resumeResources;
     let _stopped = false;
     const abortController = new AbortController();
     const flag = {
@@ -2501,6 +2577,7 @@ function createAgent(id, h, slots, agentOptions) {
               sessionId: sId,
               interruptId: iId,
               response: resp,
+              ...heldLease(lease),
               ...onStoreError !== void 0 ? { onStoreError } : {},
               failState: resumeFailState,
               run: async () => {
@@ -2523,6 +2600,7 @@ function createAgent(id, h, slots, agentOptions) {
                   agentCtx,
                   {
                     mode: "resume",
+                    checkpoint,
                     shouldStop: () => flag.stopped,
                     onBeforeStep: (n) => {
                       ref.current = n;
@@ -2574,6 +2652,7 @@ function createAgent(id, h, slots, agentOptions) {
     }
     const sessionId = readIdResource(resources, "sessionId") ?? randomUUID();
     const requestId = readIdResource(resources, "requestId");
+    const checkpoint = readCheckpointResource(resources) ?? agentCheckpoint ?? "off";
     const runId = randomUUID();
     if (debugSessions.has(sessionId)) throw new SessionInDebugError(sessionId);
     if (inFlightSessions.has(sessionId)) throw new SessionInFlightError(sessionId);
@@ -2628,6 +2707,7 @@ function createAgent(id, h, slots, agentOptions) {
     const claimOptions = parseClaimOptions(resources["claimOptions"], fallbackClaimOptions());
     const options = {
       mode: "run",
+      checkpoint,
       ...requestId !== void 0 ? { requestId } : {},
       shouldStop: () => stopFlag.stopped,
       onBeforeStep: (name, state) => {
@@ -2723,6 +2803,7 @@ function createAgent(id, h, slots, agentOptions) {
                   sessionId,
                   interruptId,
                   response,
+                  ...heldLease(lease),
                   ...onStoreError !== void 0 ? { onStoreError } : {},
                   failState: resumeFailState,
                   run: async () => {
@@ -2738,6 +2819,7 @@ function createAgent(id, h, slots, agentOptions) {
                       resumeCtx,
                       {
                         mode: "resume",
+                        checkpoint,
                         shouldStop: () => resumeStopFlag.stopped,
                         onBeforeStep: (n, s) => {
                           resumeStepRef.current = n;
@@ -2813,6 +2895,7 @@ function createAgent(id, h, slots, agentOptions) {
     resume: (response, sessionId, interruptId, resources) => {
       if (debugSessions.has(sessionId)) throw new SessionInDebugError(sessionId);
       if (inFlightSessions.has(sessionId)) throw new SessionInFlightError(sessionId);
+      const checkpoint = readCheckpointResource(resources ?? {}) ?? agentCheckpoint ?? "off";
       interruptPendingSessions.delete(sessionId);
       inFlightSessions.add(sessionId);
       const events = extractRunEvents(resources ?? {});
@@ -2822,6 +2905,7 @@ function createAgent(id, h, slots, agentOptions) {
       const claimOptions = parseClaimOptions(resources?.["claimOptions"], fallbackClaimOptions());
       return makeAgentResumeHandle(response, sessionId, interruptId, {
         claimOptions,
+        checkpoint,
         ...events.onStoreError !== void 0 ? { onStoreError: events.onStoreError } : {},
         ...observer !== void 0 ? { observer } : {},
         ...onObserverError !== void 0 ? { onObserverError } : {},