@sema-agent/core 2.10.0 → 2.12.0

package/dist/core/runner/prepare-task.js

@@ -49,7 +49,8 @@ import { capAggregateToolResults } from "../tool-result-budget.js";
 import { capAggregateMediaBytes, AGGREGATE_MEDIA_BUDGET_BYTES } from "../media-byte-cap.js";
 import { dropOrphanToolResults, guardBudget, insertTrimNotice, trimToBudget } from "../context-guard.js";
 import { StubExecutionEnv } from "../stub-env.js";
-import { hasDestroy, isIsolated, isRemoteExecutionEnv, isSuspendable } from "../remote-env.js";
+import { hasDestroy, isIsolated, isRemoteExecutionEnv, isRetryableRemoteErrorCode, isSuspendable, missingRestoreSurface, RETRYABLE_REMOTE_ERROR_CODES } from "../remote-env.js";
+import { withRetry } from "../with-retry.js";
 import { hasBackgroundShell, sweepBackgroundShells } from "../background-shell.js";
 import { createTaskOutputTool, createTaskStopTool, defaultTaskRegistry } from "../task-registry.js";
 import { createMonitorTool } from "../../tools/monitor.js";
@@ -173,6 +174,27 @@ export function rebaseWorkspacePath(p, fromRaw, toRaw) {
     const suffix = p.slice(fromPrefix.length).replace(/^\/+/, "");
     return to === "/" ? `${to}${suffix}` : `${to}/${suffix}`;
 }
+function remoteEnvFailureNote(op, error, attempts) {
+    return { op, code: error.code, retryable: isRetryableRemoteErrorCode(error.code), attempts, message: error.message };
+}
+const REMOTE_RESTORE_MAX_ATTEMPTS = 2;
+const REMOTE_RESTORE_BACKOFF_MS = 200;
+async function restoreWorkspaceWithRetry(env, snapshotId, options) {
+    let attempts = 0;
+    const outcome = await withRetry(async (attempt) => {
+        attempts = attempt;
+        return env.resumeVM(snapshotId, options);
+    }, { retryableCodes: RETRYABLE_REMOTE_ERROR_CODES, maxAttempts: REMOTE_RESTORE_MAX_ATTEMPTS, backoffMs: () => REMOTE_RESTORE_BACKOFF_MS }, { ...(options.abortSignal !== undefined ? { signal: options.abortSignal } : {}) });
+    return { outcome, attempts };
+}
+export function rebaseWorkspacePathAcross(p, froms, to) {
+    for (const from of froms) {
+        const out = rebaseWorkspacePath(p, from, to);
+        if (out !== p)
+            return out;
+    }
+    return p;
+}
 export async function prepareTask(spec, deps, sessions, resume, internals, runnerSelf) {
     const toolFaceSnapshot = {
         exclude: spec.excludeTools ? Object.freeze([...spec.excludeTools]) : undefined,
@@ -409,7 +431,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
             }));
         }
     }
-    const taskScope = spec.principal ?? "default";
+    const taskScope = internals?.registryScope ?? spec.principal ?? "default";
     defaultTaskRegistry.maybeGc();
     const forgetOnThrow = () => forgetQuietly(sessions, sessionId);
     const extraBodyCollisions = reservedCollisions(model.extraBody, reservedFor(model.api));
@@ -558,11 +580,13 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
             }
         }
         let restoredRootRebase;
-        const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p : rebaseWorkspacePath(p, restoredRootRebase.from, restoredRootRebase.to);
+        const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p : rebaseWorkspacePathAcross(p, restoredRootRebase.from, restoredRootRebase.to);
         if (resume?.workspaceHandle !== undefined) {
-            const failResume = (message, cause) => {
+            const failResume = (message, cause, note) => {
                 const e = new Error(message, cause ? { cause } : undefined);
                 e.code = "resume.env_failed";
+                if (note !== undefined)
+                    e.remoteEnvFailure = note;
                 throw e;
             };
             const handle = resume.workspaceHandle;
@@ -570,7 +594,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 failResume("resume needs a RemoteExecutionEnv from executionEnvFactory to restore the workspace snapshot");
             }
             else if (handle.snapshotId === undefined) {
-                if (handle.restoreMode !== "park_only" && isSuspendable(ownedEnv)) {
+                if (handle.restoreMode !== "park_only" && ownedEnv.capabilities.suspendable) {
                     failResume("checkpoint workspaceHandle has no snapshotId and is not a park_only handle, but the resumed env is suspendable — refusing to resume on a possibly-unrestored workspace (corrupt checkpoint?)");
                 }
                 if (handle.mountPath && handle.mountPath !== taskRootPath) {
@@ -581,12 +605,37 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 const restoreSignal = spec.signal
                     ? AbortSignal.any([abortController.signal, spec.signal])
                     : abortController.signal;
-                const restored = await ownedEnv.resumeVM(handle.snapshotId, { abortSignal: restoreSignal, priorHandle: handle });
+                const missingHere = missingRestoreSurface(ownedEnv);
+                if (missingHere.length > 0) {
+                    failResume(`the resumed execution env cannot restore a workspace snapshot: its adapter does not implement ${missingHere.join(" or ")}. The checkpoint holds snapshot "${handle.snapshotId}" — wire an adapter that implements the full RemoteExecutionEnv restore surface and re-resume.`);
+                }
+                const { outcome: restored, attempts: restoreAttempts } = await restoreWorkspaceWithRetry(ownedEnv, handle.snapshotId, {
+                    abortSignal: restoreSignal,
+                    priorHandle: handle,
+                });
                 if (!restored.ok) {
-                    failResume(`resumeVM failed (${restored.error.code}): ${restored.error.message}`, restored.error);
+                    failResume(`resumeVM failed after ${restoreAttempts} attempt(s) (${restored.error.code}): ${restored.error.message}`, restored.error, remoteEnvFailureNote("resumeVM", restored.error, restoreAttempts));
                 }
                 else {
+                    const restoredEnv = ownedEnv;
+                    const canonicalInEnv = async (p) => {
+                        try {
+                            const r = await restoredEnv.canonicalPath(p, restoreSignal);
+                            return r.ok ? r.value : undefined;
+                        }
+                        catch {
+                            return undefined;
+                        }
+                    };
+                    let checkpointedCanonical;
+                    let sameRootUnderAlias = false;
                     if (restored.value.mountPath !== handle.mountPath) {
+                        checkpointedCanonical = await canonicalInEnv(handle.mountPath);
+                        const restoredCanonical = await canonicalInEnv(restored.value.mountPath);
+                        sameRootUnderAlias =
+                            checkpointedCanonical !== undefined && restoredCanonical !== undefined && checkpointedCanonical === restoredCanonical;
+                    }
+                    if (restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
                         if (resume.executesApprovedAction === true) {
                             failResume(`resumeVM workspace-root divergence with a pending approved action: checkpointed mountPath "${handle.mountPath}" but the restored handle reports "${restored.value.mountPath}" — the approved args are bound to the checkpointed root; refusing to execute them against a moved workspace (adapter should honor priorHandle)`);
                         }
@@ -599,13 +648,23 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     if (restored.value.mountPath && restored.value.mountPath !== taskRootPath) {
                         taskRootPath = restored.value.mountPath;
                     }
-                    if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath) {
-                        restoredRootRebase = { from: handle.mountPath, to: restored.value.mountPath };
+                    if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
+                        const from = checkpointedCanonical !== undefined && checkpointedCanonical !== handle.mountPath
+                            ? [handle.mountPath, checkpointedCanonical]
+                            : [handle.mountPath];
+                        restoredRootRebase = { from, to: restored.value.mountPath };
+                        try {
+                            deps.onError?.(new Error(from.length > 1
+                                ? `resumeVM workspace-root rebase accepts both spellings of the checkpointed root (${from.map((f) => `"${f}"`).join(" and ")}) when migrating persisted paths to "${restored.value.mountPath}"`
+                                : `resumeVM workspace-root rebase is spelling-exact: it matches the checkpointed root "${handle.mountPath}" only, so persisted paths recorded under an equivalent alias of it are NOT migrated to "${restored.value.mountPath}" and stay as written`), { phase: "config", sessionId });
+                        }
+                        catch {
+                        }
                     }
                 }
                 const init = await ownedEnv.postResumeInit();
                 if (!init.ok) {
-                    failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error);
+                    failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error, remoteEnvFailureNote("postResumeInit", init.error, 1));
                 }
             }
         }
@@ -618,6 +677,11 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
         }
         const rewindNotes = [];
         const rewindCaptureRequested = spec.rewindFiles === true;
+        if (spec.resumeAt !== undefined && spec.rewindFilesTo !== undefined) {
+            const e = new Error(`rewind-files: resumeAt ("${spec.resumeAt}") and rewindFilesTo ("${spec.rewindFilesTo}") were both set — resumeAt already anchors the file restore when rewindFiles is true, so a separate rewindFilesTo target is a conflicting request. Drop one of the two.`);
+            e.code = "rewind.conflicting_targets";
+            throw e;
+        }
         let rewindTarget = spec.resumeAt !== undefined ? (rewindCaptureRequested ? spec.resumeAt : undefined) : spec.rewindFilesTo;
         const rewindBefore = rewindTarget !== undefined && spec.resumeAt !== undefined && spec.resumeAtMode === "before";
         if (spec.resumeAt !== undefined && !rewindCaptureRequested) {
@@ -943,6 +1007,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
         }
         const suspendRef = {};
         const reviewRef = {};
+        const remoteEnvFailures = [];
         const suspendLoopRef = { hit: false };
         const compactionReuseRef = { consecutive: 0 };
         const trimPressureRef = { droppedMessages: false };
@@ -1833,15 +1898,6 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
         }
         const activeTools = new Set();
         const fpRef = {};
-        if (rebuildHarnessToolsRef.current === undefined) {
-            rebuildHarnessToolsRef.current = async () => {
-                const list = [...tools];
-                await harnessRef.current.setTools(list, list.map((t) => t.name));
-                if (fpRef.current)
-                    fpRef.current.tools = toolsToFingerprintInputs(list);
-                turnSnapshotRef.current?.refreshTools(toolsToFingerprintInputs(list));
-            };
-        }
         const turnSnapshotRef = {};
         let harnessTools = tools;
         const failedMcpServers = mcp.statuses
@@ -1908,7 +1964,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     if (deferred.has(n))
                         activeTools.add(n);
             }
-            offloadReachableToolsRef.current = () => {
+            const callableToolNames = () => {
                 const s = new Set(tools.map((t) => t.name));
                 for (const n of deferred)
                     if (!activeTools.has(n))
@@ -1916,6 +1972,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 s.add(TOOL_SEARCH_NAME);
                 return s;
             };
+            offloadReachableToolsRef.current = callableToolNames;
             let toolSearch;
             const buildToolList = (active) => {
                 const list = tools.map((t) => (deferred.has(t.name) && !active.has(t.name) ? placeholders.get(t.name) : t));
@@ -1957,11 +2014,20 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 active: activeTools,
                 rematerialize,
                 listingRide: (newly) => listingRideRef.current?.(newly),
-                mountedNames: () => new Set(buildToolList(activeTools).map((t) => t.name).filter((n) => !registry.has(n) || activeTools.has(n))),
+                mountedNames: callableToolNames,
                 directCallEnabled: spec.deferSelfResolve !== false,
             });
             harnessTools = buildToolList(activeTools);
         }
+        else {
+            rebuildHarnessToolsRef.current = async () => {
+                const list = [...tools];
+                await harnessRef.current.setTools(list, list.map((t) => t.name));
+                if (fpRef.current)
+                    fpRef.current.tools = toolsToFingerprintInputs(list);
+                turnSnapshotRef.current?.refreshTools(toolsToFingerprintInputs(list));
+            };
+        }
         if (spec.agents !== undefined && spec.agents.length > 0) {
             const known = new Set(tools.flatMap((t) => [canonicalToolName(t.name), ...(t.aliases ?? []).map((a) => canonicalToolName(a))]));
             for (const def of spec.agents) {
@@ -2535,16 +2601,21 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
         const preToolContexts = new Map();
         const blockedToolCalls = new Set();
         const blockedTracked = Boolean(hooks?.postToolUse || hooks?.preToolUse || hooks?.postToolUseFailure || hooks?.postToolBatch);
+        const restoreSurfaceGap = ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && ownedEnv.capabilities.suspendable ? missingRestoreSurface(ownedEnv) : [];
+        const incompleteSuspendAdapter = restoreSurfaceGap.length > 0 ? restoreSurfaceGap : undefined;
         const resourceSuspendEligible = spec.resourceSuspend !== undefined &&
             (spec.checkpointStore ?? deps.checkpointStore) !== undefined &&
             !(offloadStore !== undefined && isVolatileOffloadStore(offloadStore)) &&
-            (ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv));
+            (ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv)) &&
+            incompleteSuspendAdapter === undefined;
         if (spec.resourceSuspend !== undefined && !resourceSuspendEligible) {
             const why = (spec.checkpointStore ?? deps.checkpointStore) === undefined
                 ? "no CheckpointStore is wired"
                 : offloadStore !== undefined && isVolatileOffloadStore(offloadStore)
                     ? "tool-result offload uses the in-memory store (a resume needs durable results)"
-                    : "the per-task execution env is not a RemoteExecutionEnv (it would be destroyed on suspend)";
+                    : incompleteSuspendAdapter !== undefined
+                        ? `the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")} (a snapshot nothing can restore is worse than no snapshot)`
+                        : "the per-task execution env is not a RemoteExecutionEnv (it would be destroyed on suspend)";
             deps.onError?.(new Error(`resourceSuspend is set but INACTIVE: ${why}; resource limits will hard-fail, not suspend`), {
                 phase: "config",
                 sessionId,
@@ -2720,14 +2791,18 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 catch (putErr) {
                     deps.onError?.(putErr, { phase: "config", sessionId });
                     if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
-                        const back = await remoteEnv.resumeVM(remoteHandle.snapshotId, { abortSignal: abortController.signal });
+                        const { outcome: back, attempts: backAttempts } = await restoreWorkspaceWithRetry(remoteEnv, remoteHandle.snapshotId, {
+                            abortSignal: abortController.signal,
+                        });
                         if (back.ok) {
                             const init = await remoteEnv.postResumeInit();
                             if (init.ok)
                                 return false;
+                            remoteEnvFailures.push(remoteEnvFailureNote("postResumeInit", init.error, 1));
                             deps.onError?.(init.error, { phase: "config", sessionId });
                         }
                         else {
+                            remoteEnvFailures.push(remoteEnvFailureNote("resumeVM", back.error, backAttempts));
                             deps.onError?.(back.error, { phase: "config", sessionId });
                         }
                         abortController.abort();
@@ -2737,10 +2812,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     return false;
                 }
             };
-            const publishCommittedSuspend = (token, gate, scope) => {
+            const publishCommittedSuspend = (token, gate, scope, remoteHandle) => {
                 const ref = gate.kind === "needs_review" || gate.kind === "plan_review" ? reviewRef : suspendRef;
                 ref.token = token;
                 ref.gate = gate;
+                if (remoteHandle !== undefined)
+                    ref.restoreMode = remoteHandle.restoreMode === "park_only" ? "park_only" : "snapshot";
                 ref.scope = scope;
             };
             const suspendLoopCapHit = (count, cap, detail) => {
@@ -2757,7 +2834,9 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                 return true;
             };
             const suspendableEnv = ownedEnv !== undefined && isSuspendable(ownedEnv) ? ownedEnv : undefined;
-            const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) ? ownedEnv : undefined;
+            const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && incompleteSuspendAdapter === undefined
+                ? ownedEnv
+                : undefined;
             const parkOnlyHandle = (env) => {
                 const { snapshotId: _lineage, ...identity } = env.workspaceHandle();
                 return { ...identity, restoreMode: "park_only" };
@@ -2782,6 +2861,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                             await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
                         const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
                         if (!snap.ok) {
+                            remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
                             deps.onError?.(new Error(`resource suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
                             return false;
                         }
@@ -2816,7 +2896,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     };
                     if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
                         return false;
-                    publishCommittedSuspend(token, gate, rs.scope);
+                    publishCommittedSuspend(token, gate, rs.scope, remoteHandle);
                     try {
                         await sessions.pin?.(sessionId);
                     }
@@ -2833,6 +2913,14 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                         return false;
                     if (abortController.signal.aborted)
                         return false;
+                    if (incompleteSuspendAdapter !== undefined) {
+                        try {
+                            deps.onError?.(new Error(`plan_review park skipped: the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so its workspace could not be restored on resume (the plan-review request was dropped; the run continues without pausing)`), { phase: "config", sessionId });
+                        }
+                        catch {
+                        }
+                        return false;
+                    }
                     if (suspendLoopCapHit(priorSuspendCount, maxSuspends, " for a plan_review (likely a resume/restart loop)."))
                         return false;
                     const leafId = await session.getLeafId();
@@ -2852,6 +2940,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                             await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
                         const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
                         if (!snap.ok) {
+                            remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
                             deps.onError?.(new Error(`plan_review suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
                             return false;
                         }
@@ -2890,7 +2979,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     };
                     if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
                         return false;
-                    publishCommittedSuspend(token, gate, scope);
+                    publishCommittedSuspend(token, gate, scope, remoteHandle);
                     try {
                         await sessions.pin?.(sessionId);
                     }
@@ -2924,10 +3013,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                                 "RunnerDeps.toolResultStore or disable offload.");
                         }
                         if (ownedEnv !== undefined && remoteEnv === undefined && parkOnlyRemoteEnv === undefined) {
-                            throw new Error("durable suspend is not supported with a non-remote per-task executionEnvFactory env: the " +
-                                "minted env is destroyed on suspend, so a resumed file/shell tool would act on a fresh " +
-                                "(empty) env. Use a RemoteExecutionEnv factory (it is paused, not destroyed) or a static, " +
-                                "caller-owned RunnerDeps.executionEnv.");
+                            throw new Error(incompleteSuspendAdapter !== undefined
+                                ? `durable suspend is not supported with this per-task executionEnvFactory env: it declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so a snapshot taken now could never be restored. Implement the full RemoteExecutionEnv restore surface, or declare capabilities.suspendable:false if the workspace is externally durable (the park-only lane).`
+                                : "durable suspend is not supported with a non-remote per-task executionEnvFactory env: the " +
+                                    "minted env is destroyed on suspend, so a resumed file/shell tool would act on a fresh " +
+                                    "(empty) env. Use a RemoteExecutionEnv factory (it is paused, not destroyed) or a static, " +
+                                    "caller-owned RunnerDeps.executionEnv.");
                         }
                         const leafId = await session.getLeafId();
                         if (!leafId) {
@@ -2945,6 +3036,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                                 await sweepBackgroundShells(remoteEnv, defaultTaskRegistry);
                             const snap = await remoteEnv.suspendVM({ abortSignal: abortController.signal });
                             if (!snap.ok) {
+                                remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
                                 throw new Error(`suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error });
                             }
                             remoteHandle = { ...remoteEnv.workspaceHandle(), snapshotId: snap.value };
@@ -3033,7 +3125,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
                     if (!(await commitSuspendSaga(token, cp, remoteEnv, remoteHandle))) {
                         return undefined;
                     }
-                    publishCommittedSuspend(token, gate, cp.scope);
+                    publishCommittedSuspend(token, gate, cp.scope, remoteHandle);
                     try {
                         await sessions.pin?.(sessionId);
                     }
@@ -3382,7 +3474,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
             : undefined;
         overheadState.promptChars = systemPrompt.length;
         const preparedHolder = {};
-        const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
+        const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, remoteEnvFailures, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
         const prepared = buildPrepared();
         preparedHolder.current = prepared;
         return prepared;

package/dist/core/runner/runtask.js

@@ -1345,6 +1345,16 @@ export class Runner {
                 queue.close();
                 return;
             }
+            const remoteEnvFailure = (() => {
+                let cur = err;
+                for (let depth = 0; cur && typeof cur === "object" && depth < 8; depth++) {
+                    const note = cur.remoteEnvFailure;
+                    if (note !== undefined)
+                        return [note];
+                    cur = cur.cause;
+                }
+                return undefined;
+            })();
             resultValue = {
                 taskId: taskIdRef.current ?? "unknown",
                 sessionId: taskIdRef.sessionId ?? "unknown",
@@ -1352,6 +1362,7 @@ export class Runner {
                 result: "",
                 errorMessage: err instanceof Error ? err.message : String(err),
                 errorCode: code,
+                ...(remoteEnvFailure !== undefined ? { remoteEnvFailures: remoteEnvFailure } : {}),
                 stats: { turns: 0, tokens: 0, toolCalls: 0, cachedTokens: 0, costMicroUsd: 0 },
             };
             emitTrace(spec.tracer ?? this.deps.tracer, () => ({
@@ -2765,6 +2776,7 @@ export class Runner {
                 model: prepared.model.id,
                 unpricedSpend: rs.telemetry.unpricedSpend,
                 rewindNotes: prepared.rewindNotes,
+                remoteEnvFailures: prepared.remoteEnvFailures,
                 abortedForTimeout: timeout.fired,
                 abortedForTurns: rs.limits.turnsExceeded,
                 abortedLive,
@@ -2774,10 +2786,18 @@ export class Runner {
                 outputInvalid: rs.degrade.outputInvalid,
                 suspendLoop: prepared.suspendLoopRef.hit,
                 suspendRef: prepared.suspendRef.token !== undefined && prepared.suspendRef.gate !== undefined
-                    ? { token: prepared.suspendRef.token, gate: prepared.suspendRef.gate }
+                    ? {
+                        token: prepared.suspendRef.token,
+                        gate: prepared.suspendRef.gate,
+                        ...(prepared.suspendRef.restoreMode !== undefined ? { restoreMode: prepared.suspendRef.restoreMode } : {}),
+                    }
                     : undefined,
                 reviewRef: prepared.reviewRef.token !== undefined && prepared.reviewRef.gate !== undefined
-                    ? { token: prepared.reviewRef.token, gate: prepared.reviewRef.gate }
+                    ? {
+                        token: prepared.reviewRef.token,
+                        gate: prepared.reviewRef.gate,
+                        ...(prepared.reviewRef.restoreMode !== undefined ? { restoreMode: prepared.reviewRef.restoreMode } : {}),
+                    }
                     : undefined,
             });
             if (resume !== undefined &&

package/dist/core/session-store.d.ts

@@ -12,6 +12,7 @@ export interface TtlSessionStoreOptions {
 export declare class TtlSessionStore implements SessionStore {
     private repo;
     private entries;
+    private owners;
     private pending;
     private pinned;
     private defaultTtlMs;
@@ -31,6 +32,8 @@ export declare class TtlSessionStore implements SessionStore {
     forget(sessionId: string): void;
     pin(sessionId: string): void;
     unpin(sessionId: string): void;
+    ownerOf(sessionId: string): Promise<string | null | undefined>;
+    register(sessionId: string, owner: string | null): Promise<void>;
     sweep(now?: number): void;
     get size(): number;
     dispose(): void;

package/dist/core/session-store.js

@@ -6,6 +6,7 @@ export { SESSION_DEFAULT_TTL_DAYS };
 export class TtlSessionStore {
     repo;
     entries = new Map();
+    owners = new Map();
     pending = new Map();
     pinned = new Set();
     defaultTtlMs;
@@ -132,6 +133,7 @@ export class TtlSessionStore {
         this.entries.delete(sessionId);
         if (this.evictPolicy === "delete") {
             await this.repo.delete(await e.session.getMetadata());
+            this.owners.delete(sessionId);
         }
     }
     forget(sessionId) {
@@ -143,6 +145,13 @@ export class TtlSessionStore {
     unpin(sessionId) {
         this.pinned.delete(sessionId);
     }
+    async ownerOf(sessionId) {
+        return this.owners.has(sessionId) ? this.owners.get(sessionId) : undefined;
+    }
+    async register(sessionId, owner) {
+        if (!this.owners.has(sessionId))
+            this.owners.set(sessionId, owner);
+    }
     sweep(now = Date.now()) {
         for (const [id, e] of this.entries) {
             if (this.pinned.has(id)) {
@@ -152,6 +161,7 @@ export class TtlSessionStore {
                 this.entries.delete(id);
                 if (this.evictPolicy === "delete") {
                     void this.repo.delete({ id, createdAt: "" });
+                    this.owners.delete(id);
                 }
             }
         }

package/dist/core/session.d.ts

@@ -31,6 +31,8 @@ export interface SessionStore {
     noteTaskRun?(sessionId: string, taskId: string): void | Promise<void>;
     list?(): Promise<SessionStoreSummary[]>;
     fork?(sourceId: string, owner?: string | null): Promise<string | null>;
+    ownerOf?(sessionId: string): Promise<string | null | undefined>;
+    register?(sessionId: string, owner: string | null): Promise<void>;
     readonly size: number;
     dispose(): void | Promise<void>;
 }

package/dist/core/tool-result-budget.js

@@ -21,6 +21,8 @@ function isOffloadedPreview(content) {
     return first?.type === "text" && typeof first.text === "string" && first.text.startsWith(PERSISTED_OUTPUT_PREFIX);
 }
 function replaceTextBlock(m, text) {
+    if (!isToolResult(m))
+        return m;
     const images = m.content.filter((b) => b.type !== "text");
     return { ...m, content: [{ type: "text", text }, ...images] };
 }

package/dist/core/types.d.ts

@@ -370,6 +370,13 @@ export interface TaskSpec {
     signal?: AbortSignal;
 }
 export type TaskStatus = "completed" | "blocked" | "failed" | "timeout" | "suspended" | "needs_review";
+export interface RemoteEnvFailureNote {
+    op: "suspendVM" | "resumeVM" | "postResumeInit";
+    code: import("./remote-env.js").RemoteExecutionErrorCode;
+    retryable: boolean;
+    attempts: number;
+    message: string;
+}
 export interface TaskResult {
     taskId: string;
     sessionId: string;
@@ -380,6 +387,8 @@ export interface TaskResult {
     blockedReason?: string;
     checkpointToken?: import("./checkpoint-store.js").CheckpointToken;
     checkpointGate?: import("./checkpoint-store.js").CheckpointGate;
+    workspaceRestoreMode?: "snapshot" | "park_only";
+    remoteEnvFailures?: RemoteEnvFailureNote[];
     errorMessage?: string;
     errorCode?: string;
     degraded?: {

package/dist/engine/harness/types.d.ts

@@ -47,7 +47,7 @@ export declare class FileError extends Error {
     path?: string;
     constructor(code: FileErrorCode, message: string, path?: string, cause?: Error);
 }
-export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "unknown";
+export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "suspended" | "auth_failed" | "unknown";
 export declare class ExecutionError extends Error {
     code: ExecutionErrorCode;
     partialStdout?: string;

package/dist/engine/session/session.js

@@ -34,8 +34,9 @@ export function buildSessionContext(pathEntries, opts) {
         }
     }
     const messages = [];
+    const isAssistantMessage = (m) => m.role === "assistant";
     const stripAssistantUsage = (message) => {
-        if (message.role !== "assistant") {
+        if (!isAssistantMessage(message)) {
             return message;
         }
         return {

package/dist/index.d.ts

@@ -56,7 +56,7 @@ export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-e
 export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
 export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
 export type { ExecutionEnv, FileInfo, Result, FileErrorCode, ExecutionErrorCode } from "./internal/harness.js";
-export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
+export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
 export { withRetry } from "./core/with-retry.js";
 export type { RetryPolicy, RetryResult } from "./core/with-retry.js";
 export type { RemoteExecutionEnv, WorkspaceHandle, SnapshotId, SessionToken, SandboxTier, OutputChunk, ExecStreamOptions, RemoteConnectConfig, VmLifecycleOptions, SecretRef, RemoteExecutionErrorCode, ExecutionEnvFactory, ExecutionEnvFactoryContext, } from "./core/remote-env.js";
@@ -189,7 +189,7 @@ export { retryBackoffMs, parseRetryAfter } from "./brain/retry.js";
 export { type BrainTimeoutConfig } from "./brain/timeout.js";
 export { createAssistantMessageEventStream } from "./internal/llm.js";
 export type { AssistantMessage, AssistantMessageEvent, Context, DocumentContent, ImageContent, Message, StopReason, StreamFn, TextContent, ThinkingContent, ToolCall, ToolResultMessage, Usage, UserMessage, } from "./internal/llm.js";
-export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
+export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, RemoteEnvFailureNote, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
 export { Type } from "typebox";
 export type { TSchema, Static } from "typebox";
 export { explainPromptAssembly, describeDefaultPack, type DefaultPackDescription, type ExplainInput } from "./prompt-assembly/explain.js";

package/dist/index.js

@@ -51,7 +51,7 @@ export { killProcessTree, signalProcessTree } from "./engine/execution-env/kill-
 export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-execution-env.js";
 export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
 export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
-export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
+export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
 export { withRetry } from "./core/with-retry.js";
 export { addWorktree, pruneWorktrees, WORKTREE_PARENT } from "./core/git-worktree-env.js";
 export { runExecGate } from "./core/exec-gate.js";

package/dist/tools/fs/fs-bash.js

@@ -229,6 +229,20 @@ async function runShell(env, cwd, toolName, command, timeoutSec, caps, signal, c
                 isError: true,
             };
         }
+        if (res.error.code === "suspended") {
+            return {
+                content: `Error (${toolName}): the execution environment is suspended (its workspace VM is paused), so the command did NOT run and no command can run in this leg. Do not retry it here — the task must be resumed first; report that the workspace is suspended. (${res.error.message})`,
+                details: { type: "bash", envSuspended: true },
+                isError: true,
+            };
+        }
+        if (res.error.code === "auth_failed") {
+            return {
+                content: `Error (${toolName}): authentication to the execution environment was rejected, so the command did NOT run. Do NOT retry — this is a permanent credential failure that an operator must fix; report it instead of trying other commands. (${res.error.message})`,
+                details: { type: "bash", authFailed: true },
+                isError: true,
+            };
+        }
         if (res.error.code === "timeout" || res.error.code === "aborted" || res.error.code === "callback_error") {
             const rawStdout = res.error.partialStdout ?? "";
             const rawStderr = res.error.partialStderr ?? "";

package/dist/tools/fs/fs-write.js

@@ -152,6 +152,7 @@ export function createEditFileTool(env, state, rootCanonical, cwdRef, additional
                 return errorResult(`Error (Edit): "${path}" has a truncated UTF-16 body; repair/convert it with bash first.`);
             const original = decoded.text;
             const entry = state.get(r.key);
+            const readWasTruncated = entry?.truncated === true;
             const stale = checkStale(entry, sha256(original));
             if (stale)
                 return errorResult(violationText("Edit", stale));
@@ -180,7 +181,7 @@ export function createEditFileTool(env, state, rootCanonical, cwdRef, additional
                         oldS = resolvedEsc;
                     }
                 }
-                const match = checkEditMatch(working, oldS, e.replace_all === true, entry.truncated);
+                const match = checkEditMatch(working, oldS, e.replace_all === true, readWasTruncated);
                 if (match) {
                     const escNote = match.code === "ambiguous_edit" && escapeMatchWasAttempted(e.old_string) && !working.includes(oldS) ? ESCAPE_MATCH_MISS_NOTE : "";
                     return errorResult(batch ? `Error (Edit): ${where}${match.message}${escNote} (no changes written — the batch is atomic).` : `${violationText("Edit", match)}${escNote}`);

package/dist/tools/fs/safety.d.ts

@@ -58,7 +58,7 @@ export declare const OVERSIZE_READ_ESCAPE_HINT = "(This file is over the Read to
 export declare const PARTIAL_VIEW_READ_ESCAPE_HINT = "(Your last Read of this file returned only a PARTIAL view \u2014 the output token cap paginated it, so a default Read will keep returning the same page. Re-read it with explicit offset/limit (start from the page marker's next-page hint) until you have seen the part you are about to change; an explicit slice that fits satisfies the read-first rule. Or inspect/transform it with bash (e.g. `sed -n`, `grep`) instead.)";
 export declare const WRITE_ENCODING_DEADLOCK_ESCAPE_HINT = "(If the Read tool refuses this file (binary/unknown encoding), overwrite or convert it with bash instead \u2014 e.g. `rm` + rewrite, or `iconv`.)";
 export declare function checkNoChange(oldString: string, newString: string): FsViolation | undefined;
-export declare function checkStale(entry: ReadEntry, currentHash: string): FsViolation | undefined;
+export declare function checkStale(entry: ReadEntry | undefined, currentHash: string): FsViolation | undefined;
 export declare function countOccurrences(haystack: string, needle: string): number;
 export declare function similarNameSuggestion(siblingNames: readonly string[], missingName: string): string | undefined;
 export declare function normalizeQuotes(s: string): string;