@sema-agent/core 2.11.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -242,6 +242,7 @@ export interface CheckpointSummary {
242
242
  contentKind?: "content_ask";
243
243
  createdAt?: number;
244
244
  toolInput?: string;
245
+ restoreMode?: "snapshot" | "park_only";
245
246
  preview?: unknown;
246
247
  }
247
248
  export declare function summarizeCheckpoint(cp: Checkpoint): CheckpointSummary;
@@ -181,6 +181,9 @@ export function summarizeCheckpoint(cp) {
181
181
  ...(tool !== undefined ? { toolCallId: tool.toolCallId, toolName: tool.toolName } : {}),
182
182
  ...(tool?.toolName === ASK_USER_QUESTION_TOOL_NAME ? { contentKind: "content_ask" } : {}),
183
183
  ...(toolInput !== undefined ? { toolInput } : {}),
184
+ ...(cp.state.workspaceHandle !== undefined
185
+ ? { restoreMode: cp.state.workspaceHandle.restoreMode === "park_only" ? "park_only" : "snapshot" }
186
+ : {}),
184
187
  ...(tool?.preview !== undefined ? { preview: tool.preview } : {}),
185
188
  };
186
189
  }
@@ -57,6 +57,7 @@ export interface McpServerStatus {
57
57
  };
58
58
  toolNames?: string[];
59
59
  error?: string;
60
+ transportClosed?: boolean;
60
61
  }
61
62
  export declare const MCP_PREFIX = "mcp__";
62
63
  export declare function resolveMcpDeclaredResultSize(meta: Record<string, unknown> | undefined): number | undefined;
@@ -69,6 +70,7 @@ export declare const MCP_IDLE_TIMEOUT_STDIO_DEFAULT_MS: number;
69
70
  export declare const MCP_IDLE_TIMEOUT_HTTP_DEFAULT_MS: number;
70
71
  export declare function mcpIdleTimeoutMs(kind: "stdio" | "http"): number;
71
72
  export declare function describeMcpSpecErrorCode(code: unknown): string | undefined;
73
+ export declare function collapseMcpErrorPrefix(message: string): string;
72
74
  export declare function normalizeMcpName(name: string): string;
73
75
  export declare function clampNameSegment(seg: string, max?: number): string;
74
76
  export * from "./image-downsample.js";
package/dist/core/mcp.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { Client } from "@modelcontextprotocol/sdk/client/index.js";
3
3
  import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
4
- import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
4
+ import { StreamableHTTPClientTransport, StreamableHTTPError } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
5
5
  import { lstat, mkdir, writeFile } from "node:fs/promises";
6
6
  import { tmpdir } from "node:os";
7
7
  import { join } from "node:path";
@@ -145,8 +145,24 @@ const MCP_SPEC_ERROR_CODE_NAMES = new Map([
145
145
  export function describeMcpSpecErrorCode(code) {
146
146
  return typeof code === "number" ? MCP_SPEC_ERROR_CODE_NAMES.get(code) : undefined;
147
147
  }
148
+ export function collapseMcpErrorPrefix(message) {
149
+ let out = message;
150
+ for (;;) {
151
+ const m = /^MCP error (-?\d+): (?=MCP error \1: )/.exec(out);
152
+ if (m === null)
153
+ return out;
154
+ out = out.slice(m[0].length);
155
+ }
156
+ }
157
+ function collapseMcpErrorStampInPlace(err) {
158
+ if (!(err instanceof McpError))
159
+ return;
160
+ const collapsed = collapseMcpErrorPrefix(err.message);
161
+ if (collapsed !== err.message)
162
+ err.message = collapsed;
163
+ }
148
164
  function namedMcpFailureText(err) {
149
- const detail = err instanceof Error ? err.message : String(err);
165
+ const detail = err instanceof Error ? collapseMcpErrorPrefix(err.message) : String(err);
150
166
  const condition = err instanceof McpError ? describeMcpSpecErrorCode(err.code) : undefined;
151
167
  return condition !== undefined ? `${condition}: ${detail}` : detail;
152
168
  }
@@ -155,6 +171,53 @@ function isTransportLost(err) {
155
171
  return true;
156
172
  return err instanceof Error && /not connected|connection closed/i.test(err.message);
157
173
  }
174
+ const NETWORK_CODES_NEVER_DELIVERED = new Set([
175
+ "ECONNREFUSED",
176
+ "ENOTFOUND",
177
+ "EAI_AGAIN",
178
+ "EHOSTUNREACH",
179
+ "ENETUNREACH",
180
+ "UND_ERR_CONNECT_TIMEOUT",
181
+ ]);
182
+ function networkErrorCode(err, depth = 0) {
183
+ if (depth > 5 || !(err instanceof Error))
184
+ return undefined;
185
+ const code = err.code;
186
+ if (typeof code === "string" && /^(?:E[A-Z]+|UND_ERR_[A-Z_]+)$/.test(code))
187
+ return code;
188
+ if (err instanceof AggregateError) {
189
+ for (const inner of err.errors) {
190
+ const found = networkErrorCode(inner, depth + 1);
191
+ if (found !== undefined)
192
+ return found;
193
+ }
194
+ }
195
+ return networkErrorCode(err.cause, depth + 1);
196
+ }
197
+ function describeHttpTransportFailure(err) {
198
+ if (err instanceof McpError)
199
+ return undefined;
200
+ if (err instanceof StreamableHTTPError) {
201
+ const status = typeof err.code === "number" && err.code > 0 ? err.code : undefined;
202
+ return {
203
+ condition: status !== undefined
204
+ ? `its HTTP endpoint answered ${status} instead of an MCP response`
205
+ : "its HTTP endpoint answered something that is not an MCP response",
206
+ delivered: "unknown",
207
+ ...(status !== undefined ? { httpStatus: status } : {}),
208
+ };
209
+ }
210
+ const code = networkErrorCode(err);
211
+ if (code !== undefined) {
212
+ return NETWORK_CODES_NEVER_DELIVERED.has(code)
213
+ ? { condition: `its HTTP endpoint could not be reached (${code})`, delivered: "no" }
214
+ : { condition: `the connection to its HTTP endpoint failed (${code})`, delivered: "unknown" };
215
+ }
216
+ if (err instanceof TypeError && /fetch failed|terminated|network/i.test(err.message)) {
217
+ return { condition: "the HTTP request to its endpoint failed at the network layer", delivered: "unknown" };
218
+ }
219
+ return undefined;
220
+ }
158
221
  function writeEffectWarning(writeEffect) {
159
222
  return writeEffect
160
223
  ? " This tool is write-capable: treat its side effects as POSSIBLY APPLIED and verify the actual state before retrying."
@@ -173,6 +236,7 @@ function rethrowHonestMcpError(err, ctx) {
173
236
  }
174
237
  if (ctx.signal?.aborted)
175
238
  throw err;
239
+ collapseMcpErrorStampInPlace(err);
176
240
  const serverLabel = inlineUntrusted(ctx.server);
177
241
  if (err instanceof McpError && err.code === ErrorCode.RequestTimeout) {
178
242
  const data = err.data;
@@ -193,6 +257,21 @@ function rethrowHonestMcpError(err, ctx) {
193
257
  e.details = { transportLost: true, server: ctx.server };
194
258
  throw e;
195
259
  }
260
+ const httpFailure = describeHttpTransportFailure(err);
261
+ if (httpFailure !== undefined) {
262
+ const detail = err instanceof Error ? err.message : String(err);
263
+ const fenced = `\nThe transport error follows as external/untrusted data:\n${delimitUntrusted(`${ctx.server} transport error`, truncateMcpErrorText(detail))}`;
264
+ const e = new Error(httpFailure.delivered === "no"
265
+ ? `${ctx.what} could not reach MCP server "${serverLabel}": ${httpFailure.condition}. The request was not delivered, so the server did not execute it. This server's tools and resources will keep failing until its endpoint is reachable again — do not retry them; use an alternative if one exists.${fenced}`
266
+ : `${ctx.what} failed at the transport layer of MCP server "${serverLabel}": ${httpFailure.condition}. The request may or may not have executed on the server — the outcome is unknown.${writeEffectWarning(ctx.writeEffect)}${fenced}`, { cause: err });
267
+ e.errorKind = httpFailure.delivered === "no" ? "server_disconnected" : "transport_lost";
268
+ e.details = {
269
+ server: ctx.server,
270
+ ...(httpFailure.delivered === "no" ? {} : { transportLost: true }),
271
+ ...(httpFailure.httpStatus !== undefined ? { httpStatus: httpFailure.httpStatus } : {}),
272
+ };
273
+ throw e;
274
+ }
196
275
  if (err instanceof McpError) {
197
276
  const condition = describeMcpSpecErrorCode(err.code);
198
277
  if (condition !== undefined) {
@@ -546,12 +625,16 @@ export async function materializeMcpTools(specs, principal, onElicit, imageResiz
546
625
  serverInstructions.push({ server: spec.name, text: s.instructions });
547
626
  }
548
627
  const hasInstructions = s.instructions !== undefined;
549
- s.announce.fn = () => {
550
- if (!disposing && hasInstructions)
628
+ const announceClose = () => {
629
+ if (disposing)
630
+ return;
631
+ s.status.transportClosed = true;
632
+ if (hasInstructions)
551
633
  instructionsDelta.pendingRemovals.push(spec.name);
552
634
  };
553
- if (s.health.dead && hasInstructions)
554
- instructionsDelta.pendingRemovals.push(spec.name);
635
+ s.announce.fn = announceClose;
636
+ if (s.health.dead)
637
+ announceClose();
555
638
  if (s.resourceServer)
556
639
  resourceServers.push(s.resourceServer);
557
640
  for (const d of s.dropped)
@@ -44,6 +44,8 @@ export interface ExecStreamOptions {
44
44
  maxOutputBytes?: number;
45
45
  }
46
46
  export type RemoteExecutionErrorCode = "suspended" | "command_in_flight" | "connect_failed" | "post_resume_failed" | "aborted" | "timeout" | "unsupported" | "auth_transient" | "auth_failed" | "transport_lost" | "unknown";
47
+ export declare const RETRYABLE_REMOTE_ERROR_CODES: readonly RemoteExecutionErrorCode[];
48
+ export declare function isRetryableRemoteErrorCode(code: RemoteExecutionErrorCode): boolean;
47
49
  export declare class RemoteExecutionError extends Error {
48
50
  readonly code: RemoteExecutionErrorCode;
49
51
  constructor(code: RemoteExecutionErrorCode, message: string, cause?: Error);
@@ -101,5 +103,6 @@ export interface ExecutionEnvFactoryContext {
101
103
  export type ExecutionEnvFactory = (ctx: ExecutionEnvFactoryContext) => ExecutionEnv | Promise<ExecutionEnv>;
102
104
  export declare function hasDestroy(env: ExecutionEnv): env is ExecutionEnv & Pick<RemoteExecutionEnv, "destroy">;
103
105
  export declare function isRemoteExecutionEnv(env: ExecutionEnv): env is RemoteExecutionEnv;
106
+ export declare function missingRestoreSurface(env: ExecutionEnv): readonly ("resumeVM" | "postResumeInit")[];
104
107
  export declare function isSuspendable(env: ExecutionEnv): env is RemoteExecutionEnv;
105
108
  export declare function isIsolated(env: ExecutionEnv): boolean;
@@ -1,3 +1,12 @@
1
+ export const RETRYABLE_REMOTE_ERROR_CODES = [
2
+ "auth_transient",
3
+ "connect_failed",
4
+ "timeout",
5
+ "transport_lost",
6
+ ];
7
+ export function isRetryableRemoteErrorCode(code) {
8
+ return RETRYABLE_REMOTE_ERROR_CODES.includes(code);
9
+ }
1
10
  export class RemoteExecutionError extends Error {
2
11
  code;
3
12
  constructor(code, message, cause) {
@@ -18,8 +27,17 @@ export function isRemoteExecutionEnv(env) {
18
27
  typeof e.capabilities === "object" &&
19
28
  e.capabilities !== null);
20
29
  }
30
+ export function missingRestoreSurface(env) {
31
+ const e = env;
32
+ const missing = [];
33
+ if (typeof e.resumeVM !== "function")
34
+ missing.push("resumeVM");
35
+ if (typeof e.postResumeInit !== "function")
36
+ missing.push("postResumeInit");
37
+ return missing;
38
+ }
21
39
  export function isSuspendable(env) {
22
- return isRemoteExecutionEnv(env) && env.capabilities.suspendable;
40
+ return isRemoteExecutionEnv(env) && env.capabilities.suspendable && missingRestoreSurface(env).length === 0;
23
41
  }
24
42
  export function isIsolated(env) {
25
43
  return isRemoteExecutionEnv(env) ? env.capabilities.isolation : false;
@@ -70,6 +70,7 @@ export interface ResultFlags {
70
70
  model?: string;
71
71
  unpricedSpend?: boolean;
72
72
  rewindNotes?: TaskResult["rewindNotes"];
73
+ remoteEnvFailures?: TaskResult["remoteEnvFailures"];
73
74
  abortedForTimeout: boolean;
74
75
  abortedForTurns: boolean;
75
76
  abortedLive?: boolean;
@@ -81,10 +82,12 @@ export interface ResultFlags {
81
82
  suspendRef?: {
82
83
  token: import("../checkpoint-store.js").CheckpointToken;
83
84
  gate: import("../checkpoint-store.js").CheckpointGate;
85
+ restoreMode?: "snapshot" | "park_only";
84
86
  };
85
87
  reviewRef?: {
86
88
  token: import("../checkpoint-store.js").CheckpointToken;
87
89
  gate: import("../checkpoint-store.js").CheckpointGate;
90
+ restoreMode?: "snapshot" | "park_only";
88
91
  };
89
92
  }
90
93
  export declare function errorCodeOf(err: unknown): string | undefined;
@@ -54,6 +54,7 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
54
54
  let salvagedOutput;
55
55
  let checkpointToken;
56
56
  let checkpointGate;
57
+ let workspaceRestoreMode;
57
58
  const isDegenerate = final?.stopReason === "error" && final.errorMessage === DEGENERATE_MESSAGE;
58
59
  const isWalltimeCutoff = final?.stopReason === "error" && final.errorMessage === WALLTIME_CUTOFF_MESSAGE;
59
60
  if (flags.outputInvalid) {
@@ -105,12 +106,14 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
105
106
  status = "suspended";
106
107
  checkpointToken = flags.suspendRef.token;
107
108
  checkpointGate = flags.suspendRef.gate;
109
+ workspaceRestoreMode = flags.suspendRef.restoreMode;
108
110
  }
109
111
  else if (flags.reviewRef) {
110
112
  status = "needs_review";
111
113
  errorCode = "review.pending";
112
114
  checkpointToken = flags.reviewRef.token;
113
115
  checkpointGate = flags.reviewRef.gate;
116
+ workspaceRestoreMode = flags.reviewRef.restoreMode;
114
117
  }
115
118
  else if (flags.abortedLive || final?.stopReason === "aborted") {
116
119
  status = flags.abortedForTimeout ? "timeout" : "failed";
@@ -143,5 +146,5 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
143
146
  void _internalCompaction;
144
147
  if (flags.unpricedSpend)
145
148
  delete publicStats.costMicroUsd;
146
- return { taskId, sessionId, status, ...(flags.model !== undefined ? { model: flags.model } : {}), result: result.trim(), salvagedOutput, blockedReason, errorMessage, errorCode, checkpointToken, checkpointGate, ...(flags.rewindNotes !== undefined && flags.rewindNotes.length > 0 ? { rewindNotes: flags.rewindNotes } : {}), stats: publicStats };
149
+ return { taskId, sessionId, status, ...(flags.model !== undefined ? { model: flags.model } : {}), result: result.trim(), salvagedOutput, blockedReason, errorMessage, errorCode, checkpointToken, checkpointGate, ...(workspaceRestoreMode !== undefined ? { workspaceRestoreMode } : {}), ...(flags.rewindNotes !== undefined && flags.rewindNotes.length > 0 ? { rewindNotes: flags.rewindNotes } : {}), ...(flags.remoteEnvFailures !== undefined && flags.remoteEnvFailures.length > 0 ? { remoteEnvFailures: flags.remoteEnvFailures } : {}), stats: publicStats };
147
150
  }
@@ -127,12 +127,15 @@ export interface Prepared {
127
127
  token?: CheckpointToken;
128
128
  gate?: CheckpointGate;
129
129
  scope?: string;
130
+ restoreMode?: "snapshot" | "park_only";
130
131
  };
131
132
  reviewRef: {
132
133
  token?: CheckpointToken;
133
134
  gate?: CheckpointGate;
134
135
  scope?: string;
136
+ restoreMode?: "snapshot" | "park_only";
135
137
  };
138
+ remoteEnvFailures: NonNullable<TaskResult["remoteEnvFailures"]>;
136
139
  suspendLoopRef: {
137
140
  hit: boolean;
138
141
  };
@@ -340,4 +343,5 @@ export declare function batchContextAt(messages: AgentMessage[], currentId: stri
340
343
  completedCallIds: string[];
341
344
  };
342
345
  export declare function rebaseWorkspacePath(p: string, fromRaw: string, toRaw: string): string;
346
+ export declare function rebaseWorkspacePathAcross(p: string, froms: readonly string[], to: string): string;
343
347
  export declare function prepareTask(spec: TaskSpec, deps: RunnerDeps, sessions: SessionStore, resume?: PrepareResume, internals?: RunInternals, runnerSelf?: Runner): Promise<Prepared>;
@@ -49,7 +49,8 @@ import { capAggregateToolResults } from "../tool-result-budget.js";
49
49
  import { capAggregateMediaBytes, AGGREGATE_MEDIA_BUDGET_BYTES } from "../media-byte-cap.js";
50
50
  import { dropOrphanToolResults, guardBudget, insertTrimNotice, trimToBudget } from "../context-guard.js";
51
51
  import { StubExecutionEnv } from "../stub-env.js";
52
- import { hasDestroy, isIsolated, isRemoteExecutionEnv, isSuspendable } from "../remote-env.js";
52
+ import { hasDestroy, isIsolated, isRemoteExecutionEnv, isRetryableRemoteErrorCode, isSuspendable, missingRestoreSurface, RETRYABLE_REMOTE_ERROR_CODES } from "../remote-env.js";
53
+ import { withRetry } from "../with-retry.js";
53
54
  import { hasBackgroundShell, sweepBackgroundShells } from "../background-shell.js";
54
55
  import { createTaskOutputTool, createTaskStopTool, defaultTaskRegistry } from "../task-registry.js";
55
56
  import { createMonitorTool } from "../../tools/monitor.js";
@@ -173,6 +174,27 @@ export function rebaseWorkspacePath(p, fromRaw, toRaw) {
173
174
  const suffix = p.slice(fromPrefix.length).replace(/^\/+/, "");
174
175
  return to === "/" ? `${to}${suffix}` : `${to}/${suffix}`;
175
176
  }
177
+ function remoteEnvFailureNote(op, error, attempts) {
178
+ return { op, code: error.code, retryable: isRetryableRemoteErrorCode(error.code), attempts, message: error.message };
179
+ }
180
+ const REMOTE_RESTORE_MAX_ATTEMPTS = 2;
181
+ const REMOTE_RESTORE_BACKOFF_MS = 200;
182
+ async function restoreWorkspaceWithRetry(env, snapshotId, options) {
183
+ let attempts = 0;
184
+ const outcome = await withRetry(async (attempt) => {
185
+ attempts = attempt;
186
+ return env.resumeVM(snapshotId, options);
187
+ }, { retryableCodes: RETRYABLE_REMOTE_ERROR_CODES, maxAttempts: REMOTE_RESTORE_MAX_ATTEMPTS, backoffMs: () => REMOTE_RESTORE_BACKOFF_MS }, { ...(options.abortSignal !== undefined ? { signal: options.abortSignal } : {}) });
188
+ return { outcome, attempts };
189
+ }
190
+ export function rebaseWorkspacePathAcross(p, froms, to) {
191
+ for (const from of froms) {
192
+ const out = rebaseWorkspacePath(p, from, to);
193
+ if (out !== p)
194
+ return out;
195
+ }
196
+ return p;
197
+ }
176
198
  export async function prepareTask(spec, deps, sessions, resume, internals, runnerSelf) {
177
199
  const toolFaceSnapshot = {
178
200
  exclude: spec.excludeTools ? Object.freeze([...spec.excludeTools]) : undefined,
@@ -558,11 +580,13 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
558
580
  }
559
581
  }
560
582
  let restoredRootRebase;
561
- const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p : rebaseWorkspacePath(p, restoredRootRebase.from, restoredRootRebase.to);
583
+ const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p : rebaseWorkspacePathAcross(p, restoredRootRebase.from, restoredRootRebase.to);
562
584
  if (resume?.workspaceHandle !== undefined) {
563
- const failResume = (message, cause) => {
585
+ const failResume = (message, cause, note) => {
564
586
  const e = new Error(message, cause ? { cause } : undefined);
565
587
  e.code = "resume.env_failed";
588
+ if (note !== undefined)
589
+ e.remoteEnvFailure = note;
566
590
  throw e;
567
591
  };
568
592
  const handle = resume.workspaceHandle;
@@ -570,7 +594,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
570
594
  failResume("resume needs a RemoteExecutionEnv from executionEnvFactory to restore the workspace snapshot");
571
595
  }
572
596
  else if (handle.snapshotId === undefined) {
573
- if (handle.restoreMode !== "park_only" && isSuspendable(ownedEnv)) {
597
+ if (handle.restoreMode !== "park_only" && ownedEnv.capabilities.suspendable) {
574
598
  failResume("checkpoint workspaceHandle has no snapshotId and is not a park_only handle, but the resumed env is suspendable — refusing to resume on a possibly-unrestored workspace (corrupt checkpoint?)");
575
599
  }
576
600
  if (handle.mountPath && handle.mountPath !== taskRootPath) {
@@ -581,12 +605,37 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
581
605
  const restoreSignal = spec.signal
582
606
  ? AbortSignal.any([abortController.signal, spec.signal])
583
607
  : abortController.signal;
584
- const restored = await ownedEnv.resumeVM(handle.snapshotId, { abortSignal: restoreSignal, priorHandle: handle });
608
+ const missingHere = missingRestoreSurface(ownedEnv);
609
+ if (missingHere.length > 0) {
610
+ failResume(`the resumed execution env cannot restore a workspace snapshot: its adapter does not implement ${missingHere.join(" or ")}. The checkpoint holds snapshot "${handle.snapshotId}" — wire an adapter that implements the full RemoteExecutionEnv restore surface and re-resume.`);
611
+ }
612
+ const { outcome: restored, attempts: restoreAttempts } = await restoreWorkspaceWithRetry(ownedEnv, handle.snapshotId, {
613
+ abortSignal: restoreSignal,
614
+ priorHandle: handle,
615
+ });
585
616
  if (!restored.ok) {
586
- failResume(`resumeVM failed (${restored.error.code}): ${restored.error.message}`, restored.error);
617
+ failResume(`resumeVM failed after ${restoreAttempts} attempt(s) (${restored.error.code}): ${restored.error.message}`, restored.error, remoteEnvFailureNote("resumeVM", restored.error, restoreAttempts));
587
618
  }
588
619
  else {
620
+ const restoredEnv = ownedEnv;
621
+ const canonicalInEnv = async (p) => {
622
+ try {
623
+ const r = await restoredEnv.canonicalPath(p, restoreSignal);
624
+ return r.ok ? r.value : undefined;
625
+ }
626
+ catch {
627
+ return undefined;
628
+ }
629
+ };
630
+ let checkpointedCanonical;
631
+ let sameRootUnderAlias = false;
589
632
  if (restored.value.mountPath !== handle.mountPath) {
633
+ checkpointedCanonical = await canonicalInEnv(handle.mountPath);
634
+ const restoredCanonical = await canonicalInEnv(restored.value.mountPath);
635
+ sameRootUnderAlias =
636
+ checkpointedCanonical !== undefined && restoredCanonical !== undefined && checkpointedCanonical === restoredCanonical;
637
+ }
638
+ if (restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
590
639
  if (resume.executesApprovedAction === true) {
591
640
  failResume(`resumeVM workspace-root divergence with a pending approved action: checkpointed mountPath "${handle.mountPath}" but the restored handle reports "${restored.value.mountPath}" — the approved args are bound to the checkpointed root; refusing to execute them against a moved workspace (adapter should honor priorHandle)`);
592
641
  }
@@ -599,13 +648,23 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
599
648
  if (restored.value.mountPath && restored.value.mountPath !== taskRootPath) {
600
649
  taskRootPath = restored.value.mountPath;
601
650
  }
602
- if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath) {
603
- restoredRootRebase = { from: handle.mountPath, to: restored.value.mountPath };
651
+ if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
652
+ const from = checkpointedCanonical !== undefined && checkpointedCanonical !== handle.mountPath
653
+ ? [handle.mountPath, checkpointedCanonical]
654
+ : [handle.mountPath];
655
+ restoredRootRebase = { from, to: restored.value.mountPath };
656
+ try {
657
+ deps.onError?.(new Error(from.length > 1
658
+ ? `resumeVM workspace-root rebase accepts both spellings of the checkpointed root (${from.map((f) => `"${f}"`).join(" and ")}) when migrating persisted paths to "${restored.value.mountPath}"`
659
+ : `resumeVM workspace-root rebase is spelling-exact: it matches the checkpointed root "${handle.mountPath}" only, so persisted paths recorded under an equivalent alias of it are NOT migrated to "${restored.value.mountPath}" and stay as written`), { phase: "config", sessionId });
660
+ }
661
+ catch {
662
+ }
604
663
  }
605
664
  }
606
665
  const init = await ownedEnv.postResumeInit();
607
666
  if (!init.ok) {
608
- failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error);
667
+ failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error, remoteEnvFailureNote("postResumeInit", init.error, 1));
609
668
  }
610
669
  }
611
670
  }
@@ -948,6 +1007,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
948
1007
  }
949
1008
  const suspendRef = {};
950
1009
  const reviewRef = {};
1010
+ const remoteEnvFailures = [];
951
1011
  const suspendLoopRef = { hit: false };
952
1012
  const compactionReuseRef = { consecutive: 0 };
953
1013
  const trimPressureRef = { droppedMessages: false };
@@ -2541,16 +2601,21 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2541
2601
  const preToolContexts = new Map();
2542
2602
  const blockedToolCalls = new Set();
2543
2603
  const blockedTracked = Boolean(hooks?.postToolUse || hooks?.preToolUse || hooks?.postToolUseFailure || hooks?.postToolBatch);
2604
+ const restoreSurfaceGap = ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && ownedEnv.capabilities.suspendable ? missingRestoreSurface(ownedEnv) : [];
2605
+ const incompleteSuspendAdapter = restoreSurfaceGap.length > 0 ? restoreSurfaceGap : undefined;
2544
2606
  const resourceSuspendEligible = spec.resourceSuspend !== undefined &&
2545
2607
  (spec.checkpointStore ?? deps.checkpointStore) !== undefined &&
2546
2608
  !(offloadStore !== undefined && isVolatileOffloadStore(offloadStore)) &&
2547
- (ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv));
2609
+ (ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv)) &&
2610
+ incompleteSuspendAdapter === undefined;
2548
2611
  if (spec.resourceSuspend !== undefined && !resourceSuspendEligible) {
2549
2612
  const why = (spec.checkpointStore ?? deps.checkpointStore) === undefined
2550
2613
  ? "no CheckpointStore is wired"
2551
2614
  : offloadStore !== undefined && isVolatileOffloadStore(offloadStore)
2552
2615
  ? "tool-result offload uses the in-memory store (a resume needs durable results)"
2553
- : "the per-task execution env is not a RemoteExecutionEnv (it would be destroyed on suspend)";
2616
+ : incompleteSuspendAdapter !== undefined
2617
+ ? `the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")} (a snapshot nothing can restore is worse than no snapshot)`
2618
+ : "the per-task execution env is not a RemoteExecutionEnv (it would be destroyed on suspend)";
2554
2619
  deps.onError?.(new Error(`resourceSuspend is set but INACTIVE: ${why}; resource limits will hard-fail, not suspend`), {
2555
2620
  phase: "config",
2556
2621
  sessionId,
@@ -2726,14 +2791,18 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2726
2791
  catch (putErr) {
2727
2792
  deps.onError?.(putErr, { phase: "config", sessionId });
2728
2793
  if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
2729
- const back = await remoteEnv.resumeVM(remoteHandle.snapshotId, { abortSignal: abortController.signal });
2794
+ const { outcome: back, attempts: backAttempts } = await restoreWorkspaceWithRetry(remoteEnv, remoteHandle.snapshotId, {
2795
+ abortSignal: abortController.signal,
2796
+ });
2730
2797
  if (back.ok) {
2731
2798
  const init = await remoteEnv.postResumeInit();
2732
2799
  if (init.ok)
2733
2800
  return false;
2801
+ remoteEnvFailures.push(remoteEnvFailureNote("postResumeInit", init.error, 1));
2734
2802
  deps.onError?.(init.error, { phase: "config", sessionId });
2735
2803
  }
2736
2804
  else {
2805
+ remoteEnvFailures.push(remoteEnvFailureNote("resumeVM", back.error, backAttempts));
2737
2806
  deps.onError?.(back.error, { phase: "config", sessionId });
2738
2807
  }
2739
2808
  abortController.abort();
@@ -2743,10 +2812,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2743
2812
  return false;
2744
2813
  }
2745
2814
  };
2746
- const publishCommittedSuspend = (token, gate, scope) => {
2815
+ const publishCommittedSuspend = (token, gate, scope, remoteHandle) => {
2747
2816
  const ref = gate.kind === "needs_review" || gate.kind === "plan_review" ? reviewRef : suspendRef;
2748
2817
  ref.token = token;
2749
2818
  ref.gate = gate;
2819
+ if (remoteHandle !== undefined)
2820
+ ref.restoreMode = remoteHandle.restoreMode === "park_only" ? "park_only" : "snapshot";
2750
2821
  ref.scope = scope;
2751
2822
  };
2752
2823
  const suspendLoopCapHit = (count, cap, detail) => {
@@ -2763,7 +2834,9 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2763
2834
  return true;
2764
2835
  };
2765
2836
  const suspendableEnv = ownedEnv !== undefined && isSuspendable(ownedEnv) ? ownedEnv : undefined;
2766
- const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) ? ownedEnv : undefined;
2837
+ const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && incompleteSuspendAdapter === undefined
2838
+ ? ownedEnv
2839
+ : undefined;
2767
2840
  const parkOnlyHandle = (env) => {
2768
2841
  const { snapshotId: _lineage, ...identity } = env.workspaceHandle();
2769
2842
  return { ...identity, restoreMode: "park_only" };
@@ -2788,6 +2861,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2788
2861
  await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
2789
2862
  const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
2790
2863
  if (!snap.ok) {
2864
+ remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
2791
2865
  deps.onError?.(new Error(`resource suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
2792
2866
  return false;
2793
2867
  }
@@ -2822,7 +2896,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2822
2896
  };
2823
2897
  if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
2824
2898
  return false;
2825
- publishCommittedSuspend(token, gate, rs.scope);
2899
+ publishCommittedSuspend(token, gate, rs.scope, remoteHandle);
2826
2900
  try {
2827
2901
  await sessions.pin?.(sessionId);
2828
2902
  }
@@ -2839,6 +2913,14 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2839
2913
  return false;
2840
2914
  if (abortController.signal.aborted)
2841
2915
  return false;
2916
+ if (incompleteSuspendAdapter !== undefined) {
2917
+ try {
2918
+ deps.onError?.(new Error(`plan_review park skipped: the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so its workspace could not be restored on resume (the plan-review request was dropped; the run continues without pausing)`), { phase: "config", sessionId });
2919
+ }
2920
+ catch {
2921
+ }
2922
+ return false;
2923
+ }
2842
2924
  if (suspendLoopCapHit(priorSuspendCount, maxSuspends, " for a plan_review (likely a resume/restart loop)."))
2843
2925
  return false;
2844
2926
  const leafId = await session.getLeafId();
@@ -2858,6 +2940,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2858
2940
  await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
2859
2941
  const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
2860
2942
  if (!snap.ok) {
2943
+ remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
2861
2944
  deps.onError?.(new Error(`plan_review suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
2862
2945
  return false;
2863
2946
  }
@@ -2896,7 +2979,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2896
2979
  };
2897
2980
  if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
2898
2981
  return false;
2899
- publishCommittedSuspend(token, gate, scope);
2982
+ publishCommittedSuspend(token, gate, scope, remoteHandle);
2900
2983
  try {
2901
2984
  await sessions.pin?.(sessionId);
2902
2985
  }
@@ -2930,10 +3013,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2930
3013
  "RunnerDeps.toolResultStore or disable offload.");
2931
3014
  }
2932
3015
  if (ownedEnv !== undefined && remoteEnv === undefined && parkOnlyRemoteEnv === undefined) {
2933
- throw new Error("durable suspend is not supported with a non-remote per-task executionEnvFactory env: the " +
2934
- "minted env is destroyed on suspend, so a resumed file/shell tool would act on a fresh " +
2935
- "(empty) env. Use a RemoteExecutionEnv factory (it is paused, not destroyed) or a static, " +
2936
- "caller-owned RunnerDeps.executionEnv.");
3016
+ throw new Error(incompleteSuspendAdapter !== undefined
3017
+ ? `durable suspend is not supported with this per-task executionEnvFactory env: it declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so a snapshot taken now could never be restored. Implement the full RemoteExecutionEnv restore surface, or declare capabilities.suspendable:false if the workspace is externally durable (the park-only lane).`
3018
+ : "durable suspend is not supported with a non-remote per-task executionEnvFactory env: the " +
3019
+ "minted env is destroyed on suspend, so a resumed file/shell tool would act on a fresh " +
3020
+ "(empty) env. Use a RemoteExecutionEnv factory (it is paused, not destroyed) or a static, " +
3021
+ "caller-owned RunnerDeps.executionEnv.");
2937
3022
  }
2938
3023
  const leafId = await session.getLeafId();
2939
3024
  if (!leafId) {
@@ -2951,6 +3036,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
2951
3036
  await sweepBackgroundShells(remoteEnv, defaultTaskRegistry);
2952
3037
  const snap = await remoteEnv.suspendVM({ abortSignal: abortController.signal });
2953
3038
  if (!snap.ok) {
3039
+ remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
2954
3040
  throw new Error(`suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error });
2955
3041
  }
2956
3042
  remoteHandle = { ...remoteEnv.workspaceHandle(), snapshotId: snap.value };
@@ -3039,7 +3125,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
3039
3125
  if (!(await commitSuspendSaga(token, cp, remoteEnv, remoteHandle))) {
3040
3126
  return undefined;
3041
3127
  }
3042
- publishCommittedSuspend(token, gate, cp.scope);
3128
+ publishCommittedSuspend(token, gate, cp.scope, remoteHandle);
3043
3129
  try {
3044
3130
  await sessions.pin?.(sessionId);
3045
3131
  }
@@ -3388,7 +3474,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
3388
3474
  : undefined;
3389
3475
  overheadState.promptChars = systemPrompt.length;
3390
3476
  const preparedHolder = {};
3391
- const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
3477
+ const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, remoteEnvFailures, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
3392
3478
  const prepared = buildPrepared();
3393
3479
  preparedHolder.current = prepared;
3394
3480
  return prepared;
@@ -1345,6 +1345,16 @@ export class Runner {
1345
1345
  queue.close();
1346
1346
  return;
1347
1347
  }
1348
+ const remoteEnvFailure = (() => {
1349
+ let cur = err;
1350
+ for (let depth = 0; cur && typeof cur === "object" && depth < 8; depth++) {
1351
+ const note = cur.remoteEnvFailure;
1352
+ if (note !== undefined)
1353
+ return [note];
1354
+ cur = cur.cause;
1355
+ }
1356
+ return undefined;
1357
+ })();
1348
1358
  resultValue = {
1349
1359
  taskId: taskIdRef.current ?? "unknown",
1350
1360
  sessionId: taskIdRef.sessionId ?? "unknown",
@@ -1352,6 +1362,7 @@ export class Runner {
1352
1362
  result: "",
1353
1363
  errorMessage: err instanceof Error ? err.message : String(err),
1354
1364
  errorCode: code,
1365
+ ...(remoteEnvFailure !== undefined ? { remoteEnvFailures: remoteEnvFailure } : {}),
1355
1366
  stats: { turns: 0, tokens: 0, toolCalls: 0, cachedTokens: 0, costMicroUsd: 0 },
1356
1367
  };
1357
1368
  emitTrace(spec.tracer ?? this.deps.tracer, () => ({
@@ -2765,6 +2776,7 @@ export class Runner {
2765
2776
  model: prepared.model.id,
2766
2777
  unpricedSpend: rs.telemetry.unpricedSpend,
2767
2778
  rewindNotes: prepared.rewindNotes,
2779
+ remoteEnvFailures: prepared.remoteEnvFailures,
2768
2780
  abortedForTimeout: timeout.fired,
2769
2781
  abortedForTurns: rs.limits.turnsExceeded,
2770
2782
  abortedLive,
@@ -2774,10 +2786,18 @@ export class Runner {
2774
2786
  outputInvalid: rs.degrade.outputInvalid,
2775
2787
  suspendLoop: prepared.suspendLoopRef.hit,
2776
2788
  suspendRef: prepared.suspendRef.token !== undefined && prepared.suspendRef.gate !== undefined
2777
- ? { token: prepared.suspendRef.token, gate: prepared.suspendRef.gate }
2789
+ ? {
2790
+ token: prepared.suspendRef.token,
2791
+ gate: prepared.suspendRef.gate,
2792
+ ...(prepared.suspendRef.restoreMode !== undefined ? { restoreMode: prepared.suspendRef.restoreMode } : {}),
2793
+ }
2778
2794
  : undefined,
2779
2795
  reviewRef: prepared.reviewRef.token !== undefined && prepared.reviewRef.gate !== undefined
2780
- ? { token: prepared.reviewRef.token, gate: prepared.reviewRef.gate }
2796
+ ? {
2797
+ token: prepared.reviewRef.token,
2798
+ gate: prepared.reviewRef.gate,
2799
+ ...(prepared.reviewRef.restoreMode !== undefined ? { restoreMode: prepared.reviewRef.restoreMode } : {}),
2800
+ }
2781
2801
  : undefined,
2782
2802
  });
2783
2803
  if (resume !== undefined &&
@@ -370,6 +370,13 @@ export interface TaskSpec {
370
370
  signal?: AbortSignal;
371
371
  }
372
372
  export type TaskStatus = "completed" | "blocked" | "failed" | "timeout" | "suspended" | "needs_review";
373
+ export interface RemoteEnvFailureNote {
374
+ op: "suspendVM" | "resumeVM" | "postResumeInit";
375
+ code: import("./remote-env.js").RemoteExecutionErrorCode;
376
+ retryable: boolean;
377
+ attempts: number;
378
+ message: string;
379
+ }
373
380
  export interface TaskResult {
374
381
  taskId: string;
375
382
  sessionId: string;
@@ -380,6 +387,8 @@ export interface TaskResult {
380
387
  blockedReason?: string;
381
388
  checkpointToken?: import("./checkpoint-store.js").CheckpointToken;
382
389
  checkpointGate?: import("./checkpoint-store.js").CheckpointGate;
390
+ workspaceRestoreMode?: "snapshot" | "park_only";
391
+ remoteEnvFailures?: RemoteEnvFailureNote[];
383
392
  errorMessage?: string;
384
393
  errorCode?: string;
385
394
  degraded?: {
@@ -47,7 +47,7 @@ export declare class FileError extends Error {
47
47
  path?: string;
48
48
  constructor(code: FileErrorCode, message: string, path?: string, cause?: Error);
49
49
  }
50
- export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "unknown";
50
+ export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "suspended" | "auth_failed" | "unknown";
51
51
  export declare class ExecutionError extends Error {
52
52
  code: ExecutionErrorCode;
53
53
  partialStdout?: string;
package/dist/index.d.ts CHANGED
@@ -56,7 +56,7 @@ export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-e
56
56
  export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
57
57
  export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
58
58
  export type { ExecutionEnv, FileInfo, Result, FileErrorCode, ExecutionErrorCode } from "./internal/harness.js";
59
- export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
59
+ export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
60
60
  export { withRetry } from "./core/with-retry.js";
61
61
  export type { RetryPolicy, RetryResult } from "./core/with-retry.js";
62
62
  export type { RemoteExecutionEnv, WorkspaceHandle, SnapshotId, SessionToken, SandboxTier, OutputChunk, ExecStreamOptions, RemoteConnectConfig, VmLifecycleOptions, SecretRef, RemoteExecutionErrorCode, ExecutionEnvFactory, ExecutionEnvFactoryContext, } from "./core/remote-env.js";
@@ -189,7 +189,7 @@ export { retryBackoffMs, parseRetryAfter } from "./brain/retry.js";
189
189
  export { type BrainTimeoutConfig } from "./brain/timeout.js";
190
190
  export { createAssistantMessageEventStream } from "./internal/llm.js";
191
191
  export type { AssistantMessage, AssistantMessageEvent, Context, DocumentContent, ImageContent, Message, StopReason, StreamFn, TextContent, ThinkingContent, ToolCall, ToolResultMessage, Usage, UserMessage, } from "./internal/llm.js";
192
- export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
192
+ export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, RemoteEnvFailureNote, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
193
193
  export { Type } from "typebox";
194
194
  export type { TSchema, Static } from "typebox";
195
195
  export { explainPromptAssembly, describeDefaultPack, type DefaultPackDescription, type ExplainInput } from "./prompt-assembly/explain.js";
package/dist/index.js CHANGED
@@ -51,7 +51,7 @@ export { killProcessTree, signalProcessTree } from "./engine/execution-env/kill-
51
51
  export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-execution-env.js";
52
52
  export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
53
53
  export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
54
- export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
54
+ export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
55
55
  export { withRetry } from "./core/with-retry.js";
56
56
  export { addWorktree, pruneWorktrees, WORKTREE_PARENT } from "./core/git-worktree-env.js";
57
57
  export { runExecGate } from "./core/exec-gate.js";
@@ -229,6 +229,20 @@ async function runShell(env, cwd, toolName, command, timeoutSec, caps, signal, c
229
229
  isError: true,
230
230
  };
231
231
  }
232
+ if (res.error.code === "suspended") {
233
+ return {
234
+ content: `Error (${toolName}): the execution environment is suspended (its workspace VM is paused), so the command did NOT run and no command can run in this leg. Do not retry it here — the task must be resumed first; report that the workspace is suspended. (${res.error.message})`,
235
+ details: { type: "bash", envSuspended: true },
236
+ isError: true,
237
+ };
238
+ }
239
+ if (res.error.code === "auth_failed") {
240
+ return {
241
+ content: `Error (${toolName}): authentication to the execution environment was rejected, so the command did NOT run. Do NOT retry — this is a permanent credential failure that an operator must fix; report it instead of trying other commands. (${res.error.message})`,
242
+ details: { type: "bash", authFailed: true },
243
+ isError: true,
244
+ };
245
+ }
232
246
  if (res.error.code === "timeout" || res.error.code === "aborted" || res.error.code === "callback_error") {
233
247
  const rawStdout = res.error.partialStdout ?? "";
234
248
  const rawStderr = res.error.partialStderr ?? "";
@@ -83,16 +83,13 @@ function expandCronField(field, { min, max }) {
83
83
  return out.size > 0 ? [...out].sort((a, b) => a - b) : null;
84
84
  }
85
85
  export function cronScheduleError(expr) {
86
- const parts = expr.trim().split(/\s+/);
87
- let std = parts;
88
- if (parts.length === 6) {
89
- if (!expandCronField(parts[0] ?? "", { min: 0, max: 59 })) {
90
- return `invalid seconds field "${parts[0]}" — allowed 0-59 (syntax: N, N-M, */N, comma lists).`;
91
- }
92
- std = parts.slice(1);
86
+ const std = expr.trim().split(/\s+/);
87
+ if (std.length === 6) {
88
+ return (`its leading seconds field is not supported — the resident scheduler fires on 5-field cron, so one ` +
89
+ `minute is the finest granularity available. Pass "${std.slice(1).join(" ")}" if that cadence works.`);
93
90
  }
94
- else if (parts.length !== 5) {
95
- return `expected 5 fields (minute hour day-of-month month day-of-week) or 6 with leading seconds, got ${parts.length}.`;
91
+ if (std.length !== 5) {
92
+ return `expected 5 fields (minute hour day-of-month month day-of-week), got ${std.length}.`;
96
93
  }
97
94
  const expanded = [];
98
95
  for (let i = 0; i < 5; i++) {
@@ -117,9 +114,12 @@ export function cronScheduleError(expr) {
117
114
  return null;
118
115
  }
119
116
  const DAY_NAMES = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
117
+ function cronExprFromSummary(when) {
118
+ const trimmed = when.trim();
119
+ return trimmed.startsWith("cron ") ? trimmed.slice("cron ".length).trim() : trimmed;
120
+ }
120
121
  export function cronToHuman(expr) {
121
- const parts = expr.trim().split(/\s+/);
122
- const std = parts.length === 6 ? parts.slice(1) : parts;
122
+ const std = cronExprFromSummary(expr).split(/\s+/);
123
123
  if (std.length !== 5)
124
124
  return expr;
125
125
  const [minute = "", hour = "", dom = "", month = "", dow = ""] = std;
@@ -191,7 +191,10 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
191
191
  parameters: Type.Object({
192
192
  prompt: Type.String({ description: "Self-contained instructions for the future task (it runs unattended)." }),
193
193
  schedule: Type.Optional(Type.Union([
194
- Type.Object({ kind: Type.Literal("cron"), expr: Type.String({ description: "5- or 6-field cron expression." }) }),
194
+ Type.Object({
195
+ kind: Type.Literal("cron"),
196
+ expr: Type.String({ description: "5-field cron expression (minute hour day-of-month month day-of-week)." }),
197
+ }),
195
198
  Type.Object({ kind: Type.Literal("at"), atMs: Type.Number({ description: "Absolute epoch ms to fire once." }) }),
196
199
  Type.Object({ kind: Type.Literal("delay"), delaySec: Type.Number({ description: "Seconds from now to fire once." }) }),
197
200
  ], { description: 'When to fire. Pass exactly one of `schedule` or `cron`. Defaults to `durable: true`.' })),
@@ -246,22 +249,32 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
246
249
  ...(!durable ? { lifetime: "session" } : {}),
247
250
  ...(a.recurring !== undefined ? { recurring: a.recurring } : {}),
248
251
  };
252
+ const before = await sched.list(schedCtx);
253
+ const knownIds = before.ok ? new Set(before.value.map((s) => s.id)) : undefined;
249
254
  const r = await sched.schedule(intent, schedCtx);
250
255
  if (!r.ok)
251
256
  return errorResult(`Error (CronCreate): ${r.error.message}`);
257
+ const replaced = knownIds?.has(r.value.id);
252
258
  const humanSchedule = when.kind === "cron"
253
259
  ? cronToHuman(when.expr)
254
260
  : when.kind === "delay"
255
261
  ? `once in ${when.delaySec}s`
256
262
  : `once at ${new Date(when.atMs).toISOString()}`;
263
+ const named = `${r.value.id}${a.label ? ` (${a.label})` : ""}`;
264
+ const content = replaced === true
265
+ ? `Updated scheduled task ${named} — it replaced an existing job with the same schedule and label, whose prompt is now gone.`
266
+ : replaced === false
267
+ ? `Scheduled task ${named}.`
268
+ : `Scheduled (or updated) task ${named} — the scheduler's listing was unavailable, so whether this replaced an existing job with the same schedule and label is unknown.`;
257
269
  return {
258
- content: `Scheduled task ${r.value.id}${a.label ? ` (${a.label})` : ""}.`,
270
+ content,
259
271
  details: {
260
272
  type: "cron-create",
261
273
  id: r.value.id,
262
274
  humanSchedule,
263
275
  recurring: when.kind === "cron" && a.recurring !== false,
264
276
  durable,
277
+ ...(replaced !== undefined ? { replaced } : {}),
265
278
  },
266
279
  };
267
280
  },
@@ -298,17 +311,18 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
298
311
  }
299
312
  const content = r.value
300
313
  .map((s) => {
314
+ const shown = cronExprFromSummary(s.when);
301
315
  const human = cronToHuman(s.when);
302
316
  const tier = s.lifetime === "session" ? " [session]" : "";
303
317
  const once = s.recurring === false ? " [once]" : "";
304
- return `- ${s.id}: ${s.when}${human !== s.when ? ` — ${human}` : ""}${s.label ? ` (${s.label})` : ""}${tier}${once}`;
318
+ return `- ${s.id}: ${shown}${human !== s.when ? ` — ${human}` : ""}${s.label ? ` (${s.label})` : ""}${tier}${once}`;
305
319
  })
306
320
  .join("\n");
307
321
  const jobs = r.value.map((s) => {
308
322
  const human = cronToHuman(s.when);
309
323
  return {
310
324
  id: s.id,
311
- cron: s.when,
325
+ cron: cronExprFromSummary(s.when),
312
326
  humanSchedule: human !== s.when ? human : s.when,
313
327
  ...(s.label !== undefined ? { label: s.label } : {}),
314
328
  ...(s.recurring !== undefined ? { recurring: s.recurring } : {}),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sema-agent/core",
3
- "version": "2.11.0",
3
+ "version": "2.12.0",
4
4
  "description": "Stateless, task-oriented AI agent core",
5
5
  "type": "module",
6
6
  "license": "BUSL-1.1",