@sema-agent/core 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/checkpoint-store.d.ts +1 -0
- package/dist/core/checkpoint-store.js +3 -0
- package/dist/core/mcp.d.ts +2 -0
- package/dist/core/mcp.js +89 -6
- package/dist/core/remote-env.d.ts +3 -0
- package/dist/core/remote-env.js +19 -1
- package/dist/core/runner/assemble-result.d.ts +3 -0
- package/dist/core/runner/assemble-result.js +4 -1
- package/dist/core/runner/prepare-task.d.ts +4 -0
- package/dist/core/runner/prepare-task.js +108 -22
- package/dist/core/runner/runtask.js +22 -2
- package/dist/core/types.d.ts +9 -0
- package/dist/engine/harness/types.d.ts +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/tools/fs/fs-bash.js +14 -0
- package/dist/tools/scheduler-tools.js +29 -15
- package/package.json +1 -1
|
@@ -242,6 +242,7 @@ export interface CheckpointSummary {
|
|
|
242
242
|
contentKind?: "content_ask";
|
|
243
243
|
createdAt?: number;
|
|
244
244
|
toolInput?: string;
|
|
245
|
+
restoreMode?: "snapshot" | "park_only";
|
|
245
246
|
preview?: unknown;
|
|
246
247
|
}
|
|
247
248
|
export declare function summarizeCheckpoint(cp: Checkpoint): CheckpointSummary;
|
|
@@ -181,6 +181,9 @@ export function summarizeCheckpoint(cp) {
|
|
|
181
181
|
...(tool !== undefined ? { toolCallId: tool.toolCallId, toolName: tool.toolName } : {}),
|
|
182
182
|
...(tool?.toolName === ASK_USER_QUESTION_TOOL_NAME ? { contentKind: "content_ask" } : {}),
|
|
183
183
|
...(toolInput !== undefined ? { toolInput } : {}),
|
|
184
|
+
...(cp.state.workspaceHandle !== undefined
|
|
185
|
+
? { restoreMode: cp.state.workspaceHandle.restoreMode === "park_only" ? "park_only" : "snapshot" }
|
|
186
|
+
: {}),
|
|
184
187
|
...(tool?.preview !== undefined ? { preview: tool.preview } : {}),
|
|
185
188
|
};
|
|
186
189
|
}
|
package/dist/core/mcp.d.ts
CHANGED
|
@@ -57,6 +57,7 @@ export interface McpServerStatus {
|
|
|
57
57
|
};
|
|
58
58
|
toolNames?: string[];
|
|
59
59
|
error?: string;
|
|
60
|
+
transportClosed?: boolean;
|
|
60
61
|
}
|
|
61
62
|
export declare const MCP_PREFIX = "mcp__";
|
|
62
63
|
export declare function resolveMcpDeclaredResultSize(meta: Record<string, unknown> | undefined): number | undefined;
|
|
@@ -69,6 +70,7 @@ export declare const MCP_IDLE_TIMEOUT_STDIO_DEFAULT_MS: number;
|
|
|
69
70
|
export declare const MCP_IDLE_TIMEOUT_HTTP_DEFAULT_MS: number;
|
|
70
71
|
export declare function mcpIdleTimeoutMs(kind: "stdio" | "http"): number;
|
|
71
72
|
export declare function describeMcpSpecErrorCode(code: unknown): string | undefined;
|
|
73
|
+
export declare function collapseMcpErrorPrefix(message: string): string;
|
|
72
74
|
export declare function normalizeMcpName(name: string): string;
|
|
73
75
|
export declare function clampNameSegment(seg: string, max?: number): string;
|
|
74
76
|
export * from "./image-downsample.js";
|
package/dist/core/mcp.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
3
3
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
4
|
-
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
4
|
+
import { StreamableHTTPClientTransport, StreamableHTTPError } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
5
5
|
import { lstat, mkdir, writeFile } from "node:fs/promises";
|
|
6
6
|
import { tmpdir } from "node:os";
|
|
7
7
|
import { join } from "node:path";
|
|
@@ -145,8 +145,24 @@ const MCP_SPEC_ERROR_CODE_NAMES = new Map([
|
|
|
145
145
|
export function describeMcpSpecErrorCode(code) {
|
|
146
146
|
return typeof code === "number" ? MCP_SPEC_ERROR_CODE_NAMES.get(code) : undefined;
|
|
147
147
|
}
|
|
148
|
+
export function collapseMcpErrorPrefix(message) {
|
|
149
|
+
let out = message;
|
|
150
|
+
for (;;) {
|
|
151
|
+
const m = /^MCP error (-?\d+): (?=MCP error \1: )/.exec(out);
|
|
152
|
+
if (m === null)
|
|
153
|
+
return out;
|
|
154
|
+
out = out.slice(m[0].length);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function collapseMcpErrorStampInPlace(err) {
|
|
158
|
+
if (!(err instanceof McpError))
|
|
159
|
+
return;
|
|
160
|
+
const collapsed = collapseMcpErrorPrefix(err.message);
|
|
161
|
+
if (collapsed !== err.message)
|
|
162
|
+
err.message = collapsed;
|
|
163
|
+
}
|
|
148
164
|
function namedMcpFailureText(err) {
|
|
149
|
-
const detail = err instanceof Error ? err.message : String(err);
|
|
165
|
+
const detail = err instanceof Error ? collapseMcpErrorPrefix(err.message) : String(err);
|
|
150
166
|
const condition = err instanceof McpError ? describeMcpSpecErrorCode(err.code) : undefined;
|
|
151
167
|
return condition !== undefined ? `${condition}: ${detail}` : detail;
|
|
152
168
|
}
|
|
@@ -155,6 +171,53 @@ function isTransportLost(err) {
|
|
|
155
171
|
return true;
|
|
156
172
|
return err instanceof Error && /not connected|connection closed/i.test(err.message);
|
|
157
173
|
}
|
|
174
|
+
const NETWORK_CODES_NEVER_DELIVERED = new Set([
|
|
175
|
+
"ECONNREFUSED",
|
|
176
|
+
"ENOTFOUND",
|
|
177
|
+
"EAI_AGAIN",
|
|
178
|
+
"EHOSTUNREACH",
|
|
179
|
+
"ENETUNREACH",
|
|
180
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
181
|
+
]);
|
|
182
|
+
function networkErrorCode(err, depth = 0) {
|
|
183
|
+
if (depth > 5 || !(err instanceof Error))
|
|
184
|
+
return undefined;
|
|
185
|
+
const code = err.code;
|
|
186
|
+
if (typeof code === "string" && /^(?:E[A-Z]+|UND_ERR_[A-Z_]+)$/.test(code))
|
|
187
|
+
return code;
|
|
188
|
+
if (err instanceof AggregateError) {
|
|
189
|
+
for (const inner of err.errors) {
|
|
190
|
+
const found = networkErrorCode(inner, depth + 1);
|
|
191
|
+
if (found !== undefined)
|
|
192
|
+
return found;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return networkErrorCode(err.cause, depth + 1);
|
|
196
|
+
}
|
|
197
|
+
function describeHttpTransportFailure(err) {
|
|
198
|
+
if (err instanceof McpError)
|
|
199
|
+
return undefined;
|
|
200
|
+
if (err instanceof StreamableHTTPError) {
|
|
201
|
+
const status = typeof err.code === "number" && err.code > 0 ? err.code : undefined;
|
|
202
|
+
return {
|
|
203
|
+
condition: status !== undefined
|
|
204
|
+
? `its HTTP endpoint answered ${status} instead of an MCP response`
|
|
205
|
+
: "its HTTP endpoint answered something that is not an MCP response",
|
|
206
|
+
delivered: "unknown",
|
|
207
|
+
...(status !== undefined ? { httpStatus: status } : {}),
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
const code = networkErrorCode(err);
|
|
211
|
+
if (code !== undefined) {
|
|
212
|
+
return NETWORK_CODES_NEVER_DELIVERED.has(code)
|
|
213
|
+
? { condition: `its HTTP endpoint could not be reached (${code})`, delivered: "no" }
|
|
214
|
+
: { condition: `the connection to its HTTP endpoint failed (${code})`, delivered: "unknown" };
|
|
215
|
+
}
|
|
216
|
+
if (err instanceof TypeError && /fetch failed|terminated|network/i.test(err.message)) {
|
|
217
|
+
return { condition: "the HTTP request to its endpoint failed at the network layer", delivered: "unknown" };
|
|
218
|
+
}
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
158
221
|
function writeEffectWarning(writeEffect) {
|
|
159
222
|
return writeEffect
|
|
160
223
|
? " This tool is write-capable: treat its side effects as POSSIBLY APPLIED and verify the actual state before retrying."
|
|
@@ -173,6 +236,7 @@ function rethrowHonestMcpError(err, ctx) {
|
|
|
173
236
|
}
|
|
174
237
|
if (ctx.signal?.aborted)
|
|
175
238
|
throw err;
|
|
239
|
+
collapseMcpErrorStampInPlace(err);
|
|
176
240
|
const serverLabel = inlineUntrusted(ctx.server);
|
|
177
241
|
if (err instanceof McpError && err.code === ErrorCode.RequestTimeout) {
|
|
178
242
|
const data = err.data;
|
|
@@ -193,6 +257,21 @@ function rethrowHonestMcpError(err, ctx) {
|
|
|
193
257
|
e.details = { transportLost: true, server: ctx.server };
|
|
194
258
|
throw e;
|
|
195
259
|
}
|
|
260
|
+
const httpFailure = describeHttpTransportFailure(err);
|
|
261
|
+
if (httpFailure !== undefined) {
|
|
262
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
263
|
+
const fenced = `\nThe transport error follows as external/untrusted data:\n${delimitUntrusted(`${ctx.server} transport error`, truncateMcpErrorText(detail))}`;
|
|
264
|
+
const e = new Error(httpFailure.delivered === "no"
|
|
265
|
+
? `${ctx.what} could not reach MCP server "${serverLabel}": ${httpFailure.condition}. The request was not delivered, so the server did not execute it. This server's tools and resources will keep failing until its endpoint is reachable again — do not retry them; use an alternative if one exists.${fenced}`
|
|
266
|
+
: `${ctx.what} failed at the transport layer of MCP server "${serverLabel}": ${httpFailure.condition}. The request may or may not have executed on the server — the outcome is unknown.${writeEffectWarning(ctx.writeEffect)}${fenced}`, { cause: err });
|
|
267
|
+
e.errorKind = httpFailure.delivered === "no" ? "server_disconnected" : "transport_lost";
|
|
268
|
+
e.details = {
|
|
269
|
+
server: ctx.server,
|
|
270
|
+
...(httpFailure.delivered === "no" ? {} : { transportLost: true }),
|
|
271
|
+
...(httpFailure.httpStatus !== undefined ? { httpStatus: httpFailure.httpStatus } : {}),
|
|
272
|
+
};
|
|
273
|
+
throw e;
|
|
274
|
+
}
|
|
196
275
|
if (err instanceof McpError) {
|
|
197
276
|
const condition = describeMcpSpecErrorCode(err.code);
|
|
198
277
|
if (condition !== undefined) {
|
|
@@ -546,12 +625,16 @@ export async function materializeMcpTools(specs, principal, onElicit, imageResiz
|
|
|
546
625
|
serverInstructions.push({ server: spec.name, text: s.instructions });
|
|
547
626
|
}
|
|
548
627
|
const hasInstructions = s.instructions !== undefined;
|
|
549
|
-
|
|
550
|
-
if (
|
|
628
|
+
const announceClose = () => {
|
|
629
|
+
if (disposing)
|
|
630
|
+
return;
|
|
631
|
+
s.status.transportClosed = true;
|
|
632
|
+
if (hasInstructions)
|
|
551
633
|
instructionsDelta.pendingRemovals.push(spec.name);
|
|
552
634
|
};
|
|
553
|
-
|
|
554
|
-
|
|
635
|
+
s.announce.fn = announceClose;
|
|
636
|
+
if (s.health.dead)
|
|
637
|
+
announceClose();
|
|
555
638
|
if (s.resourceServer)
|
|
556
639
|
resourceServers.push(s.resourceServer);
|
|
557
640
|
for (const d of s.dropped)
|
|
@@ -44,6 +44,8 @@ export interface ExecStreamOptions {
|
|
|
44
44
|
maxOutputBytes?: number;
|
|
45
45
|
}
|
|
46
46
|
export type RemoteExecutionErrorCode = "suspended" | "command_in_flight" | "connect_failed" | "post_resume_failed" | "aborted" | "timeout" | "unsupported" | "auth_transient" | "auth_failed" | "transport_lost" | "unknown";
|
|
47
|
+
export declare const RETRYABLE_REMOTE_ERROR_CODES: readonly RemoteExecutionErrorCode[];
|
|
48
|
+
export declare function isRetryableRemoteErrorCode(code: RemoteExecutionErrorCode): boolean;
|
|
47
49
|
export declare class RemoteExecutionError extends Error {
|
|
48
50
|
readonly code: RemoteExecutionErrorCode;
|
|
49
51
|
constructor(code: RemoteExecutionErrorCode, message: string, cause?: Error);
|
|
@@ -101,5 +103,6 @@ export interface ExecutionEnvFactoryContext {
|
|
|
101
103
|
export type ExecutionEnvFactory = (ctx: ExecutionEnvFactoryContext) => ExecutionEnv | Promise<ExecutionEnv>;
|
|
102
104
|
export declare function hasDestroy(env: ExecutionEnv): env is ExecutionEnv & Pick<RemoteExecutionEnv, "destroy">;
|
|
103
105
|
export declare function isRemoteExecutionEnv(env: ExecutionEnv): env is RemoteExecutionEnv;
|
|
106
|
+
export declare function missingRestoreSurface(env: ExecutionEnv): readonly ("resumeVM" | "postResumeInit")[];
|
|
104
107
|
export declare function isSuspendable(env: ExecutionEnv): env is RemoteExecutionEnv;
|
|
105
108
|
export declare function isIsolated(env: ExecutionEnv): boolean;
|
package/dist/core/remote-env.js
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
export const RETRYABLE_REMOTE_ERROR_CODES = [
|
|
2
|
+
"auth_transient",
|
|
3
|
+
"connect_failed",
|
|
4
|
+
"timeout",
|
|
5
|
+
"transport_lost",
|
|
6
|
+
];
|
|
7
|
+
export function isRetryableRemoteErrorCode(code) {
|
|
8
|
+
return RETRYABLE_REMOTE_ERROR_CODES.includes(code);
|
|
9
|
+
}
|
|
1
10
|
export class RemoteExecutionError extends Error {
|
|
2
11
|
code;
|
|
3
12
|
constructor(code, message, cause) {
|
|
@@ -18,8 +27,17 @@ export function isRemoteExecutionEnv(env) {
|
|
|
18
27
|
typeof e.capabilities === "object" &&
|
|
19
28
|
e.capabilities !== null);
|
|
20
29
|
}
|
|
30
|
+
export function missingRestoreSurface(env) {
|
|
31
|
+
const e = env;
|
|
32
|
+
const missing = [];
|
|
33
|
+
if (typeof e.resumeVM !== "function")
|
|
34
|
+
missing.push("resumeVM");
|
|
35
|
+
if (typeof e.postResumeInit !== "function")
|
|
36
|
+
missing.push("postResumeInit");
|
|
37
|
+
return missing;
|
|
38
|
+
}
|
|
21
39
|
export function isSuspendable(env) {
|
|
22
|
-
return isRemoteExecutionEnv(env) && env.capabilities.suspendable;
|
|
40
|
+
return isRemoteExecutionEnv(env) && env.capabilities.suspendable && missingRestoreSurface(env).length === 0;
|
|
23
41
|
}
|
|
24
42
|
export function isIsolated(env) {
|
|
25
43
|
return isRemoteExecutionEnv(env) ? env.capabilities.isolation : false;
|
|
@@ -70,6 +70,7 @@ export interface ResultFlags {
|
|
|
70
70
|
model?: string;
|
|
71
71
|
unpricedSpend?: boolean;
|
|
72
72
|
rewindNotes?: TaskResult["rewindNotes"];
|
|
73
|
+
remoteEnvFailures?: TaskResult["remoteEnvFailures"];
|
|
73
74
|
abortedForTimeout: boolean;
|
|
74
75
|
abortedForTurns: boolean;
|
|
75
76
|
abortedLive?: boolean;
|
|
@@ -81,10 +82,12 @@ export interface ResultFlags {
|
|
|
81
82
|
suspendRef?: {
|
|
82
83
|
token: import("../checkpoint-store.js").CheckpointToken;
|
|
83
84
|
gate: import("../checkpoint-store.js").CheckpointGate;
|
|
85
|
+
restoreMode?: "snapshot" | "park_only";
|
|
84
86
|
};
|
|
85
87
|
reviewRef?: {
|
|
86
88
|
token: import("../checkpoint-store.js").CheckpointToken;
|
|
87
89
|
gate: import("../checkpoint-store.js").CheckpointGate;
|
|
90
|
+
restoreMode?: "snapshot" | "park_only";
|
|
88
91
|
};
|
|
89
92
|
}
|
|
90
93
|
export declare function errorCodeOf(err: unknown): string | undefined;
|
|
@@ -54,6 +54,7 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
|
|
|
54
54
|
let salvagedOutput;
|
|
55
55
|
let checkpointToken;
|
|
56
56
|
let checkpointGate;
|
|
57
|
+
let workspaceRestoreMode;
|
|
57
58
|
const isDegenerate = final?.stopReason === "error" && final.errorMessage === DEGENERATE_MESSAGE;
|
|
58
59
|
const isWalltimeCutoff = final?.stopReason === "error" && final.errorMessage === WALLTIME_CUTOFF_MESSAGE;
|
|
59
60
|
if (flags.outputInvalid) {
|
|
@@ -105,12 +106,14 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
|
|
|
105
106
|
status = "suspended";
|
|
106
107
|
checkpointToken = flags.suspendRef.token;
|
|
107
108
|
checkpointGate = flags.suspendRef.gate;
|
|
109
|
+
workspaceRestoreMode = flags.suspendRef.restoreMode;
|
|
108
110
|
}
|
|
109
111
|
else if (flags.reviewRef) {
|
|
110
112
|
status = "needs_review";
|
|
111
113
|
errorCode = "review.pending";
|
|
112
114
|
checkpointToken = flags.reviewRef.token;
|
|
113
115
|
checkpointGate = flags.reviewRef.gate;
|
|
116
|
+
workspaceRestoreMode = flags.reviewRef.restoreMode;
|
|
114
117
|
}
|
|
115
118
|
else if (flags.abortedLive || final?.stopReason === "aborted") {
|
|
116
119
|
status = flags.abortedForTimeout ? "timeout" : "failed";
|
|
@@ -143,5 +146,5 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
|
|
|
143
146
|
void _internalCompaction;
|
|
144
147
|
if (flags.unpricedSpend)
|
|
145
148
|
delete publicStats.costMicroUsd;
|
|
146
|
-
return { taskId, sessionId, status, ...(flags.model !== undefined ? { model: flags.model } : {}), result: result.trim(), salvagedOutput, blockedReason, errorMessage, errorCode, checkpointToken, checkpointGate, ...(flags.rewindNotes !== undefined && flags.rewindNotes.length > 0 ? { rewindNotes: flags.rewindNotes } : {}), stats: publicStats };
|
|
149
|
+
return { taskId, sessionId, status, ...(flags.model !== undefined ? { model: flags.model } : {}), result: result.trim(), salvagedOutput, blockedReason, errorMessage, errorCode, checkpointToken, checkpointGate, ...(workspaceRestoreMode !== undefined ? { workspaceRestoreMode } : {}), ...(flags.rewindNotes !== undefined && flags.rewindNotes.length > 0 ? { rewindNotes: flags.rewindNotes } : {}), ...(flags.remoteEnvFailures !== undefined && flags.remoteEnvFailures.length > 0 ? { remoteEnvFailures: flags.remoteEnvFailures } : {}), stats: publicStats };
|
|
147
150
|
}
|
|
@@ -127,12 +127,15 @@ export interface Prepared {
|
|
|
127
127
|
token?: CheckpointToken;
|
|
128
128
|
gate?: CheckpointGate;
|
|
129
129
|
scope?: string;
|
|
130
|
+
restoreMode?: "snapshot" | "park_only";
|
|
130
131
|
};
|
|
131
132
|
reviewRef: {
|
|
132
133
|
token?: CheckpointToken;
|
|
133
134
|
gate?: CheckpointGate;
|
|
134
135
|
scope?: string;
|
|
136
|
+
restoreMode?: "snapshot" | "park_only";
|
|
135
137
|
};
|
|
138
|
+
remoteEnvFailures: NonNullable<TaskResult["remoteEnvFailures"]>;
|
|
136
139
|
suspendLoopRef: {
|
|
137
140
|
hit: boolean;
|
|
138
141
|
};
|
|
@@ -340,4 +343,5 @@ export declare function batchContextAt(messages: AgentMessage[], currentId: stri
|
|
|
340
343
|
completedCallIds: string[];
|
|
341
344
|
};
|
|
342
345
|
export declare function rebaseWorkspacePath(p: string, fromRaw: string, toRaw: string): string;
|
|
346
|
+
export declare function rebaseWorkspacePathAcross(p: string, froms: readonly string[], to: string): string;
|
|
343
347
|
export declare function prepareTask(spec: TaskSpec, deps: RunnerDeps, sessions: SessionStore, resume?: PrepareResume, internals?: RunInternals, runnerSelf?: Runner): Promise<Prepared>;
|
|
@@ -49,7 +49,8 @@ import { capAggregateToolResults } from "../tool-result-budget.js";
|
|
|
49
49
|
import { capAggregateMediaBytes, AGGREGATE_MEDIA_BUDGET_BYTES } from "../media-byte-cap.js";
|
|
50
50
|
import { dropOrphanToolResults, guardBudget, insertTrimNotice, trimToBudget } from "../context-guard.js";
|
|
51
51
|
import { StubExecutionEnv } from "../stub-env.js";
|
|
52
|
-
import { hasDestroy, isIsolated, isRemoteExecutionEnv, isSuspendable } from "../remote-env.js";
|
|
52
|
+
import { hasDestroy, isIsolated, isRemoteExecutionEnv, isRetryableRemoteErrorCode, isSuspendable, missingRestoreSurface, RETRYABLE_REMOTE_ERROR_CODES } from "../remote-env.js";
|
|
53
|
+
import { withRetry } from "../with-retry.js";
|
|
53
54
|
import { hasBackgroundShell, sweepBackgroundShells } from "../background-shell.js";
|
|
54
55
|
import { createTaskOutputTool, createTaskStopTool, defaultTaskRegistry } from "../task-registry.js";
|
|
55
56
|
import { createMonitorTool } from "../../tools/monitor.js";
|
|
@@ -173,6 +174,27 @@ export function rebaseWorkspacePath(p, fromRaw, toRaw) {
|
|
|
173
174
|
const suffix = p.slice(fromPrefix.length).replace(/^\/+/, "");
|
|
174
175
|
return to === "/" ? `${to}${suffix}` : `${to}/${suffix}`;
|
|
175
176
|
}
|
|
177
|
+
function remoteEnvFailureNote(op, error, attempts) {
|
|
178
|
+
return { op, code: error.code, retryable: isRetryableRemoteErrorCode(error.code), attempts, message: error.message };
|
|
179
|
+
}
|
|
180
|
+
const REMOTE_RESTORE_MAX_ATTEMPTS = 2;
|
|
181
|
+
const REMOTE_RESTORE_BACKOFF_MS = 200;
|
|
182
|
+
async function restoreWorkspaceWithRetry(env, snapshotId, options) {
|
|
183
|
+
let attempts = 0;
|
|
184
|
+
const outcome = await withRetry(async (attempt) => {
|
|
185
|
+
attempts = attempt;
|
|
186
|
+
return env.resumeVM(snapshotId, options);
|
|
187
|
+
}, { retryableCodes: RETRYABLE_REMOTE_ERROR_CODES, maxAttempts: REMOTE_RESTORE_MAX_ATTEMPTS, backoffMs: () => REMOTE_RESTORE_BACKOFF_MS }, { ...(options.abortSignal !== undefined ? { signal: options.abortSignal } : {}) });
|
|
188
|
+
return { outcome, attempts };
|
|
189
|
+
}
|
|
190
|
+
export function rebaseWorkspacePathAcross(p, froms, to) {
|
|
191
|
+
for (const from of froms) {
|
|
192
|
+
const out = rebaseWorkspacePath(p, from, to);
|
|
193
|
+
if (out !== p)
|
|
194
|
+
return out;
|
|
195
|
+
}
|
|
196
|
+
return p;
|
|
197
|
+
}
|
|
176
198
|
export async function prepareTask(spec, deps, sessions, resume, internals, runnerSelf) {
|
|
177
199
|
const toolFaceSnapshot = {
|
|
178
200
|
exclude: spec.excludeTools ? Object.freeze([...spec.excludeTools]) : undefined,
|
|
@@ -558,11 +580,13 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
558
580
|
}
|
|
559
581
|
}
|
|
560
582
|
let restoredRootRebase;
|
|
561
|
-
const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p :
|
|
583
|
+
const rebaseRestoredPath = (p) => restoredRootRebase === undefined ? p : rebaseWorkspacePathAcross(p, restoredRootRebase.from, restoredRootRebase.to);
|
|
562
584
|
if (resume?.workspaceHandle !== undefined) {
|
|
563
|
-
const failResume = (message, cause) => {
|
|
585
|
+
const failResume = (message, cause, note) => {
|
|
564
586
|
const e = new Error(message, cause ? { cause } : undefined);
|
|
565
587
|
e.code = "resume.env_failed";
|
|
588
|
+
if (note !== undefined)
|
|
589
|
+
e.remoteEnvFailure = note;
|
|
566
590
|
throw e;
|
|
567
591
|
};
|
|
568
592
|
const handle = resume.workspaceHandle;
|
|
@@ -570,7 +594,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
570
594
|
failResume("resume needs a RemoteExecutionEnv from executionEnvFactory to restore the workspace snapshot");
|
|
571
595
|
}
|
|
572
596
|
else if (handle.snapshotId === undefined) {
|
|
573
|
-
if (handle.restoreMode !== "park_only" &&
|
|
597
|
+
if (handle.restoreMode !== "park_only" && ownedEnv.capabilities.suspendable) {
|
|
574
598
|
failResume("checkpoint workspaceHandle has no snapshotId and is not a park_only handle, but the resumed env is suspendable — refusing to resume on a possibly-unrestored workspace (corrupt checkpoint?)");
|
|
575
599
|
}
|
|
576
600
|
if (handle.mountPath && handle.mountPath !== taskRootPath) {
|
|
@@ -581,12 +605,37 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
581
605
|
const restoreSignal = spec.signal
|
|
582
606
|
? AbortSignal.any([abortController.signal, spec.signal])
|
|
583
607
|
: abortController.signal;
|
|
584
|
-
const
|
|
608
|
+
const missingHere = missingRestoreSurface(ownedEnv);
|
|
609
|
+
if (missingHere.length > 0) {
|
|
610
|
+
failResume(`the resumed execution env cannot restore a workspace snapshot: its adapter does not implement ${missingHere.join(" or ")}. The checkpoint holds snapshot "${handle.snapshotId}" — wire an adapter that implements the full RemoteExecutionEnv restore surface and re-resume.`);
|
|
611
|
+
}
|
|
612
|
+
const { outcome: restored, attempts: restoreAttempts } = await restoreWorkspaceWithRetry(ownedEnv, handle.snapshotId, {
|
|
613
|
+
abortSignal: restoreSignal,
|
|
614
|
+
priorHandle: handle,
|
|
615
|
+
});
|
|
585
616
|
if (!restored.ok) {
|
|
586
|
-
failResume(`resumeVM failed (${restored.error.code}): ${restored.error.message}`, restored.error);
|
|
617
|
+
failResume(`resumeVM failed after ${restoreAttempts} attempt(s) (${restored.error.code}): ${restored.error.message}`, restored.error, remoteEnvFailureNote("resumeVM", restored.error, restoreAttempts));
|
|
587
618
|
}
|
|
588
619
|
else {
|
|
620
|
+
const restoredEnv = ownedEnv;
|
|
621
|
+
const canonicalInEnv = async (p) => {
|
|
622
|
+
try {
|
|
623
|
+
const r = await restoredEnv.canonicalPath(p, restoreSignal);
|
|
624
|
+
return r.ok ? r.value : undefined;
|
|
625
|
+
}
|
|
626
|
+
catch {
|
|
627
|
+
return undefined;
|
|
628
|
+
}
|
|
629
|
+
};
|
|
630
|
+
let checkpointedCanonical;
|
|
631
|
+
let sameRootUnderAlias = false;
|
|
589
632
|
if (restored.value.mountPath !== handle.mountPath) {
|
|
633
|
+
checkpointedCanonical = await canonicalInEnv(handle.mountPath);
|
|
634
|
+
const restoredCanonical = await canonicalInEnv(restored.value.mountPath);
|
|
635
|
+
sameRootUnderAlias =
|
|
636
|
+
checkpointedCanonical !== undefined && restoredCanonical !== undefined && checkpointedCanonical === restoredCanonical;
|
|
637
|
+
}
|
|
638
|
+
if (restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
|
|
590
639
|
if (resume.executesApprovedAction === true) {
|
|
591
640
|
failResume(`resumeVM workspace-root divergence with a pending approved action: checkpointed mountPath "${handle.mountPath}" but the restored handle reports "${restored.value.mountPath}" — the approved args are bound to the checkpointed root; refusing to execute them against a moved workspace (adapter should honor priorHandle)`);
|
|
592
641
|
}
|
|
@@ -599,13 +648,23 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
599
648
|
if (restored.value.mountPath && restored.value.mountPath !== taskRootPath) {
|
|
600
649
|
taskRootPath = restored.value.mountPath;
|
|
601
650
|
}
|
|
602
|
-
if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath) {
|
|
603
|
-
|
|
651
|
+
if (restored.value.mountPath && restored.value.mountPath !== handle.mountPath && !sameRootUnderAlias) {
|
|
652
|
+
const from = checkpointedCanonical !== undefined && checkpointedCanonical !== handle.mountPath
|
|
653
|
+
? [handle.mountPath, checkpointedCanonical]
|
|
654
|
+
: [handle.mountPath];
|
|
655
|
+
restoredRootRebase = { from, to: restored.value.mountPath };
|
|
656
|
+
try {
|
|
657
|
+
deps.onError?.(new Error(from.length > 1
|
|
658
|
+
? `resumeVM workspace-root rebase accepts both spellings of the checkpointed root (${from.map((f) => `"${f}"`).join(" and ")}) when migrating persisted paths to "${restored.value.mountPath}"`
|
|
659
|
+
: `resumeVM workspace-root rebase is spelling-exact: it matches the checkpointed root "${handle.mountPath}" only, so persisted paths recorded under an equivalent alias of it are NOT migrated to "${restored.value.mountPath}" and stay as written`), { phase: "config", sessionId });
|
|
660
|
+
}
|
|
661
|
+
catch {
|
|
662
|
+
}
|
|
604
663
|
}
|
|
605
664
|
}
|
|
606
665
|
const init = await ownedEnv.postResumeInit();
|
|
607
666
|
if (!init.ok) {
|
|
608
|
-
failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error);
|
|
667
|
+
failResume(`postResumeInit failed (${init.error.code}): ${init.error.message}`, init.error, remoteEnvFailureNote("postResumeInit", init.error, 1));
|
|
609
668
|
}
|
|
610
669
|
}
|
|
611
670
|
}
|
|
@@ -948,6 +1007,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
948
1007
|
}
|
|
949
1008
|
const suspendRef = {};
|
|
950
1009
|
const reviewRef = {};
|
|
1010
|
+
const remoteEnvFailures = [];
|
|
951
1011
|
const suspendLoopRef = { hit: false };
|
|
952
1012
|
const compactionReuseRef = { consecutive: 0 };
|
|
953
1013
|
const trimPressureRef = { droppedMessages: false };
|
|
@@ -2541,16 +2601,21 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2541
2601
|
const preToolContexts = new Map();
|
|
2542
2602
|
const blockedToolCalls = new Set();
|
|
2543
2603
|
const blockedTracked = Boolean(hooks?.postToolUse || hooks?.preToolUse || hooks?.postToolUseFailure || hooks?.postToolBatch);
|
|
2604
|
+
const restoreSurfaceGap = ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && ownedEnv.capabilities.suspendable ? missingRestoreSurface(ownedEnv) : [];
|
|
2605
|
+
const incompleteSuspendAdapter = restoreSurfaceGap.length > 0 ? restoreSurfaceGap : undefined;
|
|
2544
2606
|
const resourceSuspendEligible = spec.resourceSuspend !== undefined &&
|
|
2545
2607
|
(spec.checkpointStore ?? deps.checkpointStore) !== undefined &&
|
|
2546
2608
|
!(offloadStore !== undefined && isVolatileOffloadStore(offloadStore)) &&
|
|
2547
|
-
(ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv))
|
|
2609
|
+
(ownedEnv === undefined || isRemoteExecutionEnv(ownedEnv)) &&
|
|
2610
|
+
incompleteSuspendAdapter === undefined;
|
|
2548
2611
|
if (spec.resourceSuspend !== undefined && !resourceSuspendEligible) {
|
|
2549
2612
|
const why = (spec.checkpointStore ?? deps.checkpointStore) === undefined
|
|
2550
2613
|
? "no CheckpointStore is wired"
|
|
2551
2614
|
: offloadStore !== undefined && isVolatileOffloadStore(offloadStore)
|
|
2552
2615
|
? "tool-result offload uses the in-memory store (a resume needs durable results)"
|
|
2553
|
-
:
|
|
2616
|
+
: incompleteSuspendAdapter !== undefined
|
|
2617
|
+
? `the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")} (a snapshot nothing can restore is worse than no snapshot)`
|
|
2618
|
+
: "the per-task execution env is not a RemoteExecutionEnv (it would be destroyed on suspend)";
|
|
2554
2619
|
deps.onError?.(new Error(`resourceSuspend is set but INACTIVE: ${why}; resource limits will hard-fail, not suspend`), {
|
|
2555
2620
|
phase: "config",
|
|
2556
2621
|
sessionId,
|
|
@@ -2726,14 +2791,18 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2726
2791
|
catch (putErr) {
|
|
2727
2792
|
deps.onError?.(putErr, { phase: "config", sessionId });
|
|
2728
2793
|
if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
|
|
2729
|
-
const back = await remoteEnv
|
|
2794
|
+
const { outcome: back, attempts: backAttempts } = await restoreWorkspaceWithRetry(remoteEnv, remoteHandle.snapshotId, {
|
|
2795
|
+
abortSignal: abortController.signal,
|
|
2796
|
+
});
|
|
2730
2797
|
if (back.ok) {
|
|
2731
2798
|
const init = await remoteEnv.postResumeInit();
|
|
2732
2799
|
if (init.ok)
|
|
2733
2800
|
return false;
|
|
2801
|
+
remoteEnvFailures.push(remoteEnvFailureNote("postResumeInit", init.error, 1));
|
|
2734
2802
|
deps.onError?.(init.error, { phase: "config", sessionId });
|
|
2735
2803
|
}
|
|
2736
2804
|
else {
|
|
2805
|
+
remoteEnvFailures.push(remoteEnvFailureNote("resumeVM", back.error, backAttempts));
|
|
2737
2806
|
deps.onError?.(back.error, { phase: "config", sessionId });
|
|
2738
2807
|
}
|
|
2739
2808
|
abortController.abort();
|
|
@@ -2743,10 +2812,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2743
2812
|
return false;
|
|
2744
2813
|
}
|
|
2745
2814
|
};
|
|
2746
|
-
const publishCommittedSuspend = (token, gate, scope) => {
|
|
2815
|
+
const publishCommittedSuspend = (token, gate, scope, remoteHandle) => {
|
|
2747
2816
|
const ref = gate.kind === "needs_review" || gate.kind === "plan_review" ? reviewRef : suspendRef;
|
|
2748
2817
|
ref.token = token;
|
|
2749
2818
|
ref.gate = gate;
|
|
2819
|
+
if (remoteHandle !== undefined)
|
|
2820
|
+
ref.restoreMode = remoteHandle.restoreMode === "park_only" ? "park_only" : "snapshot";
|
|
2750
2821
|
ref.scope = scope;
|
|
2751
2822
|
};
|
|
2752
2823
|
const suspendLoopCapHit = (count, cap, detail) => {
|
|
@@ -2763,7 +2834,9 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2763
2834
|
return true;
|
|
2764
2835
|
};
|
|
2765
2836
|
const suspendableEnv = ownedEnv !== undefined && isSuspendable(ownedEnv) ? ownedEnv : undefined;
|
|
2766
|
-
const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv)
|
|
2837
|
+
const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && incompleteSuspendAdapter === undefined
|
|
2838
|
+
? ownedEnv
|
|
2839
|
+
: undefined;
|
|
2767
2840
|
const parkOnlyHandle = (env) => {
|
|
2768
2841
|
const { snapshotId: _lineage, ...identity } = env.workspaceHandle();
|
|
2769
2842
|
return { ...identity, restoreMode: "park_only" };
|
|
@@ -2788,6 +2861,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2788
2861
|
await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
|
|
2789
2862
|
const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
|
|
2790
2863
|
if (!snap.ok) {
|
|
2864
|
+
remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
|
|
2791
2865
|
deps.onError?.(new Error(`resource suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
|
|
2792
2866
|
return false;
|
|
2793
2867
|
}
|
|
@@ -2822,7 +2896,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2822
2896
|
};
|
|
2823
2897
|
if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
|
|
2824
2898
|
return false;
|
|
2825
|
-
publishCommittedSuspend(token, gate, rs.scope);
|
|
2899
|
+
publishCommittedSuspend(token, gate, rs.scope, remoteHandle);
|
|
2826
2900
|
try {
|
|
2827
2901
|
await sessions.pin?.(sessionId);
|
|
2828
2902
|
}
|
|
@@ -2839,6 +2913,14 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2839
2913
|
return false;
|
|
2840
2914
|
if (abortController.signal.aborted)
|
|
2841
2915
|
return false;
|
|
2916
|
+
if (incompleteSuspendAdapter !== undefined) {
|
|
2917
|
+
try {
|
|
2918
|
+
deps.onError?.(new Error(`plan_review park skipped: the per-task execution env declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so its workspace could not be restored on resume (the plan-review request was dropped; the run continues without pausing)`), { phase: "config", sessionId });
|
|
2919
|
+
}
|
|
2920
|
+
catch {
|
|
2921
|
+
}
|
|
2922
|
+
return false;
|
|
2923
|
+
}
|
|
2842
2924
|
if (suspendLoopCapHit(priorSuspendCount, maxSuspends, " for a plan_review (likely a resume/restart loop)."))
|
|
2843
2925
|
return false;
|
|
2844
2926
|
const leafId = await session.getLeafId();
|
|
@@ -2858,6 +2940,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2858
2940
|
await sweepBackgroundShells(suspendableEnv, defaultTaskRegistry);
|
|
2859
2941
|
const snap = await suspendableEnv.suspendVM({ abortSignal: abortController.signal });
|
|
2860
2942
|
if (!snap.ok) {
|
|
2943
|
+
remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
|
|
2861
2944
|
deps.onError?.(new Error(`plan_review suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error }), { phase: "config", sessionId });
|
|
2862
2945
|
return false;
|
|
2863
2946
|
}
|
|
@@ -2896,7 +2979,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2896
2979
|
};
|
|
2897
2980
|
if (!(await commitSuspendSaga(token, cp, suspendableEnv, remoteHandle)))
|
|
2898
2981
|
return false;
|
|
2899
|
-
publishCommittedSuspend(token, gate, scope);
|
|
2982
|
+
publishCommittedSuspend(token, gate, scope, remoteHandle);
|
|
2900
2983
|
try {
|
|
2901
2984
|
await sessions.pin?.(sessionId);
|
|
2902
2985
|
}
|
|
@@ -2930,10 +3013,12 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2930
3013
|
"RunnerDeps.toolResultStore or disable offload.");
|
|
2931
3014
|
}
|
|
2932
3015
|
if (ownedEnv !== undefined && remoteEnv === undefined && parkOnlyRemoteEnv === undefined) {
|
|
2933
|
-
throw new Error(
|
|
2934
|
-
|
|
2935
|
-
"
|
|
2936
|
-
|
|
3016
|
+
throw new Error(incompleteSuspendAdapter !== undefined
|
|
3017
|
+
? `durable suspend is not supported with this per-task executionEnvFactory env: it declares capabilities.suspendable but its adapter does not implement ${incompleteSuspendAdapter.join(" or ")}, so a snapshot taken now could never be restored. Implement the full RemoteExecutionEnv restore surface, or declare capabilities.suspendable:false if the workspace is externally durable (the park-only lane).`
|
|
3018
|
+
: "durable suspend is not supported with a non-remote per-task executionEnvFactory env: the " +
|
|
3019
|
+
"minted env is destroyed on suspend, so a resumed file/shell tool would act on a fresh " +
|
|
3020
|
+
"(empty) env. Use a RemoteExecutionEnv factory (it is paused, not destroyed) or a static, " +
|
|
3021
|
+
"caller-owned RunnerDeps.executionEnv.");
|
|
2937
3022
|
}
|
|
2938
3023
|
const leafId = await session.getLeafId();
|
|
2939
3024
|
if (!leafId) {
|
|
@@ -2951,6 +3036,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
2951
3036
|
await sweepBackgroundShells(remoteEnv, defaultTaskRegistry);
|
|
2952
3037
|
const snap = await remoteEnv.suspendVM({ abortSignal: abortController.signal });
|
|
2953
3038
|
if (!snap.ok) {
|
|
3039
|
+
remoteEnvFailures.push(remoteEnvFailureNote("suspendVM", snap.error, 1));
|
|
2954
3040
|
throw new Error(`suspendVM failed (${snap.error.code}): ${snap.error.message}`, { cause: snap.error });
|
|
2955
3041
|
}
|
|
2956
3042
|
remoteHandle = { ...remoteEnv.workspaceHandle(), snapshotId: snap.value };
|
|
@@ -3039,7 +3125,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
3039
3125
|
if (!(await commitSuspendSaga(token, cp, remoteEnv, remoteHandle))) {
|
|
3040
3126
|
return undefined;
|
|
3041
3127
|
}
|
|
3042
|
-
publishCommittedSuspend(token, gate, cp.scope);
|
|
3128
|
+
publishCommittedSuspend(token, gate, cp.scope, remoteHandle);
|
|
3043
3129
|
try {
|
|
3044
3130
|
await sessions.pin?.(sessionId);
|
|
3045
3131
|
}
|
|
@@ -3388,7 +3474,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
3388
3474
|
: undefined;
|
|
3389
3475
|
overheadState.promptChars = systemPrompt.length;
|
|
3390
3476
|
const preparedHolder = {};
|
|
3391
|
-
const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
|
|
3477
|
+
const buildPrepared = () => ({ harness, session, sessionId, taskRootPath, model, thinking, compModel, mcp: mcp, blockedRef, outputRef, abortController, conflictRef, blockedToolCalls, nestedStats, ...(rewindNotes.length > 0 ? { rewindNotes } : {}), cwdRef: handsCwdRef, ...(worktreeSessionRef !== undefined ? { worktreeSessionRef } : {}), ...(workspaceStateSettle !== undefined ? { workspaceStateSettle } : {}), denyNarrowingPolicy, ...(basePolicyForResumeEdit !== undefined ? { basePolicyForResumeEdit } : {}), releaseSignal, cacheBreakDetector, cacheFingerprint, promptManifest, epochDeclaredSections, activeTools, ...(deferred.size > 0 ? { deferredToolNames: deferred } : {}), ownedEnv, suspendRef, reviewRef, remoteEnvFailures, reviewRequestRef, suspendLoopRef, suspendForResource, ...(callCapRef ? { callCapRef } : {}), ...(cutKills ? { cutKills } : {}), ...(callCapRef ? { callIssuedAtRef } : {}), suspendForReview, resourceLedger: priorLedger, liveSpendRef, humanReviewRef, now, tools, toolEffects, promptOverheadTokens, readTaskFile, recentlyReadFiles, normalizeAttachmentPath, isDedupStubResult, ...(onCompactionApplied ? { onCompactionApplied } : {}), compactionReuseRef, trimPressureRef, ...(memoryEngineSession ? { memoryEngineSession } : {}), ...(subagentRetain ? { subagentRetain } : {}), ...(lspDiagnostics && nudgeLspOnEdit ? { lspDiagnostics: { registry: lspDiagnostics, nudge: nudgeLspOnEdit } } : {}), planModeRef, ...(dateChange ? { dateChange } : {}), ...(instructionSources ? { instructionSources } : {}), ...(detectExternalChanges ? { detectExternalChanges } : {}), ...(toolsDeltaRef ? { toolsDeltaRef } : {}), ...(agentListing ? { agentListing } : {}), ...(skillsListing ? { skillsListing } : {}), announcedListingsRef, listBackgroundTasks, ...(turnSnapshotRef.current !== undefined ? { turnSnapshot: turnSnapshotRef.current } : {}), ...(centerCompactionCandidate !== undefined ? { centerCompactionCandidate } : {}) });
|
|
3392
3478
|
const prepared = buildPrepared();
|
|
3393
3479
|
preparedHolder.current = prepared;
|
|
3394
3480
|
return prepared;
|
|
@@ -1345,6 +1345,16 @@ export class Runner {
|
|
|
1345
1345
|
queue.close();
|
|
1346
1346
|
return;
|
|
1347
1347
|
}
|
|
1348
|
+
const remoteEnvFailure = (() => {
|
|
1349
|
+
let cur = err;
|
|
1350
|
+
for (let depth = 0; cur && typeof cur === "object" && depth < 8; depth++) {
|
|
1351
|
+
const note = cur.remoteEnvFailure;
|
|
1352
|
+
if (note !== undefined)
|
|
1353
|
+
return [note];
|
|
1354
|
+
cur = cur.cause;
|
|
1355
|
+
}
|
|
1356
|
+
return undefined;
|
|
1357
|
+
})();
|
|
1348
1358
|
resultValue = {
|
|
1349
1359
|
taskId: taskIdRef.current ?? "unknown",
|
|
1350
1360
|
sessionId: taskIdRef.sessionId ?? "unknown",
|
|
@@ -1352,6 +1362,7 @@ export class Runner {
|
|
|
1352
1362
|
result: "",
|
|
1353
1363
|
errorMessage: err instanceof Error ? err.message : String(err),
|
|
1354
1364
|
errorCode: code,
|
|
1365
|
+
...(remoteEnvFailure !== undefined ? { remoteEnvFailures: remoteEnvFailure } : {}),
|
|
1355
1366
|
stats: { turns: 0, tokens: 0, toolCalls: 0, cachedTokens: 0, costMicroUsd: 0 },
|
|
1356
1367
|
};
|
|
1357
1368
|
emitTrace(spec.tracer ?? this.deps.tracer, () => ({
|
|
@@ -2765,6 +2776,7 @@ export class Runner {
|
|
|
2765
2776
|
model: prepared.model.id,
|
|
2766
2777
|
unpricedSpend: rs.telemetry.unpricedSpend,
|
|
2767
2778
|
rewindNotes: prepared.rewindNotes,
|
|
2779
|
+
remoteEnvFailures: prepared.remoteEnvFailures,
|
|
2768
2780
|
abortedForTimeout: timeout.fired,
|
|
2769
2781
|
abortedForTurns: rs.limits.turnsExceeded,
|
|
2770
2782
|
abortedLive,
|
|
@@ -2774,10 +2786,18 @@ export class Runner {
|
|
|
2774
2786
|
outputInvalid: rs.degrade.outputInvalid,
|
|
2775
2787
|
suspendLoop: prepared.suspendLoopRef.hit,
|
|
2776
2788
|
suspendRef: prepared.suspendRef.token !== undefined && prepared.suspendRef.gate !== undefined
|
|
2777
|
-
? {
|
|
2789
|
+
? {
|
|
2790
|
+
token: prepared.suspendRef.token,
|
|
2791
|
+
gate: prepared.suspendRef.gate,
|
|
2792
|
+
...(prepared.suspendRef.restoreMode !== undefined ? { restoreMode: prepared.suspendRef.restoreMode } : {}),
|
|
2793
|
+
}
|
|
2778
2794
|
: undefined,
|
|
2779
2795
|
reviewRef: prepared.reviewRef.token !== undefined && prepared.reviewRef.gate !== undefined
|
|
2780
|
-
? {
|
|
2796
|
+
? {
|
|
2797
|
+
token: prepared.reviewRef.token,
|
|
2798
|
+
gate: prepared.reviewRef.gate,
|
|
2799
|
+
...(prepared.reviewRef.restoreMode !== undefined ? { restoreMode: prepared.reviewRef.restoreMode } : {}),
|
|
2800
|
+
}
|
|
2781
2801
|
: undefined,
|
|
2782
2802
|
});
|
|
2783
2803
|
if (resume !== undefined &&
|
package/dist/core/types.d.ts
CHANGED
|
@@ -370,6 +370,13 @@ export interface TaskSpec {
|
|
|
370
370
|
signal?: AbortSignal;
|
|
371
371
|
}
|
|
372
372
|
export type TaskStatus = "completed" | "blocked" | "failed" | "timeout" | "suspended" | "needs_review";
|
|
373
|
+
export interface RemoteEnvFailureNote {
|
|
374
|
+
op: "suspendVM" | "resumeVM" | "postResumeInit";
|
|
375
|
+
code: import("./remote-env.js").RemoteExecutionErrorCode;
|
|
376
|
+
retryable: boolean;
|
|
377
|
+
attempts: number;
|
|
378
|
+
message: string;
|
|
379
|
+
}
|
|
373
380
|
export interface TaskResult {
|
|
374
381
|
taskId: string;
|
|
375
382
|
sessionId: string;
|
|
@@ -380,6 +387,8 @@ export interface TaskResult {
|
|
|
380
387
|
blockedReason?: string;
|
|
381
388
|
checkpointToken?: import("./checkpoint-store.js").CheckpointToken;
|
|
382
389
|
checkpointGate?: import("./checkpoint-store.js").CheckpointGate;
|
|
390
|
+
workspaceRestoreMode?: "snapshot" | "park_only";
|
|
391
|
+
remoteEnvFailures?: RemoteEnvFailureNote[];
|
|
383
392
|
errorMessage?: string;
|
|
384
393
|
errorCode?: string;
|
|
385
394
|
degraded?: {
|
|
@@ -47,7 +47,7 @@ export declare class FileError extends Error {
|
|
|
47
47
|
path?: string;
|
|
48
48
|
constructor(code: FileErrorCode, message: string, path?: string, cause?: Error);
|
|
49
49
|
}
|
|
50
|
-
export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "unknown";
|
|
50
|
+
export type ExecutionErrorCode = "aborted" | "timeout" | "shell_unavailable" | "spawn_error" | "callback_error" | "transport_lost" | "suspended" | "auth_failed" | "unknown";
|
|
51
51
|
export declare class ExecutionError extends Error {
|
|
52
52
|
code: ExecutionErrorCode;
|
|
53
53
|
partialStdout?: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -56,7 +56,7 @@ export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-e
|
|
|
56
56
|
export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
|
|
57
57
|
export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
|
|
58
58
|
export type { ExecutionEnv, FileInfo, Result, FileErrorCode, ExecutionErrorCode } from "./internal/harness.js";
|
|
59
|
-
export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
|
|
59
|
+
export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
|
|
60
60
|
export { withRetry } from "./core/with-retry.js";
|
|
61
61
|
export type { RetryPolicy, RetryResult } from "./core/with-retry.js";
|
|
62
62
|
export type { RemoteExecutionEnv, WorkspaceHandle, SnapshotId, SessionToken, SandboxTier, OutputChunk, ExecStreamOptions, RemoteConnectConfig, VmLifecycleOptions, SecretRef, RemoteExecutionErrorCode, ExecutionEnvFactory, ExecutionEnvFactoryContext, } from "./core/remote-env.js";
|
|
@@ -189,7 +189,7 @@ export { retryBackoffMs, parseRetryAfter } from "./brain/retry.js";
|
|
|
189
189
|
export { type BrainTimeoutConfig } from "./brain/timeout.js";
|
|
190
190
|
export { createAssistantMessageEventStream } from "./internal/llm.js";
|
|
191
191
|
export type { AssistantMessage, AssistantMessageEvent, Context, DocumentContent, ImageContent, Message, StopReason, StreamFn, TextContent, ThinkingContent, ToolCall, ToolResultMessage, Usage, UserMessage, } from "./internal/llm.js";
|
|
192
|
-
export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
|
|
192
|
+
export type { AgentDefinition, BeforeWriteHook, BeforeWriteRequest, BeforeWriteResult, HandsBandOptions, Brain, BrainStatus, BrainStatusPhase, ImageInput, McpElicitRequest, McpElicitResponse, McpServerSpec, OnElicit, Model, ModelRef, ProjectMemoryLoad, RunnerDeps, RuntimeCaps, BackgroundChildEvent, SkillManifest, SkillSpec, TaskEvent, TaskEventIdentity, ToolActivity, TaskResult, RemoteEnvFailureNote, TaskSpec, TaskStatus, TaskStream, CompactOutcome, ThinkingLevel, ToolExecuteContext, ToolReturn, ToolSpec, ToolEffect, WorkflowGovernanceBaseline, } from "./core/types.js";
|
|
193
193
|
export { Type } from "typebox";
|
|
194
194
|
export type { TSchema, Static } from "typebox";
|
|
195
195
|
export { explainPromptAssembly, describeDefaultPack, type DefaultPackDescription, type ExplainInput } from "./prompt-assembly/explain.js";
|
package/dist/index.js
CHANGED
|
@@ -51,7 +51,7 @@ export { killProcessTree, signalProcessTree } from "./engine/execution-env/kill-
|
|
|
51
51
|
export { getShellConfig, isWslBashLauncher } from "./engine/execution-env/node-execution-env.js";
|
|
52
52
|
export { isSecretEnvKey, scrubSecretEnv } from "./core/secret-env.js";
|
|
53
53
|
export { MAX_EXEC_OUTPUT_BYTES, RollingTailBuffer, markTruncated } from "./core/exec-output-tail.js";
|
|
54
|
-
export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, } from "./core/remote-env.js";
|
|
54
|
+
export { RemoteExecutionError, hasDestroy, isRemoteExecutionEnv, isSuspendable, isIsolated, missingRestoreSurface, isRetryableRemoteErrorCode, RETRYABLE_REMOTE_ERROR_CODES, } from "./core/remote-env.js";
|
|
55
55
|
export { withRetry } from "./core/with-retry.js";
|
|
56
56
|
export { addWorktree, pruneWorktrees, WORKTREE_PARENT } from "./core/git-worktree-env.js";
|
|
57
57
|
export { runExecGate } from "./core/exec-gate.js";
|
package/dist/tools/fs/fs-bash.js
CHANGED
|
@@ -229,6 +229,20 @@ async function runShell(env, cwd, toolName, command, timeoutSec, caps, signal, c
|
|
|
229
229
|
isError: true,
|
|
230
230
|
};
|
|
231
231
|
}
|
|
232
|
+
if (res.error.code === "suspended") {
|
|
233
|
+
return {
|
|
234
|
+
content: `Error (${toolName}): the execution environment is suspended (its workspace VM is paused), so the command did NOT run and no command can run in this leg. Do not retry it here — the task must be resumed first; report that the workspace is suspended. (${res.error.message})`,
|
|
235
|
+
details: { type: "bash", envSuspended: true },
|
|
236
|
+
isError: true,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
if (res.error.code === "auth_failed") {
|
|
240
|
+
return {
|
|
241
|
+
content: `Error (${toolName}): authentication to the execution environment was rejected, so the command did NOT run. Do NOT retry — this is a permanent credential failure that an operator must fix; report it instead of trying other commands. (${res.error.message})`,
|
|
242
|
+
details: { type: "bash", authFailed: true },
|
|
243
|
+
isError: true,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
232
246
|
if (res.error.code === "timeout" || res.error.code === "aborted" || res.error.code === "callback_error") {
|
|
233
247
|
const rawStdout = res.error.partialStdout ?? "";
|
|
234
248
|
const rawStderr = res.error.partialStderr ?? "";
|
|
@@ -83,16 +83,13 @@ function expandCronField(field, { min, max }) {
|
|
|
83
83
|
return out.size > 0 ? [...out].sort((a, b) => a - b) : null;
|
|
84
84
|
}
|
|
85
85
|
export function cronScheduleError(expr) {
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
return `invalid seconds field "${parts[0]}" — allowed 0-59 (syntax: N, N-M, */N, comma lists).`;
|
|
91
|
-
}
|
|
92
|
-
std = parts.slice(1);
|
|
86
|
+
const std = expr.trim().split(/\s+/);
|
|
87
|
+
if (std.length === 6) {
|
|
88
|
+
return (`its leading seconds field is not supported — the resident scheduler fires on 5-field cron, so one ` +
|
|
89
|
+
`minute is the finest granularity available. Pass "${std.slice(1).join(" ")}" if that cadence works.`);
|
|
93
90
|
}
|
|
94
|
-
|
|
95
|
-
return `expected 5 fields (minute hour day-of-month month day-of-week)
|
|
91
|
+
if (std.length !== 5) {
|
|
92
|
+
return `expected 5 fields (minute hour day-of-month month day-of-week), got ${std.length}.`;
|
|
96
93
|
}
|
|
97
94
|
const expanded = [];
|
|
98
95
|
for (let i = 0; i < 5; i++) {
|
|
@@ -117,9 +114,12 @@ export function cronScheduleError(expr) {
|
|
|
117
114
|
return null;
|
|
118
115
|
}
|
|
119
116
|
const DAY_NAMES = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
|
|
117
|
+
function cronExprFromSummary(when) {
|
|
118
|
+
const trimmed = when.trim();
|
|
119
|
+
return trimmed.startsWith("cron ") ? trimmed.slice("cron ".length).trim() : trimmed;
|
|
120
|
+
}
|
|
120
121
|
export function cronToHuman(expr) {
|
|
121
|
-
const
|
|
122
|
-
const std = parts.length === 6 ? parts.slice(1) : parts;
|
|
122
|
+
const std = cronExprFromSummary(expr).split(/\s+/);
|
|
123
123
|
if (std.length !== 5)
|
|
124
124
|
return expr;
|
|
125
125
|
const [minute = "", hour = "", dom = "", month = "", dow = ""] = std;
|
|
@@ -191,7 +191,10 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
|
|
|
191
191
|
parameters: Type.Object({
|
|
192
192
|
prompt: Type.String({ description: "Self-contained instructions for the future task (it runs unattended)." }),
|
|
193
193
|
schedule: Type.Optional(Type.Union([
|
|
194
|
-
Type.Object({
|
|
194
|
+
Type.Object({
|
|
195
|
+
kind: Type.Literal("cron"),
|
|
196
|
+
expr: Type.String({ description: "5-field cron expression (minute hour day-of-month month day-of-week)." }),
|
|
197
|
+
}),
|
|
195
198
|
Type.Object({ kind: Type.Literal("at"), atMs: Type.Number({ description: "Absolute epoch ms to fire once." }) }),
|
|
196
199
|
Type.Object({ kind: Type.Literal("delay"), delaySec: Type.Number({ description: "Seconds from now to fire once." }) }),
|
|
197
200
|
], { description: 'When to fire. Pass exactly one of `schedule` or `cron`. Defaults to `durable: true`.' })),
|
|
@@ -246,22 +249,32 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
|
|
|
246
249
|
...(!durable ? { lifetime: "session" } : {}),
|
|
247
250
|
...(a.recurring !== undefined ? { recurring: a.recurring } : {}),
|
|
248
251
|
};
|
|
252
|
+
const before = await sched.list(schedCtx);
|
|
253
|
+
const knownIds = before.ok ? new Set(before.value.map((s) => s.id)) : undefined;
|
|
249
254
|
const r = await sched.schedule(intent, schedCtx);
|
|
250
255
|
if (!r.ok)
|
|
251
256
|
return errorResult(`Error (CronCreate): ${r.error.message}`);
|
|
257
|
+
const replaced = knownIds?.has(r.value.id);
|
|
252
258
|
const humanSchedule = when.kind === "cron"
|
|
253
259
|
? cronToHuman(when.expr)
|
|
254
260
|
: when.kind === "delay"
|
|
255
261
|
? `once in ${when.delaySec}s`
|
|
256
262
|
: `once at ${new Date(when.atMs).toISOString()}`;
|
|
263
|
+
const named = `${r.value.id}${a.label ? ` (${a.label})` : ""}`;
|
|
264
|
+
const content = replaced === true
|
|
265
|
+
? `Updated scheduled task ${named} — it replaced an existing job with the same schedule and label, whose prompt is now gone.`
|
|
266
|
+
: replaced === false
|
|
267
|
+
? `Scheduled task ${named}.`
|
|
268
|
+
: `Scheduled (or updated) task ${named} — the scheduler's listing was unavailable, so whether this replaced an existing job with the same schedule and label is unknown.`;
|
|
257
269
|
return {
|
|
258
|
-
content
|
|
270
|
+
content,
|
|
259
271
|
details: {
|
|
260
272
|
type: "cron-create",
|
|
261
273
|
id: r.value.id,
|
|
262
274
|
humanSchedule,
|
|
263
275
|
recurring: when.kind === "cron" && a.recurring !== false,
|
|
264
276
|
durable,
|
|
277
|
+
...(replaced !== undefined ? { replaced } : {}),
|
|
265
278
|
},
|
|
266
279
|
};
|
|
267
280
|
},
|
|
@@ -298,17 +311,18 @@ Only use minute 0 or 30 when the user names that exact time and clearly means it
|
|
|
298
311
|
}
|
|
299
312
|
const content = r.value
|
|
300
313
|
.map((s) => {
|
|
314
|
+
const shown = cronExprFromSummary(s.when);
|
|
301
315
|
const human = cronToHuman(s.when);
|
|
302
316
|
const tier = s.lifetime === "session" ? " [session]" : "";
|
|
303
317
|
const once = s.recurring === false ? " [once]" : "";
|
|
304
|
-
return `- ${s.id}: ${
|
|
318
|
+
return `- ${s.id}: ${shown}${human !== s.when ? ` — ${human}` : ""}${s.label ? ` (${s.label})` : ""}${tier}${once}`;
|
|
305
319
|
})
|
|
306
320
|
.join("\n");
|
|
307
321
|
const jobs = r.value.map((s) => {
|
|
308
322
|
const human = cronToHuman(s.when);
|
|
309
323
|
return {
|
|
310
324
|
id: s.id,
|
|
311
|
-
cron: s.when,
|
|
325
|
+
cron: cronExprFromSummary(s.when),
|
|
312
326
|
humanSchedule: human !== s.when ? human : s.when,
|
|
313
327
|
...(s.label !== undefined ? { label: s.label } : {}),
|
|
314
328
|
...(s.recurring !== undefined ? { recurring: s.recurring } : {}),
|