@ferris1225/pi-subagents 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/spawn.ts CHANGED
@@ -1,557 +1,587 @@
1
- /**
2
- * Sub-agent result handling and resilient RPC launch orchestration.
3
- *
4
- * The process transport itself lives in rpc-run.ts. Each attempt starts pi in
5
- * persistent `--mode rpc`, sends commands over strict LF-delimited JSONL, and
6
- * settles only on `agent_settled`. This module preserves the existing startup
7
- * retry, same-model retry, model fallback, accounting, and result formatting
8
- * contracts around those attempts.
9
- */
10
-
11
- import { randomUUID } from "node:crypto";
12
- import { mkdirSync, writeFileSync } from "node:fs";
13
- import { mkdtemp, rm } from "node:fs/promises";
14
- import { tmpdir } from "node:os";
15
- import { basename, join } from "node:path";
16
- import type { Message } from "@earendil-works/pi-ai";
17
- import type { AgentConfig } from "./agents.ts";
18
- import { DEFAULT_THINKING_LEVEL, type ThinkingLevel } from "./config.ts";
19
- import {
20
- currentSubagentDepth,
21
- DEPTH_ENV_VAR,
22
- extractToolErrorText,
23
- getPiInvocation,
24
- RpcRunControl,
25
- runRpcAgentAttempt,
26
- sessionExists,
27
- SUBAGENT_KILL_GRACE_MS,
28
- type RpcSingleResult,
29
- type SubagentLiveEvent,
30
- type UsageStats,
31
- } from "./rpc-run.ts";
32
-
33
- export {
34
- currentSubagentDepth,
35
- DEPTH_ENV_VAR,
36
- extractToolErrorText,
37
- getPiInvocation,
38
- RpcRunControl,
39
- sessionExists,
40
- SUBAGENT_KILL_GRACE_MS,
41
- };
42
- export type { SubagentLiveEvent, UsageStats };
43
-
44
- export const SUBAGENT_THINKING_LEVEL: ThinkingLevel = DEFAULT_THINKING_LEVEL;
45
- /** 0 disables the watchdog; dispatch supplies the configured timeout. */
46
- export const SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS = 0;
47
- export const SUBAGENT_STARTUP_RETRY_DELAYS_MS = [250, 750, 1500] as const;
48
- export const MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS = 2000;
49
- export const SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 30_000] as const;
50
-
51
- export interface SingleResult extends RpcSingleResult {}
52
-
53
- export interface SubagentDetails {
54
- mode: "single" | "parallel";
55
- results: SingleResult[];
56
- background?: boolean;
57
- }
58
-
59
- export function getFinalOutput(messages: Message[]): string {
60
- for (let i = messages.length - 1; i >= 0; i--) {
61
- const msg = messages[i];
62
- if (msg.role === "assistant") {
63
- for (const part of msg.content) {
64
- if (part.type === "text") return part.text;
65
- }
66
- }
67
- }
68
- return "";
69
- }
70
-
71
- /** Only the last standalone reviewer verdict line counts. */
72
- export function reviewVerdict(output: string): "pass" | "fail" | undefined {
73
- const lines = output.split("\n");
74
- for (let index = lines.length - 1; index >= 0; index--) {
75
- const match = /^\s*VERDICT:\s*REVIEW_(PASS|FAIL)\s*$/i.exec(lines[index]);
76
- if (match) return match[1].toUpperCase() === "PASS" ? "pass" : "fail";
77
- }
78
- return undefined;
79
- }
80
-
81
- export const RESULT_LINE_MAX = 200;
82
-
83
- export interface TruncatedOutput {
84
- text: string;
85
- truncated: boolean;
86
- }
87
-
88
- export function truncateResultOutput(output: string, maxLines: number): TruncatedOutput {
89
- const lines = output.split("\n");
90
- if (lines.length <= maxLines && lines.every((line) => line.length <= RESULT_LINE_MAX)) {
91
- return { text: output, truncated: false };
92
- }
93
- const kept = lines.slice(0, maxLines).map((line) =>
94
- line.length > RESULT_LINE_MAX ? `${line.slice(0, RESULT_LINE_MAX)}…` : line,
95
- );
96
- return { text: kept.join("\n"), truncated: true };
97
- }
98
-
99
- export function writeResultArtifact(output: string, agentName: string, cwd?: string): string {
100
- const projectSlug = cwd ? basename(cwd).replace(/[^\w.-]+/g, "_") || "default" : "default";
101
- const dir = join(tmpdir(), "pi-subagents-results", projectSlug);
102
- mkdirSync(dir, { recursive: true });
103
- const safeName = agentName.replace(/[^\w.-]+/g, "_");
104
- const unique = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
105
- const filePath = join(dir, `${unique}-${safeName}.md`);
106
- writeFileSync(filePath, output, "utf8");
107
- return filePath;
108
- }
109
-
110
- export function isFailedResult(result: SingleResult): boolean {
111
- if (result.parked) return false;
112
- return result.exitCode !== 0 || result.stopReason === "error" || result.stopReason === "aborted";
113
- }
114
-
115
- function lastAssistantMessage(messages: Message[]): Extract<Message, { role: "assistant" }> | undefined {
116
- for (let index = messages.length - 1; index >= 0; index--) {
117
- const message = messages[index];
118
- if (message.role === "assistant") return message;
119
- }
120
- return undefined;
121
- }
122
-
123
- function assistantText(message: Extract<Message, { role: "assistant" }>): string {
124
- return message.content
125
- .filter((part): part is Extract<(typeof message.content)[number], { type: "text" }> => part.type === "text")
126
- .map((part) => part.text)
127
- .join("");
128
- }
129
-
130
- export function isModelLevelFailure(result: SingleResult): boolean {
131
- if (!isFailedResult(result)) return false;
132
- if (result.stopReason === "aborted") return false;
133
- if (result.dispatchFailed) return false;
134
- if (result.integrationStatus === "retained") return false;
135
- if (result.errorMessage?.includes("idle timeout")) return true;
136
- if (result.rpcPromptRejected) return true;
137
-
138
- // Classification belongs to the final assistant turn, not the whole attempt.
139
- // Earlier useful text or failed tool calls are retained session history and
140
- // must not hide a later provider error (for example a second-turn 503).
141
- const finalAssistant = lastAssistantMessage(result.messages);
142
- if (finalAssistant) {
143
- if (finalAssistant.stopReason !== "error") return false;
144
- if (assistantText(finalAssistant).trim()) return false;
145
- return Boolean(
146
- finalAssistant.errorMessage?.trim() ||
147
- result.errorMessage?.trim() ||
148
- result.stderr.trim() ||
149
- finalAssistant.content.length === 0
150
- );
151
- }
152
-
153
- if ((result.failedTools?.length ?? 0) > 0) return false;
154
- return Boolean(result.errorMessage?.trim()) || result.stderr.trim().length > 0;
155
- }
156
-
157
- const TERMINAL_MODEL_ERROR_PATTERN =
158
- /insufficient_quota|quota\s+exceeded|exceeded[^.\n]{0,40}quota|out\s+of\s+budget|billing|usage\s+limit|usage_limit|gousagelimiterror|freeusagelimiterror|monthly\s+usage\s+limit\s+reached|available\s+balance|invalid\s+(?:api\s+)?key|incorrect\s+api\s+key|unauthori[sz]ed|\b401\b|\b403\b|forbidden|permission\s+denied/i;
159
- const PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN =
160
- /model[_ -]?not[_ -]?found|no\s+models?\s+(?:found|matched)|(?:model|provider)[^.\n]{0,80}(?:not\s+found|unknown|does\s+not\s+exist|unsupported|invalid)|(?:not\s+found|unknown|unsupported|invalid)[^.\n]{0,40}(?:model|provider)|\b404\b/i;
161
-
162
- export function isTerminalModelError(result: SingleResult): boolean {
163
- const message = result.errorMessage?.trim();
164
- if (message) return TERMINAL_MODEL_ERROR_PATTERN.test(message);
165
- const stderr = result.stderr.trim();
166
- return stderr.length > 0 && TERMINAL_MODEL_ERROR_PATTERN.test(stderr);
167
- }
168
-
169
- /** A permanent failure of this model/provider reference (stale id, unknown
170
- * provider, 404 config route). Skip same-candidate backoff, but keep advancing
171
- * through backup and current-main candidates. */
172
- export function isPermanentModelCandidateError(result: SingleResult): boolean {
173
- const message = result.errorMessage?.trim();
174
- if (message) return PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN.test(message);
175
- const stderr = result.stderr.trim();
176
- return stderr.length > 0 && PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN.test(stderr);
177
- }
178
-
179
- export function isRetryableStartupFailure(result: SingleResult, durationMs: number): boolean {
180
- if (result.exitCode === 0) return false;
181
- if (result.stopReason === "aborted") return false;
182
- if (result.dispatchFailed) return false;
183
- if (result.errorMessage?.includes("idle timeout")) return false;
184
- if (getFinalOutput(result.messages)) return false;
185
- if (result.messages.length > 0) return false;
186
- const usage = result.usage;
187
- if (usage.turns || usage.input || usage.output || usage.cacheRead || usage.cacheWrite || usage.cost) return false;
188
- if (durationMs > MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS) return false;
189
- if (result.stderr.trim().length > 0) return false;
190
- if (result.errorMessage && result.errorMessage.trim().length > 0) return false;
191
- return true;
192
- }
193
-
194
- export function formatStartupRetryExhaustedError(model: string, attempts: number): string {
195
- return `Subagent failed to start after ${attempts} attempt${attempts === 1 ? "" : "s"} on ${model}: the child exited before any model, tool, output, or usage activity. This is typically a concurrent pi startup race (several sub-agents starting at once). Retry the dispatch, or temporarily lower maxConcurrency in /subagents-setup.`;
196
- }
197
-
198
- export async function waitForStartupRetry(delayMs: number, signal?: AbortSignal): Promise<boolean> {
199
- if (delayMs <= 0) return !signal?.aborted;
200
- if (!signal) {
201
- return new Promise<boolean>((resolve) => {
202
- const timer = setTimeout(() => resolve(true), delayMs);
203
- if (typeof timer.unref === "function") timer.unref();
204
- });
205
- }
206
- if (signal.aborted) return false;
207
- return new Promise<boolean>((resolve) => {
208
- let settled = false;
209
- const finish = (shouldRetry: boolean): void => {
210
- if (settled) return;
211
- settled = true;
212
- clearTimeout(timer);
213
- signal.removeEventListener("abort", onAbort);
214
- resolve(shouldRetry);
215
- };
216
- const onAbort = (): void => finish(false);
217
- const timer = setTimeout(() => finish(true), delayMs);
218
- if (typeof timer.unref === "function") timer.unref();
219
- signal.addEventListener("abort", onAbort, { once: true });
220
- });
221
- }
222
-
223
- async function waitForControlledRetry(
224
- delayMs: number,
225
- signal: AbortSignal | undefined,
226
- control: RpcRunControl | undefined,
227
- ): Promise<boolean> {
228
- let remaining = delayMs;
229
- while (remaining > 0) {
230
- if (control?.isParkRequested() || control?.isStopRequested()) return false;
231
- const slice = Math.min(remaining, 50);
232
- if (!(await waitForStartupRetry(slice, signal))) return false;
233
- remaining -= slice;
234
- }
235
- return !signal?.aborted && !control?.isParkRequested() && !control?.isStopRequested();
236
- }
237
-
238
- export function getResultOutput(result: SingleResult): string {
239
- if (isFailedResult(result)) {
240
- const error = result.errorMessage || result.stderr;
241
- const partial = getFinalOutput(result.messages);
242
- if (error && partial) return `${error}\n\n--- Partial output ---\n${partial}`;
243
- return error || partial || "(no output)";
244
- }
245
- return getFinalOutput(result.messages) || "(no output)";
246
- }
247
-
248
- export function buildResumePrompt(task: string, reason: string): string {
249
- return `You are resuming an earlier sub-agent session after ${reason}. Your earlier work — searches, reads, edits, and reasoning — is preserved in this session's history above; review it before acting. Original task: ${task}. Pick up exactly where you left off and finish it. Do NOT redo searches, reads, or edits you already completed unless a step clearly failed. Continue now.`;
250
- }
251
-
252
- export function buildFallbackResumeReason(fromModel?: string): string {
253
- return fromModel
254
- ? `the previous model (${fromModel}) failed at the model/provider level, so the next model in its configured pool is continuing`
255
- : "the previous model failed at the model/provider level, so the next model in its configured pool is continuing";
256
- }
257
-
258
- export interface RunSingleOptions {
259
- defaultCwd: string;
260
- agent: AgentConfig | undefined;
261
- agentName: string;
262
- task: string;
263
- cwd?: string;
264
- thinkingLevel?: ThinkingLevel;
265
- idleTimeoutMs?: number;
266
- startupRetryDelaysMs?: readonly number[];
267
- runLevelRetryDelaysMs?: readonly number[];
268
- sessionDir?: string;
269
- sessionId?: string;
270
- /** Initial RPC prompt. Kept under the old name to limit caller churn. */
271
- stdinText?: string;
272
- signal?: AbortSignal;
273
- onLive?: (event: SubagentLiveEvent) => void;
274
- makeDetails: (results: SingleResult[]) => SubagentDetails;
275
- env?: NodeJS.ProcessEnv;
276
- /** Stable logical-generation controller shared across retry attempts. */
277
- control?: RpcRunControl;
278
- }
279
-
280
- function controlledDisposition(options: RunSingleOptions, base?: SingleResult): SingleResult | undefined {
281
- const control = options.control;
282
- if (!control?.isParkRequested() && !control?.isStopRequested()) return undefined;
283
- const result: SingleResult = base ?? {
284
- agent: options.agentName,
285
- agentSource: options.agent?.source ?? "unknown",
286
- task: control.getObjective(),
287
- exitCode: 0,
288
- messages: [],
289
- stderr: "",
290
- usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
291
- model: options.agent?.model,
292
- thinking: options.thinkingLevel,
293
- sessionId: options.sessionId,
294
- sessionDir: options.sessionDir,
295
- };
296
- result.task = control.getObjective();
297
- if (control.isParkRequested()) {
298
- result.parked = true;
299
- result.exitCode = 0;
300
- result.stopReason = undefined;
301
- result.errorMessage = undefined;
302
- } else {
303
- result.parked = undefined;
304
- result.exitCode = 1;
305
- result.stopReason = "aborted";
306
- result.errorMessage = control.getStopMessage();
307
- }
308
- return result;
309
- }
310
-
311
- /** Spawn one RPC attempt and wait for stable settlement. */
312
- export async function runSingleAgent(options: RunSingleOptions): Promise<SingleResult> {
313
- const {
314
- agent,
315
- agentName,
316
- thinkingLevel = SUBAGENT_THINKING_LEVEL,
317
- idleTimeoutMs = SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS,
318
- control,
319
- } = options;
320
- if (!agent) {
321
- return {
322
- agent: agentName,
323
- agentSource: "unknown",
324
- task: options.task,
325
- exitCode: 1,
326
- messages: [],
327
- stderr: `Unknown agent: "${agentName}".`,
328
- usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
329
- };
330
- }
331
-
332
- const disposition = controlledDisposition(options);
333
- if (disposition) return disposition;
334
- const objective = control?.getObjective() ?? options.task;
335
- let prompt = options.stdinText ?? `Task: ${objective}`;
336
- if (control && objective !== options.task) {
337
- prompt = options.sessionDir && sessionExists(options.sessionDir, options.sessionId ?? "")
338
- ? `Abandon the previous objective. New objective: ${objective}`
339
- : `Task: ${objective}`;
340
- }
341
- const result = await runRpcAgentAttempt({
342
- defaultCwd: options.defaultCwd,
343
- agent,
344
- agentName,
345
- task: objective,
346
- cwd: options.cwd,
347
- thinkingLevel,
348
- idleTimeoutMs,
349
- sessionDir: options.sessionDir,
350
- sessionId: options.sessionId,
351
- prompt,
352
- signal: options.signal,
353
- onLive: options.onLive,
354
- env: options.env,
355
- control,
356
- });
357
- result.task = control?.getObjective() ?? result.task;
358
- return result;
359
- }
360
-
361
- /**
362
- * Run one logical generation across an ordered model pool. Every candidate gets
363
- * startup retries plus same-model retries for transient provider failures;
364
- * terminal model errors skip those retries and advance immediately. All
365
- * candidates resume the same retained pi session.
366
- */
367
- export async function runSingleAgentWithModelFallback(
368
- options: RunSingleOptions,
369
- fallbackModelRefs: readonly string[] = [],
370
- ): Promise<SingleResult> {
371
- const agent = options.agent;
372
- const launchedRef = agent?.model;
373
- const startupDelays = options.startupRetryDelaysMs ?? SUBAGENT_STARTUP_RETRY_DELAYS_MS;
374
- const runDelays = options.runLevelRetryDelaysMs ?? SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS;
375
-
376
- const sessionId = options.sessionId ?? randomUUID();
377
- const sessionDir = options.sessionDir ?? (await mkdtemp(join(tmpdir(), "pi-subagent-session-")));
378
- const baseOptions: RunSingleOptions = { ...options, sessionDir, sessionId };
379
-
380
- const dispatchFailure = async (error: unknown): Promise<SingleResult> => {
381
- const errorMessage = error instanceof Error ? error.message : String(error);
382
- const hasSession = sessionExists(sessionDir, sessionId);
383
- if (!hasSession && !options.sessionDir) {
384
- await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
385
- }
386
- return {
387
- agent: options.agentName,
388
- agentSource: options.agent?.source ?? "unknown",
389
- task: options.control?.getObjective() ?? options.task,
390
- exitCode: 1,
391
- messages: [],
392
- stderr: errorMessage,
393
- usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
394
- model: options.agent?.model,
395
- thinking: options.thinkingLevel,
396
- stopReason: "error",
397
- errorMessage,
398
- dispatchFailed: true,
399
- ...(hasSession || options.sessionDir ? { sessionId, sessionDir } : {}),
400
- };
401
- };
402
-
403
- const runWithStartupRetry = async (opts: RunSingleOptions): Promise<SingleResult> => {
404
- let lastResult: SingleResult;
405
- let retries = 0;
406
- for (let attempt = 0; ; attempt++) {
407
- const immediate = controlledDisposition(opts);
408
- if (immediate) {
409
- if (immediate.parked && !options.sessionDir && !sessionExists(sessionDir, sessionId)) {
410
- await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
411
- immediate.sessionId = undefined;
412
- immediate.sessionDir = undefined;
413
- }
414
- return immediate;
415
- }
416
- const start = Date.now();
417
- try {
418
- lastResult = await runSingleAgent(opts);
419
- } catch (error) {
420
- const failed = await dispatchFailure(error);
421
- return controlledDisposition(opts, failed) ?? failed;
422
- }
423
- const durationMs = Date.now() - start;
424
- const controlled = controlledDisposition(opts, lastResult);
425
- if (controlled) return controlled;
426
- if (lastResult.parked || lastResult.stopReason === "aborted") return lastResult;
427
- if (!isRetryableStartupFailure(lastResult, durationMs)) {
428
- if (retries > 0 && !isFailedResult(lastResult)) lastResult.startupRetries = retries;
429
- return lastResult;
430
- }
431
- const delay = startupDelays[attempt];
432
- if (delay === undefined) {
433
- lastResult.errorMessage = formatStartupRetryExhaustedError(
434
- lastResult.model ?? opts.agent?.model ?? "default",
435
- attempt + 1,
436
- );
437
- lastResult.stopReason ??= "error";
438
- lastResult.dispatchFailed = true;
439
- return lastResult;
440
- }
441
- opts.control?.markRetrying();
442
- try {
443
- opts.onLive?.({ kind: "status", status: "running" });
444
- } catch {
445
- /* never throw from event handling */
446
- }
447
- if (!(await waitForControlledRetry(delay, opts.signal, opts.control))) {
448
- return controlledDisposition(opts, lastResult) ?? lastResult;
449
- }
450
- retries++;
451
- }
452
- };
453
-
454
- const fallbackRefs: string[] = [];
455
- const seenRefs = new Set<string>();
456
- if (launchedRef?.trim()) seenRefs.add(launchedRef.trim());
457
- for (const candidate of fallbackModelRefs) {
458
- const ref = candidate.trim();
459
- if (!ref || seenRefs.has(ref)) continue;
460
- seenRefs.add(ref);
461
- fallbackRefs.push(ref);
462
- }
463
-
464
- const candidates: Array<{ agent: AgentConfig | undefined; ref?: string }> = [
465
- { agent, ref: launchedRef?.trim() || undefined },
466
- ];
467
- if (agent) {
468
- for (const ref of fallbackRefs) candidates.push({ agent: { ...agent, model: ref }, ref });
469
- }
470
-
471
- let modelRetries = 0;
472
- let fallbackUsed = false;
473
- let result: SingleResult | undefined;
474
-
475
- const finish = async (settled: SingleResult): Promise<SingleResult> => {
476
- const persistedSession = sessionExists(sessionDir, sessionId);
477
- if (!settled.dispatchFailed || persistedSession || options.sessionDir) {
478
- settled.sessionId ??= sessionId;
479
- settled.sessionDir ??= sessionDir;
480
- } else {
481
- await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
482
- settled.sessionId = undefined;
483
- settled.sessionDir = undefined;
484
- }
485
- settled.task = options.control?.getObjective() ?? settled.task;
486
- if (fallbackUsed && launchedRef) settled.modelFallbackFrom = launchedRef;
487
- settled.modelRetries = modelRetries;
488
- options.control?.markSettled();
489
- return settled;
490
- };
491
-
492
- for (let candidateIndex = 0; candidateIndex < candidates.length; candidateIndex++) {
493
- const candidate = candidates[candidateIndex];
494
- fallbackUsed ||= candidateIndex > 0;
495
- const previousModel = result?.model ?? candidates[candidateIndex - 1]?.ref;
496
- const candidateOptions: RunSingleOptions = {
497
- ...baseOptions,
498
- agent: candidate.agent,
499
- ...(candidateIndex > 0
500
- ? {
501
- stdinText: buildResumePrompt(
502
- options.control?.getObjective() ?? options.task,
503
- buildFallbackResumeReason(previousModel),
504
- ),
505
- }
506
- : {}),
507
- };
508
- try {
509
- options.onLive?.({
510
- kind: "model",
511
- model: candidate.ref,
512
- ...(candidateIndex > 0 && launchedRef ? { fallbackFrom: launchedRef } : {}),
513
- });
514
- } catch {
515
- /* never throw from event handling */
516
- }
517
-
518
- result = await runWithStartupRetry(candidateOptions);
519
- if (result.parked || result.stopReason === "aborted") return result;
520
- if (!isModelLevelFailure(result)) return finish(result);
521
-
522
- if (!isTerminalModelError(result) && !isPermanentModelCandidateError(result)) {
523
- const retryOptions: RunSingleOptions = {
524
- ...candidateOptions,
525
- stdinText: buildResumePrompt(
526
- options.control?.getObjective() ?? options.task,
527
- "a transient provider error on the same model",
528
- ),
529
- };
530
- for (const delay of runDelays) {
531
- baseOptions.control?.markRetrying();
532
- try {
533
- options.onLive?.({ kind: "status", status: "running" });
534
- } catch {
535
- /* never throw from event handling */
536
- }
537
- if (!(await waitForControlledRetry(delay, options.signal, options.control))) {
538
- return controlledDisposition(baseOptions, result) ?? result;
539
- }
540
- result = await runWithStartupRetry(retryOptions);
541
- modelRetries++;
542
- if (result.parked || result.stopReason === "aborted") return result;
543
- if (
544
- !isModelLevelFailure(result) ||
545
- isTerminalModelError(result) ||
546
- isPermanentModelCandidateError(result)
547
- ) break;
548
- }
549
- }
550
-
551
- if (!isModelLevelFailure(result)) return finish(result);
552
- // Transient exhaustion plus terminal/permanent candidate errors advance
553
- // to the next configured candidate. Ordinary task/tool failures returned above.
554
- }
555
-
556
- return finish(result ?? (await dispatchFailure("No model candidate was attempted.")));
557
- }
1
+ /**
2
+ * Sub-agent result handling and resilient RPC launch orchestration.
3
+ *
4
+ * The process transport itself lives in rpc-run.ts. Each attempt starts pi in
5
+ * persistent `--mode rpc`, sends commands over strict LF-delimited JSONL, and
6
+ * settles only on `agent_settled`. This module owns startup-race recovery,
7
+ * selected-to-main model handoff, capability-clamped thinking, accounting, and
8
+ * result formatting around those attempts.
9
+ */
10
+
11
+ import { createHash, randomUUID } from "node:crypto";
12
+ import { type Dirent, mkdirSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
13
+ import { mkdtemp, rm } from "node:fs/promises";
14
+ import { tmpdir } from "node:os";
15
+ import { basename, join, resolve } from "node:path";
16
+ import type { Message } from "@earendil-works/pi-ai";
17
+ import type { AgentConfig } from "./agents.ts";
18
+ import { DEFAULT_THINKING_LEVEL, type ThinkingLevel } from "./config.ts";
19
+ import {
20
+ currentSubagentDepth,
21
+ DEPTH_ENV_VAR,
22
+ emptyUsage,
23
+ extractToolErrorText,
24
+ getPiInvocation,
25
+ RpcRunControl,
26
+ runRpcAgentAttempt,
27
+ sessionExists,
28
+ writeChildRetryPolicyExtension,
29
+ SUBAGENT_KILL_GRACE_MS,
30
+ type RpcSingleResult,
31
+ type SubagentLiveEvent,
32
+ type UsageStats,
33
+ } from "./rpc-run.ts";
34
+
35
+ export {
36
+ currentSubagentDepth,
37
+ DEPTH_ENV_VAR,
38
+ extractToolErrorText,
39
+ getPiInvocation,
40
+ RpcRunControl,
41
+ sessionExists,
42
+ SUBAGENT_KILL_GRACE_MS,
43
+ writeChildRetryPolicyExtension,
44
+ };
45
+ export type { SubagentLiveEvent, UsageStats };
46
+
47
+ export const SUBAGENT_THINKING_LEVEL: ThinkingLevel = DEFAULT_THINKING_LEVEL;
48
+ /** 0 disables the watchdog; dispatch supplies the configured timeout. */
49
+ export const SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS = 0;
50
+ export const SUBAGENT_STARTUP_RETRY_DELAYS_MS = [250, 750, 1500] as const;
51
+ export const MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS = 2000;
52
+
53
+ export interface SingleResult extends RpcSingleResult {}
54
+
55
+ export interface SubagentDetails {
56
+ mode: "single" | "parallel";
57
+ results: SingleResult[];
58
+ background?: boolean;
59
+ }
60
+
61
+ export function getFinalOutput(messages: Message[]): string {
62
+ for (let i = messages.length - 1; i >= 0; i--) {
63
+ const msg = messages[i];
64
+ if (msg.role === "assistant") {
65
+ for (const part of msg.content) {
66
+ if (part.type === "text") return part.text;
67
+ }
68
+ }
69
+ }
70
+ return "";
71
+ }
72
+
73
+ /** Only the last standalone reviewer verdict line counts. */
74
+ export function reviewVerdict(output: string): "pass" | "fail" | undefined {
75
+ const lines = output.split("\n");
76
+ for (let index = lines.length - 1; index >= 0; index--) {
77
+ const match = /^\s*VERDICT:\s*REVIEW_(PASS|FAIL)\s*$/i.exec(lines[index]);
78
+ if (match) return match[1].toUpperCase() === "PASS" ? "pass" : "fail";
79
+ }
80
+ return undefined;
81
+ }
82
+
83
+ export const RESULT_LINE_MAX = 200;
84
+
85
+ export interface TruncatedOutput {
86
+ text: string;
87
+ truncated: boolean;
88
+ }
89
+
90
+ export function truncateResultOutput(output: string, maxLines: number): TruncatedOutput {
91
+ const lines = output.split("\n");
92
+ if (lines.length <= maxLines && lines.every((line) => line.length <= RESULT_LINE_MAX)) {
93
+ return { text: output, truncated: false };
94
+ }
95
+ const kept = lines.slice(0, maxLines).map((line) =>
96
+ line.length > RESULT_LINE_MAX ? `${line.slice(0, RESULT_LINE_MAX)}…` : line,
97
+ );
98
+ return { text: kept.join("\n"), truncated: true };
99
+ }
100
+
101
+ export const RESULT_ARTIFACT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000;
102
+ export const RESULT_ARTIFACT_MAX_FILES_PER_PROJECT = 50;
103
+ // Explicit current prefix plus the strict timestamp/token convention used by 1.1.0.
104
+ const RESULT_ARTIFACT_NAME = /^(?:pi-subagent-\d{13,}-[0-9a-f]{12}|\d{13,}-[a-z0-9]{6})-[\w.-]+\.md$/;
105
+
106
+ interface ResultArtifactRetentionOptions {
107
+ now?: number;
108
+ maxAgeMs?: number;
109
+ maxFilesPerProject?: number;
110
+ }
111
+
112
+ /** Remove only stale/overflow Markdown result artifacts. Unknown files and
113
+ * symlinks are never touched. Called on each artifact write, so storage stays
114
+ * bounded without deleting a result that the current completion just linked. */
115
+ export function pruneResultArtifacts(
116
+ rootDir: string = join(tmpdir(), "pi-subagents-results"),
117
+ options: ResultArtifactRetentionOptions = {},
118
+ ): void {
119
+ const now = options.now ?? Date.now();
120
+ const maxAgeMs = Math.max(0, options.maxAgeMs ?? RESULT_ARTIFACT_MAX_AGE_MS);
121
+ const maxFiles = Math.max(0, Math.floor(options.maxFilesPerProject ?? RESULT_ARTIFACT_MAX_FILES_PER_PROJECT));
122
+ let projects: Dirent[];
123
+ try {
124
+ projects = readdirSync(rootDir, { withFileTypes: true });
125
+ } catch {
126
+ return;
127
+ }
128
+
129
+ for (const project of projects) {
130
+ if (!project.isDirectory() || project.isSymbolicLink()) continue;
131
+ const projectDir = join(rootDir, project.name);
132
+ let entries: Dirent[];
133
+ try {
134
+ entries = readdirSync(projectDir, { withFileTypes: true });
135
+ } catch {
136
+ continue;
137
+ }
138
+ const artifacts = entries
139
+ .filter((entry) => entry.isFile() && !entry.isSymbolicLink() && RESULT_ARTIFACT_NAME.test(entry.name))
140
+ .flatMap((entry) => {
141
+ const path = join(projectDir, entry.name);
142
+ try {
143
+ return [{ path, mtimeMs: statSync(path).mtimeMs }];
144
+ } catch {
145
+ return [];
146
+ }
147
+ })
148
+ .sort((left, right) => right.mtimeMs - left.mtimeMs);
149
+
150
+ for (const [index, artifact] of artifacts.entries()) {
151
+ if (index < maxFiles && now - artifact.mtimeMs <= maxAgeMs) continue;
152
+ try {
153
+ rmSync(artifact.path, { force: true });
154
+ } catch {
155
+ // Temp cleanup is best-effort; result delivery must still succeed.
156
+ }
157
+ }
158
+ }
159
+ }
160
+
161
+ export function resultArtifactProjectKey(cwd?: string): string {
162
+ if (!cwd) return "default";
163
+ let canonical: string;
164
+ try {
165
+ canonical = realpathSync.native(cwd);
166
+ } catch {
167
+ canonical = resolve(cwd);
168
+ }
169
+ if (process.platform === "win32") canonical = canonical.toLowerCase();
170
+ const slug = basename(canonical).replace(/[^\w.-]+/g, "_") || "project";
171
+ const digest = createHash("sha256").update(canonical).digest("hex").slice(0, 12);
172
+ return `${slug}-${digest}`;
173
+ }
174
+
175
+ export function writeResultArtifact(output: string, agentName: string, cwd?: string): string {
176
+ const rootDir = join(tmpdir(), "pi-subagents-results");
177
+ const dir = join(rootDir, resultArtifactProjectKey(cwd));
178
+ mkdirSync(dir, { recursive: true });
179
+ const safeName = agentName.replace(/[^\w.-]+/g, "_") || "agent";
180
+ const unique = `pi-subagent-${Date.now()}-${randomUUID().replaceAll("-", "").slice(0, 12)}`;
181
+ const filePath = join(dir, `${unique}-${safeName}.md`);
182
+ writeFileSync(filePath, output, "utf8");
183
+ pruneResultArtifacts(rootDir);
184
+ return filePath;
185
+ }
186
+
187
+ export function isFailedResult(result: SingleResult): boolean {
188
+ if (result.parked) return false;
189
+ return result.exitCode !== 0 || result.stopReason === "error" || result.stopReason === "aborted";
190
+ }
191
+
192
+ function lastAssistantMessage(messages: Message[]): Extract<Message, { role: "assistant" }> | undefined {
193
+ for (let index = messages.length - 1; index >= 0; index--) {
194
+ const message = messages[index];
195
+ if (message.role === "assistant") return message;
196
+ }
197
+ return undefined;
198
+ }
199
+
200
+ export function isModelLevelFailure(result: SingleResult): boolean {
201
+ if (!isFailedResult(result)) return false;
202
+ if (result.stopReason === "aborted") return false;
203
+ if (result.dispatchFailed) return false;
204
+ if (result.integrationStatus === "retained") return false;
205
+ if (result.errorMessage?.includes("idle timeout")) return true;
206
+ if (result.rpcPromptRejected) return true;
207
+
208
+ // Classification belongs to the final assistant turn, not the whole attempt.
209
+ // Earlier useful text or failed tool calls are retained session history and
210
+ // must not hide a later provider error (for example a second-turn 503).
211
+ const finalAssistant = lastAssistantMessage(result.messages);
212
+ if (finalAssistant) {
213
+ // Provider streams may preserve partial text on a terminal error. The stop
214
+ // reason, not content emptiness, is the transport boundary; ordinary tool or
215
+ // task failures settle with a non-error assistant stop reason.
216
+ return finalAssistant.stopReason === "error";
217
+ }
218
+
219
+ if ((result.failedTools?.length ?? 0) > 0) return false;
220
+ return Boolean(
221
+ result.rpcPromptAccepted ||
222
+ result.rpcActivity ||
223
+ result.errorMessage?.trim() ||
224
+ result.stderr.trim(),
225
+ );
226
+ }
227
+
228
+ export function isRetryableStartupFailure(result: SingleResult, durationMs: number): boolean {
229
+ if (result.exitCode === 0) return false;
230
+ if (result.stopReason === "aborted") return false;
231
+ if (result.dispatchFailed) return false;
232
+ if (result.rpcPromptAccepted || result.rpcActivity) return false;
233
+ if (result.errorMessage?.includes("idle timeout")) return false;
234
+ if (getFinalOutput(result.messages)) return false;
235
+ if (result.messages.length > 0) return false;
236
+ const usage = result.usage;
237
+ if (usage.turns || usage.input || usage.output || usage.cacheRead || usage.cacheWrite || usage.cost) return false;
238
+ if (durationMs > MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS) return false;
239
+ if (result.stderr.trim().length > 0) return false;
240
+ if (result.errorMessage && result.errorMessage.trim().length > 0) return false;
241
+ return true;
242
+ }
243
+
244
+ export function formatStartupRetryExhaustedError(model: string, attempts: number): string {
245
+ return `Subagent failed to start after ${attempts} attempt${attempts === 1 ? "" : "s"} on ${model}: the child exited before any model, tool, output, or usage activity. This is typically a concurrent pi startup race (several sub-agents starting at once). Retry the dispatch, or temporarily lower maxConcurrency in /subagents-setup.`;
246
+ }
247
+
248
+ export async function waitForStartupRetry(delayMs: number, signal?: AbortSignal): Promise<boolean> {
249
+ if (delayMs <= 0) return !signal?.aborted;
250
+ if (!signal) {
251
+ return new Promise<boolean>((resolve) => {
252
+ const timer = setTimeout(() => resolve(true), delayMs);
253
+ if (typeof timer.unref === "function") timer.unref();
254
+ });
255
+ }
256
+ if (signal.aborted) return false;
257
+ return new Promise<boolean>((resolve) => {
258
+ let settled = false;
259
+ const finish = (shouldRetry: boolean): void => {
260
+ if (settled) return;
261
+ settled = true;
262
+ clearTimeout(timer);
263
+ signal.removeEventListener("abort", onAbort);
264
+ resolve(shouldRetry);
265
+ };
266
+ const onAbort = (): void => finish(false);
267
+ const timer = setTimeout(() => finish(true), delayMs);
268
+ if (typeof timer.unref === "function") timer.unref();
269
+ signal.addEventListener("abort", onAbort, { once: true });
270
+ });
271
+ }
272
+
273
+ async function waitForControlledRetry(
274
+ delayMs: number,
275
+ signal: AbortSignal | undefined,
276
+ control: RpcRunControl | undefined,
277
+ ): Promise<boolean> {
278
+ let remaining = delayMs;
279
+ while (remaining > 0) {
280
+ if (control?.isParkRequested() || control?.isStopRequested()) return false;
281
+ const slice = Math.min(remaining, 50);
282
+ if (!(await waitForStartupRetry(slice, signal))) return false;
283
+ remaining -= slice;
284
+ }
285
+ return !signal?.aborted && !control?.isParkRequested() && !control?.isStopRequested();
286
+ }
287
+
288
+ export function getResultOutput(result: SingleResult): string {
289
+ if (isFailedResult(result)) {
290
+ const error = result.errorMessage || result.stderr;
291
+ const partial = getFinalOutput(result.messages);
292
+ if (error && partial) return `${error}\n\n--- Partial output ---\n${partial}`;
293
+ return error || partial || "(no output)";
294
+ }
295
+ return getFinalOutput(result.messages) || "(no output)";
296
+ }
297
+
298
+ export function buildResumePrompt(task: string, reason: string): string {
299
+ return `You are resuming an earlier sub-agent session after ${reason}. Your earlier work — searches, reads, edits, and reasoning — is preserved in this session's history above; review it before acting. Original task: ${task}. Pick up exactly where you left off and finish it. Do NOT redo searches, reads, or edits you already completed unless a step clearly failed. Continue now.`;
300
+ }
301
+
302
+ export function buildFallbackResumeReason(fromModel?: string): string {
303
+ return fromModel
304
+ ? `the selected model (${fromModel}) failed at the model/provider level, so the current main model is continuing`
305
+ : "the selected model failed at the model/provider level, so the current main model is continuing";
306
+ }
307
+
308
+ export interface RunSingleOptions {
309
+ defaultCwd: string;
310
+ agent: AgentConfig;
311
+ agentName: string;
312
+ task: string;
313
+ cwd?: string;
314
+ thinkingLevel?: ThinkingLevel;
315
+ /** Resolve the effective level for each runtime model candidate. */
316
+ thinkingLevelForModel?: (modelRef?: string) => ThinkingLevel;
317
+ idleTimeoutMs?: number;
318
+ startupRetryDelaysMs?: readonly number[];
319
+ sessionDir?: string;
320
+ sessionId?: string;
321
+ /** Initial RPC prompt. Kept under the old name to limit caller churn. */
322
+ stdinText?: string;
323
+ signal?: AbortSignal;
324
+ onLive?: (event: SubagentLiveEvent) => void;
325
+ makeDetails: (results: SingleResult[]) => SubagentDetails;
326
+ env?: NodeJS.ProcessEnv;
327
+ /** Stable logical-generation controller shared across retry attempts. */
328
+ control?: RpcRunControl;
329
+ }
330
+
331
+ function controlledDisposition(options: RunSingleOptions, base?: SingleResult): SingleResult | undefined {
332
+ const control = options.control;
333
+ if (!control?.isParkRequested() && !control?.isStopRequested()) return undefined;
334
+ const result: SingleResult = base ?? {
335
+ agent: options.agentName,
336
+ task: control.getObjective(),
337
+ exitCode: 0,
338
+ messages: [],
339
+ stderr: "",
340
+ usage: emptyUsage(),
341
+ model: options.agent.model,
342
+ thinking: options.thinkingLevel,
343
+ sessionId: options.sessionId,
344
+ sessionDir: options.sessionDir,
345
+ };
346
+ result.task = control.getObjective();
347
+ if (control.isParkRequested()) {
348
+ result.parked = true;
349
+ result.exitCode = 0;
350
+ result.stopReason = undefined;
351
+ result.errorMessage = undefined;
352
+ } else {
353
+ result.parked = undefined;
354
+ result.exitCode = 1;
355
+ result.stopReason = "aborted";
356
+ result.errorMessage = control.getStopMessage();
357
+ }
358
+ return result;
359
+ }
360
+
361
+ /** Spawn one RPC attempt and wait for stable settlement. */
362
+ export async function runSingleAgent(options: RunSingleOptions): Promise<SingleResult> {
363
+ const {
364
+ agent,
365
+ agentName,
366
+ thinkingLevel = SUBAGENT_THINKING_LEVEL,
367
+ idleTimeoutMs = SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS,
368
+ control,
369
+ } = options;
370
+ const disposition = controlledDisposition(options);
371
+ if (disposition) return disposition;
372
+ const objective = control?.getObjective() ?? options.task;
373
+ let prompt = options.stdinText ?? `Task: ${objective}`;
374
+ if (control && objective !== options.task) {
375
+ prompt = options.sessionDir && sessionExists(options.sessionDir, options.sessionId ?? "")
376
+ ? `Abandon the previous objective. New objective: ${objective}`
377
+ : `Task: ${objective}`;
378
+ }
379
+ const result = await runRpcAgentAttempt({
380
+ defaultCwd: options.defaultCwd,
381
+ agent,
382
+ agentName,
383
+ task: objective,
384
+ cwd: options.cwd,
385
+ thinkingLevel,
386
+ idleTimeoutMs,
387
+ sessionDir: options.sessionDir,
388
+ sessionId: options.sessionId,
389
+ prompt,
390
+ signal: options.signal,
391
+ onLive: options.onLive,
392
+ env: options.env,
393
+ control,
394
+ });
395
+ result.task = control?.getObjective() ?? result.task;
396
+ return result;
397
+ }
398
+
399
+ /**
400
+ * Run one logical generation on the selected model, then hand directly to the
401
+ * current main model after any model/provider-level failure. Startup-race retries
402
+ * remain process-level recovery; provider/model retries and extra candidates do not.
403
+ * Both attempts resume the same retained Pi session.
404
+ */
405
+ export async function runSingleAgentWithMainFallback(
406
+ options: RunSingleOptions,
407
+ mainFallbackRef?: string,
408
+ ): Promise<SingleResult> {
409
+ const agent = options.agent;
410
+ const launchedRef = agent?.model;
411
+ const startupDelays = options.startupRetryDelaysMs ?? SUBAGENT_STARTUP_RETRY_DELAYS_MS;
412
+
413
+ const sessionId = options.sessionId ?? randomUUID();
414
+ const sessionDir = options.sessionDir ?? (await mkdtemp(join(tmpdir(), "pi-subagent-session-")));
415
+ const baseOptions: RunSingleOptions = { ...options, sessionDir, sessionId };
416
+
417
+ const dispatchFailure = async (error: unknown): Promise<SingleResult> => {
418
+ const errorMessage = error instanceof Error ? error.message : String(error);
419
+ const hasSession = sessionExists(sessionDir, sessionId);
420
+ if (!hasSession && !options.sessionDir) {
421
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
422
+ }
423
+ return {
424
+ agent: options.agentName,
425
+ task: options.control?.getObjective() ?? options.task,
426
+ exitCode: 1,
427
+ messages: [],
428
+ stderr: errorMessage,
429
+ usage: emptyUsage(),
430
+ model: options.agent.model,
431
+ thinking: options.thinkingLevel,
432
+ stopReason: "error",
433
+ errorMessage,
434
+ dispatchFailed: true,
435
+ ...(hasSession || options.sessionDir ? { sessionId, sessionDir } : {}),
436
+ };
437
+ };
438
+
439
+ const runWithStartupRetry = async (opts: RunSingleOptions): Promise<SingleResult> => {
440
+ let lastResult: SingleResult;
441
+ let retries = 0;
442
+ for (let attempt = 0; ; attempt++) {
443
+ const immediate = controlledDisposition(opts);
444
+ if (immediate) {
445
+ if (immediate.parked && !options.sessionDir && !sessionExists(sessionDir, sessionId)) {
446
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
447
+ immediate.sessionId = undefined;
448
+ immediate.sessionDir = undefined;
449
+ }
450
+ return immediate;
451
+ }
452
+ const start = Date.now();
453
+ try {
454
+ lastResult = await runSingleAgent(opts);
455
+ } catch (error) {
456
+ const failed = await dispatchFailure(error);
457
+ return controlledDisposition(opts, failed) ?? failed;
458
+ }
459
+ const durationMs = Date.now() - start;
460
+ const controlled = controlledDisposition(opts, lastResult);
461
+ if (controlled) return controlled;
462
+ if (lastResult.parked || lastResult.stopReason === "aborted") return lastResult;
463
+ if (!isRetryableStartupFailure(lastResult, durationMs)) {
464
+ if (retries > 0 && !isFailedResult(lastResult)) lastResult.startupRetries = retries;
465
+ return lastResult;
466
+ }
467
+ const delay = startupDelays[attempt];
468
+ if (delay === undefined) {
469
+ lastResult.errorMessage = formatStartupRetryExhaustedError(
470
+ lastResult.model ?? opts.agent.model ?? "default",
471
+ attempt + 1,
472
+ );
473
+ lastResult.stopReason ??= "error";
474
+ lastResult.dispatchFailed = true;
475
+ return lastResult;
476
+ }
477
+ opts.control?.markRetrying();
478
+ try {
479
+ opts.onLive?.({ kind: "status", status: "running" });
480
+ } catch {
481
+ /* never throw from event handling */
482
+ }
483
+ if (!(await waitForControlledRetry(delay, opts.signal, opts.control))) {
484
+ return controlledDisposition(opts, lastResult) ?? lastResult;
485
+ }
486
+ retries++;
487
+ }
488
+ };
489
+
490
+ const selectedRef = launchedRef?.trim() || undefined;
491
+ const normalizedMainRef = mainFallbackRef?.trim() || undefined;
492
+ const candidates: Array<{ agent: AgentConfig; ref?: string }> = [
493
+ { agent, ref: selectedRef },
494
+ ];
495
+ if (normalizedMainRef && normalizedMainRef !== selectedRef) {
496
+ candidates.push({ agent: { ...agent, model: normalizedMainRef }, ref: normalizedMainRef });
497
+ }
498
+
499
+ let fallbackUsed = false;
500
+ let result: SingleResult | undefined;
501
+ const priorFailedTools: NonNullable<SingleResult["failedTools"]> = [];
502
+ const priorUsage = emptyUsage();
503
+
504
+ const retainAttemptDiagnostics = (attempt: SingleResult): void => {
505
+ priorFailedTools.push(...(attempt.failedTools ?? []));
506
+ priorUsage.input += attempt.usage.input;
507
+ priorUsage.output += attempt.usage.output;
508
+ priorUsage.cacheRead += attempt.usage.cacheRead;
509
+ priorUsage.cacheWrite += attempt.usage.cacheWrite;
510
+ priorUsage.cost += attempt.usage.cost;
511
+ priorUsage.turns += attempt.usage.turns;
512
+ priorUsage.contextTokens = attempt.usage.contextTokens || priorUsage.contextTokens;
513
+ };
514
+
515
+ const finish = async (settled: SingleResult): Promise<SingleResult> => {
516
+ if (priorFailedTools.length > 0) {
517
+ settled.failedTools = [...priorFailedTools, ...(settled.failedTools ?? [])];
518
+ }
519
+ if (
520
+ priorUsage.turns || priorUsage.input || priorUsage.output || priorUsage.cacheRead ||
521
+ priorUsage.cacheWrite || priorUsage.cost || priorUsage.contextTokens
522
+ ) {
523
+ settled.usage = {
524
+ input: priorUsage.input + settled.usage.input,
525
+ output: priorUsage.output + settled.usage.output,
526
+ cacheRead: priorUsage.cacheRead + settled.usage.cacheRead,
527
+ cacheWrite: priorUsage.cacheWrite + settled.usage.cacheWrite,
528
+ cost: priorUsage.cost + settled.usage.cost,
529
+ turns: priorUsage.turns + settled.usage.turns,
530
+ contextTokens: settled.usage.contextTokens || priorUsage.contextTokens,
531
+ };
532
+ }
533
+ const persistedSession = sessionExists(sessionDir, sessionId);
534
+ if (!settled.dispatchFailed || persistedSession || options.sessionDir) {
535
+ settled.sessionId ??= sessionId;
536
+ settled.sessionDir ??= sessionDir;
537
+ } else {
538
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
539
+ settled.sessionId = undefined;
540
+ settled.sessionDir = undefined;
541
+ }
542
+ settled.task = options.control?.getObjective() ?? settled.task;
543
+ if (fallbackUsed && launchedRef) settled.modelFallbackFrom = launchedRef;
544
+ options.control?.markSettled();
545
+ return settled;
546
+ };
547
+
548
+ for (let candidateIndex = 0; candidateIndex < candidates.length; candidateIndex++) {
549
+ const candidate = candidates[candidateIndex];
550
+ fallbackUsed ||= candidateIndex > 0;
551
+ const previousModel = result?.model ?? candidates[candidateIndex - 1]?.ref;
552
+ const candidateThinking = options.thinkingLevelForModel?.(candidate.ref) ?? options.thinkingLevel;
553
+ const candidateOptions: RunSingleOptions = {
554
+ ...baseOptions,
555
+ agent: candidate.agent,
556
+ thinkingLevel: candidateThinking,
557
+ ...(candidateIndex > 0
558
+ ? {
559
+ stdinText: buildResumePrompt(
560
+ options.control?.getObjective() ?? options.task,
561
+ buildFallbackResumeReason(previousModel),
562
+ ),
563
+ }
564
+ : {}),
565
+ };
566
+ try {
567
+ options.onLive?.({
568
+ kind: "model",
569
+ model: candidate.ref,
570
+ thinking: candidateThinking,
571
+ ...(candidateIndex > 0 && launchedRef ? { fallbackFrom: launchedRef } : {}),
572
+ });
573
+ } catch {
574
+ /* never throw from event handling */
575
+ }
576
+
577
+ result = await runWithStartupRetry(candidateOptions);
578
+ if (result.parked || result.stopReason === "aborted") return finish(result);
579
+ if (!isModelLevelFailure(result)) return finish(result);
580
+ // Any model-level failure advances immediately to the sole fallback (the
581
+ // current main model). Retain selected-attempt tool diagnostics and usage;
582
+ // ordinary task/tool failures returned above without a handoff.
583
+ if (candidateIndex < candidates.length - 1) retainAttemptDiagnostics(result);
584
+ }
585
+
586
+ return finish(result!);
587
+ }