@ferris1225/pi-subagents 0.29.0 → 0.32.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/spawn.ts CHANGED
@@ -1,856 +1,562 @@
1
- /**
2
- * Sub-agent dispatch: each agent runs as an isolated `pi` child process
3
- * (`--mode json -p --no-session`). The agent's system prompt (the .md body) is
4
- * written to a temp file and passed via `--append-system-prompt` (which accepts a
5
- * file path). The task itself is sent through the child's stdin pipe, not another
6
- * temp file or command-line argument. Child stdout is a JSON-lines event stream;
7
- * we accumulate assistant messages from `message_end` events.
8
- *
9
- * Adapted from the official pi example `examples/extensions/subagent`.
10
- */
11
-
12
- import { spawn, type ChildProcess } from "node:child_process";
13
- import { existsSync, mkdirSync, unlinkSync, rmdirSync, writeFileSync } from "node:fs";
14
- import { mkdtemp, rm, writeFile } from "node:fs/promises";
15
- import { tmpdir } from "node:os";
16
- import { basename, join } from "node:path";
17
- import { StringDecoder } from "node:string_decoder";
18
- import type { Message } from "@earendil-works/pi-ai";
19
- import type { AgentConfig, AgentSource } from "./agents.ts";
20
- import { DEFAULT_THINKING_LEVEL, type ThinkingLevel } from "./config.ts";
21
-
22
- /**
23
- * Limits are configurable: see maxConcurrency in config.ts (default 4, via
24
- * /subagents-setup or pi-subagents.json).
25
- */
26
- /** Default thinking level for sub-agents. pi clamps it to the resolved model's support. */
27
- export const SUBAGENT_THINKING_LEVEL: ThinkingLevel = DEFAULT_THINKING_LEVEL;
28
- export const DEPTH_ENV_VAR = "PI_SUBAGENT_DEPTH";
29
- export const SUBAGENT_KILL_GRACE_MS = 5_000;
30
- /** Default idle watchdog: terminate a child whose stdout goes silent for this
31
- * many milliseconds. 0 disables it. The actual value comes from config
32
- * (idleTimeoutSec); this constant is only a fallback for tests. */
33
- export const SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS = 0;
34
-
35
- /** Backoff schedule for retrying a child that exited before any model or tool
36
- * activity — the signature of a concurrent pi startup race, where several
37
- * sub-agents contending for pi's startup lock lose and exit with nothing on
38
- * stdout. Bounded and short so persistent launch failures are not amplified
39
- * while the startup lock clears. */
40
- export const SUBAGENT_STARTUP_RETRY_DELAYS_MS = [250, 750, 1500] as const;
41
- /** A genuine startup race fails well before a model request can complete. */
42
- export const MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS = 2000;
43
-
44
- /** Backoff schedule (ms) for retrying a run whose configured model failed at the
45
- * provider level with a TRANSIENT error (503/429/timeout/network/overloaded/...) —
46
- * i.e. NOT a terminal error (quota exhausted, billing, invalid API key). The same
47
- * model is relaunched (each relaunch gets its own startup-retry inner loop), so a
48
- * one-off provider hiccup recovers without demoting the configured agent model.
49
- *
50
- * This sits OUTSIDE pi-ai's per-request provider retry (default 3 attempts, 2/4/8s
51
- * backoff): when the provider still can't recover after its own retries, the
52
- * child exits carrying the final error, and this layer relaunches the whole run
53
- * up to len(delays) more times before falling back to the main-window model.
54
- *
55
- * Bounded and capped so a stubborn outage does not stall a dispatch forever. */
56
- export const SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 30_000] as const;
57
-
58
- export interface UsageStats {
59
- input: number;
60
- output: number;
61
- cacheRead: number;
62
- cacheWrite: number;
63
- cost: number;
64
- contextTokens: number;
65
- turns: number;
66
- }
67
-
68
- export interface SingleResult {
69
- agent: string;
70
- agentSource: AgentSource | "unknown";
71
- task: string;
72
- exitCode: number; // -1 = still running
73
- messages: Message[];
74
- stderr: string;
75
- usage: UsageStats;
76
- model?: string;
77
- /** Effective thinking strength this run was launched with. */
78
- thinking?: string;
79
- stopReason?: string;
80
- errorMessage?: string;
81
- /** Model the run degraded from: set when a failed run was retried with the main-window model. */
82
- modelFallbackFrom?: string;
83
- /** True when the result was synthesized from a thrown exception (spawn infra,
84
- * temp-file/fs errors, delivery bugs) instead of being produced by the agent
85
- * process. A dispatch failure is never a model-level failure. */
86
- dispatchFailed?: boolean;
87
- /** How many times the run was relaunched after a silent, zero-activity startup
88
- * exit (a concurrent pi startup race) before it produced a result. Set only when
89
- * the run actually recovered after retrying, so callers can surface it. */
90
- startupRetries?: number;
91
- /** How many times the SAME configured model was relaunched after a transient
92
- * provider-level failure (503/429/timeout/network/...) before the run produced
93
- * a result. Set on recovery and on fall-back to the main-window model; left
94
- * undefined for a terminal (quota/billing/invalid-key) error that short-
95
- * circuits before any retry, since no relaunch happened. */
96
- modelRetries?: number;
97
- /** Tool calls that failed inside the run (from tool_execution_end events). A
98
- * clean process exit can still hide a failed build/test/tool — the completion
99
- * message must surface these so the main agent is never misled by a rosy final
100
- * text (e.g. a worker that ended with "keep waiting" while its build failed). */
101
- failedTools?: Array<{ toolName: string; error: string }>;
102
- }
103
-
104
- export interface SubagentDetails {
105
- mode: "single" | "parallel";
106
- results: SingleResult[];
107
- /** The tool returned immediately while the child process continues in the background. */
108
- background?: boolean;
109
- }
110
-
111
- export type SubagentLiveEvent =
112
- | { kind: "status"; status: "queued" | "running" | "done" | "failed" }
113
- | { kind: "usage"; usage: UsageStats; model?: string }
114
- | { kind: "tool_start"; toolName: string; args: unknown }
115
- | { kind: "tool_end"; toolName: string; isError: boolean }
116
- | { kind: "thinking" }
117
- | { kind: "text" };
118
-
119
- function emptyUsage(): UsageStats {
120
- return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 };
121
- }
122
-
123
- export function getFinalOutput(messages: Message[]): string {
124
- for (let i = messages.length - 1; i >= 0; i--) {
125
- const msg = messages[i];
126
- if (msg.role === "assistant") {
127
- for (const part of msg.content) {
128
- if (part.type === "text") return part.text;
129
- }
130
- }
131
- }
132
- return "";
133
- }
134
-
135
- /**
136
- * Parse the machine-readable verdict a reviewer emits (see agents/reviewer.md).
137
- * Only the LAST standalone `VERDICT: REVIEW_PASS/FAIL` line counts, so a report
138
- * that merely discusses the tokens cannot be misclassified. Returns undefined
139
- * when no verdict marker is present, so non-review agents are never mistaken
140
- * for reviews.
141
- */
142
- export function reviewVerdict(output: string): "pass" | "fail" | undefined {
143
- const lines = output.split("\n");
144
- for (let index = lines.length - 1; index >= 0; index--) {
145
- const match = /^\s*VERDICT:\s*REVIEW_(PASS|FAIL)\s*$/i.exec(lines[index]);
146
- if (match) return match[1].toUpperCase() === "PASS" ? "pass" : "fail";
147
- }
148
- return undefined;
149
- }
150
-
151
- /** Tool errors are usually the trailing lines of a long output (build logs);
152
- * keep the last non-empty lines, clipped to RESULT_LINE_MAX each. */
153
- export function extractToolErrorText(content: unknown): string {
154
- const parts = Array.isArray(content) ? content : [];
155
- const text = parts
156
- .filter(
157
- (part): part is { type: "text"; text: string } =>
158
- typeof part === "object" &&
159
- part !== null &&
160
- (part as { type?: unknown }).type === "text" &&
161
- typeof (part as { text?: unknown }).text === "string",
162
- )
163
- .map((part) => part.text)
164
- .join("\n");
165
- return text
166
- .split("\n")
167
- .map((line) => line.trim())
168
- .filter(Boolean)
169
- .slice(-3)
170
- .map((line) => (line.length > RESULT_LINE_MAX ? `${line.slice(0, RESULT_LINE_MAX)}…` : line))
171
- .join("\n");
172
- }
173
-
174
- /** Hard cap for a single line inside a truncated result (minified blobs must not blow up). */
175
- export const RESULT_LINE_MAX = 200;
176
-
177
- export interface TruncatedOutput {
178
- /** The result text that fits in the completion message. */
179
- text: string;
180
- /** True when lines were dropped or shortened, so the full text is written to disk. */
181
- truncated: boolean;
182
- }
183
-
184
- /** Cap result text for the main conversation: keep the first `maxLines` lines, at most RESULT_LINE_MAX chars each. */
185
- export function truncateResultOutput(output: string, maxLines: number): TruncatedOutput {
186
- const lines = output.split("\n");
187
- if (lines.length <= maxLines && lines.every((line) => line.length <= RESULT_LINE_MAX)) {
188
- return { text: output, truncated: false };
189
- }
190
- const kept = lines.slice(0, maxLines).map((line) =>
191
- line.length > RESULT_LINE_MAX ? `${line.slice(0, RESULT_LINE_MAX)}…` : line,
192
- );
193
- return { text: kept.join("\n"), truncated: true };
194
- }
195
-
196
- /** Persist the full result where the main agent can read it on demand. Returns the file path.
197
- * Results are grouped under a per-project subdirectory so concurrent projects don't
198
- * litter a single flat folder. */
199
- export function writeResultArtifact(output: string, agentName: string, cwd?: string): string {
200
- const projectSlug = cwd
201
- ? basename(cwd).replace(/[^\w.-]+/g, "_") || "default"
202
- : "default";
203
- const dir = join(tmpdir(), "pi-subagents-results", projectSlug);
204
- mkdirSync(dir, { recursive: true });
205
- const safeName = agentName.replace(/[^\w.-]+/g, "_");
206
- // A random suffix keeps same-millisecond writes from clobbering each other.
207
- const unique = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
208
- const filePath = join(dir, `${unique}-${safeName}.md`);
209
- writeFileSync(filePath, output, "utf8");
210
- return filePath;
211
- }
212
-
213
- export function isFailedResult(result: SingleResult): boolean {
214
- return result.exitCode !== 0 || result.stopReason === "error" || result.stopReason === "aborted";
215
- }
216
-
217
- /**
218
- * True when a failed run never got usable output from its model: the provider
219
- * rejected the call before the model produced any text (bad model id, auth,
220
- * thinking level, quota, ...). Task-level failures — the model worked and the
221
- * task failed — and aborts/timeouts are NOT model-level and must not degrade.
222
- */
223
- export function isModelLevelFailure(result: SingleResult): boolean {
224
- if (!isFailedResult(result)) return false;
225
- if (result.stopReason === "aborted") return false;
226
- // A result synthesized from a thrown exception (spawn infra, fs, delivery
227
- // bugs) never came from the provider: it is a dispatch failure, not a
228
- // model-level one, and must not be handed back as a model problem.
229
- if (result.dispatchFailed) return false;
230
- // An idle timeout (stdout went silent) signals a stalled provider connection,
231
- // not a task-level failure: allow model fallback even if the model produced
232
- // partial output before going quiet.
233
- if (result.errorMessage?.includes("idle timeout")) return true;
234
- // The model produced text: the failure belongs to the task, not the model.
235
- if (getFinalOutput(result.messages)) return false;
236
- // Require evidence the failure came from the model/provider (an error
237
- // message or stderr), not from the child process failing to start.
238
- return result.messages.length > 0 || result.stderr.trim().length > 0;
239
- }
240
-
241
- /** Patterns that signal a TERMINAL provider/account error: retrying the same
242
- * model (or falling back to the main-window model under the same account) cannot
243
- * fix it, so the run skips both run-level retry and model fallback and is handed
244
- * back to the main agent. This is the complement of pi-ai's transient-error set
245
- * (429/5xx/overloaded/network/timeout/...): anything NOT matching here is treated
246
- * as transient and retried on the same model before degrading.
247
- *
248
- * Mirrors pi-ai's NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN (quota/billing/
249
- * subscription-limit text) and adds the auth/credential failures the user cited
250
- * ("key无效"). Auth is account-scoped, so a fallback model on the same provider
251
- * would fail identically — hand it to the main agent immediately. */
252
- const TERMINAL_MODEL_ERROR_PATTERN =
253
- /insufficient_quota|quota\s+exceeded|exceeded[^.\n]{0,40}quota|out\s+of\s+budget|billing|usage\s+limit|usage_limit|gousagelimiterror|freeusagelimiterror|monthly\s+usage\s+limit\s+reached|available\s+balance|invalid\s+(?:api\s+)?key|incorrect\s+api\s+key|unauthori[sz]ed|\b401\b|\b403\b|forbidden|permission\s+denied/i;
254
-
255
- /** True when a model-level failure carries a TERMINAL error message quota
256
- * exhaustion, billing, an invalid API key, auth rejection. Such a run is NEVER
257
- * retried on the same model and never falls back to the main-window model: the
258
- * account is the bottleneck, so it is handed back to the main agent to fix.
259
- *
260
- * Caller must first confirm `isModelLevelFailure(result)` — aborts and
261
- * dispatch-crafted results never reach this classifier. */
262
- export function isTerminalModelError(result: SingleResult): boolean {
263
- const message = result.errorMessage?.trim();
264
- if (message) return TERMINAL_MODEL_ERROR_PATTERN.test(message);
265
- // Only consult stderr when there is no structured errorMessage: pi-ai surfaces
266
- // provider errors via message_end -> errorMessage, so a transient errorMessage
267
- // (e.g. "503 Service Unavailable") must not be overridden by noisy stderr that
268
- // happens to mention a terminal-looking word (an npm warning, a proxy banner).
269
- // This keeps transient failures retryable even when stderr is chatty.
270
- const stderr = result.stderr.trim();
271
- return stderr.length > 0 && TERMINAL_MODEL_ERROR_PATTERN.test(stderr);
272
- }
273
-
274
- /**
275
- * True when a failed run produced NO model, tool, output, or usage activity
276
- * within the startup window — the signature of a concurrent pi startup race,
277
- * where the child lost pi's startup lock and exited before doing anything.
278
- * Such a run is safe to relaunch: nothing was mutated and no provider call
279
- * completed, so retrying cannot duplicate work.
280
- *
281
- * Fails closed: any final output, assistant message, usage, stderr, structured
282
- * error message, idle-timeout, abort, dispatch crash, or run that outlived the
283
- * startup window disqualifies the run from retry (it either did real work or
284
- * carries a real error that belongs to model fallback / normal failure
285
- * delivery instead). Only a clean, SILENT, fast, zero-activity exit retries.
286
- */
287
- export function isRetryableStartupFailure(result: SingleResult, durationMs: number): boolean {
288
- if (result.exitCode === 0) return false;
289
- if (result.stopReason === "aborted") return false;
290
- if (result.dispatchFailed) return false;
291
- if (result.errorMessage?.includes("idle timeout")) return false;
292
- if (getFinalOutput(result.messages)) return false;
293
- if (result.messages.length > 0) return false;
294
- const usage = result.usage;
295
- if (usage.turns || usage.input || usage.output || usage.cacheRead || usage.cacheWrite || usage.cost) return false;
296
- if (durationMs > MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS) return false;
297
- // Any stderr or structured error could be a real provider/config error (auth,
298
- // bad model id, quota, ...) that must not be amplified by retry. A silent
299
- // zero-activity exit — no stdout, no stderr, no error message — is the race.
300
- if (result.stderr.trim().length > 0) return false;
301
- if (result.errorMessage && result.errorMessage.trim().length > 0) return false;
302
- return true;
303
- }
304
-
305
- /** Error surfaced when every startup-retry attempt still exited with no
306
- * activity. Tells the main agent the dispatch never reached a model and what to
307
- * do (retry, or lower maxConcurrency). */
308
- export function formatStartupRetryExhaustedError(model: string, attempts: number): string {
309
- return `Subagent failed to start after ${attempts} attempt${attempts === 1 ? "" : "s"} on ${model}: the child exited before any model, tool, output, or usage activity. This is typically a concurrent pi startup race (several sub-agents starting at once). Retry the dispatch, or temporarily lower maxConcurrency in /subagents-setup.`;
310
- }
311
-
312
- /** Wait out a startup-retry backoff. Resolves false immediately (do not retry)
313
- * when the signal is or becomes aborted during the wait, so cancellation never
314
- * delays delivering the last result. The timer is unref'd so it cannot keep the
315
- * event loop alive on shutdown. */
316
- export async function waitForStartupRetry(delayMs: number, signal?: AbortSignal): Promise<boolean> {
317
- if (delayMs <= 0) return !signal?.aborted;
318
- if (!signal) {
319
- return new Promise<boolean>((resolve) => {
320
- const timer = setTimeout(() => resolve(true), delayMs);
321
- if (typeof timer.unref === "function") timer.unref();
322
- });
323
- }
324
- if (signal.aborted) return false;
325
- return new Promise<boolean>((resolve) => {
326
- let settled = false;
327
- const finish = (shouldRetry: boolean): void => {
328
- if (settled) return;
329
- settled = true;
330
- clearTimeout(timer);
331
- signal.removeEventListener("abort", onAbort);
332
- resolve(shouldRetry);
333
- };
334
- const onAbort = (): void => finish(false);
335
- const timer = setTimeout(() => finish(true), delayMs);
336
- if (typeof timer.unref === "function") timer.unref();
337
- signal.addEventListener("abort", onAbort, { once: true });
338
- });
339
- }
340
-
341
- export function getResultOutput(result: SingleResult): string {
342
- if (isFailedResult(result)) {
343
- const error = result.errorMessage || result.stderr;
344
- const partial = getFinalOutput(result.messages);
345
- if (error && partial) return `${error}\n\n--- Partial output ---\n${partial}`;
346
- return error || partial || "(no output)";
347
- }
348
- return getFinalOutput(result.messages) || "(no output)";
349
- }
350
-
351
- async function writePromptToTempFile(agentName: string, prompt: string): Promise<{ dir: string; filePath: string }> {
352
- const dir = await mkdtemp(join(tmpdir(), "pi-subagents-"));
353
- const safeName = agentName.replace(/[^\w.-]+/g, "_");
354
- const filePath = join(dir, `prompt-${safeName}.md`);
355
- await writeFile(filePath, prompt, "utf8");
356
- return { dir, filePath };
357
- }
358
-
359
- /** Resolve how to invoke the SAME pi build as the current process. */
360
- export function getPiInvocation(args: string[]): { command: string; args: string[] } {
361
- const currentScript = process.argv[1];
362
- const isBunVirtualScript = currentScript?.startsWith("/$bunfs/root/");
363
- if (currentScript && !isBunVirtualScript && existsSync(currentScript)) {
364
- return { command: process.execPath, args: [currentScript, ...args] };
365
- }
366
- const execName = basename(process.execPath).toLowerCase();
367
- const isGenericRuntime = /^(node|bun)(\.exe)?$/.test(execName);
368
- if (!isGenericRuntime) return { command: process.execPath, args };
369
- return { command: "pi", args };
370
- }
371
-
372
- /** Terminate the child and any tool processes it left behind. */
373
- function terminateProcessTree(proc: ChildProcess, force: boolean): void {
374
- if (process.platform === "win32" && proc.pid !== undefined) {
375
- const killer = spawn("taskkill", ["/pid", String(proc.pid), "/t", "/f"], {
376
- stdio: "ignore",
377
- windowsHide: true,
378
- });
379
- const fallback = (): void => {
380
- try {
381
- proc.kill(force ? "SIGKILL" : "SIGTERM");
382
- } catch {
383
- /* process may already be gone */
384
- }
385
- };
386
- killer.on("error", fallback);
387
- killer.on("close", (code) => {
388
- if (code !== 0) fallback();
389
- });
390
- return;
391
- }
392
-
393
- try {
394
- proc.kill(force ? "SIGKILL" : "SIGTERM");
395
- } catch {
396
- /* process may already be gone */
397
- }
398
- }
399
-
400
- export function currentSubagentDepth(env: NodeJS.ProcessEnv = process.env): number {
401
- const raw = env[DEPTH_ENV_VAR];
402
- const parsed = raw === undefined ? 0 : Number.parseInt(raw, 10);
403
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
404
- }
405
-
406
- export interface RunSingleOptions {
407
- defaultCwd: string;
408
- agent: AgentConfig | undefined;
409
- agentName: string;
410
- task: string;
411
- cwd?: string;
412
- /** Thinking level passed to the child pi process. */
413
- thinkingLevel?: ThinkingLevel;
414
- /** Idle timeout in ms: terminate the child if its stdout produces no activity
415
- * for this duration. 0 (the default) disables the idle watchdog. */
416
- idleTimeoutMs?: number;
417
- /** Startup-retry backoff schedule (ms) for silent, zero-activity child exits
418
- * (a concurrent pi startup race). Defaults to SUBAGENT_STARTUP_RETRY_DELAYS_MS;
419
- * pass a shorter array in tests to keep them fast. */
420
- startupRetryDelaysMs?: readonly number[];
421
- /** Run-level backoff schedule (ms) for relaunching the SAME configured model
422
- * after a transient provider-level failure (503/429/timeout/network/...). Each
423
- * relaunch gets its own startup-retry inner loop. Defaults to
424
- * SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS; pass [] to disable (e.g. when an isolated
425
- * test wants to assert only the fallback path runs once). */
426
- runLevelRetryDelaysMs?: readonly number[];
427
- signal?: AbortSignal;
428
- onLive?: (e: SubagentLiveEvent) => void;
429
- makeDetails: (results: SingleResult[]) => SubagentDetails;
430
- env?: NodeJS.ProcessEnv;
431
- }
432
-
433
- /** Spawn one agent as an isolated pi child process and collect its output. */
434
- export async function runSingleAgent(options: RunSingleOptions): Promise<SingleResult> {
435
- const {
436
- agent,
437
- agentName,
438
- task,
439
- cwd,
440
- thinkingLevel = SUBAGENT_THINKING_LEVEL,
441
- idleTimeoutMs = SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS,
442
- signal,
443
- onLive,
444
- makeDetails,
445
- } = options;
446
-
447
- if (!agent) {
448
- return {
449
- agent: agentName,
450
- agentSource: "unknown",
451
- task,
452
- exitCode: 1,
453
- messages: [],
454
- stderr: `Unknown agent: "${agentName}".`,
455
- usage: emptyUsage(),
456
- };
457
- }
458
-
459
- // Defense in depth: even if another extension ignores the depth marker, a
460
- // child process can never expose a tool named `subagent` back to its model.
461
- const args: string[] = ["--mode", "json", "-p", "--no-session", "--exclude-tools", "subagent"];
462
- if (agent.model) args.push("--model", agent.model);
463
- // The configured level is clamped adaptively per model by pi.
464
- args.push("--thinking", thinkingLevel);
465
- if (agent.tools && agent.tools.length > 0) args.push("--tools", agent.tools.join(","));
466
-
467
- let tmpPromptDir: string | null = null;
468
- let tmpPromptPath: string | null = null;
469
-
470
- const currentResult: SingleResult = {
471
- agent: agentName,
472
- agentSource: agent.source,
473
- task,
474
- exitCode: 0,
475
- messages: [],
476
- stderr: "",
477
- usage: emptyUsage(),
478
- model: agent.model,
479
- thinking: thinkingLevel,
480
- };
481
-
482
- try {
483
- if (agent.systemPrompt.trim()) {
484
- const tmp = await writePromptToTempFile(agent.name, agent.systemPrompt);
485
- tmpPromptDir = tmp.dir;
486
- tmpPromptPath = tmp.filePath;
487
- args.push("--append-system-prompt", tmpPromptPath);
488
- }
489
-
490
- let wasAborted = false;
491
-
492
- // Increment depth so nested sub-agents can be guarded against runaway recursion.
493
- const childDepth = currentSubagentDepth(options.env) + 1;
494
- const childEnv: NodeJS.ProcessEnv = {
495
- ...(options.env ?? process.env),
496
- [DEPTH_ENV_VAR]: String(childDepth),
497
- };
498
-
499
- const exitCode = await new Promise<number>((resolve) => {
500
- const invocation = getPiInvocation(args);
501
- const proc = spawn(invocation.command, invocation.args, {
502
- cwd: cwd ?? options.defaultCwd,
503
- shell: false,
504
- stdio: ["pipe", "pipe", "pipe"],
505
- env: childEnv,
506
- });
507
- let buffer = "";
508
- let closed = false;
509
- let termSent = false;
510
- let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
511
- let abortHandler: (() => void) | undefined;
512
- let lastActivityAt = Date.now();
513
- let idleTimer: ReturnType<typeof setInterval> | undefined;
514
-
515
- const finish = (code: number | null): void => {
516
- if (closed) return;
517
- closed = true;
518
- if (forceKillTimer) clearTimeout(forceKillTimer);
519
- if (idleTimer) clearInterval(idleTimer);
520
- if (signal && abortHandler) signal.removeEventListener("abort", abortHandler);
521
- resolve(code ?? 1);
522
- };
523
-
524
- const terminate = (): void => {
525
- if (closed) return;
526
- if (!termSent) {
527
- termSent = true;
528
- terminateProcessTree(proc, false);
529
- }
530
- if (!forceKillTimer) {
531
- forceKillTimer = setTimeout(() => {
532
- if (!closed) terminateProcessTree(proc, true);
533
- }, SUBAGENT_KILL_GRACE_MS);
534
- }
535
- };
536
-
537
- const processLine = (line: string): void => {
538
- if (!line.trim()) return;
539
- let event: any;
540
- try {
541
- event = JSON.parse(line);
542
- } catch {
543
- return;
544
- }
545
-
546
- // Live event: agent started
547
- if (event.type === "agent_start" || event.type === "turn_start") {
548
- if (onLive) {
549
- try {
550
- onLive({ kind: "status", status: "running" });
551
- } catch { /* never throw from event handling */ }
552
- }
553
- }
554
-
555
- // Live event: streamed assistant reasoning / output text
556
- if (event.type === "message_update") {
557
- const t = event.assistantMessageEvent?.type;
558
- if (t === "thinking_delta" || t === "text_delta") {
559
- if (onLive) {
560
- try {
561
- onLive({ kind: t === "thinking_delta" ? "thinking" : "text" });
562
- } catch { /* never throw from event handling */ }
563
- }
564
- }
565
- }
566
-
567
- // Live event: tool execution started
568
- if (event.type === "tool_execution_start") {
569
- if (onLive) {
570
- try {
571
- onLive({ kind: "tool_start", toolName: event.toolName ?? "unknown", args: event.args });
572
- } catch { /* never throw from event handling */ }
573
- }
574
- }
575
-
576
- // Live event: tool execution ended
577
- if (event.type === "tool_execution_end") {
578
- if (onLive) {
579
- try {
580
- onLive({ kind: "tool_end", toolName: event.toolName ?? "unknown", isError: Boolean(event.isError) });
581
- } catch { /* never throw from event handling */ }
582
- }
583
- if (event.isError) {
584
- (currentResult.failedTools ??= []).push({
585
- toolName: event.toolName ?? "unknown",
586
- error: extractToolErrorText(event.result?.content),
587
- });
588
- }
589
- }
590
-
591
- if (event.type === "message_end" && event.message) {
592
- const msg = event.message as Message;
593
- currentResult.messages.push(msg);
594
- if (msg.role === "assistant") {
595
- currentResult.usage.turns++;
596
- const usage = (msg as any).usage;
597
- if (usage) {
598
- currentResult.usage.input += usage.input || 0;
599
- currentResult.usage.output += usage.output || 0;
600
- currentResult.usage.cacheRead += usage.cacheRead || 0;
601
- currentResult.usage.cacheWrite += usage.cacheWrite || 0;
602
- currentResult.usage.cost += usage.cost?.total || 0;
603
- currentResult.usage.contextTokens = usage.totalTokens || 0;
604
- }
605
- if (!currentResult.model && (msg as any).model) currentResult.model = (msg as any).model;
606
- if ((msg as any).stopReason) currentResult.stopReason = (msg as any).stopReason;
607
- if ((msg as any).errorMessage) currentResult.errorMessage = (msg as any).errorMessage;
608
- }
609
- // Live event: usage snapshot after accumulation
610
- if (onLive) {
611
- try {
612
- onLive({ kind: "usage", usage: { ...currentResult.usage }, model: currentResult.model });
613
- } catch { /* never throw from event handling */ }
614
- }
615
- }
616
-
617
- if (event.type === "tool_result_end" && event.message) {
618
- currentResult.messages.push(event.message as Message);
619
- }
620
- };
621
- // Send the task through the child stdin pipe instead of the process
622
- // command line. This avoids OS argument-length limits and does not
623
- // require another temporary file for conversation data.
624
- proc.stdin?.on("error", () => undefined);
625
- proc.stdin?.end(`Task: ${task}`);
626
-
627
- // Decode stdout through a StringDecoder so multi-byte UTF-8 characters
628
- // (CJK, emoji) split across chunk boundaries never produce U+FFFD
629
- // replacement characters — a corrupted JSON line would drop the whole
630
- // message (including a reviewer's verdict line) from parsing.
631
- const stdoutDecoder = new StringDecoder("utf8");
632
- proc.stdout.on("data", (data) => {
633
- lastActivityAt = Date.now();
634
- buffer += stdoutDecoder.write(data);
635
- const lines = buffer.split("\n");
636
- buffer = lines.pop() || "";
637
- for (const line of lines) processLine(line);
638
- });
639
-
640
- proc.stderr.on("data", (data) => {
641
- currentResult.stderr += data.toString();
642
- });
643
-
644
- proc.on("close", (code) => {
645
- // Flush any bytes still held by the decoder (a trailing incomplete
646
- // multi-byte sequence) before processing the final buffer.
647
- buffer += stdoutDecoder.end();
648
- if (buffer.trim()) processLine(buffer);
649
- // A null exit code means the process was terminated by a signal and
650
- // must be reported as failure, never as a false clean completion.
651
- const failed =
652
- code !== 0 ||
653
- wasAborted ||
654
- (signal?.aborted ?? false) ||
655
- currentResult.stopReason === "error" ||
656
- currentResult.stopReason === "aborted";
657
- if (onLive) {
658
- try {
659
- onLive({ kind: "status", status: failed ? "failed" : "done" });
660
- } catch { /* never throw from event handling */ }
661
- }
662
- finish(code);
663
- });
664
-
665
- proc.on("error", () => {
666
- // Spawn itself failed; close may never fire, so finish the run here.
667
- currentResult.stopReason = "error";
668
- currentResult.errorMessage ??= "Failed to start the sub-agent process.";
669
- if (onLive) {
670
- try {
671
- onLive({ kind: "status", status: "failed" });
672
- } catch { /* never throw from event handling */ }
673
- }
674
- finish(1);
675
- });
676
-
677
- if (idleTimeoutMs > 0) {
678
- const checkInterval = Math.min(10_000, Math.floor(idleTimeoutMs / 3));
679
- idleTimer = setInterval(() => {
680
- if (closed) return;
681
- if (Date.now() - lastActivityAt >= idleTimeoutMs) {
682
- if (idleTimer) clearInterval(idleTimer);
683
- currentResult.stopReason = "error";
684
- currentResult.errorMessage = `Subagent idle timeout: no activity for ${Math.ceil(idleTimeoutMs / 1000)} seconds.`;
685
- terminate();
686
- }
687
- }, checkInterval);
688
- }
689
-
690
- if (signal) {
691
- abortHandler = (): void => {
692
- wasAborted = true;
693
- terminate();
694
- };
695
- if (signal.aborted) abortHandler();
696
- else signal.addEventListener("abort", abortHandler, { once: true });
697
- }
698
- });
699
-
700
- currentResult.exitCode = exitCode;
701
- if (wasAborted) {
702
- currentResult.stopReason = "aborted";
703
- currentResult.errorMessage ??= "Subagent was aborted";
704
- if (onLive) {
705
- try {
706
- onLive({ kind: "status", status: "failed" });
707
- } catch { /* never throw from event handling */ }
708
- }
709
- }
710
- return currentResult;
711
- } finally {
712
- if (tmpPromptPath)
713
- try {
714
- unlinkSync(tmpPromptPath);
715
- } catch {
716
- /* ignore */
717
- }
718
- if (tmpPromptDir)
719
- try {
720
- rmdirSync(tmpPromptDir);
721
- } catch {
722
- /* ignore */
723
- }
724
- }
725
- }
726
-
727
- /**
728
- * Run one agent with three layers of resilience against transient dispatch failures:
729
- *
730
- * 1. Startup retry (inner loop): a concurrent pi startup race can make the child
731
- * exit before any model/tool activity. Relaunch with backoff so the startup
732
- * lock clears. The SAME model is retried — the race is in the host, not the
733
- * model — and only a clean, silent, zero-activity exit qualifies (see
734
- * isRetryableStartupFailure), so retrying can never duplicate real work.
735
- * 2. Run-level retry on the SAME configured model (middle): when the provider
736
- * rejects the model before producing output with a TRANSIENT error
737
- * (503/429/timeout/network/stream/...) — i.e. NOT a terminal one (quota
738
- * exhausted, billing, an invalid API key, auth rejected — see
739
- * isTerminalModelError) — relaunch the whole run up to
740
- * SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS.length more times with backoff, so a
741
- * one-off provider hiccup recovers without demoting the configured agent
742
- * model. Each relaunch gets its own inner startup-retry loop. This sits
743
- * outside pi-ai's per-request provider retry, which by then has already
744
- * tried (default 3 attempts) and given up.
745
- * 3. Model fallback (outer): when the same model is still failing after all its
746
- * run-level retries, retry once with the main window's current model. The
747
- * fallback gets its own startup-retry loop, since a startup race can hit any
748
- * relaunch regardless of model.
749
- *
750
- * Terminal model errors short-circuit straight to the caller (modelRetries is
751
- * set): the account is the bottleneck, so neither same-model retry nor a
752
- * same-account fallback can help, and the run is left for the main agent to fix.
753
- *
754
- * The fallback is per-run only and never persisted: a transient provider hiccup
755
- * must not silently downgrade the configured agent model.
756
- */
757
- export async function runSingleAgentWithModelFallback(
758
- options: RunSingleOptions,
759
- fallbackModelRef?: string,
760
- ): Promise<SingleResult> {
761
- const agent = options.agent;
762
- const launchedRef = agent?.model;
763
- const startupDelays = options.startupRetryDelaysMs ?? SUBAGENT_STARTUP_RETRY_DELAYS_MS;
764
- // Run-level retry is opted out of with an explicit empty array (e.g. a test
765
- // that wants to assert ONLY the fallback path runs once); undefined means
766
- // "use the default 5-attempt transient-error schedule".
767
- const runDelays = options.runLevelRetryDelaysMs ?? SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS;
768
-
769
- const runWithStartupRetry = async (opts: RunSingleOptions): Promise<SingleResult> => {
770
- let lastResult: SingleResult;
771
- let retries = 0;
772
- for (let attempt = 0; ; attempt++) {
773
- const start = Date.now();
774
- lastResult = await runSingleAgent(opts);
775
- const durationMs = Date.now() - start;
776
- if (!isRetryableStartupFailure(lastResult, durationMs)) {
777
- if (retries > 0 && !isFailedResult(lastResult)) lastResult.startupRetries = retries;
778
- return lastResult;
779
- }
780
- const delay = startupDelays[attempt];
781
- if (delay === undefined) {
782
- // Exhausted: the agent never reached a model. Surface the concurrency-race
783
- // cause as a dispatch-level failure (no model was ever reached, so this
784
- // must NOT trigger run-level retry or model fallback) so the main agent
785
- // can retry or lower maxConcurrency.
786
- lastResult.errorMessage = formatStartupRetryExhaustedError(
787
- lastResult.model ?? opts.agent?.model ?? "default",
788
- attempt + 1,
789
- );
790
- lastResult.stopReason ??= "error";
791
- lastResult.dispatchFailed = true;
792
- return lastResult;
793
- }
794
- // Flip the live status back to running so the widget does not flash a
795
- // false "failed" while we wait out the backoff and relaunch the child.
796
- try {
797
- opts.onLive?.({ kind: "status", status: "running" });
798
- } catch { /* never throw from event handling */ }
799
- const shouldRetry = await waitForStartupRetry(delay, opts.signal);
800
- if (!shouldRetry) return lastResult;
801
- retries++;
802
- }
803
- };
804
-
805
- let result = await runWithStartupRetry(options);
806
-
807
- // After a model-level failure, classify before reacting. A TERMINAL error
808
- // (quota/billing/invalid key/auth) is account-scoped: neither same-model
809
- // retry nor a same-account fallback can help, so hand the run straight back
810
- // to the main agent instead of burning its time on a doomed retry.
811
- if (agent && launchedRef && isModelLevelFailure(result) && isTerminalModelError(result)) return result;
812
-
813
- // A TRANSIENT provider failure (503/429/timeout/network/stream/...) is usually
814
- // a one-off hiccup. Relaunch the SAME configured model up to runDelays.length
815
- // more times with backoff before degrading to a fallback model — the run's own
816
- // provider retry already tried and failed, so each relaunch here is an
817
- // independent, fresh attempt that can recover without losing the configured
818
- // model's capability to review.
819
- let modelRetries = 0;
820
- if (agent && launchedRef && isModelLevelFailure(result) && runDelays.length > 0) {
821
- for (let attempt = 0; ; attempt++) {
822
- const delay = runDelays[attempt];
823
- if (delay === undefined) break;
824
- try {
825
- options.onLive?.({ kind: "status", status: "running" });
826
- } catch { /* never throw from event handling */ }
827
- const shouldRetry = await waitForStartupRetry(delay, options.signal);
828
- if (!shouldRetry) return { ...result, modelRetries };
829
- const retried = await runWithStartupRetry(options);
830
- modelRetries++;
831
- if (!isModelLevelFailure(retried) || isTerminalModelError(retried)) {
832
- // Each relaunch redoes the work; failedTools reflect ONLY the final
833
- // attempt (no stale build errors from earlier transient failures),
834
- // so the completion message's claim stays accurate.
835
- return { ...retried, modelRetries };
836
- }
837
- result = retried;
838
- }
839
- }
840
-
841
- // Same-model retries exhausted (or none configured) and still failing: fall
842
- // back to the main window's current model exactly once. Skipped when there is
843
- // no fallback ref or it equals the configured model — a same-ref rerun would
844
- // just repeat the already-exhausted failure for nothing.
845
- if (agent && launchedRef && fallbackModelRef && launchedRef !== fallbackModelRef && isModelLevelFailure(result)) {
846
- const retried = await runWithStartupRetry({ ...options, agent: { ...agent, model: fallbackModelRef } });
847
- // The fallback replaces the result wholesale: `retried.failedTools` reflect
848
- // ONLY the fallback (final) attempt. The original attempt's failedTools are
849
- // intentionally not merged — a fallback relaunch redoes the work, so attaching
850
- // the first attempt's stale build errors to a clean final attempt would
851
- // misattribute failures the worker already fixed. This makes the README's
852
- // "failed tool calls from the run's final attempt" claim accurate.
853
- return { ...retried, modelFallbackFrom: launchedRef, modelRetries };
854
- }
855
- return { ...result, modelRetries };
856
- }
1
+ /**
2
+ * Sub-agent result handling and resilient RPC launch orchestration.
3
+ *
4
+ * The process transport itself lives in rpc-run.ts. Each attempt starts pi in
5
+ * persistent `--mode rpc`, sends commands over strict LF-delimited JSONL, and
6
+ * settles only on `agent_settled`. This module preserves the existing startup
7
+ * retry, same-model retry, model fallback, accounting, and result formatting
8
+ * contracts around those attempts.
9
+ */
10
+
11
+ import { randomUUID } from "node:crypto";
12
+ import { mkdirSync, writeFileSync } from "node:fs";
13
+ import { mkdtemp, rm } from "node:fs/promises";
14
+ import { tmpdir } from "node:os";
15
+ import { basename, join } from "node:path";
16
+ import type { Message } from "@earendil-works/pi-ai";
17
+ import type { AgentConfig } from "./agents.ts";
18
+ import { DEFAULT_THINKING_LEVEL, type ThinkingLevel } from "./config.ts";
19
+ import {
20
+ currentSubagentDepth,
21
+ DEPTH_ENV_VAR,
22
+ extractToolErrorText,
23
+ getPiInvocation,
24
+ RpcRunControl,
25
+ runRpcAgentAttempt,
26
+ sessionExists,
27
+ SUBAGENT_KILL_GRACE_MS,
28
+ type RpcSingleResult,
29
+ type SubagentLiveEvent,
30
+ type SubagentRecordEvent,
31
+ type UsageStats,
32
+ } from "./rpc-run.ts";
33
+
34
+ export {
35
+ currentSubagentDepth,
36
+ DEPTH_ENV_VAR,
37
+ extractToolErrorText,
38
+ getPiInvocation,
39
+ RpcRunControl,
40
+ sessionExists,
41
+ SUBAGENT_KILL_GRACE_MS,
42
+ };
43
+ export type { SubagentLiveEvent, SubagentRecordEvent, UsageStats };
44
+
45
+ export const SUBAGENT_THINKING_LEVEL: ThinkingLevel = DEFAULT_THINKING_LEVEL;
46
+ /** 0 disables the watchdog; dispatch supplies the configured timeout. */
47
+ export const SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS = 0;
48
+ export const SUBAGENT_STARTUP_RETRY_DELAYS_MS = [250, 750, 1500] as const;
49
+ export const MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS = 2000;
50
+ export const SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 30_000] as const;
51
+
52
+ export interface SingleResult extends RpcSingleResult {}
53
+
54
+ export interface SubagentDetails {
55
+ mode: "single" | "parallel";
56
+ results: SingleResult[];
57
+ background?: boolean;
58
+ }
59
+
60
+ export function getFinalOutput(messages: Message[]): string {
61
+ for (let i = messages.length - 1; i >= 0; i--) {
62
+ const msg = messages[i];
63
+ if (msg.role === "assistant") {
64
+ for (const part of msg.content) {
65
+ if (part.type === "text") return part.text;
66
+ }
67
+ }
68
+ }
69
+ return "";
70
+ }
71
+
72
+ /** Only the last standalone reviewer verdict line counts. */
73
+ export function reviewVerdict(output: string): "pass" | "fail" | undefined {
74
+ const lines = output.split("\n");
75
+ for (let index = lines.length - 1; index >= 0; index--) {
76
+ const match = /^\s*VERDICT:\s*REVIEW_(PASS|FAIL)\s*$/i.exec(lines[index]);
77
+ if (match) return match[1].toUpperCase() === "PASS" ? "pass" : "fail";
78
+ }
79
+ return undefined;
80
+ }
81
+
82
+ export const RESULT_LINE_MAX = 200;
83
+
84
+ export interface TruncatedOutput {
85
+ text: string;
86
+ truncated: boolean;
87
+ }
88
+
89
+ export function truncateResultOutput(output: string, maxLines: number): TruncatedOutput {
90
+ const lines = output.split("\n");
91
+ if (lines.length <= maxLines && lines.every((line) => line.length <= RESULT_LINE_MAX)) {
92
+ return { text: output, truncated: false };
93
+ }
94
+ const kept = lines.slice(0, maxLines).map((line) =>
95
+ line.length > RESULT_LINE_MAX ? `${line.slice(0, RESULT_LINE_MAX)}…` : line,
96
+ );
97
+ return { text: kept.join("\n"), truncated: true };
98
+ }
99
+
100
+ export function writeResultArtifact(output: string, agentName: string, cwd?: string): string {
101
+ const projectSlug = cwd ? basename(cwd).replace(/[^\w.-]+/g, "_") || "default" : "default";
102
+ const dir = join(tmpdir(), "pi-subagents-results", projectSlug);
103
+ mkdirSync(dir, { recursive: true });
104
+ const safeName = agentName.replace(/[^\w.-]+/g, "_");
105
+ const unique = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
106
+ const filePath = join(dir, `${unique}-${safeName}.md`);
107
+ writeFileSync(filePath, output, "utf8");
108
+ return filePath;
109
+ }
110
+
111
+ export function isFailedResult(result: SingleResult): boolean {
112
+ if (result.parked) return false;
113
+ return result.exitCode !== 0 || result.stopReason === "error" || result.stopReason === "aborted";
114
+ }
115
+
116
+ function lastAssistantMessage(messages: Message[]): Extract<Message, { role: "assistant" }> | undefined {
117
+ for (let index = messages.length - 1; index >= 0; index--) {
118
+ const message = messages[index];
119
+ if (message.role === "assistant") return message;
120
+ }
121
+ return undefined;
122
+ }
123
+
124
+ function assistantText(message: Extract<Message, { role: "assistant" }>): string {
125
+ return message.content
126
+ .filter((part): part is Extract<(typeof message.content)[number], { type: "text" }> => part.type === "text")
127
+ .map((part) => part.text)
128
+ .join("");
129
+ }
130
+
131
+ export function isModelLevelFailure(result: SingleResult): boolean {
132
+ if (!isFailedResult(result)) return false;
133
+ if (result.stopReason === "aborted") return false;
134
+ if (result.dispatchFailed) return false;
135
+ if (result.integrationStatus === "retained") return false;
136
+ if (result.errorMessage?.includes("idle timeout")) return true;
137
+ if (result.rpcPromptRejected) return true;
138
+
139
+ // Classification belongs to the final assistant turn, not the whole attempt.
140
+ // Earlier useful text or failed tool calls are retained session history and
141
+ // must not hide a later provider error (for example a second-turn 503).
142
+ const finalAssistant = lastAssistantMessage(result.messages);
143
+ if (finalAssistant) {
144
+ if (finalAssistant.stopReason !== "error") return false;
145
+ if (assistantText(finalAssistant).trim()) return false;
146
+ return Boolean(
147
+ finalAssistant.errorMessage?.trim() ||
148
+ result.errorMessage?.trim() ||
149
+ result.stderr.trim() ||
150
+ finalAssistant.content.length === 0
151
+ );
152
+ }
153
+
154
+ if ((result.failedTools?.length ?? 0) > 0) return false;
155
+ return Boolean(result.errorMessage?.trim()) || result.stderr.trim().length > 0;
156
+ }
157
+
158
+ const TERMINAL_MODEL_ERROR_PATTERN =
159
+ /insufficient_quota|quota\s+exceeded|exceeded[^.\n]{0,40}quota|out\s+of\s+budget|billing|usage\s+limit|usage_limit|gousagelimiterror|freeusagelimiterror|monthly\s+usage\s+limit\s+reached|available\s+balance|invalid\s+(?:api\s+)?key|incorrect\s+api\s+key|unauthori[sz]ed|\b401\b|\b403\b|forbidden|permission\s+denied/i;
160
+ const PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN =
161
+ /model[_ -]?not[_ -]?found|no\s+models?\s+(?:found|matched)|(?:model|provider)[^.\n]{0,80}(?:not\s+found|unknown|does\s+not\s+exist|unsupported|invalid)|(?:not\s+found|unknown|unsupported|invalid)[^.\n]{0,40}(?:model|provider)|\b404\b/i;
162
+
163
+ export function isTerminalModelError(result: SingleResult): boolean {
164
+ const message = result.errorMessage?.trim();
165
+ if (message) return TERMINAL_MODEL_ERROR_PATTERN.test(message);
166
+ const stderr = result.stderr.trim();
167
+ return stderr.length > 0 && TERMINAL_MODEL_ERROR_PATTERN.test(stderr);
168
+ }
169
+
170
+ /** A permanent failure of this model/provider reference (stale id, unknown
171
+ * provider, 404 config route). Skip same-candidate backoff, but keep advancing
172
+ * through backup and current-main candidates. */
173
+ export function isPermanentModelCandidateError(result: SingleResult): boolean {
174
+ const message = result.errorMessage?.trim();
175
+ if (message) return PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN.test(message);
176
+ const stderr = result.stderr.trim();
177
+ return stderr.length > 0 && PERMANENT_MODEL_CANDIDATE_ERROR_PATTERN.test(stderr);
178
+ }
179
+
180
+ export function isRetryableStartupFailure(result: SingleResult, durationMs: number): boolean {
181
+ if (result.exitCode === 0) return false;
182
+ if (result.stopReason === "aborted") return false;
183
+ if (result.dispatchFailed) return false;
184
+ if (result.errorMessage?.includes("idle timeout")) return false;
185
+ if (getFinalOutput(result.messages)) return false;
186
+ if (result.messages.length > 0) return false;
187
+ const usage = result.usage;
188
+ if (usage.turns || usage.input || usage.output || usage.cacheRead || usage.cacheWrite || usage.cost) return false;
189
+ if (durationMs > MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS) return false;
190
+ if (result.stderr.trim().length > 0) return false;
191
+ if (result.errorMessage && result.errorMessage.trim().length > 0) return false;
192
+ return true;
193
+ }
194
+
195
+ export function formatStartupRetryExhaustedError(model: string, attempts: number): string {
196
+ return `Subagent failed to start after ${attempts} attempt${attempts === 1 ? "" : "s"} on ${model}: the child exited before any model, tool, output, or usage activity. This is typically a concurrent pi startup race (several sub-agents starting at once). Retry the dispatch, or temporarily lower maxConcurrency in /subagents-setup.`;
197
+ }
198
+
199
+ export async function waitForStartupRetry(delayMs: number, signal?: AbortSignal): Promise<boolean> {
200
+ if (delayMs <= 0) return !signal?.aborted;
201
+ if (!signal) {
202
+ return new Promise<boolean>((resolve) => {
203
+ const timer = setTimeout(() => resolve(true), delayMs);
204
+ if (typeof timer.unref === "function") timer.unref();
205
+ });
206
+ }
207
+ if (signal.aborted) return false;
208
+ return new Promise<boolean>((resolve) => {
209
+ let settled = false;
210
+ const finish = (shouldRetry: boolean): void => {
211
+ if (settled) return;
212
+ settled = true;
213
+ clearTimeout(timer);
214
+ signal.removeEventListener("abort", onAbort);
215
+ resolve(shouldRetry);
216
+ };
217
+ const onAbort = (): void => finish(false);
218
+ const timer = setTimeout(() => finish(true), delayMs);
219
+ if (typeof timer.unref === "function") timer.unref();
220
+ signal.addEventListener("abort", onAbort, { once: true });
221
+ });
222
+ }
223
+
224
+ async function waitForControlledRetry(
225
+ delayMs: number,
226
+ signal: AbortSignal | undefined,
227
+ control: RpcRunControl | undefined,
228
+ ): Promise<boolean> {
229
+ let remaining = delayMs;
230
+ while (remaining > 0) {
231
+ if (control?.isParkRequested() || control?.isStopRequested()) return false;
232
+ const slice = Math.min(remaining, 50);
233
+ if (!(await waitForStartupRetry(slice, signal))) return false;
234
+ remaining -= slice;
235
+ }
236
+ return !signal?.aborted && !control?.isParkRequested() && !control?.isStopRequested();
237
+ }
238
+
239
+ export function getResultOutput(result: SingleResult): string {
240
+ if (isFailedResult(result)) {
241
+ const error = result.errorMessage || result.stderr;
242
+ const partial = getFinalOutput(result.messages);
243
+ if (error && partial) return `${error}\n\n--- Partial output ---\n${partial}`;
244
+ return error || partial || "(no output)";
245
+ }
246
+ return getFinalOutput(result.messages) || "(no output)";
247
+ }
248
+
249
+ export function buildResumePrompt(task: string, reason: string): string {
250
+ return `You are resuming an earlier sub-agent session after ${reason}. Your earlier work — searches, reads, edits, and reasoning — is preserved in this session's history above; review it before acting. Original task: ${task}. Pick up exactly where you left off and finish it. Do NOT redo searches, reads, or edits you already completed unless a step clearly failed. Continue now.`;
251
+ }
252
+
253
+ export function buildFallbackResumeReason(fromModel?: string): string {
254
+ return fromModel
255
+ ? `the previous model (${fromModel}) failed at the model/provider level, so the next model in its configured pool is continuing`
256
+ : "the previous model failed at the model/provider level, so the next model in its configured pool is continuing";
257
+ }
258
+
259
+ export interface RunSingleOptions {
260
+ defaultCwd: string;
261
+ agent: AgentConfig | undefined;
262
+ agentName: string;
263
+ task: string;
264
+ cwd?: string;
265
+ thinkingLevel?: ThinkingLevel;
266
+ idleTimeoutMs?: number;
267
+ startupRetryDelaysMs?: readonly number[];
268
+ runLevelRetryDelaysMs?: readonly number[];
269
+ sessionDir?: string;
270
+ sessionId?: string;
271
+ /** Initial RPC prompt. Kept under the old name to limit caller churn. */
272
+ stdinText?: string;
273
+ signal?: AbortSignal;
274
+ onLive?: (event: SubagentLiveEvent) => void;
275
+ /** Receives the raw streamed text/thinking deltas for the inspector
276
+ * transcript; forwarded to the transport alongside onLive. */
277
+ onRecord?: (event: SubagentRecordEvent) => void;
278
+ makeDetails: (results: SingleResult[]) => SubagentDetails;
279
+ env?: NodeJS.ProcessEnv;
280
+ /** Stable logical-generation controller shared across retry attempts. */
281
+ control?: RpcRunControl;
282
+ }
283
+
284
+ function controlledDisposition(options: RunSingleOptions, base?: SingleResult): SingleResult | undefined {
285
+ const control = options.control;
286
+ if (!control?.isParkRequested() && !control?.isStopRequested()) return undefined;
287
+ const result: SingleResult = base ?? {
288
+ agent: options.agentName,
289
+ agentSource: options.agent?.source ?? "unknown",
290
+ task: control.getObjective(),
291
+ exitCode: 0,
292
+ messages: [],
293
+ stderr: "",
294
+ usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
295
+ model: options.agent?.model,
296
+ thinking: options.thinkingLevel,
297
+ sessionId: options.sessionId,
298
+ sessionDir: options.sessionDir,
299
+ };
300
+ result.task = control.getObjective();
301
+ if (control.isParkRequested()) {
302
+ result.parked = true;
303
+ result.exitCode = 0;
304
+ result.stopReason = undefined;
305
+ result.errorMessage = undefined;
306
+ } else {
307
+ result.parked = undefined;
308
+ result.exitCode = 1;
309
+ result.stopReason = "aborted";
310
+ result.errorMessage = control.getStopMessage();
311
+ }
312
+ return result;
313
+ }
314
+
315
+ /** Spawn one RPC attempt and wait for stable settlement. */
316
+ export async function runSingleAgent(options: RunSingleOptions): Promise<SingleResult> {
317
+ const {
318
+ agent,
319
+ agentName,
320
+ thinkingLevel = SUBAGENT_THINKING_LEVEL,
321
+ idleTimeoutMs = SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS,
322
+ control,
323
+ } = options;
324
+ if (!agent) {
325
+ return {
326
+ agent: agentName,
327
+ agentSource: "unknown",
328
+ task: options.task,
329
+ exitCode: 1,
330
+ messages: [],
331
+ stderr: `Unknown agent: "${agentName}".`,
332
+ usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
333
+ };
334
+ }
335
+
336
+ const disposition = controlledDisposition(options);
337
+ if (disposition) return disposition;
338
+ const objective = control?.getObjective() ?? options.task;
339
+ let prompt = options.stdinText ?? `Task: ${objective}`;
340
+ if (control && objective !== options.task) {
341
+ prompt = options.sessionDir && sessionExists(options.sessionDir, options.sessionId ?? "")
342
+ ? `Abandon the previous objective. New objective: ${objective}`
343
+ : `Task: ${objective}`;
344
+ }
345
+ const result = await runRpcAgentAttempt({
346
+ defaultCwd: options.defaultCwd,
347
+ agent,
348
+ agentName,
349
+ task: objective,
350
+ cwd: options.cwd,
351
+ thinkingLevel,
352
+ idleTimeoutMs,
353
+ sessionDir: options.sessionDir,
354
+ sessionId: options.sessionId,
355
+ prompt,
356
+ signal: options.signal,
357
+ onLive: options.onLive,
358
+ onRecord: options.onRecord,
359
+ env: options.env,
360
+ control,
361
+ });
362
+ result.task = control?.getObjective() ?? result.task;
363
+ return result;
364
+ }
365
+
366
+ /**
367
+ * Run one logical generation across an ordered model pool. Every candidate gets
368
+ * startup retries plus same-model retries for transient provider failures;
369
+ * terminal model errors skip those retries and advance immediately. All
370
+ * candidates resume the same retained pi session.
371
+ */
372
+ export async function runSingleAgentWithModelFallback(
373
+ options: RunSingleOptions,
374
+ fallbackModelRefs: readonly string[] = [],
375
+ ): Promise<SingleResult> {
376
+ const agent = options.agent;
377
+ const launchedRef = agent?.model;
378
+ const startupDelays = options.startupRetryDelaysMs ?? SUBAGENT_STARTUP_RETRY_DELAYS_MS;
379
+ const runDelays = options.runLevelRetryDelaysMs ?? SUBAGENT_RUN_LEVEL_RETRY_DELAYS_MS;
380
+
381
+ const sessionId = options.sessionId ?? randomUUID();
382
+ const sessionDir = options.sessionDir ?? (await mkdtemp(join(tmpdir(), "pi-subagent-session-")));
383
+ const baseOptions: RunSingleOptions = { ...options, sessionDir, sessionId };
384
+
385
+ const dispatchFailure = async (error: unknown): Promise<SingleResult> => {
386
+ const errorMessage = error instanceof Error ? error.message : String(error);
387
+ const hasSession = sessionExists(sessionDir, sessionId);
388
+ if (!hasSession && !options.sessionDir) {
389
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
390
+ }
391
+ return {
392
+ agent: options.agentName,
393
+ agentSource: options.agent?.source ?? "unknown",
394
+ task: options.control?.getObjective() ?? options.task,
395
+ exitCode: 1,
396
+ messages: [],
397
+ stderr: errorMessage,
398
+ usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 },
399
+ model: options.agent?.model,
400
+ thinking: options.thinkingLevel,
401
+ stopReason: "error",
402
+ errorMessage,
403
+ dispatchFailed: true,
404
+ ...(hasSession || options.sessionDir ? { sessionId, sessionDir } : {}),
405
+ };
406
+ };
407
+
408
+ const runWithStartupRetry = async (opts: RunSingleOptions): Promise<SingleResult> => {
409
+ let lastResult: SingleResult;
410
+ let retries = 0;
411
+ for (let attempt = 0; ; attempt++) {
412
+ const immediate = controlledDisposition(opts);
413
+ if (immediate) {
414
+ if (immediate.parked && !options.sessionDir && !sessionExists(sessionDir, sessionId)) {
415
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
416
+ immediate.sessionId = undefined;
417
+ immediate.sessionDir = undefined;
418
+ }
419
+ return immediate;
420
+ }
421
+ const start = Date.now();
422
+ try {
423
+ lastResult = await runSingleAgent(opts);
424
+ } catch (error) {
425
+ const failed = await dispatchFailure(error);
426
+ return controlledDisposition(opts, failed) ?? failed;
427
+ }
428
+ const durationMs = Date.now() - start;
429
+ const controlled = controlledDisposition(opts, lastResult);
430
+ if (controlled) return controlled;
431
+ if (lastResult.parked || lastResult.stopReason === "aborted") return lastResult;
432
+ if (!isRetryableStartupFailure(lastResult, durationMs)) {
433
+ if (retries > 0 && !isFailedResult(lastResult)) lastResult.startupRetries = retries;
434
+ return lastResult;
435
+ }
436
+ const delay = startupDelays[attempt];
437
+ if (delay === undefined) {
438
+ lastResult.errorMessage = formatStartupRetryExhaustedError(
439
+ lastResult.model ?? opts.agent?.model ?? "default",
440
+ attempt + 1,
441
+ );
442
+ lastResult.stopReason ??= "error";
443
+ lastResult.dispatchFailed = true;
444
+ return lastResult;
445
+ }
446
+ opts.control?.markRetrying();
447
+ try {
448
+ opts.onLive?.({ kind: "status", status: "running" });
449
+ } catch {
450
+ /* never throw from event handling */
451
+ }
452
+ if (!(await waitForControlledRetry(delay, opts.signal, opts.control))) {
453
+ return controlledDisposition(opts, lastResult) ?? lastResult;
454
+ }
455
+ retries++;
456
+ }
457
+ };
458
+
459
+ const fallbackRefs: string[] = [];
460
+ const seenRefs = new Set<string>();
461
+ if (launchedRef?.trim()) seenRefs.add(launchedRef.trim());
462
+ for (const candidate of fallbackModelRefs) {
463
+ const ref = candidate.trim();
464
+ if (!ref || seenRefs.has(ref)) continue;
465
+ seenRefs.add(ref);
466
+ fallbackRefs.push(ref);
467
+ }
468
+
469
+ const candidates: Array<{ agent: AgentConfig | undefined; ref?: string }> = [
470
+ { agent, ref: launchedRef?.trim() || undefined },
471
+ ];
472
+ if (agent) {
473
+ for (const ref of fallbackRefs) candidates.push({ agent: { ...agent, model: ref }, ref });
474
+ }
475
+
476
+ let modelRetries = 0;
477
+ let fallbackUsed = false;
478
+ let result: SingleResult | undefined;
479
+
480
+ const finish = async (settled: SingleResult): Promise<SingleResult> => {
481
+ const persistedSession = sessionExists(sessionDir, sessionId);
482
+ if (!settled.dispatchFailed || persistedSession || options.sessionDir) {
483
+ settled.sessionId ??= sessionId;
484
+ settled.sessionDir ??= sessionDir;
485
+ } else {
486
+ await rm(sessionDir, { recursive: true, force: true }).catch(() => undefined);
487
+ settled.sessionId = undefined;
488
+ settled.sessionDir = undefined;
489
+ }
490
+ settled.task = options.control?.getObjective() ?? settled.task;
491
+ if (fallbackUsed && launchedRef) settled.modelFallbackFrom = launchedRef;
492
+ settled.modelRetries = modelRetries;
493
+ options.control?.markSettled();
494
+ return settled;
495
+ };
496
+
497
+ for (let candidateIndex = 0; candidateIndex < candidates.length; candidateIndex++) {
498
+ const candidate = candidates[candidateIndex];
499
+ fallbackUsed ||= candidateIndex > 0;
500
+ const previousModel = result?.model ?? candidates[candidateIndex - 1]?.ref;
501
+ const candidateOptions: RunSingleOptions = {
502
+ ...baseOptions,
503
+ agent: candidate.agent,
504
+ ...(candidateIndex > 0
505
+ ? {
506
+ stdinText: buildResumePrompt(
507
+ options.control?.getObjective() ?? options.task,
508
+ buildFallbackResumeReason(previousModel),
509
+ ),
510
+ }
511
+ : {}),
512
+ };
513
+ try {
514
+ options.onLive?.({
515
+ kind: "model",
516
+ model: candidate.ref,
517
+ ...(candidateIndex > 0 && launchedRef ? { fallbackFrom: launchedRef } : {}),
518
+ });
519
+ } catch {
520
+ /* never throw from event handling */
521
+ }
522
+
523
+ result = await runWithStartupRetry(candidateOptions);
524
+ if (result.parked || result.stopReason === "aborted") return result;
525
+ if (!isModelLevelFailure(result)) return finish(result);
526
+
527
+ if (!isTerminalModelError(result) && !isPermanentModelCandidateError(result)) {
528
+ const retryOptions: RunSingleOptions = {
529
+ ...candidateOptions,
530
+ stdinText: buildResumePrompt(
531
+ options.control?.getObjective() ?? options.task,
532
+ "a transient provider error on the same model",
533
+ ),
534
+ };
535
+ for (const delay of runDelays) {
536
+ baseOptions.control?.markRetrying();
537
+ try {
538
+ options.onLive?.({ kind: "status", status: "running" });
539
+ } catch {
540
+ /* never throw from event handling */
541
+ }
542
+ if (!(await waitForControlledRetry(delay, options.signal, options.control))) {
543
+ return controlledDisposition(baseOptions, result) ?? result;
544
+ }
545
+ result = await runWithStartupRetry(retryOptions);
546
+ modelRetries++;
547
+ if (result.parked || result.stopReason === "aborted") return result;
548
+ if (
549
+ !isModelLevelFailure(result) ||
550
+ isTerminalModelError(result) ||
551
+ isPermanentModelCandidateError(result)
552
+ ) break;
553
+ }
554
+ }
555
+
556
+ if (!isModelLevelFailure(result)) return finish(result);
557
+ // Transient exhaustion plus terminal/permanent candidate errors advance
558
+ // to the next configured candidate. Ordinary task/tool failures returned above.
559
+ }
560
+
561
+ return finish(result ?? (await dispatchFailure("No model candidate was attempted.")));
562
+ }