@ferris1225/pi-subagents 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/package.json +1 -1
- package/src/background.ts +38 -9
- package/src/index.ts +173 -35
- package/src/monitor.ts +91 -1
- package/src/spawn.ts +146 -8
package/README.md
CHANGED
|
@@ -389,6 +389,13 @@ for `idleTimeoutSec` seconds) are treated as model-level failures and do trigger
|
|
|
389
389
|
since a stalled SSE stream is usually a provider-side issue. Results carry a `model fell back
|
|
390
390
|
from …` note when it happened.
|
|
391
391
|
|
|
392
|
+
If the model is unavailable or broken and the fallback retry also fails (or no fallback model
|
|
393
|
+
is available), the task is **handed back to the main window**: the completion message tells
|
|
394
|
+
the main agent to execute the task itself with its own tools instead of leaving a dead
|
|
395
|
+
failure. A background task that crashes with an exception (spawn infra, delivery API, ...)
|
|
396
|
+
is never silently swallowed either — the user gets a `✗ … 派发失败` error notification and
|
|
397
|
+
the failure is delivered to the main agent, which can re-dispatch it.
|
|
398
|
+
|
|
392
399
|
Thinking strength uses this precedence: `agentThinkingLevels` entry → agent frontmatter `thinking` → `thinkingLevel` default.
|
|
393
400
|
|
|
394
401
|
## Agent discovery and overrides
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ferris1225/pi-subagents",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Focused sub-agent delegation for pi: explore / worker / reviewer agents in isolated context, with proactive dispatch injection and per-agent model selection.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/background.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Tasks get their own AbortSignal rather than inheriting the foreground agent
|
|
5
5
|
* turn's signal. The owning extension cancels all work only on session teardown.
|
|
6
|
+
*
|
|
7
|
+
* Task exceptions are never swallowed: the per-task onError callback receives
|
|
8
|
+
* them (unless the task was cancelled) so callers can surface the failure to
|
|
9
|
+
* the user and the main agent instead of it vanishing into the queue.
|
|
6
10
|
*/
|
|
7
11
|
|
|
8
12
|
export type BackgroundTask = (signal: AbortSignal) => Promise<void>;
|
|
@@ -11,6 +15,9 @@ interface PendingTask {
|
|
|
11
15
|
task: BackgroundTask;
|
|
12
16
|
controller: AbortController;
|
|
13
17
|
onCancelled?: () => void;
|
|
18
|
+
/** Invoked when the task throws and was not cancelled (cancellation is not a
|
|
19
|
+
* failure — e.g. session shutdown races must never be reported as errors). */
|
|
20
|
+
onError?: (error: unknown) => void;
|
|
14
21
|
}
|
|
15
22
|
|
|
16
23
|
export class BackgroundTaskQueue {
|
|
@@ -33,15 +40,15 @@ export class BackgroundTaskQueue {
|
|
|
33
40
|
this.drain();
|
|
34
41
|
}
|
|
35
42
|
|
|
36
|
-
enqueue(task: BackgroundTask, onCancelled?: () => void): AbortController {
|
|
43
|
+
enqueue(task: BackgroundTask, onCancelled?: () => void, onError?: (error: unknown) => void): AbortController {
|
|
37
44
|
const controller = new AbortController();
|
|
38
45
|
if (this.stopped) {
|
|
39
46
|
controller.abort();
|
|
40
|
-
onCancelled
|
|
47
|
+
this.runCancelled(onCancelled);
|
|
41
48
|
return controller;
|
|
42
49
|
}
|
|
43
50
|
|
|
44
|
-
this.pending.push({ task, controller, onCancelled });
|
|
51
|
+
this.pending.push({ task, controller, onCancelled, onError });
|
|
45
52
|
this.drain();
|
|
46
53
|
return controller;
|
|
47
54
|
}
|
|
@@ -53,25 +60,47 @@ export class BackgroundTaskQueue {
|
|
|
53
60
|
|
|
54
61
|
for (const entry of this.pending.splice(0)) {
|
|
55
62
|
entry.controller.abort();
|
|
56
|
-
entry.onCancelled
|
|
63
|
+
this.runCancelled(entry.onCancelled);
|
|
57
64
|
}
|
|
58
65
|
for (const controller of this.active) controller.abort();
|
|
59
66
|
}
|
|
60
67
|
|
|
68
|
+
/** Cancellation callbacks are user-supplied: a throw must never break the queue
|
|
69
|
+
* (mirrors the try/catch around onError in drain). */
|
|
70
|
+
private runCancelled(callback: (() => void) | undefined): void {
|
|
71
|
+
if (!callback) return;
|
|
72
|
+
try {
|
|
73
|
+
callback();
|
|
74
|
+
} catch {
|
|
75
|
+
/* cancellation callbacks must never break the queue */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
61
79
|
private drain(): void {
|
|
62
80
|
while (!this.stopped && this.active.size < this.concurrency) {
|
|
63
81
|
const entry = this.pending.shift();
|
|
64
82
|
if (!entry) return;
|
|
65
83
|
if (entry.controller.signal.aborted) {
|
|
66
|
-
entry.onCancelled
|
|
84
|
+
this.runCancelled(entry.onCancelled);
|
|
67
85
|
continue;
|
|
68
86
|
}
|
|
69
87
|
|
|
70
88
|
this.active.add(entry.controller);
|
|
71
|
-
void entry.task(entry.controller.signal)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
89
|
+
void entry.task(entry.controller.signal)
|
|
90
|
+
.catch((error: unknown) => {
|
|
91
|
+
// Cancellation is not a failure: aborted work (e.g. session
|
|
92
|
+
// shutdown) must never be reported as an exception.
|
|
93
|
+
if (entry.controller.signal.aborted) return;
|
|
94
|
+
try {
|
|
95
|
+
entry.onError?.(error);
|
|
96
|
+
} catch {
|
|
97
|
+
/* error reporting must never break the queue */
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
.finally(() => {
|
|
101
|
+
this.active.delete(entry.controller);
|
|
102
|
+
this.drain();
|
|
103
|
+
});
|
|
75
104
|
}
|
|
76
105
|
}
|
|
77
106
|
}
|
package/src/index.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { getAgentDir, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
16
|
-
import { Text
|
|
16
|
+
import { Text } from "@earendil-works/pi-tui";
|
|
17
17
|
import { Type } from "typebox";
|
|
18
18
|
import { discoverAgents, type AgentConfig } from "./agents.ts";
|
|
19
19
|
import { BackgroundTaskQueue } from "./background.ts";
|
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
currentSubagentDepth,
|
|
33
33
|
getResultOutput,
|
|
34
34
|
isFailedResult,
|
|
35
|
+
isModelLevelFailure,
|
|
35
36
|
reviewVerdict,
|
|
36
37
|
runSingleAgentWithModelFallback,
|
|
37
38
|
truncateResultOutput,
|
|
@@ -42,7 +43,19 @@ import {
|
|
|
42
43
|
type UsageStats,
|
|
43
44
|
} from "./spawn.ts";
|
|
44
45
|
import { buildFixTaskBrief, buildReReviewBrief, shouldTriggerFixLoop } from "./fixloop.ts";
|
|
45
|
-
import {
|
|
46
|
+
import {
|
|
47
|
+
activityStateLabel,
|
|
48
|
+
compactModelRef,
|
|
49
|
+
deriveActivityState,
|
|
50
|
+
formatElapsed,
|
|
51
|
+
formatTaskSummary,
|
|
52
|
+
formatToolActivity,
|
|
53
|
+
monitor,
|
|
54
|
+
rightAlign,
|
|
55
|
+
statusIcon,
|
|
56
|
+
statusLabel,
|
|
57
|
+
type RunChainMeta,
|
|
58
|
+
} from "./monitor.ts";
|
|
46
59
|
|
|
47
60
|
const NON_BLANK_TASK_OPTIONS = { minLength: 1, pattern: "\\S" } as const;
|
|
48
61
|
|
|
@@ -92,6 +105,21 @@ function failedStartResult(agentName: string, task: string, errorMessage: string
|
|
|
92
105
|
stderr: errorMessage,
|
|
93
106
|
usage: emptyUsage(),
|
|
94
107
|
errorMessage,
|
|
108
|
+
dispatchFailed: true,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Failed result for a background task that crashed with an exception (spawn
|
|
113
|
+
* infra, delivery API, ...) instead of returning a normal result. */
|
|
114
|
+
function dispatchFailedResult(agent: AgentConfig, task: string, error: unknown, thinking?: string): SingleResult {
|
|
115
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
116
|
+
return {
|
|
117
|
+
...queuedResult(agent, task, thinking),
|
|
118
|
+
exitCode: 1,
|
|
119
|
+
stderr: errorMessage,
|
|
120
|
+
stopReason: "error",
|
|
121
|
+
errorMessage,
|
|
122
|
+
dispatchFailed: true,
|
|
95
123
|
};
|
|
96
124
|
}
|
|
97
125
|
|
|
@@ -132,7 +160,10 @@ function formatCompletionBlock(result: SingleResult, maxResultLines: number, cwd
|
|
|
132
160
|
const fallbackNote = result.modelFallbackFrom
|
|
133
161
|
? ` (model fell back from ${result.modelFallbackFrom} to ${result.model ?? "main-window model"})`
|
|
134
162
|
: "";
|
|
135
|
-
const
|
|
163
|
+
const retryNote = result.startupRetries
|
|
164
|
+
? ` (recovered after ${result.startupRetries} startup retr${result.startupRetries === 1 ? "y" : "ies"} — concurrent pi startup race)`
|
|
165
|
+
: "";
|
|
166
|
+
const lines = [`### [${result.agent}] ${status}${usage ? ` (${usage})` : ""}${fallbackNote}${retryNote}`, "", `Task: ${formatTaskSummary(result.task, 80, false)}`, "", text];
|
|
136
167
|
if (truncated) {
|
|
137
168
|
// The full text lives on disk so the main agent can read it on demand.
|
|
138
169
|
lines.push("", `(output truncated to ${maxResultLines} lines; full result: ${writeResultArtifact(output, result.agent, cwd)})`);
|
|
@@ -140,6 +171,14 @@ function formatCompletionBlock(result: SingleResult, maxResultLines: number, cwd
|
|
|
140
171
|
return lines.join("\n");
|
|
141
172
|
}
|
|
142
173
|
|
|
174
|
+
/** Instruction appended to a model-level failure: the sub-agent's provider never
|
|
175
|
+
* produced usable output (or the run stalled), so the task is handed back to the
|
|
176
|
+
* main window instead of being left as a dead failure. */
|
|
177
|
+
function modelLevelTakeoverNote(result: SingleResult): string {
|
|
178
|
+
const retry = result.modelFallbackFrom ? ", and the retry with the main-window model also failed" : "";
|
|
179
|
+
return `The sub-agent could not complete this task: its model was unavailable or failed (or the run stalled)${retry}. Please execute this task in the main window with your own tools; do not re-dispatch it as a sub-agent.`;
|
|
180
|
+
}
|
|
181
|
+
|
|
143
182
|
export default function (pi: ExtensionAPI): void {
|
|
144
183
|
const configPath = getConfigPath(getAgentDir());
|
|
145
184
|
// Init-time decisions need the config synchronously; the full (migrating)
|
|
@@ -260,29 +299,30 @@ export default function (pi: ExtensionAPI): void {
|
|
|
260
299
|
};
|
|
261
300
|
|
|
262
301
|
// Live sub-agent activity → concise one-line status ("thinking",
|
|
263
|
-
// "read src/index.ts", ...), never a raw args blob.
|
|
264
|
-
//
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
-
//
|
|
268
|
-
|
|
302
|
+
// "read src/index.ts", ...), never a raw args blob. The live handler only
|
|
303
|
+
// updates widget status; finishing (removeRun + notify) is owned by the
|
|
304
|
+
// queue task / launchInLoop. That keeps a startup retry — which fires a
|
|
305
|
+
// transient "failed" status before relaunching — from ripping the row out
|
|
306
|
+
// early, and lets the queue task decide between delivering a reviewer's
|
|
307
|
+
// result and starting an auto-fix chain (a triggered chain keeps the
|
|
308
|
+
// parent row annotated until it completes).
|
|
309
|
+
const makeLiveHandler = (runId: number) => (e: SubagentLiveEvent): void => {
|
|
269
310
|
switch (e.kind) {
|
|
270
311
|
case "status":
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
} else monitor.setStatus(runId, e.status);
|
|
312
|
+
// Only update the widget status here. Finishing (removeRun + notify) is
|
|
313
|
+
// owned by the queue task / launchInLoop so that a startup retry — which
|
|
314
|
+
// fires a transient "failed" status before relaunching the child — never
|
|
315
|
+
// rips the row out from under the retry or emits a premature "✗" toast.
|
|
316
|
+
monitor.setStatus(runId, e.status);
|
|
277
317
|
break;
|
|
278
318
|
case "usage":
|
|
279
319
|
monitor.setUsage(runId, e.usage, e.model);
|
|
280
320
|
break;
|
|
281
321
|
case "tool_start":
|
|
282
|
-
monitor.
|
|
322
|
+
monitor.recordToolStart(runId, e.toolName, formatToolActivity(e.toolName, e.args));
|
|
283
323
|
break;
|
|
284
324
|
case "tool_end":
|
|
285
|
-
|
|
325
|
+
monitor.recordToolEnd(runId, e.toolName, e.isError);
|
|
286
326
|
break;
|
|
287
327
|
case "thinking":
|
|
288
328
|
monitor.setActivity(runId, "thinking");
|
|
@@ -393,6 +433,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
393
433
|
stderr: errorMessage,
|
|
394
434
|
stopReason: signal.aborted ? "aborted" : "error",
|
|
395
435
|
errorMessage,
|
|
436
|
+
dispatchFailed: true,
|
|
396
437
|
};
|
|
397
438
|
}
|
|
398
439
|
};
|
|
@@ -438,11 +479,20 @@ export default function (pi: ExtensionAPI): void {
|
|
|
438
479
|
// success, a stuck one needs a human).
|
|
439
480
|
monitor.removeRun(parentRunId);
|
|
440
481
|
if (!sessionActive) return;
|
|
441
|
-
const items: CompletionMessageItem[] = chain.map((r) =>
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
482
|
+
const items: CompletionMessageItem[] = chain.map((r) => {
|
|
483
|
+
// A model-level chain run (worker or re-review whose provider never
|
|
484
|
+
// produced output) is handed to the main window like any other
|
|
485
|
+
// sub-agent run: the block carries the takeover note. Dispatch
|
|
486
|
+
// crashes (dispatchFailed) are excluded by the gate itself.
|
|
487
|
+
const modelLevel = isFailedResult(r) && isModelLevelFailure(r);
|
|
488
|
+
return {
|
|
489
|
+
agent: r.agent,
|
|
490
|
+
block: modelLevel
|
|
491
|
+
? `${formatCompletionBlock(r, config.maxResultLines, ctx.cwd)}\n\n${modelLevelTakeoverNote(r)}`
|
|
492
|
+
: formatCompletionBlock(r, config.maxResultLines, ctx.cwd),
|
|
493
|
+
triggerTurn: true,
|
|
494
|
+
};
|
|
495
|
+
});
|
|
446
496
|
sendCompletionGroup(items);
|
|
447
497
|
completionBatcher.flush();
|
|
448
498
|
},
|
|
@@ -451,6 +501,30 @@ export default function (pi: ExtensionAPI): void {
|
|
|
451
501
|
// in-flight chain run was already finished by its launchInLoop path).
|
|
452
502
|
monitor.removeRun(parentRunId);
|
|
453
503
|
},
|
|
504
|
+
(error) => {
|
|
505
|
+
// A crash inside the chain orchestration (failed runs are caught by
|
|
506
|
+
// launchInLoop and delivered as part of the chain) must not vanish:
|
|
507
|
+
// drop the retained parent row, notify, and deliver a failed result
|
|
508
|
+
// so the main agent knows the chain never completed.
|
|
509
|
+
monitor.removeRun(parentRunId);
|
|
510
|
+
if (!sessionActive) return;
|
|
511
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
512
|
+
try {
|
|
513
|
+
ctx.ui.notify(`✗ auto-fix chain 派发失败: ${errorMessage}`, "error");
|
|
514
|
+
// Keep the triggering review's findings: the chain crashed before any
|
|
515
|
+
// fix round ran, and the main agent needs the review to act on it.
|
|
516
|
+
sendCompletionGroup([
|
|
517
|
+
{
|
|
518
|
+
agent: initialReviewerResult.agent,
|
|
519
|
+
block: `${formatCompletionBlock(initialReviewerResult, config.maxResultLines, ctx.cwd)}\n\nAuto-fix chain crashed before completion: ${errorMessage}. The planned fix rounds did not run; the review above is the triggering reviewer's full output.`,
|
|
520
|
+
triggerTurn: true,
|
|
521
|
+
},
|
|
522
|
+
]);
|
|
523
|
+
completionBatcher.flush();
|
|
524
|
+
} catch {
|
|
525
|
+
/* a second delivery failure must not throw through the queue */
|
|
526
|
+
}
|
|
527
|
+
},
|
|
454
528
|
);
|
|
455
529
|
};
|
|
456
530
|
|
|
@@ -464,7 +538,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
464
538
|
const runId = monitor.addRun(agent.name, task, agent.model, thinkingLevel);
|
|
465
539
|
// Only a main-agent-dispatched reviewer can trigger an auto-fix chain, so
|
|
466
540
|
// only its finish is deferred to the queue task (see startFixLoop).
|
|
467
|
-
const onLive = makeLiveHandler(runId
|
|
541
|
+
const onLive = makeLiveHandler(runId);
|
|
468
542
|
|
|
469
543
|
backgroundQueue.enqueue(
|
|
470
544
|
async (backgroundSignal) => {
|
|
@@ -493,8 +567,11 @@ export default function (pi: ExtensionAPI): void {
|
|
|
493
567
|
stderr: errorMessage,
|
|
494
568
|
stopReason: backgroundSignal.aborted ? "aborted" : "error",
|
|
495
569
|
errorMessage,
|
|
570
|
+
dispatchFailed: true,
|
|
496
571
|
};
|
|
497
|
-
|
|
572
|
+
// The dedicated dispatch-failure notification below replaces the generic
|
|
573
|
+
// failure toast for dispatch crashes, so finish silently here.
|
|
574
|
+
finishRun(runId, "failed", { silent: true });
|
|
498
575
|
}
|
|
499
576
|
|
|
500
577
|
if (!sessionActive) return;
|
|
@@ -516,13 +593,31 @@ export default function (pi: ExtensionAPI): void {
|
|
|
516
593
|
return;
|
|
517
594
|
}
|
|
518
595
|
const failed = isFailedResult(result);
|
|
519
|
-
|
|
596
|
+
// Model-level failures and dispatch crashes get their own dedicated
|
|
597
|
+
// dispatch-failure notification below, so finishRun's generic failure toast is
|
|
598
|
+
// silenced for them (computed before finishRun for that reason).
|
|
599
|
+
const modelLevel = failed && isModelLevelFailure(result);
|
|
600
|
+
const dispatchFailed = result.dispatchFailed === true;
|
|
601
|
+
finishRun(runId, failed ? "failed" : "done", modelLevel || dispatchFailed ? { silent: true } : undefined);
|
|
520
602
|
if (!sessionActive) return;
|
|
603
|
+
// Model-level failure: the configured model is unavailable or broke
|
|
604
|
+
// and the retry with the main-window model (when distinct) also
|
|
605
|
+
// failed. Instead of leaving a dead failure, hand the task to the
|
|
606
|
+
// main window — the main agent executes it itself with its own tools.
|
|
521
607
|
const completion: CompletionMessageItem = {
|
|
522
608
|
agent: result.agent,
|
|
523
|
-
block:
|
|
609
|
+
block: modelLevel
|
|
610
|
+
? `${formatCompletionBlock(result, config.maxResultLines, ctx.cwd)}\n\n${modelLevelTakeoverNote(result)}`
|
|
611
|
+
: formatCompletionBlock(result, config.maxResultLines, ctx.cwd),
|
|
524
612
|
triggerTurn: completionTriggersTurn(result, config.notifyOnReviewPass),
|
|
525
613
|
};
|
|
614
|
+
if (modelLevel) {
|
|
615
|
+
ctx.ui.notify(`✗ ${result.agent} 派发失败: 模型不可用或出错,任务已交由主窗口执行`, "error");
|
|
616
|
+
} else if (dispatchFailed) {
|
|
617
|
+
// An exception inside the dispatch layer (spawn infra, temp-file/fs
|
|
618
|
+
// errors, ...): the main agent must know so it can re-dispatch.
|
|
619
|
+
ctx.ui.notify(`✗ ${result.agent} 派发失败: ${result.errorMessage ?? "dispatch crashed"}`, "error");
|
|
620
|
+
}
|
|
526
621
|
if (failed) {
|
|
527
622
|
// Failures never wait and never hide behind a success turn: deliver
|
|
528
623
|
// first so the wake-up leads with the failure; held successes follow.
|
|
@@ -533,6 +628,28 @@ export default function (pi: ExtensionAPI): void {
|
|
|
533
628
|
}
|
|
534
629
|
},
|
|
535
630
|
() => finishRun(runId, "failed"),
|
|
631
|
+
(error) => {
|
|
632
|
+
// The task body converts sub-agent failures into delivered results; an
|
|
633
|
+
// exception escaping it (spawn infra, delivery API, ...) must not
|
|
634
|
+
// vanish: notify the user and deliver a failed result so the main
|
|
635
|
+
// agent knows the dispatch failed and can re-dispatch.
|
|
636
|
+
const crashed = dispatchFailedResult(agent, task, error, thinkingLevel);
|
|
637
|
+
finishRun(runId, "failed", { silent: true });
|
|
638
|
+
if (!sessionActive) return;
|
|
639
|
+
try {
|
|
640
|
+
ctx.ui.notify(`✗ ${agent.name} 派发失败: ${crashed.errorMessage}`, "error");
|
|
641
|
+
sendCompletionGroup([
|
|
642
|
+
{
|
|
643
|
+
agent: agent.name,
|
|
644
|
+
block: formatCompletionBlock(crashed, config.maxResultLines, ctx.cwd),
|
|
645
|
+
triggerTurn: true,
|
|
646
|
+
},
|
|
647
|
+
]);
|
|
648
|
+
completionBatcher.flush();
|
|
649
|
+
} catch {
|
|
650
|
+
/* a second delivery failure must not throw through the queue */
|
|
651
|
+
}
|
|
652
|
+
},
|
|
536
653
|
);
|
|
537
654
|
|
|
538
655
|
return pending;
|
|
@@ -660,20 +777,41 @@ export default function (pi: ExtensionAPI): void {
|
|
|
660
777
|
render(width: number): string[] {
|
|
661
778
|
const runs = monitor.getRuns();
|
|
662
779
|
if (runs.length === 0) return [];
|
|
780
|
+
const now = Date.now();
|
|
663
781
|
const lines: string[] = [];
|
|
664
782
|
for (const r of runs) {
|
|
665
783
|
const icon = statusIcon(r.status, theme);
|
|
666
|
-
|
|
667
|
-
//
|
|
668
|
-
//
|
|
784
|
+
// Chain-internal runs (auto-fix worker/reviewer) indent under their parent
|
|
785
|
+
// reviewer. For those, the relationLabel ("fix round 1") is more
|
|
786
|
+
// distinguishing than the repeated worker/reviewer name.
|
|
669
787
|
const head = r.groupId ? theme.fg("dim", " ↳ ") : " ";
|
|
670
|
-
const
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
788
|
+
const name = r.groupId ? (r.relationLabel ?? r.agent) : r.agent;
|
|
789
|
+
// Inline the task's distinguishing key fragments (paths, symbols, quoted
|
|
790
|
+
// phrases) right after the agent name, so parallel runs of the SAME agent
|
|
791
|
+
// are distinguishable at a glance instead of only on a second line.
|
|
792
|
+
const title = formatTaskSummary(r.task, Math.max(16, Math.floor(width * 0.45)), true);
|
|
793
|
+
const left = `${head}${icon} ${theme.fg("dim", `#${r.id}`)} ${theme.bold(name)} ${theme.fg("dim", "›")} ${theme.fg("accent", title)}`;
|
|
794
|
+
|
|
795
|
+
// Right side: compact model (no provider prefix), tool count, elapsed,
|
|
796
|
+
// and the soft activity-state annotation (idle / long-running). Always
|
|
797
|
+
// visible — rightAlign clips the title on overflow, never this.
|
|
798
|
+
const model = compactModelRef(r.model);
|
|
799
|
+
const tools = r.toolCount ? `${r.toolCount} tool${r.toolCount === 1 ? "" : "s"}` : "";
|
|
800
|
+
const elapsed = formatElapsed(r, now);
|
|
801
|
+
const metaParts = [model, tools, elapsed].filter(Boolean);
|
|
802
|
+
// Running is conveyed by the icon + elapsed; spell out the label only for
|
|
803
|
+
// the other states (ready / done / stopped) so they are unambiguous.
|
|
804
|
+
if (r.status !== "running") metaParts.push(statusLabel(r.status));
|
|
805
|
+
const state = deriveActivityState(r, now);
|
|
806
|
+
const stateNote = state ? ` · ${activityStateLabel(state)}` : "";
|
|
807
|
+
const note = r.annotation ? ` · ${r.annotation}` : "";
|
|
808
|
+
const right = theme.fg("dim", `${metaParts.join(" · ")}${stateNote}${note}`);
|
|
809
|
+
lines.push(rightAlign(left, right, width));
|
|
810
|
+
|
|
811
|
+
// Current activity sits one indent below, only while the run is active.
|
|
812
|
+
if (r.activity && (r.status === "running" || r.status === "queued")) {
|
|
813
|
+
lines.push(rightAlign(`${head} ${theme.fg("dim", r.activity)}`, "", width));
|
|
674
814
|
}
|
|
675
|
-
// Activity sits one indent level below the agent name.
|
|
676
|
-
if (r.activity) lines.push(truncateToWidth(theme.fg("dim", ` ${r.activity}`), width, ""));
|
|
677
815
|
}
|
|
678
816
|
return lines;
|
|
679
817
|
},
|
package/src/monitor.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { stripVTControlCharacters } from "node:util";
|
|
14
14
|
import type { Theme } from "@earendil-works/pi-coding-agent";
|
|
15
|
-
import { visibleWidth } from "@earendil-works/pi-tui";
|
|
15
|
+
import { truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
|
|
16
16
|
import type { UsageStats } from "./spawn.ts";
|
|
17
17
|
|
|
18
18
|
// ---------------------------------------------------------------------------
|
|
@@ -21,6 +21,20 @@ import type { UsageStats } from "./spawn.ts";
|
|
|
21
21
|
|
|
22
22
|
export type RunStatus = "queued" | "running" | "done" | "failed";
|
|
23
23
|
|
|
24
|
+
/** Soft state-awareness signals, complementary to the hard idle-kill: a run may
|
|
25
|
+
* be alive (stdout streaming) yet "stuck thinking" (no tool running for a while),
|
|
26
|
+
* or simply taking a long time. Both are surfaced as widget annotations so the
|
|
27
|
+
* user can tell a healthy busy run from one that needs a nudge. */
|
|
28
|
+
export type ActivityState = "needs_attention" | "active_long_running";
|
|
29
|
+
|
|
30
|
+
/** A run with no tool running and no activity for this long is "needs attention"
|
|
31
|
+
* (the model may be stuck between turns). Below the idle-kill threshold so the
|
|
32
|
+
* soft signal always fires before the hard kill. */
|
|
33
|
+
export const NEEDS_ATTENTION_AFTER_MS = 60_000;
|
|
34
|
+
/** A run whose total elapsed time exceeds this is "long-running": still active
|
|
35
|
+
* but worth flagging so the user can decide whether to wait or steer. */
|
|
36
|
+
export const ACTIVE_LONG_RUNNING_AFTER_MS = 240_000;
|
|
37
|
+
|
|
24
38
|
export interface RunView {
|
|
25
39
|
id: number;
|
|
26
40
|
agent: string;
|
|
@@ -32,6 +46,14 @@ export interface RunView {
|
|
|
32
46
|
usage: UsageStats;
|
|
33
47
|
/** Concise current activity ("thinking", "read src/index.ts"); last writer wins. */
|
|
34
48
|
activity?: string;
|
|
49
|
+
/** Total tool calls started by the run so far (a progress signal). */
|
|
50
|
+
toolCount?: number;
|
|
51
|
+
/** Tool currently executing (set on tool_start, cleared on tool_end). When set,
|
|
52
|
+
* the run is NOT idle for needs-attention purposes. */
|
|
53
|
+
currentTool?: string;
|
|
54
|
+
/** Epoch ms of the last live activity (tool, usage, status). Used to derive
|
|
55
|
+
* the needs-attention state: no tool running AND now - lastActivityAt > threshold. */
|
|
56
|
+
lastActivityAt?: number;
|
|
35
57
|
/** Epoch ms when the run started executing (set on first "running" status). */
|
|
36
58
|
startedAt?: number;
|
|
37
59
|
/** Epoch ms when the run finished (set on "done"/"failed"). */
|
|
@@ -233,6 +255,47 @@ export function formatElapsed(run: RunView, now: number = Date.now()): string {
|
|
|
233
255
|
return formatDuration(end - run.startedAt);
|
|
234
256
|
}
|
|
235
257
|
|
|
258
|
+
/** Strip the provider prefix from a "provider/model-id" reference for compact
|
|
259
|
+
* widget display ("anthropic/claude-sonnet-4" → "claude-sonnet-4"). A bare id is
|
|
260
|
+
* left unchanged. */
|
|
261
|
+
export function compactModelRef(model: string | undefined): string {
|
|
262
|
+
if (!model) return "";
|
|
263
|
+
const slash = model.lastIndexOf("/");
|
|
264
|
+
return slash >= 0 ? model.slice(slash + 1) : model;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Human-readable label for a soft activity-state annotation. */
|
|
268
|
+
export function activityStateLabel(state: ActivityState): string {
|
|
269
|
+
return state === "needs_attention" ? "idle" : "long-running";
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** Derive the soft activity state of a run at render time: needs_attention
|
|
273
|
+
* (no tool running, idle past the threshold) takes priority over
|
|
274
|
+
* active_long_running (total elapsed past its threshold). Both are suppressed
|
|
275
|
+
* for non-running runs. */
|
|
276
|
+
export function deriveActivityState(run: RunView, now: number = Date.now()): ActivityState | undefined {
|
|
277
|
+
if (run.status !== "running") return undefined;
|
|
278
|
+
if (!run.currentTool) {
|
|
279
|
+
const since = run.lastActivityAt ?? run.startedAt ?? now;
|
|
280
|
+
if (now - since >= NEEDS_ATTENTION_AFTER_MS) return "needs_attention";
|
|
281
|
+
}
|
|
282
|
+
if (run.startedAt !== undefined && now - run.startedAt >= ACTIVE_LONG_RUNNING_AFTER_MS) {
|
|
283
|
+
return "active_long_running";
|
|
284
|
+
}
|
|
285
|
+
return undefined;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Left/right split a widget line so the right side (status, elapsed) is always
|
|
289
|
+
* visible and the left side (title) clips on overflow instead of pushing it off.
|
|
290
|
+
* `left`/`right` may carry ANSI styling; widths are measured display-column-wise. */
|
|
291
|
+
export function rightAlign(left: string, right: string, width: number): string {
|
|
292
|
+
const rightWidth = visibleWidth(right);
|
|
293
|
+
const leftMax = Math.max(0, width - rightWidth - 1);
|
|
294
|
+
const leftClipped = truncateToWidth(left, leftMax);
|
|
295
|
+
const gap = Math.max(1, width - visibleWidth(leftClipped) - rightWidth);
|
|
296
|
+
return truncateToWidth(`${leftClipped}${" ".repeat(gap)}${right}`, width);
|
|
297
|
+
}
|
|
298
|
+
|
|
236
299
|
/** Max length of the argument target inside a formatted activity line. */
|
|
237
300
|
export const ACTIVITY_TARGET_MAX = 60;
|
|
238
301
|
|
|
@@ -337,6 +400,7 @@ export class MonitorStore {
|
|
|
337
400
|
// A model-fallback retry after a failed attempt restarts the clock; a
|
|
338
401
|
// stale endedAt would freeze the elapsed display at the first attempt.
|
|
339
402
|
if (run.endedAt !== undefined) run.endedAt = undefined;
|
|
403
|
+
run.lastActivityAt = Date.now();
|
|
340
404
|
} else if ((status === "done" || status === "failed") && run.endedAt === undefined) {
|
|
341
405
|
run.endedAt = Date.now();
|
|
342
406
|
}
|
|
@@ -347,6 +411,7 @@ export class MonitorStore {
|
|
|
347
411
|
if (!run) return;
|
|
348
412
|
run.usage = { ...usage };
|
|
349
413
|
if (model) run.model = model;
|
|
414
|
+
run.lastActivityAt = Date.now();
|
|
350
415
|
this.notify();
|
|
351
416
|
}
|
|
352
417
|
|
|
@@ -355,6 +420,31 @@ export class MonitorStore {
|
|
|
355
420
|
const run = this.find(id);
|
|
356
421
|
if (!run) return;
|
|
357
422
|
run.activity = text;
|
|
423
|
+
run.lastActivityAt = Date.now();
|
|
424
|
+
this.notify();
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Record a tool starting: counts it, marks it current, and updates activity.
|
|
428
|
+
* A running tool means the run is NOT idle, so needs-attention is suppressed
|
|
429
|
+
* while it stays current. */
|
|
430
|
+
recordToolStart(id: number, toolName: string, activity: string): void {
|
|
431
|
+
const run = this.find(id);
|
|
432
|
+
if (!run) return;
|
|
433
|
+
run.toolCount = (run.toolCount ?? 0) + 1;
|
|
434
|
+
run.currentTool = toolName;
|
|
435
|
+
run.activity = activity;
|
|
436
|
+
run.lastActivityAt = Date.now();
|
|
437
|
+
this.notify();
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/** Record a tool ending: clears the current-tool marker (so the run becomes
|
|
441
|
+
* eligible for needs-attention again) and notes the failure in activity. */
|
|
442
|
+
recordToolEnd(id: number, toolName: string, isError: boolean): void {
|
|
443
|
+
const run = this.find(id);
|
|
444
|
+
if (!run) return;
|
|
445
|
+
run.currentTool = undefined;
|
|
446
|
+
run.lastActivityAt = Date.now();
|
|
447
|
+
if (isError) run.activity = `✗ ${toolName} failed`;
|
|
358
448
|
this.notify();
|
|
359
449
|
}
|
|
360
450
|
|
package/src/spawn.ts
CHANGED
|
@@ -32,6 +32,15 @@ export const SUBAGENT_KILL_GRACE_MS = 5_000;
|
|
|
32
32
|
* (idleTimeoutSec); this constant is only a fallback for tests. */
|
|
33
33
|
export const SUBAGENT_DEFAULT_IDLE_TIMEOUT_MS = 0;
|
|
34
34
|
|
|
35
|
+
/** Backoff schedule for retrying a child that exited before any model or tool
|
|
36
|
+
* activity — the signature of a concurrent pi startup race, where several
|
|
37
|
+
* sub-agents contending for pi's startup lock lose and exit with nothing on
|
|
38
|
+
* stdout. Bounded and short so persistent launch failures are not amplified
|
|
39
|
+
* while the startup lock clears. */
|
|
40
|
+
export const SUBAGENT_STARTUP_RETRY_DELAYS_MS = [250, 750, 1500] as const;
|
|
41
|
+
/** A genuine startup race fails well before a model request can complete. */
|
|
42
|
+
export const MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS = 2000;
|
|
43
|
+
|
|
35
44
|
export interface UsageStats {
|
|
36
45
|
input: number;
|
|
37
46
|
output: number;
|
|
@@ -57,6 +66,14 @@ export interface SingleResult {
|
|
|
57
66
|
errorMessage?: string;
|
|
58
67
|
/** Model the run degraded from: set when a failed run was retried with the main-window model. */
|
|
59
68
|
modelFallbackFrom?: string;
|
|
69
|
+
/** True when the result was synthesized from a thrown exception (spawn infra,
|
|
70
|
+
* temp-file/fs errors, delivery bugs) instead of being produced by the agent
|
|
71
|
+
* process. A dispatch failure is never a model-level failure. */
|
|
72
|
+
dispatchFailed?: boolean;
|
|
73
|
+
/** How many times the run was relaunched after a silent, zero-activity startup
|
|
74
|
+
* exit (a concurrent pi startup race) before it produced a result. Set only when
|
|
75
|
+
* the run actually recovered after retrying, so callers can surface it. */
|
|
76
|
+
startupRetries?: number;
|
|
60
77
|
}
|
|
61
78
|
|
|
62
79
|
export interface SubagentDetails {
|
|
@@ -158,6 +175,10 @@ export function isFailedResult(result: SingleResult): boolean {
|
|
|
158
175
|
export function isModelLevelFailure(result: SingleResult): boolean {
|
|
159
176
|
if (!isFailedResult(result)) return false;
|
|
160
177
|
if (result.stopReason === "aborted") return false;
|
|
178
|
+
// A result synthesized from a thrown exception (spawn infra, fs, delivery
|
|
179
|
+
// bugs) never came from the provider: it is a dispatch failure, not a
|
|
180
|
+
// model-level one, and must not be handed back as a model problem.
|
|
181
|
+
if (result.dispatchFailed) return false;
|
|
161
182
|
// An idle timeout (stdout went silent) signals a stalled provider connection,
|
|
162
183
|
// not a task-level failure: allow model fallback even if the model produced
|
|
163
184
|
// partial output before going quiet.
|
|
@@ -169,6 +190,73 @@ export function isModelLevelFailure(result: SingleResult): boolean {
|
|
|
169
190
|
return result.messages.length > 0 || result.stderr.trim().length > 0;
|
|
170
191
|
}
|
|
171
192
|
|
|
193
|
+
/**
|
|
194
|
+
* True when a failed run produced NO model, tool, output, or usage activity
|
|
195
|
+
* within the startup window — the signature of a concurrent pi startup race,
|
|
196
|
+
* where the child lost pi's startup lock and exited before doing anything.
|
|
197
|
+
* Such a run is safe to relaunch: nothing was mutated and no provider call
|
|
198
|
+
* completed, so retrying cannot duplicate work.
|
|
199
|
+
*
|
|
200
|
+
* Fails closed: any final output, assistant message, usage, stderr, structured
|
|
201
|
+
* error message, idle-timeout, abort, dispatch crash, or run that outlived the
|
|
202
|
+
* startup window disqualifies the run from retry (it either did real work or
|
|
203
|
+
* carries a real error that belongs to model fallback / normal failure
|
|
204
|
+
* delivery instead). Only a clean, SILENT, fast, zero-activity exit retries.
|
|
205
|
+
*/
|
|
206
|
+
export function isRetryableStartupFailure(result: SingleResult, durationMs: number): boolean {
|
|
207
|
+
if (result.exitCode === 0) return false;
|
|
208
|
+
if (result.stopReason === "aborted") return false;
|
|
209
|
+
if (result.dispatchFailed) return false;
|
|
210
|
+
if (result.errorMessage?.includes("idle timeout")) return false;
|
|
211
|
+
if (getFinalOutput(result.messages)) return false;
|
|
212
|
+
if (result.messages.length > 0) return false;
|
|
213
|
+
const usage = result.usage;
|
|
214
|
+
if (usage.turns || usage.input || usage.output || usage.cacheRead || usage.cacheWrite || usage.cost) return false;
|
|
215
|
+
if (durationMs > MAX_SUBAGENT_STARTUP_FAILURE_DURATION_MS) return false;
|
|
216
|
+
// Any stderr or structured error could be a real provider/config error (auth,
|
|
217
|
+
// bad model id, quota, ...) that must not be amplified by retry. A silent
|
|
218
|
+
// zero-activity exit — no stdout, no stderr, no error message — is the race.
|
|
219
|
+
if (result.stderr.trim().length > 0) return false;
|
|
220
|
+
if (result.errorMessage && result.errorMessage.trim().length > 0) return false;
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Error surfaced when every startup-retry attempt still exited with no
|
|
225
|
+
* activity. Tells the main agent the dispatch never reached a model and what to
|
|
226
|
+
* do (retry, or lower maxConcurrency). */
|
|
227
|
+
export function formatStartupRetryExhaustedError(model: string, attempts: number): string {
|
|
228
|
+
return `Subagent failed to start after ${attempts} attempt${attempts === 1 ? "" : "s"} on ${model}: the child exited before any model, tool, output, or usage activity. This is typically a concurrent pi startup race (several sub-agents starting at once). Retry the dispatch, or temporarily lower maxConcurrency in /subagents-setup.`;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Wait out a startup-retry backoff. Resolves false immediately (do not retry)
|
|
232
|
+
* when the signal is or becomes aborted during the wait, so cancellation never
|
|
233
|
+
* delays delivering the last result. The timer is unref'd so it cannot keep the
|
|
234
|
+
* event loop alive on shutdown. */
|
|
235
|
+
export async function waitForStartupRetry(delayMs: number, signal?: AbortSignal): Promise<boolean> {
|
|
236
|
+
if (delayMs <= 0) return !signal?.aborted;
|
|
237
|
+
if (!signal) {
|
|
238
|
+
return new Promise<boolean>((resolve) => {
|
|
239
|
+
const timer = setTimeout(() => resolve(true), delayMs);
|
|
240
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
if (signal.aborted) return false;
|
|
244
|
+
return new Promise<boolean>((resolve) => {
|
|
245
|
+
let settled = false;
|
|
246
|
+
const finish = (shouldRetry: boolean): void => {
|
|
247
|
+
if (settled) return;
|
|
248
|
+
settled = true;
|
|
249
|
+
clearTimeout(timer);
|
|
250
|
+
signal.removeEventListener("abort", onAbort);
|
|
251
|
+
resolve(shouldRetry);
|
|
252
|
+
};
|
|
253
|
+
const onAbort = (): void => finish(false);
|
|
254
|
+
const timer = setTimeout(() => finish(true), delayMs);
|
|
255
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
256
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
|
|
172
260
|
export function getResultOutput(result: SingleResult): string {
|
|
173
261
|
if (isFailedResult(result)) {
|
|
174
262
|
const error = result.errorMessage || result.stderr;
|
|
@@ -245,6 +333,10 @@ export interface RunSingleOptions {
|
|
|
245
333
|
/** Idle timeout in ms: terminate the child if its stdout produces no activity
|
|
246
334
|
* for this duration. 0 (the default) disables the idle watchdog. */
|
|
247
335
|
idleTimeoutMs?: number;
|
|
336
|
+
/** Startup-retry backoff schedule (ms) for silent, zero-activity child exits
|
|
337
|
+
* (a concurrent pi startup race). Defaults to SUBAGENT_STARTUP_RETRY_DELAYS_MS;
|
|
338
|
+
* pass a shorter array in tests to keep them fast. */
|
|
339
|
+
startupRetryDelaysMs?: readonly number[];
|
|
248
340
|
signal?: AbortSignal;
|
|
249
341
|
onLive?: (e: SubagentLiveEvent) => void;
|
|
250
342
|
makeDetails: (results: SingleResult[]) => SubagentDetails;
|
|
@@ -540,22 +632,68 @@ export async function runSingleAgent(options: RunSingleOptions): Promise<SingleR
|
|
|
540
632
|
}
|
|
541
633
|
|
|
542
634
|
/**
|
|
543
|
-
* Run one agent
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
547
|
-
*
|
|
548
|
-
*
|
|
635
|
+
* Run one agent with two layers of resilience against transient dispatch failures:
|
|
636
|
+
*
|
|
637
|
+
* 1. Startup retry (inner loop): a concurrent pi startup race can make the child
|
|
638
|
+
* exit before any model/tool activity. Relaunch with backoff so the startup
|
|
639
|
+
* lock clears. The SAME model is retried — the race is in the host, not the
|
|
640
|
+
* model — and only a clean, silent, zero-activity exit qualifies (see
|
|
641
|
+
* isRetryableStartupFailure), so retrying can never duplicate real work.
|
|
642
|
+
* 2. Model fallback (outer): when the provider rejects the configured model
|
|
643
|
+
* before producing output (see isModelLevelFailure), retry once with the main
|
|
644
|
+
* window's current model. The fallback gets its own startup-retry loop, since
|
|
645
|
+
* a startup race can hit any relaunch regardless of model.
|
|
646
|
+
*
|
|
647
|
+
* The fallback is per-run only and never persisted: a transient provider hiccup
|
|
648
|
+
* must not silently downgrade the configured agent model.
|
|
549
649
|
*/
|
|
550
650
|
export async function runSingleAgentWithModelFallback(
|
|
551
651
|
options: RunSingleOptions,
|
|
552
652
|
fallbackModelRef?: string,
|
|
553
653
|
): Promise<SingleResult> {
|
|
554
|
-
const result = await runSingleAgent(options);
|
|
555
654
|
const agent = options.agent;
|
|
556
655
|
const launchedRef = agent?.model;
|
|
656
|
+
const delays = options.startupRetryDelaysMs ?? SUBAGENT_STARTUP_RETRY_DELAYS_MS;
|
|
657
|
+
|
|
658
|
+
const runWithStartupRetry = async (opts: RunSingleOptions): Promise<SingleResult> => {
|
|
659
|
+
let lastResult: SingleResult;
|
|
660
|
+
let retries = 0;
|
|
661
|
+
for (let attempt = 0; ; attempt++) {
|
|
662
|
+
const start = Date.now();
|
|
663
|
+
lastResult = await runSingleAgent(opts);
|
|
664
|
+
const durationMs = Date.now() - start;
|
|
665
|
+
if (!isRetryableStartupFailure(lastResult, durationMs)) {
|
|
666
|
+
if (retries > 0 && !isFailedResult(lastResult)) lastResult.startupRetries = retries;
|
|
667
|
+
return lastResult;
|
|
668
|
+
}
|
|
669
|
+
const delay = delays[attempt];
|
|
670
|
+
if (delay === undefined) {
|
|
671
|
+
// Exhausted: the agent never reached a model. Surface the concurrency-race
|
|
672
|
+
// cause as a dispatch-level failure (no model was ever reached, so this
|
|
673
|
+
// must NOT trigger model fallback) so the main agent can retry or lower
|
|
674
|
+
// maxConcurrency.
|
|
675
|
+
lastResult.errorMessage = formatStartupRetryExhaustedError(
|
|
676
|
+
lastResult.model ?? opts.agent?.model ?? "default",
|
|
677
|
+
attempt + 1,
|
|
678
|
+
);
|
|
679
|
+
lastResult.stopReason ??= "error";
|
|
680
|
+
lastResult.dispatchFailed = true;
|
|
681
|
+
return lastResult;
|
|
682
|
+
}
|
|
683
|
+
// Flip the live status back to running so the widget does not flash a
|
|
684
|
+
// false "failed" while we wait out the backoff and relaunch the child.
|
|
685
|
+
try {
|
|
686
|
+
opts.onLive?.({ kind: "status", status: "running" });
|
|
687
|
+
} catch { /* never throw from event handling */ }
|
|
688
|
+
const shouldRetry = await waitForStartupRetry(delay, opts.signal);
|
|
689
|
+
if (!shouldRetry) return lastResult;
|
|
690
|
+
retries++;
|
|
691
|
+
}
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
const result = await runWithStartupRetry(options);
|
|
557
695
|
if (!agent || !launchedRef || !fallbackModelRef || launchedRef === fallbackModelRef) return result;
|
|
558
696
|
if (!isModelLevelFailure(result)) return result;
|
|
559
|
-
const retried = await
|
|
697
|
+
const retried = await runWithStartupRetry({ ...options, agent: { ...agent, model: fallbackModelRef } });
|
|
560
698
|
return { ...retried, modelFallbackFrom: launchedRef };
|
|
561
699
|
}
|