@mjasnikovs/pi-task 0.39.0 → 0.39.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/group-args.d.ts +24 -9
- package/dist/config/group-args.js +38 -28
- package/dist/config/register.d.ts +0 -9
- package/dist/config/register.js +17 -62
- package/dist/shared/child-process.d.ts +34 -32
- package/dist/shared/child-process.js +44 -58
- package/dist/shared/command-watchdog.d.ts +12 -4
- package/dist/shared/command-watchdog.js +6 -7
- package/dist/shared/connection-error.d.ts +7 -0
- package/dist/shared/connection-error.js +65 -0
- package/dist/shared/model-endpoint.d.ts +12 -24
- package/dist/shared/model-endpoint.js +32 -82
- package/dist/shared/model-resolve.d.ts +105 -0
- package/dist/shared/model-resolve.js +97 -0
- package/dist/shared/reasoning-capability.d.ts +20 -0
- package/dist/shared/reasoning-capability.js +32 -1
- package/dist/shared/stall-probe.d.ts +51 -0
- package/dist/shared/stall-probe.js +79 -0
- package/dist/task/child-runner.d.ts +76 -278
- package/dist/task/child-runner.js +186 -722
- package/dist/task/context-usage.js +2 -7
- package/dist/task/failure-classifier.js +53 -81
- package/dist/task/gate-child.js +1 -1
- package/dist/task/impl-widget.d.ts +2 -0
- package/dist/task/impl-widget.js +4 -0
- package/dist/task/implementation-hold.d.ts +11 -0
- package/dist/task/implementation-hold.js +20 -0
- package/dist/task/implementation-scope.d.ts +24 -0
- package/dist/task/implementation-scope.js +34 -0
- package/dist/task/loop-detector.d.ts +13 -5
- package/dist/task/loop-detector.js +11 -5
- package/dist/task/model-hold-stash.js +4 -14
- package/dist/task/orchestrator.d.ts +1 -8
- package/dist/task/orchestrator.js +11 -34
- package/dist/task/phases.js +2 -2
- package/dist/task/stall-detector.d.ts +1 -1
- package/dist/task/stall-detector.js +1 -1
- package/dist/workers/model-warning.d.ts +4 -16
- package/dist/workers/model-warning.js +14 -70
- package/dist/workers/pi-worker-core.d.ts +65 -20
- package/dist/workers/pi-worker-core.js +109 -50
- package/dist/workers/reasoning-warning.js +2 -24
- package/dist/workers/worker-failure.d.ts +2 -0
- package/dist/workers/worker-failure.js +2 -1
- package/dist/workers/worker-kill.d.ts +30 -11
- package/dist/workers/worker-kill.js +68 -20
- package/dist/workers/worker-profiles.d.ts +20 -0
- package/dist/workers/worker-profiles.js +22 -9
- package/package.json +1 -1
|
@@ -1,470 +1,92 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* The phase children's adapter over the one attempt loop.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* `runWorker` (workers/pi-worker-core.ts) owns every guard a model child runs
|
|
5
|
+
* under and every restart it may be granted; the `phase` row of WORKER_PROFILES
|
|
6
|
+
* says which. What is left here is what a PHASE child is that a research worker
|
|
7
|
+
* is not: it is named, its name picks its group, its failure is THROWN as a
|
|
8
|
+
* typed error the pipeline switches on, and its loop kills leave a trail in the
|
|
9
|
+
* task file. Nothing here re-decides how a child may die.
|
|
7
10
|
*/
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import { StallDetector, formatStallHint } from './stall-detector.js';
|
|
14
|
-
import { detectLeakedToolCall, leakedToolCallHint, MAX_LEAK_RETRIES } from '../shared/leaked-tool-call.js';
|
|
11
|
+
import { runWorker } from '../workers/pi-worker-core.js';
|
|
12
|
+
import { classifyWorkerFailure } from '../workers/worker-failure.js';
|
|
13
|
+
import { isFatalKill } from '../workers/worker-kill.js';
|
|
14
|
+
import { MAX_LOOP_RESTARTS } from './loop-detector.js';
|
|
15
|
+
import { MAX_LEAK_RETRIES } from '../shared/leaked-tool-call.js';
|
|
15
16
|
import { readSection, setTaskSection } from './task-io.js';
|
|
16
17
|
import { streamStallCause } from '../shared/stream-watchdog.js';
|
|
17
|
-
import { commandCeilingForAttempt, commandTimeoutHint, commandWatch } from '../shared/command-watchdog.js';
|
|
18
|
-
import { childModelEndpoints, probeModelEndpoints } from '../shared/model-endpoint.js';
|
|
19
|
-
import { modelSpecFromArgs } from '../config/group-models.js';
|
|
20
|
-
// VALUE import, and it is only safe because worker-profiles.ts reads its loop
|
|
21
|
-
// constants from loop-detector.ts. Point those back at this file and the graph
|
|
22
|
-
// closes into a TDZ ReferenceError that no compile step catches.
|
|
23
|
-
import { workerPolicy } from '../workers/worker-profiles.js';
|
|
24
18
|
import { getConfig } from '../config/config.js';
|
|
25
19
|
import { groupChildArgs, groupWindow } from '../config/group-args.js';
|
|
26
20
|
import { groupForChild } from '../config/groups.js';
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
*
|
|
31
|
-
* WHY OFF, AND NOT A NUMBER. A wall clock on a model child measures the
|
|
32
|
-
* MODEL'S SPEED, not its health. The same planning child that answers well in
|
|
33
|
-
* seconds on one backend takes many minutes on another, or on the same backend
|
|
34
|
-
* with thinking turned on — so any cap generous enough to be safe is too loose
|
|
35
|
-
* to catch anything, and any cap tight enough to catch a runaway kills healthy
|
|
36
|
-
* work. Assume a model that emits one token per second and the number has no
|
|
37
|
-
* defensible value at all.
|
|
38
|
-
*
|
|
39
|
-
* The runaway it was there to catch — a child forward-paging through its whole
|
|
40
|
-
* context window, past the loop detector, never going to return — is caught by
|
|
41
|
-
* StallDetector (stall-detector.ts) instead. That bounds NON-PROGRESS and
|
|
42
|
-
* CONTEXT CHURN, both properties of the pathology itself, so neither has to be
|
|
43
|
-
* re-tuned for a slower model or a bigger repo.
|
|
44
|
-
*
|
|
45
|
-
* The value and the plumbing stay for a caller that genuinely wants a hard stop
|
|
46
|
-
* (tests inject a short one), but nothing sets it in production. Pass
|
|
47
|
-
* `timeoutMs` explicitly to arm it.
|
|
48
|
-
*/
|
|
49
|
-
export const PHASE_CHILD_TIMEOUT_MS = 0;
|
|
50
|
-
/**
|
|
51
|
-
* Restart hint after a phase child burns its whole wall-clock budget. It
|
|
52
|
-
* diagnoses over-exploration, which is what the cap actually catches — the same
|
|
53
|
-
* job WORKER_TIMEOUT_HINT does for research workers.
|
|
54
|
-
*/
|
|
55
|
-
export const PHASE_TIMEOUT_HINT = '[SYSTEM NOTE: Your previous attempt ran out of time before answering — you '
|
|
56
|
-
+ 'were re-reading source material you had already seen. Read each file AT '
|
|
57
|
-
+ 'MOST ONCE, then write your answer from what you have. Do not re-open a '
|
|
58
|
-
+ 'file you have already read.]';
|
|
59
|
-
/**
|
|
60
|
-
* Combine the caller's abort signal with a wall-clock timer into one signal,
|
|
61
|
-
* keeping the two causes apart: `timedOut()` is true only when the timer fired,
|
|
62
|
-
* never when the user cancelled — so a cap can restart the child while a cancel
|
|
63
|
-
* still ends the run. `ms <= 0` disables the timer entirely.
|
|
64
|
-
*
|
|
65
|
-
* (workers/pi-worker-core.ts has the same shape for research workers. It is not
|
|
66
|
-
* shared because that module imports FROM this one; a common home for it would
|
|
67
|
-
* be worth it if a third caller ever appears.)
|
|
68
|
-
*/
|
|
69
|
-
function phaseTimeout(external, ms) {
|
|
70
|
-
const ctrl = new AbortController();
|
|
71
|
-
let firedByTimer = false;
|
|
72
|
-
const armed = ms > 0 && Number.isFinite(ms);
|
|
73
|
-
const timer = armed ?
|
|
74
|
-
setTimeout(() => {
|
|
75
|
-
firedByTimer = true;
|
|
76
|
-
ctrl.abort();
|
|
77
|
-
}, ms)
|
|
78
|
-
: undefined;
|
|
79
|
-
const onExternal = () => ctrl.abort();
|
|
80
|
-
if (external.aborted)
|
|
81
|
-
ctrl.abort();
|
|
82
|
-
else
|
|
83
|
-
external.addEventListener('abort', onExternal, { once: true });
|
|
84
|
-
return {
|
|
85
|
-
signal: ctrl.signal,
|
|
86
|
-
timedOut: () => firedByTimer,
|
|
87
|
-
cleanup: () => {
|
|
88
|
-
if (timer)
|
|
89
|
-
clearTimeout(timer);
|
|
90
|
-
external.removeEventListener('abort', onExternal);
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
/** Thrown when a phase child spends its whole restart budget hitting the cap. */
|
|
95
|
-
export class PhaseTimeoutError extends Error {
|
|
96
|
-
childName;
|
|
97
|
-
budgetMs;
|
|
98
|
-
attempts;
|
|
99
|
-
constructor(childName, budgetMs, attempts) {
|
|
100
|
-
super(`${childName} child exceeded its ${Math.round(budgetMs / 1000)}s budget on all `
|
|
101
|
-
+ `${attempts} attempt(s) — it never stopped working long enough to answer`);
|
|
102
|
-
this.childName = childName;
|
|
103
|
-
this.budgetMs = budgetMs;
|
|
104
|
-
this.attempts = attempts;
|
|
105
|
-
this.name = 'PhaseTimeoutError';
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* The terminal error for a guard kill, or null when the child was not killed.
|
|
110
|
-
*
|
|
111
|
-
* Both spawn paths must ask. A kill reports `exitCode: 0` (child-process.ts uses
|
|
112
|
-
* `code ?? 0`, and a signal gives null), so a path that tests the exit code
|
|
113
|
-
* instead returns the truncated text as the phase's answer.
|
|
114
|
-
*/
|
|
115
|
-
export function guardKillError(name, r, opts = {}) {
|
|
116
|
-
if (r.commandKill)
|
|
117
|
-
return new CommandTimeoutError(name, r.commandKill);
|
|
118
|
-
// A dead-backend verdict is only trusted once every attempt has produced it.
|
|
119
|
-
// The probe now asks about this child's own endpoint rather than ORing over
|
|
120
|
-
// every provider, so it is exact — but it is still one network call at one
|
|
121
|
-
// instant, and a blip is indistinguishable from a death in a single sample.
|
|
122
|
-
// The asymmetry settles it: a backend that really is down costs three 5s
|
|
123
|
-
// probes, a wrong verdict costs the whole run.
|
|
124
|
-
if (r.stalled)
|
|
125
|
-
return opts.finalAttempt === false ? null : new BackendDownError(name);
|
|
126
|
-
return null;
|
|
127
|
-
}
|
|
21
|
+
// Sentinel error thrown when the user dismisses a grill-me dialog or cancels a
|
|
22
|
+
// run. Defined here (not in failure-classifier.ts) to avoid a circular dependency.
|
|
23
|
+
export const USER_CANCELLED = '__user_cancelled__';
|
|
128
24
|
/**
|
|
129
|
-
*
|
|
25
|
+
* ONE error class for every way a phase child fails, carrying the cause as data.
|
|
130
26
|
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
27
|
+
* Six classes plus a string sentinel used to say the same nine things, and
|
|
28
|
+
* `classifyFailure` rebuilt the ladder by `instanceof` and then fell through
|
|
29
|
+
* to sniffing the message — the tell that the vocabulary was leaking. A catch
|
|
30
|
+
* site asks `isFatalChildCause`; the notice switches on `failure.kind`.
|
|
135
31
|
*/
|
|
136
|
-
export class
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
this.
|
|
32
|
+
export class ChildFailureError extends Error {
|
|
33
|
+
phase;
|
|
34
|
+
failure;
|
|
35
|
+
stderr;
|
|
36
|
+
constructor(phase, failure, stderr = '') {
|
|
37
|
+
super(describeChildFailure(phase, failure, stderr));
|
|
38
|
+
this.phase = phase;
|
|
39
|
+
this.failure = failure;
|
|
40
|
+
this.stderr = stderr;
|
|
41
|
+
this.name = 'ChildFailureError';
|
|
143
42
|
}
|
|
144
43
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
44
|
+
function describeChildFailure(phase, f, stderr) {
|
|
45
|
+
switch (f.kind) {
|
|
46
|
+
case 'stalled':
|
|
47
|
+
return (`${phase} child killed: no output for the stall window and the model `
|
|
48
|
+
+ `endpoint did not answer a probe`);
|
|
49
|
+
case 'command-timeout':
|
|
50
|
+
return (`${phase} child ran \`${f.toolName}\``
|
|
51
|
+
+ `${f.detail ? ` (${f.detail})` : ''} past its `
|
|
52
|
+
+ `${Math.round(f.timeoutMs / 1000)}s ceiling on every attempt`);
|
|
53
|
+
case 'stream-stall':
|
|
54
|
+
return `${phase} child: model error — ${streamStallCause(f.idleMs)}`;
|
|
55
|
+
case 'worker-timeout':
|
|
56
|
+
return (`${phase} child ran out of time on every attempt — it never stopped `
|
|
57
|
+
+ `working long enough to answer`);
|
|
58
|
+
case 'loop':
|
|
59
|
+
return `loop detected ${f.strikes} times in ${phase}`;
|
|
60
|
+
case 'leaked-tool-call':
|
|
61
|
+
return (`${phase} child wrote a tool call as text instead of invoking it `
|
|
62
|
+
+ `(${f.text.trim()}) — it never ran`);
|
|
63
|
+
case 'aborted':
|
|
64
|
+
return `${phase} child aborted`;
|
|
65
|
+
case 'exit':
|
|
66
|
+
// The exit code is the only signal when stderr is empty, and pi exits
|
|
67
|
+
// silently on several paths (143 = SIGTERM, 137 = SIGKILL/OOM).
|
|
68
|
+
return `${phase} child failed: ${stderr || `no stderr, exit ${f.code}`}`;
|
|
69
|
+
case 'model-error':
|
|
70
|
+
return `${phase} child: model error — ${f.cause}`;
|
|
71
|
+
case 'empty-answer':
|
|
72
|
+
return `${phase} child produced no output${stderr ? ' — stderr: ' + stderr : ''}`;
|
|
160
73
|
}
|
|
161
74
|
}
|
|
162
75
|
/**
|
|
163
76
|
* Causes a best-effort `catch` must NOT absorb.
|
|
164
77
|
*
|
|
165
78
|
* A phase child that merely answered badly should degrade — that is what those
|
|
166
|
-
* catches are for.
|
|
167
|
-
* and swallowing them ships a half-built spec while
|
|
168
|
-
* against the same dead
|
|
169
|
-
*
|
|
170
|
-
* unreachable.
|
|
79
|
+
* catches are for. A dead backend and a user cancel are different in kind: the
|
|
80
|
+
* run is over either way, and swallowing them ships a half-built spec while
|
|
81
|
+
* every later phase dies against the same dead server, or turns an ESC into
|
|
82
|
+
* silent progress. Which kills are fatal is the roster's column, not a list here.
|
|
171
83
|
*/
|
|
172
84
|
export function isFatalChildCause(e) {
|
|
173
|
-
if (e instanceof
|
|
174
|
-
return
|
|
85
|
+
if (e instanceof ChildFailureError)
|
|
86
|
+
return isFatalKill(e.failure.kind);
|
|
175
87
|
return e instanceof Error && e.message === USER_CANCELLED;
|
|
176
88
|
}
|
|
177
|
-
// ───
|
|
178
|
-
/**
|
|
179
|
-
* A connection-class model error is transient: a single dropped fetch to a live
|
|
180
|
-
* endpoint, not a repeatable mistake. On a local single-slot server (e.g.
|
|
181
|
-
* llama-server with `--parallel 1`) pi-task's own concurrent fan-out can briefly
|
|
182
|
-
* saturate the slot, and one request fails to connect even though the model is
|
|
183
|
-
* up and the next request succeeds. pi already retries internally, but those
|
|
184
|
-
* retries don't always absorb it on a saturated local server — and pi-task's
|
|
185
|
-
* fail-fast then kills the whole task (and, under /task-auto, the whole run) for
|
|
186
|
-
* a single blip. We retry these within the existing strike/leak budget.
|
|
187
|
-
*
|
|
188
|
-
* A NON-connection model error (bad request, context-length overflow, auth,
|
|
189
|
-
* provider 5xx that names a real fault) still fails fast: re-spawning against
|
|
190
|
-
* the same request won't fix it, so burning the budget only delays the report.
|
|
191
|
-
*/
|
|
192
|
-
/**
|
|
193
|
-
* Transport-level failures worth another attempt.
|
|
194
|
-
*
|
|
195
|
-
* SCOPE, and it is deliberate: connection classes only. pi's own
|
|
196
|
-
* `isRetryableAssistantError` (@earendil-works/pi-ai, `dist/utils/retry.js`) also
|
|
197
|
-
* retries the provider-LOAD family — `429`, `5xx`, `rate limit`, `overloaded` —
|
|
198
|
-
* which `does NOT match real, non-transient faults` in child-runner.test.ts
|
|
199
|
-
* explicitly rejects. That disagreement is real and OPEN; it is not settled here,
|
|
200
|
-
* because this backoff starts at 500ms and a 429 answered that fast is a retry
|
|
201
|
-
* storm, not a recovery.
|
|
202
|
-
*
|
|
203
|
-
* MEASURED against pi before widening: the transport entries added here — a bare
|
|
204
|
-
* `timed out`, `getaddrinfo ENOTFOUND`, `upstream connect`, `reset before
|
|
205
|
-
* headers`, a truncated Anthropic stream and a closed websocket — were all
|
|
206
|
-
* MISSES. Every one is a REMOTE-provider failure, which is why a local llama.cpp
|
|
207
|
-
* setup never surfaced the gap. The errno spellings are pi-task's own: a child
|
|
208
|
-
* reports them through stderr, and pi never sees them.
|
|
209
|
-
*
|
|
210
|
-
* pi's bare `timeout` is deliberately NOT reproduced. It matched a provider 400
|
|
211
|
-
* that merely echoed a `timeout` field back, turning a fail-fast into a full
|
|
212
|
-
* retry budget, and it caught nothing the `timed out` spellings above miss.
|
|
213
|
-
*/
|
|
214
|
-
const CONNECTION_ERROR_RE = /\b(?:connection error|connection (?:lost|closed|reset|refused|aborted)|econnreset|econnrefused|econnaborted|epipe|etimedout|enetunreach|enetdown|eai_again|socket hang up|socket connection was closed|fetch failed|network (?:error|timeout)|premature close|terminated|unreachable|getaddrinfo|enotfound|upstream.?connect|reset before headers|timed? out|ended without|stream ended before message_stop|websocket.?(?:closed|error))\b/i;
|
|
215
|
-
/**
|
|
216
|
-
* Provider LOAD, which is transient in a different way: the server is up and
|
|
217
|
-
* saying "not now". pi retries all of these; 53f0488 did not, but its own message
|
|
218
|
-
* names only "context overflow, bad request, auth" as the fail-fast set — a
|
|
219
|
-
* throttle was never argued for, it just rode along in a list written for a LOCAL
|
|
220
|
-
* server, where none of these can occur.
|
|
221
|
-
*
|
|
222
|
-
* Words carry no trailing \b (`overloaded_error` joins on `_`, which is a word
|
|
223
|
-
* character); the bare status codes carry one, or `500` matches inside `15000`.
|
|
224
|
-
*/
|
|
225
|
-
const PROVIDER_LOAD_RE = /(?:overloaded|rate.?limit|too many requests|service.?unavailable|server.?error|internal.?error|provider.?returned.?error)|\b(?:429|500|502|503|504|524)\b/i;
|
|
226
|
-
/**
|
|
227
|
-
* Account facts, not liveness: a budget does not refill on a retry. Checked FIRST,
|
|
228
|
-
* because these arrive worded as a throttle — `429 GoUsageLimitError` is a
|
|
229
|
-
* subscription limit, not a queue.
|
|
230
|
-
*/
|
|
231
|
-
const NON_RETRYABLE_RE = /\b(?:insufficient_quota|quota exceeded|out of budget|billing|usage limit reached|available balance|GoUsageLimitError|FreeUsageLimitError)\b/i;
|
|
232
|
-
/**
|
|
233
|
-
* Retry budget is three attempts at 500ms/1s/2s — three requests over 3.5s, which
|
|
234
|
-
* is not a storm even against a throttle. pi's own ladder is three at 2s/4s/8s.
|
|
235
|
-
*/
|
|
236
|
-
export function isConnectionError(cause) {
|
|
237
|
-
if (NON_RETRYABLE_RE.test(cause))
|
|
238
|
-
return false;
|
|
239
|
-
return CONNECTION_ERROR_RE.test(cause) || PROVIDER_LOAD_RE.test(cause);
|
|
240
|
-
}
|
|
241
|
-
/** Exponential backoff before a connection-error retry: 500ms, 1s, 2s, …, so a
|
|
242
|
-
* brief saturation window can drain before we re-issue the request. */
|
|
243
|
-
export function connectionRetryBackoffMs(attempt) {
|
|
244
|
-
return 500 * 2 ** attempt;
|
|
245
|
-
}
|
|
246
|
-
const defaultSleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
247
|
-
// ─── Spawn helpers ───────────────────────────────────────────────────────────
|
|
248
|
-
export function childArgs(tools, extensions = [],
|
|
249
|
-
/**
|
|
250
|
-
* This child's group fragment: `--model` then `--thinking`, either half
|
|
251
|
-
* possibly absent. Resolved by the CALLER, never here — both are properties
|
|
252
|
-
* of the child's ROLE, and this function is handed tools and extensions, not
|
|
253
|
-
* a name. Omitted ⇒ byte-identical argv to the version before group profiles.
|
|
254
|
-
*
|
|
255
|
-
* ONE field rather than a `model` beside a `thinking`, because nothing may
|
|
256
|
-
* compose the two halves by hand: `groupChildArgs` is the only producer, so a
|
|
257
|
-
* doubled `--thinking` is unreachable rather than merely unlikely.
|
|
258
|
-
*/
|
|
259
|
-
groupArgs = []) {
|
|
260
|
-
// `--mode json` puts the child into the structured event stream the
|
|
261
|
-
// unified runner parses in `mode: 'json-events'`. Without it the child
|
|
262
|
-
// emits plain text, every line fails JSON.parse, finalText stays empty,
|
|
263
|
-
// and every phase fails with "X child produced no output". A refactor has
|
|
264
|
-
// dropped it once already; do not remove it again.
|
|
265
|
-
//
|
|
266
|
-
// An empty `tools` string means "no tools at all" — emit `--no-tools`
|
|
267
|
-
// instead of `--tools ''` (which pi would reject). Used by pure-judgment
|
|
268
|
-
// phases like critique-triage that should reason only over the text we
|
|
269
|
-
// hand them, never spend time reading the repo.
|
|
270
|
-
//
|
|
271
|
-
// The prompt is NOT an argv element: it goes to the child over stdin (see
|
|
272
|
-
// runChild below / getPiInvocation), so a large inlined-design prompt cannot
|
|
273
|
-
// exceed the OS argv ceiling — which fails the spawn outright rather than
|
|
274
|
-
// truncating (`E2BIG` on this platform).
|
|
275
|
-
//
|
|
276
|
-
// `extensions` are internal `-e` loads for in-run guards (the caller supplies
|
|
277
|
-
// the path). A no-tools child cannot make a tool call, so it never carries
|
|
278
|
-
// one — the guards all hang off pi's `tool_call` hook.
|
|
279
|
-
const toolFlags = tools === '' ? ['--no-tools'] : ['--tools', tools];
|
|
280
|
-
const internal = tools === '' ? [] : extensions;
|
|
281
|
-
return [...childBaseArgs(internal), ...groupArgs, '--mode', 'json', ...toolFlags];
|
|
282
|
-
}
|
|
283
|
-
// Sentinel error thrown when the user dismisses a grill-me dialog.
|
|
284
|
-
// Defined here (not in failure-classifier.ts) to avoid circular dependency.
|
|
285
|
-
export const USER_CANCELLED = '__user_cancelled__';
|
|
286
|
-
/**
|
|
287
|
-
* The `phase` row of WORKER_PROFILES, resolved with this machine's config.
|
|
288
|
-
*
|
|
289
|
-
* Read here rather than at module load so a /task-config change reaches the next
|
|
290
|
-
* child, the same contract childBaseArgs already keeps. Both spawn paths in this
|
|
291
|
-
* file go through it, so the degraded final attempt cannot drift from the ordinary
|
|
292
|
-
* one — the mislabel class runDegradedFinalAttempt's own comment warns about.
|
|
293
|
-
*/
|
|
294
|
-
export function phasePolicy() {
|
|
295
|
-
return workerPolicy('phase', {
|
|
296
|
-
commandTimeoutMs: getConfig().requestTimeoutMs,
|
|
297
|
-
streamInactivityMs: getConfig().streamInactivityMs
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
|
-
export async function runChild({ cwd, tools, prompt, signal, onLine, onContextUsage, onToolCall, spawn: spawnFn, extensions, onToolResult, contextWindow, groupArgs, commandCeilingMs }) {
|
|
301
|
-
const invocation = getPiInvocation(childArgs(tools, extensions, groupArgs), prompt);
|
|
302
|
-
let loopHit;
|
|
303
|
-
const guards = phasePolicy().guards;
|
|
304
|
-
// Null when the user set the ceiling to `off`. Why a phase child needs this at
|
|
305
|
-
// all is the `phase` row's `why` in worker-profiles.ts.
|
|
306
|
-
const cmdWatch = commandWatch(commandCeilingMs ?? guards['command-timeout']);
|
|
307
|
-
const childSignal = cmdWatch ? AbortSignal.any([signal, cmdWatch.signal]) : signal;
|
|
308
|
-
let result;
|
|
309
|
-
try {
|
|
310
|
-
result = await runChildUnified(spawnFn ?? spawn, invocation, cwd, childSignal, {
|
|
311
|
-
mode: 'json-events',
|
|
312
|
-
// A hung model stream reports nothing at all, so without this the
|
|
313
|
-
// phase child waits forever. The kill
|
|
314
|
-
// is reported below as a connection-class cause, which routes it into
|
|
315
|
-
// the retry/backoff path this file already has for a LOUD disconnect.
|
|
316
|
-
streamInactivityMs: guards['stream-stall'],
|
|
317
|
-
...(guards.stalled === false ?
|
|
318
|
-
{}
|
|
319
|
-
: {
|
|
320
|
-
stall: {
|
|
321
|
-
afterMs: guards.stalled.afterMs,
|
|
322
|
-
probe: guards.stalled.probe
|
|
323
|
-
?? (() => probeModelEndpoints(childModelEndpoints(modelSpecFromArgs(groupArgs ?? []))))
|
|
324
|
-
}
|
|
325
|
-
}),
|
|
326
|
-
onLine,
|
|
327
|
-
onContextUsage,
|
|
328
|
-
...(contextWindow && contextWindow > 0 ? { contextWindow } : {}),
|
|
329
|
-
// ALWAYS wired, never conditional on the caller wanting results:
|
|
330
|
-
// the sink emits a tool-execution-end only when a handler exists,
|
|
331
|
-
// and without that end the command watchdog's timer is never
|
|
332
|
-
// disarmed — every healthy tool call would then look hung.
|
|
333
|
-
onToolResult: r => {
|
|
334
|
-
cmdWatch?.onEnd(r.toolCallId);
|
|
335
|
-
onToolResult?.(r.text, r.isError);
|
|
336
|
-
},
|
|
337
|
-
onToolCall: call => {
|
|
338
|
-
// Before the detectors: a call they let through still needs its
|
|
339
|
-
// clock started.
|
|
340
|
-
cmdWatch?.onStart(call);
|
|
341
|
-
if (!onToolCall)
|
|
342
|
-
return null;
|
|
343
|
-
const hit = onToolCall(call);
|
|
344
|
-
if (hit && !loopHit) {
|
|
345
|
-
loopHit = hit;
|
|
346
|
-
}
|
|
347
|
-
return hit; // propagate to unified runner so it can kill
|
|
348
|
-
}
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
finally {
|
|
352
|
-
cmdWatch?.clear();
|
|
353
|
-
}
|
|
354
|
-
const commandKill = cmdWatch?.killed();
|
|
355
|
-
// Use `||` (not `??`) so an empty string from json-events mode falls
|
|
356
|
-
// back to raw stdout. Without this, a child that exits 0 but emits no
|
|
357
|
-
// assistant text (e.g. model API error swallowed in json mode) always
|
|
358
|
-
// fails with the unhelpful "X child produced no output" — the raw
|
|
359
|
-
// stdout/stderr that might contain the real error is discarded.
|
|
360
|
-
const text = result.text || result.stdout.trim();
|
|
361
|
-
// The stream watchdog's kill leaves no provider error to report (that is the
|
|
362
|
-
// whole failure mode), so name it here rather than letting it surface as the
|
|
363
|
-
// meaningless "produced no output". Never overwrite a real reported cause.
|
|
364
|
-
const modelError = result.modelError
|
|
365
|
-
?? (result.streamStalled ? streamStallCause(result.streamStalled.idleMs) : undefined);
|
|
366
|
-
return {
|
|
367
|
-
text,
|
|
368
|
-
// WE killed this child, so its exit status is our own SIGTERM. Report 0
|
|
369
|
-
// and let the named cause carry it. EVERY guard kill must be listed: one
|
|
370
|
-
// omitted here arrives as exit 0 with partial text, and triageChildResult
|
|
371
|
-
// returns it as a successful answer.
|
|
372
|
-
exitCode: result.streamStalled || result.stalled || commandKill ? 0 : result.exitCode,
|
|
373
|
-
stderr: result.stderr.trim(),
|
|
374
|
-
loopHit,
|
|
375
|
-
modelError,
|
|
376
|
-
...(commandKill ? { commandKill } : {}),
|
|
377
|
-
...(result.stalled ? { stalled: true } : {}),
|
|
378
|
-
// A tool call the model wrote as text (wrong dialect) never executed and
|
|
379
|
-
// sailed past the structured-event guards above; flag it so the wrappers
|
|
380
|
-
// can re-prompt instead of accepting the unexecuted call. Only meaningful
|
|
381
|
-
// when the run otherwise succeeded — a loop kill truncates text mid-stream.
|
|
382
|
-
leakedToolCall: loopHit ? undefined : (detectLeakedToolCall(text) ?? undefined)
|
|
383
|
-
};
|
|
384
|
-
}
|
|
385
|
-
/**
|
|
386
|
-
* The error-triage ladder both phase wrappers run over a finished child, in
|
|
387
|
-
* this fixed order: non-zero exit → model error → empty completion → leaked
|
|
388
|
-
* tool call. Callers own the loop, the prompt and the hint; this owns the
|
|
389
|
-
* verdict, so a fix to any rung lands in every caller at once.
|
|
390
|
-
*
|
|
391
|
-
* `attempt` is the caller's 0-based counter, `budget` the matching restart
|
|
392
|
-
* allowance — MAX_LEAK_RETRIES and MAX_LOOP_RESTARTS are both 2, and one loop
|
|
393
|
-
* spends the pair — so a phase runs `budget + 1` attempts before a rung gives up.
|
|
394
|
-
*
|
|
395
|
-
* `verb` names the restart in the debug log ("retry" by default, "restart" for
|
|
396
|
-
* refine and grill-gen). It is the only externally visible thing that differed
|
|
397
|
-
* between the two loops this collapsed, and the only way to tell from a debug
|
|
398
|
-
* log which phase produced a line — so it is passed in rather than hardcoded.
|
|
399
|
-
*
|
|
400
|
-
* A loop kill (`r.loopHit`) is NOT handled here: the caller detects loops and
|
|
401
|
-
* must consume the hit before calling this.
|
|
402
|
-
*/
|
|
403
|
-
async function triageChildResult(deps, name, r, attempt, budget, verb) {
|
|
404
|
-
if (r.exitCode !== 0) {
|
|
405
|
-
// The exit code is the only signal when stderr is empty, and pi exits
|
|
406
|
-
// silently on several paths (143 = SIGTERM/loop-kill, 137 = SIGKILL/OOM).
|
|
407
|
-
// Dropping it left "(no stderr)" as the whole diagnosis.
|
|
408
|
-
const why = r.stderr || `no stderr, exit ${r.exitCode}`;
|
|
409
|
-
throw new Error(`${name} child failed: ${why}`);
|
|
410
|
-
}
|
|
411
|
-
if (r.modelError) {
|
|
412
|
-
// The model/provider failed (pi exited 0 with a stopReason "error"
|
|
413
|
-
// turn). A connection-class cause is transient — re-spawn within the
|
|
414
|
-
// caller's budget after a backoff; anything else fails fast (pi already
|
|
415
|
-
// retried, and re-spawning won't fix a real fault).
|
|
416
|
-
if (isConnectionError(r.modelError) && attempt < budget) {
|
|
417
|
-
deps.logDebug?.(`${name}: connection error "${r.modelError}" — ${verb} `
|
|
418
|
-
+ `${attempt + 1}/${budget}`);
|
|
419
|
-
await (deps.sleepFor ?? defaultSleep)(connectionRetryBackoffMs(attempt));
|
|
420
|
-
return { done: false };
|
|
421
|
-
}
|
|
422
|
-
throw new ModelError(name, r.modelError);
|
|
423
|
-
}
|
|
424
|
-
if (r.text.trim().length === 0) {
|
|
425
|
-
// An empty completion (exit 0, no assistant text, no stderr) is almost
|
|
426
|
-
// always transient — a model/API error swallowed inside --mode json,
|
|
427
|
-
// not a repeatable mistake — so re-spawn rather than fail the phase.
|
|
428
|
-
// There's nothing to correct, so we carry no hint (and leave any hint
|
|
429
|
-
// the caller already has alone). Shares the caller's budget: budget+1
|
|
430
|
-
// attempts, then surface the error.
|
|
431
|
-
if (attempt === budget) {
|
|
432
|
-
throw new Error(`${name} child produced no output${r.stderr ? ' — stderr: ' + r.stderr : ''}`);
|
|
433
|
-
}
|
|
434
|
-
return { done: false };
|
|
435
|
-
}
|
|
436
|
-
if (r.leakedToolCall) {
|
|
437
|
-
if (attempt === budget) {
|
|
438
|
-
throw new LeakedToolCallError(name, r.leakedToolCall);
|
|
439
|
-
}
|
|
440
|
-
return { done: false, hint: leakedToolCallHint(r.leakedToolCall) };
|
|
441
|
-
}
|
|
442
|
-
return { done: true, text: r.text };
|
|
443
|
-
}
|
|
444
|
-
/**
|
|
445
|
-
* Run a child pi and return its assistant text. Throws if exit code != 0.
|
|
446
|
-
*
|
|
447
|
-
* If the child leaks a tool call as plain text (wrong dialect — never executed),
|
|
448
|
-
* re-prompt with a correction hint up to MAX_LEAK_RETRIES times; if it keeps
|
|
449
|
-
* leaking, throw LeakedToolCallError rather than returning the unexecuted call.
|
|
450
|
-
* Empty completions and connection-class model errors share that same budget —
|
|
451
|
-
* see triageChildResult, which decides every one of those cases.
|
|
452
|
-
*
|
|
453
|
-
* THREE RUNAWAY GUARDS ride the same budget, because this is the runner every
|
|
454
|
-
* /task-auto planning child goes through (clarify, decompose, coverage,
|
|
455
|
-
* contract-extract), and an unguarded planning child can burn a whole run:
|
|
456
|
-
* • a LoopDetector, so an identical repeated tool call is killed and
|
|
457
|
-
* re-prompted instead of being allowed to fill the context window;
|
|
458
|
-
* • a StallDetector, the backstop for the varied-args thrash the loop
|
|
459
|
-
* detector's short window cannot see — a child that keeps calling tools with
|
|
460
|
-
* different arguments, learns nothing, and is never going to return. It bounds
|
|
461
|
-
* consecutive no-new-ground calls and total context churn, NOT elapsed time;
|
|
462
|
-
* • PHASE_CHILD_TIMEOUT_MS, a hard wall clock, OFF by default: a healthy
|
|
463
|
-
* reasoning-on planning child and a runaway one occupy the same range of
|
|
464
|
-
* elapsed times, so no threshold separates them. See its comment.
|
|
465
|
-
* All three are checked BEFORE the triage ladder: we killed the child, so its
|
|
466
|
-
* exit status describes our SIGTERM and says nothing about its verdict.
|
|
467
|
-
*/
|
|
89
|
+
// ─── The adapter ─────────────────────────────────────────────────────────────
|
|
468
90
|
/**
|
|
469
91
|
* The group fragment for a named child, or `[]` when the name is unmapped.
|
|
470
92
|
*
|
|
@@ -478,15 +100,6 @@ export function groupArgsForChild(name) {
|
|
|
478
100
|
const group = groupForChild(name);
|
|
479
101
|
return group ? groupChildArgs(group) : [];
|
|
480
102
|
}
|
|
481
|
-
/**
|
|
482
|
-
* What a PHASE child's invocation carries, said once.
|
|
483
|
-
*
|
|
484
|
-
* Both callers of `runChild` in this file are phase children — the strike
|
|
485
|
-
* attempts and the no-tools degrade that rescues them — and everything they
|
|
486
|
-
* disagree about is in `over`. Anything not there is the same by construction,
|
|
487
|
-
* which is what the degrade's own comment ("the degrade changes the TOOLS, not
|
|
488
|
-
* the role") claimed while three bare `undefined`s quietly made it false.
|
|
489
|
-
*/
|
|
490
103
|
/**
|
|
491
104
|
* The context window a child of this group runs against.
|
|
492
105
|
*
|
|
@@ -497,175 +110,119 @@ export function groupArgsForChild(name) {
|
|
|
497
110
|
function childContextWindow(deps, group) {
|
|
498
111
|
return (group === undefined ? undefined : groupWindow(group)) ?? deps.contextWindow;
|
|
499
112
|
}
|
|
500
|
-
function phaseChildRun(deps,
|
|
501
|
-
/** This child's group, for the window. Undefined ⇒ the run's own window. */
|
|
502
|
-
group, over) {
|
|
503
|
-
return {
|
|
504
|
-
cwd: deps.cwd,
|
|
505
|
-
onLine: deps.onChildOutput,
|
|
506
|
-
onContextUsage: deps.onContextUsage,
|
|
507
|
-
spawn: deps.spawn,
|
|
508
|
-
extensions: deps.childExtensions,
|
|
509
|
-
// The GROUP's window when this session resolved one, else the run's.
|
|
510
|
-
// Too small a window makes the churn rule fire early and kill a healthy
|
|
511
|
-
// child, so an unresolved group keeps the parent's number.
|
|
512
|
-
contextWindow: (group === undefined ? undefined : groupWindow(group)) ?? deps.contextWindow,
|
|
513
|
-
...over
|
|
514
|
-
};
|
|
515
|
-
}
|
|
516
113
|
export async function runPhaseChild(deps, name, tools, prompt, opts = {}) {
|
|
517
114
|
if (deps.runChild)
|
|
518
115
|
return await deps.runChild(name, tools, prompt);
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
// different experiments, and the retry exists to repeat the first one with a
|
|
522
|
-
// hint. An unmapped name inherits, which is today's argv — the build-time
|
|
523
|
-
// guard for that is config/groups.test.ts, not a throw in a user's run.
|
|
524
|
-
const groupArgs = groupArgsForChild(name);
|
|
525
|
-
const group = groupForChild(name);
|
|
116
|
+
if (deps.signal.aborted)
|
|
117
|
+
throw new Error(USER_CANCELLED);
|
|
526
118
|
const verb = opts.verb ?? 'retry';
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
:
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
//
|
|
549
|
-
//
|
|
550
|
-
// and the
|
|
551
|
-
|
|
552
|
-
//
|
|
553
|
-
//
|
|
554
|
-
//
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
let r;
|
|
561
|
-
try {
|
|
562
|
-
r = await runChild(phaseChildRun(deps, group, {
|
|
563
|
-
tools,
|
|
564
|
-
prompt: prependHint(hint, prompt),
|
|
565
|
-
signal: clock.signal,
|
|
566
|
-
groupArgs,
|
|
567
|
-
onContextUsage: snapshot => {
|
|
568
|
-
// Real window or nothing: `noteContext` ignores 0, which is
|
|
569
|
-
// why the parent's value must be supplied at spawn — a
|
|
570
|
-
// stream that only ever reports 0 leaves the churn rule
|
|
571
|
-
// permanently disarmed.
|
|
572
|
-
stall?.noteContext(snapshot.contextWindow);
|
|
573
|
-
deps.onContextUsage?.(snapshot);
|
|
574
|
-
},
|
|
575
|
-
commandCeilingMs: commandCeilingForAttempt(phasePolicy().guards['command-timeout'], hangKills),
|
|
576
|
-
onToolCall: call => detector?.record(call) ?? stall?.record(call) ?? null,
|
|
577
|
-
onToolResult: (text, isError) => stall?.noteResult(text, isError)
|
|
578
|
-
}));
|
|
579
|
-
}
|
|
580
|
-
finally {
|
|
581
|
-
clock.cleanup();
|
|
582
|
-
}
|
|
583
|
-
// A user cancel must not be mistaken for any of the guards.
|
|
584
|
-
if (deps.signal.aborted)
|
|
585
|
-
throw new Error(USER_CANCELLED);
|
|
586
|
-
if (r.loopHit) {
|
|
587
|
-
const isLastStrike = attempt === MAX_LEAK_RETRIES;
|
|
588
|
-
loopHistory.push(r.loopHit);
|
|
589
|
-
await appendLoopEvent(deps.cwd, deps.taskId, name, r.loopHit, attempt + 1, isLastStrike ?
|
|
590
|
-
opts.degradeOnExhaustion ?
|
|
591
|
-
'degraded — no-tools final attempt'
|
|
592
|
-
: 'phase failed'
|
|
593
|
-
: 'restarted with hint');
|
|
594
|
-
if (isLastStrike) {
|
|
595
|
-
if (opts.degradeOnExhaustion) {
|
|
596
|
-
return await runDegradedFinalAttempt(deps, name, prompt, r.loopHit, loopHistory);
|
|
597
|
-
}
|
|
598
|
-
throw new LoopExhaustedError(name, loopHistory);
|
|
599
|
-
}
|
|
600
|
-
deps.logDebug?.(r.loopHit.stall ?
|
|
601
|
-
`${name}: stalled (${r.loopHit.stall}) on ${r.loopHit.call.name} — `
|
|
602
|
-
+ `${verb} ${attempt + 1}/${MAX_LEAK_RETRIES}`
|
|
603
|
-
: `${name}: looped on ${r.loopHit.call.name} — ${verb} ${attempt + 1}/${MAX_LEAK_RETRIES}`);
|
|
604
|
-
hint = r.loopHit.stall ? formatStallHint(r.loopHit.stall) : formatLoopHint(r.loopHit);
|
|
605
|
-
continue;
|
|
606
|
-
}
|
|
607
|
-
// Ordered after the loop rule and before the clock, matching RESTART_ORDER
|
|
608
|
-
// (worker-kill.ts): the loop hint names the offending call and is the more
|
|
609
|
-
// specific thing to tell a re-spawn, and a watchdog kill leaves the phase
|
|
610
|
-
// clock's own flag false, so the two cannot be confused.
|
|
611
|
-
if (r.commandKill) {
|
|
612
|
-
hangKills++;
|
|
613
|
-
if (attempt === MAX_LEAK_RETRIES) {
|
|
614
|
-
throw new CommandTimeoutError(name, r.commandKill);
|
|
615
|
-
}
|
|
616
|
-
deps.logDebug?.(`${name}: ${r.commandKill.toolName} outran `
|
|
617
|
-
+ `${Math.round(r.commandKill.timeoutMs / 1000)}s — `
|
|
618
|
-
+ `${verb} ${attempt + 1}/${MAX_LEAK_RETRIES}`);
|
|
619
|
-
// Tracks the TOOLS, not the phase: verify-tooling holds `read,bash`,
|
|
620
|
-
// and a half-written node_modules survives the kill.
|
|
621
|
-
hint = commandTimeoutHint(r.commandKill.toolName, r.commandKill.timeoutMs, {
|
|
622
|
-
...(r.commandKill.detail ? { commandDetail: r.commandKill.detail } : {}),
|
|
623
|
-
editsMayPersist: /\b(?:bash|edit|write)\b/.test(tools)
|
|
624
|
-
});
|
|
625
|
-
continue;
|
|
626
|
-
}
|
|
627
|
-
// A dead-backend kill spends a strike like the others; guardKillError says
|
|
628
|
-
// when the verdict has been earned.
|
|
629
|
-
const killed = guardKillError(name, r, { finalAttempt: attempt === MAX_LEAK_RETRIES });
|
|
630
|
-
if (killed)
|
|
631
|
-
throw killed;
|
|
632
|
-
if (r.stalled) {
|
|
633
|
-
deps.logDebug?.(`${name}: no output for the stall window and no endpoint answered — `
|
|
634
|
-
+ `${verb} ${attempt + 1}/${MAX_LEAK_RETRIES}`);
|
|
635
|
-
continue;
|
|
636
|
-
}
|
|
637
|
-
if (clock.timedOut()) {
|
|
638
|
-
if (attempt === MAX_LEAK_RETRIES) {
|
|
639
|
-
throw new PhaseTimeoutError(name, budgetMs, MAX_LEAK_RETRIES + 1);
|
|
640
|
-
}
|
|
641
|
-
deps.logDebug?.(`${name}: exceeded its ${Math.round(budgetMs / 1000)}s budget — `
|
|
642
|
-
+ `${verb} ${attempt + 1}/${MAX_LEAK_RETRIES}`);
|
|
643
|
-
hint = PHASE_TIMEOUT_HINT;
|
|
644
|
-
continue;
|
|
119
|
+
const cfg = getConfig();
|
|
120
|
+
// Leak retries draw on their own budget, so `r.attempt` (one counter across
|
|
121
|
+
// every reason) would print 3/2 on a leak after two loop kills.
|
|
122
|
+
const spent = { leak: 0, shared: 0 };
|
|
123
|
+
const result = await runWorker({
|
|
124
|
+
prompt,
|
|
125
|
+
cwd: deps.cwd,
|
|
126
|
+
signal: deps.signal,
|
|
127
|
+
spawn: deps.spawn,
|
|
128
|
+
tools,
|
|
129
|
+
extensions: deps.childExtensions,
|
|
130
|
+
onLine: deps.onChildOutput,
|
|
131
|
+
onContextUsage: deps.onContextUsage,
|
|
132
|
+
contextWindow: childContextWindow(deps, groupForChild(name)) ?? 'unknown',
|
|
133
|
+
profile: 'phase',
|
|
134
|
+
policyInputs: {
|
|
135
|
+
commandTimeoutMs: cfg.requestTimeoutMs,
|
|
136
|
+
streamInactivityMs: cfg.streamInactivityMs,
|
|
137
|
+
timeoutMs: deps.timeoutMs
|
|
138
|
+
},
|
|
139
|
+
sleepFor: deps.sleepFor,
|
|
140
|
+
// Resolved ONCE per call, not per attempt: a /task-config change landing
|
|
141
|
+
// between a loop-kill and its retry would otherwise make the two attempts
|
|
142
|
+
// different experiments, and the retry exists to repeat the first one.
|
|
143
|
+
groupArgs: groupArgsForChild(name),
|
|
144
|
+
// The degrade changes the TOOLS, not the role: it runs on the same model
|
|
145
|
+
// at the same level as the attempts it rescues, or it is a different
|
|
146
|
+
// experiment from the thing it stands in for.
|
|
147
|
+
rescue: opts.degradeOnExhaustion ? { tools: '', hint: formatDegradeHint } : undefined,
|
|
148
|
+
onRestart: r => {
|
|
149
|
+
deps.logDebug?.(r.rescue ?
|
|
150
|
+
`${name}: loop budget exhausted — degrading to a no-tools final attempt`
|
|
151
|
+
: `${name}: ${describeRestart(r)} — ${verb} ${describeBudget(r, spent)}`);
|
|
645
152
|
}
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
153
|
+
});
|
|
154
|
+
await appendLoopEvents(deps.cwd, deps.taskId, name, result);
|
|
155
|
+
// A user cancel must not be mistaken for any of the guards.
|
|
156
|
+
if (deps.signal.aborted)
|
|
157
|
+
throw new Error(USER_CANCELLED);
|
|
158
|
+
const failure = phaseFailure(result);
|
|
159
|
+
if (failure)
|
|
160
|
+
throw new ChildFailureError(name, failure, result.stderr);
|
|
161
|
+
return result.text;
|
|
162
|
+
}
|
|
163
|
+
/** `n/budget` for the budget this restart spent, counted by the caller. */
|
|
164
|
+
function describeBudget(r, spent) {
|
|
165
|
+
if (r.reason === 'leaked-tool-call')
|
|
166
|
+
return `${++spent.leak}/${MAX_LEAK_RETRIES}`;
|
|
167
|
+
return `${++spent.shared}/${MAX_LOOP_RESTARTS}`;
|
|
168
|
+
}
|
|
169
|
+
/** The debug-log line for one discarded attempt. */
|
|
170
|
+
function describeRestart(r) {
|
|
171
|
+
switch (r.reason) {
|
|
172
|
+
case 'loop':
|
|
173
|
+
return r.loopHit?.stall ?
|
|
174
|
+
`stalled (${r.loopHit.stall}) on ${r.loopHit.call.name}`
|
|
175
|
+
: `looped on ${r.loopHit?.call.name ?? r.detail}`;
|
|
176
|
+
case 'command-timeout':
|
|
177
|
+
return `${r.detail} outran its ceiling`;
|
|
178
|
+
case 'stream-stall':
|
|
179
|
+
return `model stream inactivity (${r.detail})`;
|
|
180
|
+
case 'stalled':
|
|
181
|
+
return r.detail ?? 'stalled';
|
|
182
|
+
case 'worker-timeout':
|
|
183
|
+
return `exceeded its budget (${r.detail})`;
|
|
184
|
+
case 'connection-error':
|
|
185
|
+
return `connection error "${r.detail}"`;
|
|
186
|
+
case 'leaked-tool-call':
|
|
187
|
+
return 'wrote a tool call as text';
|
|
188
|
+
case 'empty-answer':
|
|
189
|
+
return 'empty completion';
|
|
651
190
|
}
|
|
652
|
-
// Unreachable: the loop returns clean text or throws on the final leak.
|
|
653
|
-
throw new LeakedToolCallError(name, '(unknown)');
|
|
654
191
|
}
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
192
|
+
/**
|
|
193
|
+
* What the pipeline is told about a finished phase child, or `undefined` for an
|
|
194
|
+
* answer.
|
|
195
|
+
*
|
|
196
|
+
* The roster's kills come first, through the one ladder every consumer uses.
|
|
197
|
+
* Then the two consumer-policy outcomes: a rescue that produced nothing is
|
|
198
|
+
* reported as the loop it stood in for — a wall-clock or watchdog kill on the
|
|
199
|
+
* rescue is still that kill, but "the no-tools attempt said nothing" is not a
|
|
200
|
+
* new fact about the child, it is the loop budget being spent.
|
|
201
|
+
*/
|
|
202
|
+
function phaseFailure(r) {
|
|
203
|
+
// Strikes the BUDGET saw. The rescue is outside it: a no-tools child cannot
|
|
204
|
+
// loop, and one that somehow did is still the budget's exhaustion, not a
|
|
205
|
+
// fourth strike.
|
|
206
|
+
const loopStrikes = r.restarts.filter(x => x.reason === 'loop').length + (r.loopHit && !r.rescued ? 1 : 0);
|
|
207
|
+
const kill = classifyWorkerFailure(r);
|
|
208
|
+
if (kill)
|
|
209
|
+
return kill.kind === 'loop' ? { ...kill, strikes: loopStrikes } : kill;
|
|
210
|
+
const answered = r.modelError === undefined && r.text.trim().length > 0;
|
|
211
|
+
if (answered)
|
|
212
|
+
return undefined;
|
|
213
|
+
if (r.rescued) {
|
|
214
|
+
const hit = r.restarts.findLast(x => x.loopHit !== undefined)?.loopHit;
|
|
215
|
+
if (hit)
|
|
216
|
+
return { kind: 'loop', hit, strikes: loopStrikes };
|
|
217
|
+
}
|
|
218
|
+
if (r.modelError !== undefined)
|
|
219
|
+
return { kind: 'model-error', cause: r.modelError };
|
|
220
|
+
return { kind: 'empty-answer' };
|
|
661
221
|
}
|
|
662
222
|
/**
|
|
663
223
|
* Terminal hint for the degrade attempt: the model has thrashed through the whole
|
|
664
224
|
* strike budget re-reading files without converging, so we strip its tools and
|
|
665
|
-
* order it to emit the deliverable NOW from what it already has.
|
|
666
|
-
* read-only analysis phases (refine) whose output is a text rewrite that never
|
|
667
|
-
* strictly required a successful read — far better to ship a best-effort spec
|
|
668
|
-
* than to hard-fail the whole /task-auto run. See countRevisits / LoopExhausted.
|
|
225
|
+
* order it to emit the deliverable NOW from what it already has.
|
|
669
226
|
*/
|
|
670
227
|
export function formatDegradeHint(hit) {
|
|
671
228
|
return (`[SYSTEM NOTE: You called ${hit.call.name}(${JSON.stringify(hit.call.args)}) `
|
|
@@ -679,77 +236,34 @@ export function prependHint(hint, prompt) {
|
|
|
679
236
|
return hint === null ? prompt : `${hint}\n\n${prompt}`;
|
|
680
237
|
}
|
|
681
238
|
/**
|
|
682
|
-
* Append one line to the task file's `loop events` section.
|
|
239
|
+
* Append one line per loop kill to the task file's `loop events` section.
|
|
683
240
|
*
|
|
684
|
-
* Best-effort by contract: it runs for EVERY phase child
|
|
685
|
-
*
|
|
686
|
-
* bare unit deps bag). A trail
|
|
241
|
+
* Best-effort by contract: it runs for EVERY phase child, and not every caller
|
|
242
|
+
* owns a task file on disk (a scripted harness, a bare unit deps bag). A trail
|
|
687
243
|
* that cannot be written must cost the phase nothing — the loop kill itself is
|
|
688
|
-
* already reported through the debug log and the thrown
|
|
244
|
+
* already reported through the debug log and the thrown error.
|
|
689
245
|
*/
|
|
690
|
-
async function
|
|
691
|
-
const
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
246
|
+
async function appendLoopEvents(cwd, taskId, phase, r) {
|
|
247
|
+
const line = (hit, strike, outcome) => `- ${new Date().toISOString()} ${phase} strike ${strike}/${MAX_LOOP_RESTARTS + 1} `
|
|
248
|
+
+ `${hit.call.name}(${JSON.stringify(hit.call.args)}) ×${hit.count} in last `
|
|
249
|
+
+ `${hit.windowSize} calls → ${outcome}`;
|
|
250
|
+
const lines = r.restarts.flatMap(x => x.loopHit ?
|
|
251
|
+
[
|
|
252
|
+
line(x.loopHit, x.attempt, x.rescue ? 'degraded — no-tools final attempt' : 'restarted with hint')
|
|
253
|
+
]
|
|
254
|
+
: []);
|
|
255
|
+
if (r.loopHit)
|
|
256
|
+
lines.push(line(r.loopHit, r.attempts, 'phase failed'));
|
|
257
|
+
if (lines.length === 0)
|
|
258
|
+
return;
|
|
695
259
|
try {
|
|
696
260
|
const existing = (await readSection(cwd, taskId, 'loop events')) ?? '';
|
|
697
|
-
|
|
698
|
-
await setTaskSection(cwd, taskId, 'loop events', next);
|
|
261
|
+
await setTaskSection(cwd, taskId, 'loop events', [existing, ...lines].filter(Boolean).join('\n'));
|
|
699
262
|
}
|
|
700
263
|
catch {
|
|
701
264
|
/* best-effort: a trail is never worth failing a phase for */
|
|
702
265
|
}
|
|
703
266
|
}
|
|
704
|
-
/**
|
|
705
|
-
* Final degrade attempt after the loop budget is spent: re-spawn the child with
|
|
706
|
-
* NO tools and a terminal hint, so a model that thrashed re-reading files is
|
|
707
|
-
* forced to emit its deliverable from what it already has. With no tools there
|
|
708
|
-
* are no tool calls, so no loop can recur — the only failure modes left are a
|
|
709
|
-
* dead turn (non-zero exit, model error) or empty output, any of which fall back
|
|
710
|
-
* to the original LoopExhaustedError so the phase still fails honestly when even
|
|
711
|
-
* the degrade produces nothing.
|
|
712
|
-
*/
|
|
713
|
-
async function runDegradedFinalAttempt(deps, name, prompt, hit, loopHistory) {
|
|
714
|
-
deps.logDebug?.(`${name}: loop budget exhausted — degrading to a no-tools final attempt`);
|
|
715
|
-
// This attempt runs under the same wall clock as the strikes that led here.
|
|
716
|
-
// Passing `deps.signal` raw instead would make the one attempt taken after a
|
|
717
|
-
// loop budget is spent the one attempt that can hang forever.
|
|
718
|
-
const clock = phaseTimeout(deps.signal, deps.timeoutMs ?? PHASE_CHILD_TIMEOUT_MS);
|
|
719
|
-
let r;
|
|
720
|
-
try {
|
|
721
|
-
r = await runChild(phaseChildRun(deps, groupForChild(name), {
|
|
722
|
-
tools: '', // --no-tools: the model cannot read/grep/list, only answer
|
|
723
|
-
prompt: prependHint(formatDegradeHint(hit), prompt),
|
|
724
|
-
signal: clock.signal,
|
|
725
|
-
// Same group as the attempts that led here. The degrade changes the
|
|
726
|
-
// TOOLS, not the role — running it on a different model, or at a
|
|
727
|
-
// different thinking level, would make the fallback a different
|
|
728
|
-
// experiment from the thing it rescues.
|
|
729
|
-
groupArgs: groupArgsForChild(name)
|
|
730
|
-
}));
|
|
731
|
-
}
|
|
732
|
-
finally {
|
|
733
|
-
clock.cleanup();
|
|
734
|
-
}
|
|
735
|
-
// BEFORE the exit-code test, not inside it: a guard kill reports exit 0, so
|
|
736
|
-
// asking afterwards would already have returned the truncated text as this
|
|
737
|
-
// phase's deliverable.
|
|
738
|
-
const killed = guardKillError(name, r);
|
|
739
|
-
if (killed)
|
|
740
|
-
throw killed;
|
|
741
|
-
if (r.exitCode !== 0 || r.modelError || r.text.trim().length === 0) {
|
|
742
|
-
// A wall-clock kill is NOT a loop. Without this check a child that outran
|
|
743
|
-
// its budget is reported as "loop budget exhausted", carrying a loop
|
|
744
|
-
// history that did not cause it — the same mislabel class the worker-kill
|
|
745
|
-
// roster exists to prevent, on the very path the clock guards.
|
|
746
|
-
if (clock.timedOut()) {
|
|
747
|
-
throw new PhaseTimeoutError(name, deps.timeoutMs ?? PHASE_CHILD_TIMEOUT_MS, 1);
|
|
748
|
-
}
|
|
749
|
-
throw new LoopExhaustedError(name, loopHistory);
|
|
750
|
-
}
|
|
751
|
-
return r.text;
|
|
752
|
-
}
|
|
753
267
|
/**
|
|
754
268
|
* Run a child up to twice; the second attempt gets `emphasized=true` to escalate
|
|
755
269
|
* the prompt. On success, return the validator's value; on two failures, throw
|
|
@@ -766,53 +280,3 @@ export async function runWithEmphasisRetry(deps, name, tools, build, validate, o
|
|
|
766
280
|
}
|
|
767
281
|
throw onFail(lastProblem);
|
|
768
282
|
}
|
|
769
|
-
// ─── LoopExhaustedError ──────────────────────────────────────────────────────
|
|
770
|
-
export class LoopExhaustedError extends Error {
|
|
771
|
-
phase;
|
|
772
|
-
history;
|
|
773
|
-
constructor(phase, history) {
|
|
774
|
-
super(`loop detected ${history.length} times in ${phase}`);
|
|
775
|
-
this.phase = phase;
|
|
776
|
-
this.history = history;
|
|
777
|
-
this.name = 'LoopExhaustedError';
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
// ─── ModelError ──────────────────────────────────────────────────────────────
|
|
781
|
-
/**
|
|
782
|
-
* Thrown when a phase child's final turn failed with stopReason "error" — the
|
|
783
|
-
* model/provider died (local model disconnect, fetch failed, socket hang up,
|
|
784
|
-
* provider 5xx) after pi exhausted its own internal retries. pi reports this as
|
|
785
|
-
* an agent_end with empty assistant text, which would otherwise surface as the
|
|
786
|
-
* misleading "produced no output"; this names the real cause instead.
|
|
787
|
-
*
|
|
788
|
-
* Fail-fast: not retried at the pi-task layer. pi already retried the retryable
|
|
789
|
-
* cases; re-spawning a fresh child against the same dead endpoint only burns
|
|
790
|
-
* time and buries the real error. Restart the model/provider, then resume.
|
|
791
|
-
*/
|
|
792
|
-
export class ModelError extends Error {
|
|
793
|
-
phase;
|
|
794
|
-
cause;
|
|
795
|
-
constructor(phase, cause) {
|
|
796
|
-
super(`${phase} child: model error — ${cause}`);
|
|
797
|
-
this.phase = phase;
|
|
798
|
-
this.cause = cause;
|
|
799
|
-
this.name = 'ModelError';
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
// ─── LeakedToolCallError ─────────────────────────────────────────────────────
|
|
803
|
-
/**
|
|
804
|
-
* Thrown when a phase child repeatedly wrote a tool call as plain text (a markup
|
|
805
|
-
* dialect pi's harness didn't parse) instead of invoking it. The call never ran,
|
|
806
|
-
* so the phase output is untrustworthy — fail loudly rather than check it off.
|
|
807
|
-
*/
|
|
808
|
-
export class LeakedToolCallError extends Error {
|
|
809
|
-
phase;
|
|
810
|
-
marker;
|
|
811
|
-
constructor(phase, marker) {
|
|
812
|
-
super(`${phase} child wrote a tool call as text instead of invoking it `
|
|
813
|
-
+ `(${marker.trim()}) — it never ran`);
|
|
814
|
-
this.phase = phase;
|
|
815
|
-
this.marker = marker;
|
|
816
|
-
this.name = 'LeakedToolCallError';
|
|
817
|
-
}
|
|
818
|
-
}
|