@mjasnikovs/pi-task 0.38.23 → 0.38.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/reasoning-args.d.ts +12 -1
- package/dist/config/reasoning-args.js +5 -2
- package/dist/config/reasoning.d.ts +47 -6
- package/dist/config/reasoning.js +84 -9
- package/dist/config/register.d.ts +50 -26
- package/dist/config/register.js +96 -80
- package/dist/shared/reasoning-capability.d.ts +2 -5
- package/dist/shared/reasoning-capability.js +31 -4
- package/dist/task/auto-orchestrator.d.ts +2 -0
- package/dist/task/auto-orchestrator.js +28 -41
- package/dist/task/child-runner.d.ts +89 -24
- package/dist/task/child-runner.js +67 -46
- package/dist/task/gate-child.js +11 -11
- package/dist/task/orchestrator.d.ts +14 -20
- package/dist/task/orchestrator.js +12 -9
- package/dist/task/phases.d.ts +0 -23
- package/dist/task/phases.js +48 -464
- package/dist/task/question-dialog.d.ts +56 -0
- package/dist/task/question-dialog.js +53 -0
- package/dist/task/research-fanout-budget.d.ts +20 -0
- package/dist/task/research-fanout-budget.js +29 -0
- package/dist/task/research-worker.d.ts +183 -0
- package/dist/task/research-worker.js +429 -0
- package/dist/workers/brave-warning.js +4 -30
- package/dist/workers/docs-core.d.ts +8 -4
- package/dist/workers/docs-core.js +30 -21
- package/dist/workers/docs-lookup.d.ts +72 -0
- package/dist/workers/docs-lookup.js +53 -0
- package/dist/workers/docs-project.d.ts +9 -0
- package/dist/workers/docs-project.js +15 -0
- package/dist/workers/pi-worker-core.d.ts +112 -109
- package/dist/workers/pi-worker-core.js +33 -48
- package/dist/workers/pi-worker-docs.js +27 -31
- package/dist/workers/pi-worker.js +6 -0
- package/dist/workers/reasoning-warning.d.ts +10 -16
- package/dist/workers/reasoning-warning.js +25 -57
- package/dist/workers/session-hint.d.ts +37 -0
- package/dist/workers/session-hint.js +82 -0
- package/dist/workers/worker-failure.d.ts +34 -0
- package/dist/workers/worker-failure.js +27 -16
- package/dist/workers/worker-kill.d.ts +84 -0
- package/dist/workers/worker-kill.js +124 -0
- package/dist/workers/worker-profiles.d.ts +314 -0
- package/dist/workers/worker-profiles.js +220 -0
- package/package.json +1 -1
- package/dist/task/reasoning-groups.d.ts +0 -36
- package/dist/task/reasoning-groups.js +0 -36
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The docs TAIL — concatenate the chunks, extract against them, verify the
|
|
3
|
+
* citation, format the answer — written once, with the corpus as a row.
|
|
4
|
+
*
|
|
5
|
+
* WHY. This sequence existed three times: the project-source arm of
|
|
6
|
+
* `pi-worker-docs`, its package arm, and `docsFocused` in `docs-core`. The fetch
|
|
7
|
+
* channel proves the shape is avoidable — `fetchFocused` is one core and
|
|
8
|
+
* `pi-worker-fetch`'s `run` is 60 lines, while the docs registration was 293 for
|
|
9
|
+
* the same job over two corpora.
|
|
10
|
+
*
|
|
11
|
+
* The copies had drifted, in the place hand-flattening always drifts: the
|
|
12
|
+
* package arm's ERROR path dropped `autoInstallPin`, which both of its sibling
|
|
13
|
+
* paths keep — so a package that was auto-installed and then failed to re-resolve
|
|
14
|
+
* lost the `versionSource`/`declaredRange` provenance the last defect in this
|
|
15
|
+
* area was about.
|
|
16
|
+
*
|
|
17
|
+
* A CORPUS is what genuinely varies: the prompt, the header the answer is
|
|
18
|
+
* introduced by, and what an abort of it is called. Everything a corpus does NOT
|
|
19
|
+
* vary — where the content comes from, the version banner, the type-only
|
|
20
|
+
* detector, the details bag — stays with its caller, because those differ in kind
|
|
21
|
+
* and not in value.
|
|
22
|
+
*/
|
|
23
|
+
import type { SpawnFn } from '../shared/child-process.js';
|
|
24
|
+
import { type FocusedAnswer, type FocusedFailure } from './focused-extractor.js';
|
|
25
|
+
/** The two corpora a docs lookup can read. A third (`page`) already has a prompt. */
|
|
26
|
+
export type DocsCorpusId = 'package' | 'project';
|
|
27
|
+
export interface DocsCorpus {
|
|
28
|
+
id: DocsCorpusId;
|
|
29
|
+
/** The extraction prompt for this corpus, over the concatenated content. */
|
|
30
|
+
buildPrompt: (query: string, content: string) => string;
|
|
31
|
+
/** The line the formatted answer is introduced by. */
|
|
32
|
+
header: string;
|
|
33
|
+
/** What an abort of this corpus's lookup is called, in the failure text. */
|
|
34
|
+
abortedMessage: string;
|
|
35
|
+
}
|
|
36
|
+
export interface DocsLookupInput {
|
|
37
|
+
corpus: DocsCorpus;
|
|
38
|
+
/** The retrieved chunks, in retrieval order. */
|
|
39
|
+
chunks: ReadonlyArray<{
|
|
40
|
+
content: string;
|
|
41
|
+
}>;
|
|
42
|
+
query: string;
|
|
43
|
+
cwd: string;
|
|
44
|
+
signal?: AbortSignal;
|
|
45
|
+
spawn?: SpawnFn;
|
|
46
|
+
/**
|
|
47
|
+
* The `extraction` group's `--thinking` fragment. Resolved by the CALLER so
|
|
48
|
+
* this module — like the extractor it wraps — never reads ambient config.
|
|
49
|
+
*/
|
|
50
|
+
thinking: readonly string[];
|
|
51
|
+
}
|
|
52
|
+
export type DocsLookup = {
|
|
53
|
+
kind: 'answer';
|
|
54
|
+
/** The formatted answer: header, answer, and the verified excerpt. */
|
|
55
|
+
body: string;
|
|
56
|
+
/** Exactly what was prompted with, and what the citation was verified against. */
|
|
57
|
+
content: string;
|
|
58
|
+
extraction: FocusedAnswer;
|
|
59
|
+
/** Undefined when there was no excerpt to check. */
|
|
60
|
+
excerptVerified?: boolean;
|
|
61
|
+
} | {
|
|
62
|
+
kind: 'failed';
|
|
63
|
+
extraction: FocusedFailure;
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Run one docs lookup over already-retrieved chunks.
|
|
67
|
+
*
|
|
68
|
+
* The citation is verified against exactly the text that was prompted with — the
|
|
69
|
+
* concatenation, not a superset. (`fetch` is the one site that verifies against a
|
|
70
|
+
* superset; see `FocusedRequest.verifyAgainst`.)
|
|
71
|
+
*/
|
|
72
|
+
export declare function docsLookup(input: DocsLookupInput): Promise<DocsLookup>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The docs TAIL — concatenate the chunks, extract against them, verify the
|
|
3
|
+
* citation, format the answer — written once, with the corpus as a row.
|
|
4
|
+
*
|
|
5
|
+
* WHY. This sequence existed three times: the project-source arm of
|
|
6
|
+
* `pi-worker-docs`, its package arm, and `docsFocused` in `docs-core`. The fetch
|
|
7
|
+
* channel proves the shape is avoidable — `fetchFocused` is one core and
|
|
8
|
+
* `pi-worker-fetch`'s `run` is 60 lines, while the docs registration was 293 for
|
|
9
|
+
* the same job over two corpora.
|
|
10
|
+
*
|
|
11
|
+
* The copies had drifted, in the place hand-flattening always drifts: the
|
|
12
|
+
* package arm's ERROR path dropped `autoInstallPin`, which both of its sibling
|
|
13
|
+
* paths keep — so a package that was auto-installed and then failed to re-resolve
|
|
14
|
+
* lost the `versionSource`/`declaredRange` provenance the last defect in this
|
|
15
|
+
* area was about.
|
|
16
|
+
*
|
|
17
|
+
* A CORPUS is what genuinely varies: the prompt, the header the answer is
|
|
18
|
+
* introduced by, and what an abort of it is called. Everything a corpus does NOT
|
|
19
|
+
* vary — where the content comes from, the version banner, the type-only
|
|
20
|
+
* detector, the details bag — stays with its caller, because those differ in kind
|
|
21
|
+
* and not in value.
|
|
22
|
+
*/
|
|
23
|
+
import { formatResultText } from '../shared/child-output.js';
|
|
24
|
+
import { runFocusedExtraction } from './focused-extractor.js';
|
|
25
|
+
/**
|
|
26
|
+
* Run one docs lookup over already-retrieved chunks.
|
|
27
|
+
*
|
|
28
|
+
* The citation is verified against exactly the text that was prompted with — the
|
|
29
|
+
* concatenation, not a superset. (`fetch` is the one site that verifies against a
|
|
30
|
+
* superset; see `FocusedRequest.verifyAgainst`.)
|
|
31
|
+
*/
|
|
32
|
+
export async function docsLookup(input) {
|
|
33
|
+
const content = input.chunks.map(c => c.content).join('\n\n');
|
|
34
|
+
const extraction = await runFocusedExtraction({
|
|
35
|
+
prompt: input.corpus.buildPrompt(input.query, content),
|
|
36
|
+
verifyAgainst: content,
|
|
37
|
+
cwd: input.cwd,
|
|
38
|
+
signal: input.signal,
|
|
39
|
+
spawn: input.spawn,
|
|
40
|
+
thinking: input.thinking,
|
|
41
|
+
abortedMessage: input.corpus.abortedMessage
|
|
42
|
+
});
|
|
43
|
+
if (!extraction.ok)
|
|
44
|
+
return { kind: 'failed', extraction };
|
|
45
|
+
const excerptVerified = extraction.excerptVerified;
|
|
46
|
+
return {
|
|
47
|
+
kind: 'answer',
|
|
48
|
+
body: formatResultText(input.corpus.header, extraction, excerptVerified),
|
|
49
|
+
content,
|
|
50
|
+
extraction,
|
|
51
|
+
excerptVerified
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { CacheHandle } from './docs-cache.js';
|
|
2
2
|
import { retrieveChunks as defaultRetrieveChunks } from './docs-retrieve.js';
|
|
3
3
|
import type { RetrievedChunk } from './docs-retrieve.js';
|
|
4
|
+
import type { DocsCorpus } from './docs-lookup.js';
|
|
4
5
|
export declare function getProjectName(cwd: string): string;
|
|
5
6
|
export declare function cwdKey(cwd: string): string;
|
|
6
7
|
/**
|
|
@@ -50,3 +51,11 @@ export declare function projectDocsRaw(cache: CacheHandle, cwd: string, query: s
|
|
|
50
51
|
/** How to enumerate the project's sources. See getProjectFiles. */
|
|
51
52
|
listFiles?: (cwd: string) => string[]): ProjectDocsRawResult;
|
|
52
53
|
export declare function buildProjectPrompt(projectName: string, query: string, content: string): string;
|
|
54
|
+
/**
|
|
55
|
+
* The PROJECT corpus row: the current repo's own indexed `.ts`/`.tsx` source.
|
|
56
|
+
*
|
|
57
|
+
* Named after the project so a reader of the answer can tell a project-source
|
|
58
|
+
* citation from a package one at a glance — they are read and cited the same way
|
|
59
|
+
* and mean very different things.
|
|
60
|
+
*/
|
|
61
|
+
export declare function projectCorpus(projectName: string): DocsCorpus;
|
|
@@ -208,3 +208,18 @@ export function buildProjectPrompt(projectName, query, content) {
|
|
|
208
208
|
content
|
|
209
209
|
});
|
|
210
210
|
}
|
|
211
|
+
/**
|
|
212
|
+
* The PROJECT corpus row: the current repo's own indexed `.ts`/`.tsx` source.
|
|
213
|
+
*
|
|
214
|
+
* Named after the project so a reader of the answer can tell a project-source
|
|
215
|
+
* citation from a package one at a glance — they are read and cited the same way
|
|
216
|
+
* and mean very different things.
|
|
217
|
+
*/
|
|
218
|
+
export function projectCorpus(projectName) {
|
|
219
|
+
return {
|
|
220
|
+
id: 'project',
|
|
221
|
+
buildPrompt: (query, content) => buildProjectPrompt(projectName, query, content),
|
|
222
|
+
header: `Per ${projectName} (project source):`,
|
|
223
|
+
abortedMessage: 'Project docs lookup aborted.'
|
|
224
|
+
};
|
|
225
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type ContextSnapshot, type LoopHit, type SpawnFn } from '../shared/child-process.js';
|
|
2
|
+
import { RESTART_ORDER } from './worker-kill.js';
|
|
3
|
+
import { type WorkerGuardOverride, type WorkerGuardPolicy, type WorkerPolicyInputs, type WorkerProfileId } from './worker-profiles.js';
|
|
2
4
|
/**
|
|
3
5
|
* Tool calls that can GROUND an APIS claim — i.e. return content a signature or
|
|
4
6
|
* command could be cited from. `pi-worker-docs` (the primary), `read` and `grep`
|
|
@@ -87,116 +89,40 @@ export interface RunWorkerInput {
|
|
|
87
89
|
*/
|
|
88
90
|
contextWindow?: number;
|
|
89
91
|
/**
|
|
90
|
-
*
|
|
91
|
-
* Pass 0 to disable the timeout entirely (run until the child exits on its
|
|
92
|
-
* own) — for a pass that must be allowed to finish however long it takes.
|
|
93
|
-
*/
|
|
94
|
-
timeoutMs?: number;
|
|
95
|
-
/**
|
|
96
|
-
* PER-TOOL-CALL wall-clock ceiling in ms — the child-side half of the command
|
|
97
|
-
* watchdog (see shared/command-watchdog.ts). Arms on each tool_execution_start
|
|
98
|
-
* and disarms on the matching end; on overrun the child is killed and, within
|
|
99
|
-
* the shared restart budget, re-spawned with commandTimeoutHint.
|
|
100
|
-
*
|
|
101
|
-
* WHY SEPARATE FROM `timeoutMs`: that one bounds the whole worker and is
|
|
102
|
-
* deliberately 0 (unbounded) for gate children, which must run to completion.
|
|
103
|
-
* Neither it nor the stall guard can catch a hung command — the stall guard
|
|
104
|
-
* treats a reachable model endpoint as proof of life, which it is, even while
|
|
105
|
-
* a `bun run dev` the model forgot to bound blocks the child forever.
|
|
92
|
+
* WHICH KIND of worker child this is — the whole guard policy, in one word.
|
|
106
93
|
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* 0 / omitted = off, so every existing caller is unchanged.
|
|
94
|
+
* REQUIRED, and required on purpose. The ten guard knobs this replaces used
|
|
95
|
+
* to sit here as independent optionals, so a caller that named none of them
|
|
96
|
+
* still got a full policy and nobody could see which one. That is how the
|
|
97
|
+
* ad-hoc `pi-worker` tool came to run the strictest wall clock of the three
|
|
98
|
+
* children without anyone deciding it should. See worker-profiles.ts.
|
|
113
99
|
*/
|
|
114
|
-
|
|
100
|
+
profile: WorkerProfileId;
|
|
115
101
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
* edit/fix pass legitimately revisits one file, so it can raise (or disable
|
|
119
|
-
* via Infinity) `pathThreshold`. Pass `false` to turn the detector OFF
|
|
120
|
-
* entirely — no tool-call pattern will ever kill the worker.
|
|
102
|
+
* The facts the profile needs that are NOT policy: the gate's two watchdog
|
|
103
|
+
* ceilings (user config) and which research worker is docs-capable.
|
|
121
104
|
*/
|
|
122
|
-
|
|
123
|
-
window?: number;
|
|
124
|
-
threshold?: number;
|
|
125
|
-
pathThreshold?: number;
|
|
126
|
-
} | false;
|
|
105
|
+
policyInputs?: WorkerPolicyInputs;
|
|
127
106
|
/**
|
|
128
|
-
* Whole
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
* to it — every key occurs once per window and the count never reaches the
|
|
133
|
-
* threshold. Measured: mx5-n 2026-08-27, worker:tooling made 550 calls over
|
|
134
|
-
* exactly 20 distinct files, ~36 reads each, and neither the exact rule nor
|
|
135
|
-
* the path rule ever tripped. It died 20 minutes later on the absolute
|
|
136
|
-
* progress ceiling, having done 25s of useful work.
|
|
137
|
-
*
|
|
138
|
-
* StallDetector judges RESULTS, which a rotating reader cannot vary. It was
|
|
139
|
-
* written for exactly this class and was wired only into phase children
|
|
140
|
-
* (task/child-runner.ts) until this option existed.
|
|
141
|
-
*
|
|
142
|
-
* Pass `false` to disable, or override the thresholds (tests, harnesses).
|
|
143
|
-
*/
|
|
144
|
-
stallGuard?: {
|
|
145
|
-
limit?: number;
|
|
146
|
-
churnFactor?: number;
|
|
147
|
-
} | false;
|
|
148
|
-
/**
|
|
149
|
-
* Dead-backend stall guard override. Default ON: no output for
|
|
150
|
-
* STALL_AFTER_MS → probe the model endpoints pi is configured with →
|
|
151
|
-
* unreachable → kill + `stalled: true`. Pass `false` to disable, or
|
|
152
|
-
* override the window/probe (tests, harnesses).
|
|
107
|
+
* Whole guard rows laid over the profile's. TESTS AND A/B HARNESSES ONLY —
|
|
108
|
+
* an override at a production call site is the hand-picked subset this
|
|
109
|
+
* design exists to stop, and `worker-profiles.test.ts` fails the build if
|
|
110
|
+
* one appears under src/ outside a test.
|
|
153
111
|
*/
|
|
154
|
-
|
|
155
|
-
afterMs?: number;
|
|
156
|
-
probe?: () => Promise<boolean>;
|
|
157
|
-
} | false;
|
|
112
|
+
override?: WorkerGuardOverride;
|
|
158
113
|
/**
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
114
|
+
* The resolved policy this run will use, reported once before the first
|
|
115
|
+
* attempt.
|
|
116
|
+
*
|
|
117
|
+
* WHY: asserting that a profile RESOLVES correctly proves nothing about
|
|
118
|
+
* whether runWorker then READS it correctly — a rewiring that turns "0 means
|
|
119
|
+
* off" into "0 means on" leaves every profile assertion green. This hook is
|
|
120
|
+
* what lets a caller's own test (gate-child.test.ts) drive the REAL call
|
|
121
|
+
* site and check the REAL policy, instead of re-typing the table.
|
|
166
122
|
*/
|
|
167
|
-
|
|
123
|
+
onPolicy?: (policy: WorkerGuardPolicy) => void;
|
|
168
124
|
/** Backoff sleep, injectable so tests don't wait out the real delays. */
|
|
169
125
|
sleepFor?: (ms: number) => Promise<void>;
|
|
170
|
-
/**
|
|
171
|
-
* SCALE arm of nexttask 5B — OFF unless set, and set only by the harness that
|
|
172
|
-
* is measuring it (src/task/research-fanout-budget.ts explains both arms).
|
|
173
|
-
* Each project-source `pi-worker-docs` call pushes this attempt's deadline out
|
|
174
|
-
* by `perLookupMs`, never past `ceilingMs` from the attempt's start: a worker
|
|
175
|
-
* that is making retrieval progress is not killed for making it, while a
|
|
176
|
-
* worker that is thrashing still hits a hard bound.
|
|
177
|
-
*/
|
|
178
|
-
fanoutTimeout?: {
|
|
179
|
-
perLookupMs: number;
|
|
180
|
-
ceilingMs: number;
|
|
181
|
-
};
|
|
182
|
-
/**
|
|
183
|
-
* Absolute backstop that turns `timeoutMs` from "total time allowed" into
|
|
184
|
-
* "time allowed WITHOUT PROGRESS". A tool call or a line of output re-arms
|
|
185
|
-
* the deadline; only a worker that goes quiet for `timeoutMs` — or exceeds
|
|
186
|
-
* this ceiling outright — is killed.
|
|
187
|
-
*
|
|
188
|
-
* This is the difference between "took too long" and "stopped working". The
|
|
189
|
-
* first is a property of the machine (a slower local model, a bigger file)
|
|
190
|
-
* and must not cost the user their answer; the second is a real fault, and
|
|
191
|
-
* one the output-stall probe already catches on its own terms.
|
|
192
|
-
*/
|
|
193
|
-
progressTimeoutCeilingMs?: number;
|
|
194
|
-
/**
|
|
195
|
-
* Carry a killed attempt's findings into the re-spawn, and never return less
|
|
196
|
-
* than the best attempt produced. OFF by default so the shipped path is
|
|
197
|
-
* unchanged while the A/B runs — see src/task/research-fanout-budget.ts.
|
|
198
|
-
*/
|
|
199
|
-
carryForward?: boolean;
|
|
200
126
|
/**
|
|
201
127
|
* Called when a carried-forward partial is INJECTED into an attempt's prompt
|
|
202
128
|
* — once per attempt that receives one. Distinct from `onRestart`, which says
|
|
@@ -233,20 +159,12 @@ export interface RunWorkerInput {
|
|
|
233
159
|
* of the `start` and `done` lines around it.
|
|
234
160
|
*/
|
|
235
161
|
onRestart?: (restart: WorkerRestart) => void;
|
|
236
|
-
/**
|
|
237
|
-
* Connection-error restart budget. Defaults to MAX_LOOP_RESTARTS, and even
|
|
238
|
-
* then the SHARED restart counter is what actually binds — a worker that
|
|
239
|
-
* already spent the budget looping does not get extra lives here. 0 turns the
|
|
240
|
-
* retry off, which is how scripts/connection-retry-ab.ts gets a baseline arm
|
|
241
|
-
* out of a build that already ships the retry.
|
|
242
|
-
*/
|
|
243
|
-
connectionRetries?: number;
|
|
244
162
|
}
|
|
245
163
|
/**
|
|
246
164
|
* Why an attempt was thrown away. One value per restart branch in runWorker, so
|
|
247
165
|
* a log line naming the reason points at exactly one piece of code.
|
|
248
166
|
*/
|
|
249
|
-
export type WorkerRestartReason =
|
|
167
|
+
export type WorkerRestartReason = (typeof RESTART_ORDER)[number];
|
|
250
168
|
/** One DISCARDED attempt: its cause and the wall clock it consumed and lost. */
|
|
251
169
|
export interface WorkerRestart {
|
|
252
170
|
/** 1-based number of the attempt being discarded (the 1st restart ends attempt 1). */
|
|
@@ -411,4 +329,89 @@ export interface RunWorkerResult {
|
|
|
411
329
|
* itself, or a caller asking for 10s would silently get 30.
|
|
412
330
|
*/
|
|
413
331
|
export declare function commandCeilingForAttempt(baseMs: number, priorHangs: number): number;
|
|
332
|
+
/**
|
|
333
|
+
* Everything the restart ladder reads about one finished attempt, plus the
|
|
334
|
+
* budgets it draws on. Assembled once per attempt so the rules below can be
|
|
335
|
+
* module-level data instead of six `if` blocks welded into `runWorker`'s closure.
|
|
336
|
+
*/
|
|
337
|
+
interface RestartState {
|
|
338
|
+
loopHit?: LoopHit;
|
|
339
|
+
commandKill?: CommandKill;
|
|
340
|
+
streamStalled?: {
|
|
341
|
+
idleMs: number;
|
|
342
|
+
};
|
|
343
|
+
timedOut: boolean;
|
|
344
|
+
modelError?: string;
|
|
345
|
+
leaked: string | null;
|
|
346
|
+
/** The cap this attempt actually died against — the SCALE arm moves it. */
|
|
347
|
+
effectiveCapMs: number;
|
|
348
|
+
/** The child's tool string, which decides whether its edits can persist. */
|
|
349
|
+
tools: string;
|
|
350
|
+
restartBudgetSpent: number;
|
|
351
|
+
connRetries: number;
|
|
352
|
+
connectionRetries: number;
|
|
353
|
+
leakRetries: number;
|
|
354
|
+
}
|
|
355
|
+
/** What a rule does to the budgets when it fires. */
|
|
356
|
+
interface RestartCounters {
|
|
357
|
+
/** Consume one of the shared loop/timeout/connection restarts. */
|
|
358
|
+
shared?: boolean;
|
|
359
|
+
/** Consume one of the leaked-tool-call retries (a separate budget). */
|
|
360
|
+
leak?: boolean;
|
|
361
|
+
/** Count a WATCHDOG kill specifically — drives the command-ceiling halving. */
|
|
362
|
+
hang?: boolean;
|
|
363
|
+
/** Count a CONNECTION restart specifically — drives the backoff schedule. */
|
|
364
|
+
connection?: boolean;
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* One restartable failure: how to spot it, what to tell the fresh child, which
|
|
368
|
+
* budget it spends, and how long to wait first.
|
|
369
|
+
*/
|
|
370
|
+
interface RestartRule {
|
|
371
|
+
reason: WorkerRestartReason;
|
|
372
|
+
/**
|
|
373
|
+
* Does this rule apply to the attempt, and is its budget unspent? Returns
|
|
374
|
+
* the restart's detail line, or null to fall through to the next rule.
|
|
375
|
+
*
|
|
376
|
+
* Detection and budget are ONE test on purpose. An out-of-budget failure must
|
|
377
|
+
* fall through to the return path, not stop the ladder — a loop kill with the
|
|
378
|
+
* shared budget spent still has to let the plain-abort return happen.
|
|
379
|
+
*/
|
|
380
|
+
detect: (s: RestartState) => {
|
|
381
|
+
detail: string;
|
|
382
|
+
} | null;
|
|
383
|
+
/**
|
|
384
|
+
* The corrective preamble prepended to the next attempt's prompt. Omitted by
|
|
385
|
+
* `connection-error` alone: nothing the model did caused a dropped socket, so
|
|
386
|
+
* there is nothing to correct — and any hint already in flight from an
|
|
387
|
+
* earlier restart must survive the retry rather than be cleared by it.
|
|
388
|
+
*/
|
|
389
|
+
hint?: (s: RestartState) => string;
|
|
390
|
+
counters: RestartCounters;
|
|
391
|
+
/** Backoff before re-spawning, in ms. Only the connection rule waits. */
|
|
392
|
+
backoffMs?: (s: RestartState) => number;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* The restart ladder, in precedence order. FIRST MATCH WINS.
|
|
396
|
+
*
|
|
397
|
+
* Read the `!loopHit` guards as "a loop kill outranks me even when it has no
|
|
398
|
+
* budget left". They are not redundant with row order: when a loop is detected
|
|
399
|
+
* but the shared budget is spent, row 1 declines, and without those guards row 2
|
|
400
|
+
* or 4 would then restart the same runaway child under a hint that does not
|
|
401
|
+
* describe why it died.
|
|
402
|
+
*
|
|
403
|
+
* The whole ritual — check the budget, set the hint, spend the counters, record
|
|
404
|
+
* and announce the discarded attempt, sleep, re-spawn — belongs to the loop in
|
|
405
|
+
* `runWorker`, so a new failure mode is one row here and cannot be added without
|
|
406
|
+
* becoming visible in `restarts`.
|
|
407
|
+
*/
|
|
408
|
+
export declare const RESTART_RULES: readonly RestartRule[];
|
|
409
|
+
/** What the command watchdog recorded when it killed an attempt. */
|
|
410
|
+
interface CommandKill {
|
|
411
|
+
toolName: string;
|
|
412
|
+
timeoutMs: number;
|
|
413
|
+
/** The command line itself, when the tool carried one — quoted into the hint
|
|
414
|
+
* so the fresh child knows which call it must not repeat unbounded. */
|
|
415
|
+
detail?: string;
|
|
416
|
+
}
|
|
414
417
|
export declare function runWorker(input: RunWorkerInput): Promise<RunWorkerResult>;
|
|
@@ -5,11 +5,13 @@ import { isGroundingRetrieval as isGrounding, workerChannel } from './worker-cha
|
|
|
5
5
|
import { childBaseArgs } from '../shared/child-extensions.js';
|
|
6
6
|
import { LoopDetector } from '../task/loop-detector.js';
|
|
7
7
|
import { StallDetector, formatStallHint } from '../task/stall-detector.js';
|
|
8
|
-
import {
|
|
8
|
+
import { MAX_LOOP_RESTARTS, formatLoopHint, isConnectionError, connectionRetryBackoffMs } from '../task/child-runner.js';
|
|
9
9
|
import { detectLeakedToolCall, leakedToolCallHint, MAX_LEAK_RETRIES } from '../shared/leaked-tool-call.js';
|
|
10
10
|
import { discoverModelEndpoints, probeModelEndpoints } from '../shared/model-endpoint.js';
|
|
11
11
|
import { streamStallHint } from '../shared/stream-watchdog.js';
|
|
12
12
|
import { classifyWorkerFailure } from './worker-failure.js';
|
|
13
|
+
import { CARRY_FORWARD_IDS } from './worker-kill.js';
|
|
14
|
+
import { applyOverride, WORKER_PROFILES } from './worker-profiles.js';
|
|
13
15
|
// `--mode json` makes pi emit structured events as they happen instead of
|
|
14
16
|
// buffering the assistant text and flushing on exit. That matters for the
|
|
15
17
|
// wait/work timing split: in text mode the first stdout chunk only arrives at
|
|
@@ -34,25 +36,9 @@ const DEFAULT_TOOLS = 'read,grep,find,ls';
|
|
|
34
36
|
// hand-kept — this was a second copy of the four tool names. Re-exported because
|
|
35
37
|
// several call sites and tests import it from here.
|
|
36
38
|
export { isGroundingRetrieval } from './worker-channels.js';
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
* that thrashes with slightly-varied calls (different grep patterns each time)
|
|
41
|
-
* slips past it and would otherwise run unbounded. This is the backstop for that
|
|
42
|
-
* case: after this long with no clean exit, abort and restart with a hint. Sized
|
|
43
|
-
* well above a healthy worker's observed runtime (~25-130s on the local backend)
|
|
44
|
-
* so it never trips a legitimately slow run.
|
|
45
|
-
*/
|
|
46
|
-
const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
|
|
47
|
-
/**
|
|
48
|
-
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
49
|
-
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
50
|
-
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
51
|
-
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
52
|
-
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
53
|
-
* alive) just gets probed and waits on.
|
|
54
|
-
*/
|
|
55
|
-
const STALL_AFTER_MS = 180_000;
|
|
39
|
+
// RESEARCH_WORKER_TIMEOUT_MS and STALL_AFTER_MS live on the profile table now
|
|
40
|
+
// (worker-profiles.ts): they are the default VALUES of two guard rows, and a
|
|
41
|
+
// default that lives apart from the table stating it is a second place to look.
|
|
56
42
|
/**
|
|
57
43
|
* Restart hint after a WHOLE-WORKER wall-clock timeout — distinct from both the
|
|
58
44
|
* loop hint and the per-command hint. This one diagnoses over-exploration, which
|
|
@@ -89,12 +75,7 @@ const CARRY_FORWARD_LIMIT = 24_000;
|
|
|
89
75
|
* first is by definition the same call repeated, the second is malformed
|
|
90
76
|
* protocol text, and replaying either would feed the failure back to itself.
|
|
91
77
|
*/
|
|
92
|
-
const CARRY_FORWARD_REASONS =
|
|
93
|
-
'worker-timeout',
|
|
94
|
-
'command-timeout',
|
|
95
|
-
'stream-stall',
|
|
96
|
-
'connection-error'
|
|
97
|
-
]);
|
|
78
|
+
const CARRY_FORWARD_REASONS = CARRY_FORWARD_IDS;
|
|
98
79
|
/**
|
|
99
80
|
* Does this partial output carry ANSWER CONTENT, or is it the model clearing its
|
|
100
81
|
* throat?
|
|
@@ -276,7 +257,7 @@ export function commandCeilingForAttempt(baseMs, priorHangs) {
|
|
|
276
257
|
* `runWorker`, so a new failure mode is one row here and cannot be added without
|
|
277
258
|
* becoming visible in `restarts`.
|
|
278
259
|
*/
|
|
279
|
-
const RESTART_RULES = [
|
|
260
|
+
export const RESTART_RULES = [
|
|
280
261
|
{
|
|
281
262
|
// A loop-kill gets the same restart-with-hint treatment every other phase
|
|
282
263
|
// already gets (runPhaseChild) — name the offending call so the
|
|
@@ -447,7 +428,14 @@ export async function runWorker(input) {
|
|
|
447
428
|
'--tools',
|
|
448
429
|
tools
|
|
449
430
|
];
|
|
450
|
-
|
|
431
|
+
// ONE resolution, before the first attempt. Every guard read below goes
|
|
432
|
+
// through `policy`, so "which knobs is this child running" has exactly one
|
|
433
|
+
// answer and it is observable (`onPolicy`) rather than inferable.
|
|
434
|
+
const policy = applyOverride(WORKER_PROFILES[input.profile].resolve(input.policyInputs ?? {}), input.override);
|
|
435
|
+
input.onPolicy?.(policy);
|
|
436
|
+
const guards = policy.guards;
|
|
437
|
+
const clock = guards['worker-timeout'];
|
|
438
|
+
const timeoutMs = clock.timeoutMs;
|
|
451
439
|
let hint = null;
|
|
452
440
|
// Loop-kill and timeout share one restart budget, mirroring
|
|
453
441
|
// runPhaseChild: a runaway worker gets re-spawned with a corrective
|
|
@@ -500,19 +488,15 @@ export async function runWorker(input) {
|
|
|
500
488
|
// loop === false turns the guard off entirely (detector is null and no
|
|
501
489
|
// tool call is ever flagged); otherwise build a detector from the override
|
|
502
490
|
// or the default research/impl thresholds.
|
|
503
|
-
const loopDetector =
|
|
491
|
+
const loopDetector = guards.loop.detector === false ?
|
|
504
492
|
null
|
|
505
|
-
: (
|
|
506
|
-
const window = input.loop?.window ?? LOOP_WINDOW;
|
|
507
|
-
const threshold = input.loop?.threshold ?? LOOP_THRESHOLD;
|
|
508
|
-
return new LoopDetector(window, threshold, input.loop?.pathThreshold ?? threshold);
|
|
509
|
-
})();
|
|
493
|
+
: new LoopDetector(guards.loop.detector.window, guards.loop.detector.threshold, guards.loop.detector.pathThreshold);
|
|
510
494
|
// Reset EACH attempt, like the loop detector: a restart discards the
|
|
511
495
|
// previous attempt's calls along with its text, so a fresh child must not
|
|
512
496
|
// inherit a dead streak it did not earn.
|
|
513
|
-
const stallDetector =
|
|
497
|
+
const stallDetector = guards.loop.progress === false ?
|
|
514
498
|
null
|
|
515
|
-
: new StallDetector(
|
|
499
|
+
: new StallDetector(guards.loop.progress.limit, guards.loop.progress.churnFactor);
|
|
516
500
|
// Arm the churn rule BEFORE the first tool call. pi's stream carries no
|
|
517
501
|
// context event (issue #16), so waiting for one leaves the rule
|
|
518
502
|
// permanently disarmed. The parent knows the window at spawn time.
|
|
@@ -526,27 +510,28 @@ export async function runWorker(input) {
|
|
|
526
510
|
// discarded with its text, so the count must describe only the attempt
|
|
527
511
|
// whose text this call returns.
|
|
528
512
|
let groundingRetrievalCount = 0;
|
|
529
|
-
const timeout = workerTimeout(input.signal, timeoutMs,
|
|
513
|
+
const timeout = workerTimeout(input.signal, timeoutMs, clock.progressCeilingMs ?? undefined);
|
|
530
514
|
// Per-tool-call watchdog for this attempt (null when off). Its abort is
|
|
531
515
|
// OR'd with the worker timeout / external cancel into the child's signal.
|
|
532
|
-
const cmdWatch = commandWatch(commandCeilingForAttempt(
|
|
516
|
+
const cmdWatch = commandWatch(commandCeilingForAttempt(guards['command-timeout'], hangKills));
|
|
533
517
|
const childSignal = cmdWatch ? AbortSignal.any([timeout.signal, cmdWatch.signal]) : timeout.signal;
|
|
534
518
|
let result;
|
|
535
519
|
try {
|
|
536
520
|
result = await runChildDefault(invocation, input.cwd, childSignal, {
|
|
537
521
|
mode: 'json-events',
|
|
538
|
-
...(
|
|
522
|
+
...(guards.stalled === false ?
|
|
539
523
|
{}
|
|
540
524
|
: {
|
|
541
525
|
stall: {
|
|
542
|
-
afterMs:
|
|
543
|
-
probe
|
|
526
|
+
afterMs: guards.stalled.afterMs,
|
|
527
|
+
// `null` in the policy means the built-in probe.
|
|
528
|
+
// Kept as data so a resolved policy stays plain
|
|
529
|
+
// comparable data — see StalledGuard.probe.
|
|
530
|
+
probe: guards.stalled.probe
|
|
544
531
|
?? (() => probeModelEndpoints(discoverModelEndpoints()))
|
|
545
532
|
}
|
|
546
533
|
}),
|
|
547
|
-
...(
|
|
548
|
-
{ streamInactivityMs: input.streamInactivityMs }
|
|
549
|
-
: {}),
|
|
534
|
+
...(guards['stream-stall'] ? { streamInactivityMs: guards['stream-stall'] } : {}),
|
|
550
535
|
onFirstByte: () => (tFirstByte = Date.now()),
|
|
551
536
|
onToolCall: call => {
|
|
552
537
|
cmdWatch?.onStart(call);
|
|
@@ -555,9 +540,9 @@ export async function runWorker(input) {
|
|
|
555
540
|
timeout.progress();
|
|
556
541
|
// The generic child runner used to name ONE tool and ONE of
|
|
557
542
|
// its parameters here. It asks the tool's own row now.
|
|
558
|
-
if (
|
|
543
|
+
if (clock.fanout
|
|
559
544
|
&& workerChannel(call.name)?.isProjectSourceLookup?.(call.args ?? {}) === true) {
|
|
560
|
-
timeout.extend(
|
|
545
|
+
timeout.extend(clock.fanout.perLookupMs, clock.fanout.ceilingMs);
|
|
561
546
|
}
|
|
562
547
|
if (isGrounding(call.name))
|
|
563
548
|
groundingRetrievalCount++;
|
|
@@ -625,7 +610,7 @@ export async function runWorker(input) {
|
|
|
625
610
|
// path cannot be added that silently drops the attempt's work.
|
|
626
611
|
// Longest-wins — a later attempt killed early should not replace a
|
|
627
612
|
// fuller answer an earlier one had already reached.
|
|
628
|
-
if (
|
|
613
|
+
if (policy.carryForward && CARRY_FORWARD_REASONS.has(reason)) {
|
|
629
614
|
const partial = text.trim();
|
|
630
615
|
// Longest-with-CONTENT wins. Length alone let a preamble sentence
|
|
631
616
|
// become the answer — see hasAnswerContent.
|
|
@@ -656,7 +641,7 @@ export async function runWorker(input) {
|
|
|
656
641
|
tools,
|
|
657
642
|
restartBudgetSpent,
|
|
658
643
|
connRetries,
|
|
659
|
-
connectionRetries:
|
|
644
|
+
connectionRetries: guards['connection-error'],
|
|
660
645
|
leakRetries
|
|
661
646
|
};
|
|
662
647
|
let restarted = false;
|