@mjasnikovs/pi-task 0.38.23 → 0.38.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/reasoning-args.d.ts +12 -1
- package/dist/config/reasoning-args.js +5 -2
- package/dist/config/reasoning.d.ts +47 -6
- package/dist/config/reasoning.js +84 -9
- package/dist/config/register.d.ts +50 -26
- package/dist/config/register.js +96 -80
- package/dist/shared/reasoning-capability.d.ts +2 -5
- package/dist/shared/reasoning-capability.js +31 -4
- package/dist/task/auto-orchestrator.d.ts +2 -0
- package/dist/task/auto-orchestrator.js +28 -41
- package/dist/task/child-runner.d.ts +89 -24
- package/dist/task/child-runner.js +67 -46
- package/dist/task/gate-child.js +11 -11
- package/dist/task/orchestrator.d.ts +14 -20
- package/dist/task/orchestrator.js +12 -9
- package/dist/task/phases.d.ts +0 -23
- package/dist/task/phases.js +48 -464
- package/dist/task/question-dialog.d.ts +56 -0
- package/dist/task/question-dialog.js +53 -0
- package/dist/task/research-fanout-budget.d.ts +20 -0
- package/dist/task/research-fanout-budget.js +29 -0
- package/dist/task/research-worker.d.ts +183 -0
- package/dist/task/research-worker.js +429 -0
- package/dist/workers/brave-warning.js +4 -30
- package/dist/workers/docs-core.d.ts +8 -4
- package/dist/workers/docs-core.js +30 -21
- package/dist/workers/docs-lookup.d.ts +72 -0
- package/dist/workers/docs-lookup.js +53 -0
- package/dist/workers/docs-project.d.ts +9 -0
- package/dist/workers/docs-project.js +15 -0
- package/dist/workers/pi-worker-core.d.ts +112 -109
- package/dist/workers/pi-worker-core.js +33 -48
- package/dist/workers/pi-worker-docs.js +27 -31
- package/dist/workers/pi-worker.js +6 -0
- package/dist/workers/reasoning-warning.d.ts +10 -16
- package/dist/workers/reasoning-warning.js +25 -57
- package/dist/workers/session-hint.d.ts +37 -0
- package/dist/workers/session-hint.js +82 -0
- package/dist/workers/worker-failure.d.ts +34 -0
- package/dist/workers/worker-failure.js +27 -16
- package/dist/workers/worker-kill.d.ts +84 -0
- package/dist/workers/worker-kill.js +124 -0
- package/dist/workers/worker-profiles.d.ts +314 -0
- package/dist/workers/worker-profiles.js +220 -0
- package/package.json +1 -1
- package/dist/task/reasoning-groups.d.ts +0 -36
- package/dist/task/reasoning-groups.js +0 -36
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ROSTER of ways a worker child can die, and what each one implies.
|
|
3
|
+
*
|
|
4
|
+
* WHY IT EXISTS. One kill cause was named in six unlinked places: a
|
|
5
|
+
* `RunWorkerInput` guard option, a `RunWorkerResult` field, the
|
|
6
|
+
* `WorkerRestartReason` union, a `RESTART_RULES` row, the `CARRY_FORWARD_REASONS`
|
|
7
|
+
* set, and a `FAILURE_RULES` row. Adding one meant six coordinated edits, and
|
|
8
|
+
* only two of them failed to compile if you skipped one. That is not
|
|
9
|
+
* hypothetical: `worker-failure.ts`'s own header records the bug it cost —
|
|
10
|
+
* *"`streamStalled` was added to the result and to `finalAttemptFailed`, but the
|
|
11
|
+
* enforce ladder never grew an arm for it, so an enforcement child killed for a
|
|
12
|
+
* hung model stream fell all the way through to `if (aborted) return
|
|
13
|
+
* USER_CANCELLED`"*. That fix closed the READER side. This closes the author
|
|
14
|
+
* side.
|
|
15
|
+
*
|
|
16
|
+
* WHAT IS AND IS NOT UNIFIED. The roster is one table. The two ORDERINGS stay
|
|
17
|
+
* two, because they genuinely disagree and each says so in its own prose: the
|
|
18
|
+
* restart ladder puts `loop` first (its hint is the most specific thing to tell a
|
|
19
|
+
* re-spawn), and the failure ladder puts `stalled` first (the diagnosis most
|
|
20
|
+
* easily lost behind the `aborted` every kill path sets). Folding two
|
|
21
|
+
* precedences into one row type would need an escape hatch per row — the same
|
|
22
|
+
* objection that got a `WriteGuard` row table rejected. What the orderings gain
|
|
23
|
+
* here is that neither can name a cause with no row, nor silently omit one.
|
|
24
|
+
*
|
|
25
|
+
* Not every cause appears in both ladders, and that asymmetry is real:
|
|
26
|
+
* `connection-error` is restartable but is reported as a `modelError`, never as a
|
|
27
|
+
* kill; `stalled`, `aborted` and `exit` end an attempt outright and no hint would
|
|
28
|
+
* help.
|
|
29
|
+
*/
|
|
30
|
+
export const WORKER_KILLS = [
|
|
31
|
+
{
|
|
32
|
+
id: 'stalled',
|
|
33
|
+
resultField: 'stalled',
|
|
34
|
+
carryForward: false,
|
|
35
|
+
restartable: false,
|
|
36
|
+
reported: true
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
id: 'command-timeout',
|
|
40
|
+
resultField: 'commandTimedOut',
|
|
41
|
+
carryForward: true,
|
|
42
|
+
restartable: true,
|
|
43
|
+
reported: true
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
id: 'stream-stall',
|
|
47
|
+
resultField: 'streamStalled',
|
|
48
|
+
carryForward: true,
|
|
49
|
+
restartable: true,
|
|
50
|
+
reported: true
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
id: 'worker-timeout',
|
|
54
|
+
resultField: 'timedOut',
|
|
55
|
+
carryForward: true,
|
|
56
|
+
restartable: true,
|
|
57
|
+
reported: true
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
id: 'connection-error',
|
|
61
|
+
resultField: null,
|
|
62
|
+
carryForward: true,
|
|
63
|
+
restartable: true,
|
|
64
|
+
reported: false
|
|
65
|
+
},
|
|
66
|
+
{ id: 'loop', resultField: 'loopHit', carryForward: false, restartable: true, reported: true },
|
|
67
|
+
{
|
|
68
|
+
id: 'leaked-tool-call',
|
|
69
|
+
resultField: 'leakedToolCall',
|
|
70
|
+
carryForward: false,
|
|
71
|
+
restartable: true,
|
|
72
|
+
reported: true
|
|
73
|
+
},
|
|
74
|
+
{ id: 'aborted', resultField: null, carryForward: false, restartable: false, reported: true },
|
|
75
|
+
{ id: 'exit', resultField: null, carryForward: false, restartable: false, reported: true }
|
|
76
|
+
];
|
|
77
|
+
/** Look one cause up. `undefined` only for an id with no row, which the suite forbids. */
|
|
78
|
+
export function workerKill(id) {
|
|
79
|
+
return WORKER_KILLS.find(k => k.id === id);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The restart ladder's precedence, as ids. `RESTART_RULES` must be exactly this,
|
|
83
|
+
* in this order.
|
|
84
|
+
*
|
|
85
|
+
* `loop` leads: its hint names the offending call, which is the most useful thing
|
|
86
|
+
* to tell a re-spawn. The two watchdogs come before the wall clock because each
|
|
87
|
+
* is the narrower diagnosis, and they cannot be confused with it — a watchdog
|
|
88
|
+
* kill leaves the worker's own timeout flag false.
|
|
89
|
+
*/
|
|
90
|
+
export const RESTART_ORDER = [
|
|
91
|
+
'loop',
|
|
92
|
+
'command-timeout',
|
|
93
|
+
'stream-stall',
|
|
94
|
+
'worker-timeout',
|
|
95
|
+
'connection-error',
|
|
96
|
+
'leaked-tool-call'
|
|
97
|
+
// `as const satisfies`, not an annotation: `WorkerRestartReason` is
|
|
98
|
+
// `(typeof RESTART_ORDER)[number]`, and a `readonly WorkerKillId[]`
|
|
99
|
+
// annotation collapses that to the whole `WorkerKillId` union — which would
|
|
100
|
+
// let `noteRestart('aborted')` compile for a cause the restart ladder has no
|
|
101
|
+
// rule for. `satisfies` keeps the membership check without the widening.
|
|
102
|
+
];
|
|
103
|
+
/**
|
|
104
|
+
* The failure ladder's precedence, as ids. `FAILURE_RULES` must be exactly this,
|
|
105
|
+
* in this order.
|
|
106
|
+
*
|
|
107
|
+
* DIFFERENT from `RESTART_ORDER`, deliberately. Every kill path also sets
|
|
108
|
+
* `aborted` and a non-zero exit, so the specific causes must all be matched
|
|
109
|
+
* before the two generic ones or a dead backend is reported as "you cancelled".
|
|
110
|
+
* `stalled` leads because it is both the most specific diagnosis and the one most
|
|
111
|
+
* easily lost.
|
|
112
|
+
*/
|
|
113
|
+
export const FAILURE_ORDER = [
|
|
114
|
+
'stalled',
|
|
115
|
+
'command-timeout',
|
|
116
|
+
'stream-stall',
|
|
117
|
+
'worker-timeout',
|
|
118
|
+
'loop',
|
|
119
|
+
'leaked-tool-call',
|
|
120
|
+
'aborted',
|
|
121
|
+
'exit'
|
|
122
|
+
];
|
|
123
|
+
/** The causes whose partial output is worth keeping. Derived, never hand-kept. */
|
|
124
|
+
export const CARRY_FORWARD_IDS = new Set(WORKER_KILLS.filter(k => k.carryForward).map(k => k.id));
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The GUARD POLICY each kind of worker child runs under, keyed on the ways it
|
|
3
|
+
* can die.
|
|
4
|
+
*
|
|
5
|
+
* WHY IT EXISTS. `RunWorkerInput` carried ten guard knobs in four different
|
|
6
|
+
* shapes — two bare millisecond numbers, three `{...} | false` unions, an
|
|
7
|
+
* optional object, a boolean and two counts — and three production callers each
|
|
8
|
+
* hand-picked a different subset of them:
|
|
9
|
+
*
|
|
10
|
+
* gate-child.ts timeoutMs 0, a per-command watchdog, a stream watchdog,
|
|
11
|
+
* and the path rule disabled. Everything else default.
|
|
12
|
+
* research-worker.ts a progress deadline and two off-by-default A/B levers.
|
|
13
|
+
* NO command watchdog, NO stream watchdog. Everything else
|
|
14
|
+
* default.
|
|
15
|
+
* pi-worker.ts nothing at all — every default, silently.
|
|
16
|
+
*
|
|
17
|
+
* So "a gate child runs unbounded but with a per-command watchdog; a research
|
|
18
|
+
* worker is the reverse" existed only as three option literals in three files,
|
|
19
|
+
* and the reasoning was attached to whichever line happened to need defending.
|
|
20
|
+
* `gate-child.ts` explained why it disables the path rule and said nothing about
|
|
21
|
+
* why it takes no progress deadline. Nothing anywhere said that the ad-hoc
|
|
22
|
+
* `pi-worker` tool is the strictest-clocked of the three. That was not a
|
|
23
|
+
* decision; it was the residue of never having had a place to write one down.
|
|
24
|
+
*
|
|
25
|
+
* WHY IT IS KEYED ON `WorkerKillId`. A guard exists to prevent a specific way a
|
|
26
|
+
* child can die, so the roster of deaths (`worker-kill.ts`) is the correct key —
|
|
27
|
+
* the same argument that roster makes for kill CAUSES, one level up. The mapped
|
|
28
|
+
* type means a tenth cause cannot be added to `WORKER_KILLS` without every
|
|
29
|
+
* profile deciding what to do about it, and it means the three causes with no
|
|
30
|
+
* dial say so in the table (`null`) instead of being absent from it.
|
|
31
|
+
*
|
|
32
|
+
* The key does NOT partition the knobs one-per-row, and pretending otherwise
|
|
33
|
+
* would be the lie:
|
|
34
|
+
*
|
|
35
|
+
* `worker-timeout` holds THREE — the cap, the progress ceiling that turns the
|
|
36
|
+
* cap from "time allowed" into "time allowed without progress", and the
|
|
37
|
+
* fan-out extension. All three move the same deadline; splitting them across
|
|
38
|
+
* rows would let a profile set a ceiling for a cap it disabled.
|
|
39
|
+
*
|
|
40
|
+
* `loop` holds TWO detectors. `StallDetector`'s hit IS a `LoopHit` with
|
|
41
|
+
* `.stall` set (child-process.ts: "so a stall rides the kill/restart plumbing
|
|
42
|
+
* the loop hit already has"), and the restart ladder has ONE rule for both.
|
|
43
|
+
* One cause, one row.
|
|
44
|
+
*
|
|
45
|
+
* WHAT IS DELIBERATELY NOT UNIFIED.
|
|
46
|
+
*
|
|
47
|
+
* `carryForward` is not a row. It is one switch over the whole run, and WHICH
|
|
48
|
+
* causes honour it is already decided by `CARRY_FORWARD_IDS`, derived from the
|
|
49
|
+
* roster. A per-cause row here would be a second copy of that set, free to
|
|
50
|
+
* disagree with it.
|
|
51
|
+
*
|
|
52
|
+
* The reasoning group is not the profile. `pi-worker.ts` runs `adhoc` guards
|
|
53
|
+
* but `groupThinkingArgs('research')`, on purpose. Guards answer "how may this
|
|
54
|
+
* child die"; `thinking` answers "how hard may it think". Folding them would
|
|
55
|
+
* silently re-level a gate child, which is the exact mistake
|
|
56
|
+
* `RunWorkerInput.thinking`'s comment already records.
|
|
57
|
+
*
|
|
58
|
+
* `projectDocsBudget()` (the CAP arm, research-fanout-budget.ts) stays out. It
|
|
59
|
+
* bounds what a worker ASKS FOR, via its prompt and its tool, not how it dies.
|
|
60
|
+
*
|
|
61
|
+
* `RESTART_ORDER` and `FAILURE_ORDER` are untouched. This is a third view of
|
|
62
|
+
* the same key, not a merge of the two orderings.
|
|
63
|
+
*/
|
|
64
|
+
import type { WorkerKillId } from './worker-kill.js';
|
|
65
|
+
/**
|
|
66
|
+
* Hard wall-clock bound on a single worker run (one spawn). The exact-match
|
|
67
|
+
* LoopDetector only catches *identical* repeated tool calls; a model that
|
|
68
|
+
* thrashes with slightly-varied calls (different grep patterns each time) slips
|
|
69
|
+
* past it and would otherwise run unbounded. This is the backstop for that case:
|
|
70
|
+
* after this long with no clean exit, abort and restart with a hint. Sized well
|
|
71
|
+
* above a healthy worker's observed runtime (~25-130s on the local backend) so
|
|
72
|
+
* it never trips a legitimately slow run.
|
|
73
|
+
*/
|
|
74
|
+
export declare const RESEARCH_WORKER_TIMEOUT_MS = 240000;
|
|
75
|
+
/**
|
|
76
|
+
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
77
|
+
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
78
|
+
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
79
|
+
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
80
|
+
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
81
|
+
* alive) just gets probed and waits on.
|
|
82
|
+
*/
|
|
83
|
+
export declare const STALL_AFTER_MS = 180000;
|
|
84
|
+
/**
|
|
85
|
+
* The dead-backend probe. No output for `afterMs` -> probe the model endpoints
|
|
86
|
+
* pi is configured with -> unreachable -> kill and set `stalled: true`. Output
|
|
87
|
+
* progress resets it, and a reachable endpoint is treated as proof of life, so
|
|
88
|
+
* this alone will not end a child that is merely quiet.
|
|
89
|
+
*/
|
|
90
|
+
export interface StalledGuard {
|
|
91
|
+
afterMs: number;
|
|
92
|
+
/**
|
|
93
|
+
* `null` means the built-in endpoint probe, and a PROFILE always writes
|
|
94
|
+
* `null`. Kept as data rather than a closure so a resolved policy is plain
|
|
95
|
+
* comparable data — which is what makes the no-behaviour-change proof in
|
|
96
|
+
* `worker-profiles.test.ts` an equality assertion rather than a hand-written
|
|
97
|
+
* comparer that skips the one field most likely to be wrong. Tests and
|
|
98
|
+
* harnesses inject a real probe through the override.
|
|
99
|
+
*/
|
|
100
|
+
probe: (() => Promise<boolean>) | null;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The whole-worker deadline. All three fields move the SAME timer, which is why
|
|
104
|
+
* they share a row: `timeoutMs` is the cap (0 = unbounded), `progressCeilingMs`
|
|
105
|
+
* turns that cap from "total time allowed" into "time allowed WITHOUT PROGRESS"
|
|
106
|
+
* up to this absolute bound, and `fanout` pushes the deadline out per
|
|
107
|
+
* project-source lookup.
|
|
108
|
+
*
|
|
109
|
+
* The progress ceiling is the difference between "took too long" and "stopped
|
|
110
|
+
* working". The first is a property of the machine — a slower local model, a
|
|
111
|
+
* bigger file — and must not cost the user their answer; the second is a real
|
|
112
|
+
* fault, and one the dead-backend probe already catches on its own terms.
|
|
113
|
+
*
|
|
114
|
+
* `fanout` is the SCALE arm of nexttask 5B and is OFF unless both its env vars
|
|
115
|
+
* are set — see task/research-fanout-budget.ts for why it was not the fix.
|
|
116
|
+
*/
|
|
117
|
+
export interface WorkerTimeoutGuard {
|
|
118
|
+
/** 0 disables the wall clock entirely: the child runs until it exits. */
|
|
119
|
+
timeoutMs: number;
|
|
120
|
+
/**
|
|
121
|
+
* A tool call or a line of output re-arms the deadline to `now + timeoutMs`,
|
|
122
|
+
* never past this many ms from the attempt's start. `null` leaves the fixed
|
|
123
|
+
* cap and makes the re-arm inert.
|
|
124
|
+
*/
|
|
125
|
+
progressCeilingMs: number | null;
|
|
126
|
+
/**
|
|
127
|
+
* Each project-source `pi-worker-docs` call pushes this attempt's deadline
|
|
128
|
+
* out by `perLookupMs`, never past `ceilingMs` from the attempt's start.
|
|
129
|
+
*/
|
|
130
|
+
fanout: {
|
|
131
|
+
perLookupMs: number;
|
|
132
|
+
ceilingMs: number;
|
|
133
|
+
} | null;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* The two runaway detectors. ONE row because they are one cause: both return a
|
|
137
|
+
* `LoopHit` and both are handled by the single `loop` restart rule.
|
|
138
|
+
*
|
|
139
|
+
* `detector` judges ARGUMENTS over a 20-call window, so a child that rotates
|
|
140
|
+
* through MORE DISTINCT CALLS THAN THE WINDOW HOLDS is invisible to it — every
|
|
141
|
+
* key occurs once per window and the count never reaches the threshold.
|
|
142
|
+
* Measured: mx5-n 2026-08-27, worker:tooling made 550 calls over exactly 20
|
|
143
|
+
* distinct files, ~36 reads each, and neither the exact rule nor the path rule
|
|
144
|
+
* ever tripped. It died 20 minutes later on the absolute progress ceiling,
|
|
145
|
+
* having done 25s of useful work.
|
|
146
|
+
*
|
|
147
|
+
* `progress` judges RESULTS, which a rotating reader cannot vary. It was written
|
|
148
|
+
* for exactly that class and was wired only into phase children until
|
|
149
|
+
* `runWorker` grew an option for it.
|
|
150
|
+
*
|
|
151
|
+
* Either can be `false` independently — a pass that legitimately revisits one
|
|
152
|
+
* file raises `pathThreshold`; a harness isolating one rule turns the other off.
|
|
153
|
+
*/
|
|
154
|
+
export interface LoopGuard {
|
|
155
|
+
detector: {
|
|
156
|
+
window: number;
|
|
157
|
+
threshold: number;
|
|
158
|
+
pathThreshold: number;
|
|
159
|
+
} | false;
|
|
160
|
+
progress: {
|
|
161
|
+
limit: number;
|
|
162
|
+
churnFactor: number;
|
|
163
|
+
} | false;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* The knob (if any) that governs each way a worker can die.
|
|
167
|
+
*
|
|
168
|
+
* The three `null` rows are not filler. They are the statement that those causes
|
|
169
|
+
* have no dial: a leaked tool call is bounded by the fixed `MAX_LEAK_RETRIES`,
|
|
170
|
+
* `aborted` is the caller's own signal, and `exit` is the child deciding to
|
|
171
|
+
* stop. No profile may tune them, and now no profile can pretend to.
|
|
172
|
+
*/
|
|
173
|
+
interface WorkerGuardShapes {
|
|
174
|
+
stalled: StalledGuard | false;
|
|
175
|
+
/**
|
|
176
|
+
* PER-TOOL-CALL ceiling, ms. 0 = off. The child-side half of the command
|
|
177
|
+
* watchdog (shared/command-watchdog.ts): arms on each `tool_execution_start`,
|
|
178
|
+
* disarms on the matching end, and on overrun kills the child and — within
|
|
179
|
+
* the shared restart budget — re-spawns it with `commandTimeoutHint`.
|
|
180
|
+
*
|
|
181
|
+
* WHY IT IS NOT THE WALL CLOCK: that one bounds the whole worker and is
|
|
182
|
+
* deliberately 0 for gate children, which must run to completion. Neither it
|
|
183
|
+
* nor the dead-backend probe can catch a hung COMMAND — the probe treats a
|
|
184
|
+
* reachable model endpoint as proof of life, which it is, even while a `bun
|
|
185
|
+
* run dev` the model forgot to bound blocks the child forever.
|
|
186
|
+
*
|
|
187
|
+
* This is the ceiling for the FIRST attempt; each HANG-caused restart halves
|
|
188
|
+
* it (`commandCeilingForAttempt` — loop-caused restarts don't count), so a
|
|
189
|
+
* model that ignores the hint cannot spend the full ceiling again every retry.
|
|
190
|
+
*/
|
|
191
|
+
'command-timeout': number;
|
|
192
|
+
/**
|
|
193
|
+
* Stream-inactivity ceiling, ms (shared/stream-watchdog.ts). 0 = off.
|
|
194
|
+
*
|
|
195
|
+
* The dead-backend probe cannot catch a HUNG stream on a HEALTHY backend —
|
|
196
|
+
* it reads a reachable endpoint as proof of life, which is exactly what run
|
|
197
|
+
* 14's three hangs looked like. This one asks nothing of the backend: no
|
|
198
|
+
* output for this long, tool executions excluded, means kill and restart the
|
|
199
|
+
* attempt with `streamStallHint`, inside the same shared restart budget.
|
|
200
|
+
*/
|
|
201
|
+
'stream-stall': number;
|
|
202
|
+
'worker-timeout': WorkerTimeoutGuard;
|
|
203
|
+
/**
|
|
204
|
+
* Connection-error restart budget. The SHARED restart counter is what
|
|
205
|
+
* actually binds — a worker that already spent the budget looping does not
|
|
206
|
+
* get extra lives here. 0 turns the retry off, which is how
|
|
207
|
+
* `scripts/connection-retry-ab.ts` gets a baseline arm out of a build that
|
|
208
|
+
* already ships the retry.
|
|
209
|
+
*/
|
|
210
|
+
'connection-error': number;
|
|
211
|
+
loop: LoopGuard;
|
|
212
|
+
'leaked-tool-call': null;
|
|
213
|
+
aborted: null;
|
|
214
|
+
exit: null;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* One row per `WorkerKillId`. Indexing the shapes BY the roster's union is the
|
|
218
|
+
* compile-time bite: drop a row and `WorkerGuardShapes[K]` stops resolving.
|
|
219
|
+
*/
|
|
220
|
+
export type WorkerGuards = {
|
|
221
|
+
[K in WorkerKillId]: WorkerGuardShapes[K];
|
|
222
|
+
};
|
|
223
|
+
export interface WorkerGuardPolicy {
|
|
224
|
+
guards: WorkerGuards;
|
|
225
|
+
/**
|
|
226
|
+
* Carry a killed attempt's findings into the re-spawn, and never return less
|
|
227
|
+
* than the best attempt produced. Which CAUSES honour it is not settable —
|
|
228
|
+
* `CARRY_FORWARD_IDS` derives that from the roster. Cross-cutting, so
|
|
229
|
+
* deliberately NOT a row; see the header.
|
|
230
|
+
*/
|
|
231
|
+
carryForward: boolean;
|
|
232
|
+
}
|
|
233
|
+
/** A partial policy. Whole rows only: no deep-partial nobody can read. */
|
|
234
|
+
export type WorkerGuardOverride = {
|
|
235
|
+
[K in WorkerKillId]?: WorkerGuardShapes[K];
|
|
236
|
+
} & {
|
|
237
|
+
carryForward?: boolean;
|
|
238
|
+
};
|
|
239
|
+
/** The shipped `detector` half of the `loop` row: the read-only research/impl guard. */
|
|
240
|
+
export declare const DEFAULT_LOOP_DETECTOR: {
|
|
241
|
+
readonly window: 20;
|
|
242
|
+
readonly threshold: 5;
|
|
243
|
+
readonly pathThreshold: 5;
|
|
244
|
+
};
|
|
245
|
+
/**
|
|
246
|
+
* The shipped `progress` half of the `loop` row.
|
|
247
|
+
*
|
|
248
|
+
* Exported for the tests that isolate ONE of the two runaway rules. The row is
|
|
249
|
+
* whole-row-overridable on purpose, so turning the argument detector off means
|
|
250
|
+
* restating the result detector; naming the default here keeps that honest
|
|
251
|
+
* instead of tempting a deep-partial that would let a test silently disable both.
|
|
252
|
+
*/
|
|
253
|
+
export declare const DEFAULT_LOOP_PROGRESS: {
|
|
254
|
+
readonly limit: 8;
|
|
255
|
+
readonly churnFactor: 2;
|
|
256
|
+
};
|
|
257
|
+
export type WorkerProfileId = 'research' | 'gate' | 'adhoc';
|
|
258
|
+
/**
|
|
259
|
+
* The facts a profile needs that are NOT policy: user config, and which of the
|
|
260
|
+
* four research workers is the docs-capable one.
|
|
261
|
+
*/
|
|
262
|
+
export interface WorkerPolicyInputs {
|
|
263
|
+
/** gate: `config.requestTimeoutMs`. */
|
|
264
|
+
commandTimeoutMs?: number;
|
|
265
|
+
/** gate: `config.streamInactivityMs`. */
|
|
266
|
+
streamInactivityMs?: number;
|
|
267
|
+
/** research: only `worker:apis` fans out, so only it can be scaled. */
|
|
268
|
+
fanoutBounded?: boolean;
|
|
269
|
+
/** research: the A/B levers' env reader. Injectable for tests. */
|
|
270
|
+
env?: (key: string) => string | undefined;
|
|
271
|
+
}
|
|
272
|
+
export interface WorkerProfile {
|
|
273
|
+
id: WorkerProfileId;
|
|
274
|
+
/** Why THIS child's guards differ. The prose no call site was carrying. */
|
|
275
|
+
why: string;
|
|
276
|
+
resolve: (inputs: WorkerPolicyInputs) => WorkerGuardPolicy;
|
|
277
|
+
}
|
|
278
|
+
export declare const WORKER_PROFILES: {
|
|
279
|
+
readonly research: {
|
|
280
|
+
readonly id: "research";
|
|
281
|
+
readonly why: string;
|
|
282
|
+
readonly resolve: (inputs: WorkerPolicyInputs) => {
|
|
283
|
+
guards: WorkerGuards;
|
|
284
|
+
carryForward: boolean;
|
|
285
|
+
};
|
|
286
|
+
};
|
|
287
|
+
readonly gate: {
|
|
288
|
+
readonly id: "gate";
|
|
289
|
+
readonly why: string;
|
|
290
|
+
readonly resolve: (inputs: WorkerPolicyInputs) => {
|
|
291
|
+
guards: WorkerGuards;
|
|
292
|
+
carryForward: false;
|
|
293
|
+
};
|
|
294
|
+
};
|
|
295
|
+
readonly adhoc: {
|
|
296
|
+
readonly id: "adhoc";
|
|
297
|
+
readonly why: string;
|
|
298
|
+
readonly resolve: () => {
|
|
299
|
+
guards: WorkerGuards;
|
|
300
|
+
carryForward: false;
|
|
301
|
+
};
|
|
302
|
+
};
|
|
303
|
+
};
|
|
304
|
+
/** Resolve one profile. The only way a caller should obtain a policy. */
|
|
305
|
+
export declare function workerPolicy(id: WorkerProfileId, inputs?: WorkerPolicyInputs): WorkerGuardPolicy;
|
|
306
|
+
/**
|
|
307
|
+
* Lay whole rows over a resolved policy.
|
|
308
|
+
*
|
|
309
|
+
* For tests and A/B harnesses ONLY. Production code names a profile: an override
|
|
310
|
+
* at a production call site is the exact "hand-pick a subset" this module exists
|
|
311
|
+
* to stop, and `worker-profiles.test.ts` fails the build if one appears.
|
|
312
|
+
*/
|
|
313
|
+
export declare function applyOverride(policy: WorkerGuardPolicy, override: WorkerGuardOverride | undefined): WorkerGuardPolicy;
|
|
314
|
+
export {};
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The GUARD POLICY each kind of worker child runs under, keyed on the ways it
|
|
3
|
+
* can die.
|
|
4
|
+
*
|
|
5
|
+
* WHY IT EXISTS. `RunWorkerInput` carried ten guard knobs in four different
|
|
6
|
+
* shapes — two bare millisecond numbers, three `{...} | false` unions, an
|
|
7
|
+
* optional object, a boolean and two counts — and three production callers each
|
|
8
|
+
* hand-picked a different subset of them:
|
|
9
|
+
*
|
|
10
|
+
* gate-child.ts timeoutMs 0, a per-command watchdog, a stream watchdog,
|
|
11
|
+
* and the path rule disabled. Everything else default.
|
|
12
|
+
* research-worker.ts a progress deadline and two off-by-default A/B levers.
|
|
13
|
+
* NO command watchdog, NO stream watchdog. Everything else
|
|
14
|
+
* default.
|
|
15
|
+
* pi-worker.ts nothing at all — every default, silently.
|
|
16
|
+
*
|
|
17
|
+
* So "a gate child runs unbounded but with a per-command watchdog; a research
|
|
18
|
+
* worker is the reverse" existed only as three option literals in three files,
|
|
19
|
+
* and the reasoning was attached to whichever line happened to need defending.
|
|
20
|
+
* `gate-child.ts` explained why it disables the path rule and said nothing about
|
|
21
|
+
* why it takes no progress deadline. Nothing anywhere said that the ad-hoc
|
|
22
|
+
* `pi-worker` tool is the strictest-clocked of the three. That was not a
|
|
23
|
+
* decision; it was the residue of never having had a place to write one down.
|
|
24
|
+
*
|
|
25
|
+
* WHY IT IS KEYED ON `WorkerKillId`. A guard exists to prevent a specific way a
|
|
26
|
+
* child can die, so the roster of deaths (`worker-kill.ts`) is the correct key —
|
|
27
|
+
* the same argument that roster makes for kill CAUSES, one level up. The mapped
|
|
28
|
+
* type means a tenth cause cannot be added to `WORKER_KILLS` without every
|
|
29
|
+
* profile deciding what to do about it, and it means the three causes with no
|
|
30
|
+
* dial say so in the table (`null`) instead of being absent from it.
|
|
31
|
+
*
|
|
32
|
+
* The key does NOT partition the knobs one-per-row, and pretending otherwise
|
|
33
|
+
* would be the lie:
|
|
34
|
+
*
|
|
35
|
+
* `worker-timeout` holds THREE — the cap, the progress ceiling that turns the
|
|
36
|
+
* cap from "time allowed" into "time allowed without progress", and the
|
|
37
|
+
* fan-out extension. All three move the same deadline; splitting them across
|
|
38
|
+
* rows would let a profile set a ceiling for a cap it disabled.
|
|
39
|
+
*
|
|
40
|
+
* `loop` holds TWO detectors. `StallDetector`'s hit IS a `LoopHit` with
|
|
41
|
+
* `.stall` set (child-process.ts: "so a stall rides the kill/restart plumbing
|
|
42
|
+
* the loop hit already has"), and the restart ladder has ONE rule for both.
|
|
43
|
+
* One cause, one row.
|
|
44
|
+
*
|
|
45
|
+
* WHAT IS DELIBERATELY NOT UNIFIED.
|
|
46
|
+
*
|
|
47
|
+
* `carryForward` is not a row. It is one switch over the whole run, and WHICH
|
|
48
|
+
* causes honour it is already decided by `CARRY_FORWARD_IDS`, derived from the
|
|
49
|
+
* roster. A per-cause row here would be a second copy of that set, free to
|
|
50
|
+
* disagree with it.
|
|
51
|
+
*
|
|
52
|
+
* The reasoning group is not the profile. `pi-worker.ts` runs `adhoc` guards
|
|
53
|
+
* but `groupThinkingArgs('research')`, on purpose. Guards answer "how may this
|
|
54
|
+
* child die"; `thinking` answers "how hard may it think". Folding them would
|
|
55
|
+
* silently re-level a gate child, which is the exact mistake
|
|
56
|
+
* `RunWorkerInput.thinking`'s comment already records.
|
|
57
|
+
*
|
|
58
|
+
* `projectDocsBudget()` (the CAP arm, research-fanout-budget.ts) stays out. It
|
|
59
|
+
* bounds what a worker ASKS FOR, via its prompt and its tool, not how it dies.
|
|
60
|
+
*
|
|
61
|
+
* `RESTART_ORDER` and `FAILURE_ORDER` are untouched. This is a third view of
|
|
62
|
+
* the same key, not a merge of the two orderings.
|
|
63
|
+
*/
|
|
64
|
+
import { LOOP_THRESHOLD, LOOP_WINDOW, MAX_LOOP_RESTARTS } from '../task/child-runner.js';
|
|
65
|
+
import { CONTEXT_CHURN_FACTOR, NO_PROGRESS_LIMIT } from '../task/stall-detector.js';
|
|
66
|
+
import { fanoutTimeoutPolicy, workerCarryForward, workerProgressCeilingMs } from '../task/research-fanout-budget.js';
|
|
67
|
+
/**
|
|
68
|
+
* Hard wall-clock bound on a single worker run (one spawn). The exact-match
|
|
69
|
+
* LoopDetector only catches *identical* repeated tool calls; a model that
|
|
70
|
+
* thrashes with slightly-varied calls (different grep patterns each time) slips
|
|
71
|
+
* past it and would otherwise run unbounded. This is the backstop for that case:
|
|
72
|
+
* after this long with no clean exit, abort and restart with a hint. Sized well
|
|
73
|
+
* above a healthy worker's observed runtime (~25-130s on the local backend) so
|
|
74
|
+
* it never trips a legitimately slow run.
|
|
75
|
+
*/
|
|
76
|
+
export const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
|
|
77
|
+
/**
|
|
78
|
+
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
79
|
+
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
80
|
+
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
81
|
+
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
82
|
+
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
83
|
+
* alive) just gets probed and waits on.
|
|
84
|
+
*/
|
|
85
|
+
export const STALL_AFTER_MS = 180_000;
|
|
86
|
+
/** The shipped `detector` half of the `loop` row: the read-only research/impl guard. */
|
|
87
|
+
export const DEFAULT_LOOP_DETECTOR = {
|
|
88
|
+
window: LOOP_WINDOW,
|
|
89
|
+
threshold: LOOP_THRESHOLD,
|
|
90
|
+
pathThreshold: LOOP_THRESHOLD
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* The shipped `progress` half of the `loop` row.
|
|
94
|
+
*
|
|
95
|
+
* Exported for the tests that isolate ONE of the two runaway rules. The row is
|
|
96
|
+
* whole-row-overridable on purpose, so turning the argument detector off means
|
|
97
|
+
* restating the result detector; naming the default here keeps that honest
|
|
98
|
+
* instead of tempting a deep-partial that would let a test silently disable both.
|
|
99
|
+
*/
|
|
100
|
+
export const DEFAULT_LOOP_PROGRESS = {
|
|
101
|
+
limit: NO_PROGRESS_LIMIT,
|
|
102
|
+
churnFactor: CONTEXT_CHURN_FACTOR
|
|
103
|
+
};
|
|
104
|
+
/**
|
|
105
|
+
* Every guard at its default. `adhoc` IS this; the other two are this plus a
|
|
106
|
+
* named departure, so a diff between two profiles is a short list rather than a
|
|
107
|
+
* re-reading of two literals.
|
|
108
|
+
*/
|
|
109
|
+
function baseGuards() {
|
|
110
|
+
return {
|
|
111
|
+
stalled: { afterMs: STALL_AFTER_MS, probe: null },
|
|
112
|
+
'command-timeout': 0,
|
|
113
|
+
'stream-stall': 0,
|
|
114
|
+
'worker-timeout': {
|
|
115
|
+
timeoutMs: RESEARCH_WORKER_TIMEOUT_MS,
|
|
116
|
+
progressCeilingMs: null,
|
|
117
|
+
fanout: null
|
|
118
|
+
},
|
|
119
|
+
'connection-error': MAX_LOOP_RESTARTS,
|
|
120
|
+
loop: { detector: { ...DEFAULT_LOOP_DETECTOR }, progress: { ...DEFAULT_LOOP_PROGRESS } },
|
|
121
|
+
'leaked-tool-call': null,
|
|
122
|
+
aborted: null,
|
|
123
|
+
exit: null
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
export const WORKER_PROFILES = {
|
|
127
|
+
research: {
|
|
128
|
+
id: 'research',
|
|
129
|
+
why: 'The four read-only survey workers. Their fault is over-EXPLORATION, not '
|
|
130
|
+
+ 'a hung command: they get no bash, so no tool call can block forever, '
|
|
131
|
+
+ 'and the command and stream watchdogs stay off. What they do hit is the '
|
|
132
|
+
+ 'clock — mx5 run 18 measured r(project lookups, wall clock) = 0.909, '
|
|
133
|
+
+ 'with every worker past 46 lookups burning all three attempts. Hence the '
|
|
134
|
+
+ 'progress deadline (nexttask 9, 42 trials/arm: worker-timeout restarts '
|
|
135
|
+
+ '22/24 -> 0/24): the 240s cap now means 240s WITHOUT PROGRESS, up to a '
|
|
136
|
+
+ '20-minute backstop. The fan-out extension and carry-forward remain OFF '
|
|
137
|
+
+ 'unless their env var is set — see research-fanout-budget.ts.',
|
|
138
|
+
resolve: inputs => {
|
|
139
|
+
const guards = baseGuards();
|
|
140
|
+
guards['worker-timeout'] = {
|
|
141
|
+
timeoutMs: RESEARCH_WORKER_TIMEOUT_MS,
|
|
142
|
+
progressCeilingMs: workerProgressCeilingMs(inputs.env),
|
|
143
|
+
fanout: inputs.fanoutBounded === true ? fanoutTimeoutPolicy(inputs.env) : null
|
|
144
|
+
};
|
|
145
|
+
return { guards, carryForward: workerCarryForward(inputs.env) };
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
gate: {
|
|
149
|
+
id: 'gate',
|
|
150
|
+
why: 'The post-implementation verify/enforce/critique children. They WRITE, '
|
|
151
|
+
+ 'and they legitimately read and edit the same file many times, so the '
|
|
152
|
+
+ 'research guards mislabel the job as a runaway and kill good work (mx5 '
|
|
153
|
+
+ 'TASK_0002). Two departures follow from that. The wall clock is OFF — '
|
|
154
|
+
+ 'these passes must be allowed to finish however long they take. And the '
|
|
155
|
+
+ 'path-revisit rule is disabled (pathThreshold Infinity), leaving only '
|
|
156
|
+
+ 'the exact-match rule, so revisiting one file never trips but a '
|
|
157
|
+
+ 'literally-identical call repeated past threshold still does. What '
|
|
158
|
+
+ 'replaces the wall clock is the pair the research workers do not need: '
|
|
159
|
+
+ 'a per-command watchdog, because a gate child HAS bash and a `bun run '
|
|
160
|
+
+ 'dev` it forgot to bound blocks it forever while the stall probe reads '
|
|
161
|
+
+ 'the live model endpoint as proof of life; and a stream watchdog, for '
|
|
162
|
+
+ "run 14's three hangs on a HEALTHY backend. Both take their ceilings "
|
|
163
|
+
+ 'from user config, so they are inputs, not policy.',
|
|
164
|
+
resolve: inputs => {
|
|
165
|
+
const guards = baseGuards();
|
|
166
|
+
guards['command-timeout'] = inputs.commandTimeoutMs ?? 0;
|
|
167
|
+
guards['stream-stall'] = inputs.streamInactivityMs ?? 0;
|
|
168
|
+
guards['worker-timeout'] = { timeoutMs: 0, progressCeilingMs: null, fanout: null };
|
|
169
|
+
guards.loop = {
|
|
170
|
+
...guards.loop,
|
|
171
|
+
detector: { ...DEFAULT_LOOP_DETECTOR, pathThreshold: Number.POSITIVE_INFINITY }
|
|
172
|
+
};
|
|
173
|
+
return { guards, carryForward: false };
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
adhoc: {
|
|
177
|
+
id: 'adhoc',
|
|
178
|
+
why: 'The model-dispatched `pi-worker` tool. Every guard at its default, and '
|
|
179
|
+
+ 'this row exists so that is a DECISION rather than the absence of one — '
|
|
180
|
+
+ 'the call site passed nothing, and nobody could see what it therefore '
|
|
181
|
+
+ 'got. What the table now makes visible is an asymmetry: this is the '
|
|
182
|
+
+ 'strictest-clocked of the three children. It runs a FIXED 240s cap, '
|
|
183
|
+
+ 'because `progressCeilingMs` is null and the deadline re-arm is inert '
|
|
184
|
+
+ 'without one, while a research worker doing the same read-only '
|
|
185
|
+
+ 'exploration gets 240s WITHOUT PROGRESS up to 20 minutes. That '
|
|
186
|
+
+ 'difference is preserved exactly here and is NOT defended: the progress '
|
|
187
|
+
+ 'deadline was measured for the research workers (nexttask 9) and has '
|
|
188
|
+
+ 'never been measured for this tool. It is a candidate, not a bug.',
|
|
189
|
+
resolve: () => ({ guards: baseGuards(), carryForward: false })
|
|
190
|
+
// `as const satisfies`, not an annotation — the same reason RESTART_ORDER
|
|
191
|
+
// gives: an annotation widens each row back to `WorkerProfile`, and the
|
|
192
|
+
// `why` strings and literal ids stop being visible to a reader or a test.
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
/** Resolve one profile. The only way a caller should obtain a policy. */
|
|
196
|
+
export function workerPolicy(id, inputs = {}) {
|
|
197
|
+
return WORKER_PROFILES[id].resolve(inputs);
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Lay whole rows over a resolved policy.
|
|
201
|
+
*
|
|
202
|
+
* For tests and A/B harnesses ONLY. Production code names a profile: an override
|
|
203
|
+
* at a production call site is the exact "hand-pick a subset" this module exists
|
|
204
|
+
* to stop, and `worker-profiles.test.ts` fails the build if one appears.
|
|
205
|
+
*/
|
|
206
|
+
export function applyOverride(policy, override) {
|
|
207
|
+
if (override === undefined)
|
|
208
|
+
return policy;
|
|
209
|
+
const { carryForward, ...rows } = override;
|
|
210
|
+
// Present-but-`undefined` is DROPPED, not laid down. A conditional row is
|
|
211
|
+
// the natural way to write a swept arm — `{'command-timeout': on ? ms :
|
|
212
|
+
// undefined}` — and a plain spread would put `undefined` into the policy,
|
|
213
|
+
// which either disarms the guard silently or throws on `clock.timeoutMs`.
|
|
214
|
+
// The repo has no `exactOptionalPropertyTypes`, so the compiler allows it.
|
|
215
|
+
const set = Object.fromEntries(Object.entries(rows).filter(([, v]) => v !== undefined));
|
|
216
|
+
return {
|
|
217
|
+
guards: { ...policy.guards, ...set },
|
|
218
|
+
carryForward: carryForward ?? policy.carryForward
|
|
219
|
+
};
|
|
220
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.25",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|