@mjasnikovs/pi-task 0.38.24 → 0.38.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/gate-child.js +11 -11
- package/dist/task/phases.js +5 -8
- package/dist/task/research-fanout-budget.d.ts +20 -0
- package/dist/task/research-fanout-budget.js +29 -0
- package/dist/task/research-worker.d.ts +10 -7
- package/dist/task/research-worker.js +11 -14
- package/dist/workers/pi-worker-core.d.ts +25 -108
- package/dist/workers/pi-worker-core.js +30 -41
- package/dist/workers/pi-worker.js +6 -0
- package/dist/workers/worker-profiles.d.ts +314 -0
- package/dist/workers/worker-profiles.js +220 -0
- package/package.json +1 -1
package/dist/task/gate-child.js
CHANGED
|
@@ -88,18 +88,18 @@ export function makeGateChild(deps) {
|
|
|
88
88
|
cwd: deps.cwd,
|
|
89
89
|
...(sig ? { signal: sig } : {}),
|
|
90
90
|
tools,
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
91
|
+
// The four guard literals that used to sit here — run to
|
|
92
|
+
// completion, a per-command watchdog, a stream watchdog, and
|
|
93
|
+
// the path rule disabled — are the `gate` row of
|
|
94
|
+
// WORKER_PROFILES (workers/worker-profiles.ts), which carries
|
|
95
|
+
// the reasoning for each. The two ceilings stay inputs
|
|
96
|
+
// because they are user config, not policy.
|
|
97
|
+
profile: 'gate',
|
|
98
|
+
policyInputs: {
|
|
99
|
+
commandTimeoutMs: deps.commandTimeoutMs,
|
|
100
|
+
streamInactivityMs: deps.streamInactivityMs
|
|
101
|
+
},
|
|
97
102
|
thinking: deps.thinking,
|
|
98
|
-
// Exact-match loop guard only: pathThreshold Infinity disables
|
|
99
|
-
// the path-revisit heuristic, so revisiting one file (which IS
|
|
100
|
-
// the job) never trips — only a literally-identical call
|
|
101
|
-
// repeated past threshold does.
|
|
102
|
-
loop: { pathThreshold: Number.POSITIVE_INFINITY },
|
|
103
103
|
// A discarded attempt is otherwise invisible: the returned
|
|
104
104
|
// exitCode/text describe the FINAL attempt, so a child that
|
|
105
105
|
// burned two attempts reads exactly like one that ran clean.
|
package/dist/task/phases.js
CHANGED
|
@@ -9,7 +9,7 @@ import { runWorker } from '../workers/pi-worker-core.js';
|
|
|
9
9
|
import { findPhantomImports, formatApiCorrections, rewritePhantomSpecifiers } from '../workers/phantom-imports.js';
|
|
10
10
|
import { searchProviderKey } from '../workers/search-types.js';
|
|
11
11
|
import { channelSet } from '../workers/worker-channels.js';
|
|
12
|
-
import {
|
|
12
|
+
import { snapshotLeverEnv, workerProgressCeilingMs, projectDocsBudget, projectDocsBudgetNotice } from './research-fanout-budget.js';
|
|
13
13
|
import { isIntegrationUnknown } from './unknown-routing.js';
|
|
14
14
|
import { extractUserDirectives, preserveDirectivesBlock, enforceDirectives } from './user-directives.js';
|
|
15
15
|
import { demoteUnsourcedAttributions } from './context-attribution.js';
|
|
@@ -488,10 +488,9 @@ export async function phaseResearch(deps, refined) {
|
|
|
488
488
|
// phase, so every worker in a run sees the same policy and a harness cannot
|
|
489
489
|
// half-apply an arm. CAP, SCALE and carry-forward are null/false in the
|
|
490
490
|
// shipped configuration; the progress deadline shipped ON (nexttask 9).
|
|
491
|
-
const
|
|
492
|
-
const
|
|
493
|
-
const
|
|
494
|
-
const progressCeilingMs = workerProgressCeilingMs();
|
|
491
|
+
const leverEnv = snapshotLeverEnv();
|
|
492
|
+
const fanoutBudget = projectDocsBudget(leverEnv);
|
|
493
|
+
const progressCeilingMs = workerProgressCeilingMs(leverEnv);
|
|
495
494
|
// Which deadline policy was in force is a fact about how every number below
|
|
496
495
|
// was produced. Run 18's 120 discarded minutes were only recoverable because
|
|
497
496
|
// 5A started writing down what the workers actually did; a run whose logs do
|
|
@@ -695,9 +694,7 @@ export async function phaseResearch(deps, refined) {
|
|
|
695
694
|
onDone: updateProgress,
|
|
696
695
|
readCached: async (heading) => (await readSection(deps.cwd, deps.taskId, heading)) ?? '',
|
|
697
696
|
persistSection,
|
|
698
|
-
|
|
699
|
-
fanoutTimeout,
|
|
700
|
-
progressCeilingMs
|
|
697
|
+
leverEnv
|
|
701
698
|
}, prior);
|
|
702
699
|
const sections = [];
|
|
703
700
|
if (!getConfig().parallelResearchWorkers) {
|
|
@@ -81,6 +81,26 @@ export declare const WORKER_CARRY_FORWARD_ENV = "PI_TASK_WORKER_CARRY_FORWARD";
|
|
|
81
81
|
/** RESCUE: deadline on lack of progress instead of elapsed time. Value = absolute ceiling, ms. */
|
|
82
82
|
export declare const WORKER_PROGRESS_CEILING_ENV = "PI_TASK_WORKER_PROGRESS_CEILING_MS";
|
|
83
83
|
type Env = (key: string) => string | undefined;
|
|
84
|
+
/**
|
|
85
|
+
* Every lever env var this module owns.
|
|
86
|
+
*
|
|
87
|
+
* Exists so `snapshotLeverEnv` cannot drift from the levers: adding a lever
|
|
88
|
+
* without adding it here would leave that one lever read LATE, which is the
|
|
89
|
+
* half-applied arm the snapshot exists to prevent.
|
|
90
|
+
*/
|
|
91
|
+
export declare const RESEARCH_LEVER_ENVS: readonly string[];
|
|
92
|
+
/**
|
|
93
|
+
* The levers, read ONCE, as a reader the profile table can be handed.
|
|
94
|
+
*
|
|
95
|
+
* WHY A SNAPSHOT AND NOT `process.env`. Every worker in one research phase must
|
|
96
|
+
* see the same arm. The three lever values used to be resolved once in
|
|
97
|
+
* `phases.ts` and threaded down as three separate `ResearchWorkerRun` fields for
|
|
98
|
+
* exactly that reason; moving the resolution into the `research` profile would
|
|
99
|
+
* have moved the READ down to each worker with it, and a harness that flips a
|
|
100
|
+
* var mid-phase would then half-apply its own arm. Freezing the reader keeps the
|
|
101
|
+
* read-once property while letting the profile own what the values MEAN.
|
|
102
|
+
*/
|
|
103
|
+
export declare function snapshotLeverEnv(env?: Env): Env;
|
|
84
104
|
/**
|
|
85
105
|
* The CAP arm's budget, or null when the lever is off (the shipped default).
|
|
86
106
|
* A non-numeric or non-positive value is off too: a typo'd env var must not
|
|
@@ -81,6 +81,35 @@ export const WORKER_CARRY_FORWARD_ENV = 'PI_TASK_WORKER_CARRY_FORWARD';
|
|
|
81
81
|
/** RESCUE: deadline on lack of progress instead of elapsed time. Value = absolute ceiling, ms. */
|
|
82
82
|
export const WORKER_PROGRESS_CEILING_ENV = 'PI_TASK_WORKER_PROGRESS_CEILING_MS';
|
|
83
83
|
const defaultEnv = key => process.env[key];
|
|
84
|
+
/**
|
|
85
|
+
* Every lever env var this module owns.
|
|
86
|
+
*
|
|
87
|
+
* Exists so `snapshotLeverEnv` cannot drift from the levers: adding a lever
|
|
88
|
+
* without adding it here would leave that one lever read LATE, which is the
|
|
89
|
+
* half-applied arm the snapshot exists to prevent.
|
|
90
|
+
*/
|
|
91
|
+
export const RESEARCH_LEVER_ENVS = [
|
|
92
|
+
PROJECT_DOCS_BUDGET_ENV,
|
|
93
|
+
FANOUT_TIMEOUT_PER_LOOKUP_ENV,
|
|
94
|
+
FANOUT_TIMEOUT_CEILING_ENV,
|
|
95
|
+
WORKER_CARRY_FORWARD_ENV,
|
|
96
|
+
WORKER_PROGRESS_CEILING_ENV
|
|
97
|
+
];
|
|
98
|
+
/**
|
|
99
|
+
* The levers, read ONCE, as a reader the profile table can be handed.
|
|
100
|
+
*
|
|
101
|
+
* WHY A SNAPSHOT AND NOT `process.env`. Every worker in one research phase must
|
|
102
|
+
* see the same arm. The three lever values used to be resolved once in
|
|
103
|
+
* `phases.ts` and threaded down as three separate `ResearchWorkerRun` fields for
|
|
104
|
+
* exactly that reason; moving the resolution into the `research` profile would
|
|
105
|
+
* have moved the READ down to each worker with it, and a harness that flips a
|
|
106
|
+
* var mid-phase would then half-apply its own arm. Freezing the reader keeps the
|
|
107
|
+
* read-once property while letting the profile own what the values MEAN.
|
|
108
|
+
*/
|
|
109
|
+
export function snapshotLeverEnv(env = defaultEnv) {
|
|
110
|
+
const snap = new Map(RESEARCH_LEVER_ENVS.map(k => [k, env(k)]));
|
|
111
|
+
return key => snap.get(key);
|
|
112
|
+
}
|
|
84
113
|
function positiveInt(raw) {
|
|
85
114
|
if (raw === undefined)
|
|
86
115
|
return null;
|
|
@@ -88,13 +88,16 @@ export interface ResearchWorkerRun {
|
|
|
88
88
|
readCached: (heading: string) => Promise<string>;
|
|
89
89
|
/** Write one validated section to the task file. Serialised by the caller. */
|
|
90
90
|
persistSection: (heading: string, text: string) => Promise<void>;
|
|
91
|
-
/**
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
91
|
+
/**
|
|
92
|
+
* The 5B lever env vars, READ ONCE for the whole phase.
|
|
93
|
+
*
|
|
94
|
+
* Was three resolved values (`carryForward`, `fanoutTimeout`,
|
|
95
|
+
* `progressCeilingMs`). It is one frozen reader now because the `research`
|
|
96
|
+
* profile owns what those values mean; what this layer still owns is that
|
|
97
|
+
* every worker in a run sees the SAME arm, which a live `process.env` read
|
|
98
|
+
* per worker would lose. See `snapshotLeverEnv`.
|
|
99
|
+
*/
|
|
100
|
+
leverEnv: (key: string) => string | undefined;
|
|
98
101
|
}
|
|
99
102
|
/**
|
|
100
103
|
* Task-file heading under which a research worker's validated output is cached.
|
|
@@ -263,20 +263,17 @@ export async function runResearchWorker(spec, run, prior = []) {
|
|
|
263
263
|
thinking: run.thinkingFor(spec.label),
|
|
264
264
|
...(spec.tools ? { tools: spec.tools } : {}),
|
|
265
265
|
...(spec.extensions ? { extensions: spec.extensions } : {}),
|
|
266
|
-
// 5B
|
|
267
|
-
//
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
//
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
...(run.progressCeilingMs !== null ?
|
|
278
|
-
{ progressTimeoutCeilingMs: run.progressCeilingMs }
|
|
279
|
-
: {}),
|
|
266
|
+
// The three 5B lever spreads that used to sit here are the
|
|
267
|
+
// `research` row of WORKER_PROFILES (workers/worker-profiles.ts).
|
|
268
|
+
// Two facts still come from here, and only these two: which of
|
|
269
|
+
// the four workers is docs-capable (only it can be scaled), and
|
|
270
|
+
// the phase's FROZEN lever reader, so every worker in one run
|
|
271
|
+
// sees the same arm.
|
|
272
|
+
profile: 'research',
|
|
273
|
+
policyInputs: {
|
|
274
|
+
...(spec.fanoutBounded ? { fanoutBounded: true } : {}),
|
|
275
|
+
env: run.leverEnv
|
|
276
|
+
},
|
|
280
277
|
// One line per DISCARDED attempt. The `done` line below reports
|
|
281
278
|
// the final attempt only, so a worker that timed out twice at
|
|
282
279
|
// 240s and then answered used to log exactly like a clean one —
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ContextSnapshot, type LoopHit, type SpawnFn } from '../shared/child-process.js';
|
|
2
2
|
import { RESTART_ORDER } from './worker-kill.js';
|
|
3
|
+
import { type WorkerGuardOverride, type WorkerGuardPolicy, type WorkerPolicyInputs, type WorkerProfileId } from './worker-profiles.js';
|
|
3
4
|
/**
|
|
4
5
|
* Tool calls that can GROUND an APIS claim — i.e. return content a signature or
|
|
5
6
|
* command could be cited from. `pi-worker-docs` (the primary), `read` and `grep`
|
|
@@ -88,116 +89,40 @@ export interface RunWorkerInput {
|
|
|
88
89
|
*/
|
|
89
90
|
contextWindow?: number;
|
|
90
91
|
/**
|
|
91
|
-
*
|
|
92
|
-
* Pass 0 to disable the timeout entirely (run until the child exits on its
|
|
93
|
-
* own) — for a pass that must be allowed to finish however long it takes.
|
|
94
|
-
*/
|
|
95
|
-
timeoutMs?: number;
|
|
96
|
-
/**
|
|
97
|
-
* PER-TOOL-CALL wall-clock ceiling in ms — the child-side half of the command
|
|
98
|
-
* watchdog (see shared/command-watchdog.ts). Arms on each tool_execution_start
|
|
99
|
-
* and disarms on the matching end; on overrun the child is killed and, within
|
|
100
|
-
* the shared restart budget, re-spawned with commandTimeoutHint.
|
|
101
|
-
*
|
|
102
|
-
* WHY SEPARATE FROM `timeoutMs`: that one bounds the whole worker and is
|
|
103
|
-
* deliberately 0 (unbounded) for gate children, which must run to completion.
|
|
104
|
-
* Neither it nor the stall guard can catch a hung command — the stall guard
|
|
105
|
-
* treats a reachable model endpoint as proof of life, which it is, even while
|
|
106
|
-
* a `bun run dev` the model forgot to bound blocks the child forever.
|
|
92
|
+
* WHICH KIND of worker child this is — the whole guard policy, in one word.
|
|
107
93
|
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* 0 / omitted = off, so every existing caller is unchanged.
|
|
114
|
-
*/
|
|
115
|
-
commandTimeoutMs?: number;
|
|
116
|
-
/**
|
|
117
|
-
* Per-worker loop-detector tuning. Defaults to the read-only research/impl
|
|
118
|
-
* guard (LOOP_WINDOW / LOOP_THRESHOLD, path threshold = exact threshold). An
|
|
119
|
-
* edit/fix pass legitimately revisits one file, so it can raise (or disable
|
|
120
|
-
* via Infinity) `pathThreshold`. Pass `false` to turn the detector OFF
|
|
121
|
-
* entirely — no tool-call pattern will ever kill the worker.
|
|
94
|
+
* REQUIRED, and required on purpose. The ten guard knobs this replaces used
|
|
95
|
+
* to sit here as independent optionals, so a caller that named none of them
|
|
96
|
+
* still got a full policy and nobody could see which one. That is how the
|
|
97
|
+
* ad-hoc `pi-worker` tool came to run the strictest wall clock of the three
|
|
98
|
+
* children without anyone deciding it should. See worker-profiles.ts.
|
|
122
99
|
*/
|
|
123
|
-
|
|
124
|
-
window?: number;
|
|
125
|
-
threshold?: number;
|
|
126
|
-
pathThreshold?: number;
|
|
127
|
-
} | false;
|
|
100
|
+
profile: WorkerProfileId;
|
|
128
101
|
/**
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* WHY BOTH. LoopDetector judges ARGUMENTS over a 20-call window, so a child
|
|
132
|
-
* that rotates through MORE DISTINCT CALLS THAN THE WINDOW HOLDS is invisible
|
|
133
|
-
* to it — every key occurs once per window and the count never reaches the
|
|
134
|
-
* threshold. Measured: mx5-n 2026-08-27, worker:tooling made 550 calls over
|
|
135
|
-
* exactly 20 distinct files, ~36 reads each, and neither the exact rule nor
|
|
136
|
-
* the path rule ever tripped. It died 20 minutes later on the absolute
|
|
137
|
-
* progress ceiling, having done 25s of useful work.
|
|
138
|
-
*
|
|
139
|
-
* StallDetector judges RESULTS, which a rotating reader cannot vary. It was
|
|
140
|
-
* written for exactly this class and was wired only into phase children
|
|
141
|
-
* (task/child-runner.ts) until this option existed.
|
|
142
|
-
*
|
|
143
|
-
* Pass `false` to disable, or override the thresholds (tests, harnesses).
|
|
102
|
+
* The facts the profile needs that are NOT policy: the gate's two watchdog
|
|
103
|
+
* ceilings (user config) and which research worker is docs-capable.
|
|
144
104
|
*/
|
|
145
|
-
|
|
146
|
-
limit?: number;
|
|
147
|
-
churnFactor?: number;
|
|
148
|
-
} | false;
|
|
105
|
+
policyInputs?: WorkerPolicyInputs;
|
|
149
106
|
/**
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
107
|
+
* Whole guard rows laid over the profile's. TESTS AND A/B HARNESSES ONLY —
|
|
108
|
+
* an override at a production call site is the hand-picked subset this
|
|
109
|
+
* design exists to stop, and `worker-profiles.test.ts` fails the build if
|
|
110
|
+
* one appears under src/ outside a test.
|
|
154
111
|
*/
|
|
155
|
-
|
|
156
|
-
afterMs?: number;
|
|
157
|
-
probe?: () => Promise<boolean>;
|
|
158
|
-
} | false;
|
|
112
|
+
override?: WorkerGuardOverride;
|
|
159
113
|
/**
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
114
|
+
* The resolved policy this run will use, reported once before the first
|
|
115
|
+
* attempt.
|
|
116
|
+
*
|
|
117
|
+
* WHY: asserting that a profile RESOLVES correctly proves nothing about
|
|
118
|
+
* whether runWorker then READS it correctly — a rewiring that turns "0 means
|
|
119
|
+
* off" into "0 means on" leaves every profile assertion green. This hook is
|
|
120
|
+
* what lets a caller's own test (gate-child.test.ts) drive the REAL call
|
|
121
|
+
* site and check the REAL policy, instead of re-typing the table.
|
|
167
122
|
*/
|
|
168
|
-
|
|
123
|
+
onPolicy?: (policy: WorkerGuardPolicy) => void;
|
|
169
124
|
/** Backoff sleep, injectable so tests don't wait out the real delays. */
|
|
170
125
|
sleepFor?: (ms: number) => Promise<void>;
|
|
171
|
-
/**
|
|
172
|
-
* SCALE arm of nexttask 5B — OFF unless set, and set only by the harness that
|
|
173
|
-
* is measuring it (src/task/research-fanout-budget.ts explains both arms).
|
|
174
|
-
* Each project-source `pi-worker-docs` call pushes this attempt's deadline out
|
|
175
|
-
* by `perLookupMs`, never past `ceilingMs` from the attempt's start: a worker
|
|
176
|
-
* that is making retrieval progress is not killed for making it, while a
|
|
177
|
-
* worker that is thrashing still hits a hard bound.
|
|
178
|
-
*/
|
|
179
|
-
fanoutTimeout?: {
|
|
180
|
-
perLookupMs: number;
|
|
181
|
-
ceilingMs: number;
|
|
182
|
-
};
|
|
183
|
-
/**
|
|
184
|
-
* Absolute backstop that turns `timeoutMs` from "total time allowed" into
|
|
185
|
-
* "time allowed WITHOUT PROGRESS". A tool call or a line of output re-arms
|
|
186
|
-
* the deadline; only a worker that goes quiet for `timeoutMs` — or exceeds
|
|
187
|
-
* this ceiling outright — is killed.
|
|
188
|
-
*
|
|
189
|
-
* This is the difference between "took too long" and "stopped working". The
|
|
190
|
-
* first is a property of the machine (a slower local model, a bigger file)
|
|
191
|
-
* and must not cost the user their answer; the second is a real fault, and
|
|
192
|
-
* one the output-stall probe already catches on its own terms.
|
|
193
|
-
*/
|
|
194
|
-
progressTimeoutCeilingMs?: number;
|
|
195
|
-
/**
|
|
196
|
-
* Carry a killed attempt's findings into the re-spawn, and never return less
|
|
197
|
-
* than the best attempt produced. OFF by default so the shipped path is
|
|
198
|
-
* unchanged while the A/B runs — see src/task/research-fanout-budget.ts.
|
|
199
|
-
*/
|
|
200
|
-
carryForward?: boolean;
|
|
201
126
|
/**
|
|
202
127
|
* Called when a carried-forward partial is INJECTED into an attempt's prompt
|
|
203
128
|
* — once per attempt that receives one. Distinct from `onRestart`, which says
|
|
@@ -234,14 +159,6 @@ export interface RunWorkerInput {
|
|
|
234
159
|
* of the `start` and `done` lines around it.
|
|
235
160
|
*/
|
|
236
161
|
onRestart?: (restart: WorkerRestart) => void;
|
|
237
|
-
/**
|
|
238
|
-
* Connection-error restart budget. Defaults to MAX_LOOP_RESTARTS, and even
|
|
239
|
-
* then the SHARED restart counter is what actually binds — a worker that
|
|
240
|
-
* already spent the budget looping does not get extra lives here. 0 turns the
|
|
241
|
-
* retry off, which is how scripts/connection-retry-ab.ts gets a baseline arm
|
|
242
|
-
* out of a build that already ships the retry.
|
|
243
|
-
*/
|
|
244
|
-
connectionRetries?: number;
|
|
245
162
|
}
|
|
246
163
|
/**
|
|
247
164
|
* Why an attempt was thrown away. One value per restart branch in runWorker, so
|
|
@@ -5,12 +5,13 @@ import { isGroundingRetrieval as isGrounding, workerChannel } from './worker-cha
|
|
|
5
5
|
import { childBaseArgs } from '../shared/child-extensions.js';
|
|
6
6
|
import { LoopDetector } from '../task/loop-detector.js';
|
|
7
7
|
import { StallDetector, formatStallHint } from '../task/stall-detector.js';
|
|
8
|
-
import {
|
|
8
|
+
import { MAX_LOOP_RESTARTS, formatLoopHint, isConnectionError, connectionRetryBackoffMs } from '../task/child-runner.js';
|
|
9
9
|
import { detectLeakedToolCall, leakedToolCallHint, MAX_LEAK_RETRIES } from '../shared/leaked-tool-call.js';
|
|
10
10
|
import { discoverModelEndpoints, probeModelEndpoints } from '../shared/model-endpoint.js';
|
|
11
11
|
import { streamStallHint } from '../shared/stream-watchdog.js';
|
|
12
12
|
import { classifyWorkerFailure } from './worker-failure.js';
|
|
13
13
|
import { CARRY_FORWARD_IDS } from './worker-kill.js';
|
|
14
|
+
import { applyOverride, WORKER_PROFILES } from './worker-profiles.js';
|
|
14
15
|
// `--mode json` makes pi emit structured events as they happen instead of
|
|
15
16
|
// buffering the assistant text and flushing on exit. That matters for the
|
|
16
17
|
// wait/work timing split: in text mode the first stdout chunk only arrives at
|
|
@@ -35,25 +36,9 @@ const DEFAULT_TOOLS = 'read,grep,find,ls';
|
|
|
35
36
|
// hand-kept — this was a second copy of the four tool names. Re-exported because
|
|
36
37
|
// several call sites and tests import it from here.
|
|
37
38
|
export { isGroundingRetrieval } from './worker-channels.js';
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
* that thrashes with slightly-varied calls (different grep patterns each time)
|
|
42
|
-
* slips past it and would otherwise run unbounded. This is the backstop for that
|
|
43
|
-
* case: after this long with no clean exit, abort and restart with a hint. Sized
|
|
44
|
-
* well above a healthy worker's observed runtime (~25-130s on the local backend)
|
|
45
|
-
* so it never trips a legitimately slow run.
|
|
46
|
-
*/
|
|
47
|
-
const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
|
|
48
|
-
/**
|
|
49
|
-
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
50
|
-
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
51
|
-
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
52
|
-
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
53
|
-
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
54
|
-
* alive) just gets probed and waits on.
|
|
55
|
-
*/
|
|
56
|
-
const STALL_AFTER_MS = 180_000;
|
|
39
|
+
// RESEARCH_WORKER_TIMEOUT_MS and STALL_AFTER_MS live on the profile table now
|
|
40
|
+
// (worker-profiles.ts): they are the default VALUES of two guard rows, and a
|
|
41
|
+
// default that lives apart from the table stating it is a second place to look.
|
|
57
42
|
/**
|
|
58
43
|
* Restart hint after a WHOLE-WORKER wall-clock timeout — distinct from both the
|
|
59
44
|
* loop hint and the per-command hint. This one diagnoses over-exploration, which
|
|
@@ -443,7 +428,14 @@ export async function runWorker(input) {
|
|
|
443
428
|
'--tools',
|
|
444
429
|
tools
|
|
445
430
|
];
|
|
446
|
-
|
|
431
|
+
// ONE resolution, before the first attempt. Every guard read below goes
|
|
432
|
+
// through `policy`, so "which knobs is this child running" has exactly one
|
|
433
|
+
// answer and it is observable (`onPolicy`) rather than inferable.
|
|
434
|
+
const policy = applyOverride(WORKER_PROFILES[input.profile].resolve(input.policyInputs ?? {}), input.override);
|
|
435
|
+
input.onPolicy?.(policy);
|
|
436
|
+
const guards = policy.guards;
|
|
437
|
+
const clock = guards['worker-timeout'];
|
|
438
|
+
const timeoutMs = clock.timeoutMs;
|
|
447
439
|
let hint = null;
|
|
448
440
|
// Loop-kill and timeout share one restart budget, mirroring
|
|
449
441
|
// runPhaseChild: a runaway worker gets re-spawned with a corrective
|
|
@@ -496,19 +488,15 @@ export async function runWorker(input) {
|
|
|
496
488
|
// loop === false turns the guard off entirely (detector is null and no
|
|
497
489
|
// tool call is ever flagged); otherwise build a detector from the override
|
|
498
490
|
// or the default research/impl thresholds.
|
|
499
|
-
const loopDetector =
|
|
491
|
+
const loopDetector = guards.loop.detector === false ?
|
|
500
492
|
null
|
|
501
|
-
: (
|
|
502
|
-
const window = input.loop?.window ?? LOOP_WINDOW;
|
|
503
|
-
const threshold = input.loop?.threshold ?? LOOP_THRESHOLD;
|
|
504
|
-
return new LoopDetector(window, threshold, input.loop?.pathThreshold ?? threshold);
|
|
505
|
-
})();
|
|
493
|
+
: new LoopDetector(guards.loop.detector.window, guards.loop.detector.threshold, guards.loop.detector.pathThreshold);
|
|
506
494
|
// Reset EACH attempt, like the loop detector: a restart discards the
|
|
507
495
|
// previous attempt's calls along with its text, so a fresh child must not
|
|
508
496
|
// inherit a dead streak it did not earn.
|
|
509
|
-
const stallDetector =
|
|
497
|
+
const stallDetector = guards.loop.progress === false ?
|
|
510
498
|
null
|
|
511
|
-
: new StallDetector(
|
|
499
|
+
: new StallDetector(guards.loop.progress.limit, guards.loop.progress.churnFactor);
|
|
512
500
|
// Arm the churn rule BEFORE the first tool call. pi's stream carries no
|
|
513
501
|
// context event (issue #16), so waiting for one leaves the rule
|
|
514
502
|
// permanently disarmed. The parent knows the window at spawn time.
|
|
@@ -522,27 +510,28 @@ export async function runWorker(input) {
|
|
|
522
510
|
// discarded with its text, so the count must describe only the attempt
|
|
523
511
|
// whose text this call returns.
|
|
524
512
|
let groundingRetrievalCount = 0;
|
|
525
|
-
const timeout = workerTimeout(input.signal, timeoutMs,
|
|
513
|
+
const timeout = workerTimeout(input.signal, timeoutMs, clock.progressCeilingMs ?? undefined);
|
|
526
514
|
// Per-tool-call watchdog for this attempt (null when off). Its abort is
|
|
527
515
|
// OR'd with the worker timeout / external cancel into the child's signal.
|
|
528
|
-
const cmdWatch = commandWatch(commandCeilingForAttempt(
|
|
516
|
+
const cmdWatch = commandWatch(commandCeilingForAttempt(guards['command-timeout'], hangKills));
|
|
529
517
|
const childSignal = cmdWatch ? AbortSignal.any([timeout.signal, cmdWatch.signal]) : timeout.signal;
|
|
530
518
|
let result;
|
|
531
519
|
try {
|
|
532
520
|
result = await runChildDefault(invocation, input.cwd, childSignal, {
|
|
533
521
|
mode: 'json-events',
|
|
534
|
-
...(
|
|
522
|
+
...(guards.stalled === false ?
|
|
535
523
|
{}
|
|
536
524
|
: {
|
|
537
525
|
stall: {
|
|
538
|
-
afterMs:
|
|
539
|
-
probe
|
|
526
|
+
afterMs: guards.stalled.afterMs,
|
|
527
|
+
// `null` in the policy means the built-in probe.
|
|
528
|
+
// Kept as data so a resolved policy stays plain
|
|
529
|
+
// comparable data — see StalledGuard.probe.
|
|
530
|
+
probe: guards.stalled.probe
|
|
540
531
|
?? (() => probeModelEndpoints(discoverModelEndpoints()))
|
|
541
532
|
}
|
|
542
533
|
}),
|
|
543
|
-
...(
|
|
544
|
-
{ streamInactivityMs: input.streamInactivityMs }
|
|
545
|
-
: {}),
|
|
534
|
+
...(guards['stream-stall'] ? { streamInactivityMs: guards['stream-stall'] } : {}),
|
|
546
535
|
onFirstByte: () => (tFirstByte = Date.now()),
|
|
547
536
|
onToolCall: call => {
|
|
548
537
|
cmdWatch?.onStart(call);
|
|
@@ -551,9 +540,9 @@ export async function runWorker(input) {
|
|
|
551
540
|
timeout.progress();
|
|
552
541
|
// The generic child runner used to name ONE tool and ONE of
|
|
553
542
|
// its parameters here. It asks the tool's own row now.
|
|
554
|
-
if (
|
|
543
|
+
if (clock.fanout
|
|
555
544
|
&& workerChannel(call.name)?.isProjectSourceLookup?.(call.args ?? {}) === true) {
|
|
556
|
-
timeout.extend(
|
|
545
|
+
timeout.extend(clock.fanout.perLookupMs, clock.fanout.ceilingMs);
|
|
557
546
|
}
|
|
558
547
|
if (isGrounding(call.name))
|
|
559
548
|
groundingRetrievalCount++;
|
|
@@ -621,7 +610,7 @@ export async function runWorker(input) {
|
|
|
621
610
|
// path cannot be added that silently drops the attempt's work.
|
|
622
611
|
// Longest-wins — a later attempt killed early should not replace a
|
|
623
612
|
// fuller answer an earlier one had already reached.
|
|
624
|
-
if (
|
|
613
|
+
if (policy.carryForward && CARRY_FORWARD_REASONS.has(reason)) {
|
|
625
614
|
const partial = text.trim();
|
|
626
615
|
// Longest-with-CONTENT wins. Length alone let a preamble sentence
|
|
627
616
|
// become the answer — see hasAnswerContent.
|
|
@@ -652,7 +641,7 @@ export async function runWorker(input) {
|
|
|
652
641
|
tools,
|
|
653
642
|
restartBudgetSpent,
|
|
654
643
|
connRetries,
|
|
655
|
-
connectionRetries:
|
|
644
|
+
connectionRetries: guards['connection-error'],
|
|
656
645
|
leakRetries
|
|
657
646
|
};
|
|
658
647
|
let restarted = false;
|
|
@@ -47,6 +47,12 @@ export function registerPiWorker(pi) {
|
|
|
47
47
|
prompt: params.prompt,
|
|
48
48
|
cwd: ctx.cwd,
|
|
49
49
|
signal,
|
|
50
|
+
// `adhoc` is every guard at its default — which is what this call
|
|
51
|
+
// site already got by naming none of them. Named now so it is a
|
|
52
|
+
// decision, and so the one asymmetry it carries (a FIXED 240s cap
|
|
53
|
+
// where a research worker gets 240s without progress) is written
|
|
54
|
+
// down. See the `adhoc` row in workers/worker-profiles.ts.
|
|
55
|
+
profile: 'adhoc',
|
|
50
56
|
thinking: groupThinkingArgs('research')
|
|
51
57
|
});
|
|
52
58
|
const details = { exitCode: result.exitCode };
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The GUARD POLICY each kind of worker child runs under, keyed on the ways it
|
|
3
|
+
* can die.
|
|
4
|
+
*
|
|
5
|
+
* WHY IT EXISTS. `RunWorkerInput` carried ten guard knobs in four different
|
|
6
|
+
* shapes — two bare millisecond numbers, three `{...} | false` unions, an
|
|
7
|
+
* optional object, a boolean and two counts — and three production callers each
|
|
8
|
+
* hand-picked a different subset of them:
|
|
9
|
+
*
|
|
10
|
+
* gate-child.ts timeoutMs 0, a per-command watchdog, a stream watchdog,
|
|
11
|
+
* and the path rule disabled. Everything else default.
|
|
12
|
+
* research-worker.ts a progress deadline and two off-by-default A/B levers.
|
|
13
|
+
* NO command watchdog, NO stream watchdog. Everything else
|
|
14
|
+
* default.
|
|
15
|
+
* pi-worker.ts nothing at all — every default, silently.
|
|
16
|
+
*
|
|
17
|
+
* So "a gate child runs unbounded but with a per-command watchdog; a research
|
|
18
|
+
* worker is the reverse" existed only as three option literals in three files,
|
|
19
|
+
* and the reasoning was attached to whichever line happened to need defending.
|
|
20
|
+
* `gate-child.ts` explained why it disables the path rule and said nothing about
|
|
21
|
+
* why it takes no progress deadline. Nothing anywhere said that the ad-hoc
|
|
22
|
+
* `pi-worker` tool is the strictest-clocked of the three. That was not a
|
|
23
|
+
* decision; it was the residue of never having had a place to write one down.
|
|
24
|
+
*
|
|
25
|
+
* WHY IT IS KEYED ON `WorkerKillId`. A guard exists to prevent a specific way a
|
|
26
|
+
* child can die, so the roster of deaths (`worker-kill.ts`) is the correct key —
|
|
27
|
+
* the same argument that roster makes for kill CAUSES, one level up. The mapped
|
|
28
|
+
* type means a tenth cause cannot be added to `WORKER_KILLS` without every
|
|
29
|
+
* profile deciding what to do about it, and it means the three causes with no
|
|
30
|
+
* dial say so in the table (`null`) instead of being absent from it.
|
|
31
|
+
*
|
|
32
|
+
* The key does NOT partition the knobs one-per-row, and pretending otherwise
|
|
33
|
+
* would be the lie:
|
|
34
|
+
*
|
|
35
|
+
* `worker-timeout` holds THREE — the cap, the progress ceiling that turns the
|
|
36
|
+
* cap from "time allowed" into "time allowed without progress", and the
|
|
37
|
+
* fan-out extension. All three move the same deadline; splitting them across
|
|
38
|
+
* rows would let a profile set a ceiling for a cap it disabled.
|
|
39
|
+
*
|
|
40
|
+
* `loop` holds TWO detectors. `StallDetector`'s hit IS a `LoopHit` with
|
|
41
|
+
* `.stall` set (child-process.ts: "so a stall rides the kill/restart plumbing
|
|
42
|
+
* the loop hit already has"), and the restart ladder has ONE rule for both.
|
|
43
|
+
* One cause, one row.
|
|
44
|
+
*
|
|
45
|
+
* WHAT IS DELIBERATELY NOT UNIFIED.
|
|
46
|
+
*
|
|
47
|
+
* `carryForward` is not a row. It is one switch over the whole run, and WHICH
|
|
48
|
+
* causes honour it is already decided by `CARRY_FORWARD_IDS`, derived from the
|
|
49
|
+
* roster. A per-cause row here would be a second copy of that set, free to
|
|
50
|
+
* disagree with it.
|
|
51
|
+
*
|
|
52
|
+
* The reasoning group is not the profile. `pi-worker.ts` runs `adhoc` guards
|
|
53
|
+
* but `groupThinkingArgs('research')`, on purpose. Guards answer "how may this
|
|
54
|
+
* child die"; `thinking` answers "how hard may it think". Folding them would
|
|
55
|
+
* silently re-level a gate child, which is the exact mistake
|
|
56
|
+
* `RunWorkerInput.thinking`'s comment already records.
|
|
57
|
+
*
|
|
58
|
+
* `projectDocsBudget()` (the CAP arm, research-fanout-budget.ts) stays out. It
|
|
59
|
+
* bounds what a worker ASKS FOR, via its prompt and its tool, not how it dies.
|
|
60
|
+
*
|
|
61
|
+
* `RESTART_ORDER` and `FAILURE_ORDER` are untouched. This is a third view of
|
|
62
|
+
* the same key, not a merge of the two orderings.
|
|
63
|
+
*/
|
|
64
|
+
import type { WorkerKillId } from './worker-kill.js';
|
|
65
|
+
/**
|
|
66
|
+
* Hard wall-clock bound on a single worker run (one spawn). The exact-match
|
|
67
|
+
* LoopDetector only catches *identical* repeated tool calls; a model that
|
|
68
|
+
* thrashes with slightly-varied calls (different grep patterns each time) slips
|
|
69
|
+
* past it and would otherwise run unbounded. This is the backstop for that case:
|
|
70
|
+
* after this long with no clean exit, abort and restart with a hint. Sized well
|
|
71
|
+
* above a healthy worker's observed runtime (~25-130s on the local backend) so
|
|
72
|
+
* it never trips a legitimately slow run.
|
|
73
|
+
*/
|
|
74
|
+
export declare const RESEARCH_WORKER_TIMEOUT_MS = 240000;
|
|
75
|
+
/**
|
|
76
|
+
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
77
|
+
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
78
|
+
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
79
|
+
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
80
|
+
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
81
|
+
* alive) just gets probed and waits on.
|
|
82
|
+
*/
|
|
83
|
+
export declare const STALL_AFTER_MS = 180000;
|
|
84
|
+
/**
|
|
85
|
+
* The dead-backend probe. No output for `afterMs` -> probe the model endpoints
|
|
86
|
+
* pi is configured with -> unreachable -> kill and set `stalled: true`. Output
|
|
87
|
+
* progress resets it, and a reachable endpoint is treated as proof of life, so
|
|
88
|
+
* this alone will not end a child that is merely quiet.
|
|
89
|
+
*/
|
|
90
|
+
export interface StalledGuard {
|
|
91
|
+
afterMs: number;
|
|
92
|
+
/**
|
|
93
|
+
* `null` means the built-in endpoint probe, and a PROFILE always writes
|
|
94
|
+
* `null`. Kept as data rather than a closure so a resolved policy is plain
|
|
95
|
+
* comparable data — which is what makes the no-behaviour-change proof in
|
|
96
|
+
* `worker-profiles.test.ts` an equality assertion rather than a hand-written
|
|
97
|
+
* comparer that skips the one field most likely to be wrong. Tests and
|
|
98
|
+
* harnesses inject a real probe through the override.
|
|
99
|
+
*/
|
|
100
|
+
probe: (() => Promise<boolean>) | null;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The whole-worker deadline. All three fields move the SAME timer, which is why
|
|
104
|
+
* they share a row: `timeoutMs` is the cap (0 = unbounded), `progressCeilingMs`
|
|
105
|
+
* turns that cap from "total time allowed" into "time allowed WITHOUT PROGRESS"
|
|
106
|
+
* up to this absolute bound, and `fanout` pushes the deadline out per
|
|
107
|
+
* project-source lookup.
|
|
108
|
+
*
|
|
109
|
+
* The progress ceiling is the difference between "took too long" and "stopped
|
|
110
|
+
* working". The first is a property of the machine — a slower local model, a
|
|
111
|
+
* bigger file — and must not cost the user their answer; the second is a real
|
|
112
|
+
* fault, and one the dead-backend probe already catches on its own terms.
|
|
113
|
+
*
|
|
114
|
+
* `fanout` is the SCALE arm of nexttask 5B and is OFF unless both its env vars
|
|
115
|
+
* are set — see task/research-fanout-budget.ts for why it was not the fix.
|
|
116
|
+
*/
|
|
117
|
+
export interface WorkerTimeoutGuard {
|
|
118
|
+
/** 0 disables the wall clock entirely: the child runs until it exits. */
|
|
119
|
+
timeoutMs: number;
|
|
120
|
+
/**
|
|
121
|
+
* A tool call or a line of output re-arms the deadline to `now + timeoutMs`,
|
|
122
|
+
* never past this many ms from the attempt's start. `null` leaves the fixed
|
|
123
|
+
* cap and makes the re-arm inert.
|
|
124
|
+
*/
|
|
125
|
+
progressCeilingMs: number | null;
|
|
126
|
+
/**
|
|
127
|
+
* Each project-source `pi-worker-docs` call pushes this attempt's deadline
|
|
128
|
+
* out by `perLookupMs`, never past `ceilingMs` from the attempt's start.
|
|
129
|
+
*/
|
|
130
|
+
fanout: {
|
|
131
|
+
perLookupMs: number;
|
|
132
|
+
ceilingMs: number;
|
|
133
|
+
} | null;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* The two runaway detectors. ONE row because they are one cause: both return a
|
|
137
|
+
* `LoopHit` and both are handled by the single `loop` restart rule.
|
|
138
|
+
*
|
|
139
|
+
* `detector` judges ARGUMENTS over a 20-call window, so a child that rotates
|
|
140
|
+
* through MORE DISTINCT CALLS THAN THE WINDOW HOLDS is invisible to it — every
|
|
141
|
+
* key occurs once per window and the count never reaches the threshold.
|
|
142
|
+
* Measured: mx5-n 2026-08-27, worker:tooling made 550 calls over exactly 20
|
|
143
|
+
* distinct files, ~36 reads each, and neither the exact rule nor the path rule
|
|
144
|
+
* ever tripped. It died 20 minutes later on the absolute progress ceiling,
|
|
145
|
+
* having done 25s of useful work.
|
|
146
|
+
*
|
|
147
|
+
* `progress` judges RESULTS, which a rotating reader cannot vary. It was written
|
|
148
|
+
* for exactly that class and was wired only into phase children until
|
|
149
|
+
* `runWorker` grew an option for it.
|
|
150
|
+
*
|
|
151
|
+
* Either can be `false` independently — a pass that legitimately revisits one
|
|
152
|
+
* file raises `pathThreshold`; a harness isolating one rule turns the other off.
|
|
153
|
+
*/
|
|
154
|
+
export interface LoopGuard {
|
|
155
|
+
detector: {
|
|
156
|
+
window: number;
|
|
157
|
+
threshold: number;
|
|
158
|
+
pathThreshold: number;
|
|
159
|
+
} | false;
|
|
160
|
+
progress: {
|
|
161
|
+
limit: number;
|
|
162
|
+
churnFactor: number;
|
|
163
|
+
} | false;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* The knob (if any) that governs each way a worker can die.
|
|
167
|
+
*
|
|
168
|
+
* The three `null` rows are not filler. They are the statement that those causes
|
|
169
|
+
* have no dial: a leaked tool call is bounded by the fixed `MAX_LEAK_RETRIES`,
|
|
170
|
+
* `aborted` is the caller's own signal, and `exit` is the child deciding to
|
|
171
|
+
* stop. No profile may tune them, and now no profile can pretend to.
|
|
172
|
+
*/
|
|
173
|
+
interface WorkerGuardShapes {
|
|
174
|
+
stalled: StalledGuard | false;
|
|
175
|
+
/**
|
|
176
|
+
* PER-TOOL-CALL ceiling, ms. 0 = off. The child-side half of the command
|
|
177
|
+
* watchdog (shared/command-watchdog.ts): arms on each `tool_execution_start`,
|
|
178
|
+
* disarms on the matching end, and on overrun kills the child and — within
|
|
179
|
+
* the shared restart budget — re-spawns it with `commandTimeoutHint`.
|
|
180
|
+
*
|
|
181
|
+
* WHY IT IS NOT THE WALL CLOCK: that one bounds the whole worker and is
|
|
182
|
+
* deliberately 0 for gate children, which must run to completion. Neither it
|
|
183
|
+
* nor the dead-backend probe can catch a hung COMMAND — the probe treats a
|
|
184
|
+
* reachable model endpoint as proof of life, which it is, even while a `bun
|
|
185
|
+
* run dev` the model forgot to bound blocks the child forever.
|
|
186
|
+
*
|
|
187
|
+
* This is the ceiling for the FIRST attempt; each HANG-caused restart halves
|
|
188
|
+
* it (`commandCeilingForAttempt` — loop-caused restarts don't count), so a
|
|
189
|
+
* model that ignores the hint cannot spend the full ceiling again every retry.
|
|
190
|
+
*/
|
|
191
|
+
'command-timeout': number;
|
|
192
|
+
/**
|
|
193
|
+
* Stream-inactivity ceiling, ms (shared/stream-watchdog.ts). 0 = off.
|
|
194
|
+
*
|
|
195
|
+
* The dead-backend probe cannot catch a HUNG stream on a HEALTHY backend —
|
|
196
|
+
* it reads a reachable endpoint as proof of life, which is exactly what run
|
|
197
|
+
* 14's three hangs looked like. This one asks nothing of the backend: no
|
|
198
|
+
* output for this long, tool executions excluded, means kill and restart the
|
|
199
|
+
* attempt with `streamStallHint`, inside the same shared restart budget.
|
|
200
|
+
*/
|
|
201
|
+
'stream-stall': number;
|
|
202
|
+
'worker-timeout': WorkerTimeoutGuard;
|
|
203
|
+
/**
|
|
204
|
+
* Connection-error restart budget. The SHARED restart counter is what
|
|
205
|
+
* actually binds — a worker that already spent the budget looping does not
|
|
206
|
+
* get extra lives here. 0 turns the retry off, which is how
|
|
207
|
+
* `scripts/connection-retry-ab.ts` gets a baseline arm out of a build that
|
|
208
|
+
* already ships the retry.
|
|
209
|
+
*/
|
|
210
|
+
'connection-error': number;
|
|
211
|
+
loop: LoopGuard;
|
|
212
|
+
'leaked-tool-call': null;
|
|
213
|
+
aborted: null;
|
|
214
|
+
exit: null;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* One row per `WorkerKillId`. Indexing the shapes BY the roster's union is the
|
|
218
|
+
* compile-time bite: drop a row and `WorkerGuardShapes[K]` stops resolving.
|
|
219
|
+
*/
|
|
220
|
+
export type WorkerGuards = {
|
|
221
|
+
[K in WorkerKillId]: WorkerGuardShapes[K];
|
|
222
|
+
};
|
|
223
|
+
export interface WorkerGuardPolicy {
|
|
224
|
+
guards: WorkerGuards;
|
|
225
|
+
/**
|
|
226
|
+
* Carry a killed attempt's findings into the re-spawn, and never return less
|
|
227
|
+
* than the best attempt produced. Which CAUSES honour it is not settable —
|
|
228
|
+
* `CARRY_FORWARD_IDS` derives that from the roster. Cross-cutting, so
|
|
229
|
+
* deliberately NOT a row; see the header.
|
|
230
|
+
*/
|
|
231
|
+
carryForward: boolean;
|
|
232
|
+
}
|
|
233
|
+
/** A partial policy. Whole rows only: no deep-partial nobody can read. */
|
|
234
|
+
export type WorkerGuardOverride = {
|
|
235
|
+
[K in WorkerKillId]?: WorkerGuardShapes[K];
|
|
236
|
+
} & {
|
|
237
|
+
carryForward?: boolean;
|
|
238
|
+
};
|
|
239
|
+
/** The shipped `detector` half of the `loop` row: the read-only research/impl guard. */
|
|
240
|
+
export declare const DEFAULT_LOOP_DETECTOR: {
|
|
241
|
+
readonly window: 20;
|
|
242
|
+
readonly threshold: 5;
|
|
243
|
+
readonly pathThreshold: 5;
|
|
244
|
+
};
|
|
245
|
+
/**
|
|
246
|
+
* The shipped `progress` half of the `loop` row.
|
|
247
|
+
*
|
|
248
|
+
* Exported for the tests that isolate ONE of the two runaway rules. The row is
|
|
249
|
+
* whole-row-overridable on purpose, so turning the argument detector off means
|
|
250
|
+
* restating the result detector; naming the default here keeps that honest
|
|
251
|
+
* instead of tempting a deep-partial that would let a test silently disable both.
|
|
252
|
+
*/
|
|
253
|
+
export declare const DEFAULT_LOOP_PROGRESS: {
|
|
254
|
+
readonly limit: 8;
|
|
255
|
+
readonly churnFactor: 2;
|
|
256
|
+
};
|
|
257
|
+
export type WorkerProfileId = 'research' | 'gate' | 'adhoc';
|
|
258
|
+
/**
|
|
259
|
+
* The facts a profile needs that are NOT policy: user config, and which of the
|
|
260
|
+
* four research workers is the docs-capable one.
|
|
261
|
+
*/
|
|
262
|
+
export interface WorkerPolicyInputs {
|
|
263
|
+
/** gate: `config.requestTimeoutMs`. */
|
|
264
|
+
commandTimeoutMs?: number;
|
|
265
|
+
/** gate: `config.streamInactivityMs`. */
|
|
266
|
+
streamInactivityMs?: number;
|
|
267
|
+
/** research: only `worker:apis` fans out, so only it can be scaled. */
|
|
268
|
+
fanoutBounded?: boolean;
|
|
269
|
+
/** research: the A/B levers' env reader. Injectable for tests. */
|
|
270
|
+
env?: (key: string) => string | undefined;
|
|
271
|
+
}
|
|
272
|
+
export interface WorkerProfile {
|
|
273
|
+
id: WorkerProfileId;
|
|
274
|
+
/** Why THIS child's guards differ. The prose no call site was carrying. */
|
|
275
|
+
why: string;
|
|
276
|
+
resolve: (inputs: WorkerPolicyInputs) => WorkerGuardPolicy;
|
|
277
|
+
}
|
|
278
|
+
export declare const WORKER_PROFILES: {
|
|
279
|
+
readonly research: {
|
|
280
|
+
readonly id: "research";
|
|
281
|
+
readonly why: string;
|
|
282
|
+
readonly resolve: (inputs: WorkerPolicyInputs) => {
|
|
283
|
+
guards: WorkerGuards;
|
|
284
|
+
carryForward: boolean;
|
|
285
|
+
};
|
|
286
|
+
};
|
|
287
|
+
readonly gate: {
|
|
288
|
+
readonly id: "gate";
|
|
289
|
+
readonly why: string;
|
|
290
|
+
readonly resolve: (inputs: WorkerPolicyInputs) => {
|
|
291
|
+
guards: WorkerGuards;
|
|
292
|
+
carryForward: false;
|
|
293
|
+
};
|
|
294
|
+
};
|
|
295
|
+
readonly adhoc: {
|
|
296
|
+
readonly id: "adhoc";
|
|
297
|
+
readonly why: string;
|
|
298
|
+
readonly resolve: () => {
|
|
299
|
+
guards: WorkerGuards;
|
|
300
|
+
carryForward: false;
|
|
301
|
+
};
|
|
302
|
+
};
|
|
303
|
+
};
|
|
304
|
+
/** Resolve one profile. The only way a caller should obtain a policy. */
|
|
305
|
+
export declare function workerPolicy(id: WorkerProfileId, inputs?: WorkerPolicyInputs): WorkerGuardPolicy;
|
|
306
|
+
/**
|
|
307
|
+
* Lay whole rows over a resolved policy.
|
|
308
|
+
*
|
|
309
|
+
* For tests and A/B harnesses ONLY. Production code names a profile: an override
|
|
310
|
+
* at a production call site is the exact "hand-pick a subset" this module exists
|
|
311
|
+
* to stop, and `worker-profiles.test.ts` fails the build if one appears.
|
|
312
|
+
*/
|
|
313
|
+
export declare function applyOverride(policy: WorkerGuardPolicy, override: WorkerGuardOverride | undefined): WorkerGuardPolicy;
|
|
314
|
+
export {};
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The GUARD POLICY each kind of worker child runs under, keyed on the ways it
|
|
3
|
+
* can die.
|
|
4
|
+
*
|
|
5
|
+
* WHY IT EXISTS. `RunWorkerInput` carried ten guard knobs in four different
|
|
6
|
+
* shapes — two bare millisecond numbers, three `{...} | false` unions, an
|
|
7
|
+
* optional object, a boolean and two counts — and three production callers each
|
|
8
|
+
* hand-picked a different subset of them:
|
|
9
|
+
*
|
|
10
|
+
* gate-child.ts timeoutMs 0, a per-command watchdog, a stream watchdog,
|
|
11
|
+
* and the path rule disabled. Everything else default.
|
|
12
|
+
* research-worker.ts a progress deadline and two off-by-default A/B levers.
|
|
13
|
+
* NO command watchdog, NO stream watchdog. Everything else
|
|
14
|
+
* default.
|
|
15
|
+
* pi-worker.ts nothing at all — every default, silently.
|
|
16
|
+
*
|
|
17
|
+
* So "a gate child runs unbounded but with a per-command watchdog; a research
|
|
18
|
+
* worker is the reverse" existed only as three option literals in three files,
|
|
19
|
+
* and the reasoning was attached to whichever line happened to need defending.
|
|
20
|
+
* `gate-child.ts` explained why it disables the path rule and said nothing about
|
|
21
|
+
* why it takes no progress deadline. Nothing anywhere said that the ad-hoc
|
|
22
|
+
* `pi-worker` tool is the strictest-clocked of the three. That was not a
|
|
23
|
+
* decision; it was the residue of never having had a place to write one down.
|
|
24
|
+
*
|
|
25
|
+
* WHY IT IS KEYED ON `WorkerKillId`. A guard exists to prevent a specific way a
|
|
26
|
+
* child can die, so the roster of deaths (`worker-kill.ts`) is the correct key —
|
|
27
|
+
* the same argument that roster makes for kill CAUSES, one level up. The mapped
|
|
28
|
+
* type means a tenth cause cannot be added to `WORKER_KILLS` without every
|
|
29
|
+
* profile deciding what to do about it, and it means the three causes with no
|
|
30
|
+
* dial say so in the table (`null`) instead of being absent from it.
|
|
31
|
+
*
|
|
32
|
+
* The key does NOT partition the knobs one-per-row, and pretending otherwise
|
|
33
|
+
* would be the lie:
|
|
34
|
+
*
|
|
35
|
+
* `worker-timeout` holds THREE — the cap, the progress ceiling that turns the
|
|
36
|
+
* cap from "time allowed" into "time allowed without progress", and the
|
|
37
|
+
* fan-out extension. All three move the same deadline; splitting them across
|
|
38
|
+
* rows would let a profile set a ceiling for a cap it disabled.
|
|
39
|
+
*
|
|
40
|
+
* `loop` holds TWO detectors. `StallDetector`'s hit IS a `LoopHit` with
|
|
41
|
+
* `.stall` set (child-process.ts: "so a stall rides the kill/restart plumbing
|
|
42
|
+
* the loop hit already has"), and the restart ladder has ONE rule for both.
|
|
43
|
+
* One cause, one row.
|
|
44
|
+
*
|
|
45
|
+
* WHAT IS DELIBERATELY NOT UNIFIED.
|
|
46
|
+
*
|
|
47
|
+
* `carryForward` is not a row. It is one switch over the whole run, and WHICH
|
|
48
|
+
* causes honour it is already decided by `CARRY_FORWARD_IDS`, derived from the
|
|
49
|
+
* roster. A per-cause row here would be a second copy of that set, free to
|
|
50
|
+
* disagree with it.
|
|
51
|
+
*
|
|
52
|
+
* The reasoning group is not the profile. `pi-worker.ts` runs `adhoc` guards
|
|
53
|
+
* but `groupThinkingArgs('research')`, on purpose. Guards answer "how may this
|
|
54
|
+
* child die"; `thinking` answers "how hard may it think". Folding them would
|
|
55
|
+
* silently re-level a gate child, which is the exact mistake
|
|
56
|
+
* `RunWorkerInput.thinking`'s comment already records.
|
|
57
|
+
*
|
|
58
|
+
* `projectDocsBudget()` (the CAP arm, research-fanout-budget.ts) stays out. It
|
|
59
|
+
* bounds what a worker ASKS FOR, via its prompt and its tool, not how it dies.
|
|
60
|
+
*
|
|
61
|
+
* `RESTART_ORDER` and `FAILURE_ORDER` are untouched. This is a third view of
|
|
62
|
+
* the same key, not a merge of the two orderings.
|
|
63
|
+
*/
|
|
64
|
+
import { LOOP_THRESHOLD, LOOP_WINDOW, MAX_LOOP_RESTARTS } from '../task/child-runner.js';
|
|
65
|
+
import { CONTEXT_CHURN_FACTOR, NO_PROGRESS_LIMIT } from '../task/stall-detector.js';
|
|
66
|
+
import { fanoutTimeoutPolicy, workerCarryForward, workerProgressCeilingMs } from '../task/research-fanout-budget.js';
|
|
67
|
+
/**
|
|
68
|
+
* Hard wall-clock bound on a single worker run (one spawn). The exact-match
|
|
69
|
+
* LoopDetector only catches *identical* repeated tool calls; a model that
|
|
70
|
+
* thrashes with slightly-varied calls (different grep patterns each time) slips
|
|
71
|
+
* past it and would otherwise run unbounded. This is the backstop for that case:
|
|
72
|
+
* after this long with no clean exit, abort and restart with a hint. Sized well
|
|
73
|
+
* above a healthy worker's observed runtime (~25-130s on the local backend) so
|
|
74
|
+
* it never trips a legitimately slow run.
|
|
75
|
+
*/
|
|
76
|
+
export const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
|
|
77
|
+
/**
|
|
78
|
+
* Output-stall window before the dead-backend probe fires (mx5 run 7: model
|
|
79
|
+
* server died mid-gate-child, the child hung MUTE for 64 minutes). This is NOT
|
|
80
|
+
* a wall-clock cap — output progress resets it, and even a fully stalled child
|
|
81
|
+
* is only killed when the model endpoint is actually unreachable. Sized so a
|
|
82
|
+
* long local prompt-processing pass (minutes of legitimate silence, server
|
|
83
|
+
* alive) just gets probed and waits on.
|
|
84
|
+
*/
|
|
85
|
+
export const STALL_AFTER_MS = 180_000;
|
|
86
|
+
/** The shipped `detector` half of the `loop` row: the read-only research/impl guard. */
|
|
87
|
+
export const DEFAULT_LOOP_DETECTOR = {
|
|
88
|
+
window: LOOP_WINDOW,
|
|
89
|
+
threshold: LOOP_THRESHOLD,
|
|
90
|
+
pathThreshold: LOOP_THRESHOLD
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* The shipped `progress` half of the `loop` row.
|
|
94
|
+
*
|
|
95
|
+
* Exported for the tests that isolate ONE of the two runaway rules. The row is
|
|
96
|
+
* whole-row-overridable on purpose, so turning the argument detector off means
|
|
97
|
+
* restating the result detector; naming the default here keeps that honest
|
|
98
|
+
* instead of tempting a deep-partial that would let a test silently disable both.
|
|
99
|
+
*/
|
|
100
|
+
export const DEFAULT_LOOP_PROGRESS = {
|
|
101
|
+
limit: NO_PROGRESS_LIMIT,
|
|
102
|
+
churnFactor: CONTEXT_CHURN_FACTOR
|
|
103
|
+
};
|
|
104
|
+
/**
|
|
105
|
+
* Every guard at its default. `adhoc` IS this; the other two are this plus a
|
|
106
|
+
* named departure, so a diff between two profiles is a short list rather than a
|
|
107
|
+
* re-reading of two literals.
|
|
108
|
+
*/
|
|
109
|
+
function baseGuards() {
|
|
110
|
+
return {
|
|
111
|
+
stalled: { afterMs: STALL_AFTER_MS, probe: null },
|
|
112
|
+
'command-timeout': 0,
|
|
113
|
+
'stream-stall': 0,
|
|
114
|
+
'worker-timeout': {
|
|
115
|
+
timeoutMs: RESEARCH_WORKER_TIMEOUT_MS,
|
|
116
|
+
progressCeilingMs: null,
|
|
117
|
+
fanout: null
|
|
118
|
+
},
|
|
119
|
+
'connection-error': MAX_LOOP_RESTARTS,
|
|
120
|
+
loop: { detector: { ...DEFAULT_LOOP_DETECTOR }, progress: { ...DEFAULT_LOOP_PROGRESS } },
|
|
121
|
+
'leaked-tool-call': null,
|
|
122
|
+
aborted: null,
|
|
123
|
+
exit: null
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
export const WORKER_PROFILES = {
|
|
127
|
+
research: {
|
|
128
|
+
id: 'research',
|
|
129
|
+
why: 'The four read-only survey workers. Their fault is over-EXPLORATION, not '
|
|
130
|
+
+ 'a hung command: they get no bash, so no tool call can block forever, '
|
|
131
|
+
+ 'and the command and stream watchdogs stay off. What they do hit is the '
|
|
132
|
+
+ 'clock — mx5 run 18 measured r(project lookups, wall clock) = 0.909, '
|
|
133
|
+
+ 'with every worker past 46 lookups burning all three attempts. Hence the '
|
|
134
|
+
+ 'progress deadline (nexttask 9, 42 trials/arm: worker-timeout restarts '
|
|
135
|
+
+ '22/24 -> 0/24): the 240s cap now means 240s WITHOUT PROGRESS, up to a '
|
|
136
|
+
+ '20-minute backstop. The fan-out extension and carry-forward remain OFF '
|
|
137
|
+
+ 'unless their env var is set — see research-fanout-budget.ts.',
|
|
138
|
+
resolve: inputs => {
|
|
139
|
+
const guards = baseGuards();
|
|
140
|
+
guards['worker-timeout'] = {
|
|
141
|
+
timeoutMs: RESEARCH_WORKER_TIMEOUT_MS,
|
|
142
|
+
progressCeilingMs: workerProgressCeilingMs(inputs.env),
|
|
143
|
+
fanout: inputs.fanoutBounded === true ? fanoutTimeoutPolicy(inputs.env) : null
|
|
144
|
+
};
|
|
145
|
+
return { guards, carryForward: workerCarryForward(inputs.env) };
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
gate: {
|
|
149
|
+
id: 'gate',
|
|
150
|
+
why: 'The post-implementation verify/enforce/critique children. They WRITE, '
|
|
151
|
+
+ 'and they legitimately read and edit the same file many times, so the '
|
|
152
|
+
+ 'research guards mislabel the job as a runaway and kill good work (mx5 '
|
|
153
|
+
+ 'TASK_0002). Two departures follow from that. The wall clock is OFF — '
|
|
154
|
+
+ 'these passes must be allowed to finish however long they take. And the '
|
|
155
|
+
+ 'path-revisit rule is disabled (pathThreshold Infinity), leaving only '
|
|
156
|
+
+ 'the exact-match rule, so revisiting one file never trips but a '
|
|
157
|
+
+ 'literally-identical call repeated past threshold still does. What '
|
|
158
|
+
+ 'replaces the wall clock is the pair the research workers do not need: '
|
|
159
|
+
+ 'a per-command watchdog, because a gate child HAS bash and a `bun run '
|
|
160
|
+
+ 'dev` it forgot to bound blocks it forever while the stall probe reads '
|
|
161
|
+
+ 'the live model endpoint as proof of life; and a stream watchdog, for '
|
|
162
|
+
+ "run 14's three hangs on a HEALTHY backend. Both take their ceilings "
|
|
163
|
+
+ 'from user config, so they are inputs, not policy.',
|
|
164
|
+
resolve: inputs => {
|
|
165
|
+
const guards = baseGuards();
|
|
166
|
+
guards['command-timeout'] = inputs.commandTimeoutMs ?? 0;
|
|
167
|
+
guards['stream-stall'] = inputs.streamInactivityMs ?? 0;
|
|
168
|
+
guards['worker-timeout'] = { timeoutMs: 0, progressCeilingMs: null, fanout: null };
|
|
169
|
+
guards.loop = {
|
|
170
|
+
...guards.loop,
|
|
171
|
+
detector: { ...DEFAULT_LOOP_DETECTOR, pathThreshold: Number.POSITIVE_INFINITY }
|
|
172
|
+
};
|
|
173
|
+
return { guards, carryForward: false };
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
adhoc: {
|
|
177
|
+
id: 'adhoc',
|
|
178
|
+
why: 'The model-dispatched `pi-worker` tool. Every guard at its default, and '
|
|
179
|
+
+ 'this row exists so that is a DECISION rather than the absence of one — '
|
|
180
|
+
+ 'the call site passed nothing, and nobody could see what it therefore '
|
|
181
|
+
+ 'got. What the table now makes visible is an asymmetry: this is the '
|
|
182
|
+
+ 'strictest-clocked of the three children. It runs a FIXED 240s cap, '
|
|
183
|
+
+ 'because `progressCeilingMs` is null and the deadline re-arm is inert '
|
|
184
|
+
+ 'without one, while a research worker doing the same read-only '
|
|
185
|
+
+ 'exploration gets 240s WITHOUT PROGRESS up to 20 minutes. That '
|
|
186
|
+
+ 'difference is preserved exactly here and is NOT defended: the progress '
|
|
187
|
+
+ 'deadline was measured for the research workers (nexttask 9) and has '
|
|
188
|
+
+ 'never been measured for this tool. It is a candidate, not a bug.',
|
|
189
|
+
resolve: () => ({ guards: baseGuards(), carryForward: false })
|
|
190
|
+
// `as const satisfies`, not an annotation — the same reason RESTART_ORDER
|
|
191
|
+
// gives: an annotation widens each row back to `WorkerProfile`, and the
|
|
192
|
+
// `why` strings and literal ids stop being visible to a reader or a test.
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
/** Resolve one profile. The only way a caller should obtain a policy. */
|
|
196
|
+
export function workerPolicy(id, inputs = {}) {
|
|
197
|
+
return WORKER_PROFILES[id].resolve(inputs);
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Lay whole rows over a resolved policy.
|
|
201
|
+
*
|
|
202
|
+
* For tests and A/B harnesses ONLY. Production code names a profile: an override
|
|
203
|
+
* at a production call site is the exact "hand-pick a subset" this module exists
|
|
204
|
+
* to stop, and `worker-profiles.test.ts` fails the build if one appears.
|
|
205
|
+
*/
|
|
206
|
+
export function applyOverride(policy, override) {
|
|
207
|
+
if (override === undefined)
|
|
208
|
+
return policy;
|
|
209
|
+
const { carryForward, ...rows } = override;
|
|
210
|
+
// Present-but-`undefined` is DROPPED, not laid down. A conditional row is
|
|
211
|
+
// the natural way to write a swept arm — `{'command-timeout': on ? ms :
|
|
212
|
+
// undefined}` — and a plain spread would put `undefined` into the policy,
|
|
213
|
+
// which either disarms the guard silently or throws on `clock.timeoutMs`.
|
|
214
|
+
// The repo has no `exactOptionalPropertyTypes`, so the compiler allows it.
|
|
215
|
+
const set = Object.fromEntries(Object.entries(rows).filter(([, v]) => v !== undefined));
|
|
216
|
+
return {
|
|
217
|
+
guards: { ...policy.guards, ...set },
|
|
218
|
+
carryForward: carryForward ?? policy.carryForward
|
|
219
|
+
};
|
|
220
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.25",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|