@mjasnikovs/pi-task 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/accept-debt.d.ts +15 -1
- package/dist/task/accept-debt.js +18 -0
- package/dist/task/enforce-attribution.d.ts +128 -0
- package/dist/task/enforce-attribution.js +213 -0
- package/dist/task/final-gate.d.ts +3 -1
- package/dist/task/final-gate.js +12 -4
- package/dist/task/gate-deps.js +48 -2
- package/dist/task/phases.js +62 -1
- package/dist/task/repo-health-check.js +7 -5
- package/dist/task/research-fanout-budget.d.ts +121 -0
- package/dist/task/research-fanout-budget.js +148 -0
- package/dist/task/runner-resolve.d.ts +25 -0
- package/dist/task/runner-resolve.js +31 -0
- package/dist/task/task-gates.d.ts +22 -5
- package/dist/task/task-gates.js +73 -6
- package/dist/workers/pi-worker-core.d.ts +136 -2
- package/dist/workers/pi-worker-core.js +274 -34
- package/dist/workers/pi-worker-docs.js +18 -0
- package/package.json +1 -1
|
@@ -2,6 +2,35 @@ import { type ContextSnapshot, type SpawnFn } from '../shared/child-process.js';
|
|
|
2
2
|
import { type LoopHit } from '../task/loop-detector.js';
|
|
3
3
|
/** True when a tool call retrieves content an APIS entry could be grounded in. */
|
|
4
4
|
export declare function isGroundingRetrieval(toolName: string): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Does this partial output carry ANSWER CONTENT, or is it the model clearing its
|
|
7
|
+
* throat?
|
|
8
|
+
*
|
|
9
|
+
* Salvage originally kept the LONGEST partial, which is not the same question. On
|
|
10
|
+
* the live carry arm, TASK_0020 and TASK_0021 both timed out on all three
|
|
11
|
+
* attempts and salvage shipped this as the section:
|
|
12
|
+
*
|
|
13
|
+
* "Now let me get more details on the specific APIs and components I need:"
|
|
14
|
+
*
|
|
15
|
+
* — a preamble sentence, which beats an empty string on length and carries
|
|
16
|
+
* nothing. Both trials scored 2 entries and DEGRADED, against 22 and 5 for the
|
|
17
|
+
* same fixtures in baseline.
|
|
18
|
+
*
|
|
19
|
+
* A research worker's answer is a list of lines that each name something and
|
|
20
|
+
* describe it. The test is therefore structural, not lexical: at least two lines
|
|
21
|
+
* that look like entries — a name, then a gap, then a description. Prose wraps
|
|
22
|
+
* at no particular column and does not repeat that shape.
|
|
23
|
+
*/
|
|
24
|
+
export declare function hasAnswerContent(text: string): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Frame a discarded attempt's output as work already done.
|
|
27
|
+
*
|
|
28
|
+
* Kept deliberately blunt about status. Appended text loses to preserved text
|
|
29
|
+
* when the two disagree, so the carry must not read as a finished answer the
|
|
30
|
+
* model can simply re-emit: it is labelled partial, unverified, and truncated
|
|
31
|
+
* when it is.
|
|
32
|
+
*/
|
|
33
|
+
export declare function formatCarryForward(text: string): string | null;
|
|
5
34
|
export interface RunWorkerInput {
|
|
6
35
|
prompt: string;
|
|
7
36
|
cwd: string;
|
|
@@ -88,15 +117,90 @@ export interface RunWorkerInput {
|
|
|
88
117
|
streamInactivityMs?: number;
|
|
89
118
|
/** Backoff sleep, injectable so tests don't wait out the real delays. */
|
|
90
119
|
sleepFor?: (ms: number) => Promise<void>;
|
|
120
|
+
/**
|
|
121
|
+
* SCALE arm of nexttask 5B — OFF unless set, and set only by the harness that
|
|
122
|
+
* is measuring it (src/task/research-fanout-budget.ts explains both arms).
|
|
123
|
+
* Each project-source `pi-worker-docs` call pushes this attempt's deadline out
|
|
124
|
+
* by `perLookupMs`, never past `ceilingMs` from the attempt's start: a worker
|
|
125
|
+
* that is making retrieval progress is not killed for making it, while a
|
|
126
|
+
* worker that is thrashing still hits a hard bound.
|
|
127
|
+
*/
|
|
128
|
+
fanoutTimeout?: {
|
|
129
|
+
perLookupMs: number;
|
|
130
|
+
ceilingMs: number;
|
|
131
|
+
};
|
|
132
|
+
/**
|
|
133
|
+
* Absolute backstop that turns `timeoutMs` from "total time allowed" into
|
|
134
|
+
* "time allowed WITHOUT PROGRESS". A tool call or a line of output re-arms
|
|
135
|
+
* the deadline; only a worker that goes quiet for `timeoutMs` — or exceeds
|
|
136
|
+
* this ceiling outright — is killed.
|
|
137
|
+
*
|
|
138
|
+
* This is the difference between "took too long" and "stopped working". The
|
|
139
|
+
* first is a property of the machine (a slower local model, a bigger file)
|
|
140
|
+
* and must not cost the user their answer; the second is a real fault, and
|
|
141
|
+
* one the output-stall probe already catches on its own terms.
|
|
142
|
+
*/
|
|
143
|
+
progressTimeoutCeilingMs?: number;
|
|
144
|
+
/**
|
|
145
|
+
* Carry a killed attempt's findings into the re-spawn, and never return less
|
|
146
|
+
* than the best attempt produced. OFF by default so the shipped path is
|
|
147
|
+
* unchanged while the A/B runs — see src/task/research-fanout-budget.ts.
|
|
148
|
+
*/
|
|
149
|
+
carryForward?: boolean;
|
|
150
|
+
/**
|
|
151
|
+
* Called when a carried-forward partial is INJECTED into an attempt's prompt
|
|
152
|
+
* — once per attempt that receives one. Distinct from `onRestart`, which says
|
|
153
|
+
* an attempt was thrown away; this says the next one was actually handed its
|
|
154
|
+
* findings. The two are separately observable because they can diverge: a
|
|
155
|
+
* restart whose partial had no answer content injects nothing.
|
|
156
|
+
*/
|
|
157
|
+
onCarryForward?: (info: {
|
|
158
|
+
attempt: number;
|
|
159
|
+
chars: number;
|
|
160
|
+
promptCharsBefore: number;
|
|
161
|
+
}) => void;
|
|
162
|
+
/**
|
|
163
|
+
* Called once per DISCARDED attempt, at the moment the worker decides to
|
|
164
|
+
* re-spawn — the only window in which a restart is observable at all.
|
|
165
|
+
*
|
|
166
|
+
* WHY: every restart branch below throws away a whole attempt's wall clock
|
|
167
|
+
* along with its text, and `waitMs`/`workMs` describe the FINAL attempt only.
|
|
168
|
+
* With no hook here those attempts were structurally invisible: mx5 run 18
|
|
169
|
+
* burned 30 wall-clock timeouts / 120 minutes of compute that appeared in no
|
|
170
|
+
* log and no timing widget, and 21 of the 23 affected workers reported
|
|
171
|
+
* `exit=0` — clean successes as far as the run could tell. The discrepancy
|
|
172
|
+
* was only recoverable by subtracting reported wait+work from the timestamps
|
|
173
|
+
* of the `start` and `done` lines around it.
|
|
174
|
+
*/
|
|
175
|
+
onRestart?: (restart: WorkerRestart) => void;
|
|
91
176
|
/**
|
|
92
177
|
* Connection-error restart budget. Defaults to MAX_LOOP_RESTARTS, and even
|
|
93
|
-
* then the SHARED
|
|
178
|
+
* then the SHARED restart counter is what actually binds — a worker that
|
|
94
179
|
* already spent the budget looping does not get extra lives here. 0 turns the
|
|
95
180
|
* retry off, which is how scripts/connection-retry-ab.ts gets a baseline arm
|
|
96
181
|
* out of a build that already ships the retry.
|
|
97
182
|
*/
|
|
98
183
|
connectionRetries?: number;
|
|
99
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* Why an attempt was thrown away. One value per restart branch in runWorker, so
|
|
187
|
+
* a log line naming the reason points at exactly one piece of code.
|
|
188
|
+
*/
|
|
189
|
+
export type WorkerRestartReason = 'loop' | 'command-timeout' | 'stream-stall' | 'worker-timeout' | 'connection-error' | 'leaked-tool-call';
|
|
190
|
+
/** One DISCARDED attempt: its cause and the wall clock it consumed and lost. */
|
|
191
|
+
export interface WorkerRestart {
|
|
192
|
+
/** 1-based number of the attempt being discarded (the 1st restart ends attempt 1). */
|
|
193
|
+
attempt: number;
|
|
194
|
+
reason: WorkerRestartReason;
|
|
195
|
+
/** Wall clock this attempt spent before it was killed — time with no output. */
|
|
196
|
+
wallMs: number;
|
|
197
|
+
/** The discarded attempt's own spawn → first-byte split. */
|
|
198
|
+
waitMs: number;
|
|
199
|
+
/** The discarded attempt's own first-byte → exit split. */
|
|
200
|
+
workMs: number;
|
|
201
|
+
/** Reason-specific diagnosis: the looping call, the hung tool, the error text. */
|
|
202
|
+
detail?: string;
|
|
203
|
+
}
|
|
100
204
|
export interface RunWorkerResult {
|
|
101
205
|
text: string;
|
|
102
206
|
exitCode: number;
|
|
@@ -130,14 +234,44 @@ export interface RunWorkerResult {
|
|
|
130
234
|
* Milliseconds between spawn and the child's first stdout chunk. When
|
|
131
235
|
* multiple workers run concurrently and the upstream model API queues at
|
|
132
236
|
* some concurrency cap, this is the queue-wait portion of the run.
|
|
237
|
+
*
|
|
238
|
+
* FINAL ATTEMPT ONLY — a restarted attempt's clock is discarded with its
|
|
239
|
+
* text. `waitMs + workMs` is therefore NOT the worker's wall clock whenever
|
|
240
|
+
* `attempts > 1`; `totalWallMs` is.
|
|
133
241
|
*/
|
|
134
242
|
waitMs: number;
|
|
135
243
|
/**
|
|
136
244
|
* Milliseconds between first stdout chunk and process exit — the
|
|
137
245
|
* generation/tool-call portion, independent of queue wait. Equals total
|
|
138
|
-
* elapsed when the child never produced output.
|
|
246
|
+
* elapsed when the child never produced output. Final attempt only, same as
|
|
247
|
+
* `waitMs`.
|
|
139
248
|
*/
|
|
140
249
|
workMs: number;
|
|
250
|
+
/**
|
|
251
|
+
* How many attempts (spawns) this call made, including the one that produced
|
|
252
|
+
* `text`. 1 for a worker that ran clean. Always `restarts.length + 1`.
|
|
253
|
+
*/
|
|
254
|
+
attempts: number;
|
|
255
|
+
/**
|
|
256
|
+
* The worker's TRUE wall clock: entry to return, spanning every discarded
|
|
257
|
+
* attempt and every connection backoff. `totalWallMs - waitMs - workMs` is
|
|
258
|
+
* the time this worker spent on output that was thrown away.
|
|
259
|
+
*/
|
|
260
|
+
totalWallMs: number;
|
|
261
|
+
/**
|
|
262
|
+
* One entry per discarded attempt, in order — empty on a clean run. The only
|
|
263
|
+
* record that a restart happened: the returned `exitCode`/`text` describe the
|
|
264
|
+
* final attempt and look identical whether it was the first or the third.
|
|
265
|
+
*/
|
|
266
|
+
restarts: ReadonlyArray<WorkerRestart>;
|
|
267
|
+
/**
|
|
268
|
+
* True when `text` came from a DISCARDED attempt rather than the final one,
|
|
269
|
+
* because the final attempt returned less. The answer is real output the
|
|
270
|
+
* worker produced, but it was cut off mid-flight, so it is likelier to be
|
|
271
|
+
* incomplete than a clean return — callers that grade completeness should
|
|
272
|
+
* treat it as partial rather than as a finished answer.
|
|
273
|
+
*/
|
|
274
|
+
salvagedFromDiscardedAttempt: boolean;
|
|
141
275
|
/**
|
|
142
276
|
* How many GROUNDING retrieval tool calls the FINAL attempt made — the calls
|
|
143
277
|
* that returned content an APIS entry could be cited from (see
|
|
@@ -66,6 +66,91 @@ const STALL_AFTER_MS = 180_000;
|
|
|
66
66
|
const WORKER_TIMEOUT_HINT = '[SYSTEM NOTE: Your previous attempt ran out of time before answering — you '
|
|
67
67
|
+ 'were exploring too long. Be decisive: do the minimum reads/greps needed, '
|
|
68
68
|
+ 'then write your answer now. Do not re-explore ground you have already covered.]';
|
|
69
|
+
/**
|
|
70
|
+
* How much of a discarded attempt's answer is carried into the next one.
|
|
71
|
+
*
|
|
72
|
+
* A restart used to hand the re-spawn nothing but a hint — which is why
|
|
73
|
+
* WORKER_TIMEOUT_HINT above can tell a worker "do not re-explore ground you have
|
|
74
|
+
* already covered" while giving it no record of what that ground was. It could
|
|
75
|
+
* not comply. mx5 run 18 shows the cost: on tasks with >=46 project-source
|
|
76
|
+
* lookups, 5 of 5 workers burned the FULL restart budget, because every attempt
|
|
77
|
+
* re-read the same files against the same clock and died in the same place.
|
|
78
|
+
*
|
|
79
|
+
* Carrying the partial answer forward is what makes a restart converge instead
|
|
80
|
+
* of repeat. The risk it takes is real and is the thing the A/B measures: a
|
|
81
|
+
* half-written or speculative entry, replayed under "already established", is
|
|
82
|
+
* exactly how a fabrication gets laundered into a final answer. That is what the
|
|
83
|
+
* ungrounded-symbol and anti-synthesis guards are pointed at, so the carry is
|
|
84
|
+
* framed as findings to VERIFY-or-DROP rather than as settled fact.
|
|
85
|
+
*/
|
|
86
|
+
const CARRY_FORWARD_LIMIT = 24_000;
|
|
87
|
+
/**
|
|
88
|
+
* Restart reasons whose partial output is worth keeping.
|
|
89
|
+
*
|
|
90
|
+
* A clock kill (`worker-timeout`), a hung tool (`command-timeout`), an idle
|
|
91
|
+
* stream (`stream-stall`) and a dropped socket (`connection-error`) all discard
|
|
92
|
+
* work the model genuinely did. A loop kill and a leaked tool call do not — the
|
|
93
|
+
* first is by definition the same call repeated, the second is malformed
|
|
94
|
+
* protocol text, and replaying either would feed the failure back to itself.
|
|
95
|
+
*/
|
|
96
|
+
const CARRY_FORWARD_REASONS = new Set([
|
|
97
|
+
'worker-timeout',
|
|
98
|
+
'command-timeout',
|
|
99
|
+
'stream-stall',
|
|
100
|
+
'connection-error'
|
|
101
|
+
]);
|
|
102
|
+
/**
|
|
103
|
+
* Does this partial output carry ANSWER CONTENT, or is it the model clearing its
|
|
104
|
+
* throat?
|
|
105
|
+
*
|
|
106
|
+
* Salvage originally kept the LONGEST partial, which is not the same question. On
|
|
107
|
+
* the live carry arm, TASK_0020 and TASK_0021 both timed out on all three
|
|
108
|
+
* attempts and salvage shipped this as the section:
|
|
109
|
+
*
|
|
110
|
+
* "Now let me get more details on the specific APIs and components I need:"
|
|
111
|
+
*
|
|
112
|
+
* — a preamble sentence, which beats an empty string on length and carries
|
|
113
|
+
* nothing. Both trials scored 2 entries and DEGRADED, against 22 and 5 for the
|
|
114
|
+
* same fixtures in baseline.
|
|
115
|
+
*
|
|
116
|
+
* A research worker's answer is a list of lines that each name something and
|
|
117
|
+
* describe it. The test is therefore structural, not lexical: at least two lines
|
|
118
|
+
* that look like entries — a name, then a gap, then a description. Prose wraps
|
|
119
|
+
* at no particular column and does not repeat that shape.
|
|
120
|
+
*/
|
|
121
|
+
export function hasAnswerContent(text) {
|
|
122
|
+
const entryish = text
|
|
123
|
+
.split('\n')
|
|
124
|
+
.map(l => l.replace(/^\s*(?:[-*•]|\d+[.)])\s+/, '').trim())
|
|
125
|
+
.filter(l => /^\S.*?(?:\s{2,}|\s+[—–-]\s+)\S/.test(l) && !/[.:]$/.test(l));
|
|
126
|
+
return entryish.length >= 2;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Frame a discarded attempt's output as work already done.
|
|
130
|
+
*
|
|
131
|
+
* Kept deliberately blunt about status. Appended text loses to preserved text
|
|
132
|
+
* when the two disagree, so the carry must not read as a finished answer the
|
|
133
|
+
* model can simply re-emit: it is labelled partial, unverified, and truncated
|
|
134
|
+
* when it is.
|
|
135
|
+
*/
|
|
136
|
+
export function formatCarryForward(text) {
|
|
137
|
+
const body = text.trim();
|
|
138
|
+
if (body.length === 0)
|
|
139
|
+
return null;
|
|
140
|
+
const truncated = body.length > CARRY_FORWARD_LIMIT;
|
|
141
|
+
// Keep the TAIL: the model writes progressively, so the end of a partial
|
|
142
|
+
// answer is the furthest it got and the best statement of where to resume.
|
|
143
|
+
const kept = truncated ? body.slice(body.length - CARRY_FORWARD_LIMIT) : body;
|
|
144
|
+
return ('[WORK ALREADY DONE — from your previous attempt, which was cut off before '
|
|
145
|
+
+ 'it could answer. These findings came from real reads of this project, so '
|
|
146
|
+
+ 'do NOT gather them again; spend your time on what is still missing. They '
|
|
147
|
+
+ 'are PARTIAL and UNVERIFIED: keep every item you can confirm, and drop any '
|
|
148
|
+
+ 'item you cannot — do not carry an unconfirmed item into your answer, and '
|
|
149
|
+
+ 'do not treat this as your answer.'
|
|
150
|
+
+ (truncated ? ' (Earlier portion omitted; this is the most recent part.)' : '')
|
|
151
|
+
+ ']\n'
|
|
152
|
+
+ kept);
|
|
153
|
+
}
|
|
69
154
|
const defaultSleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
70
155
|
/**
|
|
71
156
|
* Combine an external abort signal with an internal wall-clock timeout into one
|
|
@@ -73,17 +158,25 @@ const defaultSleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
|
73
158
|
* only when the timer fired (not when the external signal aborted), so the caller
|
|
74
159
|
* can restart on a timeout but not on a user cancel.
|
|
75
160
|
*/
|
|
76
|
-
function workerTimeout(external, ms
|
|
161
|
+
function workerTimeout(external, ms,
|
|
162
|
+
/**
|
|
163
|
+
* Absolute backstop for the progress-based deadline. When set, `ms` stops
|
|
164
|
+
* meaning "total time allowed" and starts meaning "time allowed WITHOUT
|
|
165
|
+
* PROGRESS"; this is the hard limit no amount of progress can pass.
|
|
166
|
+
*/
|
|
167
|
+
absoluteCeilingMs) {
|
|
77
168
|
const ctrl = new AbortController();
|
|
78
169
|
let timedOut = false;
|
|
170
|
+
const armed = ms > 0 && Number.isFinite(ms);
|
|
171
|
+
const started = Date.now();
|
|
172
|
+
let deadline = started + ms;
|
|
173
|
+
const fire = () => {
|
|
174
|
+
timedOut = true;
|
|
175
|
+
ctrl.abort();
|
|
176
|
+
};
|
|
79
177
|
// ms <= 0 (or non-finite) disables the wall-clock timeout: no timer is armed,
|
|
80
178
|
// so only the external signal can abort and timedOut() stays false forever.
|
|
81
|
-
|
|
82
|
-
setTimeout(() => {
|
|
83
|
-
timedOut = true;
|
|
84
|
-
ctrl.abort();
|
|
85
|
-
}, ms)
|
|
86
|
-
: undefined;
|
|
179
|
+
let timer = armed ? setTimeout(fire, ms) : undefined;
|
|
87
180
|
const onExternal = () => ctrl.abort();
|
|
88
181
|
if (external) {
|
|
89
182
|
if (external.aborted)
|
|
@@ -94,6 +187,41 @@ function workerTimeout(external, ms) {
|
|
|
94
187
|
return {
|
|
95
188
|
signal: ctrl.signal,
|
|
96
189
|
timedOut: () => timedOut,
|
|
190
|
+
// SCALE arm of nexttask 5B, inert unless a caller calls it: push the
|
|
191
|
+
// deadline out, never past `started + ceilingMs`. A disabled timeout
|
|
192
|
+
// (nothing armed) stays disabled — extending "never" is meaningless — and
|
|
193
|
+
// an already-fired timer is not resurrected.
|
|
194
|
+
extend: (byMs, ceilingMs) => {
|
|
195
|
+
if (!armed || timedOut || ctrl.signal.aborted)
|
|
196
|
+
return;
|
|
197
|
+
const next = Math.min(deadline + byMs, started + ceilingMs);
|
|
198
|
+
if (next <= deadline)
|
|
199
|
+
return;
|
|
200
|
+
deadline = next;
|
|
201
|
+
clearTimeout(timer);
|
|
202
|
+
timer = setTimeout(fire, Math.max(0, deadline - Date.now()));
|
|
203
|
+
},
|
|
204
|
+
// PROGRESS-BASED DEADLINE. A worker that is making tool calls and
|
|
205
|
+
// emitting text is not stuck — it is slow, and how slow is a property of
|
|
206
|
+
// the user's machine, not of the task. Killing it on total elapsed time
|
|
207
|
+
// makes answer quality depend on the hardware: the same task on a slower
|
|
208
|
+
// local model loses its work and degrades, which no per-file constant can
|
|
209
|
+
// fix. Being STUCK is already detected separately and correctly, by the
|
|
210
|
+
// output-stall probe (STALL_AFTER_MS), which resets on progress and only
|
|
211
|
+
// kills when the model endpoint is unreachable.
|
|
212
|
+
progress: () => {
|
|
213
|
+
if (absoluteCeilingMs === undefined)
|
|
214
|
+
return;
|
|
215
|
+
if (!armed || timedOut || ctrl.signal.aborted)
|
|
216
|
+
return;
|
|
217
|
+
const next = Math.min(Date.now() + ms, started + absoluteCeilingMs);
|
|
218
|
+
if (next <= deadline)
|
|
219
|
+
return;
|
|
220
|
+
deadline = next;
|
|
221
|
+
clearTimeout(timer);
|
|
222
|
+
timer = setTimeout(fire, Math.max(0, deadline - Date.now()));
|
|
223
|
+
},
|
|
224
|
+
budgetMs: () => deadline - started,
|
|
97
225
|
cleanup: () => {
|
|
98
226
|
clearTimeout(timer);
|
|
99
227
|
external?.removeEventListener('abort', onExternal);
|
|
@@ -188,19 +316,49 @@ export async function runWorker(input) {
|
|
|
188
316
|
// runPhaseWithLoopGuard: a runaway worker gets re-spawned with a corrective
|
|
189
317
|
// hint up to MAX_LOOP_RESTARTS times before we give up. Leaked tool calls
|
|
190
318
|
// keep their own MAX_LEAK_RETRIES budget below — a different failure mode.
|
|
191
|
-
let
|
|
319
|
+
let restartBudgetSpent = 0;
|
|
192
320
|
// Watchdog kills specifically — drives the ceiling halving. Kept apart from
|
|
193
|
-
// `
|
|
321
|
+
// `restartBudgetSpent` (the shared budget) so a loop-caused restart doesn't shorten
|
|
194
322
|
// the rope of a child that has never hung (see commandCeilingForAttempt).
|
|
195
323
|
let hangKills = 0;
|
|
196
324
|
// Connection-error restarts specifically — drives the backoff schedule (and
|
|
197
325
|
// lets a harness set the budget to 0 without touching the shared counter).
|
|
198
326
|
let connRetries = 0;
|
|
199
327
|
let leakRetries = 0;
|
|
328
|
+
// Entry-to-return wall clock. tAttemptStart below is per-attempt (it is what
|
|
329
|
+
// waitMs/workMs are measured from); this one is the only thing that sees the
|
|
330
|
+
// attempts that were killed and re-spawned.
|
|
331
|
+
const tRunStart = Date.now();
|
|
332
|
+
const restarts = [];
|
|
333
|
+
// The best partial answer any discarded attempt produced, RAW. Without this a
|
|
334
|
+
// restart is amnesiac: it re-reads the same files against the same clock and
|
|
335
|
+
// dies in the same place (see CARRY_FORWARD_LIMIT). Held unformatted because
|
|
336
|
+
// it has two consumers — the next attempt's prompt, which wants it wrapped in
|
|
337
|
+
// the carry-forward framing, and the final return, which must never emit that
|
|
338
|
+
// framing as if it were the worker's answer.
|
|
339
|
+
// Held in a box, not a bare `let`: the only writer is the `noteRestart`
|
|
340
|
+
// closure below, and TypeScript narrows a closure-assigned `let` back to its
|
|
341
|
+
// initialiser at the return site.
|
|
342
|
+
const salvage = { text: null };
|
|
200
343
|
for (;;) {
|
|
201
|
-
const
|
|
344
|
+
const carried = salvage.text === null ? null : formatCarryForward(salvage.text);
|
|
345
|
+
// Announce the INJECTION, not just the restart. Without this, "the carry
|
|
346
|
+
// reached the re-spawn" can only be inferred from entry counts — and
|
|
347
|
+
// inferring what a worker did from what it produced is the exact gap 5A
|
|
348
|
+
// exists to close. The prompt goes to the child on stdin, so no log
|
|
349
|
+
// downstream of here can show it.
|
|
350
|
+
if (carried !== null) {
|
|
351
|
+
input.onCarryForward?.({
|
|
352
|
+
attempt: restarts.length + 1,
|
|
353
|
+
chars: carried.length,
|
|
354
|
+
promptCharsBefore: input.prompt.length
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
const prompt = [hint, carried, input.prompt]
|
|
358
|
+
.filter((p) => p !== null)
|
|
359
|
+
.join('\n\n');
|
|
202
360
|
const invocation = getPiInvocation([...baseArgs], prompt);
|
|
203
|
-
const
|
|
361
|
+
const tAttemptStart = Date.now();
|
|
204
362
|
let tFirstByte = null;
|
|
205
363
|
// loop === false turns the guard off entirely (detector is null and no
|
|
206
364
|
// tool call is ever flagged); otherwise build a detector from the override
|
|
@@ -221,7 +379,7 @@ export async function runWorker(input) {
|
|
|
221
379
|
// discarded with its text, so the count must describe only the attempt
|
|
222
380
|
// whose text this call returns.
|
|
223
381
|
let groundingRetrievalCount = 0;
|
|
224
|
-
const timeout = workerTimeout(input.signal, timeoutMs);
|
|
382
|
+
const timeout = workerTimeout(input.signal, timeoutMs, input.progressTimeoutCeilingMs);
|
|
225
383
|
// Per-tool-call watchdog for this attempt (null when off). Its abort is
|
|
226
384
|
// OR'd with the worker timeout / external cancel into the child's signal.
|
|
227
385
|
const cmdWatch = commandWatch(commandCeilingForAttempt(input.commandTimeoutMs ?? 0, hangKills));
|
|
@@ -245,6 +403,14 @@ export async function runWorker(input) {
|
|
|
245
403
|
onFirstByte: () => (tFirstByte = Date.now()),
|
|
246
404
|
onToolCall: call => {
|
|
247
405
|
cmdWatch?.onStart(call);
|
|
406
|
+
// A tool call is the worker working. Inert unless the
|
|
407
|
+
// caller opted into a progress-based deadline.
|
|
408
|
+
timeout.progress();
|
|
409
|
+
if (input.fanoutTimeout
|
|
410
|
+
&& call.name === 'pi-worker-docs'
|
|
411
|
+
&& call.args?.module === '.') {
|
|
412
|
+
timeout.extend(input.fanoutTimeout.perLookupMs, input.fanoutTimeout.ceilingMs);
|
|
413
|
+
}
|
|
248
414
|
if (isGroundingRetrieval(call.name))
|
|
249
415
|
groundingRetrievalCount++;
|
|
250
416
|
if (!loopDetector)
|
|
@@ -254,16 +420,23 @@ export async function runWorker(input) {
|
|
|
254
420
|
loopHit = hit;
|
|
255
421
|
return hit;
|
|
256
422
|
},
|
|
257
|
-
|
|
258
|
-
//
|
|
259
|
-
//
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
423
|
+
// Output is the other half of "still working": a worker
|
|
424
|
+
// writing its answer is making progress even when it has no
|
|
425
|
+
// more tool calls to make.
|
|
426
|
+
onLine: line => {
|
|
427
|
+
timeout.progress();
|
|
428
|
+
input.onLine?.(line);
|
|
429
|
+
},
|
|
430
|
+
// Always wired now (it used to be conditional on the command
|
|
431
|
+
// watchdog): the sink only emits tool_execution_end if a
|
|
432
|
+
// handler exists, and a completed tool call is the clearest
|
|
433
|
+
// progress signal there is. Without it a worker whose tool
|
|
434
|
+
// calls all succeed would still look idle to the deadline.
|
|
435
|
+
onToolResult: r => {
|
|
436
|
+
timeout.progress();
|
|
437
|
+
cmdWatch?.onEnd(r.toolCallId);
|
|
438
|
+
input.onToolResult?.(r);
|
|
439
|
+
},
|
|
267
440
|
onContextUsage: input.onContextUsage
|
|
268
441
|
}, input.spawn);
|
|
269
442
|
}
|
|
@@ -272,8 +445,36 @@ export async function runWorker(input) {
|
|
|
272
445
|
cmdWatch?.clear();
|
|
273
446
|
}
|
|
274
447
|
const tEnd = Date.now();
|
|
275
|
-
const
|
|
448
|
+
const effectiveCapMs = timeout.budgetMs();
|
|
449
|
+
const waitMs = tFirstByte === null ? tEnd - tAttemptStart : tFirstByte - tAttemptStart;
|
|
276
450
|
const workMs = tFirstByte === null ? 0 : tEnd - tFirstByte;
|
|
451
|
+
// Record + announce a discarded attempt. Called from every `continue`
|
|
452
|
+
// branch below, so a restart cannot be added without becoming visible.
|
|
453
|
+
const noteRestart = (reason, detail) => {
|
|
454
|
+
const record = {
|
|
455
|
+
attempt: restarts.length + 1,
|
|
456
|
+
reason,
|
|
457
|
+
wallMs: tEnd - tAttemptStart,
|
|
458
|
+
waitMs,
|
|
459
|
+
workMs,
|
|
460
|
+
...(detail ? { detail } : {})
|
|
461
|
+
};
|
|
462
|
+
restarts.push(record);
|
|
463
|
+
input.onRestart?.(record);
|
|
464
|
+
// Harvest here rather than in each branch: `noteRestart` is the one
|
|
465
|
+
// place every `continue` already has to pass through, so a restart
|
|
466
|
+
// path cannot be added that silently drops the attempt's work.
|
|
467
|
+
// Longest-wins — a later attempt killed early should not replace a
|
|
468
|
+
// fuller answer an earlier one had already reached.
|
|
469
|
+
if (input.carryForward === true && CARRY_FORWARD_REASONS.has(reason)) {
|
|
470
|
+
const partial = text.trim();
|
|
471
|
+
// Longest-with-CONTENT wins. Length alone let a preamble sentence
|
|
472
|
+
// become the answer — see hasAnswerContent.
|
|
473
|
+
if (partial.length > (salvage.text?.length ?? 0) && hasAnswerContent(partial)) {
|
|
474
|
+
salvage.text = partial;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
};
|
|
277
478
|
const text = result.text ?? '';
|
|
278
479
|
const timedOut = timeout.timedOut();
|
|
279
480
|
const commandKill = cmdWatch?.killed();
|
|
@@ -281,16 +482,17 @@ export async function runWorker(input) {
|
|
|
281
482
|
// A loop-kill gets the same restart-with-hint treatment every other phase
|
|
282
483
|
// already gets (runPhaseWithLoopGuard) — name the offending call so the
|
|
283
484
|
// re-spawn avoids it. Bounded by the shared restart budget.
|
|
284
|
-
if (loopHit &&
|
|
485
|
+
if (loopHit && restartBudgetSpent < MAX_LOOP_RESTARTS) {
|
|
285
486
|
hint = formatLoopHint(loopHit);
|
|
286
|
-
|
|
487
|
+
restartBudgetSpent++;
|
|
488
|
+
noteRestart('loop', `${loopHit.call.name} ×${loopHit.count}/${loopHit.windowSize}`);
|
|
287
489
|
continue;
|
|
288
490
|
}
|
|
289
491
|
// A hung COMMAND is restartable too, on the same budget, but checked
|
|
290
492
|
// before the whole-worker timeout because its hint is the specific one:
|
|
291
493
|
// bound the command. (The two can't be confused — a watchdog kill leaves
|
|
292
494
|
// timeout.timedOut() false, since that flag tracks only its own timer.)
|
|
293
|
-
if (commandKill && !loopHit &&
|
|
495
|
+
if (commandKill && !loopHit && restartBudgetSpent < MAX_LOOP_RESTARTS) {
|
|
294
496
|
hint = commandTimeoutHint(commandKill.toolName, commandKill.timeoutMs, {
|
|
295
497
|
commandDetail: commandKill.detail,
|
|
296
498
|
// Nothing reverts the tree between attempts, so a child that can
|
|
@@ -299,24 +501,30 @@ export async function runWorker(input) {
|
|
|
299
501
|
// gate logger uses — decided by tools, not by phase.
|
|
300
502
|
editsMayPersist: /\b(?:edit|bash|write)\b/.test(tools)
|
|
301
503
|
});
|
|
302
|
-
|
|
504
|
+
restartBudgetSpent++;
|
|
303
505
|
hangKills++;
|
|
506
|
+
noteRestart('command-timeout', `${commandKill.toolName} > ${commandKill.timeoutMs}ms`
|
|
507
|
+
+ (commandKill.detail ? `: ${commandKill.detail}` : ''));
|
|
304
508
|
continue;
|
|
305
509
|
}
|
|
306
510
|
// A hung model stream is restartable on the same budget. Checked before
|
|
307
511
|
// the wall-clock timeout because it is the more specific diagnosis (and
|
|
308
512
|
// its hint does not blame the model: nothing it did caused the hang).
|
|
309
|
-
if (streamStalled && !loopHit &&
|
|
513
|
+
if (streamStalled && !loopHit && restartBudgetSpent < MAX_LOOP_RESTARTS) {
|
|
310
514
|
hint = streamStallHint(streamStalled.idleMs);
|
|
311
|
-
|
|
515
|
+
restartBudgetSpent++;
|
|
516
|
+
noteRestart('stream-stall', `idle ${streamStalled.idleMs}ms`);
|
|
312
517
|
continue;
|
|
313
518
|
}
|
|
314
519
|
// A wall-clock timeout (the backstop for varied thrash the exact-match
|
|
315
520
|
// detector misses) is also restartable, sharing the same budget. Skip when
|
|
316
521
|
// a loop also tripped — the loop hint above is more specific.
|
|
317
|
-
if (timedOut && !loopHit &&
|
|
522
|
+
if (timedOut && !loopHit && restartBudgetSpent < MAX_LOOP_RESTARTS) {
|
|
318
523
|
hint = WORKER_TIMEOUT_HINT;
|
|
319
|
-
|
|
524
|
+
restartBudgetSpent++;
|
|
525
|
+
// The EFFECTIVE cap, which the SCALE arm moves — reporting the
|
|
526
|
+
// configured one would misname why this attempt died.
|
|
527
|
+
noteRestart('worker-timeout', `cap ${effectiveCapMs}ms`);
|
|
320
528
|
continue;
|
|
321
529
|
}
|
|
322
530
|
// A connection-class model error is restartable on the same budget, exactly
|
|
@@ -342,10 +550,13 @@ export async function runWorker(input) {
|
|
|
342
550
|
// would only delay the report.
|
|
343
551
|
if (result.modelError
|
|
344
552
|
&& isConnectionError(result.modelError)
|
|
345
|
-
&&
|
|
553
|
+
&& restartBudgetSpent < MAX_LOOP_RESTARTS
|
|
346
554
|
&& connRetries < (input.connectionRetries ?? MAX_LOOP_RESTARTS)) {
|
|
555
|
+
// Noted BEFORE the backoff sleep, so the record's wallMs stays the
|
|
556
|
+
// attempt's own clock; the sleep lands in totalWallMs, where it belongs.
|
|
557
|
+
noteRestart('connection-error', result.modelError.slice(0, 120));
|
|
347
558
|
await (input.sleepFor ?? defaultSleep)(connectionRetryBackoffMs(connRetries));
|
|
348
|
-
|
|
559
|
+
restartBudgetSpent++;
|
|
349
560
|
connRetries++;
|
|
350
561
|
continue;
|
|
351
562
|
}
|
|
@@ -356,15 +567,44 @@ export async function runWorker(input) {
|
|
|
356
567
|
if (leaked && leakRetries < MAX_LEAK_RETRIES) {
|
|
357
568
|
hint = leakedToolCallHint(leaked);
|
|
358
569
|
leakRetries++;
|
|
570
|
+
noteRestart('leaked-tool-call', leaked.trim().slice(0, 80));
|
|
359
571
|
continue;
|
|
360
572
|
}
|
|
573
|
+
// SALVAGE. The run used to return the LAST attempt's text unconditionally,
|
|
574
|
+
// so a worker whose final attempt was killed early reported nothing at all
|
|
575
|
+
// — even when a discarded attempt had produced a usable answer that was
|
|
576
|
+
// still in hand at the moment it was thrown away. A restart budget is
|
|
577
|
+
// meant to buy more chances at an answer, not to overwrite a good attempt
|
|
578
|
+
// with a worse one.
|
|
579
|
+
//
|
|
580
|
+
// Gated on the final attempt having FAILED, not on it being shorter. A
|
|
581
|
+
// worker that finished cleanly has answered, and a short answer is a
|
|
582
|
+
// legitimate answer — length would let a long half-finished fragment
|
|
583
|
+
// override a concise correct one, which is the opposite of the fix.
|
|
584
|
+
const finalAttemptFailed = timedOut === true
|
|
585
|
+
|| result.aborted
|
|
586
|
+
|| result.modelError !== undefined
|
|
587
|
+
|| result.stalled === true
|
|
588
|
+
|| streamStalled !== undefined
|
|
589
|
+
|| commandKill !== undefined
|
|
590
|
+
|| loopHit !== undefined
|
|
591
|
+
|| text.trim().length === 0;
|
|
592
|
+
const answer = (finalAttemptFailed
|
|
593
|
+
&& salvage.text !== null
|
|
594
|
+
&& salvage.text.length > text.trim().length) ?
|
|
595
|
+
salvage.text
|
|
596
|
+
: text;
|
|
361
597
|
return {
|
|
362
|
-
text,
|
|
598
|
+
text: answer,
|
|
599
|
+
salvagedFromDiscardedAttempt: answer !== text,
|
|
363
600
|
exitCode: result.exitCode,
|
|
364
601
|
stderr: result.stderr.trim(),
|
|
365
602
|
aborted: result.aborted,
|
|
366
603
|
waitMs,
|
|
367
604
|
workMs,
|
|
605
|
+
attempts: restarts.length + 1,
|
|
606
|
+
totalWallMs: Date.now() - tRunStart,
|
|
607
|
+
restarts,
|
|
368
608
|
sawOutput: tFirstByte !== null,
|
|
369
609
|
groundingRetrievalCount,
|
|
370
610
|
...(result.modelError ? { modelError: result.modelError } : {}),
|
|
@@ -14,6 +14,7 @@ import { isTypeOnlyAnswer } from '../task/type-only-answer.js';
|
|
|
14
14
|
import { logDocsAnswer } from './typeonly-log.js';
|
|
15
15
|
import { normalizeQuery } from './research-cache.js';
|
|
16
16
|
import { projectDocsRaw, buildProjectPrompt } from './docs-project.js';
|
|
17
|
+
import { projectDocsBudget, projectDocsBudgetExhausted } from '../task/research-fanout-budget.js';
|
|
17
18
|
const childArgs = () => [...childBaseArgs(), '--no-tools'];
|
|
18
19
|
const RENDER_QUERY_MAX = 100;
|
|
19
20
|
const Params = Type.Object({
|
|
@@ -57,6 +58,14 @@ function pinDetails(pin) {
|
|
|
57
58
|
return pin ? { versionSource: pin.source, declaredRange: pin.range } : {};
|
|
58
59
|
}
|
|
59
60
|
export function registerPiWorkerDocs(pi, internals = {}) {
|
|
61
|
+
// CAP arm of nexttask 5B — OFF unless PI_TASK_PROJECT_DOCS_BUDGET is set, and
|
|
62
|
+
// then per-ATTEMPT by construction: the extension is loaded into a fresh pi
|
|
63
|
+
// child on every spawn, so a restarted attempt starts this counter at 0. The
|
|
64
|
+
// budget it enforces is the one the worker was told about in its prompt
|
|
65
|
+
// (projectDocsBudgetNotice) — enforcement without the notice would be a
|
|
66
|
+
// silent tool failure, and the notice without enforcement is what run 18
|
|
67
|
+
// already shows does not bind.
|
|
68
|
+
let projectLookups = 0;
|
|
60
69
|
makeWorkerTool(pi, {
|
|
61
70
|
name: 'pi-worker-docs',
|
|
62
71
|
label: 'Pi Worker Docs',
|
|
@@ -103,6 +112,15 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
103
112
|
const spawn = internals.spawn ?? defaultSpawn;
|
|
104
113
|
// ── Project source lookup ───────────────────────────────────────
|
|
105
114
|
if (params.module === '.') {
|
|
115
|
+
const budget = projectDocsBudget();
|
|
116
|
+
if (budget !== null && ++projectLookups > budget) {
|
|
117
|
+
// Refused BEFORE any work: the point of the cap is the child
|
|
118
|
+
// spawn and the model pass this branch would otherwise run.
|
|
119
|
+
return {
|
|
120
|
+
text: projectDocsBudgetExhausted(budget),
|
|
121
|
+
details: { budgetSpent: true }
|
|
122
|
+
};
|
|
123
|
+
}
|
|
106
124
|
const openCache = internals.openCache ?? defaultOpenCache;
|
|
107
125
|
let cache;
|
|
108
126
|
let cacheError;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.28.0",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|