@mjasnikovs/pi-task 0.18.27 → 0.18.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/config.d.ts +11 -7
- package/dist/config/register.js +3 -1
- package/dist/shared/child-process.d.ts +8 -0
- package/dist/shared/child-process.js +4 -2
- package/dist/shared/command-watchdog.d.ts +103 -0
- package/dist/shared/command-watchdog.js +158 -0
- package/dist/task/command-watchdog.d.ts +13 -43
- package/dist/task/command-watchdog.js +56 -72
- package/dist/task/enforce-guidelines.d.ts +4 -0
- package/dist/task/enforce-guidelines.js +9 -0
- package/dist/task/gate-deps.js +14 -0
- package/dist/task/orchestrator.d.ts +35 -0
- package/dist/task/orchestrator.js +74 -1
- package/dist/workers/pi-worker-core.d.ts +55 -0
- package/dist/workers/pi-worker-core.js +133 -4
- package/package.json +1 -1
package/dist/config/config.d.ts
CHANGED
|
@@ -43,16 +43,20 @@ export interface PiTaskConfig {
|
|
|
43
43
|
*/
|
|
44
44
|
extensionWhitelist: string[];
|
|
45
45
|
/**
|
|
46
|
-
* Wall-clock ceiling (ms) on a SINGLE tool execution
|
|
47
|
-
*
|
|
48
|
-
* own timeout. Local models routinely run a command that never returns
|
|
46
|
+
* Wall-clock ceiling (ms) on a SINGLE tool execution before the command
|
|
47
|
+
* watchdog steps in. Local models routinely run a command that never returns
|
|
49
48
|
* (e.g. `godot --headless` with no timeout, a dev server, a hung test) and
|
|
50
49
|
* the run wedges until the user manually aborts. pi's bash tool has an
|
|
51
50
|
* OPTIONAL timeout with NO default (bash.js), so a command the model didn't
|
|
52
|
-
* bound runs forever; this is the missing default
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
51
|
+
* bound runs forever; this is the missing default.
|
|
52
|
+
*
|
|
53
|
+
* ONE knob, TWO surfaces (shared/command-watchdog.ts): in the MAIN session
|
|
54
|
+
* the overrun call is cancelled via ctx.abort() (kills the tool's whole
|
|
55
|
+
* process tree) plus an auto-reminder turn; in the verify/fix GATE children
|
|
56
|
+
* (gate-deps.ts) the child is killed and re-spawned with a hint, the ceiling
|
|
57
|
+
* halving on repeat hangs. 0 = off — which unguards BOTH surfaces, gates
|
|
58
|
+
* included. Tool-agnostic: it arms on any tool execution, though in practice
|
|
59
|
+
* only bash runs long enough to trip it.
|
|
56
60
|
* DEFAULT 15 min: long enough for a real build/test suite, short enough that
|
|
57
61
|
* a true hang doesn't cost half an hour of dead time.
|
|
58
62
|
*/
|
package/dist/config/register.js
CHANGED
|
@@ -102,7 +102,9 @@ const ITEMS = [
|
|
|
102
102
|
description: 'Cancel a single command that runs longer than this and remind the model to set '
|
|
103
103
|
+ 'its own timeout. Catches a local model that runs a command which never returns '
|
|
104
104
|
+ '(hung build, dev server, no-timeout check) so the run stops itself instead of '
|
|
105
|
-
+ 'waiting for a manual abort.
|
|
105
|
+
+ 'waiting for a manual abort. One knob for both surfaces: the main session AND '
|
|
106
|
+
+ 'the verify/fix gate children. off disables it everywhere — gates can then hang '
|
|
107
|
+
+ 'unbounded',
|
|
106
108
|
// Display human labels; the stored config value stays the ms number.
|
|
107
109
|
values: COMMAND_TIMEOUT_OPTIONS.map(o => o.label)
|
|
108
110
|
}
|
|
@@ -58,6 +58,13 @@ export interface ChildResult {
|
|
|
58
58
|
export interface ToolCall {
|
|
59
59
|
name: string;
|
|
60
60
|
args: unknown;
|
|
61
|
+
/**
|
|
62
|
+
* pi's id for this tool call, carried on both `tool_execution_start` and
|
|
63
|
+
* `tool_execution_end` so a caller can pair them (the command watchdog arms
|
|
64
|
+
* on start and disarms on the matching end). Optional because the loop
|
|
65
|
+
* detector — the other consumer — keys on name+args and never needs it.
|
|
66
|
+
*/
|
|
67
|
+
toolCallId?: string;
|
|
61
68
|
}
|
|
62
69
|
export interface LoopHit {
|
|
63
70
|
call: ToolCall;
|
|
@@ -100,6 +107,7 @@ export interface RunChildJsonEventsOptions {
|
|
|
100
107
|
name: string;
|
|
101
108
|
isError: boolean;
|
|
102
109
|
text: string;
|
|
110
|
+
toolCallId?: string;
|
|
103
111
|
}) => void;
|
|
104
112
|
onFirstByte?: () => void;
|
|
105
113
|
/**
|
|
@@ -150,12 +150,13 @@ export class JsonEventSink {
|
|
|
150
150
|
}
|
|
151
151
|
if (t === 'tool_execution_start') {
|
|
152
152
|
const tn = typeof evt.toolName === 'string' ? evt.toolName : 'tool';
|
|
153
|
+
const id = typeof evt.toolCallId === 'string' ? evt.toolCallId : undefined;
|
|
153
154
|
if (opts.onLine) {
|
|
154
155
|
const detail = summarizeToolArgs(tn, evt.args);
|
|
155
156
|
opts.onLine(detail ? `${tn}: ${detail}` : tn);
|
|
156
157
|
}
|
|
157
158
|
if (opts.onToolCall) {
|
|
158
|
-
const hit = opts.onToolCall({ name: tn, args: evt.args });
|
|
159
|
+
const hit = opts.onToolCall({ name: tn, args: evt.args, toolCallId: id });
|
|
159
160
|
if (hit)
|
|
160
161
|
this.onLoopKill();
|
|
161
162
|
}
|
|
@@ -163,9 +164,10 @@ export class JsonEventSink {
|
|
|
163
164
|
}
|
|
164
165
|
if (t === 'tool_execution_end' && opts.onToolResult) {
|
|
165
166
|
const tn = typeof evt.toolName === 'string' ? evt.toolName : 'tool';
|
|
167
|
+
const id = typeof evt.toolCallId === 'string' ? evt.toolCallId : undefined;
|
|
166
168
|
const res = evt.result;
|
|
167
169
|
const text = toolResultText(res?.content);
|
|
168
|
-
opts.onToolResult({ name: tn, isError: evt.isError === true, text });
|
|
170
|
+
opts.onToolResult({ name: tn, isError: evt.isError === true, text, toolCallId: id });
|
|
169
171
|
}
|
|
170
172
|
}
|
|
171
173
|
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command watchdog — the per-tool-call wall-clock machine, shared by both
|
|
3
|
+
* surfaces that can run a command which never returns.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS LIVES IN shared/: pi's bash tool takes an OPTIONAL `timeout` with NO
|
|
6
|
+
* default (pi-coding-agent core/tools/bash.js), so ANY command the model didn't
|
|
7
|
+
* bound runs forever. That is true in two places, and they are disjoint:
|
|
8
|
+
*
|
|
9
|
+
* MAIN SESSION — the implementation turn, handed off via sendUserMessage
|
|
10
|
+
* (task/orchestrator.ts). Guarded by registerCommandWatchdog
|
|
11
|
+
* in task/command-watchdog.ts.
|
|
12
|
+
* CHILD pi — the verify / lint-fix / recommend / final-fix gate children
|
|
13
|
+
* (task/gate-deps.ts). Children are spawned `--no-extensions`
|
|
14
|
+
* (CHILD_BASE_ARGS), so the host's extension-event watchdog
|
|
15
|
+
* cannot see them at all. Guarded inside runWorker.
|
|
16
|
+
*
|
|
17
|
+
* Neither registration can cover the other's surface, so both exist — but the
|
|
18
|
+
* TIMER STATE MACHINE is identical, and lives here once. What differs is only
|
|
19
|
+
* the `onFire` side effect, which each adapter supplies:
|
|
20
|
+
*
|
|
21
|
+
* main session — ctx.abort() cancels just that tool call and the session
|
|
22
|
+
* survives to receive a follow-up reminder turn.
|
|
23
|
+
* child — there is no per-tool cancellation channel into a child, so
|
|
24
|
+
* the whole child is killed and re-spawned with
|
|
25
|
+
* {@link commandTimeoutHint} prepended. Coarser by necessity:
|
|
26
|
+
* the child's accumulated context is lost.
|
|
27
|
+
*/
|
|
28
|
+
/** Opaque timer handle — a real `setTimeout` return in production, anything the
|
|
29
|
+
* test's fake scheduler hands back under test. */
|
|
30
|
+
export type TimerHandle = unknown;
|
|
31
|
+
export interface WatchdogDeps {
|
|
32
|
+
/**
|
|
33
|
+
* The ceiling in ms, read PER command-start. How live that read is depends
|
|
34
|
+
* on the adapter: the main session passes a config read, so a /task-config
|
|
35
|
+
* change takes effect on the next command with no reload; the child adapter
|
|
36
|
+
* passes a constant frozen per attempt (the halved ceiling), so there a
|
|
37
|
+
* config change lands at the next attempt/gate, not the next command.
|
|
38
|
+
* 0 (or any non-positive value) means the watchdog is off and never arms.
|
|
39
|
+
*/
|
|
40
|
+
getTimeoutMs: () => number;
|
|
41
|
+
schedule: (fn: () => void, ms: number) => TimerHandle;
|
|
42
|
+
cancel: (handle: TimerHandle) => void;
|
|
43
|
+
/** Invoked when a command overruns: each adapter aborts/kills here. */
|
|
44
|
+
onFire: (toolCallId: string, toolName: string, timeoutMs: number) => void;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Stable substring of {@link reminderMessage}, used by the steer loop
|
|
48
|
+
* (orchestrator steerUntilDone) to recognise the watchdog's follow-up turn in
|
|
49
|
+
* the session entries — the artifact that distinguishes a watchdog abort from a
|
|
50
|
+
* human ESC. Interpolated into the message so the detector and the text cannot
|
|
51
|
+
* drift apart.
|
|
52
|
+
*/
|
|
53
|
+
export declare const WATCHDOG_CANCEL_MARKER = "was automatically cancelled \u2014 it looked stuck.";
|
|
54
|
+
/**
|
|
55
|
+
* MAIN-SESSION reminder, delivered as a follow-up turn after ctx.abort() has
|
|
56
|
+
* cancelled the offending tool call. The session is still alive and remembers
|
|
57
|
+
* the call, so this addresses it in the second person, present tense.
|
|
58
|
+
*/
|
|
59
|
+
export declare function reminderMessage(toolName: string, timeoutMs: number): string;
|
|
60
|
+
/**
|
|
61
|
+
* CHILD restart hint, prepended to the prompt of a re-spawned gate child. The
|
|
62
|
+
* killed child is GONE — this one never saw the command — so it is framed as
|
|
63
|
+
* "your previous attempt" and names the command, which the fresh child would
|
|
64
|
+
* otherwise have no way to know it must avoid repeating unbounded.
|
|
65
|
+
*
|
|
66
|
+
* `editsMayPersist` — set for a write-capable child (edit/write tools, or bash,
|
|
67
|
+
* whose commands have side effects). Nothing reverts the working tree between
|
|
68
|
+
* attempts, so telling such a child its previous attempt was "discarded" is
|
|
69
|
+
* false: partial edits and command side effects survive the kill, and a fresh
|
|
70
|
+
* child that believes it starts clean may re-apply them or misread the tree.
|
|
71
|
+
* Only the CONVERSATION is gone; the hint must say so precisely.
|
|
72
|
+
*
|
|
73
|
+
* Replaces the generic worker-timeout hint for this case: that one blames
|
|
74
|
+
* "exploring too long", which is the wrong diagnosis for a hung command and
|
|
75
|
+
* never mentions the timeout parameter.
|
|
76
|
+
*/
|
|
77
|
+
export declare function commandTimeoutHint(toolName: string, timeoutMs: number, opts?: {
|
|
78
|
+
commandDetail?: string;
|
|
79
|
+
editsMayPersist?: boolean;
|
|
80
|
+
}): string;
|
|
81
|
+
export declare class CommandWatchdog {
|
|
82
|
+
private readonly deps;
|
|
83
|
+
/** Armed timers, keyed by the tool call they guard. Tool executions are
|
|
84
|
+
* sequential, so this holds at most one entry in normal operation, but the
|
|
85
|
+
* map keeps it correct even if pi ever overlaps two calls. */
|
|
86
|
+
private readonly active;
|
|
87
|
+
constructor(deps: WatchdogDeps);
|
|
88
|
+
/** Arm a timer for a starting tool. No-op when the watchdog is off. */
|
|
89
|
+
onStart(toolCallId: string, toolName: string): void;
|
|
90
|
+
/** Disarm the timer for a finished tool. */
|
|
91
|
+
onEnd(toolCallId: string): void;
|
|
92
|
+
/** Cancel every armed timer — a turn-end / session-shutdown / child-exit
|
|
93
|
+
* safety net so no stray timer can fire into a later, unrelated command. */
|
|
94
|
+
clearAll(): void;
|
|
95
|
+
private disarm;
|
|
96
|
+
private fire;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Real-clock schedule/cancel, unref'd so a pending watchdog timer can never
|
|
100
|
+
* itself keep the process alive on exit. Shared by both adapters; tests
|
|
101
|
+
* substitute a fake scheduler instead.
|
|
102
|
+
*/
|
|
103
|
+
export declare const realTimerDeps: Pick<WatchdogDeps, 'schedule' | 'cancel'>;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command watchdog — the per-tool-call wall-clock machine, shared by both
|
|
3
|
+
* surfaces that can run a command which never returns.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS LIVES IN shared/: pi's bash tool takes an OPTIONAL `timeout` with NO
|
|
6
|
+
* default (pi-coding-agent core/tools/bash.js), so ANY command the model didn't
|
|
7
|
+
* bound runs forever. That is true in two places, and they are disjoint:
|
|
8
|
+
*
|
|
9
|
+
* MAIN SESSION — the implementation turn, handed off via sendUserMessage
|
|
10
|
+
* (task/orchestrator.ts). Guarded by registerCommandWatchdog
|
|
11
|
+
* in task/command-watchdog.ts.
|
|
12
|
+
* CHILD pi — the verify / lint-fix / recommend / final-fix gate children
|
|
13
|
+
* (task/gate-deps.ts). Children are spawned `--no-extensions`
|
|
14
|
+
* (CHILD_BASE_ARGS), so the host's extension-event watchdog
|
|
15
|
+
* cannot see them at all. Guarded inside runWorker.
|
|
16
|
+
*
|
|
17
|
+
* Neither registration can cover the other's surface, so both exist — but the
|
|
18
|
+
* TIMER STATE MACHINE is identical, and lives here once. What differs is only
|
|
19
|
+
* the `onFire` side effect, which each adapter supplies:
|
|
20
|
+
*
|
|
21
|
+
* main session — ctx.abort() cancels just that tool call and the session
|
|
22
|
+
* survives to receive a follow-up reminder turn.
|
|
23
|
+
* child — there is no per-tool cancellation channel into a child, so
|
|
24
|
+
* the whole child is killed and re-spawned with
|
|
25
|
+
* {@link commandTimeoutHint} prepended. Coarser by necessity:
|
|
26
|
+
* the child's accumulated context is lost.
|
|
27
|
+
*/
|
|
28
|
+
/** Whole minutes, floored at 1, for the human-facing ceiling in both messages. */
|
|
29
|
+
function minutes(timeoutMs) {
|
|
30
|
+
const mins = Math.max(1, Math.round(timeoutMs / 60_000));
|
|
31
|
+
return `${mins} minute${mins === 1 ? '' : 's'}`;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The core correction, shared by both adapters' messages. Two jobs, both
|
|
35
|
+
* learned from live runs: stop the model reporting a killed command as a
|
|
36
|
+
* success, and name the ONE mechanism that prevents a repeat (the bash tool's
|
|
37
|
+
* own `timeout` parameter) rather than leaving "be faster" as the takeaway.
|
|
38
|
+
*/
|
|
39
|
+
function correction() {
|
|
40
|
+
return (`The command was killed before it finished and produced NO result, so do not `
|
|
41
|
+
+ `report it as completed or successful, and do not claim that anything it would `
|
|
42
|
+
+ `have started (a server, build, or process) is now running. `
|
|
43
|
+
+ `If it was a genuinely long-running command, you MUST re-run it with an explicit `
|
|
44
|
+
+ `timeout — set the bash tool's \`timeout\` parameter (in seconds) so it cannot hang `
|
|
45
|
+
+ `again — or break it into smaller steps. Do NOT simply retry the same unbounded command.`);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Stable substring of {@link reminderMessage}, used by the steer loop
|
|
49
|
+
* (orchestrator steerUntilDone) to recognise the watchdog's follow-up turn in
|
|
50
|
+
* the session entries — the artifact that distinguishes a watchdog abort from a
|
|
51
|
+
* human ESC. Interpolated into the message so the detector and the text cannot
|
|
52
|
+
* drift apart.
|
|
53
|
+
*/
|
|
54
|
+
export const WATCHDOG_CANCEL_MARKER = 'was automatically cancelled — it looked stuck.';
|
|
55
|
+
/**
|
|
56
|
+
* MAIN-SESSION reminder, delivered as a follow-up turn after ctx.abort() has
|
|
57
|
+
* cancelled the offending tool call. The session is still alive and remembers
|
|
58
|
+
* the call, so this addresses it in the second person, present tense.
|
|
59
|
+
*/
|
|
60
|
+
export function reminderMessage(toolName, timeoutMs) {
|
|
61
|
+
return (`[SYSTEM] Your \`${toolName}\` call ran longer than ${minutes(timeoutMs)} `
|
|
62
|
+
+ `and ${WATCHDOG_CANCEL_MARKER} `
|
|
63
|
+
+ correction());
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* CHILD restart hint, prepended to the prompt of a re-spawned gate child. The
|
|
67
|
+
* killed child is GONE — this one never saw the command — so it is framed as
|
|
68
|
+
* "your previous attempt" and names the command, which the fresh child would
|
|
69
|
+
* otherwise have no way to know it must avoid repeating unbounded.
|
|
70
|
+
*
|
|
71
|
+
* `editsMayPersist` — set for a write-capable child (edit/write tools, or bash,
|
|
72
|
+
* whose commands have side effects). Nothing reverts the working tree between
|
|
73
|
+
* attempts, so telling such a child its previous attempt was "discarded" is
|
|
74
|
+
* false: partial edits and command side effects survive the kill, and a fresh
|
|
75
|
+
* child that believes it starts clean may re-apply them or misread the tree.
|
|
76
|
+
* Only the CONVERSATION is gone; the hint must say so precisely.
|
|
77
|
+
*
|
|
78
|
+
* Replaces the generic worker-timeout hint for this case: that one blames
|
|
79
|
+
* "exploring too long", which is the wrong diagnosis for a hung command and
|
|
80
|
+
* never mentions the timeout parameter.
|
|
81
|
+
*/
|
|
82
|
+
export function commandTimeoutHint(toolName, timeoutMs, opts) {
|
|
83
|
+
const what = opts?.commandDetail ? ` (${opts.commandDetail})` : '';
|
|
84
|
+
const aftermath = opts?.editsMayPersist ?
|
|
85
|
+
`killed and that attempt's conversation was discarded — you are starting over, `
|
|
86
|
+
+ `BUT any file edits or command side effects it made before the kill are still `
|
|
87
|
+
+ `in the working tree. Check the current state of files before assuming they `
|
|
88
|
+
+ `are untouched or re-applying changes. `
|
|
89
|
+
: `killed and that attempt was discarded — you are seeing this task again from `
|
|
90
|
+
+ `the start. `;
|
|
91
|
+
return (`[SYSTEM NOTE: Your previous attempt ran a \`${toolName}\` command${what} that had not `
|
|
92
|
+
+ `returned after ${minutes(timeoutMs)}, so it was `
|
|
93
|
+
+ aftermath
|
|
94
|
+
+ correction()
|
|
95
|
+
+ `]`);
|
|
96
|
+
}
|
|
97
|
+
export class CommandWatchdog {
|
|
98
|
+
deps;
|
|
99
|
+
/** Armed timers, keyed by the tool call they guard. Tool executions are
|
|
100
|
+
* sequential, so this holds at most one entry in normal operation, but the
|
|
101
|
+
* map keeps it correct even if pi ever overlaps two calls. */
|
|
102
|
+
active = new Map();
|
|
103
|
+
constructor(deps) {
|
|
104
|
+
this.deps = deps;
|
|
105
|
+
}
|
|
106
|
+
/** Arm a timer for a starting tool. No-op when the watchdog is off. */
|
|
107
|
+
onStart(toolCallId, toolName) {
|
|
108
|
+
const ms = this.deps.getTimeoutMs();
|
|
109
|
+
if (!(ms > 0))
|
|
110
|
+
return;
|
|
111
|
+
// A duplicate start for the same id must not leak the previous timer.
|
|
112
|
+
this.disarm(toolCallId);
|
|
113
|
+
const handle = this.deps.schedule(() => this.fire(toolCallId, toolName, ms), ms);
|
|
114
|
+
this.active.set(toolCallId, handle);
|
|
115
|
+
}
|
|
116
|
+
/** Disarm the timer for a finished tool. */
|
|
117
|
+
onEnd(toolCallId) {
|
|
118
|
+
this.disarm(toolCallId);
|
|
119
|
+
}
|
|
120
|
+
/** Cancel every armed timer — a turn-end / session-shutdown / child-exit
|
|
121
|
+
* safety net so no stray timer can fire into a later, unrelated command. */
|
|
122
|
+
clearAll() {
|
|
123
|
+
for (const handle of this.active.values())
|
|
124
|
+
this.deps.cancel(handle);
|
|
125
|
+
this.active.clear();
|
|
126
|
+
}
|
|
127
|
+
disarm(toolCallId) {
|
|
128
|
+
const handle = this.active.get(toolCallId);
|
|
129
|
+
if (handle !== undefined) {
|
|
130
|
+
this.deps.cancel(handle);
|
|
131
|
+
this.active.delete(toolCallId);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
fire(toolCallId, toolName, ms) {
|
|
135
|
+
// If the tool ended in the same tick the timer fired, its entry is gone
|
|
136
|
+
// already — never abort a command that has just finished cleanly.
|
|
137
|
+
if (!this.active.has(toolCallId))
|
|
138
|
+
return;
|
|
139
|
+
this.active.delete(toolCallId);
|
|
140
|
+
this.deps.onFire(toolCallId, toolName, ms);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Real-clock schedule/cancel, unref'd so a pending watchdog timer can never
|
|
145
|
+
* itself keep the process alive on exit. Shared by both adapters; tests
|
|
146
|
+
* substitute a fake scheduler instead.
|
|
147
|
+
*/
|
|
148
|
+
export const realTimerDeps = {
|
|
149
|
+
schedule: (fn, ms) => {
|
|
150
|
+
const handle = setTimeout(fn, ms);
|
|
151
|
+
if (typeof handle.unref === 'function') {
|
|
152
|
+
;
|
|
153
|
+
handle.unref();
|
|
154
|
+
}
|
|
155
|
+
return handle;
|
|
156
|
+
},
|
|
157
|
+
cancel: handle => clearTimeout(handle)
|
|
158
|
+
};
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
* configured ceiling and reminds the model to bound its own commands.
|
|
3
|
+
* MAIN-SESSION adapter for the command watchdog.
|
|
5
4
|
*
|
|
6
5
|
* WHY: a local model in the MAIN session routinely runs a command that never
|
|
7
6
|
* returns — `godot --headless --check-only` with no timeout, a dev server, a
|
|
@@ -17,48 +16,19 @@ import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
|
17
16
|
* what happened so it retries with a timeout instead of hanging again.
|
|
18
17
|
*
|
|
19
18
|
* Tool-agnostic: it arms on ANY tool, honouring "any command can run forever",
|
|
20
|
-
* though in practice only bash runs long enough to trip it.
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
*
|
|
27
|
-
export type TimerHandle = unknown;
|
|
28
|
-
export interface WatchdogDeps {
|
|
29
|
-
/**
|
|
30
|
-
* The ceiling in ms, read PER command-start so a /task-config change takes
|
|
31
|
-
* effect on the next command with no reload. 0 (or any non-positive value)
|
|
32
|
-
* means the watchdog is off and never arms.
|
|
33
|
-
*/
|
|
34
|
-
getTimeoutMs: () => number;
|
|
35
|
-
schedule: (fn: () => void, ms: number) => TimerHandle;
|
|
36
|
-
cancel: (handle: TimerHandle) => void;
|
|
37
|
-
/** Invoked when a command overruns: the registration aborts + reminds here. */
|
|
38
|
-
onFire: (toolCallId: string, toolName: string, timeoutMs: number) => void;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* The reminder delivered to the model after its command is cancelled. Kept pure
|
|
42
|
-
* and exported so a test can assert its shape without driving the whole session.
|
|
19
|
+
* though in practice only bash runs long enough to trip it.
|
|
20
|
+
*
|
|
21
|
+
* SCOPE — this covers the main session ONLY, which is where the implementation
|
|
22
|
+
* turn runs (orchestrator hands the spec off via sendUserMessage). Gate
|
|
23
|
+
* children are spawned `--no-extensions`, so no host extension exists inside
|
|
24
|
+
* them; their equivalent guard lives in runWorker (workers/pi-worker-core.ts)
|
|
25
|
+
* and shares the same machine from shared/command-watchdog.ts.
|
|
43
26
|
*/
|
|
44
|
-
export
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
* map keeps it correct even if pi ever overlaps two calls. */
|
|
50
|
-
private readonly active;
|
|
51
|
-
constructor(deps: WatchdogDeps);
|
|
52
|
-
/** Arm a timer for a starting tool. No-op when the watchdog is off. */
|
|
53
|
-
onStart(toolCallId: string, toolName: string): void;
|
|
54
|
-
/** Disarm the timer for a finished tool. */
|
|
55
|
-
onEnd(toolCallId: string): void;
|
|
56
|
-
/** Cancel every armed timer — a turn-end / session-shutdown safety net so no
|
|
57
|
-
* stray timer can fire into a later, unrelated command. */
|
|
58
|
-
clearAll(): void;
|
|
59
|
-
private disarm;
|
|
60
|
-
private fire;
|
|
61
|
-
}
|
|
27
|
+
export { CommandWatchdog, commandTimeoutHint, realTimerDeps, reminderMessage, WATCHDOG_CANCEL_MARKER, type TimerHandle, type WatchdogDeps } from '../shared/command-watchdog.js';
|
|
28
|
+
/** @internal Set by onFire when it aborts a turn. Exported for the adapter and tests. */
|
|
29
|
+
export declare function noteWatchdogAbort(): void;
|
|
30
|
+
/** True exactly once per watchdog abort; clears the flag. */
|
|
31
|
+
export declare function consumeWatchdogAbort(): boolean;
|
|
62
32
|
/**
|
|
63
33
|
* Wire the watchdog into the main session. Only ever active in the host session
|
|
64
34
|
* (children run `--no-extensions`), which is exactly where the observed hangs
|
|
@@ -1,67 +1,56 @@
|
|
|
1
1
|
import { getConfig } from '../config/config.js';
|
|
2
|
+
import { CommandWatchdog, realTimerDeps, reminderMessage } from '../shared/command-watchdog.js';
|
|
2
3
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
4
|
+
* MAIN-SESSION adapter for the command watchdog.
|
|
5
|
+
*
|
|
6
|
+
* WHY: a local model in the MAIN session routinely runs a command that never
|
|
7
|
+
* returns — `godot --headless --check-only` with no timeout, a dev server, a
|
|
8
|
+
* hung test — and the run wedges until the user manually aborts and tells the
|
|
9
|
+
* model to add a timeout. pi's bash tool takes an OPTIONAL `timeout` with NO
|
|
10
|
+
* default (see pi-coding-agent tools/bash.js), so any command the model didn't
|
|
11
|
+
* bound runs forever. This supplies the missing default from the host side.
|
|
12
|
+
*
|
|
13
|
+
* HOW: arm a wall-clock timer on `tool_execution_start`, disarm it on
|
|
14
|
+
* `tool_execution_end`. If it elapses, `ctx.abort()` cancels the in-flight
|
|
15
|
+
* operation — which fires the tool's AbortSignal, and pi's bash executor kills
|
|
16
|
+
* the whole process tree on abort — then a follow-up user turn tells the model
|
|
17
|
+
* what happened so it retries with a timeout instead of hanging again.
|
|
18
|
+
*
|
|
19
|
+
* Tool-agnostic: it arms on ANY tool, honouring "any command can run forever",
|
|
20
|
+
* though in practice only bash runs long enough to trip it.
|
|
21
|
+
*
|
|
22
|
+
* SCOPE — this covers the main session ONLY, which is where the implementation
|
|
23
|
+
* turn runs (orchestrator hands the spec off via sendUserMessage). Gate
|
|
24
|
+
* children are spawned `--no-extensions`, so no host extension exists inside
|
|
25
|
+
* them; their equivalent guard lives in runWorker (workers/pi-worker-core.ts)
|
|
26
|
+
* and shares the same machine from shared/command-watchdog.ts.
|
|
5
27
|
*/
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
28
|
+
// Re-exported so existing importers (and the machine's own tests) keep their
|
|
29
|
+
// entry point while the implementation lives in shared/.
|
|
30
|
+
export { CommandWatchdog, commandTimeoutHint, realTimerDeps, reminderMessage, WATCHDOG_CANCEL_MARKER } from '../shared/command-watchdog.js';
|
|
31
|
+
/**
|
|
32
|
+
* One-shot marker: the most recent turn abort was issued BY THE WATCHDOG, not by
|
|
33
|
+
* a human ESC. Both end the assistant turn with stopReason 'aborted' — the only
|
|
34
|
+
* signal steerUntilDone's wasInterrupted() can read — so without this flag the
|
|
35
|
+
* steer loop can win the race against the watchdog's queued follow-up turn and
|
|
36
|
+
* show a steering prompt to an empty room (wedging an unattended run).
|
|
37
|
+
*
|
|
38
|
+
* Set synchronously in onFire BEFORE ctx.abort(), so it is observable by the
|
|
39
|
+
* time any waitForIdle resolves; consumed (cleared) by the first reader. A stale
|
|
40
|
+
* flag (the aborted turn wasn't one the steer loop was watching) only costs the
|
|
41
|
+
* consumer a bounded wait before it falls back to prompting — it can never
|
|
42
|
+
* permanently suppress a human's steer prompt.
|
|
43
|
+
*/
|
|
44
|
+
let watchdogAbortPending = false;
|
|
45
|
+
/** @internal Set by onFire when it aborts a turn. Exported for the adapter and tests. */
|
|
46
|
+
export function noteWatchdogAbort() {
|
|
47
|
+
watchdogAbortPending = true;
|
|
19
48
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
active = new Map();
|
|
26
|
-
constructor(deps) {
|
|
27
|
-
this.deps = deps;
|
|
28
|
-
}
|
|
29
|
-
/** Arm a timer for a starting tool. No-op when the watchdog is off. */
|
|
30
|
-
onStart(toolCallId, toolName) {
|
|
31
|
-
const ms = this.deps.getTimeoutMs();
|
|
32
|
-
if (!(ms > 0))
|
|
33
|
-
return;
|
|
34
|
-
// A duplicate start for the same id must not leak the previous timer.
|
|
35
|
-
this.disarm(toolCallId);
|
|
36
|
-
const handle = this.deps.schedule(() => this.fire(toolCallId, toolName, ms), ms);
|
|
37
|
-
this.active.set(toolCallId, handle);
|
|
38
|
-
}
|
|
39
|
-
/** Disarm the timer for a finished tool. */
|
|
40
|
-
onEnd(toolCallId) {
|
|
41
|
-
this.disarm(toolCallId);
|
|
42
|
-
}
|
|
43
|
-
/** Cancel every armed timer — a turn-end / session-shutdown safety net so no
|
|
44
|
-
* stray timer can fire into a later, unrelated command. */
|
|
45
|
-
clearAll() {
|
|
46
|
-
for (const handle of this.active.values())
|
|
47
|
-
this.deps.cancel(handle);
|
|
48
|
-
this.active.clear();
|
|
49
|
-
}
|
|
50
|
-
disarm(toolCallId) {
|
|
51
|
-
const handle = this.active.get(toolCallId);
|
|
52
|
-
if (handle !== undefined) {
|
|
53
|
-
this.deps.cancel(handle);
|
|
54
|
-
this.active.delete(toolCallId);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
fire(toolCallId, toolName, ms) {
|
|
58
|
-
// If the tool ended in the same tick the timer fired, its entry is gone
|
|
59
|
-
// already — never abort a command that has just finished cleanly.
|
|
60
|
-
if (!this.active.has(toolCallId))
|
|
61
|
-
return;
|
|
62
|
-
this.active.delete(toolCallId);
|
|
63
|
-
this.deps.onFire(toolCallId, toolName, ms);
|
|
64
|
-
}
|
|
49
|
+
/** True exactly once per watchdog abort; clears the flag. */
|
|
50
|
+
export function consumeWatchdogAbort() {
|
|
51
|
+
const was = watchdogAbortPending;
|
|
52
|
+
watchdogAbortPending = false;
|
|
53
|
+
return was;
|
|
65
54
|
}
|
|
66
55
|
/**
|
|
67
56
|
* Wire the watchdog into the main session. Only ever active in the host session
|
|
@@ -75,23 +64,18 @@ export function registerCommandWatchdog(pi) {
|
|
|
75
64
|
const ctxByCall = new Map();
|
|
76
65
|
const watchdog = new CommandWatchdog({
|
|
77
66
|
getTimeoutMs: () => getConfig().requestTimeoutMs,
|
|
78
|
-
|
|
79
|
-
const handle = setTimeout(fn, ms);
|
|
80
|
-
// Don't let a pending watchdog timer keep the process alive on exit.
|
|
81
|
-
if (typeof handle.unref === 'function') {
|
|
82
|
-
;
|
|
83
|
-
handle.unref();
|
|
84
|
-
}
|
|
85
|
-
return handle;
|
|
86
|
-
},
|
|
87
|
-
cancel: handle => clearTimeout(handle),
|
|
67
|
+
...realTimerDeps,
|
|
88
68
|
onFire: (toolCallId, toolName, timeoutMs) => {
|
|
89
69
|
const ctx = ctxByCall.get(toolCallId);
|
|
90
70
|
ctxByCall.delete(toolCallId);
|
|
91
71
|
// Cancel the stuck command (kills the tool's whole process tree via
|
|
92
72
|
// the turn's AbortSignal), then start a fresh turn telling the model
|
|
93
|
-
// to bound its next attempt.
|
|
94
|
-
|
|
73
|
+
// to bound its next attempt. The flag must precede the abort so the
|
|
74
|
+
// steer loop can never observe the 'aborted' turn before the flag.
|
|
75
|
+
if (ctx) {
|
|
76
|
+
noteWatchdogAbort();
|
|
77
|
+
ctx.abort();
|
|
78
|
+
}
|
|
95
79
|
pi.sendUserMessage(reminderMessage(toolName, timeoutMs), { deliverAs: 'followUp' });
|
|
96
80
|
}
|
|
97
81
|
});
|
|
@@ -90,6 +90,10 @@ export interface EnforceChildResult {
|
|
|
90
90
|
loopHit?: unknown;
|
|
91
91
|
leakedToolCall?: unknown;
|
|
92
92
|
stalled?: boolean;
|
|
93
|
+
commandTimedOut?: {
|
|
94
|
+
toolName: string;
|
|
95
|
+
timeoutMs: number;
|
|
96
|
+
};
|
|
93
97
|
}
|
|
94
98
|
/**
|
|
95
99
|
* Map the enforcement child's runWorker result to a fatal error message, or null
|
|
@@ -224,6 +224,15 @@ export function classifyEnforceChildFailure(r) {
|
|
|
224
224
|
if (r.stalled) {
|
|
225
225
|
return 'model server unreachable — the child produced no output and the model endpoint did not respond';
|
|
226
226
|
}
|
|
227
|
+
// Same rule, same reason: the command watchdog's kill sets `aborted` too, so
|
|
228
|
+
// a child killed for a command that never returned would otherwise report as
|
|
229
|
+
// a user cancel. Its text is truncated mid-run — the verdict in it is partial
|
|
230
|
+
// and must never be parsed as a real one.
|
|
231
|
+
if (r.commandTimedOut) {
|
|
232
|
+
const mins = Math.max(1, Math.round(r.commandTimedOut.timeoutMs / 60_000));
|
|
233
|
+
return (`child ran a \`${r.commandTimedOut.toolName}\` command that had not returned after `
|
|
234
|
+
+ `${mins} minute${mins === 1 ? '' : 's'} and was killed — it never bounded the command`);
|
|
235
|
+
}
|
|
227
236
|
if (r.timedOut)
|
|
228
237
|
return 'enforcement child timed out';
|
|
229
238
|
if (r.loopHit)
|
package/dist/task/gate-deps.js
CHANGED
|
@@ -263,6 +263,15 @@ export function buildGateDeps(params) {
|
|
|
263
263
|
signal: sig,
|
|
264
264
|
tools,
|
|
265
265
|
timeoutMs: 0,
|
|
266
|
+
// The gate child runs to completion (timeoutMs 0), but a
|
|
267
|
+
// single command inside it must still be bounded: pi's bash
|
|
268
|
+
// tool has no default timeout, so a `bun run dev` / hung
|
|
269
|
+
// check the model forgot to bound wedges the gate forever.
|
|
270
|
+
// The stall guard cannot see it — a reachable model endpoint
|
|
271
|
+
// reads as proof of life while the command blocks. Same
|
|
272
|
+
// ceiling the main session uses, so one /task-config knob
|
|
273
|
+
// covers implementation and gates alike.
|
|
274
|
+
commandTimeoutMs: getConfig().requestTimeoutMs,
|
|
266
275
|
loop: { pathThreshold: Number.POSITIVE_INFINITY },
|
|
267
276
|
onLine: line => {
|
|
268
277
|
lastLine = line;
|
|
@@ -404,6 +413,11 @@ export function buildGateDeps(params) {
|
|
|
404
413
|
signal: sig,
|
|
405
414
|
tools,
|
|
406
415
|
timeoutMs: 0, // no wall-clock timeout — run to completion
|
|
416
|
+
// …but still bound any SINGLE command (see makeGateChild).
|
|
417
|
+
// enforce is read,edit today, so nothing here can hang on
|
|
418
|
+
// bash — wired anyway so a future tool grant can't quietly
|
|
419
|
+
// re-open the hole.
|
|
420
|
+
commandTimeoutMs: getConfig().requestTimeoutMs,
|
|
407
421
|
// Exact-match loop guard only: pathThreshold Infinity
|
|
408
422
|
// disables the path-revisit heuristic, so revisiting one
|
|
409
423
|
// file (which IS this pass's job) never trips — only a
|
|
@@ -212,6 +212,41 @@ export declare const MAX_COMPACTION_RESUMES = 20;
|
|
|
212
212
|
* of resumes performed (0 when the turn did not end on a compaction).
|
|
213
213
|
*/
|
|
214
214
|
export declare function resumeAcrossCompactions(ctx: SteerCtx): Promise<number>;
|
|
215
|
+
/**
|
|
216
|
+
* Timing knobs for the watchdog-abort guard in {@link steerUntilDone}, injectable
|
|
217
|
+
* so tests exercise the grace expiry without a 10-second wait. `graceMs` bounds
|
|
218
|
+
* how long the loop waits for the watchdog's follow-up to be DELIVERED (not to
|
|
219
|
+
* finish — its turn may legitimately run for minutes afterwards); delivery is
|
|
220
|
+
* normally near-instant, so the grace only expires on a stale flag.
|
|
221
|
+
*/
|
|
222
|
+
export interface SteerWatchdogDeps {
|
|
223
|
+
consume: () => boolean;
|
|
224
|
+
graceMs: number;
|
|
225
|
+
pollMs: number;
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* After the implementation turn settles, honour a user ESC by letting them steer.
|
|
229
|
+
*
|
|
230
|
+
* `waitForIdle` resolves both on natural completion AND on an ESC (which aborts
|
|
231
|
+
* the turn → idle). When the last turn was aborted, the host's main input loop is
|
|
232
|
+
* blocked inside our command handler, so a message typed in the editor would only
|
|
233
|
+
* queue, never run (interactive-mode routes idle input through onInputCallback,
|
|
234
|
+
* which is unset while we hold the loop). We therefore solicit the steering text
|
|
235
|
+
* ourselves and feed it back as another turn via sendUserMessage — which runs to
|
|
236
|
+
* completion when the session is idle. Repeat until a turn finishes uninterrupted.
|
|
237
|
+
*
|
|
238
|
+
* A WATCHDOG abort also ends the turn with stopReason 'aborted' — indistinguishable
|
|
239
|
+
* from a human ESC by the session entries alone at that instant. The watchdog
|
|
240
|
+
* queues its own recovery follow-up, so prompting there would show a steering
|
|
241
|
+
* dialog to an empty room and wedge an unattended run on the race. The one-shot
|
|
242
|
+
* flag (set synchronously before the abort) routes that case to
|
|
243
|
+
* {@link awaitWatchdogFollowUp} instead; a stale flag degrades to a bounded wait
|
|
244
|
+
* followed by the ordinary prompt, never to a suppressed one.
|
|
245
|
+
*
|
|
246
|
+
* Returns true when the user declined to steer (empty/cancelled) and the run
|
|
247
|
+
* should pause; false when the implementation completed (steered or not).
|
|
248
|
+
*/
|
|
249
|
+
export declare function steerUntilDone(ctx: SteerCtx, promptSteer?: (ctx: ExtensionCommandContext) => Promise<string | undefined>, watchdog?: Partial<SteerWatchdogDeps>): Promise<boolean>;
|
|
215
250
|
/**
|
|
216
251
|
* Run one prompt through the full single-task pipeline in a fresh session and
|
|
217
252
|
* deliver its spec. With waitForImplementation, block until the agent finishes
|
|
@@ -28,6 +28,7 @@ import { armImplWidget, disarmImplWidget, setupImplWidget } from './impl-widget.
|
|
|
28
28
|
import { publishViewer, publishNotify, publishLifecycleNotice, registerBridgeCommand, getBridge, SessionUI } from '../remote/bridge.js';
|
|
29
29
|
import { pushNotify } from '../remote/push.js';
|
|
30
30
|
import { getConfig } from '../config/config.js';
|
|
31
|
+
import { consumeWatchdogAbort, WATCHDOG_CANCEL_MARKER } from './command-watchdog.js';
|
|
31
32
|
import { buildGateDeps } from './gate-deps.js';
|
|
32
33
|
import { runGatesForTask } from './task-gates.js';
|
|
33
34
|
import { parseVerifyBlock } from './spec-validation.js';
|
|
@@ -462,6 +463,67 @@ export async function resumeAcrossCompactions(ctx) {
|
|
|
462
463
|
}
|
|
463
464
|
return resumes;
|
|
464
465
|
}
|
|
466
|
+
/**
|
|
467
|
+
* True when the watchdog's reminder follow-up has been DELIVERED into the session
|
|
468
|
+
* after the aborted assistant turn but its own turn has not finished yet — the
|
|
469
|
+
* artifact that confirms a pending watchdog recovery. Scoped after the LAST
|
|
470
|
+
* assistant entry so an earlier fire's reminder (already answered by its own
|
|
471
|
+
* turn) never matches.
|
|
472
|
+
*/
|
|
473
|
+
function watchdogReminderDelivered(ctx) {
|
|
474
|
+
const entries = ctx.sessionManager.getEntries();
|
|
475
|
+
let lastAssistant = -1;
|
|
476
|
+
for (let i = 0; i < entries.length; i++) {
|
|
477
|
+
const e = entries[i];
|
|
478
|
+
if ('message' in e && 'role' in e.message && e.message.role === 'assistant') {
|
|
479
|
+
lastAssistant = i;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
for (let i = lastAssistant + 1; i < entries.length; i++) {
|
|
483
|
+
const e = entries[i];
|
|
484
|
+
if (!('message' in e) || !('role' in e.message) || e.message.role !== 'user')
|
|
485
|
+
continue;
|
|
486
|
+
const content = e.message.content;
|
|
487
|
+
const text = typeof content === 'string' ? content
|
|
488
|
+
: Array.isArray(content) ?
|
|
489
|
+
content
|
|
490
|
+
.map(b => b !== null && typeof b === 'object' && 'text' in b ?
|
|
491
|
+
String(b.text)
|
|
492
|
+
: '')
|
|
493
|
+
.join(' ')
|
|
494
|
+
: '';
|
|
495
|
+
if (text.includes(WATCHDOG_CANCEL_MARKER))
|
|
496
|
+
return true;
|
|
497
|
+
}
|
|
498
|
+
return false;
|
|
499
|
+
}
|
|
500
|
+
const STEER_WATCHDOG_DEFAULTS = {
|
|
501
|
+
consume: consumeWatchdogAbort,
|
|
502
|
+
graceMs: 10_000,
|
|
503
|
+
pollMs: 100
|
|
504
|
+
};
|
|
505
|
+
/**
|
|
506
|
+
* Wait for a watchdog abort's queued follow-up turn instead of prompting. The
|
|
507
|
+
* abort and the reminder follow-up are two separate steps in the watchdog's
|
|
508
|
+
* onFire, so the steer loop can observe the aborted turn before the reminder is
|
|
509
|
+
* delivered — poll (bounded) until it lands or the follow-up turn has already
|
|
510
|
+
* completed. True = recovery observed, re-check the loop; false = grace expired
|
|
511
|
+
* with no reminder (stale flag) — fall back to the human prompt.
|
|
512
|
+
*/
|
|
513
|
+
async function awaitWatchdogFollowUp(ctx, wd) {
|
|
514
|
+
const deadline = Date.now() + wd.graceMs;
|
|
515
|
+
for (;;) {
|
|
516
|
+
if (!wasInterrupted(ctx))
|
|
517
|
+
return true; // follow-up turn already completed
|
|
518
|
+
if (watchdogReminderDelivered(ctx)) {
|
|
519
|
+
await ctx.waitForIdle(); // let the follow-up turn run to completion
|
|
520
|
+
return true;
|
|
521
|
+
}
|
|
522
|
+
if (Date.now() >= deadline)
|
|
523
|
+
return false;
|
|
524
|
+
await new Promise(r => setTimeout(r, wd.pollMs));
|
|
525
|
+
}
|
|
526
|
+
}
|
|
465
527
|
/**
|
|
466
528
|
* After the implementation turn settles, honour a user ESC by letting them steer.
|
|
467
529
|
*
|
|
@@ -473,10 +535,19 @@ export async function resumeAcrossCompactions(ctx) {
|
|
|
473
535
|
* ourselves and feed it back as another turn via sendUserMessage — which runs to
|
|
474
536
|
* completion when the session is idle. Repeat until a turn finishes uninterrupted.
|
|
475
537
|
*
|
|
538
|
+
* A WATCHDOG abort also ends the turn with stopReason 'aborted' — indistinguishable
|
|
539
|
+
* from a human ESC by the session entries alone at that instant. The watchdog
|
|
540
|
+
* queues its own recovery follow-up, so prompting there would show a steering
|
|
541
|
+
* dialog to an empty room and wedge an unattended run on the race. The one-shot
|
|
542
|
+
* flag (set synchronously before the abort) routes that case to
|
|
543
|
+
* {@link awaitWatchdogFollowUp} instead; a stale flag degrades to a bounded wait
|
|
544
|
+
* followed by the ordinary prompt, never to a suppressed one.
|
|
545
|
+
*
|
|
476
546
|
* Returns true when the user declined to steer (empty/cancelled) and the run
|
|
477
547
|
* should pause; false when the implementation completed (steered or not).
|
|
478
548
|
*/
|
|
479
|
-
async function steerUntilDone(ctx, promptSteer) {
|
|
549
|
+
export async function steerUntilDone(ctx, promptSteer, watchdog) {
|
|
550
|
+
const wd = { ...STEER_WATCHDOG_DEFAULTS, ...watchdog };
|
|
480
551
|
// Fan the prompt out through the bridge (local TUI input + remote browser
|
|
481
552
|
// card, first answer wins) instead of a raw ctx.ui.input: an interrupt can
|
|
482
553
|
// come from the remote Stop button just as well as a terminal ESC, and a
|
|
@@ -491,6 +562,8 @@ async function steerUntilDone(ctx, promptSteer) {
|
|
|
491
562
|
allowSkip: true
|
|
492
563
|
}));
|
|
493
564
|
while (wasInterrupted(ctx)) {
|
|
565
|
+
if (wd.consume() && (await awaitWatchdogFollowUp(ctx, wd)))
|
|
566
|
+
continue;
|
|
494
567
|
const steer = await ask(ctx);
|
|
495
568
|
if (steer === undefined || steer.trim().length === 0)
|
|
496
569
|
return true; // pause
|
|
@@ -17,6 +17,7 @@ export interface RunWorkerInput {
|
|
|
17
17
|
name: string;
|
|
18
18
|
isError: boolean;
|
|
19
19
|
text: string;
|
|
20
|
+
toolCallId?: string;
|
|
20
21
|
}) => void;
|
|
21
22
|
/**
|
|
22
23
|
* Called for each context_usage snapshot the child emits (same `--mode json`
|
|
@@ -31,6 +32,26 @@ export interface RunWorkerInput {
|
|
|
31
32
|
* own) — for a pass that must be allowed to finish however long it takes.
|
|
32
33
|
*/
|
|
33
34
|
timeoutMs?: number;
|
|
35
|
+
/**
|
|
36
|
+
* PER-TOOL-CALL wall-clock ceiling in ms — the child-side half of the command
|
|
37
|
+
* watchdog (see shared/command-watchdog.ts). Arms on each tool_execution_start
|
|
38
|
+
* and disarms on the matching end; on overrun the child is killed and, within
|
|
39
|
+
* the shared restart budget, re-spawned with commandTimeoutHint.
|
|
40
|
+
*
|
|
41
|
+
* WHY SEPARATE FROM `timeoutMs`: that one bounds the whole worker and is
|
|
42
|
+
* deliberately 0 (unbounded) for gate children, which must run to completion.
|
|
43
|
+
* Neither it nor the stall guard can catch a hung command — the stall guard
|
|
44
|
+
* treats a reachable model endpoint as proof of life, which it is, even while
|
|
45
|
+
* a `bun run dev` the model forgot to bound blocks the child forever.
|
|
46
|
+
*
|
|
47
|
+
* This is the ceiling for the FIRST attempt; each HANG-caused restart halves
|
|
48
|
+
* it (see commandCeilingForAttempt — loop-caused restarts don't count), so a
|
|
49
|
+
* model that ignores the hint cannot spend the full ceiling again on every
|
|
50
|
+
* retry.
|
|
51
|
+
*
|
|
52
|
+
* 0 / omitted = off, so every existing caller is unchanged.
|
|
53
|
+
*/
|
|
54
|
+
commandTimeoutMs?: number;
|
|
34
55
|
/**
|
|
35
56
|
* Per-worker loop-detector tuning. Defaults to the read-only research/impl
|
|
36
57
|
* guard (LOOP_WINDOW / LOOP_THRESHOLD, path threshold = exact threshold). An
|
|
@@ -96,5 +117,39 @@ export interface RunWorkerResult {
|
|
|
96
117
|
* aborted too, and mislabeling this as a user cancel hides a dead backend.
|
|
97
118
|
*/
|
|
98
119
|
stalled?: boolean;
|
|
120
|
+
/**
|
|
121
|
+
* Set when the command watchdog killed the worker's FINAL attempt: one tool
|
|
122
|
+
* call outran `commandTimeoutMs` (a command the model never bounded). Like
|
|
123
|
+
* loopHit/timedOut the text is partial — treat as a failure. Names the tool
|
|
124
|
+
* so the caller's trail says which call hung rather than just "aborted".
|
|
125
|
+
*
|
|
126
|
+
* Check BEFORE `aborted`, same reasoning as `stalled`: the kill aborts too.
|
|
127
|
+
*/
|
|
128
|
+
commandTimedOut?: {
|
|
129
|
+
toolName: string;
|
|
130
|
+
timeoutMs: number;
|
|
131
|
+
};
|
|
99
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* The per-command ceiling for attempt N, halving each time a hang recurs.
|
|
135
|
+
*
|
|
136
|
+
* The first attempt gets the full configured ceiling — a genuinely slow build or
|
|
137
|
+
* test suite deserves it. But every hang-caused restart carries
|
|
138
|
+
* commandTimeoutHint, which tells the model in as many words to bound its
|
|
139
|
+
* command; a SECOND hang means it ignored an explicit instruction, and a third
|
|
140
|
+
* means it ignored it twice. Giving a non-complying child the full ceiling again
|
|
141
|
+
* would put the worst case at 3 × 15 min = 45 minutes of dead time, resting
|
|
142
|
+
* entirely on the model obeying prose. Halving bounds it at ~26 min while
|
|
143
|
+
* costing a complying child nothing.
|
|
144
|
+
*
|
|
145
|
+
* `priorHangs` counts watchdog kills specifically, NOT total restarts — the
|
|
146
|
+
* restart budget is shared with loop kills, and a child restarted for LOOPING
|
|
147
|
+
* never received the bound-your-command hint, so its first hang still deserves
|
|
148
|
+
* the full ceiling. Only a hang after a hang is defiance.
|
|
149
|
+
*
|
|
150
|
+
* Floored at 30s so repeated halving cannot shrink the ceiling to something no
|
|
151
|
+
* real command could finish inside — but never ABOVE the configured ceiling
|
|
152
|
+
* itself, or a caller asking for 10s would silently get 30.
|
|
153
|
+
*/
|
|
154
|
+
export declare function commandCeilingForAttempt(baseMs: number, priorHangs: number): number;
|
|
100
155
|
export declare function runWorker(input: RunWorkerInput): Promise<RunWorkerResult>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { getPiInvocation } from '../shared/pi-invocation.js';
|
|
2
2
|
import { runChildDefault } from '../shared/child-process.js';
|
|
3
|
+
import { CommandWatchdog, commandTimeoutHint, realTimerDeps } from '../shared/command-watchdog.js';
|
|
3
4
|
import { childBaseArgs } from '../shared/child-extensions.js';
|
|
4
5
|
import { LoopDetector } from '../task/loop-detector.js';
|
|
5
6
|
import { LOOP_WINDOW, LOOP_THRESHOLD, MAX_LOOP_RESTARTS, formatLoopHint } from '../task/child-runner.js';
|
|
@@ -32,7 +33,12 @@ const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
|
|
|
32
33
|
* alive) just gets probed and waits on.
|
|
33
34
|
*/
|
|
34
35
|
const STALL_AFTER_MS = 180_000;
|
|
35
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Restart hint after a WHOLE-WORKER wall-clock timeout — distinct from both the
|
|
38
|
+
* loop hint and the per-command hint. This one diagnoses over-exploration, which
|
|
39
|
+
* is what the whole-worker cap actually catches. A single hung COMMAND is a
|
|
40
|
+
* different fault with a different fix, and gets commandTimeoutHint instead.
|
|
41
|
+
*/
|
|
36
42
|
const WORKER_TIMEOUT_HINT = '[SYSTEM NOTE: Your previous attempt ran out of time before answering — you '
|
|
37
43
|
+ 'were exploring too long. Be decisive: do the minimum reads/greps needed, '
|
|
38
44
|
+ 'then write your answer now. Do not re-explore ground you have already covered.]';
|
|
@@ -69,6 +75,85 @@ function workerTimeout(external, ms) {
|
|
|
69
75
|
}
|
|
70
76
|
};
|
|
71
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* The per-command ceiling for attempt N, halving each time a hang recurs.
|
|
80
|
+
*
|
|
81
|
+
* The first attempt gets the full configured ceiling — a genuinely slow build or
|
|
82
|
+
* test suite deserves it. But every hang-caused restart carries
|
|
83
|
+
* commandTimeoutHint, which tells the model in as many words to bound its
|
|
84
|
+
* command; a SECOND hang means it ignored an explicit instruction, and a third
|
|
85
|
+
* means it ignored it twice. Giving a non-complying child the full ceiling again
|
|
86
|
+
* would put the worst case at 3 × 15 min = 45 minutes of dead time, resting
|
|
87
|
+
* entirely on the model obeying prose. Halving bounds it at ~26 min while
|
|
88
|
+
* costing a complying child nothing.
|
|
89
|
+
*
|
|
90
|
+
* `priorHangs` counts watchdog kills specifically, NOT total restarts — the
|
|
91
|
+
* restart budget is shared with loop kills, and a child restarted for LOOPING
|
|
92
|
+
* never received the bound-your-command hint, so its first hang still deserves
|
|
93
|
+
* the full ceiling. Only a hang after a hang is defiance.
|
|
94
|
+
*
|
|
95
|
+
* Floored at 30s so repeated halving cannot shrink the ceiling to something no
|
|
96
|
+
* real command could finish inside — but never ABOVE the configured ceiling
|
|
97
|
+
* itself, or a caller asking for 10s would silently get 30.
|
|
98
|
+
*/
|
|
99
|
+
export function commandCeilingForAttempt(baseMs, priorHangs) {
|
|
100
|
+
if (!(baseMs > 0))
|
|
101
|
+
return 0;
|
|
102
|
+
const floor = Math.min(baseMs, 30_000);
|
|
103
|
+
return Math.max(floor, Math.round(baseMs / 2 ** priorHangs));
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Build the child-side command watchdog for ONE attempt: a per-tool-call timer
|
|
107
|
+
* machine (shared with the main session) whose `onFire` aborts `signal`, which
|
|
108
|
+
* runChild turns into a process-GROUP kill — reaping the hung command itself,
|
|
109
|
+
* not just the pi child holding it.
|
|
110
|
+
*
|
|
111
|
+
* LIMIT: the group kill only reaches processes still IN the group. A hung
|
|
112
|
+
* command that detached a daemon (setsid/nohup dev server) leaves it running —
|
|
113
|
+
* the fresh attempt can then hit a port the dead attempt's escapee still holds
|
|
114
|
+
* (the run-9 orphan-dev-server → false-EADDRINUSE shape). No cheap fix from
|
|
115
|
+
* here; the restart hint's "check current state" line is the mitigation.
|
|
116
|
+
*
|
|
117
|
+
* Returns null when the watchdog is off, so the caller keeps the plain timeout
|
|
118
|
+
* signal and no per-call bookkeeping happens at all.
|
|
119
|
+
*/
|
|
120
|
+
function commandWatch(timeoutMs) {
|
|
121
|
+
if (!(timeoutMs > 0))
|
|
122
|
+
return null;
|
|
123
|
+
const ctrl = new AbortController();
|
|
124
|
+
// pi's toolCallId pairs start↔end. When it is absent (a fake stream in a
|
|
125
|
+
// test, an older pi), fall back to one shared slot: tool executions in a
|
|
126
|
+
// child are sequential, so a single slot is still correctly paired.
|
|
127
|
+
const key = (id) => id ?? 'anon';
|
|
128
|
+
const details = new Map();
|
|
129
|
+
let killed;
|
|
130
|
+
const watchdog = new CommandWatchdog({
|
|
131
|
+
getTimeoutMs: () => timeoutMs,
|
|
132
|
+
...realTimerDeps,
|
|
133
|
+
onFire: (toolCallId, toolName, ms) => {
|
|
134
|
+
killed = {
|
|
135
|
+
toolName,
|
|
136
|
+
timeoutMs: ms,
|
|
137
|
+
...(details.has(toolCallId) ? { detail: details.get(toolCallId) } : {})
|
|
138
|
+
};
|
|
139
|
+
ctrl.abort();
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
return {
|
|
143
|
+
onStart: call => {
|
|
144
|
+
const id = key(call.toolCallId);
|
|
145
|
+
const args = call.args;
|
|
146
|
+
if (typeof args?.command === 'string') {
|
|
147
|
+
details.set(id, args.command.slice(0, 120));
|
|
148
|
+
}
|
|
149
|
+
watchdog.onStart(id, call.name);
|
|
150
|
+
},
|
|
151
|
+
onEnd: id => watchdog.onEnd(key(id)),
|
|
152
|
+
killed: () => killed,
|
|
153
|
+
signal: ctrl.signal,
|
|
154
|
+
clear: () => watchdog.clearAll()
|
|
155
|
+
};
|
|
156
|
+
}
|
|
72
157
|
export async function runWorker(input) {
|
|
73
158
|
const tools = input.tools ?? DEFAULT_TOOLS;
|
|
74
159
|
const baseArgs = [...childBaseArgs(input.extensions ?? []), '--mode', 'json', '--tools', tools];
|
|
@@ -79,6 +164,10 @@ export async function runWorker(input) {
|
|
|
79
164
|
// hint up to MAX_LOOP_RESTARTS times before we give up. Leaked tool calls
|
|
80
165
|
// keep their own MAX_LEAK_RETRIES budget below — a different failure mode.
|
|
81
166
|
let restarts = 0;
|
|
167
|
+
// Watchdog kills specifically — drives the ceiling halving. Kept apart from
|
|
168
|
+
// `restarts` (the shared budget) so a loop-caused restart doesn't shorten
|
|
169
|
+
// the rope of a child that has never hung (see commandCeilingForAttempt).
|
|
170
|
+
let hangKills = 0;
|
|
82
171
|
let leakRetries = 0;
|
|
83
172
|
for (;;) {
|
|
84
173
|
const prompt = hint === null ? input.prompt : `${hint}\n\n${input.prompt}`;
|
|
@@ -101,9 +190,13 @@ export async function runWorker(input) {
|
|
|
101
190
|
// caller couldn't distinguish from a crash.
|
|
102
191
|
let loopHit;
|
|
103
192
|
const timeout = workerTimeout(input.signal, timeoutMs);
|
|
193
|
+
// Per-tool-call watchdog for this attempt (null when off). Its abort is
|
|
194
|
+
// OR'd with the worker timeout / external cancel into the child's signal.
|
|
195
|
+
const cmdWatch = commandWatch(commandCeilingForAttempt(input.commandTimeoutMs ?? 0, hangKills));
|
|
196
|
+
const childSignal = cmdWatch ? AbortSignal.any([timeout.signal, cmdWatch.signal]) : timeout.signal;
|
|
104
197
|
let result;
|
|
105
198
|
try {
|
|
106
|
-
result = await runChildDefault(invocation, input.cwd,
|
|
199
|
+
result = await runChildDefault(invocation, input.cwd, childSignal, {
|
|
107
200
|
mode: 'json-events',
|
|
108
201
|
...(input.stall === false ?
|
|
109
202
|
{}
|
|
@@ -116,6 +209,7 @@ export async function runWorker(input) {
|
|
|
116
209
|
}),
|
|
117
210
|
onFirstByte: () => (tFirstByte = Date.now()),
|
|
118
211
|
onToolCall: call => {
|
|
212
|
+
cmdWatch?.onStart(call);
|
|
119
213
|
if (!loopDetector)
|
|
120
214
|
return null;
|
|
121
215
|
const hit = loopDetector.record(call);
|
|
@@ -124,18 +218,28 @@ export async function runWorker(input) {
|
|
|
124
218
|
return hit;
|
|
125
219
|
},
|
|
126
220
|
onLine: input.onLine,
|
|
127
|
-
|
|
221
|
+
// Always wired when the watchdog is on — the sink only emits
|
|
222
|
+
// tool_execution_end if a handler exists, and without it every
|
|
223
|
+
// timer would stay armed and fire on a finished command.
|
|
224
|
+
onToolResult: cmdWatch ?
|
|
225
|
+
r => {
|
|
226
|
+
cmdWatch.onEnd(r.toolCallId);
|
|
227
|
+
input.onToolResult?.(r);
|
|
228
|
+
}
|
|
229
|
+
: input.onToolResult,
|
|
128
230
|
onContextUsage: input.onContextUsage
|
|
129
231
|
}, input.spawn);
|
|
130
232
|
}
|
|
131
233
|
finally {
|
|
132
234
|
timeout.cleanup();
|
|
235
|
+
cmdWatch?.clear();
|
|
133
236
|
}
|
|
134
237
|
const tEnd = Date.now();
|
|
135
238
|
const waitMs = tFirstByte === null ? tEnd - tStart : tFirstByte - tStart;
|
|
136
239
|
const workMs = tFirstByte === null ? 0 : tEnd - tFirstByte;
|
|
137
240
|
const text = result.text ?? '';
|
|
138
241
|
const timedOut = timeout.timedOut();
|
|
242
|
+
const commandKill = cmdWatch?.killed();
|
|
139
243
|
// A loop-kill gets the same restart-with-hint treatment every other phase
|
|
140
244
|
// already gets (runPhaseWithLoopGuard) — name the offending call so the
|
|
141
245
|
// re-spawn avoids it. Bounded by the shared restart budget.
|
|
@@ -144,6 +248,23 @@ export async function runWorker(input) {
|
|
|
144
248
|
restarts++;
|
|
145
249
|
continue;
|
|
146
250
|
}
|
|
251
|
+
// A hung COMMAND is restartable too, on the same budget, but checked
|
|
252
|
+
// before the whole-worker timeout because its hint is the specific one:
|
|
253
|
+
// bound the command. (The two can't be confused — a watchdog kill leaves
|
|
254
|
+
// timeout.timedOut() false, since that flag tracks only its own timer.)
|
|
255
|
+
if (commandKill && !loopHit && restarts < MAX_LOOP_RESTARTS) {
|
|
256
|
+
hint = commandTimeoutHint(commandKill.toolName, commandKill.timeoutMs, {
|
|
257
|
+
commandDetail: commandKill.detail,
|
|
258
|
+
// Nothing reverts the tree between attempts, so a child that can
|
|
259
|
+
// mutate it (edit/write, or bash side effects) must not be told
|
|
260
|
+
// its previous attempt left no trace. Same capability test the
|
|
261
|
+
// gate logger uses — decided by tools, not by phase.
|
|
262
|
+
editsMayPersist: /\b(?:edit|bash|write)\b/.test(tools)
|
|
263
|
+
});
|
|
264
|
+
restarts++;
|
|
265
|
+
hangKills++;
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
147
268
|
// A wall-clock timeout (the backstop for varied thrash the exact-match
|
|
148
269
|
// detector misses) is also restartable, sharing the same budget. Skip when
|
|
149
270
|
// a loop also tripped — the loop hint above is more specific.
|
|
@@ -171,7 +292,15 @@ export async function runWorker(input) {
|
|
|
171
292
|
...(leaked ? { leakedToolCall: leaked } : {}),
|
|
172
293
|
...(loopHit ? { loopHit } : {}),
|
|
173
294
|
...(timedOut ? { timedOut: true } : {}),
|
|
174
|
-
...(result.stalled ? { stalled: true } : {})
|
|
295
|
+
...(result.stalled ? { stalled: true } : {}),
|
|
296
|
+
...(commandKill ?
|
|
297
|
+
{
|
|
298
|
+
commandTimedOut: {
|
|
299
|
+
toolName: commandKill.toolName,
|
|
300
|
+
timeoutMs: commandKill.timeoutMs
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
: {})
|
|
175
304
|
};
|
|
176
305
|
}
|
|
177
306
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.29",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|