brl-subagent 2.5.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT.md +9 -2
- package/README.md +6 -3
- package/package.json +6 -6
- package/src/history.ts +19 -9
- package/src/index.ts +174 -34
- package/src/kill-escalation.ts +338 -6
- package/src/metrics.ts +5 -3
- package/src/notify-completion.ts +23 -0
- package/src/proc.ts +270 -20
- package/src/recovery-engine.ts +100 -14
- package/src/recovery-surface.ts +374 -0
- package/src/recovery.ts +19 -1
- package/src/run-registry.ts +64 -3
- package/src/runner.ts +216 -13
- package/src/session-manager.ts +129 -7
- package/src/state.ts +32 -2
- package/src/tui-format.ts +57 -1
- package/src/tui.ts +102 -22
- package/src/types.ts +66 -1
- package/src/unit-run.ts +8 -1
package/src/runner.ts
CHANGED
|
@@ -34,7 +34,16 @@ import {
|
|
|
34
34
|
SUBAGENT_ABORTED_MESSAGE,
|
|
35
35
|
SUBAGENT_SIGNAL_KILLED_MESSAGE,
|
|
36
36
|
} from "./types";
|
|
37
|
-
import {
|
|
37
|
+
import {
|
|
38
|
+
escalateKill,
|
|
39
|
+
pidTarget,
|
|
40
|
+
processTarget,
|
|
41
|
+
supportsProcessGroupKill,
|
|
42
|
+
groupTarget,
|
|
43
|
+
type EscalationTarget,
|
|
44
|
+
type SignalGuard,
|
|
45
|
+
} from "./kill-escalation";
|
|
46
|
+
import { findByMarker, pidAlive, pidHasMarker, pidGone, groupHasMembers, killProcessGroup } from "./proc";
|
|
38
47
|
import { getSafeEnv, DEPTH_ENV_KEY, CHILD_MARKER_ENV_KEY, sanitizeErrorMessage } from "./sanitize";
|
|
39
48
|
import type { Logger } from "./logging";
|
|
40
49
|
import type { Intercom } from "./messaging";
|
|
@@ -83,7 +92,10 @@ export function activeChildCount(): number {
|
|
|
83
92
|
|
|
84
93
|
/** True once the child has really exited (a signal was delivered is not enough). */
|
|
85
94
|
function childExited(proc: ChildProcess): boolean {
|
|
86
|
-
|
|
95
|
+
// `!= null` (not `!== null`): a real ChildProcess uses `null` before exit, but
|
|
96
|
+
// narrow test doubles omit the field (`undefined`), which must also read as
|
|
97
|
+
// "not exited" so the liveness re-check does not skip a live child's TERM.
|
|
98
|
+
return proc.exitCode != null || proc.signalCode != null;
|
|
87
99
|
}
|
|
88
100
|
|
|
89
101
|
/** Default real sleep between SIGTERM and the escalation re-check. */
|
|
@@ -121,41 +133,198 @@ export function __setEscalationTimingForTest(timing?: { graceMs?: number; pollMs
|
|
|
121
133
|
escalationTiming.pollMs = timing?.pollMs ?? SIGKILL_POLL_MS;
|
|
122
134
|
}
|
|
123
135
|
|
|
136
|
+
/**
|
|
137
|
+
* #322: TEST-ONLY registry seam. Register an arbitrary child (and optional
|
|
138
|
+
* marker) under a fresh unique spawn token so a test can drive the shutdown reap
|
|
139
|
+
* in isolation — with no `runSubagent` in the picture, the run's own exit sweep
|
|
140
|
+
* cannot be the killer and the reap's death assertion is attributable.
|
|
141
|
+
*
|
|
142
|
+
* Returns a deregister function that deletes that token. Throws outside the
|
|
143
|
+
* test environment (same runtime gate as `__setEscalationTimingForTest`): the
|
|
144
|
+
* registry is production state, and a stray import must never be able to point
|
|
145
|
+
* a real reap at a process the runner did not spawn.
|
|
146
|
+
*/
|
|
147
|
+
export function __registerActiveChildForTest(proc: ChildProcess, marker?: string): () => void {
|
|
148
|
+
if (!isTestEnvironment()) {
|
|
149
|
+
throw new Error(
|
|
150
|
+
"__registerActiveChildForTest() is test-only and cannot run in production.",
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
const token = `spawn-${process.pid}-${++spawnTokenSeq}`;
|
|
154
|
+
activeChildren.set(token, { proc, marker });
|
|
155
|
+
return () => {
|
|
156
|
+
activeChildren.delete(token);
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
124
160
|
/**
|
|
125
161
|
* Escalation target for one spawned child. `verifyDeath` defaults to the
|
|
126
|
-
* REAL-exit check (`childExited`)
|
|
127
|
-
* timeout paths must use the real
|
|
128
|
-
* when a signal is *sent*, not when the
|
|
129
|
-
* it would let a SIGTERM-ignoring child
|
|
162
|
+
* REAL-exit check (`childExited`) AND — on a group target — group emptiness, and
|
|
163
|
+
* may be overridden per site. The abort and timeout paths must use the real
|
|
164
|
+
* check (issue #303): `child.killed` flips when a signal is *sent*, not when the
|
|
165
|
+
* process dies, so gating the SIGKILL on it would let a SIGTERM-ignoring child
|
|
166
|
+
* survive the escalation.
|
|
167
|
+
*
|
|
168
|
+
* #299 Option B D2: on POSIX the target ALSO signals the child's process group
|
|
169
|
+
* (best-effort) and unconditionally SIGKILLs it after the grace window
|
|
170
|
+
* (`groupTarget`), so an env-scrubbed grandchild — invisible to the marker scan
|
|
171
|
+
* — dies with the child on abort/timeout/shutdown. The direct `proc.kill`
|
|
172
|
+
* signal is never replaced. On Windows (`supportsProcessGroupKill()` false,
|
|
173
|
+
* D6) this is exactly today's direct-only target.
|
|
174
|
+
*
|
|
175
|
+
* #299 review (Major 1 + Major 2):
|
|
176
|
+
* - The tracked `ChildProcess` handle is its OWN identity class — it is not
|
|
177
|
+
* marker-derived, so there is no marker guard here; terminate/forceKill stay
|
|
178
|
+
* unconditional (D5). We add only a TERMINATE-time liveness re-check (skip
|
|
179
|
+
* when the child is already confirmed exited). The force-kill window is
|
|
180
|
+
* in-session and µs-class: the live handle cannot be reused by a foreign
|
|
181
|
+
* process while we hold it.
|
|
182
|
+
* - Major 2: the default death check becomes `childExited(proc) &&
|
|
183
|
+
* groupHasMembers(pid) === false` so a descendant still cleaning up keeps the
|
|
184
|
+
* grace window open. #299 fix A.2 makes that reader THREE-state: only a
|
|
185
|
+
* POSITIVELY empty group (`false`) permits the early exit; an UNKNOWN
|
|
186
|
+
* inspection (`undefined`) keeps the target alive. When /proc is unavailable
|
|
187
|
+
* as a PLATFORM (`groupHasMembers` false) it
|
|
188
|
+
* degrades to child-exited liveness — the documented platform boundary.
|
|
130
189
|
*/
|
|
131
|
-
function childTarget(proc: ChildProcess, verifyDeath
|
|
190
|
+
function childTarget(proc: ChildProcess, verifyDeath?: () => boolean): EscalationTarget {
|
|
191
|
+
const pid = proc.pid ?? 0;
|
|
192
|
+
const killDirect = (_pid: number, signal: NodeJS.Signals): void => {
|
|
193
|
+
proc.kill(signal);
|
|
194
|
+
};
|
|
195
|
+
const useGroup = supportsProcessGroupKill() && proc.pid !== undefined;
|
|
196
|
+
const base = useGroup
|
|
197
|
+
? groupTarget(pid, killDirect, killProcessGroup, pidAlive, groupHasMembers)
|
|
198
|
+
: pidTarget(pid, killDirect, pidAlive);
|
|
199
|
+
const defaultVerifyDeath = useGroup
|
|
200
|
+
? () => childExited(proc) && groupHasMembers(pid) === false
|
|
201
|
+
: () => childExited(proc);
|
|
132
202
|
return {
|
|
203
|
+
...base,
|
|
133
204
|
terminate: () => {
|
|
134
|
-
|
|
205
|
+
// #299 review Major 1: an already-exited child is not signaled again.
|
|
206
|
+
if (childExited(proc)) return;
|
|
207
|
+
base.terminate();
|
|
135
208
|
},
|
|
136
|
-
|
|
137
|
-
proc.kill("SIGKILL");
|
|
138
|
-
},
|
|
139
|
-
verifyDeath,
|
|
209
|
+
verifyDeath: verifyDeath ?? defaultVerifyDeath,
|
|
140
210
|
};
|
|
141
211
|
}
|
|
142
212
|
|
|
213
|
+
/** Send a signal to a pid; the ONE production kill primitive for marker sweeps. */
|
|
214
|
+
const killPid = (pid: number, signal: NodeJS.Signals): void => {
|
|
215
|
+
process.kill(pid, signal);
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* #299 Option A: union the marker-carrying descendants of the given markers into
|
|
220
|
+
* pid-based escalation targets. A spawned child stamps its own subprocesses with
|
|
221
|
+
* the same per-run marker env, so `findByMarker` surfaces grandchildren the
|
|
222
|
+
* `ChildProcess`-based targets cannot see. `excludePids` drops anything already
|
|
223
|
+
* targeted (the tracked direct children). Markers are per-run UUIDs, so a sweep
|
|
224
|
+
* cannot reach another run's tree.
|
|
225
|
+
*
|
|
226
|
+
* #299 review Major 1: every pid here is marker-derived, so its targets carry a
|
|
227
|
+
* `SignalGuard` that re-reads the LIVE marker (`pidHasMarker`) immediately before
|
|
228
|
+
* each signal. A pid that exited and was reused by a foreign process no longer
|
|
229
|
+
* carries the marker, so both the group and direct signals are SKIPPED and the
|
|
230
|
+
* skip is logged — never an unverified signal. `log` is optional (the sweep and
|
|
231
|
+
* shutdown calls pass their logger).
|
|
232
|
+
*/
|
|
233
|
+
function collectMarkerTargets(
|
|
234
|
+
markers: Iterable<string | undefined>,
|
|
235
|
+
excludePids: ReadonlySet<number>,
|
|
236
|
+
log?: Logger,
|
|
237
|
+
): { pids: number[]; targets: EscalationTarget[] } {
|
|
238
|
+
const seen = new Set<number>();
|
|
239
|
+
const pids: number[] = [];
|
|
240
|
+
const targets: EscalationTarget[] = [];
|
|
241
|
+
// D3 (#299): a marker-verified pid is identity proof, so its group is also
|
|
242
|
+
// signaled best-effort. When the carrier is a group leader (the spawned child
|
|
243
|
+
// or a descendant that `setsid`'d) `-pid` reaches its env-scrubbed
|
|
244
|
+
// descendants; when it is merely a member, `-pid` is ESRCH and the direct
|
|
245
|
+
// signal still lands.
|
|
246
|
+
const groupKill = supportsProcessGroupKill() ? killProcessGroup : undefined;
|
|
247
|
+
for (const marker of markers) {
|
|
248
|
+
if (!marker) continue;
|
|
249
|
+
for (const pid of findByMarker(marker)) {
|
|
250
|
+
if (!Number.isInteger(pid) || pid <= 0) continue;
|
|
251
|
+
if (excludePids.has(pid) || seen.has(pid)) continue;
|
|
252
|
+
seen.add(pid);
|
|
253
|
+
pids.push(pid);
|
|
254
|
+
const guard: SignalGuard = {
|
|
255
|
+
isOwn: (targetPid) => pidHasMarker(targetPid, marker),
|
|
256
|
+
isGone: (targetPid) => pidGone(targetPid),
|
|
257
|
+
onSkip: (targetPid, signal, phase) =>
|
|
258
|
+
log?.warn("Skipped marker signal: pid no longer carries the run marker", {
|
|
259
|
+
pid: targetPid,
|
|
260
|
+
signal,
|
|
261
|
+
phase,
|
|
262
|
+
marker,
|
|
263
|
+
}),
|
|
264
|
+
};
|
|
265
|
+
targets.push(processTarget(pid, killPid, pidAlive, groupKill, groupHasMembers, guard));
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return { pids, targets };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* #299 Option A: kill every pid still carrying `marker` once the direct child has
|
|
273
|
+
* exited — the marker-inheriting descendants (grandchildren) that would
|
|
274
|
+
* otherwise survive the run in-session (probe 1c). No marker / no match is a
|
|
275
|
+
* no-op. Uses the same SIGTERM → grace → SIGKILL window (and test-only timing
|
|
276
|
+
* seam) as every other kill path; never rejects.
|
|
277
|
+
*/
|
|
278
|
+
async function sweepMarkerChildren(
|
|
279
|
+
marker: string | undefined,
|
|
280
|
+
excludePids: ReadonlySet<number>,
|
|
281
|
+
log?: Logger,
|
|
282
|
+
): Promise<number[]> {
|
|
283
|
+
const { pids, targets } = collectMarkerTargets([marker], excludePids, log);
|
|
284
|
+
if (targets.length === 0) return [];
|
|
285
|
+
try {
|
|
286
|
+
await escalateKill(targets, {
|
|
287
|
+
graceMs: escalationTiming.graceMs,
|
|
288
|
+
pollMs: escalationTiming.pollMs,
|
|
289
|
+
sleep: realSleep,
|
|
290
|
+
});
|
|
291
|
+
log?.info("Swept leftover marker children after run exit", { marker, pids });
|
|
292
|
+
} catch (err) {
|
|
293
|
+
// A sweep must never block the run result from settling.
|
|
294
|
+
log?.warn("Marker sweep failed after run exit", { marker, error: String(err) });
|
|
295
|
+
}
|
|
296
|
+
return pids;
|
|
297
|
+
}
|
|
298
|
+
|
|
143
299
|
/**
|
|
144
300
|
* Terminate every registered in-flight child: SIGTERM all, wait up to the shared
|
|
145
301
|
* grace period (returning as soon as every child is dead — B4), then SIGKILL any
|
|
146
302
|
* survivor. Returns the pids targeted. Safe to call when nothing is in flight.
|
|
303
|
+
*
|
|
304
|
+
* #299 Option A: each tracked entry's marker is also swept (`findByMarker`),
|
|
305
|
+
* because a marker-carrying grandchild is invisible to the `ChildProcess` handle
|
|
306
|
+
* and would otherwise survive a clean session_shutdown (probe 1b). Marker pids
|
|
307
|
+
* are deduped against the tracked pids and share the ONE escalation window, so
|
|
308
|
+
* the whole inherited tree dies together.
|
|
147
309
|
*/
|
|
148
310
|
export async function reapActiveChildren(log?: Logger): Promise<number[]> {
|
|
149
311
|
const pids: number[] = [];
|
|
312
|
+
const trackedPids = new Set<number>();
|
|
150
313
|
const targets: EscalationTarget[] = [];
|
|
314
|
+
const markers: Array<string | undefined> = [];
|
|
151
315
|
for (const [token, entry] of activeChildren) {
|
|
152
316
|
if (entry.proc.pid === undefined) {
|
|
153
317
|
activeChildren.delete(token);
|
|
154
318
|
continue;
|
|
155
319
|
}
|
|
320
|
+
trackedPids.add(entry.proc.pid);
|
|
156
321
|
pids.push(entry.proc.pid);
|
|
157
322
|
targets.push(childTarget(entry.proc));
|
|
323
|
+
markers.push(entry.marker);
|
|
158
324
|
}
|
|
325
|
+
const swept = collectMarkerTargets(markers, trackedPids, log);
|
|
326
|
+
pids.push(...swept.pids);
|
|
327
|
+
targets.push(...swept.targets);
|
|
159
328
|
if (targets.length === 0) return [];
|
|
160
329
|
|
|
161
330
|
await escalateKill(targets, {
|
|
@@ -955,6 +1124,19 @@ ${msgBlock}`;
|
|
|
955
1124
|
shell: false,
|
|
956
1125
|
stdio: ["ignore", "pipe", "pipe"],
|
|
957
1126
|
env: getSafeEnv(hasEnvOverrides ? envOverrides : undefined), // F2: Environment isolation + depth tracking
|
|
1127
|
+
// #299 Option B D1: POSIX only — the child becomes its OWN session and
|
|
1128
|
+
// process-group leader (`pgid == pid`), so every kill path can signal
|
|
1129
|
+
// the whole tree (including env-scrubbed descendants) with
|
|
1130
|
+
// `process.kill(-pid, sig)`. No `unref()`: we track the child, and the
|
|
1131
|
+
// boot scan is its recovery path if the conductor crashes. Boundaries:
|
|
1132
|
+
// (a) an env-scrubbed subtree whose leader is already dead before any
|
|
1133
|
+
// sweep stays unreachable safely (only a surviving marker carrier is
|
|
1134
|
+
// swept); (b) a descendant that calls `setsid` leaves the group by
|
|
1135
|
+
// definition; (c) macOS has no /proc, so the boot scan is unchanged
|
|
1136
|
+
// while the in-session group kill still works; (d) detached children no
|
|
1137
|
+
// longer receive terminal-generated signals — the boot reap is their
|
|
1138
|
+
// recovery path. Windows passes `false` (D6), exactly today's behavior.
|
|
1139
|
+
detached: supportsProcessGroupKill(),
|
|
958
1140
|
});
|
|
959
1141
|
|
|
960
1142
|
// Option B U1: register the child for shutdown reap. Keyed by the
|
|
@@ -982,6 +1164,24 @@ ${msgBlock}`;
|
|
|
982
1164
|
result.stderr += data.toString();
|
|
983
1165
|
});
|
|
984
1166
|
|
|
1167
|
+
// #299 Option A: sweep on `exit`, not `close`. `exit` is where the direct
|
|
1168
|
+
// child is confirmed dead; `close` can be delayed indefinitely when a
|
|
1169
|
+
// grandchild inherited the child's stdio and keeps the pipes open. Killing
|
|
1170
|
+
// the leftover marker tree here also releases those pipes so the run can
|
|
1171
|
+
// settle. Success, timeout, and abort all terminate the direct child and so
|
|
1172
|
+
// all funnel through this ONE exit sweep (probe 1c). The per-run marker
|
|
1173
|
+
// keeps it foreign-tree-safe; already-tracked concurrent children are
|
|
1174
|
+
// excluded because two children may legitimately share a marker (B5).
|
|
1175
|
+
let sweepPromise: Promise<number[]> | undefined;
|
|
1176
|
+
proc.on("exit", () => {
|
|
1177
|
+
const excludePids = new Set<number>();
|
|
1178
|
+
for (const entry of activeChildren.values()) {
|
|
1179
|
+
if (entry.proc.pid !== undefined) excludePids.add(entry.proc.pid);
|
|
1180
|
+
}
|
|
1181
|
+
if (proc.pid !== undefined) excludePids.add(proc.pid);
|
|
1182
|
+
sweepPromise = sweepMarkerChildren(childMarker, excludePids, log);
|
|
1183
|
+
});
|
|
1184
|
+
|
|
985
1185
|
proc.on("close", (code, signal) => {
|
|
986
1186
|
activeChildren.delete(registryToken);
|
|
987
1187
|
if (buffer.trim()) {
|
|
@@ -1003,7 +1203,10 @@ ${msgBlock}`;
|
|
|
1003
1203
|
result.errorMessage = `${SUBAGENT_SIGNAL_KILLED_MESSAGE} (${signal ?? "unknown"})`;
|
|
1004
1204
|
log?.warn("Subagent killed by external signal", { signal, pid: proc.pid });
|
|
1005
1205
|
}
|
|
1006
|
-
|
|
1206
|
+
// #299 Option A: settle only AFTER the exit sweep finished, so no
|
|
1207
|
+
// in-session marker tree outlives the run (probe 1c). Result semantics
|
|
1208
|
+
// above are untouched.
|
|
1209
|
+
void (sweepPromise ?? Promise.resolve([])).then(() => resolve(result.exitCode));
|
|
1007
1210
|
});
|
|
1008
1211
|
|
|
1009
1212
|
proc.on("error", (err) => {
|
package/src/session-manager.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// Purpose: Background execution: SDK sessions (`createAgentSession`), agent records, timeouts, steering, and settle paths.
|
|
2
2
|
import { randomUUID } from 'crypto';
|
|
3
|
-
import { join } from 'path';
|
|
4
|
-
import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync } from 'fs';
|
|
5
|
-
import type { BackgroundAgent, AgentStatus, GitMode, SubagentResult, SubagentRun, ThinkingLevel, SubagentToolOptions, UsageStats, ErrorCategory } from './types';
|
|
3
|
+
import { basename, dirname, join, resolve } from 'path';
|
|
4
|
+
import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, rmSync } from 'fs';
|
|
5
|
+
import type { BackgroundAgent, AgentStatus, GitMode, RecoveryMode, SubagentResult, SubagentRun, ThinkingLevel, SubagentToolOptions, UsageStats, ErrorCategory } from './types';
|
|
6
6
|
import { EMPTY_USAGE, classifyError, classifyTerminalOutcome, isProviderError, coherentFailureReason, SUBAGENT_ABORTED_MESSAGE } from './types';
|
|
7
7
|
import { accumulateUsage } from './runner';
|
|
8
8
|
import * as eventBus from './event-bus';
|
|
@@ -78,6 +78,90 @@ function ensureStorageDir(): void {
|
|
|
78
78
|
mkdirSync(STORAGE_DIR, { recursive: true, mode: 0o700 });
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
// Option B U2 (D1/D2): file-backed background sessions + retention.
|
|
83
|
+
//
|
|
84
|
+
// Background runs are persisted by the SDK's SessionManager-created files, one
|
|
85
|
+
// per-run directory BESIDE the agent records (`.pi/subagents/sessions/<id>/`).
|
|
86
|
+
// The directory NAME is the run→file mapping — no new record field is needed,
|
|
87
|
+
// and no resume/re-drive/UI logic lives here (that is U3/U4). The per-run dir is
|
|
88
|
+
// removed when the run reaches a TERMINAL state; a run marked interrupted (U1's
|
|
89
|
+
// boot scan, D3) KEEPS its file until it is resolved by a later unit.
|
|
90
|
+
//
|
|
91
|
+
// Boundaries (stated once, referenced by the PR): no resume (U4); no UI/notice/
|
|
92
|
+
// abandon (U3); background is in-process only; a session file has a single
|
|
93
|
+
// writer (the settled run's own process).
|
|
94
|
+
//
|
|
95
|
+
// D5 measurement (one week of use):
|
|
96
|
+
// du -sh .pi/subagents/sessions
|
|
97
|
+
// find .pi/subagents/sessions -name '*.jsonl' | wc -l
|
|
98
|
+
// The expected steady state is EMPTY between runs (terminal runs prune their
|
|
99
|
+
// dir) plus one dir per currently-interrupted run — a non-empty count of
|
|
100
|
+
// settled runs means a prune path regressed. There is no runtime metric.
|
|
101
|
+
// ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
/** Subdirectory of the agent-record store that holds per-run session dirs. */
|
|
104
|
+
const SESSIONS_SUBDIR = 'sessions';
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The per-run session directory for `id`: `<STORAGE_DIR>/sessions/<id>`.
|
|
108
|
+
* The id is validated first — this path is fed to SessionManager.create() and
|
|
109
|
+
* to rmSync(), so a traversal id must never reach either.
|
|
110
|
+
*/
|
|
111
|
+
export function backgroundSessionDir(id: string): string {
|
|
112
|
+
assertSafeAgentId(id);
|
|
113
|
+
return join(STORAGE_DIR, SESSIONS_SUBDIR, id);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Remove ONE per-run session directory. Defensive by construction: the target
|
|
118
|
+
* must look like `<...>/sessions/<id>` (basename === id, parent basename ===
|
|
119
|
+
* 'sessions') before rmSync runs, so a caller can never delete a path outside
|
|
120
|
+
* the sessions tree. Idempotent (force: true); best-effort (never throws).
|
|
121
|
+
*/
|
|
122
|
+
function pruneSessionDir(id: string, dir: string): void {
|
|
123
|
+
try {
|
|
124
|
+
assertSafeAgentId(id);
|
|
125
|
+
} catch {
|
|
126
|
+
log.error(`Refusing to prune session dir for invalid id: ${id}`);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const target = resolve(dir);
|
|
130
|
+
if (basename(target) !== id || basename(dirname(target)) !== SESSIONS_SUBDIR) {
|
|
131
|
+
log.error(`Refusing to prune session dir outside ${SESSIONS_SUBDIR}/: ${target}`);
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
rmSync(target, { recursive: true, force: true });
|
|
136
|
+
} catch (err) {
|
|
137
|
+
// Pruning is hygiene — it must never break execution or a settle path.
|
|
138
|
+
log.warn(`Failed to prune session dir for ${id}`, {
|
|
139
|
+
error: err instanceof Error ? err.message : String(err),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* U2 (D2): remove the persisted session for a finalized/abandoned background
|
|
146
|
+
* run. Validated (throws on an unsafe id) and idempotent — a missing directory
|
|
147
|
+
* is a success. Exposed for U3's abandon action to reuse; the internal settle
|
|
148
|
+
* path prunes the CAPTURED directory instead so a mid-run STORAGE_DIR change
|
|
149
|
+
* (tests swap it per case) can never mis-target.
|
|
150
|
+
*
|
|
151
|
+
* U3 (D5) documented boundary — abandon after a STORAGE_DIR change: this
|
|
152
|
+
* PUBLIC helper recomputes the root from the CURRENT STORAGE_DIR, so if the
|
|
153
|
+
* root moved between the spawn and the abandon it prunes the new root (a
|
|
154
|
+
* no-op when nothing lives there) and leaves the old-root session dir behind
|
|
155
|
+
* as debris. That is prod-safe today (only tests swap the root; production
|
|
156
|
+
* uses one root per process), so U3 does NOT try to track the captured dir
|
|
157
|
+
* across sessions — the run's `_sessionFile` is live-only process state and is
|
|
158
|
+
* not persisted (D4). Debris, if any, is bounded and inspectable under the old
|
|
159
|
+
* root.
|
|
160
|
+
*/
|
|
161
|
+
export function pruneBackgroundSession(id: string): void {
|
|
162
|
+
pruneSessionDir(id, backgroundSessionDir(id));
|
|
163
|
+
}
|
|
164
|
+
|
|
81
165
|
/**
|
|
82
166
|
* Persist agent record to disk
|
|
83
167
|
*/
|
|
@@ -95,8 +179,10 @@ function persistAgent(agent: BackgroundAgent): void {
|
|
|
95
179
|
const filePath = join(STORAGE_DIR, `${agent.id}.json`);
|
|
96
180
|
// Strip live/non-serializable fields before stringify — `_sessionRef` holds the
|
|
97
181
|
// ENTIRE live pi session graph (1.4MB+ for an empty session, and uninitialized
|
|
98
|
-
// getters like Theme can make JSON.stringify throw).
|
|
99
|
-
|
|
182
|
+
// getters like Theme can make JSON.stringify throw). `_sessionFile` (U2) is the
|
|
183
|
+
// on-disk session path — live-only bookkeeping for precise pruning, never a
|
|
184
|
+
// persisted record field (D4).
|
|
185
|
+
const { _sessionRef, _sessionFile, ...persistable } = agent;
|
|
100
186
|
try {
|
|
101
187
|
// F6 (issue #29): records contain task, error, result.messages (full
|
|
102
188
|
// conversation) and finalOutput — write owner-only (0o600) so other local
|
|
@@ -413,6 +499,10 @@ export async function spawnBackgroundSession(
|
|
|
413
499
|
dispatchId?: string;
|
|
414
500
|
resumeOf?: string;
|
|
415
501
|
attempt?: number;
|
|
502
|
+
/** Option B U3 (D4): recovery posture at dispatch — snapshotted onto the
|
|
503
|
+
* agent record + run entry so the boot notice reads the posture the run
|
|
504
|
+
* was actually dispatched under. Absent reads as "offer". */
|
|
505
|
+
recoveryPosture?: RecoveryMode;
|
|
416
506
|
}
|
|
417
507
|
): Promise<BackgroundAgent> {
|
|
418
508
|
// Serialize access to pi API to prevent concurrent import races
|
|
@@ -442,8 +532,16 @@ export async function spawnBackgroundSession(
|
|
|
442
532
|
}
|
|
443
533
|
}
|
|
444
534
|
|
|
445
|
-
// Create session manager for this background agent
|
|
446
|
-
|
|
535
|
+
// Create the session manager for this background agent. U2 (D1): an explicit
|
|
536
|
+
// per-run sessionDir is REQUIRED — `inMemory` persists nothing (no post-hoc
|
|
537
|
+
// inspection, no U4 resume) and the SDK default (`~/.pi/agent/sessions/<cwd-key>/`,
|
|
538
|
+
// passed as `undefined`) would scatter our runs into the global store. The
|
|
539
|
+
// directory lives beside the agent record: `.pi/subagents/sessions/<id>/`.
|
|
540
|
+
const sessionManager = SessionManager.create(effectiveCwd, backgroundSessionDir(id));
|
|
541
|
+
// Capture the exact path the SDK will write. The settle path derives the
|
|
542
|
+
// prune target from THIS (not from a freshly-computed STORAGE_DIR path) so a
|
|
543
|
+
// change of STORAGE_DIR between spawn and settle cannot mis-target the delete.
|
|
544
|
+
const sessionFile: string | undefined = sessionManager.getSessionFile();
|
|
447
545
|
const settingsManager = SettingsManager.create(effectiveCwd);
|
|
448
546
|
|
|
449
547
|
// E20: Inject the subagent prompt into the session's system prompt.
|
|
@@ -546,6 +644,11 @@ export async function spawnBackgroundSession(
|
|
|
546
644
|
resumeOf: params.resumeOf,
|
|
547
645
|
attempt,
|
|
548
646
|
owner,
|
|
647
|
+
// Option B U3 (D4): per-run recovery posture snapshot. Additive — absent
|
|
648
|
+
// reads as "offer" via `runRecoveryPosture`/the notice collector.
|
|
649
|
+
recoveryPosture: params.recoveryPosture,
|
|
650
|
+
// U2: live-only pointer to this run's session file (stripped before persist).
|
|
651
|
+
_sessionFile: sessionFile,
|
|
549
652
|
};
|
|
550
653
|
|
|
551
654
|
agents.set(id, agent);
|
|
@@ -575,6 +678,7 @@ export async function spawnBackgroundSession(
|
|
|
575
678
|
resumeOf: params.resumeOf,
|
|
576
679
|
attempt,
|
|
577
680
|
owner,
|
|
681
|
+
recoveryPosture: params.recoveryPosture,
|
|
578
682
|
};
|
|
579
683
|
persistRunRecord(pi, run, "background");
|
|
580
684
|
|
|
@@ -582,6 +686,15 @@ export async function spawnBackgroundSession(
|
|
|
582
686
|
// every terminal branch that flips agent.status also finalizes the run entry
|
|
583
687
|
// ("done"/"failed"). The guard keeps the entry single-finalized even when a
|
|
584
688
|
// settle handler throws after finalizing (markTerminalBestEffort catch-all).
|
|
689
|
+
//
|
|
690
|
+
// U2 (D2) retention: this is the SINGLE terminal choke point for a background
|
|
691
|
+
// run — git-setup refusal, a synchronous prompt throw, the .then completed /
|
|
692
|
+
// provider-failed / aborted branches, the .catch stopped / failed branches and
|
|
693
|
+
// the markTerminalBestEffort catch-all ALL land here exactly once (the
|
|
694
|
+
// runFinalized guard). Pruning here therefore covers every settle path. The
|
|
695
|
+
// index.ts poller crash notices are NOT settlements (they flip only the live
|
|
696
|
+
// UI state and leave the run/agent record to this settle); an interrupted run
|
|
697
|
+
// (U1 boot-scan mark) never reaches this function and KEEPS its file (D3).
|
|
585
698
|
let runFinalized = false;
|
|
586
699
|
const finalizeRunEntry = (
|
|
587
700
|
status: "done" | "failed",
|
|
@@ -590,6 +703,15 @@ export async function spawnBackgroundSession(
|
|
|
590
703
|
): void => {
|
|
591
704
|
if (runFinalized) return;
|
|
592
705
|
runFinalized = true;
|
|
706
|
+
// U2 (D2): drop the persisted session now that the run is terminal. Done
|
|
707
|
+
// FIRST (before the run-entry write below) so a persist throw cannot leave
|
|
708
|
+
// a terminal run's session file behind. Interrupted runs keep theirs.
|
|
709
|
+
if (!agent.interruptedAt) {
|
|
710
|
+
pruneSessionDir(
|
|
711
|
+
id,
|
|
712
|
+
agent._sessionFile ? dirname(agent._sessionFile) : backgroundSessionDir(id),
|
|
713
|
+
);
|
|
714
|
+
}
|
|
593
715
|
const usage = agent.result?.usage ?? EMPTY_USAGE;
|
|
594
716
|
const finalOutput = agent.finalOutput ?? "";
|
|
595
717
|
// Issue #179 (D1/D2): the classifier's explicit category wins (it knows the
|
package/src/state.ts
CHANGED
|
@@ -17,6 +17,7 @@ import type {
|
|
|
17
17
|
ThinkingLevel,
|
|
18
18
|
ApprovalMode,
|
|
19
19
|
CompletionNotifyMode,
|
|
20
|
+
RecoveryMode,
|
|
20
21
|
CircuitBreakerState,
|
|
21
22
|
Priority,
|
|
22
23
|
} from "./types";
|
|
@@ -24,12 +25,15 @@ import type { TranscriptMessage } from "./transcript-tail";
|
|
|
24
25
|
import {
|
|
25
26
|
isSubagentStateShape,
|
|
26
27
|
isSubagentRunShape,
|
|
28
|
+
isInterruptedRun,
|
|
27
29
|
CUSTOM_ENTRY_TYPES,
|
|
28
30
|
MAX_RUN_HISTORY_ENTRIES,
|
|
29
31
|
DEFAULT_MAX_SUBAGENT_DEPTH,
|
|
30
32
|
DEFAULT_SESSION_COST_LIMIT,
|
|
31
33
|
DEFAULT_COMPLETION_NOTIFY,
|
|
32
34
|
COMPLETION_NOTIFY_MODES,
|
|
35
|
+
DEFAULT_RECOVERY,
|
|
36
|
+
RECOVERY_MODES,
|
|
33
37
|
MAX_CONSECUTIVE_FAILURES,
|
|
34
38
|
CIRCUIT_BREAKER_RESET_MS,
|
|
35
39
|
CIRCUIT_DEGRADED_THINKING,
|
|
@@ -107,7 +111,10 @@ export function resolveTerminalRunEntry(
|
|
|
107
111
|
* renders every run twice and the spawn row shows a stale "running" forever.
|
|
108
112
|
* This collapses each id to its terminal-preferred entry via the ONE shared
|
|
109
113
|
* rule (`resolveTerminalRunEntry`). Runs still in flight (spawn entry, no
|
|
110
|
-
* terminal entry yet) are omitted: live state belongs to the monitor.
|
|
114
|
+
* terminal entry yet) are omitted: live state belongs to the monitor. U3 (D1)
|
|
115
|
+
* carves ONE exception — an INTERRUPTED run (running + `interruptedAt`, per
|
|
116
|
+
* `isInterruptedRun`) is surfaced, because the human must see it to recover or
|
|
117
|
+
* abandon it.
|
|
111
118
|
*
|
|
112
119
|
* Ordering is preserved from `entries`; callers pass `getRunEntries` output
|
|
113
120
|
* (newest-first via `cleanupRuns`), which stays the single ordering authority —
|
|
@@ -120,11 +127,28 @@ export function collapseRunsForHistory(entries: SubagentRun[]): SubagentRun[] {
|
|
|
120
127
|
if (seen.has(entry.id)) continue;
|
|
121
128
|
seen.add(entry.id);
|
|
122
129
|
const resolved = resolveTerminalRunEntry(entries, entry.id);
|
|
123
|
-
if (resolved && resolved.status !== "running"
|
|
130
|
+
if (resolved && (resolved.status !== "running" || isInterruptedRun(resolved))) {
|
|
131
|
+
display.push(resolved);
|
|
132
|
+
}
|
|
124
133
|
}
|
|
125
134
|
return display;
|
|
126
135
|
}
|
|
127
136
|
|
|
137
|
+
/**
|
|
138
|
+
* U3 (D3): the retry surface's candidate projection — settled FAILED runs plus
|
|
139
|
+
* unresolved INTERRUPTED runs, one row per id. Built on the SAME
|
|
140
|
+
* terminal-preferred collapse the history list uses, so the two surfaces can
|
|
141
|
+
* never disagree about which record represents a run. The retry offer reuses
|
|
142
|
+
* the existing `retryRunId` machinery (recorded params restored by
|
|
143
|
+
* `resolveRetryParams`), so an interrupted run is recoverable without any new
|
|
144
|
+
* delegate_task parameter.
|
|
145
|
+
*/
|
|
146
|
+
export function retryableRuns(entries: SubagentRun[]): SubagentRun[] {
|
|
147
|
+
return collapseRunsForHistory(entries).filter(
|
|
148
|
+
(r) => r.status === "failed" || isInterruptedRun(r),
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
128
152
|
// ---------------------------------------------------------------------------
|
|
129
153
|
// SessionState — session-bound mutable state
|
|
130
154
|
// ---------------------------------------------------------------------------
|
|
@@ -196,6 +220,7 @@ export class SessionState {
|
|
|
196
220
|
gitMode: "none",
|
|
197
221
|
approvalMode: "writes",
|
|
198
222
|
completionNotify: DEFAULT_COMPLETION_NOTIFY,
|
|
223
|
+
recovery: DEFAULT_RECOVERY,
|
|
199
224
|
maxHistoryEntries: MAX_RUN_HISTORY_ENTRIES,
|
|
200
225
|
sessionCostLimit: DEFAULT_SESSION_COST_LIMIT,
|
|
201
226
|
perTaskCostEstimate: 0,
|
|
@@ -221,6 +246,7 @@ export class SessionState {
|
|
|
221
246
|
gitMode: this.config.gitMode,
|
|
222
247
|
approvalMode: this.config.approvalMode,
|
|
223
248
|
completionNotify: this.config.completionNotify,
|
|
249
|
+
recovery: this.config.recovery,
|
|
224
250
|
maxHistoryEntries: this.config.maxHistoryEntries,
|
|
225
251
|
sessionCostLimit: this.config.sessionCostLimit,
|
|
226
252
|
perTaskCostEstimate: this.config.perTaskCostEstimate,
|
|
@@ -288,6 +314,9 @@ export class SessionState {
|
|
|
288
314
|
COMPLETION_NOTIFY_MODES.includes(data.completionNotify as CompletionNotifyMode)
|
|
289
315
|
? data.completionNotify
|
|
290
316
|
: DEFAULT_COMPLETION_NOTIFY;
|
|
317
|
+
this.config.recovery = RECOVERY_MODES.includes(data.recovery as RecoveryMode)
|
|
318
|
+
? data.recovery
|
|
319
|
+
: DEFAULT_RECOVERY;
|
|
291
320
|
if (data.maxHistoryEntries !== undefined) this.config.maxHistoryEntries = data.maxHistoryEntries;
|
|
292
321
|
if (data.sessionCostLimit !== undefined) this.config.sessionCostLimit = data.sessionCostLimit;
|
|
293
322
|
if (data.perTaskCostEstimate !== undefined) this.config.perTaskCostEstimate = data.perTaskCostEstimate;
|
|
@@ -577,6 +606,7 @@ export class SessionState {
|
|
|
577
606
|
this.config.gitMode = "none";
|
|
578
607
|
this.config.approvalMode = "writes";
|
|
579
608
|
this.config.completionNotify = DEFAULT_COMPLETION_NOTIFY;
|
|
609
|
+
this.config.recovery = DEFAULT_RECOVERY;
|
|
580
610
|
this.config.maxHistoryEntries = MAX_RUN_HISTORY_ENTRIES;
|
|
581
611
|
this.config.sessionCostLimit = DEFAULT_SESSION_COST_LIMIT;
|
|
582
612
|
this.config.perTaskCostEstimate = 0;
|
package/src/tui-format.ts
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
* without a TUI harness.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { formatTokens } from "./types";
|
|
10
|
+
import { formatTokens, isInterruptedRun } from "./types";
|
|
11
|
+
import type { SubagentRun } from "./types";
|
|
11
12
|
|
|
12
13
|
/** Braille spinner frames, indexed by Math.floor(now / 150) % 10. */
|
|
13
14
|
export const LIVE_SPINNER_FRAMES: readonly string[] = [
|
|
@@ -50,3 +51,58 @@ export function formatLiveRowDim(
|
|
|
50
51
|
): string {
|
|
51
52
|
return ` [${id.slice(0, 8)}] \u2191${formatTokens(usage.input)} \u2193${formatTokens(usage.output)} ${elapsedStr}`;
|
|
52
53
|
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Display name for a Run History / retry-surface row, shared by BOTH surfaces
|
|
57
|
+
* so they can never name the same run differently (Fix B item 5). Explicit
|
|
58
|
+
* label wins, else the task truncated to 60 chars, else the short id — the last
|
|
59
|
+
* case being a task-less foreground-interrupted row (no task persisted).
|
|
60
|
+
*/
|
|
61
|
+
export function runDisplayName(run: Pick<SubagentRun, "id" | "label" | "task">): string {
|
|
62
|
+
return run.label || run.task.slice(0, 60) || run.id.slice(0, 8);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// U3 (D1/D3): Run History / retry-surface row projection
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The human status of a Run History row. Keys on `isInterruptedRun` FIRST, so a
|
|
71
|
+
* run marked interrupted renders as `interrupted` and NEVER falls through to its
|
|
72
|
+
* raw `\"running\"` status (D1). The raw status is only consulted for an
|
|
73
|
+
* unmarked record.
|
|
74
|
+
*/
|
|
75
|
+
export type RunHistoryStatus = "interrupted" | "running" | "done" | "failed";
|
|
76
|
+
|
|
77
|
+
export function runHistoryStatus(run: SubagentRun): RunHistoryStatus {
|
|
78
|
+
if (isInterruptedRun(run)) return "interrupted";
|
|
79
|
+
if (run.status === "done" || run.status === "failed") return run.status;
|
|
80
|
+
return "running";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Human label for a Run History row's status. */
|
|
84
|
+
export function runHistoryStatusLabel(run: SubagentRun): string {
|
|
85
|
+
switch (runHistoryStatus(run)) {
|
|
86
|
+
case "interrupted":
|
|
87
|
+
return "Interrupted";
|
|
88
|
+
case "done":
|
|
89
|
+
return "Completed";
|
|
90
|
+
case "failed":
|
|
91
|
+
return "Failed";
|
|
92
|
+
default:
|
|
93
|
+
return "Running";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The offer text shown when a retry-surface row is selected. Names the run id
|
|
99
|
+
* and the `retryRunId` parameter explicitly (D3), so the conductor re-dispatches
|
|
100
|
+
* the recorded task/params without a new delegate_task parameter.
|
|
101
|
+
*/
|
|
102
|
+
export function buildRetryOfferMessage(run: SubagentRun): string {
|
|
103
|
+
const statusWord = isInterruptedRun(run) ? "interrupted" : "failed";
|
|
104
|
+
return (
|
|
105
|
+
`Ask the LLM to retry: "Use delegate_task with retryRunId: ${run.id} ` +
|
|
106
|
+
`to retry the ${statusWord} run ${run.id.slice(0, 8)} with its recorded params."`
|
|
107
|
+
);
|
|
108
|
+
}
|