brl-subagent 2.5.0 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT.md +9 -2
- package/README.md +6 -3
- package/package.json +6 -6
- package/src/history.ts +19 -9
- package/src/index.ts +206 -35
- package/src/kill-escalation.ts +338 -6
- package/src/metrics.ts +5 -3
- package/src/notify-completion.ts +23 -0
- package/src/proc.ts +270 -20
- package/src/recovery-engine.ts +100 -14
- package/src/recovery-surface.ts +374 -0
- package/src/recovery.ts +19 -1
- package/src/run-registry.ts +64 -3
- package/src/runner.ts +216 -13
- package/src/session-manager.ts +129 -7
- package/src/state.ts +32 -2
- package/src/tui-format.ts +57 -1
- package/src/tui.ts +102 -22
- package/src/types.ts +66 -1
- package/src/unit-run.ts +8 -1
package/src/proc.ts
CHANGED
|
@@ -21,9 +21,10 @@
|
|
|
21
21
|
import * as fs from "node:fs";
|
|
22
22
|
import * as path from "node:path";
|
|
23
23
|
import type { ProcessOwner } from "./types";
|
|
24
|
-
import { SIGKILL_GRACE_MS } from "./types";
|
|
24
|
+
import { SIGKILL_GRACE_MS, SIGKILL_POLL_MS } from "./types";
|
|
25
25
|
import { CHILD_MARKER_ENV_KEY } from "./sanitize";
|
|
26
26
|
import type { RecoveryDeps } from "./recovery-engine";
|
|
27
|
+
import { supportsProcessGroupKill } from "./kill-escalation";
|
|
27
28
|
|
|
28
29
|
const PROC_ROOT = "/proc";
|
|
29
30
|
|
|
@@ -57,27 +58,217 @@ export function isProcAvailable(): boolean {
|
|
|
57
58
|
return procAvailable;
|
|
58
59
|
}
|
|
59
60
|
|
|
61
|
+
/** The outcome of a `/proc/<pid>/stat` read: the fields, or why it failed. */
|
|
62
|
+
type StatFieldsRead =
|
|
63
|
+
| { ok: true; fields: string[] }
|
|
64
|
+
| { ok: false; code: string | undefined };
|
|
65
|
+
|
|
60
66
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
67
|
+
* The `readStatFields` primitive WITH the failure cause preserved: `code` is the
|
|
68
|
+
* errno of a failed read (`"ENOENT"` when the pid is absent) or undefined when
|
|
69
|
+
* /proc is unavailable, the pid is invalid, or the read failed for a reason
|
|
70
|
+
* other than a missing file. `pidGone` needs the cause to separate "the pid was
|
|
71
|
+
* REAPED" (ENOENT — absence-of-process evidence) from "the read failed and we
|
|
72
|
+
* cannot tell" (everything else).
|
|
66
73
|
*/
|
|
67
|
-
|
|
68
|
-
if (!isProcAvailable() || !Number.isInteger(pid) || pid <= 0)
|
|
74
|
+
function readStatFieldsDetailed(pid: number): StatFieldsRead {
|
|
75
|
+
if (!isProcAvailable() || !Number.isInteger(pid) || pid <= 0) {
|
|
76
|
+
return { ok: false, code: undefined };
|
|
77
|
+
}
|
|
69
78
|
try {
|
|
70
79
|
const stat = fs.readFileSync(path.join(PROC_ROOT, String(pid), "stat"), "utf-8");
|
|
71
80
|
const close = stat.lastIndexOf(")");
|
|
72
|
-
if (close < 0) return undefined;
|
|
81
|
+
if (close < 0) return { ok: false, code: undefined };
|
|
73
82
|
const rest = stat.slice(close + 1).trim();
|
|
74
|
-
if (!rest) return undefined;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
return
|
|
83
|
+
if (!rest) return { ok: false, code: undefined };
|
|
84
|
+
return { ok: true, fields: rest.split(/\s+/) };
|
|
85
|
+
} catch (err) {
|
|
86
|
+
return { ok: false, code: (err as NodeJS.ErrnoException).code };
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The whitespace-split `/proc/<pid>/stat` fields AFTER the `(comm)` field, or
|
|
92
|
+
* undefined when /proc is unavailable, the pid is invalid, or the process
|
|
93
|
+
* vanished. Shared by the start-token, process-group and group-liveness readers.
|
|
94
|
+
* The comm field is skipped by slicing at the LAST `)` so a name containing
|
|
95
|
+
* spaces or parens cannot shift the parse.
|
|
96
|
+
*/
|
|
97
|
+
function readStatFields(pid: number): string[] | undefined {
|
|
98
|
+
const read = readStatFieldsDetailed(pid);
|
|
99
|
+
return read.ok ? read.fields : undefined;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Read a process's start-time token — `/proc/<pid>/stat` field 22 (1-indexed),
|
|
104
|
+
* i.e. index 19 of the whitespace-split fields AFTER the `(comm)` field.
|
|
105
|
+
* Returns undefined when /proc is unavailable, the pid is invalid, or the
|
|
106
|
+
* process vanished.
|
|
107
|
+
*/
|
|
108
|
+
export function readStartToken(pid: number): string | undefined {
|
|
109
|
+
const token = readStatFields(pid)?.[19];
|
|
110
|
+
return token && /^\d+$/.test(token) ? token : undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Read a process's process-group id — `/proc/<pid>/stat` field 5 (index 2 after
|
|
115
|
+
* `(comm)`). Returns undefined when /proc is unavailable, the pid is invalid, or
|
|
116
|
+
* the process vanished.
|
|
117
|
+
*/
|
|
118
|
+
export function readProcessGroup(pid: number): number | undefined {
|
|
119
|
+
const pgid = Number(readStatFields(pid)?.[2]);
|
|
120
|
+
return Number.isInteger(pgid) && pgid > 0 ? pgid : undefined;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* #299 fix A.2 — the THREE-state answer to "does process group `pgid` still have
|
|
125
|
+
* a LIVE member?":
|
|
126
|
+
*
|
|
127
|
+
* - `true` — at least one live member was observed.
|
|
128
|
+
* - `false` — a COMPLETED inspection found no live member: POSITIVELY empty.
|
|
129
|
+
* - `undefined` — the inspection was UNAVAILABLE or PARTIAL. Absence of
|
|
130
|
+
* evidence is never evidence of absence, so callers must treat `undefined`
|
|
131
|
+
* as UNKNOWN — never as empty.
|
|
132
|
+
*
|
|
133
|
+
* `/proc` host-level unavailability is the documented platform boundary, not
|
|
134
|
+
* `undefined`: with no `/proc` at all the caller degrades to leader liveness
|
|
135
|
+
* (this returns `false`). `undefined` is reserved for an inspection that FAILED
|
|
136
|
+
* on a `/proc`-capable host — a `/proc` listing failure, or a member's stat read
|
|
137
|
+
* that failed for a reason other than the process being gone (`EACCES`, `EIO`, a
|
|
138
|
+
* parse anomaly). A member that genuinely vanished during the scan is skipped
|
|
139
|
+
* (ENOENT — it cannot be a live member).
|
|
140
|
+
*/
|
|
141
|
+
export type GroupMembership = boolean | undefined;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* #299 fix A.4 — the kernel's process-state letters, i.e. the valid values of
|
|
145
|
+
* field 3 of `/proc/<pid>/stat`. A state token OUTSIDE this set means the
|
|
146
|
+
* record was mis-parsed (a truncated or field-shifted read), so the inspection
|
|
147
|
+
* is PARTIAL rather than a positive non-membership. The set is the union of the
|
|
148
|
+
* Linux `task_state_array`: R (running), S (sleeping), D (uninterruptible
|
|
149
|
+
* sleep), Z (zombie), T (stopped), t (tracing stop), X/x (dead), K (wakekill),
|
|
150
|
+
* W (waking, pre-3.8), P (parked), I (idle).
|
|
151
|
+
*/
|
|
152
|
+
const PROCESS_STATE_LETTERS = new Set(["R", "S", "D", "Z", "T", "t", "X", "x", "K", "W", "P", "I"]);
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* The kernel's `PID_MAX_LIMIT` on 64-bit Linux. A pgrp above this is not a real
|
|
156
|
+
* process-group id, so a record carrying one is mis-parsed — the inspection is
|
|
157
|
+
* PARTIAL, not a positive non-membership. `pid_max` can be configured lower; a
|
|
158
|
+
* value above the kernel CEILING can never be genuine at all.
|
|
159
|
+
*/
|
|
160
|
+
const PID_MAX_LIMIT = 4_194_304;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* #299 fix A.5 — the SINGLE source of the `/proc/<pid>/stat` PREFIX contract,
|
|
164
|
+
* shared by every group-membership read. Given the whitespace-split fields after
|
|
165
|
+
* `(comm)`, it returns `{ state, pgrp }` ONLY when the record is WELL-FORMED
|
|
166
|
+
* enough to trust, and `undefined` (malformed/truncated/shifted) otherwise.
|
|
167
|
+
*
|
|
168
|
+
* The contract is deliberately strict, because `groupHasMembers` turns a clean
|
|
169
|
+
* scan into the actionable claim "the group is POSITIVELY empty" (an escalation
|
|
170
|
+
* early-exit and a clean survivor report). Every field the prefix relies on is
|
|
171
|
+
* therefore validated by SHAPE and ALIGNMENT:
|
|
172
|
+
*
|
|
173
|
+
* - `fields[0]` (state) must be a kernel process-state letter — numerics,
|
|
174
|
+
* punctuation, empty and multi-char tokens are all malformed.
|
|
175
|
+
* - `fields[1]` (ppid) must be a non-negative integer — this is the ALIGNMENT
|
|
176
|
+
* check. A valid state followed by a non-numeric ppid (`R S 1 9999`) is a
|
|
177
|
+
* SHIFTED record whose "pgrp" slot happens to hold a real integer; trusting
|
|
178
|
+
* it would read a malformed record as a justified non-member.
|
|
179
|
+
* - `fields[2]` (pgrp) must be an integer in `[0, PID_MAX_LIMIT]` — negatives
|
|
180
|
+
* and oversized values are impossible pgrps. `0` is LEGAL: kernel threads
|
|
181
|
+
* carry pgrp 0 and must not taint an otherwise-clean scan.
|
|
182
|
+
*/
|
|
183
|
+
function parseStatPrefix(fields: string[]): { state: string; pgrp: number } | undefined {
|
|
184
|
+
const state = fields[0];
|
|
185
|
+
const ppid = fields[1];
|
|
186
|
+
const pgrpField = fields[2];
|
|
187
|
+
if (state === undefined || !PROCESS_STATE_LETTERS.has(state)) return undefined;
|
|
188
|
+
// Strict decimal shape: `Number("1e3")`/`Number("0x10")`/`Number("")` would
|
|
189
|
+
// otherwise sneak through as integers, so validate the TOKEN, not the cast.
|
|
190
|
+
if (ppid === undefined || !/^\d+$/.test(ppid)) return undefined;
|
|
191
|
+
if (pgrpField === undefined || !/^\d+$/.test(pgrpField)) return undefined;
|
|
192
|
+
const pgrp = Number(pgrpField);
|
|
193
|
+
if (pgrp > PID_MAX_LIMIT) return undefined;
|
|
194
|
+
return { state, pgrp };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** The states whose processes are NOT live group members (an unreaped corpse
|
|
198
|
+
* cannot hold a group non-empty): `Z` (zombie), `X`/`x` (dead). Only ever
|
|
199
|
+
* consulted for a record that `parseStatPrefix` already accepted. */
|
|
200
|
+
const NON_MEMBER_STATES = new Set(["Z", "X", "x"]);
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Scans `/proc` for processes whose pgrp (field 5) equals `pgid`, skipping the
|
|
204
|
+
* leader itself and non-live states (`Z`/`X`/`x` — an unreaped corpse is not a
|
|
205
|
+
* live member and must not hold the group non-empty). A group's leader exiting
|
|
206
|
+
* does NOT empty the group, so this is what lets `escalateKill` keep waiting for
|
|
207
|
+
* a member's cleanup window.
|
|
208
|
+
*
|
|
209
|
+
* The return contract is EXACT (see `GroupMembership`): `false` may be returned
|
|
210
|
+
* ONLY after a COMPLETE scan in which every record parsed as a well-formed
|
|
211
|
+
* prefix. Any unparseable record taints the scan to `undefined` (unknown) — the
|
|
212
|
+
* check lives in `parseStatPrefix`, so there is no fall-through that can turn
|
|
213
|
+
* malformed input into a positively-empty `false`.
|
|
214
|
+
*/
|
|
215
|
+
export function groupHasMembers(pgid: number): GroupMembership {
|
|
216
|
+
// Platform boundary: no /proc at all → the caller falls back to leader
|
|
217
|
+
// liveness (documented non-Linux boundary). Same for an invalid pgid.
|
|
218
|
+
if (!Number.isInteger(pgid) || pgid <= 0 || !isProcAvailable()) return false;
|
|
219
|
+
let entries: string[];
|
|
220
|
+
try {
|
|
221
|
+
entries = fs.readdirSync(PROC_ROOT);
|
|
78
222
|
} catch {
|
|
223
|
+
// A listing failure on a /proc-capable host leaves the scan PARTIAL: we
|
|
224
|
+
// cannot tell whether a member exists, so the answer is unknown.
|
|
79
225
|
return undefined;
|
|
80
226
|
}
|
|
227
|
+
for (const entry of entries) {
|
|
228
|
+
if (!/^\d+$/.test(entry)) continue;
|
|
229
|
+
const pid = Number(entry);
|
|
230
|
+
if (pid === pgid) continue; // the leader (possibly a zombie) is not a member
|
|
231
|
+
const read = readStatFieldsDetailed(pid);
|
|
232
|
+
if (!read.ok) {
|
|
233
|
+
// A member gone mid-scan is skippable (it cannot be a live member); any
|
|
234
|
+
// OTHER failure makes the inspection partial → unknown.
|
|
235
|
+
if (read.code === "ENOENT") continue;
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
// #299 fix A.5: ONE strict prefix validation. A truncated, shifted or
|
|
239
|
+
// otherwise malformed record (too few fields, non-letter state, non-numeric
|
|
240
|
+
// ppid, impossible pgrp) makes the inspection PARTIAL → unknown, never a
|
|
241
|
+
// positively-empty `false`. Well-formed dead states are skipped; a live
|
|
242
|
+
// member is a match.
|
|
243
|
+
const parsed = parseStatPrefix(read.fields);
|
|
244
|
+
if (parsed === undefined) return undefined;
|
|
245
|
+
if (NON_MEMBER_STATES.has(parsed.state)) continue;
|
|
246
|
+
if (parsed.pgrp === pgid) return true;
|
|
247
|
+
}
|
|
248
|
+
return false;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* #299 review Major 1 — signal-time identity re-check. True only when the LIVE
|
|
253
|
+
* process `pid` carries `CHILD_MARKER_ENV_KEY=<marker>` in its environment.
|
|
254
|
+
* `findByMarker` proves the pid was ours at SCAN time; this proves it is ours at
|
|
255
|
+
* the SIGNAL time, closing the scan→signal pid-reuse window (a pid that exited
|
|
256
|
+
* and was reused by a foreign process no longer carries the marker). Returns
|
|
257
|
+
* false when /proc is unavailable, the pid is invalid, or the process vanished.
|
|
258
|
+
*
|
|
259
|
+
* RESIDUAL: this narrows the reuse window to the read→`kill` syscall boundary —
|
|
260
|
+
* the same exposure the direct signals have carried since Option A. It does NOT
|
|
261
|
+
* eliminate it (the kernel may reuse the pid between this read and the signal).
|
|
262
|
+
*/
|
|
263
|
+
export function pidHasMarker(pid: number, marker: string): boolean {
|
|
264
|
+
if (!marker || !isProcAvailable() || !Number.isInteger(pid) || pid <= 0) return false;
|
|
265
|
+
const needle = `${CHILD_MARKER_ENV_KEY}=${marker}`;
|
|
266
|
+
try {
|
|
267
|
+
const environ = fs.readFileSync(path.join(PROC_ROOT, String(pid), "environ"), "utf-8");
|
|
268
|
+
return environ.split("\0").includes(needle);
|
|
269
|
+
} catch {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
81
272
|
}
|
|
82
273
|
|
|
83
274
|
/** The CURRENT process's owner identity (pid + start token; empty string when unavailable). */
|
|
@@ -85,6 +276,21 @@ export function currentProcessOwner(): ProcessOwner {
|
|
|
85
276
|
return { pid: process.pid, start: readStartToken(process.pid) ?? "" };
|
|
86
277
|
}
|
|
87
278
|
|
|
279
|
+
/**
|
|
280
|
+
* #299 D4: the ONE production process-group signal (`kill(-pgid, sig)`), shared
|
|
281
|
+
* by the recovery deps and `runner.ts`'s child/marker group targets. ESRCH means
|
|
282
|
+
* the group is already empty — the desired end state, so it is swallowed; every
|
|
283
|
+
* other errno is rethrown for the caller's own tolerance. On a platform without
|
|
284
|
+
* process groups neither caller wires this in (`supportsProcessGroupKill`).
|
|
285
|
+
*/
|
|
286
|
+
export function killProcessGroup(pgid: number, signal: NodeJS.Signals): void {
|
|
287
|
+
try {
|
|
288
|
+
process.kill(-pgid, signal);
|
|
289
|
+
} catch (err) {
|
|
290
|
+
if ((err as NodeJS.ErrnoException).code !== "ESRCH") throw err;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
88
294
|
/** Cross-platform pid liveness (`kill -0`). EPERM means the process exists. */
|
|
89
295
|
export function pidAlive(pid: number): boolean {
|
|
90
296
|
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
@@ -96,6 +302,39 @@ export function pidAlive(pid: number): boolean {
|
|
|
96
302
|
}
|
|
97
303
|
}
|
|
98
304
|
|
|
305
|
+
/**
|
|
306
|
+
* #299 review Major 1: true when the pid is GONE — it no longer exists OR is a
|
|
307
|
+
* not-yet-reaped zombie (`Z`/`X`), which `pidAlive` (kill -0) still reports as
|
|
308
|
+
* alive. The signal-time guard uses this to separate a REUSED pid (alive, marker
|
|
309
|
+
* gone) from a merely dead/zombie one, so a zombie leader's pgid is still
|
|
310
|
+
* signaled to reach its surviving members (D5). Without /proc the zombie
|
|
311
|
+
* distinction is unavailable and this degrades to `!pidAlive`.
|
|
312
|
+
*
|
|
313
|
+
* #299 re-review A.1: a POSITIVE liveness sample immediately followed by ENOENT
|
|
314
|
+
* on `/proc/<pid>/stat` means the pid was REAPED between the two reads — it is
|
|
315
|
+
* GONE. `/proc/<pid>/stat` is readable for any process that still exists (a
|
|
316
|
+
* not-yet-reaped zombie included), so ENOENT is absence-of-process evidence;
|
|
317
|
+
* classifying it as "not gone" made the guard treat our own reaped leader as a
|
|
318
|
+
* live FOREIGN pid and refuse BOTH the group SIGTERM and the final group
|
|
319
|
+
* SIGKILL while its group still had live members. Every OTHER read failure
|
|
320
|
+
* (EACCES, a parse anomaly, /proc unavailable) stays conservatively not-gone.
|
|
321
|
+
*
|
|
322
|
+
* PLATFORM BOUNDARY (hidepid=2): on a `/proc` mounted with `hidepid=2` a foreign
|
|
323
|
+
* LIVE pid we do not own can read `kill -0` EPERM (alive) and then
|
|
324
|
+
* `/proc/<pid>/stat` ENOENT (hidden — it looks GONE). The signal guard may then
|
|
325
|
+
* ATTEMPT a group signal despite the marker mismatch. This is a documented
|
|
326
|
+
* platform boundary, NOT a signal-safety guarantee: the kernel's EPERM on the
|
|
327
|
+
* signal constrains the unprivileged blast radius, and the group signal still
|
|
328
|
+
* targets only the pgid we derived (it does not widen the target set).
|
|
329
|
+
*/
|
|
330
|
+
export function pidGone(pid: number): boolean {
|
|
331
|
+
if (!pidAlive(pid)) return true;
|
|
332
|
+
const read = readStatFieldsDetailed(pid);
|
|
333
|
+
if (!read.ok) return read.code === "ENOENT";
|
|
334
|
+
const state = read.fields[0];
|
|
335
|
+
return state === "Z" || state === "X";
|
|
336
|
+
}
|
|
337
|
+
|
|
99
338
|
/**
|
|
100
339
|
* Pids whose environment carries the marker. Scans `/proc/<pid>/environ` (NUL
|
|
101
340
|
* separated). Skips this process; unreadable/vanished processes are ignored.
|
|
@@ -103,7 +342,6 @@ export function pidAlive(pid: number): boolean {
|
|
|
103
342
|
*/
|
|
104
343
|
export function findByMarker(marker: string): number[] {
|
|
105
344
|
if (!marker || !isProcAvailable()) return [];
|
|
106
|
-
const needle = `${CHILD_MARKER_ENV_KEY}=${marker}`;
|
|
107
345
|
const found: number[] = [];
|
|
108
346
|
let entries: string[];
|
|
109
347
|
try {
|
|
@@ -115,12 +353,7 @@ export function findByMarker(marker: string): number[] {
|
|
|
115
353
|
if (!/^\d+$/.test(entry)) continue;
|
|
116
354
|
const pid = Number(entry);
|
|
117
355
|
if (pid === process.pid) continue;
|
|
118
|
-
|
|
119
|
-
const environ = fs.readFileSync(path.join(PROC_ROOT, entry, "environ"), "utf-8");
|
|
120
|
-
if (environ.split("\0").includes(needle)) found.push(pid);
|
|
121
|
-
} catch {
|
|
122
|
-
// Vanished or not readable by this user — not a verified match.
|
|
123
|
-
}
|
|
356
|
+
if (pidHasMarker(pid, marker)) found.push(pid);
|
|
124
357
|
}
|
|
125
358
|
return found;
|
|
126
359
|
}
|
|
@@ -134,7 +367,24 @@ export function defaultRecoveryDeps(): RecoveryDeps {
|
|
|
134
367
|
kill: (pid, signal) => {
|
|
135
368
|
process.kill(pid, signal);
|
|
136
369
|
},
|
|
370
|
+
// D4 (#299): group-reaching targets on POSIX. The marker identifies the
|
|
371
|
+
// orphan tree; the process group is only the kill mechanism that reaches
|
|
372
|
+
// env-scrubbed descendants. Omitted on Windows (D6) so `reapPids` keeps
|
|
373
|
+
// today's direct pid targets.
|
|
374
|
+
killGroup: supportsProcessGroupKill() ? killProcessGroup : undefined,
|
|
375
|
+
// #299 review Major 2: a group's leader exiting does not empty the group,
|
|
376
|
+
// so the group target's death check counts live members.
|
|
377
|
+
groupHasMembers,
|
|
378
|
+
// #299 review Major 1: signal-time marker re-check for marker-derived
|
|
379
|
+
// targets (closes the scan→signal pid-reuse window).
|
|
380
|
+
verifyMarker: pidHasMarker,
|
|
381
|
+
// #299 review Major 1: the guard separates a reused pid (alive, marker
|
|
382
|
+
// gone) from a dead/zombie one, so a zombie leader's group is still killed.
|
|
383
|
+
pidGone,
|
|
137
384
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
138
385
|
graceMs: SIGKILL_GRACE_MS,
|
|
386
|
+
// Item 4 (#299): poll so the boot scan returns as soon as its orphans die
|
|
387
|
+
// instead of always blocking the full grace window (a ~5 s session_start).
|
|
388
|
+
pollMs: SIGKILL_POLL_MS,
|
|
139
389
|
};
|
|
140
390
|
}
|
package/src/recovery-engine.ts
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
|
|
26
26
|
import type { ProcessOwner, SubagentRun } from "./types";
|
|
27
27
|
import { resolveTerminalRunEntry } from "./state";
|
|
28
|
-
import { escalateKill, type
|
|
28
|
+
import { escalateKill, processTarget, type SignalGuard, type SignalPhase } from "./kill-escalation";
|
|
29
|
+
import type { Logger } from "./logging";
|
|
29
30
|
|
|
30
31
|
// ---------------------------------------------------------------------------
|
|
31
32
|
// Dependency injection surface
|
|
@@ -41,10 +42,52 @@ export interface RecoveryDeps {
|
|
|
41
42
|
findByMarker(marker: string): number[];
|
|
42
43
|
/** Send a signal to a pid (may throw when the pid is already gone). */
|
|
43
44
|
kill(pid: number, signal: NodeJS.Signals): void;
|
|
45
|
+
/**
|
|
46
|
+
* #299 D4: best-effort process-group signal (`kill(-pgid, sig)`; ESRCH
|
|
47
|
+
* tolerated). Present only where POSIX process groups exist — the effect is
|
|
48
|
+
* that `reapPids` targets become group-reaching (verified marker pids signal
|
|
49
|
+
* their group as well as the pid). When absent (Windows, D6) `reapPids`
|
|
50
|
+
* falls back to direct pid targets, exactly today's behavior.
|
|
51
|
+
*/
|
|
52
|
+
killGroup?(pgid: number, signal: NodeJS.Signals): void;
|
|
53
|
+
/**
|
|
54
|
+
* #299 review Major 2: true when process group `pgid` still has a LIVE member.
|
|
55
|
+
* Threaded into the group target's `verifyDeath` so a leader that exits
|
|
56
|
+
* promptly no longer reads as "group empty" — the escalation then honours the
|
|
57
|
+
* grace window for a member's cleanup. Absent/false on a platform without
|
|
58
|
+
* /proc → leader-only fallback (documented boundary).
|
|
59
|
+
*
|
|
60
|
+
* #299 fix A.2: THREE-state. `true` = live members observed; `false` =
|
|
61
|
+
* POSITIVELY empty (a completed inspection); `undefined` = UNKNOWN (the
|
|
62
|
+
* inspection was unavailable or partial). Callers treat `undefined` as NOT
|
|
63
|
+
* empty — absence of evidence is never evidence of absence.
|
|
64
|
+
*/
|
|
65
|
+
groupHasMembers?(pgid: number): boolean | undefined;
|
|
66
|
+
/**
|
|
67
|
+
* #299 review Major 1: signal-time marker re-check — true when the LIVE `pid`
|
|
68
|
+
* still carries `CHILD_MARKER_ENV_KEY=<marker>`. Threaded into the marker
|
|
69
|
+
* targets' signal guard so a pid that exited and was reused by a foreign
|
|
70
|
+
* process is never signaled. Absent → no signal-time gate (tests without /proc
|
|
71
|
+
* wiring); production always provides it via `defaultRecoveryDeps`.
|
|
72
|
+
*/
|
|
73
|
+
verifyMarker?(pid: number, marker: string): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* #299 review Major 1: true when the pid is gone — reaped OR a not-yet-reaped
|
|
76
|
+
* zombie. The signal guard uses this to separate a REUSED pid from a merely
|
|
77
|
+
* dead/zombie one, so a zombie leader's pgid is still signaled (D5). Absent →
|
|
78
|
+
* `!pidAlive` (no zombie distinction).
|
|
79
|
+
*/
|
|
80
|
+
pidGone?(pid: number): boolean;
|
|
44
81
|
/** Sleep between SIGTERM and SIGKILL. */
|
|
45
82
|
sleep(ms: number): Promise<void>;
|
|
46
83
|
/** Grace period in ms between SIGTERM and SIGKILL. */
|
|
47
84
|
graceMs: number;
|
|
85
|
+
/**
|
|
86
|
+
* Poll interval for the early-exit wait (B4/Item 4): when > 0 the reap returns
|
|
87
|
+
* as soon as every target verifies dead, bounded by `graceMs`. Omitted/0 keeps
|
|
88
|
+
* the historical wait-the-full-grace-once shape.
|
|
89
|
+
*/
|
|
90
|
+
pollMs?: number;
|
|
48
91
|
}
|
|
49
92
|
|
|
50
93
|
/** Which persisted store a candidate record came from. */
|
|
@@ -203,25 +246,50 @@ export function decideRecovery(record: RecoveryRecord, deps: RecoveryDeps): Reco
|
|
|
203
246
|
}
|
|
204
247
|
|
|
205
248
|
/**
|
|
206
|
-
* Escalate a set of pids together: SIGTERM every one, wait
|
|
207
|
-
*
|
|
249
|
+
* Escalate a set of pids together: SIGTERM every one, wait (with `deps.pollMs`,
|
|
250
|
+
* at most) ONE grace window returning as soon as all are dead, then SIGKILL any
|
|
251
|
+
* survivor. Returns the pids still alive afterwards. Thin
|
|
208
252
|
* adapter over the shared `escalateKill` helper (src/kill-escalation.ts), which
|
|
209
253
|
* `recoverRecord` (one record) and `runBootScan` (the whole scan) both call, so
|
|
210
254
|
* a boot with N orphans waits one grace period total, never N.
|
|
255
|
+
*
|
|
256
|
+
* #299 review Major 1: `markerOfPid` resolves the marker that verified a pid at
|
|
257
|
+
* scan time; when `deps.verifyMarker` is also wired (production), every signal
|
|
258
|
+
* on that pid is gated on an immediate re-read of the live marker (`SignalGuard`),
|
|
259
|
+
* so a pid reused by a foreign process is skipped, never signaled unverified.
|
|
260
|
+
* `onSkip` records a refused signal. Both are optional so direct `reapPids`
|
|
261
|
+
* callers without /proc wiring keep the historical unconditional behavior.
|
|
211
262
|
*/
|
|
212
|
-
export async function reapPids(
|
|
263
|
+
export async function reapPids(
|
|
264
|
+
pids: number[],
|
|
265
|
+
deps: RecoveryDeps,
|
|
266
|
+
markerOfPid?: (pid: number) => string | undefined,
|
|
267
|
+
onSkip?: (pid: number, signal: NodeJS.Signals, phase: SignalPhase) => void,
|
|
268
|
+
): Promise<number[]> {
|
|
213
269
|
const unique = [...new Set(pids)];
|
|
214
|
-
const
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
270
|
+
const isGone = deps.pidGone ?? ((pid: number) => !deps.pidAlive(pid));
|
|
271
|
+
const targets = unique.map((pid) => {
|
|
272
|
+
const marker = markerOfPid?.(pid);
|
|
273
|
+
const guard: SignalGuard | undefined =
|
|
274
|
+
marker && deps.verifyMarker
|
|
275
|
+
? {
|
|
276
|
+
isOwn: (targetPid) => deps.verifyMarker!(targetPid, marker),
|
|
277
|
+
isGone,
|
|
278
|
+
onSkip,
|
|
279
|
+
}
|
|
280
|
+
: undefined;
|
|
281
|
+
return processTarget(
|
|
282
|
+
pid,
|
|
283
|
+
deps.kill,
|
|
284
|
+
deps.pidAlive,
|
|
285
|
+
deps.killGroup,
|
|
286
|
+
deps.groupHasMembers,
|
|
287
|
+
guard,
|
|
288
|
+
);
|
|
289
|
+
});
|
|
223
290
|
const survivors = await escalateKill(targets, {
|
|
224
291
|
graceMs: deps.graceMs,
|
|
292
|
+
pollMs: deps.pollMs,
|
|
225
293
|
sleep: deps.sleep,
|
|
226
294
|
});
|
|
227
295
|
const survivedSet = new Set(survivors);
|
|
@@ -231,10 +299,15 @@ export async function reapPids(pids: number[], deps: RecoveryDeps): Promise<numb
|
|
|
231
299
|
/**
|
|
232
300
|
* Apply a record's plan: SIGTERM every marker-verified orphan, wait the grace
|
|
233
301
|
* period, SIGKILL any survivor, then report. Never throws on a vanished pid.
|
|
302
|
+
*
|
|
303
|
+
* #299 review Major 1: every signal on this record's marker-verified pids is
|
|
304
|
+
* re-checked against the live marker at signal time; a refused signal is logged
|
|
305
|
+
* when `log` is supplied.
|
|
234
306
|
*/
|
|
235
307
|
export async function recoverRecord(
|
|
236
308
|
record: RecoveryRecord,
|
|
237
309
|
deps: RecoveryDeps,
|
|
310
|
+
log?: Pick<Logger, "warn">,
|
|
238
311
|
): Promise<RecoveryOutcome> {
|
|
239
312
|
const plan = decideRecovery(record, deps);
|
|
240
313
|
if (plan.decision === "skip") {
|
|
@@ -249,7 +322,20 @@ export async function recoverRecord(
|
|
|
249
322
|
}
|
|
250
323
|
|
|
251
324
|
const reaped = [...new Set(plan.reap)];
|
|
252
|
-
const
|
|
325
|
+
const marker = record.childMarker;
|
|
326
|
+
const survived = await reapPids(
|
|
327
|
+
reaped,
|
|
328
|
+
deps,
|
|
329
|
+
marker ? () => marker : undefined,
|
|
330
|
+
(pid, signal, phase) =>
|
|
331
|
+
log?.warn("Recovery: skipped a marker signal (pid no longer carries the run marker)", {
|
|
332
|
+
id: record.id,
|
|
333
|
+
pid,
|
|
334
|
+
signal,
|
|
335
|
+
phase,
|
|
336
|
+
marker,
|
|
337
|
+
}),
|
|
338
|
+
);
|
|
253
339
|
|
|
254
340
|
return {
|
|
255
341
|
id: record.id,
|