brl-subagent 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +5 -5
- package/src/event-bus.ts +7 -2
- package/src/git.ts +67 -0
- package/src/history.ts +58 -1
- package/src/index.ts +184 -32
- package/src/kill-escalation.ts +94 -0
- package/src/logging.ts +37 -3
- package/src/preflight.ts +29 -14
- package/src/proc.ts +140 -0
- package/src/recovery-engine.ts +262 -0
- package/src/recovery.ts +325 -0
- package/src/run-registry.ts +269 -0
- package/src/runner.ts +206 -16
- package/src/sanitize.ts +53 -3
- package/src/session-manager.ts +106 -23
- package/src/state.ts +51 -7
- package/src/transcript-path.ts +41 -0
- package/src/transcript.ts +13 -12
- package/src/tui.ts +228 -88
- package/src/types.ts +107 -0
- package/src/unit-run.ts +8 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Purpose: The one SIGTERM → grace → SIGKILL escalation shared by foreground reap, abort/timeout, and boot recovery.
|
|
2
|
+
/**
|
|
3
|
+
* Process kill escalation (Option B U1, review items B4 + B7).
|
|
4
|
+
*
|
|
5
|
+
* The SIGTERM → grace → SIGKILL sequence was duplicated four times with
|
|
6
|
+
* inconsistent wait semantics. This module owns the ONE sequence; each site
|
|
7
|
+
* supplies its own targets and death check:
|
|
8
|
+
*
|
|
9
|
+
* - `runner.ts` `reapActiveChildren` — every tracked foreground child at once
|
|
10
|
+
* - `runner.ts` abort handler — one child
|
|
11
|
+
* - `runner.ts` timeout handler — one child
|
|
12
|
+
* - `recovery.ts` `reapPids` — every marker-verified orphan at once
|
|
13
|
+
*
|
|
14
|
+
* Order is terminate → wait → force-kill. With `pollMs` set the wait ends as
|
|
15
|
+
* soon as every target verifies dead (B4: the shutdown reap no longer blocks
|
|
16
|
+
* the full 5 s when children exit promptly), bounded by `graceMs`; without it
|
|
17
|
+
* the helper waits the full grace once (the boot scan's wait-once shape).
|
|
18
|
+
*
|
|
19
|
+
* `verifyDeath` is supplied per site but must always answer "is the process
|
|
20
|
+
* REALLY dead" — for a child, `exitCode`/`signalCode`; for a pid, `kill -0`
|
|
21
|
+
* liveness. It must never be a "a signal was delivered" check: `child.killed`
|
|
22
|
+
* flips when SIGTERM is SENT, so gating SIGKILL on it skips escalation for a
|
|
23
|
+
* SIGTERM-ignoring child (issue #303). What is shared is the
|
|
24
|
+
* terminate-then-escalate order and the no-throw-on-vanished discipline.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** A process handle the escalation helper can signal and liveness-check. */
|
|
28
|
+
export interface EscalationTarget {
|
|
29
|
+
/** Send SIGTERM; may throw when the target is already gone. */
|
|
30
|
+
terminate(): void;
|
|
31
|
+
/** Send SIGKILL; may throw when the target is already gone. */
|
|
32
|
+
forceKill(): void;
|
|
33
|
+
/** True when the target is confirmed dead (so no SIGKILL is needed). */
|
|
34
|
+
verifyDeath(): boolean;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface EscalationOptions {
|
|
38
|
+
/** Grace window between SIGTERM and SIGKILL (ms). */
|
|
39
|
+
graceMs: number;
|
|
40
|
+
/** Sleep between SIGTERM and the liveness re-check (injectable). */
|
|
41
|
+
sleep(ms: number): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Poll interval for the early-exit wait (B4). When > 0 the helper returns as
|
|
44
|
+
* soon as every target verifies dead, bounded by `graceMs`; when omitted the
|
|
45
|
+
* helper waits the full `graceMs` once.
|
|
46
|
+
*/
|
|
47
|
+
pollMs?: number;
|
|
48
|
+
/** Injectable clock for the deadline (defaults to Date.now). */
|
|
49
|
+
now?(): number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Escalate every target together: SIGTERM all, wait (at most) one grace window,
|
|
54
|
+
* then SIGKILL every target that still verifies alive. Returns the survivors.
|
|
55
|
+
* Never throws on a target that vanished mid-sequence.
|
|
56
|
+
*/
|
|
57
|
+
export async function escalateKill(
|
|
58
|
+
targets: EscalationTarget[],
|
|
59
|
+
options: EscalationOptions,
|
|
60
|
+
): Promise<EscalationTarget[]> {
|
|
61
|
+
for (const target of targets) {
|
|
62
|
+
try {
|
|
63
|
+
target.terminate();
|
|
64
|
+
} catch {
|
|
65
|
+
// Already gone — the desired end state.
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (targets.length > 0) {
|
|
70
|
+
const pollMs = options.pollMs ?? 0;
|
|
71
|
+
if (pollMs > 0) {
|
|
72
|
+
const now = options.now ?? Date.now;
|
|
73
|
+
const deadline = now() + options.graceMs;
|
|
74
|
+
while (now() < deadline && !targets.every((target) => target.verifyDeath())) {
|
|
75
|
+
const remaining = Math.max(deadline - now(), 1);
|
|
76
|
+
await options.sleep(Math.min(pollMs, remaining));
|
|
77
|
+
}
|
|
78
|
+
} else {
|
|
79
|
+
await options.sleep(options.graceMs);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const survivors: EscalationTarget[] = [];
|
|
84
|
+
for (const target of targets) {
|
|
85
|
+
if (target.verifyDeath()) continue;
|
|
86
|
+
try {
|
|
87
|
+
target.forceKill();
|
|
88
|
+
} catch {
|
|
89
|
+
// Exited between the liveness check and the signal.
|
|
90
|
+
}
|
|
91
|
+
if (!target.verifyDeath()) survivors.push(target);
|
|
92
|
+
}
|
|
93
|
+
return survivors;
|
|
94
|
+
}
|
package/src/logging.ts
CHANGED
|
@@ -48,6 +48,12 @@ const LOG_LEVEL_NAMES: readonly LogLevel[] = ["debug", "info", "warn", "error"];
|
|
|
48
48
|
/** Current minimum log level — can be changed at runtime */
|
|
49
49
|
let minLevel: LogLevel = resolveInitialLogLevel();
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Whether the console mirror is enabled — opt-in via BRL_LOG_CONSOLE (issue #265).
|
|
53
|
+
* File-only is the default so extension logging never corrupts the pi TUI.
|
|
54
|
+
*/
|
|
55
|
+
let consoleOutput: boolean = resolveInitialConsoleOutput();
|
|
56
|
+
|
|
51
57
|
/**
|
|
52
58
|
* Issue #235: the log verbosity knob was never wired, so the three `log.debug()`
|
|
53
59
|
* call sites were permanently suppressed with no way to turn them on.
|
|
@@ -64,6 +70,22 @@ function resolveInitialLogLevel(): LogLevel {
|
|
|
64
70
|
return "info";
|
|
65
71
|
}
|
|
66
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Issue #265: extension code runs inside pi's TUI process, so a raw console
|
|
75
|
+
* write lands wherever the renderer's cursor sits and corrupts the display.
|
|
76
|
+
* File-only is the default; `BRL_LOG_CONSOLE` opts the terminal mirror back in.
|
|
77
|
+
* Read ONCE at module load for the same reason as BRL_LOG_LEVEL — logger call
|
|
78
|
+
* sites are spread across the codebase. Accepts `1|true|yes`; anything else is
|
|
79
|
+
* off. `setConsoleOutput()` remains the programmatic override.
|
|
80
|
+
*
|
|
81
|
+
* Explicitly NOT gated on process.stdout.isTTY: in pi print mode stdout IS the
|
|
82
|
+
* result channel, so that test is exactly backwards.
|
|
83
|
+
*/
|
|
84
|
+
function resolveInitialConsoleOutput(): boolean {
|
|
85
|
+
const raw = process.env.BRL_LOG_CONSOLE?.trim().toLowerCase();
|
|
86
|
+
return raw === "1" || raw === "true" || raw === "yes";
|
|
87
|
+
}
|
|
88
|
+
|
|
67
89
|
// ---------------------------------------------------------------------------
|
|
68
90
|
// Logger
|
|
69
91
|
// ---------------------------------------------------------------------------
|
|
@@ -79,6 +101,15 @@ export function setLogLevel(level: LogLevel): void {
|
|
|
79
101
|
minLevel = level;
|
|
80
102
|
}
|
|
81
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Programmatic override for the console mirror (issue #265). Mirrors
|
|
106
|
+
* setLogLevel: module-level state, so tests can toggle it without re-reading
|
|
107
|
+
* the environment.
|
|
108
|
+
*/
|
|
109
|
+
export function setConsoleOutput(enabled: boolean): void {
|
|
110
|
+
consoleOutput = enabled;
|
|
111
|
+
}
|
|
112
|
+
|
|
82
113
|
// Issue #179 (D6): createLogger() runs at MODULE LOAD (index.ts,
|
|
83
114
|
// session-manager.ts) where the session cwd does not exist yet. The cwd is set
|
|
84
115
|
// later from the session_start hook (ctx.cwd). All logger instances share this
|
|
@@ -121,9 +152,12 @@ export function createLogger(prefix: string, cwd?: string): Logger {
|
|
|
121
152
|
|
|
122
153
|
const line = JSON.stringify(entry);
|
|
123
154
|
|
|
124
|
-
// Console
|
|
125
|
-
|
|
126
|
-
|
|
155
|
+
// Console mirror — opt-in (issue #265). Off by default so a logger call
|
|
156
|
+
// can never corrupt the pi TUI renderer; BRL_LOG_CONSOLE opts back in.
|
|
157
|
+
if (consoleOutput) {
|
|
158
|
+
const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log;
|
|
159
|
+
consoleMethod(`[${prefix}] ${level.toUpperCase()}: ${message}`);
|
|
160
|
+
}
|
|
127
161
|
|
|
128
162
|
// File output — resolved on each call because the cwd may be set after
|
|
129
163
|
// createLogger ran (module load) but before the first entry.
|
package/src/preflight.ts
CHANGED
|
@@ -22,6 +22,33 @@ export type PreflightResult = { ok: true } | { ok: false; error: string };
|
|
|
22
22
|
// Checks
|
|
23
23
|
// ---------------------------------------------------------------------------
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Walk PATH for an executable `pi`, returning the first match.
|
|
27
|
+
*
|
|
28
|
+
* Mirrors the spawn-time fallback in `getPiInvocation` (the bare `"pi"`
|
|
29
|
+
* case). Shared with the Tier-2 e2e harness (issue #271) so the preflight
|
|
30
|
+
* check and the harness resolve the real binary identically.
|
|
31
|
+
*/
|
|
32
|
+
export function resolvePiOnPath(): string | undefined {
|
|
33
|
+
const pathDirs = (process.env.PATH || "").split(path.delimiter);
|
|
34
|
+
const isWindows = process.platform === "win32";
|
|
35
|
+
const piNames = isWindows ? ["pi.cmd", "pi.exe", "pi"] : ["pi"];
|
|
36
|
+
|
|
37
|
+
for (const dir of pathDirs) {
|
|
38
|
+
for (const name of piNames) {
|
|
39
|
+
const fullPath = path.join(dir, name);
|
|
40
|
+
try {
|
|
41
|
+
fs.accessSync(fullPath, fs.constants.X_OK);
|
|
42
|
+
return fullPath;
|
|
43
|
+
} catch {
|
|
44
|
+
// Not found in this directory — continue searching
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
25
52
|
/**
|
|
26
53
|
* Verify the pi binary is accessible via the same resolution logic used
|
|
27
54
|
* at spawn time. When `getPiInvocation` falls back to "pi" on PATH, we
|
|
@@ -45,20 +72,8 @@ function checkPiBinary(): PreflightResult {
|
|
|
45
72
|
}
|
|
46
73
|
|
|
47
74
|
// "pi" fallback — search PATH
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const piNames = isWindows ? ["pi.cmd", "pi.exe", "pi"] : ["pi"];
|
|
51
|
-
|
|
52
|
-
for (const dir of pathDirs) {
|
|
53
|
-
for (const name of piNames) {
|
|
54
|
-
const fullPath = path.join(dir, name);
|
|
55
|
-
try {
|
|
56
|
-
fs.accessSync(fullPath, fs.constants.X_OK);
|
|
57
|
-
return { ok: true };
|
|
58
|
-
} catch {
|
|
59
|
-
// Not found in this directory — continue searching
|
|
60
|
-
}
|
|
61
|
-
}
|
|
75
|
+
if (resolvePiOnPath()) {
|
|
76
|
+
return { ok: true };
|
|
62
77
|
}
|
|
63
78
|
|
|
64
79
|
return {
|
package/src/proc.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// Purpose: Production process primitives — Linux/POSIX /proc reads, feature detection, and the default recovery deps.
|
|
2
|
+
/**
|
|
3
|
+
* Option B U1 — production process wiring for the recovery engine.
|
|
4
|
+
*
|
|
5
|
+
* The recovery decision engine (`src/recovery-engine.ts`) is pure over the
|
|
6
|
+
* injected `RecoveryDeps`. This module supplies the REAL deps: the Linux/POSIX
|
|
7
|
+
* `/proc` start-time token (`/proc/<pid>/stat` field 22), the child-marker scan
|
|
8
|
+
* (`/proc/<pid>/environ`), cross-platform pid liveness, and the default
|
|
9
|
+
* `RecoveryDeps` assembly.
|
|
10
|
+
*
|
|
11
|
+
* PLATFORM BOUNDARY (required): the start token and marker scan are Linux/POSIX
|
|
12
|
+
* reads. `isProcAvailable()` feature-detects them (cached; the answer cannot
|
|
13
|
+
* change mid-process). Where they are unavailable, `pidAlive` still works but
|
|
14
|
+
* on its own cannot rule out pid reuse — so the boot scan skips entirely and
|
|
15
|
+
* nothing is killed or marked unverified (`recoverProduction`, src/recovery.ts).
|
|
16
|
+
*
|
|
17
|
+
* `__setProcAvailableForTest` is the injectable seam that lets a test exercise
|
|
18
|
+
* that platform guard without mocking `node:fs` module-wide.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import * as fs from "node:fs";
|
|
22
|
+
import * as path from "node:path";
|
|
23
|
+
import type { ProcessOwner } from "./types";
|
|
24
|
+
import { SIGKILL_GRACE_MS } from "./types";
|
|
25
|
+
import { CHILD_MARKER_ENV_KEY } from "./sanitize";
|
|
26
|
+
import type { RecoveryDeps } from "./recovery-engine";
|
|
27
|
+
|
|
28
|
+
const PROC_ROOT = "/proc";
|
|
29
|
+
|
|
30
|
+
let procAvailable: boolean | undefined;
|
|
31
|
+
let procAvailableOverride: boolean | undefined;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* TEST-ONLY: force the `/proc` feature-detection answer (undefined restores the
|
|
35
|
+
* real detection and clears the cache). The production scan consults this
|
|
36
|
+
* through `isProcAvailable`, so the platform guard is exercisable in-repo.
|
|
37
|
+
*/
|
|
38
|
+
export function __setProcAvailableForTest(value: boolean | undefined): void {
|
|
39
|
+
procAvailableOverride = value;
|
|
40
|
+
procAvailable = undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Feature-detect the Linux/POSIX `/proc` surface this module needs. Cached:
|
|
45
|
+
* the answer cannot change mid-process. Exported so the entry point can skip
|
|
46
|
+
* the whole scan with one log line on platforms without /proc.
|
|
47
|
+
*/
|
|
48
|
+
export function isProcAvailable(): boolean {
|
|
49
|
+
if (procAvailableOverride !== undefined) return procAvailableOverride;
|
|
50
|
+
if (procAvailable === undefined) {
|
|
51
|
+
try {
|
|
52
|
+
procAvailable = fs.existsSync(path.join(PROC_ROOT, "self", "stat"));
|
|
53
|
+
} catch {
|
|
54
|
+
procAvailable = false;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return procAvailable;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Read a process's start-time token — `/proc/<pid>/stat` field 22 (1-indexed),
|
|
62
|
+
* i.e. index 19 of the whitespace-split fields AFTER the `(comm)` field. The
|
|
63
|
+
* comm field is skipped by slicing at the LAST `)` so a name containing spaces
|
|
64
|
+
* or parens cannot shift the parse. Returns undefined when /proc is
|
|
65
|
+
* unavailable, the pid is invalid, or the process vanished.
|
|
66
|
+
*/
|
|
67
|
+
export function readStartToken(pid: number): string | undefined {
|
|
68
|
+
if (!isProcAvailable() || !Number.isInteger(pid) || pid <= 0) return undefined;
|
|
69
|
+
try {
|
|
70
|
+
const stat = fs.readFileSync(path.join(PROC_ROOT, String(pid), "stat"), "utf-8");
|
|
71
|
+
const close = stat.lastIndexOf(")");
|
|
72
|
+
if (close < 0) return undefined;
|
|
73
|
+
const rest = stat.slice(close + 1).trim();
|
|
74
|
+
if (!rest) return undefined;
|
|
75
|
+
const fields = rest.split(/\s+/);
|
|
76
|
+
const token = fields[19];
|
|
77
|
+
return token && /^\d+$/.test(token) ? token : undefined;
|
|
78
|
+
} catch {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The CURRENT process's owner identity (pid + start token; empty string when unavailable). */
|
|
84
|
+
export function currentProcessOwner(): ProcessOwner {
|
|
85
|
+
return { pid: process.pid, start: readStartToken(process.pid) ?? "" };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Cross-platform pid liveness (`kill -0`). EPERM means the process exists. */
|
|
89
|
+
export function pidAlive(pid: number): boolean {
|
|
90
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
91
|
+
try {
|
|
92
|
+
process.kill(pid, 0);
|
|
93
|
+
return true;
|
|
94
|
+
} catch (err) {
|
|
95
|
+
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Pids whose environment carries the marker. Scans `/proc/<pid>/environ` (NUL
|
|
101
|
+
* separated). Skips this process; unreadable/vanished processes are ignored.
|
|
102
|
+
* Returns [] — never a kill — when /proc is unavailable.
|
|
103
|
+
*/
|
|
104
|
+
export function findByMarker(marker: string): number[] {
|
|
105
|
+
if (!marker || !isProcAvailable()) return [];
|
|
106
|
+
const needle = `${CHILD_MARKER_ENV_KEY}=${marker}`;
|
|
107
|
+
const found: number[] = [];
|
|
108
|
+
let entries: string[];
|
|
109
|
+
try {
|
|
110
|
+
entries = fs.readdirSync(PROC_ROOT);
|
|
111
|
+
} catch {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
for (const entry of entries) {
|
|
115
|
+
if (!/^\d+$/.test(entry)) continue;
|
|
116
|
+
const pid = Number(entry);
|
|
117
|
+
if (pid === process.pid) continue;
|
|
118
|
+
try {
|
|
119
|
+
const environ = fs.readFileSync(path.join(PROC_ROOT, entry, "environ"), "utf-8");
|
|
120
|
+
if (environ.split("\0").includes(needle)) found.push(pid);
|
|
121
|
+
} catch {
|
|
122
|
+
// Vanished or not readable by this user — not a verified match.
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return found;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** The real production dependencies (Linux/POSIX /proc reads + process signals). */
|
|
129
|
+
export function defaultRecoveryDeps(): RecoveryDeps {
|
|
130
|
+
return {
|
|
131
|
+
pidAlive,
|
|
132
|
+
startTokenOf: readStartToken,
|
|
133
|
+
findByMarker,
|
|
134
|
+
kill: (pid, signal) => {
|
|
135
|
+
process.kill(pid, signal);
|
|
136
|
+
},
|
|
137
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
138
|
+
graceMs: SIGKILL_GRACE_MS,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
// Purpose: Pure recovery decision engine — identity, owner liveness classification, and the boot-scan plan over injected process deps.
|
|
2
|
+
/**
|
|
3
|
+
* Option B U1 — recovery decision engine (pure over `RecoveryDeps`).
|
|
4
|
+
*
|
|
5
|
+
* The engine decides, for one persisted `running` record, whether its
|
|
6
|
+
* conductor is still live, whether to reject ownership, and whether to reap a
|
|
7
|
+
* marker-verified orphan child. It performs NO process work itself: every
|
|
8
|
+
* effect goes through the injected `RecoveryDeps`, so the full matrix is
|
|
9
|
+
* unit-testable without spawning processes. The boot orchestration lives in
|
|
10
|
+
* `src/recovery.ts`; the production `/proc` wiring in `src/proc.ts`.
|
|
11
|
+
*
|
|
12
|
+
* Decision order (`decideRecovery`):
|
|
13
|
+
* - status not "running" → skip
|
|
14
|
+
* - `interruptedAt` already set → never re-mark, but re-check for a leftover
|
|
15
|
+
* live marker child and reap it again (B2 revisit)
|
|
16
|
+
* - absent owner → skip (`no-owner`)
|
|
17
|
+
* - owner present but unparsable → mark-only (`owner-unverifiable`), no kill
|
|
18
|
+
* - owner alive (pid + start token match) → leave untouched
|
|
19
|
+
* - owner alive but token unreadable → skip (`ownership-unverifiable`)
|
|
20
|
+
* - owner dead → reap marker-verified orphan children, then mark
|
|
21
|
+
*
|
|
22
|
+
* Reaping is the ONE `escalateKill` sequence (src/kill-escalation.ts): SIGTERM
|
|
23
|
+
* every pid, wait one grace window, SIGKILL any survivor.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import type { ProcessOwner, SubagentRun } from "./types";
|
|
27
|
+
import { resolveTerminalRunEntry } from "./state";
|
|
28
|
+
import { escalateKill, type EscalationTarget } from "./kill-escalation";
|
|
29
|
+
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// Dependency injection surface
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
/** The process primitives the decision engine needs (injected for testability). */
|
|
35
|
+
export interface RecoveryDeps {
|
|
36
|
+
/** True when the pid exists (SIGCONT/`kill -0` semantics; EPERM counts as alive). */
|
|
37
|
+
pidAlive(pid: number): boolean;
|
|
38
|
+
/** The process start-time token, or undefined when /proc is unavailable/unreadable. */
|
|
39
|
+
startTokenOf(pid: number): string | undefined;
|
|
40
|
+
/** Pids whose environment carries `CHILD_MARKER_ENV_KEY=<marker>`; [] when /proc is unavailable. */
|
|
41
|
+
findByMarker(marker: string): number[];
|
|
42
|
+
/** Send a signal to a pid (may throw when the pid is already gone). */
|
|
43
|
+
kill(pid: number, signal: NodeJS.Signals): void;
|
|
44
|
+
/** Sleep between SIGTERM and SIGKILL. */
|
|
45
|
+
sleep(ms: number): Promise<void>;
|
|
46
|
+
/** Grace period in ms between SIGTERM and SIGKILL. */
|
|
47
|
+
graceMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Which persisted store a candidate record came from. */
|
|
51
|
+
export type RecoveryKind = "run" | "agent";
|
|
52
|
+
|
|
53
|
+
/** A persisted `running` record projected into the shape the engine reasons over. */
|
|
54
|
+
export interface RecoveryRecord {
|
|
55
|
+
id: string;
|
|
56
|
+
kind: RecoveryKind;
|
|
57
|
+
status: string;
|
|
58
|
+
interruptedAt?: string;
|
|
59
|
+
owner?: ProcessOwner;
|
|
60
|
+
childMarker?: string;
|
|
61
|
+
/** Original record for the caller's write-back; opaque to this module. */
|
|
62
|
+
source?: unknown;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export type RecoverySkipReason =
|
|
66
|
+
| "not-running"
|
|
67
|
+
| "already-interrupted"
|
|
68
|
+
| "no-owner"
|
|
69
|
+
| "owner-alive"
|
|
70
|
+
| "ownership-unverifiable";
|
|
71
|
+
|
|
72
|
+
export type RecoveryMarkReason = "owner-dead" | "child-orphaned" | "owner-unverifiable";
|
|
73
|
+
|
|
74
|
+
/** What the engine decided to do with one record (before any process work). */
|
|
75
|
+
export type RecoveryPlan =
|
|
76
|
+
| { decision: "skip"; reason: RecoverySkipReason }
|
|
77
|
+
| { decision: "mark"; reason: RecoveryMarkReason; reap: number[] }
|
|
78
|
+
/** Already-marked record re-checked this boot: reap leftovers, never re-mark. */
|
|
79
|
+
| { decision: "reap"; reason: "revisit-interrupted"; reap: number[] };
|
|
80
|
+
|
|
81
|
+
/** The engine's verdict on an owner identity. */
|
|
82
|
+
export type OwnerState = "alive" | "dead" | "unknown";
|
|
83
|
+
|
|
84
|
+
/** The result of applying a plan (kills performed, survivors logged). */
|
|
85
|
+
export interface RecoveryOutcome {
|
|
86
|
+
id: string;
|
|
87
|
+
kind: RecoveryKind;
|
|
88
|
+
decision: "skip" | "mark" | "reap";
|
|
89
|
+
reason: string;
|
|
90
|
+
reaped: number[];
|
|
91
|
+
survived: number[];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// Candidate projection
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Durability identity for a newly dispatched run record. A retry inherits the
|
|
100
|
+
* original's dispatchId and records `resumeOf`/`attempt`; a fresh dispatch
|
|
101
|
+
* mints a new dispatchId with attempt 1. These are RECORD fields, not
|
|
102
|
+
* delegate_task params (D4).
|
|
103
|
+
*/
|
|
104
|
+
export function newDispatchIdentity(source: SubagentRun | undefined): {
|
|
105
|
+
dispatchId: string;
|
|
106
|
+
resumeOf?: string;
|
|
107
|
+
attempt: number;
|
|
108
|
+
} {
|
|
109
|
+
return {
|
|
110
|
+
dispatchId: source?.dispatchId ?? crypto.randomUUID(),
|
|
111
|
+
resumeOf: source?.id,
|
|
112
|
+
attempt: source ? (source.attempt ?? 1) + 1 : 1,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Collapse run entries to ONE per id before the scan, using the ONE shared
|
|
118
|
+
* preference rule (`resolveTerminalRunEntry`, src/state.ts): terminal-first;
|
|
119
|
+
* among non-terminal entries the D3 `interruptedAt` mark wins over the
|
|
120
|
+
* original spawn entry; otherwise the first entry seen. Ids keep their
|
|
121
|
+
* first-seen order. Delegating keeps the recovery scan and U3's display/retry
|
|
122
|
+
* lookup from diverging (the historical duplicate rule picked the marked clone
|
|
123
|
+
* while the resolver picked the unmarked spawn).
|
|
124
|
+
*/
|
|
125
|
+
export function dedupeRunEntriesById(runs: SubagentRun[]): SubagentRun[] {
|
|
126
|
+
const ids: string[] = [];
|
|
127
|
+
const seen = new Set<string>();
|
|
128
|
+
for (const run of runs) {
|
|
129
|
+
if (seen.has(run.id)) continue;
|
|
130
|
+
seen.add(run.id);
|
|
131
|
+
ids.push(run.id);
|
|
132
|
+
}
|
|
133
|
+
const collapsed: SubagentRun[] = [];
|
|
134
|
+
for (const id of ids) {
|
|
135
|
+
const resolved = resolveTerminalRunEntry(runs, id);
|
|
136
|
+
if (resolved) collapsed.push(resolved);
|
|
137
|
+
}
|
|
138
|
+
return collapsed;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
// The decision engine (pure over RecoveryDeps)
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Classify the recorded owner's liveness. `unknown` means the pid is alive but
|
|
147
|
+
* the start token could not be read, so pid reuse cannot be excluded — the
|
|
148
|
+
* caller must not act on it.
|
|
149
|
+
*/
|
|
150
|
+
export function classifyOwner(owner: ProcessOwner, deps: RecoveryDeps): OwnerState {
|
|
151
|
+
if (!deps.pidAlive(owner.pid)) return "dead";
|
|
152
|
+
// An empty recorded token means the WRITER could not read one (non-/proc
|
|
153
|
+
// platform). Even on a host that can read tokens now, the record's identity
|
|
154
|
+
// is unverifiable — never act on it.
|
|
155
|
+
if (!owner.start) return "unknown";
|
|
156
|
+
const token = deps.startTokenOf(owner.pid);
|
|
157
|
+
if (token === undefined) return "unknown";
|
|
158
|
+
return token === owner.start ? "alive" : "dead";
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* True when an owner carries a usable pid (positive integer). Anything else —
|
|
163
|
+
* a missing pid, a string, 0, a negative, NaN, a fraction — is UNPARSABLE
|
|
164
|
+
* ownership: the record is marked but the kill path is never touched.
|
|
165
|
+
*/
|
|
166
|
+
function hasParsableOwnerPid(owner: ProcessOwner): boolean {
|
|
167
|
+
return Number.isInteger(owner.pid) && owner.pid > 0;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Decide what to do with one persisted record. Pure — no process is touched.
|
|
172
|
+
* Reaping pids come ONLY from a marker scan: without a verified marker match
|
|
173
|
+
* the engine never proposes a kill. An unparsable owner is mark-only (see the
|
|
174
|
+
* module header's three-way owner policy): interrupt is certain, ownership is
|
|
175
|
+
* not, so `reap` is empty and the kill path stays untouched.
|
|
176
|
+
*/
|
|
177
|
+
export function decideRecovery(record: RecoveryRecord, deps: RecoveryDeps): RecoveryPlan {
|
|
178
|
+
if (record.status !== "running") return { decision: "skip", reason: "not-running" };
|
|
179
|
+
if (record.interruptedAt) {
|
|
180
|
+
// Already marked: NEVER re-mark (idempotent), but a SIGKILL survivor or a
|
|
181
|
+
// child whose marker was unreadable at mark time must stay revisitable.
|
|
182
|
+
// The registry entry is retained, so every later boot re-checks it.
|
|
183
|
+
const reap = record.childMarker ? deps.findByMarker(record.childMarker) : [];
|
|
184
|
+
return reap.length > 0
|
|
185
|
+
? { decision: "reap", reason: "revisit-interrupted", reap }
|
|
186
|
+
: { decision: "skip", reason: "already-interrupted" };
|
|
187
|
+
}
|
|
188
|
+
if (!record.owner) return { decision: "skip", reason: "no-owner" };
|
|
189
|
+
if (!hasParsableOwnerPid(record.owner)) {
|
|
190
|
+
return { decision: "mark", reason: "owner-unverifiable", reap: [] };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const ownerState = classifyOwner(record.owner, deps);
|
|
194
|
+
if (ownerState === "alive") return { decision: "skip", reason: "owner-alive" };
|
|
195
|
+
if (ownerState === "unknown") return { decision: "skip", reason: "ownership-unverifiable" };
|
|
196
|
+
|
|
197
|
+
const reap = record.childMarker ? deps.findByMarker(record.childMarker) : [];
|
|
198
|
+
return {
|
|
199
|
+
decision: "mark",
|
|
200
|
+
reason: reap.length > 0 ? "child-orphaned" : "owner-dead",
|
|
201
|
+
reap,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Escalate a set of pids together: SIGTERM every one, wait ONE grace window,
|
|
207
|
+
* then SIGKILL any survivor. Returns the pids still alive afterwards. Thin
|
|
208
|
+
* adapter over the shared `escalateKill` helper (src/kill-escalation.ts), which
|
|
209
|
+
* `recoverRecord` (one record) and `runBootScan` (the whole scan) both call, so
|
|
210
|
+
* a boot with N orphans waits one grace period total, never N.
|
|
211
|
+
*/
|
|
212
|
+
export async function reapPids(pids: number[], deps: RecoveryDeps): Promise<number[]> {
|
|
213
|
+
const unique = [...new Set(pids)];
|
|
214
|
+
const targets: EscalationTarget[] = unique.map((pid) => ({
|
|
215
|
+
terminate: () => {
|
|
216
|
+
deps.kill(pid, "SIGTERM");
|
|
217
|
+
},
|
|
218
|
+
forceKill: () => {
|
|
219
|
+
deps.kill(pid, "SIGKILL");
|
|
220
|
+
},
|
|
221
|
+
verifyDeath: () => !deps.pidAlive(pid),
|
|
222
|
+
}));
|
|
223
|
+
const survivors = await escalateKill(targets, {
|
|
224
|
+
graceMs: deps.graceMs,
|
|
225
|
+
sleep: deps.sleep,
|
|
226
|
+
});
|
|
227
|
+
const survivedSet = new Set(survivors);
|
|
228
|
+
return unique.filter((_, index) => survivedSet.has(targets[index]));
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Apply a record's plan: SIGTERM every marker-verified orphan, wait the grace
|
|
233
|
+
* period, SIGKILL any survivor, then report. Never throws on a vanished pid.
|
|
234
|
+
*/
|
|
235
|
+
export async function recoverRecord(
|
|
236
|
+
record: RecoveryRecord,
|
|
237
|
+
deps: RecoveryDeps,
|
|
238
|
+
): Promise<RecoveryOutcome> {
|
|
239
|
+
const plan = decideRecovery(record, deps);
|
|
240
|
+
if (plan.decision === "skip") {
|
|
241
|
+
return {
|
|
242
|
+
id: record.id,
|
|
243
|
+
kind: record.kind,
|
|
244
|
+
decision: "skip",
|
|
245
|
+
reason: plan.reason,
|
|
246
|
+
reaped: [],
|
|
247
|
+
survived: [],
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const reaped = [...new Set(plan.reap)];
|
|
252
|
+
const survived = await reapPids(reaped, deps);
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
id: record.id,
|
|
256
|
+
kind: record.kind,
|
|
257
|
+
decision: plan.decision,
|
|
258
|
+
reason: plan.reason,
|
|
259
|
+
reaped,
|
|
260
|
+
survived,
|
|
261
|
+
};
|
|
262
|
+
}
|