brl-subagent 2.4.0 → 2.5.0

package/src/recovery.ts

@@ -0,0 +1,325 @@
+// Purpose: Boot-time recovery scan — decides whether a persisted running record is live, reaps orphaned children, and marks interrupted records.
+/**
+ * Option B U1 — recovery boot orchestration.
+ *
+ * A run record is written BEFORE its effect (the spawn), so a crashed
+ * conductor leaves `running` records and — on the foreground path — an
+ * orphaned child process. This module runs the boot-time scan that decides, in
+ * a FRESH process, which of those records are genuinely live:
+ *
+ *   - `interruptedAt` already set  → never re-mark, but re-check for a leftover
+ *     live marker child and reap it again (B2 revisit)
+ *   - owner alive (pid + start token match) → leave untouched
+ *   - owner dead → reap a marker-verified orphan child (SIGTERM → grace →
+ *     SIGKILL), then mark `interruptedAt` on the record
+ *
+ * The decision logic itself is the pure engine in `src/recovery-engine.ts`
+ * (type/external surface unchanged); this module owns candidate sourcing and
+ * the scan loop. `recoverInflightRuns` sources candidates from the durable
+ * registry (`src/run-registry.ts`) — NOT from the current session's run
+ * entries, which a fresh process cannot see after a crash — and
+ * `recoverProduction` is the entry point that also enforces the platform guard.
+ *
+ * Reaping waits ONE grace window for the whole scan (`runBootScan` collects
+ * every orphan pid and escalates them together), so N orphans cost one grace
+ * period, not N.
+ *
+ * PLATFORM BOUNDARY (required): the `/proc` reads live in `src/proc.ts`.
+ * Where `/proc` is unavailable, `recoverProduction` SKIPS the whole scan with
+ * one log line — `process.kill(pid, 0)` liveness alone cannot rule out pid
+ * reuse, so we NEVER kill an unverified process and NEVER mark a record when
+ * ownership cannot be verified.
+ */
+
+import { listInflightRuns, type InflightRun } from "./run-registry";
+import { assertSafeAgentId } from "./sanitize";
+import { isProcAvailable, defaultRecoveryDeps } from "./proc";
+import { decideRecovery, reapPids, type RecoveryDeps, type RecoveryRecord } from "./recovery-engine";
+import type { Logger } from "./logging";
+
+// Re-export the pure engine and the production wiring so existing importers
+// (`index.ts`, the recovery/registry suites) keep their single entry point.
+export * from "./recovery-engine";
+export * from "./proc";
+
+// ---------------------------------------------------------------------------
+// Boot scan orchestration
+// ---------------------------------------------------------------------------
+
+/** Aggregate counts from one boot scan (also the shape the log line carries). */
+export interface BootScanSummary {
+	scanned: number;
+	skipped: number;
+	/** Records whose interruption mark was DURABLY written (issue #304). */
+	marked: number;
+	/** Records whose mark the engine decided but whose durable write failed (#304). */
+	markFailures: number;
+	reaped: number;
+	/** Already-marked records that had leftover children reaped again this boot. */
+	revisited: number;
+	skippedUnverifiable: number;
+	skippedNoOwner: number;
+	/** Registry entries skipped fail-closed because their id failed the write guard. */
+	skippedUnsafeIds: number;
+}
+
+/** The all-zero summary (a skipped scan / an empty registry). */
+export function emptyBootScanSummary(): BootScanSummary {
+	return {
+		scanned: 0,
+		skipped: 0,
+		marked: 0,
+		markFailures: 0,
+		reaped: 0,
+		revisited: 0,
+		skippedUnverifiable: 0,
+		skippedNoOwner: 0,
+		skippedUnsafeIds: 0,
+	};
+}
+
+export interface BootScanOptions {
+	records: RecoveryRecord[];
+	deps: RecoveryDeps;
+	/**
+	 * Persist the D3 mark for this record (registry entry; agent record for
+	 * background). Returns `false` when the durable write did NOT land — the scan
+	 * then counts a `markFailure` instead of claiming `marked` (issue #304). A
+	 * `void` return keeps the historical assume-success contract.
+	 */
+	mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
+	now?: () => Date;
+	log?: Pick<Logger, "debug" | "info" | "warn">;
+}
+
+/**
+ * The warning context shared by the two mark outcomes (issue #306 nit 3): the
+ * record identity, why it was marked, which orphan pids were reaped, and which
+ * of those survived the escalation. One construction so the "marked" and
+ * "mark failed" warnings cannot drift apart.
+ */
+function markWarningPayload(
+	record: RecoveryRecord,
+	reason: string,
+	reaped: number[],
+	survived: ReadonlySet<number>,
+): { id: string; kind: RecoveryRecord["kind"]; reason: string; reaped: number[]; survived: number[] } {
+	return {
+		id: record.id,
+		kind: record.kind,
+		reason,
+		reaped,
+		survived: reaped.filter((pid) => survived.has(pid)),
+	};
+}
+
+/**
+ * Run the decision engine over every candidate, reap markers, and mark the
+ * records whose conductor is dead. Every decision is logged; the UI is
+ * deliberately silent (the notice/offer surface lands in U3).
+ *
+ * Idempotent: a marked record has `interruptedAt` set in memory before the
+ * next iteration, so a second scan over the same array is a no-op.
+ *
+ * Wait-once: every orphan pid across every record is reaped in ONE
+ * SIGTERM → grace → SIGKILL window (the `reapActiveChildren` shape), so a boot
+ * with N orphans is not an N × graceMs stall.
+ */
+export async function runBootScan(options: BootScanOptions): Promise<BootScanSummary> {
+	const { records, deps, mark } = options;
+	const nowIso = (options.now?.() ?? new Date()).toISOString();
+	const summary = emptyBootScanSummary();
+	summary.scanned = records.length;
+
+	// Plan every record first (pure), then reap ALL proposed pids together —
+	// including leftovers of already-marked records (never re-marked).
+	const planned = records.map((record) => ({ record, plan: decideRecovery(record, deps) }));
+	const allReaps = planned.flatMap(({ plan }) =>
+		plan.decision === "mark" || plan.decision === "reap" ? plan.reap : [],
+	);
+	const survived = new Set(await reapPids(allReaps, deps));
+
+	for (const { record, plan } of planned) {
+		if (plan.decision === "mark") {
+			const durable = mark(record, nowIso) !== false;
+			// In-memory mark first: guarantees the scan is idempotent even when
+			// the caller's write-back is append-only (run entries) or deferred.
+			record.interruptedAt = nowIso;
+			const reaped = [...new Set(plan.reap)];
+			summary.reaped += reaped.length;
+			const markWarning = markWarningPayload(record, plan.reason, reaped, survived);
+			if (durable) {
+				summary.marked++;
+				options.log?.warn("Recovery: marked interrupted record", markWarning);
+			} else {
+				// Issue #304: the mark did NOT land. Count the failure and warn with
+				// the id — this must be visible even when nothing was marked, so it
+				// does not hide behind the caller's `summary.marked > 0` gate.
+				summary.markFailures++;
+				options.log?.warn("Recovery: registry mark failed", markWarning);
+			}
+		} else if (plan.decision === "reap") {
+			// B2 revisit: the record was already marked (so it is never re-marked),
+			// but it still carried a live marker child — reap it again this boot.
+			const reaped = [...new Set(plan.reap)];
+			summary.revisited++;
+			summary.reaped += reaped.length;
+			options.log?.warn("Recovery: reaped a leftover child of an interrupted record", {
+				id: record.id,
+				kind: record.kind,
+				reaped,
+				survived: reaped.filter((pid) => survived.has(pid)),
+			});
+		} else {
+			summary.skipped++;
+			if (plan.reason === "ownership-unverifiable") summary.skippedUnverifiable++;
+			if (plan.reason === "no-owner") summary.skippedNoOwner++;
+			options.log?.debug("Recovery: left record untouched", {
+				id: record.id,
+				kind: record.kind,
+				reason: plan.reason,
+			});
+		}
+	}
+
+	return summary;
+}
+
+// ---------------------------------------------------------------------------
+// Registry-sourced scan (the production candidate source)
+// ---------------------------------------------------------------------------
+
+/** The subset of a persisted background agent record the scan reasons over. */
+export interface PersistedAgentLike {
+	status: string;
+	interruptedAt?: string;
+	owner?: RecoveryRecord["owner"];
+	childMarker?: string;
+}
+
+export interface RegistryScanOptions {
+	deps: RecoveryDeps;
+	/**
+	 * Persist the D3 mark for this record (registry entry; agent record for
+	 * background). Returns `false` when the durable write did not land.
+	 */
+	mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
+	/**
+	 * Read ONE background run's agent record by id (never the whole store).
+	 * Returns null when the record is absent; absent defaults to an in-flight
+	 * entry so a registry without an agent file is still marked, not skipped.
+	 */
+	readAgentRecord?(id: string): PersistedAgentLike | null;
+	now?: () => Date;
+	log?: Pick<Logger, "debug" | "info" | "warn">;
+}
+
+/**
+ * The production boot scan: candidate runs come from the durable registry
+ * (`listInflightRuns`), NOT from the current session's entries and NOT from a
+ * full parse of `.pi/subagents/`. This is what makes detection work on pi's
+ * default fresh startup after a conductor crash.
+ *
+ * Foreground/unit entries become `kind: "run"` candidates straight from the
+ * registry; background entries become `kind: "agent"` candidates enriched by
+ * reading THAT ONE agent file (so a stale registry entry whose agent record is
+ * already terminal is dropped). Legacy records with no registry entry are
+ * invisible here — consistent with the maintained `no-owner` policy (don't act
+ * on what we cannot verify).
+ */
+export async function recoverInflightRuns(options: RegistryScanOptions): Promise<BootScanSummary> {
+	// Issue #304 read/write asymmetry, fail-closed: `listInflightRuns` accepts
+	// any string id, but every write (and `markInterrupted`) resolves through
+	// `assertSafeAgentId`. Acting on an entry the writer could never have
+	// produced is unsafe, so SKIP those here — counted and reported in ONE log
+	// line — instead of attempting a mark that would silently no-op.
+	const addressable: InflightRun[] = [];
+	const unsafeIds: string[] = [];
+	for (const entry of listInflightRuns()) {
+		try {
+			assertSafeAgentId(entry.id);
+			addressable.push(entry);
+		} catch {
+			unsafeIds.push(entry.id);
+		}
+	}
+	if (unsafeIds.length > 0) {
+		options.log?.warn("Recovery: skipped registry entries whose id fails the UUID guard", {
+			count: unsafeIds.length,
+			ids: unsafeIds,
+		});
+	}
+
+	const records: RecoveryRecord[] = [];
+	for (const entry of addressable) {
+		if (entry.kind === "background") {
+			const agent = options.readAgentRecord?.(entry.id) ?? null;
+			if (agent && agent.status !== "running") continue;
+			records.push({
+				id: entry.id,
+				kind: "agent",
+				status: "running",
+				interruptedAt: agent?.interruptedAt ?? entry.interruptedAt,
+				owner: agent?.owner ?? entry.owner,
+				childMarker: agent?.childMarker ?? entry.childMarker,
+				source: entry,
+			});
+		} else {
+			records.push({
+				id: entry.id,
+				kind: "run",
+				status: "running",
+				interruptedAt: entry.interruptedAt,
+				owner: entry.owner,
+				childMarker: entry.childMarker,
+				source: entry,
+			});
+		}
+	}
+	const summary = await runBootScan({
+		records,
+		deps: options.deps,
+		mark: options.mark,
+		now: options.now,
+		log: options.log,
+	});
+	summary.skippedUnsafeIds = unsafeIds.length;
+	return summary;
+}
+
+// ---------------------------------------------------------------------------
+// Production entry (owns the /proc platform guard)
+// ---------------------------------------------------------------------------
+
+export interface ProductionRecoveryOptions {
+	/** Process deps; defaults to the real /proc wiring (`defaultRecoveryDeps`). */
+	deps?: RecoveryDeps;
+	/** Persist the D3 mark for this record; `false` = the durable write failed. */
+	mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
+	readAgentRecord?(id: string): PersistedAgentLike | null;
+	now?: () => Date;
+	log?: Pick<Logger, "debug" | "info" | "warn">;
+}
+
+/**
+ * The production boot-recovery entry. It owns the `/proc` platform guard so the
+ * guarantee "nothing is marked or killed without /proc ownership verification"
+ * is enforced — and testable — BELOW the extension entry point: on a platform
+ * without `/proc` it returns an empty summary without scanning, instead of
+ * relying on the caller's early return.
+ */
+export async function recoverProduction(options: ProductionRecoveryOptions): Promise<BootScanSummary> {
+	if (!isProcAvailable()) {
+		options.log?.info(
+			"Boot recovery skipped: /proc start-token verification unavailable on this platform",
+		);
+		return emptyBootScanSummary();
+	}
+	return recoverInflightRuns({
+		deps: options.deps ?? defaultRecoveryDeps(),
+		mark: options.mark,
+		readAgentRecord: options.readAgentRecord,
+		now: options.now,
+		log: options.log,
+	});
+}

package/src/run-registry.ts

@@ -0,0 +1,269 @@
+// Purpose: Durable per-run in-flight registry + the single run-record persist choke point that mirrors into it.
+/**
+ * Option B U1 — durable in-flight run registry.
+ *
+ * A foreground/chain/parallel/graph run's `SubagentRun` record lives ONLY as a
+ * session custom entry, so after a conductor crash a fresh process (pi's
+ * default `SessionManager.create` startup) cannot see it at all. The agent
+ * store (`.pi/subagents/<id>.json`) covers background runs only. This module
+ * closes that gap: one small JSON per IN-FLIGHT run at the cwd-relative path
+ * `.pi/run-registry/<runId>.json`, written before the run's effect and cleared
+ * when it reaches a terminal state.
+ *
+ * `persistRunRecord` is the single choke point every run-record write goes
+ * through (`state.persistRun` delegates here; the background path in
+ * session-manager calls it directly). It mirrors the record into the registry:
+ * a `running` record with a kind REGISTERS (or refreshes) an entry; a terminal
+ * record CLEARS it; a `running` record without a kind is left alone (the boot
+ * scan's `interruptedAt` mark is written through `markInterrupted`).
+ *
+ * WHERE THE INTERRUPTION MARK LIVES (commitment for U3): for foreground/unit
+ * runs the registry entry IS the durable interruption mark — the boot scan
+ * writes `interruptedAt` there, not into a session entry it cannot reach.
+ * Background runs additionally mark their `.pi/subagents/<id>.json` agent
+ * record (the existing double-mark). U3's interrupted-run surfacing reads the
+ * registry.
+ *
+ * Reads are parse-tolerant and never throw on a malformed or vanished entry;
+ * writes are atomic (temp file + rename) and owner-only (0o600), mirroring the
+ * agent store's F6 discipline.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { ProcessOwner, SubagentRun } from "./types";
+import { CUSTOM_ENTRY_TYPES } from "./types";
+import { assertSafeAgentId } from "./sanitize";
+
+/** Which execution path owns an in-flight run (drives the boot scan's mark). */
+export type InflightRunKind = "foreground" | "unit" | "background";
+
+/** The durable per-run registry entry (one small JSON file per run id). */
+export interface InflightRun {
+	id: string;
+	dispatchId?: string;
+	attempt?: number;
+	kind: InflightRunKind;
+	owner?: ProcessOwner;
+	childMarker?: string;
+	startedAt: string;
+	cwd: string;
+	interruptedAt?: string;
+}
+
+/** Fields accepted when registering/updating an entry (cwd defaults to the process cwd). */
+export interface InflightRunInput {
+	id: string;
+	dispatchId?: string;
+	attempt?: number;
+	kind: InflightRunKind;
+	owner?: ProcessOwner;
+	childMarker?: string;
+	startedAt: string;
+	cwd?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Storage location
+// ---------------------------------------------------------------------------
+
+/**
+ * Registry directory, cwd-relative like the agent store (`.pi/subagents`).
+ * Test-only override, mirroring `__setStorageDir` — the shared temp-lifecycle
+ * fixture points it at a throwaway dir.
+ */
+let REGISTRY_DIR = ".pi/run-registry";
+
+/** TEST-ONLY: point the registry at an alternate directory. */
+export function __setRegistryDir(dir: string): void {
+	REGISTRY_DIR = dir;
+}
+
+/** The active registry directory (absolute or cwd-relative as configured). */
+export function registryDir(): string {
+	return REGISTRY_DIR;
+}
+
+// ---------------------------------------------------------------------------
+// Paths + atomic IO
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve an entry's file path, refusing anything that is not a safe UUID —
+ * ids reach this module from records loaded off disk, and `path.join` does not
+ * sanitize traversal segments (same F24 rule as the agent store).
+ */
+function resolvePath(id: string): string | null {
+	try {
+		assertSafeAgentId(id);
+	} catch {
+		return null;
+	}
+	return path.join(REGISTRY_DIR, `${id}.json`);
+}
+
+function ensureDir(): void {
+	fs.mkdirSync(REGISTRY_DIR, { recursive: true, mode: 0o700 });
+}
+
+/** True when `value` looks like a registry entry (id + kind + startedAt). */
+function isInflightRun(value: unknown): value is InflightRun {
+	if (!value || typeof value !== "object") return false;
+	const v = value as Record<string, unknown>;
+	if (typeof v.id !== "string" || typeof v.startedAt !== "string") return false;
+	return v.kind === "foreground" || v.kind === "unit" || v.kind === "background";
+}
+
+/** Parse one entry file; null on missing/corrupt/malformed. */
+function readEntry(file: string): InflightRun | null {
+	try {
+		const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf-8"));
+		return isInflightRun(parsed) ? parsed : null;
+	} catch {
+		return null;
+	}
+}
+
+/** Atomic, owner-only write: temp file in the same directory, then rename. */
+function writeAtomic(file: string, entry: InflightRun): boolean {
+	try {
+		ensureDir();
+		const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+		fs.writeFileSync(tmp, JSON.stringify(entry, null, 2), { encoding: "utf-8", mode: 0o600 });
+		fs.renameSync(tmp, file);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/** Copy only defined fields from an input onto an entry (never clobber with undefined). */
+function mergeInput(base: InflightRun, input: InflightRunInput): InflightRun {
+	const next: InflightRun = { ...base, id: input.id, kind: input.kind };
+	if (input.dispatchId !== undefined) next.dispatchId = input.dispatchId;
+	if (input.attempt !== undefined) next.attempt = input.attempt;
+	if (input.owner !== undefined) next.owner = input.owner;
+	if (input.childMarker !== undefined) next.childMarker = input.childMarker;
+	if (input.startedAt) next.startedAt = input.startedAt;
+	if (input.cwd !== undefined) next.cwd = input.cwd;
+	return next;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Register (or refresh) an in-flight run. A re-register preserves an existing
+ * `interruptedAt` so a later write cannot erase a boot-scan mark.
+ */
+export function registerInflightRun(input: InflightRunInput): boolean {
+	const file = resolvePath(input.id);
+	if (!file) return false;
+	const existing = readEntry(file);
+	const entry: InflightRun = {
+		...mergeInput(existing ?? { id: input.id, kind: input.kind, startedAt: input.startedAt, cwd: input.cwd ?? process.cwd() }, input),
+	};
+	return writeAtomic(file, entry);
+}
+
+/**
+ * Merge a defined-field patch into an existing entry. A missing entry is left
+ * missing (registration is `registerInflightRun`'s job, so a stray update can
+ * never resurrect a cleared run).
+ */
+export function updateInflightRun(input: InflightRunInput): boolean {
+	const file = resolvePath(input.id);
+	if (!file) return false;
+	const existing = readEntry(file);
+	if (!existing) return false;
+	return writeAtomic(file, mergeInput(existing, input));
+}
+
+/** Remove an entry (terminal finalize). Missing file is a no-op. */
+export function clearInflightRun(id: string): void {
+	const file = resolvePath(id);
+	if (!file) return;
+	try {
+		fs.unlinkSync(file);
+	} catch {
+		// Already cleared (or never registered) — the desired end state.
+	}
+}
+
+/** Every parseable in-flight entry; [] when the registry is absent. */
+export function listInflightRuns(): InflightRun[] {
+	let files: string[];
+	try {
+		files = fs.readdirSync(REGISTRY_DIR);
+	} catch {
+		return [];
+	}
+	const entries: InflightRun[] = [];
+	for (const file of files) {
+		if (!file.endsWith(".json")) continue;
+		const entry = readEntry(path.join(REGISTRY_DIR, file));
+		if (entry) entries.push(entry);
+	}
+	return entries;
+}
+
+/**
+ * Write the D3 `interruptedAt` mark onto an existing entry. Idempotent; returns
+ * false when the entry is absent, its id fails the write guard, or the write
+ * fails. Issue #304: this MUST never throw — the boot scan treats a false as a
+ * durable-write failure and counts it, whereas a thrown FS error would abort
+ * the whole scan via `performBootRecovery`'s catch-all. The boolean is the
+ * only truthful success signal, so callers must not assume the mark landed.
+ */
+export function markInterrupted(id: string, interruptedAt: string): boolean {
+	try {
+		const file = resolvePath(id);
+		if (!file) return false;
+		const existing = readEntry(file);
+		if (!existing) return false;
+		if (existing.interruptedAt) return true;
+		return writeAtomic(file, { ...existing, interruptedAt });
+	} catch {
+		// Belt-and-braces: the inner read/write helpers already swallow FS errors,
+		// but the scan's no-throw contract must not depend on that staying true.
+		return false;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// The persistence choke point
+// ---------------------------------------------------------------------------
+
+/** Project a run record into the registry input shape. */
+function toRegistryInput(run: SubagentRun, kind: InflightRunKind): InflightRunInput {
+	return {
+		id: run.id,
+		dispatchId: run.dispatchId,
+		attempt: run.attempt,
+		kind,
+		owner: run.owner,
+		childMarker: run.childMarker,
+		startedAt: run.startedAt,
+		cwd: process.cwd(),
+	};
+}
+
+/**
+ * The ONE run-record writer: append the session entry AND keep the registry in
+ * lockstep. `state.persistRun` delegates here and the background path calls it
+ * directly, so every write site is mirrored by construction.
+ *
+ * `kind` is required only at dispatch (`running`); terminal writes clear the
+ * registry and ignore it. A `running` write without a kind is a refresh with
+ * nothing to register — the boot scan's mark is a separate `markInterrupted`.
+ */
+export function persistRunRecord(pi: ExtensionAPI, run: SubagentRun, kind?: InflightRunKind): void {
+	if (run.status !== "running") {
+		clearInflightRun(run.id);
+	} else if (kind) {
+		registerInflightRun(toRegistryInput(run, kind));
+	}
+	pi.appendEntry(CUSTOM_ENTRY_TYPES.run, run);
+}