@fitchmultz/pi-subagents 0.0.0-stage → 0.44.4

package/src/runs/background/stale-run-reconciler.ts

@@ -0,0 +1,369 @@
+import * as fs from "node:fs";
+import { readOutputPage } from "../../shared/journal-reader.ts";
+import * as path from "node:path";
+import { writeAtomicJson } from "../../shared/atomic-json.ts";
+import { RESULTS_DIR, RUNNER_ERROR_LOG_FILE, type AsyncParallelGroupStatus, type AsyncResultChild, type AsyncResultTerminalState, type AsyncStatus, type NestedRunSummary, type SubagentRunMode } from "../../shared/types.ts";
+import { normalizeParallelGroups } from "./parallel-groups.ts";
+import { nestedSummaryFromAsyncStatus, projectNestedEvents, resolveNestedAsyncDir, writeNestedEvent, type NestedRoute } from "../shared/nested-events.ts";
+import { isDurableRun, readAsyncResultFileIfExists } from "./async-result-file.ts";
+import { readRunJson } from "../shared/supervisor-questions.ts";
+import { readStatus } from "../../shared/utils.ts";
+import type { AsyncRunRecord } from "./async-resume.ts";
+
+export type PidLiveness = "alive" | "dead" | "unknown";
+
+type KillFn = (pid: number, signal?: NodeJS.Signals | 0) => boolean;
+
+interface StartedRunMetadata {
+	runId: string;
+	pid?: number;
+	sessionId?: string;
+	mode?: SubagentRunMode;
+	agents?: string[];
+	chainStepCount?: number;
+	parallelGroups?: AsyncParallelGroupStatus[];
+	startedAt?: number;
+	sessionFile?: string;
+}
+
+interface ReconcileAsyncRunOptions {
+	resultsDir?: string;
+	kill?: KillFn;
+	now?: () => number;
+	startedRun?: StartedRunMetadata;
+	missingStatusGraceMs?: number;
+	staleAlivePidMs?: number;
+}
+
+interface ReconcileAsyncRunResult {
+	status: AsyncStatus | null;
+	repaired: boolean;
+	resultPath?: string;
+	message?: string;
+}
+
+function isNotFoundError(error: unknown): boolean {
+	return typeof error === "object"
+		&& error !== null
+		&& "code" in error
+		&& (error as NodeJS.ErrnoException).code === "ENOENT";
+}
+
+function appendJsonl(filePath: string, payload: object): void {
+	fs.mkdirSync(path.dirname(filePath), { recursive: true });
+	fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, "utf-8");
+}
+
+function readStatusFile(asyncDir: string): AsyncStatus | null {
+	return readStatus(asyncDir);
+}
+
+interface ResultRepairData {
+	state: AsyncResultTerminalState;
+	timestamp?: number;
+	results?: AsyncResultChild[];
+}
+
+function readResultRepairData(resultPath: string): ResultRepairData | undefined {
+	const data = readAsyncResultFileIfExists(resultPath);
+	if (!data) return undefined;
+	return { state: data.terminalState, timestamp: data.timestamp, ...(Array.isArray(data.results) ? { results: data.results } : {}) };
+}
+
+function childState(overallState: ResultRepairData["state"], child: AsyncResultChild | undefined): "complete" | "failed" | "blocked" | "paused" {
+	if (child?.success === true) return "complete";
+	if (child?.interrupted === true) return "paused";
+	if (child?.acceptance?.status === "blocked" && (child.exitCode === 0 || child.exitCode === undefined)) return "blocked";
+	if (child?.success === false) return "failed";
+	return overallState;
+}
+
+function withoutLiveActivity<T extends { currentTool?: string; currentToolArgs?: string; currentToolStartedAt?: number; currentPath?: string; activityState?: string }>(value: T): T {
+	return {
+		...value,
+		activityState: undefined,
+		currentTool: undefined,
+		currentToolArgs: undefined,
+		currentToolStartedAt: undefined,
+		currentPath: undefined,
+	};
+}
+
+function terminalStatusFromResult(status: AsyncStatus, resultPath: string, now: number): AsyncStatus | undefined {
+	const repair = readResultRepairData(resultPath);
+	if (!repair) return undefined;
+	now = repair.timestamp ?? now;
+	const steps = (status.steps ?? []).map((step, index) => {
+		if (step.status !== "running" && step.status !== "pending") return withoutLiveActivity(step);
+		const child = repair.results?.[index];
+		if (isDurableRun(status) && step.status === "pending" && !child) return withoutLiveActivity(step);
+		const state = childState(repair.state, child);
+		return withoutLiveActivity({
+			...step,
+			status: state === "complete" ? "complete" as const : state,
+			endedAt: step.endedAt ?? now,
+			durationMs: step.startedAt !== undefined && step.durationMs === undefined ? Math.max(0, now - step.startedAt) : step.durationMs,
+			exitCode: step.exitCode ?? (state === "complete" || state === "blocked" || state === "paused" ? 0 : 1),
+			error: state === "failed" ? step.error ?? child?.error : step.error,
+			sessionFile: step.sessionFile ?? child?.sessionFile,
+			model: step.model ?? child?.model,
+			attemptedModels: step.attemptedModels ?? child?.attemptedModels,
+			modelAttempts: step.modelAttempts ?? child?.modelAttempts,
+			acceptance: step.acceptance ?? child?.acceptance,
+			agentProcessExit: step.agentProcessExit ?? child?.agentProcessExit,
+		});
+	});
+	return withoutLiveActivity({
+		...status,
+		state: repair.state,
+		lastUpdate: now,
+		endedAt: status.endedAt ?? now,
+		steps,
+	});
+}
+
+function buildStartedStatus(asyncDir: string, startedRun: StartedRunMetadata, now: number): AsyncStatus {
+	const startedAt = startedRun.startedAt ?? now;
+	const agents = startedRun.agents?.length ? startedRun.agents : ["subagent"];
+	const chainStepCount = startedRun.chainStepCount;
+	const parallelGroups = chainStepCount !== undefined
+		? normalizeParallelGroups(startedRun.parallelGroups, agents.length, chainStepCount)
+		: [];
+	return {
+		runId: startedRun.runId || path.basename(asyncDir),
+		...(startedRun.sessionId ? { sessionId: startedRun.sessionId } : {}),
+		mode: startedRun.mode ?? "single",
+		state: "running",
+		pid: startedRun.pid,
+		startedAt,
+		lastUpdate: now,
+		currentStep: 0,
+		...(chainStepCount !== undefined ? { chainStepCount } : {}),
+		...(parallelGroups.length ? { parallelGroups } : {}),
+		steps: agents.map((agent) => ({
+			agent,
+			status: "running" as const,
+			startedAt,
+		})),
+		...(startedRun.sessionFile ? { sessionFile: startedRun.sessionFile } : {}),
+	};
+}
+
+function readCompletedStepOutput(asyncDir: string, index: number): string {
+	try {
+		return readOutputPage(path.join(asyncDir, `output-${index}.log`)).text.trim();
+	} catch (error) {
+		if (isNotFoundError(error)) return "";
+		throw error;
+	}
+}
+
+const RUNNER_STDERR_EXCERPT_BYTES = 16 * 1024;
+const UNSAFE_CONTROL_CHARACTERS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g;
+
+function readRunnerStderr(asyncDir: string): string | undefined {
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(path.join(asyncDir, RUNNER_ERROR_LOG_FILE), "r");
+		const size = fs.fstatSync(fd).size;
+		const length = Math.min(size, RUNNER_STDERR_EXCERPT_BYTES);
+		const buffer = Buffer.alloc(length);
+		const bytesRead = fs.readSync(fd, buffer, 0, length, Math.max(0, size - length));
+		const excerpt = buffer.subarray(0, bytesRead).toString("utf-8").replace(UNSAFE_CONTROL_CHARACTERS, "�").trim();
+		if (!excerpt) return undefined;
+		return size > length ? `[runner stderr truncated to last ${length} bytes]\n${excerpt}` : excerpt;
+	} catch {
+		return undefined;
+	} finally {
+		if (fd !== undefined) {
+			try { fs.closeSync(fd); } catch {}
+		}
+	}
+}
+
+function buildFailedRepair(status: AsyncStatus, asyncDir: string, now: number, reason?: string): { status: AsyncStatus; result: object; message: string } {
+	const runId = status.runId || path.basename(asyncDir);
+	const pid = typeof status.pid === "number" ? status.pid : "unknown";
+	const defaultMessage = `Async runner process ${pid} exited or disappeared before writing a result. Completion is unconfirmed.`;
+	const runnerStderr = reason ? undefined : readRunnerStderr(asyncDir);
+	const message = reason ?? (runnerStderr ? `${defaultMessage}\n\nRunner stderr:\n${runnerStderr}` : defaultMessage);
+	const steps = status.steps?.length ? status.steps : [{ agent: "subagent", status: "running" as const }];
+	const repairedSteps = steps.map((step) => step.status === "running" || String(step.status) === "queued" || step.status === "pending"
+		? withoutLiveActivity({
+			...step,
+			status: "failed" as const,
+			endedAt: step.endedAt ?? now,
+			durationMs: step.startedAt !== undefined && step.durationMs === undefined ? Math.max(0, now - step.startedAt) : step.durationMs,
+			exitCode: step.exitCode ?? 1,
+			error: step.error ?? message,
+		})
+		: withoutLiveActivity(step));
+	const repairedStatus: AsyncStatus = withoutLiveActivity({
+		...status,
+		state: "failed",
+		lastUpdate: now,
+		endedAt: now,
+		steps: repairedSteps,
+	});
+	const resultAgent = repairedSteps[status.currentStep ?? 0]?.agent ?? repairedSteps[0]?.agent ?? "subagent";
+	return {
+		status: repairedStatus,
+		message,
+		result: {
+			id: runId,
+			agent: resultAgent,
+			mode: status.mode,
+			success: false,
+			state: "failed",
+			summary: message,
+			results: repairedSteps.map((step, index) => {
+				const success = step.status === "complete" || step.status === "completed";
+				return {
+					agent: step.agent,
+					output: success ? readCompletedStepOutput(asyncDir, index) : message,
+					error: success ? undefined : step.error ?? message,
+					success,
+					model: step.model,
+					attemptedModels: step.attemptedModels,
+					modelAttempts: step.modelAttempts,
+					sessionFile: step.sessionFile,
+				};
+			}),
+			exitCode: 1,
+			timestamp: now,
+			durationMs: Math.max(0, now - status.startedAt),
+			asyncDir,
+			sessionId: status.sessionId,
+			cwd: status.cwd,
+			sessionFile: status.sessionFile,
+		},
+	};
+}
+
+function writeFailedRepair(asyncDir: string, status: AsyncStatus, resultPath: string, now: number, reason?: string): ReconcileAsyncRunResult {
+	const repair = buildFailedRepair(status, asyncDir, now, reason);
+	writeAtomicJson(resultPath, repair.result);
+	writeAtomicJson(path.join(asyncDir, "status.json"), repair.status);
+	appendJsonl(path.join(asyncDir, "events.jsonl"), {
+		type: "subagent.run.repaired_stale",
+		ts: now,
+		runId: repair.status.runId,
+		pid: status.pid,
+		resultPath,
+		message: repair.message,
+	});
+	return { status: repair.status, repaired: true, resultPath, message: repair.message };
+}
+
+function terminal(state: AsyncStatus["state"]): boolean {
+	return state === "complete" || state === "failed" || state === "blocked" || state === "paused";
+}
+
+function* nestedRuns(children: NestedRunSummary[] | undefined): Generator<NestedRunSummary> {
+	for (const child of children ?? []) {
+		yield child;
+		yield* nestedRuns(child.children);
+		yield* nestedRuns(child.steps?.flatMap((step) => step.children ?? []));
+	}
+}
+
+export function reconcileNestedAsyncDescendants(route: NestedRoute, options: ReconcileAsyncRunOptions = {}): NestedRunSummary[] {
+	const registry = projectNestedEvents(route);
+	for (const run of nestedRuns(registry.children)) {
+		if (run.state !== "running" && run.state !== "queued") continue;
+		const asyncDir = resolveNestedAsyncDir(route.rootRunId, run);
+		if (!asyncDir) continue;
+		const result = reconcileAsyncRun(asyncDir, {
+			...options,
+			resultsDir: path.join(options.resultsDir ?? RESULTS_DIR, "nested", route.rootRunId),
+		});
+		const status = result.status;
+		if (!status) continue;
+		const ts = options.now?.() ?? Date.now();
+		const child = nestedSummaryFromAsyncStatus(status, asyncDir, {
+			id: run.id, parentRunId: run.parentRunId, parentStepIndex: run.parentStepIndex,
+			depth: run.depth, path: run.path, mode: run.mode, ts,
+		});
+		child.steps?.forEach((step, index) => { step.children = run.steps?.[index]?.children; });
+		Object.assign(run, child, result.message ? { error: result.message } : {});
+		if (isDurableRun(status)) continue;
+		if (!result.repaired && !terminal(status.state)) continue;
+		writeNestedEvent(route, {
+			type: terminal(status.state) ? "subagent.nested.completed" : "subagent.nested.updated",
+			ts,
+			parentRunId: run.parentRunId,
+			parentStepIndex: run.parentStepIndex,
+			child,
+		});
+	}
+	return registry.children;
+}
+
+export function checkPidLiveness(pid: number, kill: KillFn = process.kill): PidLiveness {
+	try {
+		kill(pid, 0);
+		return "alive";
+	} catch (error) {
+		const code = typeof error === "object" && error !== null && "code" in error
+			? (error as NodeJS.ErrnoException).code
+			: undefined;
+		if (code === "ESRCH") return "dead";
+		if (code === "EPERM") return "unknown";
+		return "unknown";
+	}
+}
+
+export function reconcileAsyncRun(asyncDir: string, options: ReconcileAsyncRunOptions = {}, record?: Pick<AsyncRunRecord, "status" | "durable">): ReconcileAsyncRunResult {
+	const now = options.now?.() ?? Date.now();
+	const status = record ? record.status : readStatusFile(asyncDir);
+	const startedStatus = !status && options.startedRun ? buildStartedStatus(asyncDir, options.startedRun, now) : undefined;
+	const effectiveStatus = status ?? startedStatus;
+	if (!effectiveStatus) return { status: null, repaired: false };
+
+	const runId = effectiveStatus.runId || path.basename(asyncDir);
+	const durable = isDurableRun(effectiveStatus) || (record ? record.durable : isDurableRun(readRunJson<object>(path.join(asyncDir, "launch.json"))));
+	const resultPath = durable ? path.join(asyncDir, "result.json") : path.join(options.resultsDir ?? RESULTS_DIR, `${runId}.json`);
+	if (durable) {
+		// The detached owner is the only execution writer. Inspection projects evidence.
+		const final = terminalStatusFromResult(effectiveStatus, resultPath, effectiveStatus.endedAt ?? now);
+		if (final) return { status: final, repaired: false, resultPath };
+		if (typeof effectiveStatus.pid === "number" && checkPidLiveness(effectiveStatus.pid, options.kill) === "dead") {
+			const failure = buildFailedRepair(effectiveStatus, asyncDir, effectiveStatus.lastUpdate ?? effectiveStatus.startedAt);
+			return { status: failure.status, repaired: false, message: failure.message };
+		}
+		if (terminal(effectiveStatus.state)) return { status: { ...effectiveStatus, state: "running" }, repaired: false, message: "Waiting for the owner's final result. Completion is unconfirmed." };
+		return { status: effectiveStatus, repaired: false };
+	}
+	if (fs.existsSync(resultPath)) {
+		const terminalStatus = effectiveStatus.state === "running" || effectiveStatus.state === "queued"
+			? terminalStatusFromResult(effectiveStatus, resultPath, now)
+			: undefined;
+		if (terminalStatus) {
+			writeAtomicJson(path.join(asyncDir, "status.json"), terminalStatus);
+			return { status: terminalStatus, repaired: true, resultPath, message: "Existing async result file was used to repair stale running status." };
+		}
+		return { status: effectiveStatus, repaired: false, resultPath };
+	}
+
+	if ((effectiveStatus.state !== "running" && effectiveStatus.state !== "queued") || typeof effectiveStatus.pid !== "number") {
+		return { status: status ?? null, repaired: false, resultPath };
+	}
+
+	if (!status) {
+		const startedAt = options.startedRun?.startedAt ?? effectiveStatus.startedAt;
+		if (now - startedAt < (options.missingStatusGraceMs ?? 1000)) {
+			return { status: null, repaired: false, resultPath };
+		}
+	}
+
+	const liveness = checkPidLiveness(effectiveStatus.pid, options.kill);
+	if (liveness !== "dead") {
+		const staleAfterMs = options.staleAlivePidMs ?? 24 * 60 * 60 * 1000;
+		const lastUpdate = effectiveStatus.lastUpdate ?? effectiveStatus.startedAt;
+		if (now - lastUpdate <= staleAfterMs) return { status: status ?? null, repaired: false, resultPath };
+		const message = `Async runner process ${effectiveStatus.pid} still has a live PID, but status has not updated for ${now - lastUpdate}ms. Marked run failed by stale-run reconciliation because PID ownership cannot be verified.`;
+		return writeFailedRepair(asyncDir, effectiveStatus, resultPath, now, message);
+	}
+
+	return writeFailedRepair(asyncDir, effectiveStatus, resultPath, now);
+}

package/src/runs/background/subagent-runner-launcher.ts

@@ -0,0 +1,56 @@
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+
+const RETRY_WINDOW_MS = 30_000;
+const RETRY_DELAY_MS = 500;
+const MAX_STDERR_LENGTH = 64 * 1024;
+
+function run(runnerPath: string, configPath: string, piPackageRoot?: string): Promise<{ code: number; stderr: string }> {
+	return new Promise((resolve) => {
+		const child = spawn(process.execPath, [runnerPath, configPath], {
+			stdio: ["ignore", "inherit", "pipe"],
+			env: piPackageRoot ? { ...process.env, PI_PACKAGE_DIR: process.env.PI_PACKAGE_DIR ?? piPackageRoot } : process.env,
+		});
+		let stderr = "";
+		child.stderr.on("data", (chunk: Buffer) => {
+			process.stderr.write(chunk);
+			stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_LENGTH);
+		});
+		child.once("error", (error) => {
+			const stderr = error.stack ?? error.message;
+			process.stderr.write(`${stderr}\n`);
+			resolve({ code: 1, stderr });
+		});
+		child.once("close", (code) => resolve({ code: code ?? 1, stderr }));
+	});
+}
+
+const [runnerPath, configPath] = process.argv.slice(2);
+if (!runnerPath || !configPath) throw new Error("Usage: subagent-runner-launcher <runner> <config>");
+
+let statusPath: string | undefined;
+let piPackageRoot: string | undefined;
+try {
+	const config = JSON.parse(fs.readFileSync(configPath, "utf-8")) as { asyncDir?: unknown; piPackageRoot?: unknown };
+	if (typeof config.piPackageRoot === "string") piPackageRoot = config.piPackageRoot;
+	if (typeof config.asyncDir === "string") statusPath = path.join(config.asyncDir, "status.json");
+} catch {}
+
+// ponytail: bridge short in-place extension updates; use shared update locking if longer gaps appear.
+const deadline = Date.now() + RETRY_WINDOW_MS;
+let announcedRetry = false;
+while (true) {
+	const result = await run(runnerPath, configPath, piPackageRoot);
+	const failedBeforeStartup = !statusPath || !fs.existsSync(statusPath);
+	if (result.code === 0 || !failedBeforeStartup || !result.stderr.includes("ERR_MODULE_NOT_FOUND") || Date.now() >= deadline) {
+		process.exitCode = result.code;
+		break;
+	}
+	if (!announcedRetry) {
+		process.stderr.write("[pi-subagents] Runner dependencies are being updated; retrying startup.\n");
+		announcedRetry = true;
+	}
+	await delay(RETRY_DELAY_MS);
+}