@katyella/legio 0.1.0

package/src/watchdog/health.test.ts

@@ -0,0 +1,371 @@
+import { describe, expect, test } from "vitest";
+import type { AgentSession } from "../types.ts";
+import { evaluateHealth, isProcessRunning, transitionState } from "./health.ts";
+
+/**
+ * Tests for the ZFC-based health evaluation and state machine.
+ *
+ * evaluateHealth is a pure function that takes session state + tmux liveness +
+ * thresholds and returns a HealthCheck. No mocks needed for the core logic.
+ *
+ * isProcessRunning uses process.kill(pid, 0) which is safe to test with real PIDs:
+ * the current process PID (alive) and a known-dead PID (not alive).
+ *
+ * Note: evaluateHealth calls isProcessRunning internally. For tests that need
+ * to control pid liveness independently of the actual OS process table, we set
+ * session.pid to known-alive (current process) or known-dead PIDs.
+ */
+
+const THRESHOLDS = { zombieMs: 120_000 };
+
+/** PID that is guaranteed to be alive during tests: our own process. */
+const ALIVE_PID = process.pid;
+
+/**
+ * PID that is very likely dead. PID 2147483647 (max 32-bit signed int) is
+ * almost never in use. If by some miracle it is, the test still works because
+ * we use it only for the "pid dead" path and the test validates behavior, not
+ * the exact PID value.
+ */
+const DEAD_PID = 2147483647;
+
+function makeSession(overrides: Partial<AgentSession> = {}): AgentSession {
+	return {
+		id: "session-test",
+		agentName: "test-agent",
+		capability: "builder",
+		worktreePath: "/tmp/test",
+		branchName: "legio/test-agent/test-task",
+		beadId: "test-task",
+		tmuxSession: "legio-test-agent",
+		state: "booting",
+		pid: ALIVE_PID,
+		parentAgent: null,
+		depth: 0,
+		runId: null,
+		startedAt: new Date().toISOString(),
+		lastActivity: new Date().toISOString(),
+		escalationLevel: 0,
+		stalledSince: null,
+		...overrides,
+	};
+}
+
+// === isProcessRunning ===
+
+describe("isProcessRunning", () => {
+	test("returns true for the current process PID", () => {
+		expect(isProcessRunning(process.pid)).toBe(true);
+	});
+
+	test("returns false for a PID that does not exist", () => {
+		// PID 2147483647 is max 32-bit signed — extremely unlikely to be alive
+		expect(isProcessRunning(DEAD_PID)).toBe(false);
+	});
+});
+
+// === evaluateHealth ===
+
+describe("evaluateHealth", () => {
+	// --- ZFC Rule 1: tmux dead → zombie (observable state wins) ---
+
+	test("ZFC: tmux dead + sessions.json says working → zombie with reconciliation note", () => {
+		const session = makeSession({ state: "working" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+		expect(check.tmuxAlive).toBe(false);
+		expect(check.processAlive).toBe(false);
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain("tmux dead");
+		expect(check.reconciliationNote).toContain('"working"');
+	});
+
+	test("ZFC: tmux dead + sessions.json says booting → zombie with reconciliation note", () => {
+		const session = makeSession({ state: "booting" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain('"booting"');
+	});
+
+	// --- ZFC Rule 2: tmux alive + sessions.json says zombie → investigate ---
+
+	test("ZFC: tmux alive + sessions.json says zombie → investigate (don't auto-kill)", () => {
+		const session = makeSession({ state: "zombie", pid: ALIVE_PID });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("investigate");
+		expect(check.processAlive).toBe(true);
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain("investigation needed");
+		expect(check.reconciliationNote).toContain("don't auto-kill");
+	});
+
+	// --- ZFC Rule 3: pid dead + tmux alive → zombie ---
+
+	test("ZFC: pid dead + tmux alive → zombie (agent process exited, shell survived)", () => {
+		const session = makeSession({ state: "working", pid: DEAD_PID });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+		expect(check.processAlive).toBe(false);
+		expect(check.pidAlive).toBe(false);
+		expect(check.tmuxAlive).toBe(true);
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain("pid");
+		expect(check.reconciliationNote).toContain("shell survived");
+	});
+
+	// --- pid null (unavailable) ---
+
+	test("pid null does not trigger pid-based zombie detection", () => {
+		const session = makeSession({ state: "working", pid: null });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+		expect(check.pidAlive).toBeNull();
+	});
+
+	// --- Time-based checks (both tmux and pid alive) ---
+
+	test("activity older than zombieMs → zombie", () => {
+		const oldActivity = new Date(Date.now() - 200_000).toISOString();
+		const session = makeSession({ state: "working", lastActivity: oldActivity });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+		expect(check.reconciliationNote).toBeNull();
+	});
+
+	// --- Normal state transitions ---
+
+	test("booting with recent activity → transitions to working", () => {
+		const recentActivity = new Date(Date.now() - 5_000).toISOString();
+		const session = makeSession({ state: "booting", lastActivity: recentActivity });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+		expect(check.reconciliationNote).toBeNull();
+	});
+
+	test("working with recent activity → stays working", () => {
+		const recentActivity = new Date(Date.now() - 5_000).toISOString();
+		const session = makeSession({ state: "working", lastActivity: recentActivity });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	// --- Persistent capabilities (coordinator, monitor) ---
+
+	test("persistent capability: coordinator with stale activity → still working, no escalation", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			capability: "coordinator",
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("persistent capability: coordinator with zombie-level staleness → still working", () => {
+		const oldActivity = new Date(Date.now() - 200_000).toISOString();
+		const session = makeSession({
+			capability: "coordinator",
+			state: "working",
+			lastActivity: oldActivity,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("persistent capability: monitor with stale activity → still working", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			capability: "monitor",
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("persistent capability: gateway with stale activity → still working", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			capability: "gateway",
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("persistent capability: coordinator booting → transitions to working", () => {
+		const session = makeSession({
+			capability: "coordinator",
+			state: "booting",
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("persistent capability: coordinator with tmux dead → still zombie (ZFC Rule 1 applies)", () => {
+		const session = makeSession({
+			capability: "coordinator",
+			state: "working",
+		});
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+	});
+
+	test("persistent capability: coordinator with pid dead → still zombie (ZFC Rule 3 applies)", () => {
+		const session = makeSession({
+			capability: "coordinator",
+			state: "working",
+			pid: DEAD_PID,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+	});
+
+	// --- Completed agents ---
+
+	test("completed agents skip monitoring", () => {
+		const session = makeSession({ state: "completed" });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("completed");
+		expect(check.action).toBe("none");
+		expect(check.reconciliationNote).toBeNull();
+	});
+
+	// --- pidAlive field is populated ---
+
+	test("pidAlive reflects actual process state for alive PID", () => {
+		const session = makeSession({ pid: ALIVE_PID, state: "working" });
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.pidAlive).toBe(true);
+	});
+
+	test("pidAlive reflects actual process state for dead PID", () => {
+		// Use dead pid but also tmux dead to avoid pid-zombie path intercepting
+		const session = makeSession({ pid: DEAD_PID, state: "working" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		// tmux dead takes priority, so state is zombie via ZFC Rule 1
+		expect(check.state).toBe("zombie");
+		expect(check.pidAlive).toBe(false);
+	});
+});
+
+// === transitionState ===
+
+describe("transitionState", () => {
+	test("advances from booting to working", () => {
+		const check = {
+			state: "working" as const,
+			agentName: "a",
+			timestamp: "",
+			tmuxAlive: true,
+			pidAlive: true as boolean | null,
+			lastActivity: "",
+			processAlive: true,
+			action: "none" as const,
+			reconciliationNote: null,
+		};
+		expect(transitionState("booting", check)).toBe("working");
+	});
+
+	test("never regresses from zombie to booting", () => {
+		const check = {
+			state: "booting" as const,
+			agentName: "a",
+			timestamp: "",
+			tmuxAlive: true,
+			pidAlive: true as boolean | null,
+			lastActivity: "",
+			processAlive: true,
+			action: "none" as const,
+			reconciliationNote: null,
+		};
+		expect(transitionState("zombie", check)).toBe("zombie");
+	});
+
+	test("same state stays the same", () => {
+		const check = {
+			state: "working" as const,
+			agentName: "a",
+			timestamp: "",
+			tmuxAlive: true,
+			pidAlive: true as boolean | null,
+			lastActivity: "",
+			processAlive: true,
+			action: "none" as const,
+			reconciliationNote: null,
+		};
+		expect(transitionState("working", check)).toBe("working");
+	});
+
+	// --- ZFC: investigate holds state ---
+
+	test("ZFC: investigate action holds current state (does not advance)", () => {
+		const check = {
+			state: "zombie" as const,
+			agentName: "a",
+			timestamp: "",
+			tmuxAlive: true,
+			pidAlive: true as boolean | null,
+			lastActivity: "",
+			processAlive: true,
+			action: "investigate" as const,
+			reconciliationNote: "ZFC: tmux alive but sessions.json says zombie",
+		};
+		// Even though check.state is zombie (order 4) and current is zombie (order 4),
+		// investigate should hold — not advance
+		expect(transitionState("zombie", check)).toBe("zombie");
+	});
+
+	test("ZFC: investigate prevents forward transition", () => {
+		const check = {
+			state: "zombie" as const,
+			agentName: "a",
+			timestamp: "",
+			tmuxAlive: true,
+			pidAlive: true as boolean | null,
+			lastActivity: "",
+			processAlive: true,
+			action: "investigate" as const,
+			reconciliationNote: "ZFC conflict",
+		};
+		// If something were at "working" and check says zombie with investigate,
+		// the state should NOT advance
+		expect(transitionState("working", check)).toBe("working");
+	});
+});

package/src/watchdog/health.ts

@@ -0,0 +1,248 @@
+/**
+ * Health check state machine and evaluation logic for agent monitoring.
+ *
+ * ZFC Principle (Zero Failure Crash)
+ * ==================================
+ * Observable state is the source of truth, not recorded state.
+ *
+ * Signal priority (highest to lowest):
+ *   1. tmux session liveness  — Is the tmux session actually running?
+ *   2. Process liveness (pid) — Is the Claude Code process still alive?
+ *   3. Recorded state         — What does sessions.json claim?
+ *
+ * When signals conflict, always trust what you can observe:
+ *   - tmux dead + sessions.json says "working" → mark zombie immediately.
+ *     The recorded state is stale; the process is gone.
+ *   - tmux alive + sessions.json says "zombie" → investigate, don't auto-kill.
+ *     Something marked it zombie but the process recovered or was misclassified.
+ *   - pid dead + tmux alive → the pane's shell survived but the agent process
+ *     exited. Treat as zombie (the agent is not doing work).
+ *   - pid alive + tmux dead → should not happen (tmux owns the pid), but if it
+ *     does, trust tmux (the session is gone).
+ *
+ * The rationale: sessions.json is updated asynchronously by hooks and can become
+ * stale if the agent crashes between hook invocations. tmux and the OS process
+ * table are always up-to-date because they reflect real kernel state.
+ */
+
+import type { AgentSession, AgentState, HealthCheck } from "../types.ts";
+
+/**
+ * Agent capabilities that run as persistent interactive sessions.
+ * These agents are expected to have long idle periods (e.g. coordinator waiting
+ * for worker mail) and should NOT be flagged stale/zombie based on lastActivity.
+ * Only tmux/pid liveness checks apply to them.
+ *
+ * Shared concept with src/commands/log.ts:PERSISTENT_CAPABILITIES.
+ */
+const PERSISTENT_CAPABILITIES = new Set(["coordinator", "monitor", "gateway"]);
+
+/** Numeric ordering for forward-only state transitions. */
+const STATE_ORDER: Partial<Record<AgentState, number>> = {
+	booting: 0,
+	working: 1,
+	completed: 2,
+	zombie: 3,
+};
+
+/**
+ * Check whether a process with the given PID is still running.
+ *
+ * Uses signal 0 which does not kill the process — it only checks
+ * whether it exists and we have permission to signal it.
+ *
+ * @param pid - The process ID to check
+ * @returns true if the process exists, false otherwise
+ */
+export function isProcessRunning(pid: number): boolean {
+	try {
+		// Signal 0 doesn't kill the process — just checks if it exists
+		process.kill(pid, 0);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Evaluate the health of an agent session.
+ *
+ * Implements the ZFC principle: observable state (tmux liveness, pid liveness)
+ * takes priority over recorded state (sessions.json fields).
+ *
+ * Decision logic (in priority order):
+ *
+ * 1. Completed agents skip monitoring entirely.
+ * 2. tmux dead → zombie, terminate (regardless of what sessions.json says).
+ * 3. tmux alive + sessions.json says zombie → investigate (don't auto-kill).
+ *    Something external marked this zombie, but the process is still running.
+ * 4. pid dead + tmux alive → zombie, terminate. The agent process exited but
+ *    the tmux pane shell survived. The agent is not doing work.
+ * 5. lastActivity older than zombieMs → zombie, terminate.
+ * 6. booting with recent activity → working.
+ * 7. Otherwise → working, healthy.
+ *
+ * @param session - The agent session to evaluate
+ * @param tmuxAlive - Whether the agent's tmux session is still running
+ * @param thresholds - Zombie time threshold in milliseconds
+ * @returns A HealthCheck describing the agent's current state and recommended action
+ */
+export function evaluateHealth(
+	session: AgentSession,
+	tmuxAlive: boolean,
+	thresholds: { zombieMs: number },
+): HealthCheck {
+	const now = new Date();
+	const lastActivityTime = new Date(session.lastActivity).getTime();
+	const elapsedMs = now.getTime() - lastActivityTime;
+
+	// Check pid liveness as secondary signal (null if pid unavailable)
+	const pidAlive = session.pid !== null ? isProcessRunning(session.pid) : null;
+
+	const base: Pick<
+		HealthCheck,
+		"agentName" | "timestamp" | "tmuxAlive" | "pidAlive" | "lastActivity"
+	> = {
+		agentName: session.agentName,
+		timestamp: now.toISOString(),
+		tmuxAlive,
+		pidAlive,
+		lastActivity: session.lastActivity,
+	};
+
+	// Completed agents don't need health monitoring
+	if (session.state === "completed") {
+		return {
+			...base,
+			processAlive: tmuxAlive,
+			state: "completed",
+			action: "none",
+			reconciliationNote: null,
+		};
+	}
+
+	// ZFC Rule 1: tmux dead → zombie immediately, regardless of recorded state.
+	// Observable state says the process is gone.
+	if (!tmuxAlive) {
+		const note =
+			session.state === "working" || session.state === "booting"
+				? `ZFC: tmux dead but sessions.json says "${session.state}" — marking zombie (observable state wins)`
+				: null;
+
+		return {
+			...base,
+			processAlive: false,
+			state: "zombie",
+			action: "terminate",
+			reconciliationNote: note,
+		};
+	}
+
+	// ZFC Rule 2: tmux alive but sessions.json says zombie → investigate.
+	// Something marked it zombie but the process is still running. Don't auto-kill;
+	// a human or higher-tier agent should decide.
+	if (session.state === "zombie") {
+		return {
+			...base,
+			processAlive: true,
+			state: "zombie",
+			action: "investigate",
+			reconciliationNote:
+				"ZFC: tmux alive but sessions.json says zombie — investigation needed (don't auto-kill)",
+		};
+	}
+
+	// ZFC Rule 3: pid dead but tmux alive → the agent process exited but the
+	// tmux pane shell survived. The agent is not doing work.
+	if (pidAlive === false) {
+		return {
+			...base,
+			processAlive: false,
+			state: "zombie",
+			action: "terminate",
+			reconciliationNote: `ZFC: pid ${session.pid} dead but tmux alive — agent process exited, shell survived`,
+		};
+	}
+
+	// Persistent capabilities (coordinator, monitor) are expected to have long idle
+	// periods waiting for mail/events. Skip time-based zombie detection for
+	// them — only tmux/pid liveness matters (checked above).
+	if (PERSISTENT_CAPABILITIES.has(session.capability)) {
+		// Transition booting → working if we reach here (tmux alive, pid alive)
+		const state = session.state === "booting" ? "working" : session.state;
+		return {
+			...base,
+			processAlive: true,
+			state,
+			action: "none",
+			reconciliationNote: null,
+		};
+	}
+
+	// Time-based checks (both tmux and pid confirmed alive, or pid unavailable)
+
+	// lastActivity older than zombieMs → zombie
+	if (elapsedMs > thresholds.zombieMs) {
+		return {
+			...base,
+			processAlive: true,
+			state: "zombie",
+			action: "terminate",
+			reconciliationNote: null,
+		};
+	}
+
+	// booting → transition to working once there's recent activity
+	if (session.state === "booting") {
+		return {
+			...base,
+			processAlive: true,
+			state: "working",
+			action: "none",
+			reconciliationNote: null,
+		};
+	}
+
+	// Default: healthy and working
+	return {
+		...base,
+		processAlive: true,
+		state: "working",
+		action: "none",
+		reconciliationNote: null,
+	};
+}
+
+/**
+ * Compute the next agent state based on a health check.
+ *
+ * State transitions are strictly forward-only using the ordering:
+ *   booting(0) → working(1) → completed(2) → zombie(3)
+ *
+ * A state can only advance forward, never move backwards.
+ * For example, a zombie can never become working again.
+ *
+ * Exception (ZFC): When the health check action is "investigate", the state
+ * is NOT advanced. This allows a human or higher-tier agent to review the
+ * conflicting signals before making a state change.
+ *
+ * @param currentState - The agent's current state
+ * @param check - The latest health check result
+ * @returns The new state (always >= currentState in ordering)
+ */
+export function transitionState(currentState: AgentState, check: HealthCheck): AgentState {
+	// ZFC: investigate means signals conflict — hold state until reviewed
+	if (check.action === "investigate") {
+		return currentState;
+	}
+
+	const currentOrder = STATE_ORDER[currentState] ?? 0;
+	const checkOrder = STATE_ORDER[check.state] ?? 0;
+
+	// Only move forward — never regress
+	if (checkOrder > currentOrder) {
+		return check.state;
+	}
+
+	return currentState;
+}