@os-eco/overstory-cli 0.8.0 → 0.8.2

package/src/runtimes/types.ts

@@ -110,6 +110,30 @@ export interface RuntimeConnection {
 	close(): void;
 }
 
+// === Headless Spawn ===
+
+/** Options for spawning a headless (non-tmux) agent subprocess directly. */
+export interface DirectSpawnOpts {
+	/** Working directory for the spawned process. */
+	cwd: string;
+	/** Environment variables for the subprocess. */
+	env: Record<string, string>;
+	/** Model ref (alias or provider-qualified). */
+	model: string;
+	/** Path to the instruction/overlay file for this agent. */
+	instructionPath: string;
+}
+
+/** Structured event emitted by a headless agent on stdout (NDJSON). */
+export interface AgentEvent {
+	/** Event type (e.g. 'tool_start', 'tool_end', 'result', 'error', 'ready'). */
+	type: string;
+	/** ISO 8601 timestamp. */
+	timestamp: string;
+	/** Event-specific payload. */
+	[key: string]: unknown;
+}
+
 // === Runtime Interface ===
 
 /**
@@ -187,4 +211,25 @@ export interface AgentRuntime {
 	 * Orchestrator checks `if (runtime.connect)` before calling, falls back to tmux when absent.
 	 */
 	connect?(process: RpcProcessHandle): RuntimeConnection;
+
+	/**
+	 * Whether this runtime is headless (no tmux, direct subprocess).
+	 * Headless runtimes bypass all tmux session management and use Bun.spawn directly.
+	 * Default: false (absent means interactive/tmux-based).
+	 */
+	readonly headless?: boolean;
+
+	/**
+	 * Build the argv array for Bun.spawn() to launch a headless agent subprocess.
+	 * Only headless runtimes implement this method.
+	 * The returned array is passed directly to Bun.spawn() — no shell interpolation.
+	 */
+	buildDirectSpawn?(opts: DirectSpawnOpts): string[];
+
+	/**
+	 * Parse NDJSON stdout from a headless agent subprocess into typed AgentEvent objects.
+	 * Only headless runtimes implement this method.
+	 * The caller provides the raw stdout ReadableStream from Bun.spawn().
+	 */
+	parseEvents?(stream: ReadableStream<Uint8Array>): AsyncIterable<AgentEvent>;
 }

package/src/types.ts

@@ -477,7 +477,11 @@ export type EventType =
 	| "mail_received"
 	| "spawn"
 	| "error"
-	| "custom";
+	| "custom"
+	| "turn_start"
+	| "turn_end"
+	| "progress"
+	| "result";
 
 /** Severity levels for events. */
 export type EventLevel = "debug" | "info" | "warn" | "error";

package/src/watchdog/daemon.ts

@@ -24,6 +24,8 @@ import { join } from "node:path";
 import { nudgeAgent } from "../commands/nudge.ts";
 import { createEventStore } from "../events/store.ts";
 import { createMulchClient } from "../mulch/client.ts";
+import { getConnection, removeConnection } from "../runtimes/connections.ts";
+import type { RuntimeConnection } from "../runtimes/types.ts";
 import { openSessionStore } from "../sessions/compat.ts";
 import type { AgentSession, EventStore, HealthCheck } from "../types.ts";
 import { isSessionAlive, killSession } from "../worktree/tmux.ts";
@@ -282,6 +284,10 @@ export interface DaemonOptions {
 		tier: 0 | 1,
 		triageSuggestion?: string,
 	) => Promise<void>;
+	/** Dependency injection for testing. Uses real getConnection when omitted. */
+	_getConnection?: (name: string) => RuntimeConnection | undefined;
+	/** Dependency injection for testing. Uses real removeConnection when omitted. */
+	_removeConnection?: (name: string) => void;
 }
 
 /**
@@ -345,6 +351,8 @@ export async function runDaemonTick(options: DaemonOptions): Promise<void> {
 	const triage = options._triage ?? triageAgent;
 	const nudge = options._nudge ?? nudgeAgent;
 	const recordFailureFn = options._recordFailure ?? recordFailure;
+	const getConn = options._getConnection ?? getConnection;
+	const removeConn = options._removeConnection ?? removeConnection;
 
 	const overstoryDir = join(root, ".overstory");
 	const { store } = openSessionStore(overstoryDir);
@@ -386,6 +394,32 @@ export async function runDaemonTick(options: DaemonOptions): Promise<void> {
 			// ZFC: Don't skip zombies. Re-check tmux liveness on every tick.
 			// A zombie with a live tmux session needs investigation, not silence.
 
+			// RPC health check: for headless agents with an active connection,
+			// call getState() to refresh lastActivity before evaluateHealth().
+			// This prevents false-positive stale/zombie classification for agents
+			// that are actively working but haven't updated lastActivity via hooks.
+			if (session.tmuxSession === "" && session.pid !== null) {
+				const conn = getConn(session.agentName);
+				if (conn) {
+					try {
+						const state = await Promise.race([
+							conn.getState(),
+							new Promise<never>((_, reject) =>
+								setTimeout(() => reject(new Error("getState timed out")), 5000),
+							),
+						]);
+						if (state.status === "idle" || state.status === "working") {
+							store.updateLastActivity(session.agentName);
+							// Refresh the session object so evaluateHealth sees updated lastActivity
+							session.lastActivity = new Date().toISOString();
+						}
+					} catch {
+						// getState() failed or timed out — remove stale connection
+						removeConn(session.agentName);
+					}
+				}
+			}
+
 			const tmuxAlive = await tmux.isSessionAlive(session.tmuxSession);
 			const check = evaluateHealth(session, tmuxAlive, thresholds);
 

package/src/watchdog/health.test.ts

@@ -316,6 +316,108 @@ describe("evaluateHealth", () => {
 	});
 });
 
+// === Headless agents (tmuxSession === '', PID-based lifecycle) ===
+
+describe("headless agents (tmuxSession empty, PID-based lifecycle)", () => {
+	// Headless agents always have tmuxAlive=false passed by the caller (no tmux).
+	// PID is the primary liveness signal.
+
+	test("headless agent with alive PID → working (NOT zombie)", () => {
+		const session = makeSession({ tmuxSession: "", pid: ALIVE_PID, state: "working" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+		expect(check.processAlive).toBe(true);
+		expect(check.pidAlive).toBe(true);
+		// tmuxAlive is always false for headless
+		expect(check.tmuxAlive).toBe(false);
+	});
+
+	test("headless agent with dead PID → zombie, terminate", () => {
+		const session = makeSession({ tmuxSession: "", pid: DEAD_PID, state: "working" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+		expect(check.processAlive).toBe(false);
+		expect(check.pidAlive).toBe(false);
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain("headless");
+		expect(check.reconciliationNote).toContain("dead");
+	});
+
+	test("headless agent with alive PID + state=zombie → investigate (don't auto-kill)", () => {
+		const session = makeSession({ tmuxSession: "", pid: ALIVE_PID, state: "zombie" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("investigate");
+		expect(check.processAlive).toBe(true);
+		expect(check.reconciliationNote).toContain("ZFC");
+		expect(check.reconciliationNote).toContain("don't auto-kill");
+	});
+
+	test("headless booting agent with alive PID → transitions to working", () => {
+		const session = makeSession({ tmuxSession: "", pid: ALIVE_PID, state: "booting" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("headless agent with stale activity → stalled", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			tmuxSession: "",
+			pid: ALIVE_PID,
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("stalled");
+		expect(check.action).toBe("escalate");
+	});
+
+	test("headless agent with zombie-level staleness → zombie", () => {
+		const oldActivity = new Date(Date.now() - 200_000).toISOString();
+		const session = makeSession({
+			tmuxSession: "",
+			pid: ALIVE_PID,
+			state: "working",
+			lastActivity: oldActivity,
+		});
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("zombie");
+		expect(check.action).toBe("terminate");
+	});
+
+	test("headless persistent capability (coordinator) with stale activity → still working", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			tmuxSession: "",
+			pid: ALIVE_PID,
+			capability: "coordinator",
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
+	test("headless completed agent → skips monitoring", () => {
+		const session = makeSession({ tmuxSession: "", pid: ALIVE_PID, state: "completed" });
+		const check = evaluateHealth(session, false, THRESHOLDS);
+
+		expect(check.state).toBe("completed");
+		expect(check.action).toBe("none");
+	});
+});
+
 // === transitionState ===
 
 describe("transitionState", () => {

package/src/watchdog/health.ts

@@ -20,6 +20,11 @@
  *   - pid alive + tmux dead → should not happen (tmux owns the pid), but if it
  *     does, trust tmux (the session is gone).
  *
+ * Headless agents (tmuxSession === ''):
+ *   Headless agents have no tmux session. For these, PID is the PRIMARY liveness
+ *   signal. The tmuxAlive parameter is meaningless and ignored. ZFC rules are
+ *   applied using PID liveness instead of tmux liveness.
+ *
  * The rationale: sessions.json is updated asynchronously by hooks and can become
  * stale if the agent crashes between hook invocations. tmux and the OS process
  * table are always up-to-date because they reflect real kernel state.
@@ -65,6 +70,92 @@ export function isProcessRunning(pid: number): boolean {
 	}
 }
 
+/**
+ * Detect whether a session is a headless agent.
+ *
+ * Headless agents are spawned without a tmux session (tmuxSession === '') and
+ * are tracked solely by PID. For these agents, PID is the primary liveness signal.
+ */
+function isHeadlessSession(session: AgentSession): boolean {
+	return session.tmuxSession === "" && session.pid !== null;
+}
+
+/**
+ * Evaluate time-based health (persistent capability exemptions, stale, zombie thresholds,
+ * booting→working transition). Called after liveness is confirmed for both TUI and headless paths.
+ *
+ * Assumes that by the time this is called:
+ * - The agent is not completed
+ * - The agent is not in a liveness-based zombie state
+ * - The agent is not in a zombie state that needs investigation
+ */
+function evaluateTimeBased(
+	session: AgentSession,
+	base: Pick<HealthCheck, "agentName" | "timestamp" | "tmuxAlive" | "pidAlive" | "lastActivity">,
+	elapsedMs: number,
+	thresholds: { staleMs: number; zombieMs: number },
+): HealthCheck {
+	// Persistent capabilities (coordinator, monitor) are expected to have long idle
+	// periods waiting for mail/events. Skip time-based stale/zombie detection for
+	// them — only tmux/pid liveness matters (checked above).
+	if (PERSISTENT_CAPABILITIES.has(session.capability)) {
+		// Transition booting → working if we reach here (process alive)
+		const state = session.state === "booting" ? "working" : session.state;
+		return {
+			...base,
+			processAlive: true,
+			state: state === "stalled" ? "working" : state,
+			action: "none",
+			reconciliationNote:
+				session.state === "stalled"
+					? `Persistent capability "${session.capability}" exempted from stale detection — resetting to working`
+					: null,
+		};
+	}
+
+	// lastActivity older than zombieMs → zombie
+	if (elapsedMs > thresholds.zombieMs) {
+		return {
+			...base,
+			processAlive: true,
+			state: "zombie",
+			action: "terminate",
+			reconciliationNote: null,
+		};
+	}
+
+	// lastActivity older than staleMs → stalled
+	if (elapsedMs > thresholds.staleMs) {
+		return {
+			...base,
+			processAlive: true,
+			state: "stalled",
+			action: "escalate",
+			reconciliationNote: null,
+		};
+	}
+
+	// booting → transition to working once there's recent activity
+	if (session.state === "booting") {
+		return {
+			...base,
+			processAlive: true,
+			state: "working",
+			action: "none",
+			reconciliationNote: null,
+		};
+	}
+
+	// Default: healthy and working
+	return {
+		...base,
+		processAlive: true,
+		state: "working",
+		action: "none",
+		reconciliationNote: null,
+	};
+}
+
 /**
  * Evaluate the health of an agent session.
  *
@@ -74,18 +165,23 @@ export function isProcessRunning(pid: number): boolean {
  * Decision logic (in priority order):
  *
  * 1. Completed agents skip monitoring entirely.
- * 2. tmux dead → zombie, terminate (regardless of what sessions.json says).
- * 3. tmux alive + sessions.json says zombie → investigate (don't auto-kill).
+ * 2. Headless agents (tmuxSession === ''): PID is primary liveness signal.
+ *    - pid dead → zombie, terminate.
+ *    - pid alive + state zombie → investigate.
+ *    - pid alive → fall through to time-based checks.
+ * 3. tmux dead → zombie, terminate (regardless of what sessions.json says).
+ * 4. tmux alive + sessions.json says zombie → investigate (don't auto-kill).
  *    Something external marked this zombie, but the process is still running.
- * 4. pid dead + tmux alive → zombie, terminate. The agent process exited but
+ * 5. pid dead + tmux alive → zombie, terminate. The agent process exited but
  *    the tmux pane shell survived. The agent is not doing work.
- * 5. lastActivity older than zombieMs → zombie, terminate.
- * 6. lastActivity older than staleMs → stalled, escalate.
- * 7. booting with recent activity → working.
- * 8. Otherwise → working, healthy.
+ * 6. lastActivity older than zombieMs → zombie, terminate.
+ * 7. lastActivity older than staleMs → stalled, escalate.
+ * 8. booting with recent activity → working.
+ * 9. Otherwise → working, healthy.
  *
  * @param session - The agent session to evaluate
  * @param tmuxAlive - Whether the agent's tmux session is still running
+ *                    (ignored for headless agents where tmuxSession === '')
  * @param thresholds - Staleness and zombie time thresholds in milliseconds
  * @returns A HealthCheck describing the agent's current state and recommended action
  */
@@ -101,13 +197,16 @@ export function evaluateHealth(
 	// Check pid liveness as secondary signal (null if pid unavailable)
 	const pidAlive = session.pid !== null ? isProcessRunning(session.pid) : null;
 
+	// Headless agents have no tmux session; tmuxAlive is always false for them.
+	const effectiveTmuxAlive = isHeadlessSession(session) ? false : tmuxAlive;
+
 	const base: Pick<
 		HealthCheck,
 		"agentName" | "timestamp" | "tmuxAlive" | "pidAlive" | "lastActivity"
 	> = {
 		agentName: session.agentName,
 		timestamp: now.toISOString(),
-		tmuxAlive,
+		tmuxAlive: effectiveTmuxAlive,
 		pidAlive,
 		lastActivity: session.lastActivity,
 	};
@@ -116,13 +215,44 @@ export function evaluateHealth(
 	if (session.state === "completed") {
 		return {
 			...base,
-			processAlive: tmuxAlive,
+			processAlive: effectiveTmuxAlive,
 			state: "completed",
 			action: "none",
 			reconciliationNote: null,
 		};
 	}
 
+	// === Headless path: PID is the primary liveness signal ===
+	if (isHeadlessSession(session)) {
+		// pid dead → zombie immediately (equivalent to ZFC Rule 1 for headless)
+		if (pidAlive === false) {
+			return {
+				...base,
+				processAlive: false,
+				state: "zombie",
+				action: "terminate",
+				reconciliationNote: `ZFC: headless agent pid ${session.pid} dead — marking zombie`,
+			};
+		}
+
+		// pid alive + state zombie → investigate (equivalent to ZFC Rule 2 for headless)
+		if (session.state === "zombie") {
+			return {
+				...base,
+				processAlive: true,
+				state: "zombie",
+				action: "investigate",
+				reconciliationNote:
+					"ZFC: headless pid alive but sessions.json says zombie — investigation needed (don't auto-kill)",
+			};
+		}
+
+		// pid alive → fall through to time-based checks
+		return evaluateTimeBased(session, base, elapsedMs, thresholds);
+	}
+
+	// === TUI/tmux path ===
+
 	// ZFC Rule 1: tmux dead → zombie immediately, regardless of recorded state.
 	// Observable state says the process is gone.
 	if (!tmuxAlive) {
@@ -166,67 +296,8 @@ export function evaluateHealth(
 		};
 	}
 
-	// Persistent capabilities (coordinator, monitor) are expected to have long idle
-	// periods waiting for mail/events. Skip time-based stale/zombie detection for
-	// them — only tmux/pid liveness matters (checked above).
-	if (PERSISTENT_CAPABILITIES.has(session.capability)) {
-		// Transition booting → working if we reach here (tmux alive, pid alive)
-		const state = session.state === "booting" ? "working" : session.state;
-		return {
-			...base,
-			processAlive: true,
-			state: state === "stalled" ? "working" : state,
-			action: "none",
-			reconciliationNote:
-				session.state === "stalled"
-					? `Persistent capability "${session.capability}" exempted from stale detection — resetting to working`
-					: null,
-		};
-	}
-
 	// Time-based checks (both tmux and pid confirmed alive, or pid unavailable)
-
-	// lastActivity older than zombieMs → zombie
-	if (elapsedMs > thresholds.zombieMs) {
-		return {
-			...base,
-			processAlive: true,
-			state: "zombie",
-			action: "terminate",
-			reconciliationNote: null,
-		};
-	}
-
-	// lastActivity older than staleMs → stalled
-	if (elapsedMs > thresholds.staleMs) {
-		return {
-			...base,
-			processAlive: true,
-			state: "stalled",
-			action: "escalate",
-			reconciliationNote: null,
-		};
-	}
-
-	// booting → transition to working once there's recent activity
-	if (session.state === "booting") {
-		return {
-			...base,
-			processAlive: true,
-			state: "working",
-			action: "none",
-			reconciliationNote: null,
-		};
-	}
-
-	// Default: healthy and working
-	return {
-		...base,
-		processAlive: true,
-		state: "working",
-		action: "none",
-		reconciliationNote: null,
-	};
+	return evaluateTimeBased(session, base, elapsedMs, thresholds);
 }
 
 /**

package/src/worktree/process.test.ts

@@ -0,0 +1,101 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnHeadlessAgent } from "./process.ts";
+
+describe("spawnHeadlessAgent", () => {
+	it("spawns a command and returns a valid PID", async () => {
+		const proc = await spawnHeadlessAgent(["echo", "hello"], {
+			cwd: process.cwd(),
+			env: { ...(process.env as Record<string, string>) },
+		});
+		expect(typeof proc.pid).toBe("number");
+		expect(proc.pid).toBeGreaterThan(0);
+		expect(proc.stdout).toBeDefined();
+		expect(proc.stdin).toBeDefined();
+	});
+
+	it("throws AgentError when argv is empty", async () => {
+		await expect(spawnHeadlessAgent([], { cwd: process.cwd(), env: {} })).rejects.toThrow(
+			"empty argv",
+		);
+	});
+
+	describe("file redirect mode", () => {
+		let tmpDir: string;
+
+		beforeEach(async () => {
+			tmpDir = await mkdtemp(join(tmpdir(), "ov-process-test-"));
+		});
+
+		afterEach(async () => {
+			await rm(tmpDir, { recursive: true, force: true });
+		});
+
+		it("redirects stdout to file when stdoutFile is provided", async () => {
+			const stdoutFile = join(tmpDir, "stdout.log");
+			const proc = await spawnHeadlessAgent(["echo", "hello from file"], {
+				cwd: process.cwd(),
+				env: { ...(process.env as Record<string, string>) },
+				stdoutFile,
+			});
+
+			expect(typeof proc.pid).toBe("number");
+			expect(proc.pid).toBeGreaterThan(0);
+			// stdout is null when redirected to file — no pipe, no backpressure
+			expect(proc.stdout).toBeNull();
+			expect(proc.stdin).toBeDefined();
+
+			// Wait for process to finish, then check file content
+			const exitProc = Bun.spawn(["sh", "-c", "true"], { stdout: "pipe" });
+			await exitProc.exited;
+			// Give echo a moment to flush
+			await Bun.sleep(100);
+
+			const content = await Bun.file(stdoutFile).text();
+			expect(content.trim()).toBe("hello from file");
+		});
+
+		it("redirects stderr to file when stderrFile is provided", async () => {
+			const stderrFile = join(tmpDir, "stderr.log");
+			// Write to stderr via sh -c
+			const proc = await spawnHeadlessAgent(["sh", "-c", "echo error output >&2"], {
+				cwd: process.cwd(),
+				env: { ...(process.env as Record<string, string>) },
+				stderrFile,
+			});
+
+			expect(typeof proc.pid).toBe("number");
+			// stdout still piped (no stdoutFile provided)
+			expect(proc.stdout).not.toBeNull();
+
+			// Drain stdout to let process exit cleanly
+			if (proc.stdout) {
+				const reader = proc.stdout.getReader();
+				while (!(await reader.read()).done) {
+					// drain
+				}
+				reader.releaseLock();
+			}
+
+			await Bun.sleep(100);
+			const content = await Bun.file(stderrFile).text();
+			expect(content.trim()).toBe("error output");
+		});
+
+		it("stdout remains a ReadableStream when no stdoutFile provided (default mode)", async () => {
+			const proc = await spawnHeadlessAgent(["echo", "piped"], {
+				cwd: process.cwd(),
+				env: { ...(process.env as Record<string, string>) },
+			});
+
+			expect(proc.stdout).not.toBeNull();
+			expect(proc.stdout).toBeInstanceOf(ReadableStream);
+
+			// Read the content via the stream
+			const text = await new Response(proc.stdout as ReadableStream<Uint8Array>).text();
+			expect(text.trim()).toBe("piped");
+		});
+	});
+});