@os-eco/overstory-cli 0.9.4 → 0.10.3

package/src/runtimes/claude.ts

@@ -9,7 +9,9 @@ import { estimateCost } from "../metrics/pricing.ts";
 import { parseTranscriptUsage } from "../metrics/transcript.ts";
 import type { ResolvedModel } from "../types.ts";
 import type {
+	AgentEvent,
 	AgentRuntime,
+	DirectSpawnOpts,
 	HooksDef,
 	OverlayContent,
 	ReadyState,
@@ -126,7 +128,22 @@ export class ClaudeRuntime implements AgentRuntime {
 			await Bun.write(claudeMdPath, overlay.content);
 		}
 
-		await deployHooks(hooks.worktreePath, hooks.agentName, hooks.capability, hooks.qualityGates);
+		// Always deploy hooks — headless Claude Code DOES dispatch settings.local.json
+		// PreToolUse hooks (verified empirically against `claude -p --output-format stream-json`).
+		// The original design (overstory-1c32 / docs/headless-hooks-design.md Q6) wrongly assumed
+		// hooks don't fire in headless mode and skipped deployment, leaving headless agents with
+		// none of the destructive-command guards. overstory-e24b reverses that: in headless mode
+		// we deploy a settings.local.json containing only PreToolUse security guards (path boundary,
+		// capability blocks, bash danger patterns, tracker close, lead close gate). The other
+		// hook types are dropped because they have headless equivalents already wired up
+		// (initial stdin prompt, serve mail injection loop, stream-json event capture).
+		await deployHooks(
+			hooks.worktreePath,
+			hooks.agentName,
+			hooks.capability,
+			hooks.qualityGates,
+			hooks.isHeadless ?? false,
+		);
 	}
 
 	/**
@@ -218,6 +235,311 @@ export class ClaudeRuntime implements AgentRuntime {
 		}
 	}
 
+	/**
+	 * Build the argv array for Bun.spawn() to launch a Claude Code agent in headless mode.
+	 *
+	 * Returns the exact flags Multica uses for headless Claude Code sessions:
+	 * `-p --output-format stream-json --input-format stream-json --verbose
+	 * --strict-mcp-config --permission-mode bypassPermissions [--model <m>]`
+	 *
+	 * Claude Code reads `.claude/CLAUDE.md` from cwd automatically — do NOT
+	 * pass `--append-system-prompt` or consume `opts.instructionPath`.
+	 * The initial stdin prompt is the caller's responsibility.
+	 *
+	 * @param opts - Direct spawn options; only `model` is consumed
+	 * @returns Argv array for Bun.spawn — do not shell-interpolate
+	 */
+	buildDirectSpawn(opts: DirectSpawnOpts): string[] {
+		const argv = [
+			"claude",
+			"-p",
+			"--output-format",
+			"stream-json",
+			"--input-format",
+			"stream-json",
+			"--verbose",
+			"--strict-mcp-config",
+			"--permission-mode",
+			"bypassPermissions",
+		];
+		if (opts.model !== undefined) {
+			argv.push("--model", opts.model);
+		}
+		// Phase 1 (overstory-b835): emit --resume on follow-up spawns. Mirrors
+		// multica/server/pkg/agent/claude.go:434 (positional). Empty string and
+		// null are treated as "no resume" — only non-empty strings activate it.
+		if (typeof opts.resumeSessionId === "string" && opts.resumeSessionId.length > 0) {
+			argv.push("--resume", opts.resumeSessionId);
+		}
+		return argv;
+	}
+
+	/**
+	 * Parse stream-json stdout from a Claude Code headless subprocess into typed AgentEvent objects.
+	 *
+	 * Reads the ReadableStream from Bun.spawn() stdout, buffers partial lines,
+	 * and yields a typed AgentEvent for each complete JSON line. Malformed lines
+	 * and unknown message types are silently skipped.
+	 *
+	 * Adjacent assistant_text deltas are coalesced into a single assistant_message
+	 * event using a batching window. The batch flushes on timer expiry, size cap,
+	 * any non-text event, or stream end — whichever comes first.
+	 *
+	 * Event mapping (Claude stream-json → AgentEvent):
+	 * - assistant/text     → buffered; emitted as one assistant_message per batch
+	 * - assistant/tool_use → { type: "tool_use", callId, name, input }
+	 * - assistant/thinking → (skipped)
+	 * - user/tool_result   → { type: "tool_result", toolUseId, content }
+	 * - system             → { type: "status", sessionId, subtype }
+	 * - result             → { type: "result", sessionId, result, isError, durationMs, numTurns }
+	 *
+	 * @param stream - ReadableStream<Uint8Array> from Bun.spawn stdout
+	 * @param opts - Optional hooks and tuning:
+	 *   - `onSessionId` is invoked once, synchronously, on the first event that carries a
+	 *     non-empty `sessionId`. Consumer errors are swallowed so they cannot crash the parser.
+	 *   - `flushIntervalMs` — Max ms to buffer text before emitting (default 500).
+	 *   - `flushSizeBytes` — Max UTF-8 byte size of a text batch (default 4096).
+	 * @yields Parsed AgentEvent objects in emission order
+	 */
+	async *parseEvents(
+		stream: ReadableStream<Uint8Array>,
+		opts?: {
+			onSessionId?: (sessionId: string) => void;
+			flushIntervalMs?: number;
+			flushSizeBytes?: number;
+		},
+	): AsyncIterable<AgentEvent> {
+		const flushIntervalMs = opts?.flushIntervalMs ?? 500;
+		const flushSizeBytes = opts?.flushSizeBytes ?? 4096;
+
+		const reader = stream.getReader();
+		const decoder = new TextDecoder();
+		let buffer = "";
+		let sessionIdPinned = false;
+
+		// Batch state for adjacent assistant_text deltas.
+		let pendingText: string[] = [];
+		let pendingByteSize = 0;
+		let pendingStartTs: string | null = null;
+		let pendingModel: string | undefined;
+		let pendingUsage: unknown;
+
+		// Returns batched assistant_message event and resets state; null when buffer is empty.
+		const flushText = (): AgentEvent | null => {
+			if (pendingText.length === 0) return null;
+			const text = pendingText.join("");
+			const event: AgentEvent = {
+				type: "assistant_message",
+				timestamp: pendingStartTs ?? new Date().toISOString(),
+				text,
+			};
+			if (pendingModel !== undefined) event.model = pendingModel;
+			if (pendingUsage !== undefined) event.usage = pendingUsage;
+			pendingText = [];
+			pendingByteSize = 0;
+			pendingStartTs = null;
+			pendingModel = undefined;
+			pendingUsage = undefined;
+			return event;
+		};
+
+		// Sync generator: parses one JSON line, yielding AgentEvents in order.
+		// Mutates batch state via closure — do not call concurrently.
+		function* processLine(line: string): Generator<AgentEvent> {
+			const trimmed = line.trim();
+			if (!trimmed) return;
+
+			let msg: Record<string, unknown>;
+			try {
+				msg = JSON.parse(trimmed) as Record<string, unknown>;
+			} catch {
+				return;
+			}
+
+			const timestamp = new Date().toISOString();
+
+			if (msg.type === "assistant") {
+				const message =
+					typeof msg.message === "object" && msg.message !== null
+						? (msg.message as Record<string, unknown>)
+						: null;
+				if (!message) return;
+				const content = Array.isArray(message.content) ? message.content : [];
+				const model = typeof message.model === "string" ? message.model : undefined;
+				const usage = message.usage !== undefined ? message.usage : undefined;
+
+				for (const block of content) {
+					if (typeof block !== "object" || block === null) continue;
+					const b = block as Record<string, unknown>;
+					if (b.type === "text") {
+						const text = typeof b.text === "string" ? b.text : String(b.text);
+						const textByteSize = new TextEncoder().encode(text).byteLength;
+
+						// Size-cap: if appending would exceed cap and buffer is non-empty, flush first.
+						if (pendingByteSize > 0 && pendingByteSize + textByteSize > flushSizeBytes) {
+							const ev = flushText();
+							if (ev) yield ev;
+						}
+
+						// Record the start-of-batch timestamp on the first fragment.
+						if (pendingByteSize === 0) pendingStartTs = timestamp;
+
+						pendingText.push(text);
+						pendingByteSize += textByteSize;
+						// Latest contributing message wins for model/usage.
+						if (model !== undefined) pendingModel = model;
+						if (usage !== undefined) pendingUsage = usage;
+
+						// Immediate flush when a single fragment meets or exceeds the size cap.
+						if (pendingByteSize >= flushSizeBytes) {
+							const ev = flushText();
+							if (ev) yield ev;
+						}
+					} else if (b.type === "tool_use") {
+						// Non-text block: flush pending text first to preserve in-order delivery.
+						const ev = flushText();
+						if (ev) yield ev;
+						yield {
+							type: "tool_use",
+							timestamp,
+							callId: b.id,
+							name: b.name,
+							input: b.input,
+						};
+					}
+					// thinking and other block types → skip
+				}
+			} else if (msg.type === "user") {
+				const message =
+					typeof msg.message === "object" && msg.message !== null
+						? (msg.message as Record<string, unknown>)
+						: null;
+				if (!message) return;
+				const content = Array.isArray(message.content) ? message.content : [];
+
+				const flushEv = flushText();
+				if (flushEv) yield flushEv;
+
+				for (const block of content) {
+					if (typeof block !== "object" || block === null) continue;
+					const b = block as Record<string, unknown>;
+					if (b.type === "tool_result") {
+						yield {
+							type: "tool_result",
+							timestamp,
+							toolUseId: b.tool_use_id,
+							content: b.content,
+						};
+					}
+				}
+			} else if (msg.type === "system") {
+				const flushEv = flushText();
+				if (flushEv) yield flushEv;
+				yield { type: "status", timestamp, sessionId: msg.session_id, subtype: msg.subtype };
+			} else if (msg.type === "result") {
+				const flushEv = flushText();
+				if (flushEv) yield flushEv;
+				yield {
+					type: "result",
+					timestamp,
+					sessionId: msg.session_id,
+					result: msg.result,
+					isError: msg.is_error,
+					durationMs: msg.duration_ms,
+					numTurns: msg.num_turns,
+				};
+			}
+		}
+
+		const maybePinSession = (event: AgentEvent): void => {
+			if (sessionIdPinned || !opts?.onSessionId) return;
+			const sid = event.sessionId;
+			if (typeof sid !== "string" || sid.length === 0) return;
+			sessionIdPinned = true;
+			try {
+				opts.onSessionId(sid);
+			} catch {
+				// Consumer errors must not crash the parser.
+			}
+		};
+
+		try {
+			// Use the inferred return type of reader.read() to stay compatible with
+			// both the standard Web Streams API and Bun's slightly divergent typings.
+			type ReadResult = Awaited<ReturnType<typeof reader.read>>;
+			type Race = { kind: "read"; result: ReadResult } | { kind: "timeout" };
+
+			let readPromise = reader.read();
+
+			while (true) {
+				if (pendingText.length > 0 && pendingStartTs !== null) {
+					// Race the next read chunk against the flush timer.
+					const elapsed = Date.now() - new Date(pendingStartTs).getTime();
+					const remaining = Math.max(0, flushIntervalMs - elapsed);
+
+					let timerId: ReturnType<typeof setTimeout> | undefined;
+					const wrappedRead: Promise<Race> = readPromise.then((result) => ({
+						kind: "read" as const,
+						result,
+					}));
+					const wrappedTimer = new Promise<Race>((resolve) => {
+						timerId = setTimeout(() => resolve({ kind: "timeout" }), remaining);
+					});
+
+					const winner = await Promise.race([wrappedRead, wrappedTimer]);
+					if (timerId !== undefined) clearTimeout(timerId);
+
+					if (winner.kind === "timeout") {
+						const ev = flushText();
+						if (ev) yield ev;
+						continue; // readPromise still in flight — re-enter loop without refreshing
+					}
+
+					const result = winner.result;
+					if (result.done) break;
+					buffer += decoder.decode(result.value, { stream: true });
+					const lines = buffer.split("\n");
+					buffer = lines.pop() ?? "";
+					for (const line of lines) {
+						for (const event of processLine(line)) {
+							maybePinSession(event);
+							yield event;
+						}
+					}
+					readPromise = reader.read();
+				} else {
+					const result = await readPromise;
+					if (result.done) break;
+					buffer += decoder.decode(result.value, { stream: true });
+					const lines = buffer.split("\n");
+					// Last element is either empty or an incomplete line — keep in buffer.
+					buffer = lines.pop() ?? "";
+					for (const line of lines) {
+						for (const event of processLine(line)) {
+							maybePinSession(event);
+							yield event;
+						}
+					}
+					readPromise = reader.read();
+				}
+			}
+
+			// Flush remaining buffer on clean stream end (no trailing newline).
+			if (buffer.trim()) {
+				for (const event of processLine(buffer)) {
+					maybePinSession(event);
+					yield event;
+				}
+			}
+
+			// Drain any remaining buffered text after the read loop exits.
+			const finalEv = flushText();
+			if (finalEv) yield finalEv;
+		} finally {
+			reader.releaseLock();
+		}
+	}
+
 	/**
 	 * Build runtime-specific environment variables for model/provider routing.
 	 *

package/src/runtimes/connections.test.ts

@@ -1,5 +1,11 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { getConnection, removeConnection, setConnection } from "./connections.ts";
+import {
+	addHeadlessConnectionListener,
+	getConnection,
+	registerHeadlessConnection,
+	removeConnection,
+	setConnection,
+} from "./connections.ts";
 import type { ConnectionState, RuntimeConnection } from "./types.ts";
 
 /** Minimal RuntimeConnection stub for testing the registry. */
@@ -72,3 +78,137 @@ describe("connection registry", () => {
 		expect(getConnection("agent-delta")).toBe(conn2);
 	});
 });
+
+describe("headless connection listener API", () => {
+	const usedNames: string[] = [];
+	const unsubscribers: Array<() => void> = [];
+
+	afterEach(() => {
+		for (const unsub of unsubscribers.splice(0)) unsub();
+		for (const name of usedNames.splice(0)) {
+			if (getConnection(name)) {
+				removeConnection(name);
+			}
+		}
+	});
+
+	function makeStdin(): { write(): Promise<number> } {
+		return { write: () => Promise.resolve(0) };
+	}
+
+	test("onRegister fires when registerHeadlessConnection runs", () => {
+		const seen: Array<{ name: string }> = [];
+		unsubscribers.push(
+			addHeadlessConnectionListener({
+				onRegister(agentName) {
+					seen.push({ name: agentName });
+				},
+			}),
+		);
+
+		usedNames.push("listener-agent-1");
+		registerHeadlessConnection("listener-agent-1", { pid: 99999, stdin: makeStdin() });
+
+		expect(seen).toEqual([{ name: "listener-agent-1" }]);
+	});
+
+	test("onRemove fires when a headless connection is removed", () => {
+		const removed: string[] = [];
+		unsubscribers.push(
+			addHeadlessConnectionListener({
+				onRegister: () => {},
+				onRemove(agentName) {
+					removed.push(agentName);
+				},
+			}),
+		);
+
+		usedNames.push("listener-agent-2");
+		registerHeadlessConnection("listener-agent-2", { pid: 99999, stdin: makeStdin() });
+		removeConnection("listener-agent-2");
+
+		expect(removed).toEqual(["listener-agent-2"]);
+	});
+
+	test("onRemove does NOT fire for non-headless connections", () => {
+		const removed: string[] = [];
+		unsubscribers.push(
+			addHeadlessConnectionListener({
+				onRegister: () => {},
+				onRemove(agentName) {
+					removed.push(agentName);
+				},
+			}),
+		);
+
+		const conn: RuntimeConnection = {
+			sendPrompt: async () => {},
+			followUp: async () => {},
+			abort: async () => {},
+			getState: async (): Promise<ConnectionState> => ({ status: "idle" }),
+			close: () => {},
+		};
+		usedNames.push("non-headless-agent");
+		setConnection("non-headless-agent", conn);
+		removeConnection("non-headless-agent");
+
+		expect(removed).toEqual([]);
+	});
+
+	test("listener added after registration sees existing agents via catch-up", () => {
+		usedNames.push("preexisting-agent");
+		registerHeadlessConnection("preexisting-agent", { pid: 99999, stdin: makeStdin() });
+
+		const seen: string[] = [];
+		unsubscribers.push(
+			addHeadlessConnectionListener({
+				onRegister(agentName) {
+					seen.push(agentName);
+				},
+			}),
+		);
+
+		expect(seen).toContain("preexisting-agent");
+	});
+
+	test("unsubscribe stops further notifications", () => {
+		const seen: string[] = [];
+		const unsub = addHeadlessConnectionListener({
+			onRegister(agentName) {
+				seen.push(agentName);
+			},
+		});
+		unsub();
+
+		usedNames.push("after-unsub-agent");
+		registerHeadlessConnection("after-unsub-agent", { pid: 99999, stdin: makeStdin() });
+
+		expect(seen).toEqual([]);
+	});
+
+	test("onRegister receives the stdin handle from the spawned process", async () => {
+		const writes: string[] = [];
+		const stdin = {
+			write(data: string | Uint8Array) {
+				writes.push(typeof data === "string" ? data : new TextDecoder().decode(data));
+				return Promise.resolve(0);
+			},
+		};
+
+		const observed: Array<{ write(data: string | Uint8Array): number | Promise<number> }> = [];
+		unsubscribers.push(
+			addHeadlessConnectionListener({
+				onRegister(_agentName, s) {
+					observed.push(s);
+				},
+			}),
+		);
+
+		usedNames.push("stdin-pass-agent");
+		registerHeadlessConnection("stdin-pass-agent", { pid: 99999, stdin });
+		expect(observed.length).toBe(1);
+		expect(observed[0]).toBe(stdin);
+		await observed[0]?.write("hello");
+		expect(writes).toEqual(["hello"]);
+	});
+});

package/src/runtimes/connections.ts

@@ -1,15 +1,32 @@
 /**
  * Module-level connection registry for active RuntimeConnection instances.
  *
- * Tracks RPC connections to headless agent processes (e.g., Sapling).
+ * Tracks RPC connections to headless agent processes (e.g., Sapling, headless Claude).
  * Keyed by agent name — same namespace as AgentSession.agentName.
  *
  * Thread safety: single-threaded Bun runtime; no locking needed.
  */
 
+import { HeadlessClaudeConnection } from "./headless-connection.ts";
 import type { RuntimeConnection } from "./types.ts";
 
+/** Writable handle exposed to headless connection listeners. */
+export type HeadlessStdin = { write(data: string | Uint8Array): number | Promise<number> };
+
+/**
+ * Listener fired on headless Claude connection lifecycle events.
+ * Fires only for connections created via registerHeadlessConnection() — Sapling
+ * and other RuntimeConnection registrants via setConnection() are not surfaced
+ * because their wire format differs from headless Claude's stream-json stdin.
+ */
+export interface HeadlessConnectionListener {
+	onRegister(agentName: string, stdin: HeadlessStdin): void;
+	onRemove?(agentName: string): void;
+}
+
 const connections = new Map<string, RuntimeConnection>();
+const headlessAgents = new Map<string, HeadlessStdin>();
+const listeners = new Set<HeadlessConnectionListener>();
 
 /** Retrieve the active connection for a given agent, or undefined if none. */
 export function getConnection(agentName: string): RuntimeConnection | undefined {
@@ -27,8 +44,60 @@ export function setConnection(agentName: string, conn: RuntimeConnection): void
  */
 export function removeConnection(agentName: string): void {
 	const conn = connections.get(agentName);
-	if (conn) {
-		conn.close();
-		connections.delete(agentName);
+	if (!conn) return;
+	if (headlessAgents.delete(agentName)) {
+		for (const listener of listeners) {
+			listener.onRemove?.(agentName);
+		}
+	}
+	conn.close();
+	connections.delete(agentName);
+}
+
+/**
+ * Create a HeadlessClaudeConnection from a spawned process handle and register it.
+ *
+ * Called by spawnHeadlessAgent() (process.ts) when an agentName is provided.
+ * The registered connection is retrievable via getConnection(agentName) for
+ * follow-up delivery, state polling, and abort — all without tmux.
+ *
+ * This is the sibling registration path to Sapling's connect() flow. It does NOT
+ * generalize RpcProcessHandle and does NOT touch other runtime adapters.
+ *
+ * @param agentName - Unique agent identifier (same namespace as AgentSession.agentName)
+ * @param proc - Spawned headless process with pid and stdin
+ * @returns The newly created and registered RuntimeConnection
+ */
+export function registerHeadlessConnection(
+	agentName: string,
+	proc: { pid: number; stdin: HeadlessStdin },
+): RuntimeConnection {
+	const conn = new HeadlessClaudeConnection(proc.pid, proc.stdin);
+	setConnection(agentName, conn);
+	headlessAgents.set(agentName, proc.stdin);
+	for (const listener of listeners) {
+		listener.onRegister(agentName, proc.stdin);
+	}
+	return conn;
+}
+
+/**
+ * Subscribe to headless Claude connection lifecycle events.
+ *
+ * onRegister fires synchronously after registerHeadlessConnection() inserts the
+ * connection. onRemove fires synchronously before removeConnection() closes the
+ * connection. Listeners observing already-registered agents at subscribe time
+ * receive an immediate onRegister for each — this lets late subscribers (e.g.,
+ * runServe started after agents already exist) catch up without rescanning.
+ *
+ * @returns Unsubscribe function that removes this listener.
+ */
+export function addHeadlessConnectionListener(listener: HeadlessConnectionListener): () => void {
+	listeners.add(listener);
+	for (const [agentName, stdin] of headlessAgents) {
+		listener.onRegister(agentName, stdin);
 	}
+	return () => {
+		listeners.delete(listener);
+	};
 }