@os-eco/overstory-cli 0.9.3 → 0.10.3

package/src/agents/turn-lock.ts

@@ -0,0 +1,235 @@
+/**
+ * Per-agent serialization lock for the spawn-per-turn engine.
+ *
+ * Ensures that two `runTurn` calls for the same agent never overlap. The lock
+ * is layered:
+ *
+ * 1. **In-process layer** — a module-level `Map<agentName, Promise<void>>`
+ *    chained via `.then`. Concurrent calls inside the same Bun process queue up.
+ *
+ * 2. **Cross-process layer** — a SQLite-backed lease at
+ *    `{overstoryDir}/turn-locks.db`. Each agent has at most one row in
+ *    `turn_locks(agent_name, held_by_pid, acquired_at)`. Acquire wraps the
+ *    state change in `BEGIN IMMEDIATE` so two processes cannot claim the same
+ *    row in the same instant. The transaction itself is short — it does not
+ *    span the whole turn — so other agents' acquires are not blocked.
+ *
+ * Stale leases (where `held_by_pid` is no longer alive) are stolen on the
+ * next acquire attempt. Both layers are released when `release()` is called.
+ */
+
+import { Database } from "bun:sqlite";
+import { join } from "node:path";
+
+/** In-process serialization tail per agent. Holds the latest queued promise. */
+const inProcessTails = new Map<string, Promise<void>>();
+
+export interface TurnLockHandle {
+	readonly agentName: string;
+	/** Release both layers. Idempotent. */
+	release(): void;
+}
+
+export interface AcquireTurnLockOpts {
+	agentName: string;
+	overstoryDir: string;
+	/** Process id recorded as the holder. Defaults to `process.pid`. */
+	ownerPid?: number;
+	/** Maximum time to wait for cross-process acquisition, in ms. Default 60_000. */
+	timeoutMs?: number;
+	/** Polling interval between cross-process retries when contended. Default 50ms. */
+	pollMs?: number;
+	/** Test injection: time source. */
+	_now?: () => number;
+	/** Test injection: liveness check (default uses `process.kill(pid, 0)`). */
+	_isProcessAlive?: (pid: number) => boolean;
+	/** Test injection: explicit DB path (overrides `{overstoryDir}/turn-locks.db`). */
+	_dbPath?: string;
+}
+
+const CREATE_TABLE = `
+CREATE TABLE IF NOT EXISTS turn_locks (
+  agent_name TEXT PRIMARY KEY,
+  held_by_pid INTEGER,
+  acquired_at TEXT
+)`;
+
+/** Default cross-process database path. */
+export function turnLockDbPath(overstoryDir: string): string {
+	return join(overstoryDir, "turn-locks.db");
+}
+
+function defaultIsProcessAlive(pid: number): boolean {
+	if (!Number.isFinite(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (err) {
+		// EPERM means the process exists but we lack permission to signal it.
+		// Treat as alive so we don't steal an active lock.
+		const code = (err as NodeJS.ErrnoException).code;
+		return code === "EPERM";
+	}
+}
+
+function openDb(path: string): Database {
+	const db = new Database(path);
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec("PRAGMA synchronous = NORMAL");
+	db.exec("PRAGMA busy_timeout = 5000");
+	db.exec(CREATE_TABLE);
+	return db;
+}
+
+/**
+ * Acquire the per-agent turn lock. Resolves once both layers are held.
+ *
+ * The returned handle MUST be released — failure to do so leaves a stale row
+ * that future acquires will treat as held until the holder's pid expires.
+ */
+export async function acquireTurnLock(opts: AcquireTurnLockOpts): Promise<TurnLockHandle> {
+	const { agentName, overstoryDir } = opts;
+	const ownerPid = opts.ownerPid ?? process.pid;
+	const now = opts._now ?? (() => Date.now());
+	const isProcessAlive = opts._isProcessAlive ?? defaultIsProcessAlive;
+	const dbPath = opts._dbPath ?? turnLockDbPath(overstoryDir);
+	const timeoutMs = opts.timeoutMs ?? 60_000;
+	const pollMs = opts.pollMs ?? 50;
+
+	// === Layer 1: in-process serialization ===
+	const previous = inProcessTails.get(agentName) ?? Promise.resolve();
+	let inProcessRelease!: () => void;
+	const current = new Promise<void>((resolve) => {
+		inProcessRelease = resolve;
+	});
+	inProcessTails.set(
+		agentName,
+		previous.then(() => current),
+	);
+	await previous;
+
+	// === Layer 2: cross-process SQLite lease ===
+	const db = openDb(dbPath);
+	const ensureRowStmt = db.prepare<void, { $n: string }>(
+		"INSERT OR IGNORE INTO turn_locks (agent_name, held_by_pid, acquired_at) VALUES ($n, NULL, NULL)",
+	);
+	const selectStmt = db.prepare<
+		{ held_by_pid: number | null; acquired_at: string | null },
+		{ $n: string }
+	>("SELECT held_by_pid, acquired_at FROM turn_locks WHERE agent_name = $n");
+	const claimStmt = db.prepare<void, { $n: string; $p: number; $a: string }>(
+		"UPDATE turn_locks SET held_by_pid = $p, acquired_at = $a WHERE agent_name = $n",
+	);
+	const releaseStmt = db.prepare<void, { $n: string; $p: number }>(
+		"UPDATE turn_locks SET held_by_pid = NULL, acquired_at = NULL WHERE agent_name = $n AND held_by_pid = $p",
+	);
+
+	const tearDown = (): void => {
+		try {
+			db.close();
+		} catch {
+			// best-effort
+		}
+		inProcessRelease();
+	};
+
+	const deadline = now() + timeoutMs;
+	let acquired = false;
+
+	while (!acquired) {
+		try {
+			db.exec("BEGIN IMMEDIATE");
+		} catch (err) {
+			// busy_timeout exhausted — fall through to retry until our own deadline.
+			if (now() >= deadline) {
+				tearDown();
+				throw err;
+			}
+			await Bun.sleep(pollMs);
+			continue;
+		}
+
+		try {
+			ensureRowStmt.run({ $n: agentName });
+			const row = selectStmt.get({ $n: agentName });
+			const held = row?.held_by_pid ?? null;
+			const stale = held !== null && !isProcessAlive(held);
+			if (held === null || stale || held === ownerPid) {
+				claimStmt.run({
+					$n: agentName,
+					$p: ownerPid,
+					$a: new Date(now()).toISOString(),
+				});
+				db.exec("COMMIT");
+				acquired = true;
+			} else {
+				db.exec("ROLLBACK");
+			}
+		} catch (err) {
+			try {
+				db.exec("ROLLBACK");
+			} catch {
+				// ignore
+			}
+			tearDown();
+			throw err;
+		}
+
+		if (!acquired) {
+			if (now() >= deadline) {
+				tearDown();
+				throw new Error(
+					`turn-lock: timed out after ${timeoutMs}ms acquiring lock for "${agentName}"`,
+				);
+			}
+			await Bun.sleep(pollMs);
+		}
+	}
+
+	let released = false;
+	return {
+		agentName,
+		release(): void {
+			if (released) return;
+			released = true;
+			try {
+				releaseStmt.run({ $n: agentName, $p: ownerPid });
+			} catch {
+				// best-effort: SQL failure must not block in-process release.
+			}
+			try {
+				db.close();
+			} catch {
+				// best-effort
+			}
+			inProcessRelease();
+		},
+	};
+}
+
+/** Inspect the persisted lock state. Used by tests and diagnostics. */
+export function readTurnLock(
+	overstoryDir: string,
+	agentName: string,
+	dbPath?: string,
+): { heldByPid: number | null; acquiredAt: string | null } {
+	const db = openDb(dbPath ?? turnLockDbPath(overstoryDir));
+	try {
+		const stmt = db.prepare<
+			{ held_by_pid: number | null; acquired_at: string | null },
+			{ $n: string }
+		>("SELECT held_by_pid, acquired_at FROM turn_locks WHERE agent_name = $n");
+		const row = stmt.get({ $n: agentName });
+		return {
+			heldByPid: row?.held_by_pid ?? null,
+			acquiredAt: row?.acquired_at ?? null,
+		};
+	} finally {
+		db.close();
+	}
+}
+
+/** Reset the in-process tail map. Used by tests; not exported through index. */
+export function _resetInProcessLocks(): void {
+	inProcessTails.clear();
+}

package/src/agents/turn-runner-dispatch.test.ts

@@ -0,0 +1,182 @@
+import { describe, expect, test } from "bun:test";
+import type { AgentRuntime } from "../runtimes/types.ts";
+import type { AgentManifest, AgentSession, OverstoryConfig } from "../types.ts";
+import { buildRunTurnOptsFactory, isSpawnPerTurnAgent } from "./turn-runner-dispatch.ts";
+
+function makeSession(overrides: Partial<AgentSession> = {}): AgentSession {
+	return {
+		id: "session-1",
+		agentName: "build-agent-1",
+		capability: "builder",
+		worktreePath: "/tmp/wt-build",
+		branchName: "overstory/b/task-1",
+		taskId: "task-1",
+		tmuxSession: "",
+		state: "working",
+		pid: 12345,
+		parentAgent: "lead-1",
+		depth: 1,
+		runId: "run-abc",
+		startedAt: new Date().toISOString(),
+		lastActivity: new Date().toISOString(),
+		escalationLevel: 0,
+		stalledSince: null,
+		transcriptPath: null,
+		...overrides,
+	};
+}
+
+function makeConfig(overrides: Partial<OverstoryConfig> = {}): OverstoryConfig {
+	const base: OverstoryConfig = {
+		project: {
+			name: "test",
+			root: "/tmp/proj",
+			canonicalBranch: "main",
+		},
+		agents: {
+			baseDir: "agents",
+			manifestPath: ".overstory/agent-manifest.json",
+			maxConcurrent: 5,
+			maxSessionsPerRun: 0,
+			maxAgentsPerLead: 5,
+			maxDepth: 2,
+			staggerDelayMs: 0,
+			autoNudgeOnMail: false,
+		},
+		worktrees: { baseDir: ".overstory/worktrees" },
+		merge: { mode: "manual" },
+		mulch: { enabled: false, domains: {} },
+		canopy: { enabled: false },
+		taskTracker: { backend: "seeds", enabled: true },
+		watchdog: {
+			tier0Enabled: false,
+			tier0IntervalMs: 30_000,
+			tier1Enabled: false,
+			maxEscalationLevel: 3,
+		},
+		models: {},
+		logging: { verbose: false, redactSecrets: true },
+		runtime: {
+			default: "claude",
+		},
+		providers: {},
+	} as unknown as OverstoryConfig;
+	return { ...base, ...overrides } as OverstoryConfig;
+}
+
+function makeManifest(): AgentManifest {
+	return {
+		version: "1",
+		agents: {
+			builder: {
+				file: "builder.md",
+				model: "claude-sonnet",
+				tools: [],
+				capabilities: ["build"],
+				canSpawn: false,
+				constraints: [],
+			},
+		},
+		capabilityIndex: { build: ["builder"] },
+	} as AgentManifest;
+}
+
+function fakeRuntime(overrides: Partial<AgentRuntime> = {}): AgentRuntime {
+	return {
+		id: "claude",
+		stability: "stable",
+		instructionPath: ".claude/CLAUDE.md",
+		buildSpawnCommand: () => "",
+		buildPrintCommand: () => [],
+		deployConfig: async () => {},
+		buildEnv: () => ({}),
+		buildDirectSpawn: () => ["claude"],
+		parseEvents: async function* () {},
+		...overrides,
+	} as unknown as AgentRuntime;
+}
+
+describe("buildRunTurnOptsFactory", () => {
+	test("threads session metadata + project paths into runTurn opts", () => {
+		const session = makeSession();
+		const config = makeConfig();
+		const manifest = makeManifest();
+		const runtime = fakeRuntime();
+
+		const factory = buildRunTurnOptsFactory({
+			session,
+			config,
+			manifest,
+			overstoryDir: "/tmp/proj/.overstory",
+			_getRuntime: () => runtime,
+			_resolveModel: () => ({ model: "claude-sonnet", isExplicitOverride: false }),
+		});
+
+		const opts = factory.build('{"type":"user"}\n');
+		expect(opts.agentName).toBe("build-agent-1");
+		expect(opts.capability).toBe("builder");
+		expect(opts.worktreePath).toBe("/tmp/wt-build");
+		expect(opts.taskId).toBe("task-1");
+		expect(opts.runId).toBe("run-abc");
+		expect(opts.projectRoot).toBe("/tmp/proj");
+		expect(opts.mailDbPath).toBe("/tmp/proj/.overstory/mail.db");
+		expect(opts.eventsDbPath).toBe("/tmp/proj/.overstory/events.db");
+		expect(opts.sessionsDbPath).toBe("/tmp/proj/.overstory/sessions.db");
+		expect(opts.userTurnNdjson).toBe('{"type":"user"}\n');
+		expect(opts.runtime).toBe(runtime);
+	});
+
+	test("threads merger capability through to runTurn opts", () => {
+		const session = makeSession({ capability: "merger", agentName: "merge-1" });
+		const factory = buildRunTurnOptsFactory({
+			session,
+			config: makeConfig(),
+			manifest: makeManifest(),
+			overstoryDir: "/tmp/proj/.overstory",
+			_getRuntime: () => fakeRuntime(),
+			_resolveModel: () => ({ model: "claude-sonnet", isExplicitOverride: false }),
+		});
+		expect(factory.build("x").capability).toBe("merger");
+	});
+});
+
+describe("isSpawnPerTurnAgent", () => {
+	const runtime = fakeRuntime();
+	const config = makeConfig();
+
+	test("returns true for a builder in non-terminal state on a claude-like runtime", () => {
+		const ok = isSpawnPerTurnAgent(
+			makeSession({ capability: "builder", state: "working" }),
+			config,
+			runtime,
+		);
+		expect(ok).toBe(true);
+	});
+
+	test("admits all task-scoped capabilities (scout, reviewer, merger, lead, builder)", () => {
+		for (const cap of ["builder", "scout", "reviewer", "merger", "lead"]) {
+			expect(isSpawnPerTurnAgent(makeSession({ capability: cap }), config, runtime)).toBe(true);
+		}
+	});
+
+	test("rejects persistent capabilities (coordinator, orchestrator, monitor)", () => {
+		for (const cap of ["coordinator", "orchestrator", "monitor"]) {
+			expect(isSpawnPerTurnAgent(makeSession({ capability: cap }), config, runtime)).toBe(false);
+		}
+	});
+
+	test("returns false for terminal states (completed, zombie)", () => {
+		expect(isSpawnPerTurnAgent(makeSession({ state: "completed" }), config, runtime)).toBe(false);
+		expect(isSpawnPerTurnAgent(makeSession({ state: "zombie" }), config, runtime)).toBe(false);
+	});
+
+	test("returns false when runtime cannot direct-spawn", () => {
+		const noDirectSpawn = fakeRuntime({ buildDirectSpawn: undefined });
+		expect(isSpawnPerTurnAgent(makeSession(), config, noDirectSpawn)).toBe(false);
+	});
+
+	test("returns false when runtime cannot parseEvents", () => {
+		const noParser = fakeRuntime({ parseEvents: undefined });
+		expect(isSpawnPerTurnAgent(makeSession(), config, noParser)).toBe(false);
+	});
+});

package/src/agents/turn-runner-dispatch.ts

@@ -0,0 +1,105 @@
+/**
+ * Helpers for dispatching user turns through `runTurn` (Phase 2 spawn-per-turn).
+ *
+ * `runTurn` (src/agents/turn-runner.ts) needs ~12 inputs to drive one builder
+ * turn. Most of those derive from the agent's `AgentSession` row plus
+ * project-level paths. Consumers that route mail or nudges through the
+ * spawn-per-turn engine — `ov serve`, `ov nudge` — need the same setup
+ * boilerplate, so it lives here.
+ *
+ * The exported `buildRunTurnOptsFactory()` resolves the runtime adapter and
+ * model once per agent and returns a `(userTurnNdjson) => RunTurnOpts`
+ * closure. The closure is small and disposable; callers re-resolve it when
+ * the underlying session metadata changes (e.g. on rescan).
+ */
+
+import { join } from "node:path";
+import { getRuntime } from "../runtimes/registry.ts";
+import type { AgentRuntime } from "../runtimes/types.ts";
+import type { AgentManifest, AgentSession, OverstoryConfig, ResolvedModel } from "../types.ts";
+import { isTaskScopedCapability } from "./capabilities.ts";
+import { resolveModel } from "./manifest.ts";
+import type { RunTurnOpts } from "./turn-runner.ts";
+
+export interface BuildOptsFactoryInput {
+	session: AgentSession;
+	config: OverstoryConfig;
+	manifest: AgentManifest;
+	overstoryDir: string;
+	/** Optional override (test injection). Defaults to `getRuntime`. */
+	_getRuntime?: typeof getRuntime;
+	/** Optional override (test injection). Defaults to `resolveModel`. */
+	_resolveModel?: typeof resolveModel;
+}
+
+export interface BuiltOptsFactory {
+	runtime: AgentRuntime;
+	resolvedModel: ResolvedModel;
+	/** Produces a fresh `RunTurnOpts` for the given user turn payload. */
+	build: (userTurnNdjson: string) => RunTurnOpts;
+}
+
+/**
+ * Resolve runtime + model for a session and return a factory that produces
+ * `RunTurnOpts` given a user-turn NDJSON payload.
+ *
+ * The factory closes over the per-agent metadata (worktreePath, capability,
+ * runId, etc.) so callers only have to supply the dynamic payload. The
+ * factory's outputs are otherwise identical run-to-run.
+ */
+export function buildRunTurnOptsFactory(input: BuildOptsFactoryInput): BuiltOptsFactory {
+	const { session, config, manifest, overstoryDir } = input;
+	const _getRuntime = input._getRuntime ?? getRuntime;
+	const _resolveModel = input._resolveModel ?? resolveModel;
+
+	const runtime = _getRuntime(undefined, config, session.capability);
+	const fallback = manifest.agents[session.capability]?.model ?? "claude-sonnet";
+	const resolvedModel = _resolveModel(config, manifest, session.capability, fallback);
+
+	const mailDbPath = join(overstoryDir, "mail.db");
+	const eventsDbPath = join(overstoryDir, "events.db");
+	const sessionsDbPath = join(overstoryDir, "sessions.db");
+
+	const build = (userTurnNdjson: string): RunTurnOpts => ({
+		agentName: session.agentName,
+		capability: session.capability,
+		overstoryDir,
+		worktreePath: session.worktreePath,
+		projectRoot: config.project.root,
+		taskId: session.taskId,
+		userTurnNdjson,
+		runtime,
+		resolvedModel,
+		runId: session.runId,
+		mailDbPath,
+		eventsDbPath,
+		sessionsDbPath,
+	});
+
+	return { runtime, resolvedModel, build };
+}
+
+/**
+ * Predicate: is this agent eligible for spawn-per-turn dispatch?
+ *
+ * Capability gate: any task-scoped worker (builder, scout, reviewer, merger,
+ * lead — see {@link isTaskScopedCapability}) running on a runtime that
+ * implements `buildDirectSpawn`/`parseEvents` (today: claude). Non-terminal
+ * state required — completed/zombie sessions are never re-spawned.
+ *
+ * `_config` is unused at this layer (the project-level gate flag was removed in
+ * Phase 3 once spawn-per-turn became the only path for task-scoped Claude) but
+ * the parameter is preserved so call sites that already thread config in don't
+ * need to change.
+ */
+export function isSpawnPerTurnAgent(
+	session: AgentSession,
+	_config: OverstoryConfig,
+	runtime: AgentRuntime,
+): boolean {
+	if (!isTaskScopedCapability(session.capability)) return false;
+	if (session.state === "completed" || session.state === "zombie") return false;
+	if (typeof runtime.buildDirectSpawn !== "function") return false;
+	if (typeof runtime.parseEvents !== "function") return false;
+	return true;
+}