@os-eco/overstory-cli 0.9.1 → 0.9.3

package/src/watchdog/daemon.ts

@@ -37,7 +37,7 @@ import { openSessionStore } from "../sessions/compat.ts";
 import type { AgentSession, EventStore, HealthCheck } from "../types.ts";
 import { isProcessAlive, isSessionAlive, killProcessTree, killSession } from "../worktree/tmux.ts";
 import { evaluateHealth, transitionState } from "./health.ts";
-import { triageAgent } from "./triage.ts";
+import { type TriageResult, triageAgent } from "./triage.ts";
 
 /** Maximum escalation level (terminate). */
 const MAX_ESCALATION_LEVEL = 3;
@@ -46,7 +46,7 @@ const MAX_ESCALATION_LEVEL = 3;
  * Persistent agent capabilities that are excluded from run-level completion checks.
  * These agents are long-running and should not count toward "all workers done".
  */
-const PERSISTENT_CAPABILITIES = new Set(["coordinator", "monitor"]);
+const PERSISTENT_CAPABILITIES = new Set(["coordinator", "orchestrator", "monitor"]);
 
 /**
  * Module-level registry of active event tailers for headless agents.
@@ -280,7 +280,9 @@ export interface DaemonOptions {
 		agentName: string;
 		root: string;
 		lastActivity: string;
-	}) => Promise<"retry" | "terminate" | "extend">;
+	}) => Promise<TriageResult | "retry" | "terminate" | "extend">;
+	/** Max triage calls per daemon tick (prevents runaway AI usage). Default: 3. */
+	_maxTriagePerTick?: number;
 	/** Dependency injection for testing. Uses real nudgeAgent when omitted. */
 	_nudge?: (
 		projectRoot: string,
@@ -340,6 +342,7 @@ export interface DaemonOptions {
  */
 export function startDaemon(options: DaemonOptions & { intervalMs: number }): { stop: () => void } {
 	const { intervalMs } = options;
+	const tailerRegistry = options._tailerRegistry ?? _defaultTailerRegistry;
 
 	// Run the first tick immediately, then on interval
 	runDaemonTick(options).catch(() => {
@@ -355,6 +358,10 @@ export function startDaemon(options: DaemonOptions & { intervalMs: number }): {
 	return {
 		stop(): void {
 			clearInterval(interval);
+			for (const [name, handle] of tailerRegistry) {
+				handle.stop();
+				tailerRegistry.delete(name);
+			}
 		},
 	};
 }
@@ -416,6 +423,8 @@ export async function runDaemonTick(options: DaemonOptions): Promise<void> {
 	const tailerRegistry = options._tailerRegistry ?? _defaultTailerRegistry;
 	const tailerFactory = options._tailerFactory ?? startEventTailer;
 	const findStdoutLog = options._findLatestStdoutLog ?? findLatestStdoutLog;
+	const maxTriagePerTick = options._maxTriagePerTick ?? 3;
+	const triageCount = { value: 0 };
 
 	const overstoryDir = join(root, ".overstory");
 	const { store } = openSessionStore(overstoryDir);
@@ -624,6 +633,8 @@ export async function runDaemonTick(options: DaemonOptions): Promise<void> {
 					eventStore,
 					runId,
 					recordFailure: recordFailureFn,
+					triageCount,
+					maxTriagePerTick,
 				});
 
 				if (actionResult.terminated) {
@@ -710,7 +721,11 @@ async function executeEscalationAction(ctx: {
 		agentName: string;
 		root: string;
 		lastActivity: string;
-	}) => Promise<"retry" | "terminate" | "extend">;
+	}) => Promise<TriageResult | "retry" | "terminate" | "extend">;
+	/** Shared counter across escalation calls in a single tick — enforces maxTriagePerTick. */
+	triageCount: { value: number };
+	/** Maximum number of triage calls allowed in one daemon tick. Default: 3. */
+	maxTriagePerTick: number;
 	nudge: (
 		projectRoot: string,
 		agentName: string,
@@ -739,6 +754,8 @@ async function executeEscalationAction(ctx: {
 		eventStore,
 		runId,
 		recordFailure,
+		triageCount,
+		maxTriagePerTick,
 	} = ctx;
 
 	switch (session.escalationLevel) {
@@ -785,29 +802,49 @@ async function executeEscalationAction(ctx: {
 				return { terminated: false, stateChanged: false };
 			}
 
-			const verdict = await triage({
+			// Concurrency guard: limit triage calls per tick to avoid runaway AI usage
+			if (triageCount.value >= maxTriagePerTick) {
+				return { terminated: false, stateChanged: false };
+			}
+			triageCount.value++;
+
+			const raw = await triage({
 				agentName: session.agentName,
 				root,
 				lastActivity: session.lastActivity,
 			});
+			// Normalize: accept bare string (backward compat) or TriageResult
+			const result: TriageResult =
+				typeof raw === "string" ? { verdict: raw, fallback: false } : raw;
 
 			recordEvent(eventStore, {
 				runId,
 				agentName: session.agentName,
 				eventType: "custom",
 				level: "warn",
-				data: { type: "triage", escalationLevel: 2, verdict },
+				data: {
+					type: "triage",
+					escalationLevel: 2,
+					verdict: result.verdict,
+					triageFailed: result.fallback,
+				},
 			});
 
-			if (verdict === "terminate") {
+			if (result.verdict === "terminate") {
 				// Record the failure via mulch (Tier 1 AI triage)
-				await recordFailure(root, session, "AI triage classified as terminal failure", 1, verdict);
+				await recordFailure(
+					root,
+					session,
+					"AI triage classified as terminal failure",
+					1,
+					result.verdict,
+				);
 
 				await killAgent({ session, tmuxAlive, tmux, process: proc });
 				return { terminated: true, stateChanged: true };
 			}
 
-			if (verdict === "retry") {
+			if (result.verdict === "retry") {
 				// Send a nudge with a recovery message
 				try {
 					await nudge(

package/src/watchdog/health.test.ts

@@ -196,7 +196,7 @@ describe("evaluateHealth", () => {
 		expect(check.action).toBe("escalate");
 	});
 
-	// --- Persistent capabilities (coordinator, monitor) ---
+	// --- Persistent capabilities (coordinator, orchestrator, monitor) ---
 
 	test("persistent capability: coordinator with stale activity → still working, no escalation", () => {
 		const staleActivity = new Date(Date.now() - 60_000).toISOString();
@@ -237,6 +237,20 @@ describe("evaluateHealth", () => {
 		expect(check.action).toBe("none");
 	});
 
+	test("persistent capability: orchestrator with stale activity → still working", () => {
+		const staleActivity = new Date(Date.now() - 60_000).toISOString();
+		const session = makeSession({
+			agentName: "orchestrator",
+			capability: "orchestrator",
+			state: "working",
+			lastActivity: staleActivity,
+		});
+		const check = evaluateHealth(session, true, THRESHOLDS);
+
+		expect(check.state).toBe("working");
+		expect(check.action).toBe("none");
+	});
+
 	test("persistent capability: coordinator booting → transitions to working", () => {
 		const session = makeSession({
 			capability: "coordinator",

package/src/watchdog/health.ts

@@ -40,7 +40,7 @@ import type { AgentSession, AgentState, HealthCheck } from "../types.ts";
  *
  * Shared concept with src/commands/log.ts:PERSISTENT_CAPABILITIES.
  */
-const PERSISTENT_CAPABILITIES = new Set(["coordinator", "monitor"]);
+const PERSISTENT_CAPABILITIES = new Set(["coordinator", "orchestrator", "monitor"]);
 
 /** Numeric ordering for forward-only state transitions. */
 const STATE_ORDER: Record<AgentState, number> = {

package/src/watchdog/triage.test.ts

@@ -6,7 +6,7 @@
  * spawnClaude is NOT mocked — we rely on it failing naturally in tests.
  */
 
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
 import { mkdir, mkdtemp } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -101,16 +101,18 @@ describe("triageAgent", () => {
 		await cleanupTempDir(tempRoot);
 	});
 
-	test("returns 'extend' when no logs directory exists", async () => {
+	test("returns fallback TriageResult when no logs directory exists", async () => {
 		const result = await triageAgent({
 			agentName: "test-agent",
 			root: tempRoot,
 			lastActivity: "2026-02-13T10:00:00Z",
 		});
-		expect(result).toBe("extend");
+		expect(result.verdict).toBe("extend");
+		expect(result.fallback).toBe(true);
+		expect(result.reason).toBe("No logs available");
 	});
 
-	test("returns 'extend' when logs directory exists but is empty", async () => {
+	test("returns fallback TriageResult when logs directory exists but is empty", async () => {
 		const logsDir = join(tempRoot, ".overstory", "logs", "test-agent");
 		await mkdir(logsDir, { recursive: true });
 
@@ -119,10 +121,11 @@ describe("triageAgent", () => {
 			root: tempRoot,
 			lastActivity: "2026-02-13T10:00:00Z",
 		});
-		expect(result).toBe("extend");
+		expect(result.verdict).toBe("extend");
+		expect(result.fallback).toBe(true);
 	});
 
-	test("returns 'extend' when logs directory has session dir but no session.log", async () => {
+	test("returns fallback TriageResult when logs directory has session dir but no session.log", async () => {
 		const logsDir = join(tempRoot, ".overstory", "logs", "test-agent", "2026-02-13T10-00-00");
 		await Bun.write(join(logsDir, ".gitkeep"), "");
 
@@ -131,10 +134,11 @@ describe("triageAgent", () => {
 			root: tempRoot,
 			lastActivity: "2026-02-13T10:00:00Z",
 		});
-		expect(result).toBe("extend");
+		expect(result.verdict).toBe("extend");
+		expect(result.fallback).toBe(true);
 	});
 
-	test("returns 'extend' when session.log exists but claude binary fails", async () => {
+	test("returns fallback TriageResult when session.log exists but claude binary fails", async () => {
 		const timestamp = "2026-02-13T10-00-00";
 		const sessionLogPath = join(
 			tempRoot,
@@ -160,6 +164,42 @@ describe("triageAgent", () => {
 			lastActivity: "2026-02-13T10:00:00Z",
 			timeoutMs: 500,
 		});
-		expect(result).toBe("extend");
+		expect(result.verdict).toBe("extend");
+		expect(result.fallback).toBe(true);
+		expect(result.reason).toBe("Claude unavailable");
+	});
+
+	test("writes stderr warning when claude is unavailable (fallback path)", async () => {
+		const timestamp = "2026-02-13T10-00-00";
+		const sessionLogPath = join(
+			tempRoot,
+			".overstory",
+			"logs",
+			"test-agent",
+			timestamp,
+			"session.log",
+		);
+		await Bun.write(sessionLogPath, "some log content\n");
+
+		const written: string[] = [];
+		const spy = spyOn(process.stderr, "write").mockImplementation((chunk: unknown) => {
+			written.push(String(chunk));
+			return true;
+		});
+
+		try {
+			await triageAgent({
+				agentName: "test-agent",
+				root: tempRoot,
+				lastActivity: "2026-02-13T10:00:00Z",
+				timeoutMs: 500,
+			});
+		} finally {
+			spy.mockRestore();
+		}
+
+		expect(written.some((s) => s.includes("triage fallback") && s.includes("test-agent"))).toBe(
+			true,
+		);
 	});
 });

package/src/watchdog/triage.ts

@@ -12,6 +12,19 @@ import { AgentError } from "../errors.ts";
 import { getRuntime } from "../runtimes/registry.ts";
 import type { OverstoryConfig } from "../types.ts";
 
+/**
+ * Structured result returned by triageAgent.
+ * Replaces bare string for observability — callers can distinguish AI verdicts from fallbacks.
+ */
+export interface TriageResult {
+	/** Classification: "retry", "terminate", or "extend". */
+	verdict: "retry" | "terminate" | "extend";
+	/** True when the verdict is a safe default (no AI analysis performed). */
+	fallback: boolean;
+	/** Optional human-readable reason, set on fallback paths. */
+	reason?: string;
+}
+
 /**
  * Triage a stalled agent by analyzing its recent log output with Claude.
  *
@@ -24,7 +37,7 @@ import type { OverstoryConfig } from "../types.ts";
  * @param options.agentName - Name of the agent to triage
  * @param options.root - Project root directory (contains .overstory/)
  * @param options.lastActivity - ISO timestamp of the agent's last recorded activity
- * @returns "retry" if recoverable, "terminate" if fatal, "extend" if likely long-running
+ * @returns TriageResult with verdict ("retry" | "terminate" | "extend"), fallback flag, and optional reason
  */
 export async function triageAgent(options: {
 	agentName: string;
@@ -34,7 +47,7 @@ export async function triageAgent(options: {
 	timeoutMs?: number;
 	/** Overstory config for runtime resolution. */
 	config?: OverstoryConfig;
-}): Promise<"retry" | "terminate" | "extend"> {
+}): Promise<TriageResult> {
 	const { agentName, root, lastActivity, timeoutMs, config } = options;
 	const logsDir = join(root, ".overstory", "logs", agentName);
 
@@ -43,17 +56,20 @@ export async function triageAgent(options: {
 		logContent = await readRecentLog(logsDir);
 	} catch {
 		// No logs available — assume long-running operation
-		return "extend";
+		return { verdict: "extend", fallback: true, reason: "No logs available" };
 	}
 
 	const prompt = buildTriagePrompt(agentName, lastActivity, logContent);
 
 	try {
 		const response = await spawnClaude(prompt, timeoutMs, config);
-		return classifyResponse(response);
+		return { verdict: classifyResponse(response), fallback: false };
 	} catch {
 		// Claude not available — default to extend (safe fallback)
-		return "extend";
+		process.stderr.write(
+			`[watchdog] triage fallback for ${agentName}: Claude unavailable, defaulting to extend\n`,
+		);
+		return { verdict: "extend", fallback: true, reason: "Claude unavailable" };
 	}
 }