@os-eco/overstory-cli 0.6.1

package/src/doctor/logs.test.ts

@@ -0,0 +1,251 @@
+/**
+ * Tests for logs doctor checks.
+ *
+ * Uses temp directories with real filesystem operations.
+ * No mocks needed -- all operations are cheap and local.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { OverstoryConfig } from "../types.ts";
+import { checkLogs } from "./logs.ts";
+
+describe("checkLogs", () => {
+	let tempDir: string;
+	let overstoryDir: string;
+	let logsDir: string;
+	let mockConfig: OverstoryConfig;
+
+	beforeEach(async () => {
+		tempDir = await mkdtemp(join(tmpdir(), "logs-test-"));
+		overstoryDir = join(tempDir, ".overstory");
+		logsDir = join(overstoryDir, "logs");
+
+		mockConfig = {
+			project: {
+				name: "test-project",
+				root: tempDir,
+				canonicalBranch: "main",
+			},
+			agents: {
+				manifestPath: ".overstory/agent-manifest.json",
+				baseDir: ".overstory/agent-defs",
+				maxConcurrent: 5,
+				staggerDelayMs: 1000,
+				maxDepth: 2,
+				maxSessionsPerRun: 0,
+			},
+			worktrees: {
+				baseDir: ".overstory/worktrees",
+			},
+			taskTracker: {
+				backend: "auto",
+				enabled: true,
+			},
+			mulch: {
+				enabled: true,
+				domains: [],
+				primeFormat: "markdown",
+			},
+			merge: {
+				aiResolveEnabled: false,
+				reimagineEnabled: false,
+			},
+			providers: {
+				anthropic: { type: "native" },
+			},
+			watchdog: {
+				tier0Enabled: true,
+				tier0IntervalMs: 30000,
+				tier1Enabled: false,
+				tier2Enabled: false,
+				staleThresholdMs: 300000,
+				zombieThresholdMs: 600000,
+				nudgeIntervalMs: 60000,
+			},
+			models: {},
+			logging: {
+				verbose: false,
+				redactSecrets: true,
+			},
+		};
+
+		await mkdir(overstoryDir, { recursive: true });
+	});
+
+	afterEach(async () => {
+		await rm(tempDir, { recursive: true, force: true });
+	});
+
+	test("warns when logs/ directory does not exist", async () => {
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const dirCheck = checks.find((c) => c.name === "logs/ directory");
+		expect(dirCheck).toBeDefined();
+		expect(dirCheck?.status).toBe("warn");
+		expect(dirCheck?.message).toContain("missing");
+	});
+
+	test("passes when logs/ directory exists but is empty", async () => {
+		await mkdir(logsDir, { recursive: true });
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const dirCheck = checks.find((c) => c.name === "logs/ directory");
+		expect(dirCheck?.status).toBe("pass");
+
+		const usageCheck = checks.find((c) => c.name === "Total disk usage");
+		expect(usageCheck?.status).toBe("pass");
+		expect(usageCheck?.message).toContain("0B");
+	});
+
+	test("calculates total disk usage correctly", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		// Create log files with known sizes
+		await writeFile(join(agentDir, "session.log"), "a".repeat(1024)); // 1KB
+		await writeFile(join(agentDir, "events.ndjson"), "b".repeat(2048)); // 2KB
+		await writeFile(join(agentDir, "tools.ndjson"), "c".repeat(512)); // 512B
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const usageCheck = checks.find((c) => c.name === "Total disk usage");
+		expect(usageCheck).toBeDefined();
+		expect(usageCheck?.status).toBe("pass");
+		expect(usageCheck?.message).toContain("3.5KB"); // 1024 + 2048 + 512 = 3584 bytes
+	});
+
+	test("warns when disk usage exceeds threshold", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		// Create a sparse file that reports as 600MB without allocating real disk space
+		const filePath = join(agentDir, "session.log");
+		const { promises: fsp } = await import("node:fs");
+		const fd = await fsp.open(filePath, "w");
+		await fd.truncate(600 * 1024 * 1024); // 600MB sparse file
+		await fd.close();
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const usageCheck = checks.find((c) => c.name === "Total disk usage");
+		expect(usageCheck).toBeDefined();
+		expect(usageCheck?.status).toBe("warn");
+		expect(usageCheck?.details).toBeDefined();
+		expect(usageCheck?.details?.some((d) => d.includes("threshold"))).toBe(true);
+	});
+
+	test("reports per-agent log sizes", async () => {
+		await mkdir(logsDir, { recursive: true });
+
+		// Create logs for multiple agents
+		const agent1Dir = join(logsDir, "agent-1", "session-1");
+		await mkdir(agent1Dir, { recursive: true });
+		await writeFile(join(agent1Dir, "session.log"), "a".repeat(10000));
+
+		const agent2Dir = join(logsDir, "agent-2", "session-1");
+		await mkdir(agent2Dir, { recursive: true });
+		await writeFile(join(agent2Dir, "session.log"), "b".repeat(5000));
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const sizesCheck = checks.find((c) => c.name === "Per-agent log sizes");
+		expect(sizesCheck).toBeDefined();
+		expect(sizesCheck?.status).toBe("pass");
+		expect(sizesCheck?.message).toContain("2 agent(s)");
+		expect(sizesCheck?.details).toBeDefined();
+		expect(sizesCheck?.details?.some((d) => d.includes("agent-1"))).toBe(true);
+		expect(sizesCheck?.details?.some((d) => d.includes("agent-2"))).toBe(true);
+	});
+
+	test("detects malformed NDJSON in events.ndjson", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		// Valid and invalid JSON lines
+		const content = `{"event":"start","timestamp":"2024-01-01T00:00:00Z"}
+invalid json line here
+{"event":"end","timestamp":"2024-01-01T00:01:00Z"}`;
+
+		await writeFile(join(agentDir, "events.ndjson"), content);
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const integrityCheck = checks.find((c) => c.name === "NDJSON integrity");
+		expect(integrityCheck).toBeDefined();
+		expect(integrityCheck?.status).toBe("warn");
+		expect(integrityCheck?.message).toContain("malformed JSON");
+		expect(integrityCheck?.details?.some((d) => d.includes("events.ndjson"))).toBe(true);
+	});
+
+	test("passes when all NDJSON files are valid", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		const validContent = `{"event":"start","timestamp":"2024-01-01T00:00:00Z"}
+{"event":"end","timestamp":"2024-01-01T00:01:00Z"}`;
+
+		await writeFile(join(agentDir, "events.ndjson"), validContent);
+		await writeFile(join(agentDir, "tools.ndjson"), validContent);
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const integrityCheck = checks.find((c) => c.name === "NDJSON integrity");
+		expect(integrityCheck).toBeDefined();
+		expect(integrityCheck?.status).toBe("pass");
+	});
+
+	test("detects orphaned toolStart events", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		// toolStart without matching toolEnd
+		const content = `{"event":"toolStart","tool":"Read","timestamp":"2024-01-01T00:00:00Z"}
+{"event":"toolStart","tool":"Write","timestamp":"2024-01-01T00:01:00Z"}
+{"event":"toolEnd","tool":"Write","timestamp":"2024-01-01T00:02:00Z"}`;
+
+		await writeFile(join(agentDir, "tools.ndjson"), content);
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		const orphanCheck = checks.find((c) => c.name === "Orphaned tool events");
+		expect(orphanCheck).toBeDefined();
+		expect(orphanCheck?.status).toBe("warn");
+		expect(orphanCheck?.message).toContain("incomplete");
+		expect(orphanCheck?.details?.some((d) => d.includes("Read"))).toBe(true);
+	});
+
+	test("handles missing tools.ndjson gracefully", async () => {
+		await mkdir(logsDir, { recursive: true });
+		const agentDir = join(logsDir, "test-agent", "session-1");
+		await mkdir(agentDir, { recursive: true });
+
+		// Only create events.ndjson, no tools.ndjson
+		await writeFile(join(agentDir, "events.ndjson"), '{"event":"start"}');
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		// Should not crash, should pass NDJSON integrity
+		const integrityCheck = checks.find((c) => c.name === "NDJSON integrity");
+		expect(integrityCheck).toBeDefined();
+		expect(integrityCheck?.status).toBe("pass");
+	});
+
+	test("handles empty logs directory gracefully", async () => {
+		await mkdir(logsDir, { recursive: true });
+
+		const checks = await checkLogs(mockConfig, overstoryDir);
+
+		expect(checks.length).toBeGreaterThan(0);
+		const dirCheck = checks.find((c) => c.name === "logs/ directory");
+		expect(dirCheck?.status).toBe("pass");
+	});
+});

package/src/doctor/logs.ts

@@ -0,0 +1,295 @@
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import type { DoctorCheck, DoctorCheckFn } from "./types.ts";
+
+const DISK_USAGE_WARN_THRESHOLD = 500 * 1024 * 1024; // 500MB in bytes
+
+/**
+ * Check if a path exists.
+ */
+async function pathExists(path: string): Promise<boolean> {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Calculate total disk usage for a directory recursively.
+ */
+async function calculateDiskUsage(dirPath: string): Promise<number> {
+	let totalBytes = 0;
+
+	try {
+		const entries = await readdir(dirPath, { withFileTypes: true });
+
+		for (const entry of entries) {
+			const fullPath = join(dirPath, entry.name);
+
+			if (entry.isDirectory()) {
+				totalBytes += await calculateDiskUsage(fullPath);
+			} else if (entry.isFile()) {
+				const stats = await stat(fullPath);
+				totalBytes += stats.size;
+			}
+		}
+	} catch {
+		// Ignore errors (permission denied, etc.)
+	}
+
+	return totalBytes;
+}
+
+/**
+ * Format bytes as human-readable size.
+ */
+function formatBytes(bytes: number): string {
+	if (bytes < 1024) return `${bytes}B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+	if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+	return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+}
+
+/**
+ * Check NDJSON file for malformed lines.
+ */
+async function checkNDJSON(filePath: string): Promise<string[]> {
+	const errors: string[] = [];
+
+	try {
+		const content = await Bun.file(filePath).text();
+		const lines = content.split("\n").filter((line) => line.trim() !== "");
+
+		for (let i = 0; i < lines.length; i++) {
+			const line = lines[i];
+			if (!line) continue;
+
+			try {
+				JSON.parse(line);
+			} catch {
+				errors.push(`Line ${i + 1}: malformed JSON`);
+			}
+		}
+	} catch {
+		// File doesn't exist or can't be read
+	}
+
+	return errors;
+}
+
+/**
+ * Check tools.ndjson for orphaned toolStart events.
+ */
+async function checkOrphanedToolEvents(filePath: string): Promise<string[]> {
+	const orphans: string[] = [];
+
+	try {
+		const content = await Bun.file(filePath).text();
+		const lines = content.split("\n").filter((line) => line.trim() !== "");
+
+		const startEvents = new Set<string>();
+
+		for (const line of lines) {
+			if (!line) continue;
+
+			try {
+				const event = JSON.parse(line) as { event?: string; tool?: string; data?: unknown };
+
+				if (event.event === "toolStart" && typeof event.tool === "string") {
+					startEvents.add(event.tool);
+				} else if (event.event === "toolEnd" && typeof event.tool === "string") {
+					startEvents.delete(event.tool);
+				}
+			} catch {
+				// Ignore malformed lines
+			}
+		}
+
+		for (const tool of startEvents) {
+			orphans.push(tool);
+		}
+	} catch {
+		// File doesn't exist or can't be read
+	}
+
+	return orphans;
+}
+
+/**
+ * Get per-agent log sizes.
+ */
+async function getPerAgentSizes(logsDir: string): Promise<Map<string, number>> {
+	const sizes = new Map<string, number>();
+
+	try {
+		const agentDirs = await readdir(logsDir, { withFileTypes: true });
+
+		for (const agentDir of agentDirs) {
+			if (!agentDir.isDirectory()) continue;
+
+			const agentPath = join(logsDir, agentDir.name);
+			const size = await calculateDiskUsage(agentPath);
+			sizes.set(agentDir.name, size);
+		}
+	} catch {
+		// Logs directory doesn't exist or can't be read
+	}
+
+	return sizes;
+}
+
+/**
+ * Log directory health checks.
+ * Validates log directory structure and detects excessive log accumulation.
+ */
+export const checkLogs: DoctorCheckFn = async (_config, overstoryDir): Promise<DoctorCheck[]> => {
+	const checks: DoctorCheck[] = [];
+	const logsDir = join(overstoryDir, "logs");
+
+	// Check 1: logs/ directory exists
+	const logsDirExists = await pathExists(logsDir);
+	checks.push({
+		name: "logs/ directory",
+		category: "logs",
+		status: logsDirExists ? "pass" : "warn",
+		message: logsDirExists
+			? "Directory exists"
+			: "Directory missing (will be created on first log)",
+		details: logsDirExists ? undefined : ["Not an error - created automatically on agent spawn"],
+		fixable: false,
+	});
+
+	// If logs/ doesn't exist, no further checks
+	if (!logsDirExists) {
+		return checks;
+	}
+
+	// Check 2: Total disk usage
+	const totalBytes = await calculateDiskUsage(logsDir);
+	checks.push({
+		name: "Total disk usage",
+		category: "logs",
+		status: totalBytes > DISK_USAGE_WARN_THRESHOLD ? "warn" : "pass",
+		message: `Using ${formatBytes(totalBytes)}`,
+		details:
+			totalBytes > DISK_USAGE_WARN_THRESHOLD
+				? [
+						`Exceeds ${formatBytes(DISK_USAGE_WARN_THRESHOLD)} threshold`,
+						"Consider running 'overstory worktree clean --completed' to remove old logs",
+					]
+				: undefined,
+		fixable: totalBytes > DISK_USAGE_WARN_THRESHOLD,
+	});
+
+	// Check 3: Per-agent log sizes
+	const perAgentSizes = await getPerAgentSizes(logsDir);
+	if (perAgentSizes.size > 0) {
+		const details = Array.from(perAgentSizes.entries())
+			.sort((a, b) => b[1] - a[1]) // Sort by size descending
+			.map(([agent, size]) => `${agent}: ${formatBytes(size)}`);
+
+		checks.push({
+			name: "Per-agent log sizes",
+			category: "logs",
+			status: "pass",
+			message: `${perAgentSizes.size} agent(s) with logs`,
+			details,
+			fixable: false,
+		});
+	}
+
+	// Check 4: Sample NDJSON files for malformed JSON
+	// We'll check a few random session directories
+	try {
+		const agentDirs = await readdir(logsDir, { withFileTypes: true });
+		const ndjsonErrors: string[] = [];
+
+		for (const agentDir of agentDirs) {
+			if (!agentDir.isDirectory()) continue;
+
+			const agentPath = join(logsDir, agentDir.name);
+			const sessionDirs = await readdir(agentPath, { withFileTypes: true });
+
+			for (const sessionDir of sessionDirs) {
+				if (!sessionDir.isDirectory()) continue;
+
+				const sessionPath = join(agentPath, sessionDir.name);
+
+				// Check events.ndjson
+				const eventsErrors = await checkNDJSON(join(sessionPath, "events.ndjson"));
+				if (eventsErrors.length > 0) {
+					ndjsonErrors.push(
+						`${agentDir.name}/${sessionDir.name}/events.ndjson: ${eventsErrors.length} error(s)`,
+					);
+				}
+
+				// Check tools.ndjson
+				const toolsErrors = await checkNDJSON(join(sessionPath, "tools.ndjson"));
+				if (toolsErrors.length > 0) {
+					ndjsonErrors.push(
+						`${agentDir.name}/${sessionDir.name}/tools.ndjson: ${toolsErrors.length} error(s)`,
+					);
+				}
+			}
+		}
+
+		if (ndjsonErrors.length > 0) {
+			checks.push({
+				name: "NDJSON integrity",
+				category: "logs",
+				status: "warn",
+				message: `Found ${ndjsonErrors.length} file(s) with malformed JSON`,
+				details: ndjsonErrors.slice(0, 10), // Limit to first 10
+				fixable: false,
+			});
+		} else {
+			checks.push({
+				name: "NDJSON integrity",
+				category: "logs",
+				status: "pass",
+				message: "All sampled NDJSON files are valid",
+				fixable: false,
+			});
+		}
+
+		// Check 5: Orphaned toolStart events
+		const orphanedTools: string[] = [];
+
+		for (const agentDir of agentDirs) {
+			if (!agentDir.isDirectory()) continue;
+
+			const agentPath = join(logsDir, agentDir.name);
+			const sessionDirs = await readdir(agentPath, { withFileTypes: true });
+
+			for (const sessionDir of sessionDirs) {
+				if (!sessionDir.isDirectory()) continue;
+
+				const sessionPath = join(agentPath, sessionDir.name);
+				const orphans = await checkOrphanedToolEvents(join(sessionPath, "tools.ndjson"));
+
+				if (orphans.length > 0) {
+					orphanedTools.push(
+						`${agentDir.name}/${sessionDir.name}: ${orphans.join(", ")} (incomplete session)`,
+					);
+				}
+			}
+		}
+
+		if (orphanedTools.length > 0) {
+			checks.push({
+				name: "Orphaned tool events",
+				category: "logs",
+				status: "warn",
+				message: `Found ${orphanedTools.length} session(s) with incomplete tool logs`,
+				details: orphanedTools.slice(0, 10), // Limit to first 10
+				fixable: false,
+			});
+		}
+	} catch {
+		// Ignore errors reading log directories
+	}
+
+	return checks;
+};