@alphacoder-v0/pi-loops 0.17.3

package/src/job-health.ts

@@ -0,0 +1,29 @@
+/**
+ * Which loops have stopped working, and the one line that says so.
+ *
+ * `consecutiveFailures` drives the scheduler's backoff and nothing else reads it, so a loop that has
+ * failed forty nights in a row looks exactly like a healthy one until someone types `/cron`. This is
+ * what the badge, the session-start line and `/cron` print instead — a question about a list of jobs,
+ * answerable without a scheduler, a session or a clock.
+ */
+import { FAILURE_BACKOFF_AFTER } from "./scheduler.ts";
+import type { LoopJob } from "./store.ts";
+
+/**
+ * Jobs failing on `hostName`, worst first. The threshold is the scheduler's backoff threshold, which
+ * is the honest place to draw the line: below it a failure is a bad night, at it the scheduler has
+ * already started widening the gap. A job pinned to another host (a shared $HOME) is that host's to
+ * report, not ours.
+ */
+export function failingJobs(jobs: LoopJob[], hostName: string): LoopJob[] {
+	return jobs.filter((j) => j.enabled && (!j.host || j.host === hostName) && (j.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER).sort((a, b) => (b.consecutiveFailures ?? 0) - (a.consecutiveFailures ?? 0));
+}
+
+/** `2 job(s) failing (check-issues ×7)` — one clause, the worst one named, whatever the count. */
+export function failingSummary(jobs: LoopJob[], hostName: string): string | undefined {
+	const failing = failingJobs(jobs, hostName);
+	const worst = failing[0];
+	if (!worst) return undefined;
+	const label = worst.name ?? worst.id;
+	return `${failing.length} job(s) failing (${label.length > 24 ? `${label.slice(0, 23)}…` : label} ×${worst.consecutiveFailures})`;
+}

package/src/lock.ts

@@ -0,0 +1,193 @@
+/**
+ * Small cross-process primitives: a mkdir-based file lock, atomic writes,
+ * and pid liveness. No native deps, works on any POSIX fs (and NTFS).
+ */
+import { randomBytes } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+export interface LockOptions {
+	/** A lock older than this is considered abandoned and is broken. */
+	staleMs?: number;
+	/** Give up after this long. */
+	timeoutMs?: number;
+}
+
+/**
+ * A token identifying who holds a lock. Breaking a stale lock and creating your own is not enough:
+ * the original holder's `finally` would delete *your* directory and let a third caller in. Each
+ * holder writes its own token and only removes the lock while that token is still there.
+ */
+function writeOwner(lockPath: string): string | undefined {
+	const token = `${process.pid}.${randomBytes(6).toString("hex")}`;
+	try {
+		fs.writeFileSync(path.join(lockPath, "owner"), token, "utf8");
+		return token;
+	} catch {
+		/* an unwritable lock dir still serialises via mkdir; the token is the extra safety */
+		return undefined;
+	}
+}
+
+function releaseOwned(lockPath: string, token: string | undefined): void {
+	// No token means `writeOwner` could not write one (a full or read-only filesystem). Removing the
+	// directory blind is the same cross-holder deletion from the other side, so leave it: the stale
+	// window is a few seconds and whoever is waiting breaks it themselves.
+	if (!token) return;
+	let current: string;
+	try {
+		current = fs.readFileSync(path.join(lockPath, "owner"), "utf8");
+	} catch {
+		// We wrote a token and it is gone, so this directory is no longer the one we created:
+		// someone broke our lock as stale and has not written their token yet. Removing it here
+		// would delete their lock and reopen the very race the token exists to close.
+		return;
+	}
+	if (current !== token) return; // someone broke our lock and took it: theirs to release
+	try {
+		fs.rmSync(lockPath, { recursive: true, force: true });
+	} catch {
+		/* the stale window will break it */
+	}
+}
+
+/**
+ * Synchronous sibling of `withFileLock`, for the append paths that must stay synchronous
+ * (`Inbox.append` is called from hook callbacks that cannot await). The critical section is a
+ * single `appendFileSync`, so the spin never lasts more than a few milliseconds.
+ */
+export function withFileLockSync<T>(lockPath: string, fn: () => T, opts: LockOptions = {}): T {
+	const staleMs = opts.staleMs ?? 10_000;
+	// Longer than `staleMs` on purpose: a lock left by a process killed between mkdir and rm can
+	// only be broken after it goes stale, and a shorter deadline would spin and then throw instead.
+	const deadline = Date.now() + (opts.timeoutMs ?? staleMs + 5_000);
+	fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+	const spin = new Int32Array(new SharedArrayBuffer(4));
+	for (;;) {
+		try {
+			fs.mkdirSync(lockPath);
+			break;
+		} catch (err: any) {
+			if (err?.code !== "EEXIST") throw err;
+			// The deadline is checked on every path: a lock directory we can neither stat nor remove
+			// (EACCES, or a Windows EPERM) must not turn into an event-loop-blocking hot spin.
+			if (Date.now() > deadline) throw new Error(`timed out waiting for lock ${lockPath}`);
+			try {
+				if (Date.now() - fs.statSync(lockPath).mtimeMs > staleMs) {
+					fs.rmSync(lockPath, { recursive: true, force: true });
+					continue;
+				}
+			} catch {
+				/* vanished, or unreadable: wait and try again */
+			}
+			Atomics.wait(spin, 0, 0, 5 + Math.floor(Math.random() * 10));
+		}
+	}
+	const token = writeOwner(lockPath);
+	try {
+		return fn();
+	} finally {
+		releaseOwned(lockPath, token);
+	}
+}
+
+/** Run `fn` while holding `<lockPath>` (a directory created with mkdir, which is atomic). */
+export async function withFileLock<T>(lockPath: string, fn: () => Promise<T> | T, opts: LockOptions = {}): Promise<T> {
+	const staleMs = opts.staleMs ?? 10_000;
+	// Longer than `staleMs`, for the same reason as the synchronous sibling: a lock left behind by
+	// a killed process can only be broken once it goes stale, and a shorter deadline would give up
+	// just before that moment and throw instead.
+	const timeoutMs = opts.timeoutMs ?? staleMs + 5_000;
+	const deadline = Date.now() + timeoutMs;
+	fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+	for (;;) {
+		try {
+			fs.mkdirSync(lockPath);
+			break;
+		} catch (err: any) {
+			if (err?.code !== "EEXIST") throw err;
+			// The deadline is checked on every path, as in the synchronous sibling: a lock path that
+			// exists for mkdir and not for stat — a dangling symlink is the way this happens — would
+			// otherwise loop with neither a sleep nor a way out, at 100 % of a core, forever.
+			if (Date.now() > deadline) throw new Error(`timed out waiting for lock ${lockPath}`);
+			try {
+				const age = Date.now() - fs.statSync(lockPath).mtimeMs;
+				if (age > staleMs) {
+					fs.rmSync(lockPath, { recursive: true, force: true });
+					continue;
+				}
+			} catch {
+				/* vanished between EEXIST and stat, or unreadable: wait and try again */
+			}
+			await sleep(20 + Math.floor(Math.random() * 30));
+		}
+	}
+	const token = writeOwner(lockPath);
+	try {
+		return await fn();
+	} finally {
+		releaseOwned(lockPath, token);
+	}
+}
+
+/** Write via temp file + rename so readers never see a torn file. */
+export function writeFileAtomic(file: string, data: string): void {
+	fs.mkdirSync(path.dirname(file), { recursive: true });
+	const tmp = `${file}.${process.pid}.${Date.now().toString(36)}.tmp`;
+	// fsync before the rename: without it a crash can leave the rename applied and the data not,
+	// i.e. an empty file where the store used to be. The directory is synced too so the rename
+	// itself survives. Both are best-effort — a filesystem that refuses them is not a reason to fail.
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(tmp, "w");
+		fs.writeFileSync(fd, data, "utf8");
+		try {
+			fs.fsyncSync(fd);
+		} catch {
+			/* not supported here */
+		}
+	} catch (err) {
+		try {
+			fs.rmSync(tmp, { force: true });
+		} catch {
+			/* nothing more to do */
+		}
+		throw err;
+	} finally {
+		if (fd !== undefined) fs.closeSync(fd);
+	}
+	try {
+		fs.renameSync(tmp, file);
+	} catch (err) {
+		// ENOSPC/EACCES here leaves the temp behind; on a full disk that is one abandoned file per
+		// process per tick, eating inodes long after the write itself was reported and handled.
+		try {
+			fs.rmSync(tmp, { force: true });
+		} catch {
+			/* nothing more to do */
+		}
+		throw err;
+	}
+	try {
+		const dir = fs.openSync(path.dirname(file), "r");
+		try {
+			fs.fsyncSync(dir);
+		} finally {
+			fs.closeSync(dir);
+		}
+	} catch {
+		/* directory fsync is not portable; the file fsync above is the important half */
+	}
+}
+
+export function pidAlive(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (err: any) {
+		return err?.code === "EPERM";
+	}
+}

package/src/log.ts

@@ -0,0 +1,108 @@
+/**
+ * A file every process writes its diagnostics to.
+ *
+ * Everything except the headless host went to `ctx.ui.notify`, which is not written to the
+ * session file — so `/new`, `/resume` or a crash erased every warning the automation had produced.
+ * A loop that failed at 03:00 left nothing behind to read at 09:00.
+ *
+ * Rotated in place past `MAX_BYTES`, like the run log and the audit.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { pidAlive } from "./lock.ts";
+import { capRedacted } from "./redact.ts";
+import { stamp } from "./schedule.ts";
+
+export const MAX_LOG_BYTES = 2_000_000;
+
+/**
+ * What one log line may say, in characters — the same bound `hooks.ts` puts on a hook's captured
+ * output, for the same reason: the text usually comes from something else's stdout, and a diagnostic
+ * nobody bounded is how a 2 MB log becomes one line.
+ */
+export const MAX_LOG_LINE_CHARS = 4000;
+
+/**
+ * Halve a log file in place once it passes `maxBytes`. One home for "rotated at 2 MB": the headless
+ * host writes its own file (its stdout and stderr go there too, so an MCP server's chatter lands in
+ * it) and used to carry a second copy of this, free to drift from the rule every other log follows.
+ */
+export function rotateInPlace(file: string, maxBytes: number = MAX_LOG_BYTES): void {
+	try {
+		if (fs.statSync(file).size < maxBytes) return;
+		const lines = fs.readFileSync(file, "utf8").split("\n").filter(Boolean);
+		const keep = Math.floor(lines.length / 2);
+		// One line can be over the limit on its own, and `slice(-0)` is the whole array — so without
+		// this the file would be rewritten identical forever, and every later write would pay a full
+		// read and write of it. The same guard the run log and the inbox carry.
+		fs.writeFileSync(file, keep === 0 ? "" : `${lines.slice(-keep).join("\n")}\n`);
+	} catch {
+		/* best effort */
+	}
+}
+
+export type LogLevel = "info" | "warn" | "error";
+
+export class LoopsLog {
+	readonly file: string;
+	private failed = false;
+
+	constructor(dir: string, name: string) {
+		this.file = path.join(dir, "logs", name);
+	}
+
+	write(level: LogLevel, message: string): void {
+		if (this.failed) return; // a log that cannot be written must never become the loudest problem
+		const line = `${stamp()} ${level.padEnd(5)} ${capRedacted(message, MAX_LOG_LINE_CHARS)}\n`;
+		try {
+			fs.mkdirSync(path.dirname(this.file), { recursive: true });
+			fs.appendFileSync(this.file, line);
+			rotateInPlace(this.file);
+		} catch {
+			this.failed = true;
+		}
+	}
+
+	info(message: string): void {
+		this.write("info", message);
+	}
+	warn(message: string): void {
+		this.write("warn", message);
+	}
+	error(message: string): void {
+		this.write("error", message);
+	}
+
+	/** The last `n` lines, for `/cron scheduler` and a future bug report. */
+	tail(n = 40): string[] {
+		try {
+			return fs.readFileSync(this.file, "utf8").split("\n").filter(Boolean).slice(-n);
+		} catch {
+			return [];
+		}
+	}
+
+}
+
+/**
+ * Old per-process logs are not interesting once their process is gone; keep the newest few. A log
+ * whose pid is still running is never deleted, however old it looks: a headless host can sit idle
+ * for days while newer sessions come and go, and its log is the one a user goes looking for.
+ */
+export function pruneLogs(dir: string, keep = 5): void {
+	const logsDir = path.join(dir, "logs");
+	try {
+		const files = fs
+			.readdirSync(logsDir)
+			.filter((f) => f.startsWith("pi-") && f.endsWith(".log"))
+			.map((f) => ({ f, mtime: fs.statSync(path.join(logsDir, f)).mtimeMs }))
+			.sort((a, b) => b.mtime - a.mtime);
+		for (const { f } of files.slice(keep)) {
+			const pid = Number(f.slice(3, -4));
+			if (Number.isFinite(pid) && pid > 0 && pidAlive(pid)) continue;
+			fs.rmSync(path.join(logsDir, f), { force: true });
+		}
+	} catch {
+		/* nothing to prune */
+	}
+}

package/src/mcp-pool.ts

@@ -0,0 +1,141 @@
+/**
+ * Project-level MCP servers for runs outside this process's own project.
+ *
+ * A pi process loads `<its cwd>/.pi/mcp.toml` once and shares those clients. But a loop belongs to
+ * a project, not to whichever pi owns the clock: a job in project B run by a pi open in project A
+ * used to get A's servers and never B's, and the headless host (no project at all) got none. A
+ * single-project process never has this problem — one process, one cwd, one project; this pool is
+ * what gives a run its own project's servers when the two differ. It connects a project's servers
+ * lazily, keyed by directory, and hands their tools to runs in that directory.
+ *
+ * User-level servers are not pooled: those are shared and already connected by the process.
+ */
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { McpSource, loadProjectMcpConfig, mcpToolDefinitions } from "./mcp.ts";
+import { previewRedacted } from "./redact.ts";
+
+export interface McpPoolOptions {
+	/** Whether project-local config in `cwd` may be loaded at all (pi's saved trust decisions). */
+	isTrusted: (cwd: string) => boolean;
+	/** Where an untrusted project or a failed connection is reported (one line, not one per run). */
+	log?: (message: string) => void;
+	resolveToken?: (ref: string) => string | undefined;
+}
+
+interface Entry {
+	sources: McpSource[];
+	defs: Array<{ name: string; def: ToolDefinition<any, any>; server: string }>;
+	ready: Promise<void>;
+	lastUsed: number;
+}
+
+/** Projects kept connected at once; the least recently used is dropped past this. */
+const MAX_POOLED_PROJECTS = 8;
+
+export class McpPool {
+	private readonly entries = new Map<string, Entry>();
+	/** Projects we already said were untrusted: say it once, not once per run. */
+	private readonly warned = new Set<string>();
+	private readonly opts: McpPoolOptions;
+	constructor(opts: McpPoolOptions) {
+		this.opts = opts;
+	}
+
+	/**
+	 * The project-level MCP tools for a run in `cwd`, connecting that project's servers on first use.
+	 * `taken` is the tool names already in use for that run (built-ins, user-level MCP tools, the
+	 * automation tools), so a project's server cannot shadow one of them.
+	 */
+	async toolsFor(cwd: string, taken: Set<string>): Promise<ToolDefinition<any, any>[]> {
+		if (!cwd) return [];
+		// Trust is re-checked every time: a decision the user revoked must stop lending servers,
+		// not keep the ones an earlier run happened to connect.
+		if (!this.opts.isTrusted(cwd)) {
+			const stale = this.entries.get(cwd);
+			if (stale) {
+				this.entries.delete(cwd);
+				void Promise.all(stale.sources.map((s) => s.stop())).catch(() => undefined);
+			}
+			if (!this.warned.has(cwd)) {
+				this.warned.add(cwd);
+				this.opts.log?.(`project MCP config in ${cwd} ignored: project is not trusted`);
+			}
+			return [];
+		}
+		const entry = this.entries.get(cwd) ?? this.connect(cwd);
+		entry.lastUsed = Date.now();
+		await entry.ready;
+		// Names are settled per run: a project tool that collides with a built-in or a user-level
+		// server's tool is offered under `<server>_<tool>` rather than dropped, exactly as the
+		// interactive path does — a project exposing `read` used to silently lose it here.
+		const out: ToolDefinition<any, any>[] = [];
+		for (const { name, def, server } of entry.defs) {
+			const unique = taken.has(name) ? `${server}_${name}` : name;
+			if (taken.has(unique)) {
+				this.opts.log?.(`tool ${name} from ${server} (${cwd}) is not available: both ${name} and ${unique} are taken`);
+				continue;
+			}
+			taken.add(unique);
+			out.push(unique === name ? def : { ...def, name: unique });
+		}
+		return out;
+	}
+
+	private connect(cwd: string): Entry {
+		const entry: Entry = { sources: [], defs: [], ready: Promise.resolve(), lastUsed: Date.now() };
+		this.warned.delete(cwd);
+		this.evictIfFull();
+		this.entries.set(cwd, entry);
+		const loaded = loadProjectMcpConfig(cwd);
+		for (const d of loaded.diagnostics) this.opts.log?.(d);
+		if (!loaded.servers.length) return entry;
+		const waits: Promise<void>[] = [];
+		for (const cfg of loaded.servers) {
+			let settle!: () => void;
+			waits.push(new Promise<void>((r) => (settle = r)));
+			const source = new McpSource(cfg, {
+				onConnected: async (src) => {
+					try {
+						const tools = await src.listTools();
+						const taken = new Set(entry.defs.map((d) => d.name));
+						entry.defs.push(...mcpToolDefinitions(src, tools, taken, []).map((d) => ({ ...d, server: cfg.name })));
+						this.opts.log?.(`mcp ${cfg.name} (${cwd}): ${tools.length} tool(s)`);
+					} catch (err: any) {
+						src.status.lastError = `tools/list failed: ${err?.message ?? err}`;
+					} finally {
+						settle();
+					}
+				},
+				// A project server's pushes belong to the process that owns that project's rules;
+				// the pool exists to lend tools to a run, not to open a second notification path.
+				onNotification: () => undefined,
+				resolveToken: this.opts.resolveToken,
+				log: (msg) => this.opts.log?.(`mcp ${cfg.name} (${cwd}): ${previewRedacted(msg, 200)}`),
+			});
+			entry.sources.push(source);
+			source.start();
+			// Never hold a run hostage to a server that will not come up.
+			const timer = setTimeout(settle, 10_000);
+			timer.unref?.();
+		}
+		entry.ready = Promise.all(waits).then(() => undefined);
+		return entry;
+	}
+
+	/** Keep the pool bounded: one loop per project on a busy machine must not pin every project. */
+	private evictIfFull(): void {
+		while (this.entries.size >= MAX_POOLED_PROJECTS) {
+			const oldest = [...this.entries.entries()].sort((a, b) => a[1].lastUsed - b[1].lastUsed)[0];
+			if (!oldest) return;
+			this.entries.delete(oldest[0]);
+			this.opts.log?.(`disconnected ${oldest[0]}'s MCP servers (pool limit ${MAX_POOLED_PROJECTS})`);
+			void Promise.all(oldest[1].sources.map((s) => s.stop())).catch(() => undefined);
+		}
+	}
+
+	async stopAll(): Promise<void> {
+		const sources = [...this.entries.values()].flatMap((e) => e.sources);
+		this.entries.clear();
+		await Promise.all(sources.map((s) => s.stop()));
+	}
+}