@alphacoder-v0/pi-loops 0.22.8 → 0.24.1

package/src/host-control.ts

@@ -1,116 +1,23 @@
 /**
- * The headless host: a `node` process running `src/host.ts` that keeps the clock (loops,
- * trigger checks, MCP pushes) while no interactive pi is open on this machine. The last
- * interactive pi to quit starts it; the first to open takes the clock back and the host exits.
- * `host.json` under the loops dir names the running host; `host.log` is its output.
+ * The always-on host: a `node` process running `src/host.ts` that runs every loop, rule check and
+ * MCP push for one loops directory. A pi window starts it when there is automation to run
+ * (`ensureHost`), and it never yields to one. `host-service.json` names the running host;
+ * `host.log` is its output.
  */
 import { execFileSync, spawn } from "node:child_process";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { pidAlive, writeFileAtomic } from "./lock.ts";
-import type { PresenceEntry } from "./presence.ts";
-import { stamp } from "./schedule.ts";
+import { pidAlive, withFileLockSync, writeFileAtomic } from "./lock.ts";
+import { procStart } from "./lease.ts";
+import { compareVersions } from "./pi-floor.ts";
+import { askHost, type StopReason } from "./host-control-channel.ts";
+import { formatLocal, stamp } from "./schedule.ts";
+import { previewRedacted } from "./redact.ts";
+import { readUiPrefs } from "./ui-prefs.ts";
 
-export interface HostRecord {
-	pid: number;
-	host: string;
-	startedAt: string;
-	node: string;
-	/** The host entry as it appears in the process's argv, for an exact command-line match. */
-	entry?: string;
-}
-
-export const HOST_FILE = "host.json";
 export const HOST_LOG = "host.log";
 
-export function readHost(dir: string): HostRecord | undefined {
-	try {
-		const rec = JSON.parse(fs.readFileSync(path.join(dir, HOST_FILE), "utf8")) as HostRecord;
-		return rec && typeof rec.pid === "number" ? rec : undefined;
-	} catch {
-		return undefined;
-	}
-}
-
-/**
- * Is `pid` really our host? A record can outlive its process (SIGKILL, power loss) and the pid
- * be recycled, so besides liveness we require the record to postdate the last boot and, where
- * /proc exists, the command line to be the host entry.
- */
-export function hostProcessMatches(rec: HostRecord): boolean {
-	if (!pidAlive(rec.pid)) return false;
-	const booted = Date.now() - os.uptime() * 1000;
-	if (Number.isFinite(booted) && Date.parse(rec.startedAt) < booted - 60_000) return false;
-	const isOurs = (argv: string[]) => (rec.entry ? argv.includes(rec.entry) : argv.some((a) => a.endsWith("host.ts")));
-	try {
-		return isOurs(fs.readFileSync(`/proc/${rec.pid}/cmdline`, "utf8").split("\0"));
-	} catch {
-		/* no /proc (macOS): ask ps */
-	}
-	try {
-		const command = execFileSync("ps", ["-o", "command=", "-p", String(rec.pid)], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
-		return isOurs(command.trim().split(/\s+/));
-	} catch {
-		return true; // liveness + boot time is all we can check
-	}
-}
-
-/** The host process on this machine, if it is alive. A stale record is removed. */
-export function liveHost(dir: string): HostRecord | undefined {
-	const rec = readHost(dir);
-	if (!rec) return undefined;
-	if (rec.host !== os.hostname()) return undefined;
-	if (hostProcessMatches(rec)) return rec;
-	fs.rmSync(path.join(dir, HOST_FILE), { force: true });
-	return undefined;
-}
-
-/** A record whose process is gone means the host crashed (it removes the record on a clean exit). */
-export function crashedHost(dir: string): HostRecord | undefined {
-	const rec = readHost(dir);
-	if (!rec || rec.host !== os.hostname()) return undefined;
-	if (hostProcessMatches(rec)) return undefined;
-	fs.rmSync(path.join(dir, HOST_FILE), { force: true });
-	return rec;
-}
-
-export function writeHostRecord(dir: string, rec: HostRecord): void {
-	writeFileAtomic(path.join(dir, HOST_FILE), `${JSON.stringify(rec, null, 2)}\n`);
-}
-
-export function clearHostRecord(dir: string, pid?: number): void {
-	const rec = readHost(dir);
-	if (pid !== undefined && rec && rec.pid !== pid) return;
-	fs.rmSync(path.join(dir, HOST_FILE), { force: true });
-}
-
-/** SIGTERM the host (it stops its scheduler and exits); returns the pid or undefined when none ran. */
-export function stopHost(dir: string): number | undefined {
-	const rec = liveHost(dir);
-	if (!rec) return undefined;
-	try {
-		process.kill(rec.pid, "SIGTERM");
-	} catch {
-		/* already gone */
-	}
-	fs.rmSync(path.join(dir, HOST_FILE), { force: true });
-	return rec.pid;
-}
-
-/**
- * Hand the clock to a headless host when the last interactive pi on this machine quits and
- * there is something for it to keep running. Pure, so the decision is testable.
- */
-export function shouldHandOff(input: { auto: boolean; presence: PresenceEntry[]; selfPid: number; selfInstance?: string; hostName: string; enabledLoops: number; enabledRules: number; pushServers: number; hostAlive: boolean }): { handOff: boolean; reason: string } {
-	if (!input.auto) return { handOff: false, reason: "[host] auto = false" };
-	if (input.hostAlive) return { handOff: false, reason: "a host is already running" };
-	const others = input.presence.filter((e) => e.host === input.hostName && e.kind !== "host" && !(e.pid === input.selfPid && (!input.selfInstance || e.instance === input.selfInstance)));
-	if (others.length) return { handOff: false, reason: `${others.length} other interactive pi still open` };
-	if (!input.enabledLoops && !input.enabledRules && !input.pushServers) return { handOff: false, reason: "nothing to keep running" };
-	return { handOff: true, reason: `${input.enabledLoops} loop(s), ${input.enabledRules} rule(s), ${input.pushServers} push source(s)` };
-}
-
 /**
  * MCP servers the host can do something with: a push either injects (inbox) or is evaluated
  * against rules — a tool-only server with no rules is nothing to stay up for.
@@ -151,25 +58,89 @@ export interface SpawnHostOptions {
 	dir: string;
 	packageDir: string;
 	piPackage?: string;
+	/** Who starts it, said by `pi-loops status`: its environment is theirs. Default: this process. */
+	startedBy?: string;
 	/** The handing-off pi's model and thinking level: the host's defaults for unpinned work. */
 	model?: string;
 	thinking?: string;
 	node?: string;
 	nodeVersion?: string;
+	/** A person started it by name: with `[host] auto = false` it stays until stopped, not only while a window is open. */
+	asked?: boolean;
+}
+
+/** An executable `node` in one of `pathEnv`'s directories, for a pi that is not running under node itself. */
+export function nodeOnPath(pathEnv: string = process.env.PATH ?? "", isExecutable: (file: string) => boolean = (file) => {
+	try {
+		fs.accessSync(file, fs.constants.X_OK);
+		return fs.statSync(file).isFile();
+	} catch {
+		return false; // not there, or not ours to run
+	}
+}, cwd: string = process.cwd()): string | undefined {
+	// Absolute entries only: a relative one (`.`, `node_modules/.bin`) resolves against the project pi
+	// was opened in, and would run a binary from an untrusted checkout. So would an absolute one that
+	// `bun run`, `npx` or direnv put there: anything inside that project, or under a node_modules.
+	// The project is the repository around cwd (its nearest `.git` below home), compared by real path so
+	// a symlink into it counts. Not when pi was opened in the home directory or at the root: that is
+	// no project, and node itself (nvm, fnm, a local prefix) lives under home.
+	const real = (p: string): string => {
+		try {
+			return fs.realpathSync(p);
+		} catch {
+			return path.resolve(p); // not there: compared as written
+		}
+	};
+	const project = projectRoot(real(cwd), real(os.homedir()));
+	const projects = (d: string) => d.split(path.sep).includes("node_modules") || (project !== undefined && (d === project || d.startsWith(project + path.sep)));
+	return pathEnv.split(path.delimiter).filter((d) => d && path.isAbsolute(d) && !projects(real(d))).map((d) => path.join(d, "node")).find(isExecutable);
+}
+
+/** The repository `dir` is in (its nearest ancestor with `.git`, below home), else `dir`; none for home or the root. */
+function projectRoot(dir: string, home: string): string | undefined {
+	if (dir === home || dir === path.parse(dir).root) return undefined;
+	for (let at = dir; at !== home && at !== path.parse(at).root; at = path.dirname(at)) {
+		if (fs.existsSync(path.join(at, ".git"))) return at;
+	}
+	return dir;
+}
+
+/**
+ * The command line that runs the host: node (the one on PATH under bun), pi's module resolver, the
+ * entry, and the environment it needs. `spawnHost` starts it detached; `pi-loops start --foreground`
+ * runs it in the terminal, for a service manager the person set up.
+ */
+export function hostCommand(opts: SpawnHostOptions): { node: string; args: string[]; env: NodeJS.ProcessEnv } {
+	let node = opts.node ?? process.execPath;
+	let nodeVersion = opts.nodeVersion ?? process.version;
+	// Under bun the running binary is not node; the host needs node, so the one on PATH runs it.
+	if ((process.versions as any).bun && !opts.node) {
+		const onPath = nodeOnPath();
+		if (!onPath) throw new Error("pi-loops needs node in the background, and pi is running under bun with no node on PATH");
+		// The file that is probed is the file that is started: a shim or symlink is resolved once.
+		const found = fs.realpathSync(onPath);
+		const version = execFileSync(found, ["--version"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000, cwd: os.homedir() }).trim();
+		if (!/^v\d+\./.test(version)) throw new Error(`pi-loops needs node in the background, and ${onPath} does not answer like node (${previewRedacted(version, 40)})`);
+		node = found;
+		nodeVersion = version;
+	}
+	return {
+		node,
+		args: hostSpawnArgs(nodeVersion, path.join(opts.packageDir, "src", "register-pi.mjs"), path.join(opts.packageDir, "src", "host-entry.mjs")),
+		env: { ...process.env, PI_LOOPS_STARTED_BY: opts.startedBy ?? `pid ${process.pid}`, PI_LOOPS_DIR: path.resolve(opts.dir), ...(opts.piPackage ? { PI_LOOPS_PI_PACKAGE: opts.piPackage } : {}), ...(opts.model ? { PI_LOOPS_HOST_MODEL: opts.model } : {}), ...(opts.thinking ? { PI_LOOPS_HOST_THINKING: opts.thinking } : {}), ...(opts.asked ? { PI_LOOPS_HOST_ASKED: "1" } : {}) },
+	};
 }
 
 /** Start the host detached, logging to `host.log`; returns its pid. */
 export function spawnHost(opts: SpawnHostOptions): number {
-	if ((process.versions as any).bun) throw new Error("the headless host needs node (pi is running under bun)");
-	const node = opts.node ?? process.execPath;
-	const args = hostSpawnArgs(opts.nodeVersion ?? process.version, path.join(opts.packageDir, "src", "register-pi.mjs"), path.join(opts.packageDir, "src", "host-entry.mjs"));
+	const { node, args, env } = hostCommand(opts);
 	fs.mkdirSync(opts.dir, { recursive: true });
 	const log = fs.openSync(path.join(opts.dir, HOST_LOG), "a");
 	const child = spawn(node, args, {
 		detached: true,
 		stdio: ["ignore", log, log],
 		cwd: os.homedir(),
-		env: { ...process.env, PI_LOOPS_DIR: opts.dir, ...(opts.piPackage ? { PI_LOOPS_PI_PACKAGE: opts.piPackage } : {}), ...(opts.model ? { PI_LOOPS_HOST_MODEL: opts.model } : {}), ...(opts.thinking ? { PI_LOOPS_HOST_THINKING: opts.thinking } : {}) },
+		env,
 	});
 	fs.closeSync(log);
 	// A spawn failure surfaces on the next tick; without a listener it would crash the quitting pi.
@@ -181,7 +152,7 @@ export function spawnHost(opts: SpawnHostOptions): number {
 		}
 	});
 	child.unref();
-	if (!child.pid) throw new Error("could not start the headless host");
+	if (!child.pid) throw new Error("could not start pi-loops");
 	// The host writes host.json itself, under a lock, once it knows no other host is running.
 	return child.pid;
 }
@@ -195,9 +166,529 @@ export function spawnHost(opts: SpawnHostOptions): number {
 export async function waitForHost(dir: string, pid: number, timeoutMs = 3000): Promise<boolean> {
 	const deadline = Date.now() + timeoutMs;
 	while (Date.now() < deadline) {
-		if (readHost(dir)?.pid === pid) return true;
+		if (readHostService(dir)?.pid === pid) return true;
 		if (!pidAlive(pid)) return false;
 		await new Promise((r) => setTimeout(r, 100));
 	}
-	return readHost(dir)?.pid === pid;
+	return readHostService(dir)?.pid === pid;
+}
+
+export const HOST_SERVICE_FILE = "host-service.json";
+export const HOST_SERVICE_LOCK = "host-service.lock";
+export const HOST_SPAWNS_FILE = "host-spawns.json";
+
+export interface HostServiceRecord {
+	v: 1;
+	pid: number;
+	/** OS process start time (src/lease.ts procStart), against pid recycling. */
+	procStart?: string;
+	startedAt: string;
+	/** PI_LOOPS_VERSION of the running host. */
+	version: string;
+	piVersion?: string;
+	/** argv entry, for the command-line match hostProcessMatches already does. */
+	entry: string;
+	node: string;
+	/** The loops directory this host serves. */
+	dir: string;
+	socket: string;
+	/** The browser page's address, without its token, once its port is bound. */
+	url?: string;
+	/** The page's token (`web-token`): `pi-loops status --url` prints the address with it. */
+	token: string;
+	/** Why the page is not served yet: an older pi-loops page holds the port, or another program does. */
+	pageWaiting?: { reason: "older-page" | "other-program"; since: string };
+	/** Written by the host on every tick: its loop is turning, whatever its socket says (see decideEviction). */
+	tickAt?: string;
+	/** "foreground": `pi-loops start --foreground`, under a service manager the person set up. */
+	managed: "spawned" | "foreground";
+}
+
+export function readHostService(dir: string): HostServiceRecord | undefined {
+	try {
+		const value: unknown = JSON.parse(fs.readFileSync(path.join(dir, HOST_SERVICE_FILE), "utf8"));
+		if (!value || typeof value !== "object") return undefined;
+		const rec = value as HostServiceRecord;
+		return rec.v === 1 && typeof rec.pid === "number" ? rec : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function commandMatches(pid: number, entry: string): boolean {
+	const isOurs = (argv: string[]) => argv.includes(entry);
+	try {
+		return isOurs(fs.readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0"));
+	} catch {
+		/* no /proc (macOS): ask ps */
+	}
+	try {
+		// ps joins argv with spaces, so an install path with a space in it cannot be split back out:
+		// look for the entry as a whole word of the line instead.
+		const command = ` ${execFileSync("ps", ["-o", "command=", "-p", String(pid)], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim()} `;
+		return command.includes(` ${entry} `);
+	} catch {
+		return true; // liveness is all we can check
+	}
+}
+
+export function hostServiceAlive(rec: HostServiceRecord): boolean {
+	if (!pidAlive(rec.pid)) return false;
+	// A start time that cannot be read now (a `ps` that failed to start) is unknown, not a different
+	// process: as `holderAlive` takes it, with its fallback. Read as a mismatch, one failed `ps` had a
+	// window remove a healthy host's record and count a crash; trusted alone, a pid reused since a
+	// reboot could be sent SIGKILL. A record written before this boot names some other process.
+	const started = rec.procStart === undefined ? undefined : procStart(rec.pid);
+	if (started !== undefined && started !== rec.procStart) return false;
+	if (started === undefined && Date.parse(rec.startedAt) < Date.now() - os.uptime() * 1000 - 60_000) return false;
+	return commandMatches(rec.pid, rec.entry);
+}
+
+export function claimHostService(dir: string, rec: HostServiceRecord, alive: (r: HostServiceRecord) => boolean = hostServiceAlive): { claimed: true } | { claimed: false; other: HostServiceRecord } {
+	return withFileLockSync(path.join(dir, HOST_SERVICE_LOCK), () => {
+		const other = readHostService(dir);
+		if (other && other.pid !== rec.pid && alive(other)) return { claimed: false, other };
+		// A record naming a dead host is one that never got to its clean exit: counted here when the new
+		// host is the first to find it (a window that found it first removed it and counted it).
+		if (other && other.pid !== rec.pid) recordCrash(dir, `pi-loops (pid ${other.pid}) went away without a clean exit; see ${HOST_LOG}`);
+		const file = path.join(dir, HOST_SERVICE_FILE);
+		writeFileAtomic(file, `${JSON.stringify(rec, null, 2)}\n`, 0o600);
+		fs.chmodSync(file, 0o600);
+		return { claimed: true };
+	});
+}
+
+export function removeHostService(dir: string, pid: number): boolean {
+	return withFileLockSync(path.join(dir, HOST_SERVICE_LOCK), () => {
+		if (readHostService(dir)?.pid !== pid) return false;
+		fs.rmSync(path.join(dir, HOST_SERVICE_FILE), { force: true });
+		return true;
+	});
+}
+
+/**
+ * Add what the host learns after its claim (the page's URL, why it waits for its port, the heartbeat)
+ * to its own record, and only to its own. A key given as `undefined` is removed.
+ */
+export function updateHostService(dir: string, pid: number, patch: Partial<Pick<HostServiceRecord, "url" | "tickAt" | "pageWaiting">>): boolean {
+	return withFileLockSync(path.join(dir, HOST_SERVICE_LOCK), () => {
+		const rec = readHostService(dir);
+		if (rec?.pid !== pid) return false;
+		const file = path.join(dir, HOST_SERVICE_FILE);
+		writeFileAtomic(file, `${JSON.stringify({ ...rec, ...patch }, null, 2)}\n`, 0o600);
+		fs.chmodSync(file, 0o600);
+		return true;
+	});
+}
+
+export function hostServiceNames(dir: string, pid: number, startedAt: string): boolean {
+	const rec = readHostService(dir);
+	return rec?.pid === pid && rec.startedAt === startedAt;
+}
+
+export type SpawnOutcome = "up" | "died" | "no-node" | "evicted" | "stopped";
+export interface HostSpawnLog {
+	attempts: Array<{ at: string; by: number; outcome?: SpawnOutcome }>;
+	notBefore?: string;
+	lastError?: string;
+	/**
+	 * When a host that had started went away without a clean exit: it crashed (and said so first), a
+	 * window found its record naming a dead pid, or a window evicted it. The crash budget: three within
+	 * `CRASH_BUDGET.windowMs` pause the host (`pauseEndsAt`), instead of a restart every tick that
+	 * aborts every run in flight each time.
+	 */
+	crashes?: string[];
+	/** Set when the crash budget ran out. The pause ends on its own (`pauseEndsAt`), or when a person starts the host (`clearCrashes`). */
+	pausedAt?: string;
+	/** Pauses in a row, each twice as long as the one before: a host that crashes again as soon as it is back waits longer. */
+	pauses?: number;
+	/** The last pid a window evicted, so several windows evicting it count one crash. */
+	evictedPid?: number;
+	/** A newer pi-loops asked the host to make way: older clients do not start their own while it lives. */
+	upgradedBy?: { pid: number; version: string; procStart?: string };
+}
+
+/**
+ * Crashes inside this window that pause the host. Sized to the respawn cadence: a pi window restarts
+ * a dead host on its 30 s tick, so three back-to-back crashes take about 90 s, and a 60 s window
+ * could never catch them.
+ */
+export const CRASH_BUDGET = { count: 3, windowMs: 5 * 60_000 };
+
+/**
+ * How long a crash-pause lasts: half an hour, doubled for each pause in a row, a day at most. A pause
+ * that lasted until a person came back stopped every project's loops for one bad one, overnight; a
+ * pause that ends lets them run again between the crashes.
+ */
+export const CRASH_PAUSE = { firstMs: 30 * 60_000, maxMs: 24 * 60 * 60_000 };
+
+/** When the pause in `log` ends, if it is paused. */
+export function pauseEndsAt(log: Pick<HostSpawnLog, "pausedAt" | "pauses">): number | undefined {
+	if (!log.pausedAt) return undefined;
+	const n = Math.max(1, log.pauses ?? 1);
+	return Date.parse(log.pausedAt) + Math.min(CRASH_PAUSE.firstMs * 2 ** (n - 1), CRASH_PAUSE.maxMs);
+}
+
+/** What a crash-pause is, in the words every surface uses for it. */
+export function crashPauseText(log: Pick<HostSpawnLog, "pausedAt" | "pauses">): string {
+	const ends = pauseEndsAt(log);
+	return `pi-loops stopped unexpectedly ${CRASH_BUDGET.count} times within ${CRASH_BUDGET.windowMs / 60_000} minutes — see ${HOST_LOG}; it tries again at ${ends === undefined ? "?" : formatLocal(ends)}, or /pi-loops start resumes it now`;
+}
+
+/** Whether the host is paused at `now` by its crash budget. */
+export function crashPaused(log: Pick<HostSpawnLog, "pausedAt" | "pauses">, now: number): boolean {
+	const ends = pauseEndsAt(log);
+	return ends !== undefined && now < ends;
+}
+
+/** How many crashes count against the budget at `now`; the host is paused when that reaches the count. */
+export function recentCrashes(log: HostSpawnLog, now: number): number {
+	return (log.crashes ?? []).filter((at) => now - Date.parse(at) < CRASH_BUDGET.windowMs).length;
+}
+
+/** Record a host going away uncleanly, with why. Synchronous and locked: the crashing host calls it on its way out. */
+export function recordCrash(dir: string, reason: string, now = Date.now()): HostSpawnLog {
+	return withFileLockSync(path.join(dir, "host-spawns.lock"), () => {
+		const log = readSpawnLog(dir);
+		log.crashes = [...(log.crashes ?? []), stamp(now)].slice(-10);
+		log.lastError = reason;
+		const ended = pauseEndsAt(log);
+		if (recentCrashes(log, now) >= CRASH_BUDGET.count && (ended === undefined || now >= ended)) {
+			// In a row: the last pause ended within a day. Later than that, this is a new run of crashes.
+			log.pauses = ended !== undefined && now - ended < CRASH_PAUSE.maxMs ? (log.pauses ?? 1) + 1 : 1;
+			log.pausedAt = stamp(now);
+		}
+		writeFileAtomic(path.join(dir, HOST_SPAWNS_FILE), `${JSON.stringify(log, null, 2)}\n`);
+		return log;
+	});
+}
+
+/** True for the first caller to evict `pid`: the eviction is counted once, however many windows made it. */
+function claimEviction(dir: string, pid: number): boolean {
+	return withFileLockSync(path.join(dir, "host-spawns.lock"), () => {
+		const log = readSpawnLog(dir);
+		if (log.evictedPid === pid) return false;
+		log.evictedPid = pid;
+		writeFileAtomic(path.join(dir, HOST_SPAWNS_FILE), `${JSON.stringify(log, null, 2)}\n`);
+		return true;
+	});
+}
+
+/** A person started the host by name: the crash budget and any upgrade claim start over. */
+export function clearCrashes(dir: string): void {
+	withFileLockSync(path.join(dir, "host-spawns.lock"), () => {
+		const log = readSpawnLog(dir);
+		if (!log.crashes?.length && !log.upgradedBy && !log.pausedAt) return;
+		delete log.crashes;
+		delete log.pausedAt;
+		delete log.pauses;
+		delete log.upgradedBy;
+		writeFileAtomic(path.join(dir, HOST_SPAWNS_FILE), `${JSON.stringify(log, null, 2)}\n`);
+	});
+}
+
+/** Record which newer pi-loops asked the host to make way (see `HostSpawnLog.upgradedBy`). */
+export function recordUpgrader(dir: string, upgrader: { pid: number; version: string; procStart?: string } | undefined): void {
+	withFileLockSync(path.join(dir, "host-spawns.lock"), () => {
+		const log = readSpawnLog(dir);
+		if (upgrader) log.upgradedBy = upgrader;
+		else if (!log.upgradedBy) return;
+		else delete log.upgradedBy;
+		writeFileAtomic(path.join(dir, HOST_SPAWNS_FILE), `${JSON.stringify(log, null, 2)}\n`);
+	});
+}
+
+export function readSpawnLog(dir: string): HostSpawnLog {
+	try {
+		const value: unknown = JSON.parse(fs.readFileSync(path.join(dir, HOST_SPAWNS_FILE), "utf8"));
+		const log = value as HostSpawnLog;
+		return log && Array.isArray(log.attempts) ? log : { attempts: [] };
+	} catch {
+		return { attempts: [] };
+	}
+}
+
+export function recordSpawn(dir: string, event: { kind: "attempt"; by: number } | { kind: "outcome"; by: number; outcome: SpawnOutcome; error?: string }, now = Date.now()): HostSpawnLog {
+	return withFileLockSync(path.join(dir, "host-spawns.lock"), () => {
+		const log = readSpawnLog(dir);
+		if (event.kind === "attempt") log.attempts.push({ at: stamp(now), by: event.by });
+		else {
+			let attempt = [...log.attempts].reverse().find((a) => a.by === event.by && a.outcome === undefined);
+			if (!attempt) {
+				attempt = { at: stamp(now), by: event.by };
+				log.attempts.push(attempt);
+			}
+			attempt.outcome = event.outcome;
+			if (event.error !== undefined) log.lastError = event.error;
+			let failures = 0;
+			for (const a of [...log.attempts].reverse()) {
+				if (a.outcome === "evicted") continue;
+				if (a.outcome === "died" || a.outcome === "no-node") failures++;
+				else break;
+			}
+			if (!failures) {
+				delete log.notBefore;
+				if (event.outcome === "up") delete log.lastError;
+			} else log.notBefore = stamp(now + (failures === 1 ? 60_000 : failures === 2 ? 300_000 : 1_800_000));
+		}
+		log.attempts = log.attempts.slice(-20);
+		writeFileAtomic(path.join(dir, HOST_SPAWNS_FILE), `${JSON.stringify(log, null, 2)}\n`);
+		return log;
+	});
+}
+
+export interface EnsureInput {
+	/** `closedSessionJobs`: enabled plain jobs of conversations no window holds, which the host fires. */
+	automation: { loops: number; rules: number; pushServers: number; closedSessionJobs?: number; closedMailboxes?: number };
+	auto: boolean;
+	stopped: boolean;
+	envOff: boolean;
+	now: number;
+	spawns: HostSpawnLog;
+	host?: { version: string; piVersion?: string };
+	myVersion: string;
+	myPiVersion?: string;
+	/** A person asked for the host by name (`/cron host start`, `pi-loops host start`): start it even with nothing to run, `[host] auto = false` or a backoff pending. */
+	asked?: boolean;
+	/** `pi-loops` opening the browser page: the page is work to start for, and nothing more (no `asked`, so `[host] auto = false` still ends it with the last window). */
+	page?: true;
+	/** The newer pi-loops that last asked a host to make way, and whether its process is alive. */
+	upgrader?: { pid: number; version: string; alive: boolean };
+}
+
+export type EnsureAction = { kind: "none"; reason: string } | { kind: "spawn" } | { kind: "upgrade" };
+
+export function decideEnsure(input: EnsureInput): EnsureAction {
+	if (input.envOff) return { kind: "none", reason: "PI_LOOPS_HOST=off" };
+	if (input.host) {
+		const cmp = compareVersions(input.myVersion, input.host.version);
+		if (cmp > 0 || (cmp === 0 && input.myPiVersion !== undefined && input.host.piVersion !== undefined && compareVersions(input.myPiVersion, input.host.piVersion) > 0)) return { kind: "upgrade" };
+		return { kind: "none", reason: "running" };
+	}
+	const work = input.automation;
+	if (!input.asked && !input.page && !work.loops && !work.rules && !work.pushServers && !work.closedSessionJobs && !work.closedMailboxes) return { kind: "none", reason: "nothing to run" };
+	// The newer client that asked the old host to make way starts its own; an older one starting the
+	// old code in the gap is how two versions traded the host back and forth, aborting runs each time.
+	if (input.upgrader?.alive && compareVersions(input.myVersion, input.upgrader.version) < 0) return { kind: "none", reason: `a newer pi-loops (pid ${input.upgrader.pid}) is already running` };
+	if (!input.asked && crashPaused(input.spawns, input.now)) return { kind: "none", reason: `paused: ${crashPauseText(input.spawns)}` };
+	// `[host] auto = false` does not keep a window from starting it: it makes the process leave once
+	// no window is open (`leavesWithWindows`), as 0.22.8 ran loops only while a pi was open.
+	if (input.stopped) return { kind: "none", reason: "stopped (/pi-loops start resumes it)" };
+	if (!input.asked && !input.page && input.spawns.notBefore && input.now < Date.parse(input.spawns.notBefore)) return { kind: "none", reason: `backing off until ${input.spawns.notBefore}` };
+	if (input.spawns.attempts.some((a) => a.outcome === undefined && input.now - Date.parse(a.at) < 10_000 && input.now >= Date.parse(a.at))) return { kind: "none", reason: "a start is in progress" };
+	return { kind: "spawn" };
+}
+
+/** How long a window-started process waits for the window that started it to announce itself before it may leave for want of one. */
+export const WINDOWS_GRACE_MS = 60_000;
+
+/**
+ * Pure. Whether the background process leaves because no pi window is open: `[host] auto = false`
+ * means what it meant in 0.22.8, loops run only while a window is open. The first window starts the
+ * process (as `auto = true` does) and the last one closing ends it; nothing runs with none open. One a
+ * service manager runs (`pi-loops start --foreground`) is the person's to stop and stays. `windows`
+ * is the live pi windows this machine's presence lists; `upForMs` gives the window that started the
+ * process its first tick to be listed.
+ */
+export function leavesWithWindows(input: { auto: boolean; managed: HostServiceRecord["managed"]; windows: number; upForMs: number; asked?: boolean }): boolean {
+	return !input.auto && !input.asked && input.managed !== "foreground" && input.windows === 0 && input.upForMs >= WINDOWS_GRACE_MS;
+}
+
+/**
+ * Pure. The windows `[host] auto = false` waits for: terminal windows that hold a lease, the page's
+ * pis, and tabs on the page. A page pi still starting, or one a terminal took its conversation from,
+ * holds no lease, so the pool lists it; one that holds a lease is counted once.
+ */
+export function openWindows(input: { terminalPids: Iterable<number>; pageChildPids: Iterable<number>; tabs: number }): number {
+	return new Set([...input.terminalPids, ...input.pageChildPids]).size + input.tabs;
+}
+
+/**
+ * Whether three unanswered probes mean a wedged host. Only with a second signal: the host writes a
+ * heartbeat into its record every tick, so a socket that does not answer while the heartbeat is
+ * fresh is a host that runs with a broken control channel (a removed socket, a temp-dir cleaner) —
+ * killing that one only to start another with the same broken channel looped every ~100 s.
+ */
+export function decideEviction(input: { failures: number; probeOk: boolean; fromTick: boolean; pidMatches: boolean; heartbeatFresh?: boolean }): { failures: number; evict: boolean; channelBroken?: boolean } {
+	if (input.probeOk) return { failures: 0, evict: false };
+	if (!input.fromTick) return { failures: input.failures, evict: false };
+	const failures = input.failures + 1;
+	if (failures < 3 || !input.pidMatches) return { failures, evict: false };
+	if (input.heartbeatFresh) return { failures, evict: false, channelBroken: true };
+	return { failures: 0, evict: true };
+}
+
+/** A heartbeat younger than three ticks is a host whose loop still turns. */
+export function heartbeatFresh(rec: Pick<HostServiceRecord, "tickAt">, now: number, tickMs = 30_000): boolean {
+	return !!rec.tickAt && now - Date.parse(rec.tickAt) < 3 * tickMs;
+}
+
+export async function evictHost(dir: string, pid: number, opts: { graceMs?: number; kill?: (pid: number, signal: NodeJS.Signals) => void; alive?: (pid: number) => boolean } = {}): Promise<void> {
+	const kill = opts.kill ?? ((target, signal) => process.kill(target, signal));
+	const alive = opts.alive ?? pidAlive;
+	try {
+		kill(pid, "SIGTERM");
+	} catch (err: any) {
+		if (err?.code !== "ESRCH") throw err;
+		/* ESRCH means it exited before the signal arrived. */
+	}
+	const deadline = Date.now() + (opts.graceMs ?? 10_000);
+	while (alive(pid) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 100));
+	if (alive(pid)) {
+		try {
+			kill(pid, "SIGKILL");
+		} catch (err: any) {
+			if (err?.code !== "ESRCH") throw err;
+			/* ESRCH means it exited before the signal arrived. */
+		}
+	}
+	removeHostService(dir, pid);
+	// Every window counts its own unanswered probes, so several evict one wedged process together:
+	// the first to get here counts the crash, once per pid. Who removed the record says nothing, since
+	// a process that still turns removes its own on SIGTERM.
+	if (!claimEviction(dir, pid)) return;
+	recordSpawn(dir, { kind: "outcome", by: process.pid, outcome: "evicted" });
+	recordCrash(dir, `pi-loops (pid ${pid}) stopped answering and its heartbeat stopped; it was replaced`);
+}
+
+/**
+ * How long a restart or an upgrade lets a turn in the browser page finish before the process leaves
+ * anyway (`waitForPageTurns`). A plain stop and a signal do not wait.
+ */
+export const PAGE_TURN_WAIT_MS = 120_000;
+
+/**
+ * How long the side that asked for a restart or an upgrade waits for the old process to go: the
+ * page's turns, then the shutdown, which gives up on its own after 10 s, and room to spare. A
+ * restart that gives up sends SIGKILL, so this must outlast `PAGE_TURN_WAIT_MS`.
+ */
+export const LEAVE_WAIT_MS = PAGE_TURN_WAIT_MS + 60_000;
+
+/** What `pi-loops restart` and `/pi-loops restart` say before they wait. */
+export const RESTART_WAIT_LINE = `waiting up to ${PAGE_TURN_WAIT_MS / 60_000} min for a turn in the browser page to finish…`;
+
+/** Whether a stop waits for the page's turns: a restart and an upgrade do, a person's plain stop does not. */
+export function stopWaitsForPage(reason: StopReason): boolean {
+	return reason === "restart" || reason === "upgrade";
+}
+
+/**
+ * Wait until no conversation in the page is busy (a turn or a compaction), at most `boundMs`. Says
+ * once that it waits and once if it gives up; "idle" when nothing was busy or everything finished.
+ */
+export async function waitForPageTurns(opts: { busy: () => number; log: (line: string) => void; boundMs?: number; pollMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void> }): Promise<"idle" | "gave up"> {
+	const boundMs = opts.boundMs ?? PAGE_TURN_WAIT_MS;
+	const pollMs = opts.pollMs ?? 500;
+	const now = opts.now ?? Date.now;
+	const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+	let busy = opts.busy();
+	if (!busy) return "idle";
+	opts.log(`waiting up to ${Math.round(boundMs / 1000)} s for ${busy === 1 ? "a turn" : `${busy} turns`} in the browser page to finish`);
+	const deadline = now() + boundMs;
+	while (now() < deadline) {
+		await sleep(Math.min(pollMs, Math.max(0, deadline - now())));
+		busy = opts.busy();
+		if (!busy) return "idle";
+	}
+	opts.log(`gave up waiting: ${busy === 1 ? "a turn" : `${busy} turns`} in the browser page still running after ${Math.round(boundMs / 1000)} s; leaving`);
+	return "gave up";
+}
+
+export interface EnsureHostOptions extends SpawnHostOptions {
+	automation: EnsureInput["automation"];
+	auto: boolean;
+	stopped: boolean;
+	myVersion: string;
+	myPiVersion?: string;
+	asked?: boolean;
+	page?: true;
+	/** How long an upgrade waits for the old process to leave; `LEAVE_WAIT_MS` unless a test says. */
+	leaveWaitMs?: number;
+	/**
+	 * Whether a start waits for the process to record itself (up to 10 s). Every caller waits but a
+	 * window that is quitting: it has nothing to do with the answer, and waiting made quitting slow.
+	 * Not waiting leaves the spawn log's attempt open, which the next window reads as a start in
+	 * progress for 10 s, and `up` undefined.
+	 */
+	wait?: boolean;
+}
+
+/**
+ * Make sure the host is running when there is automation for it: start it, ask an older one to make
+ * way for this version, or leave it alone (src/host-control.ts decideEnsure decides which). Never
+ * throws: a host that cannot start is a result, recorded in the spawn log with its backoff.
+ */
+export async function ensureHost(opts: EnsureHostOptions): Promise<{ action: EnsureAction; pid?: number; up?: boolean; error?: string }> {
+	let rec = readHostService(opts.dir);
+	// A clean exit removes the record, so one naming a dead process is a host that crashed, was killed
+	// (SIGKILL, OOM) or lost power: counted against the crash budget once, then cleared.
+	if (rec && !hostServiceAlive(rec)) {
+		// Only the caller that removes it counts it: several windows find the same dead record at once.
+		if (removeHostService(opts.dir, rec.pid)) recordCrash(opts.dir, `pi-loops (pid ${rec.pid}) went away without a clean exit; see ${HOST_LOG}`);
+		rec = undefined;
+	}
+	const spawns = readSpawnLog(opts.dir);
+	const action = decideEnsure({
+		automation: opts.automation,
+		auto: opts.auto,
+		stopped: opts.stopped,
+		envOff: process.env.PI_LOOPS_HOST?.toLowerCase() === "off",
+		now: Date.now(),
+		spawns,
+		host: rec ? { version: rec.version, piVersion: rec.piVersion } : undefined,
+		myVersion: opts.myVersion,
+		myPiVersion: opts.myPiVersion,
+		asked: opts.asked,
+		page: opts.page,
+		upgrader: spawns.upgradedBy ? { ...spawns.upgradedBy, alive: spawns.upgradedBy.pid === process.pid || (pidAlive(spawns.upgradedBy.pid) && (spawns.upgradedBy.procStart === undefined || procStart(spawns.upgradedBy.pid) === spawns.upgradedBy.procStart)) } : undefined,
+	});
+	if (action.kind === "upgrade") {
+		recordUpgrader(opts.dir, { pid: process.pid, version: opts.myVersion, procStart: procStart(process.pid) });
+		await askHost(opts.dir, { op: "upgrade" });
+		// Wait for it to go and start this version at once: the gap until the next tick is where an
+		// older window used to start the old code again. It may first let a turn in the page finish.
+		const old = rec!.pid;
+		const deadline = Date.now() + (opts.leaveWaitMs ?? LEAVE_WAIT_MS);
+		while (Date.now() < deadline) {
+			const now = readHostService(opts.dir);
+			if (!now || now.pid !== old || !hostServiceAlive(now)) break;
+			await new Promise((r) => setTimeout(r, 100));
+		}
+		const still = readHostService(opts.dir);
+		if (still && hostServiceAlive(still)) return { action };
+		const started = await startHost(opts, action);
+		// This version's host is up: the claim that held older clients back has done its job. Only then:
+		// the one that answers can still be the old host, slow to leave, and the claim must outlive it.
+		const now = readHostService(opts.dir);
+		if (started.up && now && compareVersions(now.version, opts.myVersion) >= 0) recordUpgrader(opts.dir, undefined);
+		return started;
+	}
+	if (action.kind !== "spawn") return { action };
+	if (opts.asked) clearCrashes(opts.dir);
+	return startHost(opts, action);
+}
+
+/** Start the host and, unless `opts.wait` is false, wait for it to record itself. */
+async function startHost(opts: EnsureHostOptions, action: EnsureAction): Promise<{ action: EnsureAction; pid?: number; up?: boolean; error?: string }> {
+	recordSpawn(opts.dir, { kind: "attempt", by: process.pid });
+	let pid: number;
+	try {
+		pid = spawnHost(opts);
+	} catch (err: any) {
+		const error = err?.message ?? String(err);
+		recordSpawn(opts.dir, { kind: "outcome", by: process.pid, outcome: "no-node", error });
+		return { action, error };
+	}
+	if (opts.wait === false) return { action, pid };
+	const up = await waitForHost(opts.dir, pid, 10_000);
+	// A second host that lost the claim exits 0 at once: it is not a failed start, the winner is up.
+	const winner = readHostService(opts.dir);
+	const ok = up || (!!winner && hostServiceAlive(winner));
+	// One that found the stop mark a person set meanwhile left as asked (src/host.ts): not a failure to back off from.
+	if (!ok && readUiPrefs(opts.dir).host === "stopped") {
+		recordSpawn(opts.dir, { kind: "outcome", by: process.pid, outcome: "stopped" });
+		return { action, pid, up: false, error: "pi-loops was stopped by hand while it started" };
+	}
+	recordSpawn(opts.dir, { kind: "outcome", by: process.pid, outcome: ok ? "up" : "died", error: ok ? undefined : `pi-loops (pid ${pid}) exited during startup; see ${HOST_LOG}` });
+	return { action, pid, up: ok };
 }