oh-my-opencode 5.0.0-beta.5 → 5.0.0-beta.51

package/packages/lsp-core/src/lsp/controlled-clock-test-support.ts

@@ -0,0 +1,61 @@
+import type { TimerProvider } from "./timer-provider.js";
+
+interface ControlledTimer {
+	readonly at: number;
+	readonly callback: () => void;
+	cancelled: boolean;
+}
+
+interface TimerWaiter {
+	readonly delayMs: number | undefined;
+	readonly resolve: () => void;
+}
+
+/**
+ * Deterministic TimerProvider for tests: timers fire only through advanceBy(),
+ * and waitForTimer(delayMs) resolves once a timer with exactly that delay is
+ * scheduled, so tests order themselves behind the timer they intend to fire.
+ */
+export class ControlledClock implements TimerProvider {
+	private readonly timers: ControlledTimer[] = [];
+	private currentTime = 0;
+	private readonly timerWaiters = new Set<TimerWaiter>();
+	readonly scheduledDelays: number[] = [];
+
+	readonly now = (): number => this.currentTime;
+	readonly setTimeout = (callback: () => void, delayMs: number): ReturnType<typeof setTimeout> => {
+		const timer: ControlledTimer = { at: this.currentTime + delayMs, callback, cancelled: false };
+		this.scheduledDelays.push(delayMs);
+		this.timers.push(timer);
+		for (const waiter of this.timerWaiters) {
+			if (waiter.delayMs === undefined || waiter.delayMs === delayMs) {
+				this.timerWaiters.delete(waiter);
+				waiter.resolve();
+			}
+		}
+		return timer as unknown as ReturnType<typeof setTimeout>;
+	};
+	readonly clearTimeout = (handle: ReturnType<typeof setTimeout>): void => {
+		(handle as unknown as { cancelled: boolean }).cancelled = true;
+	};
+
+	waitForTimer(delayMs?: number): Promise<void> {
+		const hasTimer = this.timers.some(
+			(timer) => !timer.cancelled && (delayMs === undefined || timer.at === this.currentTime + delayMs),
+		);
+		if (hasTimer) return Promise.resolve();
+		return new Promise((resolve) => {
+			this.timerWaiters.add({ delayMs, resolve });
+		});
+	}
+
+	advanceBy(delayMs: number): void {
+		this.currentTime += delayMs;
+		for (;;) {
+			const index = this.timers.findIndex((timer) => timer.at <= this.currentTime);
+			if (index < 0) return;
+			const timer = this.timers.splice(index, 1)[0];
+			if (timer && !timer.cancelled) timer.callback();
+		}
+	}
+}

package/packages/lsp-core/src/lsp/errors.ts

@@ -69,6 +69,21 @@ export class LspProcessSpawnError extends Error {
 	override readonly name = "LspProcessSpawnError";
 }
 
+export class LspClientRespawnBudgetExceededError extends Error {
+	override readonly name = "LspClientRespawnBudgetExceededError";
+
+	constructor(
+		readonly serverId: string,
+		readonly root: string,
+		readonly retryLimit: number,
+	) {
+		super(
+			`LSP server ${serverId} at ${root} failed to stay alive; respawn budget exhausted ` +
+				`(${retryLimit} consecutive dead generations). Retrying after the cooldown may succeed.`,
+		);
+	}
+}
+
 export function isLspDeadConnectionError(err: unknown): err is LspConnectionClosedError | LspProcessExitedError {
 	return err instanceof LspConnectionClosedError || err instanceof LspProcessExitedError;
 }

package/packages/lsp-core/src/lsp/fixtures/workspace-edit-server.mjs

@@ -8,6 +8,7 @@ let nextServerRequestId = 10_000;
 let renameIndex = 0;
 let diagnosticIndex = 0;
 const pendingServerRequests = new Map();
+const releasedDiagnosticResponses = [];
 
 function record(event) {
 	appendFileSync(eventsPath, `${JSON.stringify(event)}\n`, "utf-8");
@@ -146,8 +147,7 @@ function handleRequest(message) {
 			setTimeout(() => process.exit(0), delay);
 			return;
 		}
-		const delay = Number(step?.delayMs ?? 0);
-		setTimeout(() => {
+		const sendDiagnosticResponse = () => {
 			if (step?.error) {
 				record({ type: "serverError", method: "textDocument/diagnostic", error: step.error });
 				sendError(message.id, step.error.code, step.error.message);
@@ -156,7 +156,13 @@ function handleRequest(message) {
 			const result = step?.report ?? { items: scenario.diagnostics ?? [] };
 			record({ type: "serverResponse", method: "textDocument/diagnostic", result });
 			sendResponse(message.id, result);
-		}, delay);
+		};
+		if (step?.releaseOnDidChange === true) {
+			releasedDiagnosticResponses.push(sendDiagnosticResponse);
+		} else {
+			const delay = Number(step?.delayMs ?? 0);
+			setTimeout(sendDiagnosticResponse, delay);
+		}
 		return;
 	}
 	if (message.method === "shutdown") {
@@ -190,6 +196,7 @@ function handleNotification(message) {
 	}
 	if (message.method === "textDocument/didChange") {
 		applyPublishSteps("didChange", message.params.textDocument.uri, message.params.textDocument.version);
+		for (const release of releasedDiagnosticResponses.splice(0)) release();
 	}
 	if (message.method === "exit") process.exit(0);
 }

package/packages/lsp-core/src/lsp/format-document.test.ts

@@ -0,0 +1,143 @@
+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { afterEach, describe, expect, it } from "bun:test";
+
+import type { LspClient } from "./client.js";
+import { formatDocumentWithClient } from "./format-document.js";
+import type { TextEdit } from "./types.js";
+
+const workspaces: string[] = [];
+
+afterEach(() => {
+	for (const workspace of workspaces.splice(0)) rmSync(workspace, { recursive: true, force: true });
+});
+
+function workspaceWith(fileName: string, content: string): { readonly root: string; readonly filePath: string } {
+	const root = mkdtempSync(join(tmpdir(), "lsp-format-"));
+	workspaces.push(root);
+	const filePath = join(root, fileName);
+	writeFileSync(filePath, content, "utf-8");
+	return { root, filePath };
+}
+
+function stubClient(edits: TextEdit[] | null): LspClient {
+	return {
+		async formatDocument() {
+			return edits;
+		},
+		async openFile() {},
+	} as Pick<LspClient, "formatDocument" | "openFile"> as LspClient;
+}
+
+describe("formatDocumentWithClient", () => {
+	it("#given a server returning text edits #when the document is formatted #then the file is rewritten and line deltas are reported", async () => {
+		// given
+		const { filePath } = workspaceWith("sample.ts", "const a=1\nconst b   =2\nconst c=3\n");
+		const edits: TextEdit[] = [
+			{ range: { start: { line: 0, character: 0 }, end: { line: 1, character: 12 } }, newText: "const a = 1;\nconst b = 2;" },
+		];
+
+		// when
+		const result = await formatDocumentWithClient(stubClient(edits), filePath);
+
+		// then
+		expect(result.status).toBe("formatted");
+		expect(readFileSync(filePath, "utf-8")).toBe("const a = 1;\nconst b = 2;\nconst c=3\n");
+		expect(result).toMatchObject({ status: "formatted", linesAdded: 2, linesRemoved: 2 });
+	});
+
+	it("#given a server returning an edit that adds lines #when the document is formatted #then the added and removed line counts differ", async () => {
+		// given
+		const { filePath } = workspaceWith("sample.ts", "const a=1;const b=2\n");
+		const edits: TextEdit[] = [
+			{ range: { start: { line: 0, character: 0 }, end: { line: 0, character: 19 } }, newText: "const a = 1;\nconst b = 2;" },
+		];
+
+		// when
+		const result = await formatDocumentWithClient(stubClient(edits), filePath);
+
+		// then
+		expect(result).toMatchObject({ status: "formatted", linesAdded: 2, linesRemoved: 1 });
+		expect(readFileSync(filePath, "utf-8")).toBe("const a = 1;\nconst b = 2;\n");
+	});
+
+	it("#given a server that does not advertise document formatting #when the document is formatted #then the result is unavailable and the file bytes are untouched", async () => {
+		// given
+		const original = "const a=1\n";
+		const { filePath } = workspaceWith("sample.ts", original);
+		const before = statSync(filePath);
+
+		// when
+		const result = await formatDocumentWithClient(stubClient(null), filePath);
+
+		// then
+		expect(result).toEqual({ status: "unavailable", reason: "capability_not_advertised" });
+		expect(readFileSync(filePath)).toEqual(Buffer.from(original));
+		expect(statSync(filePath).mtimeMs).toBe(before.mtimeMs);
+	});
+
+	it("#given a server returning no edits #when the document is formatted #then the result is unchanged and the file bytes are untouched", async () => {
+		// given
+		const original = "const a = 1;\n";
+		const { filePath } = workspaceWith("sample.ts", original);
+
+		// when
+		const result = await formatDocumentWithClient(stubClient([]), filePath);
+
+		// then
+		expect(result).toEqual({ status: "unchanged", linesAdded: 0, linesRemoved: 0 });
+		expect(readFileSync(filePath)).toEqual(Buffer.from(original));
+	});
+
+	it("#given edits that reproduce the current bytes #when the document is formatted #then the result is unchanged", async () => {
+		// given
+		const original = "const a = 1;\n";
+		const { filePath } = workspaceWith("sample.ts", original);
+		const edits: TextEdit[] = [
+			{ range: { start: { line: 0, character: 0 }, end: { line: 0, character: 12 } }, newText: "const a = 1;" },
+		];
+
+		// when
+		const result = await formatDocumentWithClient(stubClient(edits), filePath);
+
+		// then
+		expect(result.status).toBe("unchanged");
+		expect(readFileSync(filePath, "utf-8")).toBe(original);
+	});
+
+	it("#given the file shrinks after the server computed its edits #when the edits are applied #then the stale edit is rejected instead of corrupting the file", async () => {
+		// given
+		const { filePath } = workspaceWith("sample.ts", "const a=1\nconst b=2\n");
+		const staleEdits: TextEdit[] = [
+			{ range: { start: { line: 1, character: 0 }, end: { line: 1, character: 9 } }, newText: "const b = 2;" },
+		];
+		const racingClient = {
+			async formatDocument() {
+				writeFileSync(filePath, "short\n", "utf-8");
+				return staleEdits;
+			},
+			async openFile() {},
+		} as Pick<LspClient, "formatDocument" | "openFile"> as LspClient;
+
+		// when / then
+		await expect(formatDocumentWithClient(racingClient, filePath)).rejects.toThrow(/outside line/);
+		expect(readFileSync(filePath, "utf-8")).toBe("short\n");
+	});
+
+	it("#given a formatted document #when the write completes #then no temporary file is left behind", async () => {
+		// given
+		const { root, filePath } = workspaceWith("sample.ts", "const a=1\n");
+		const edits: TextEdit[] = [
+			{ range: { start: { line: 0, character: 0 }, end: { line: 0, character: 9 } }, newText: "const a = 1;" },
+		];
+
+		// when
+		await formatDocumentWithClient(stubClient(edits), filePath);
+
+		// then
+		expect(readFileSync(filePath, "utf-8")).toBe("const a = 1;\n");
+		expect(() => statSync(join(root, "sample.ts.tmp"))).toThrow();
+	});
+});

package/packages/lsp-core/src/lsp/format-document.ts

@@ -0,0 +1,90 @@
+import { readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+
+import type { LspClient } from "./client.js";
+import type { FormattingOptions, TextEdit } from "./types.js";
+import { normalizeTextEdits } from "./workspace-edit-text.js";
+
+export const DEFAULT_FORMATTING_OPTIONS: FormattingOptions = {
+	tabSize: 4,
+	insertSpaces: false,
+	trimTrailingWhitespace: true,
+	insertFinalNewline: true,
+	trimFinalNewlines: true,
+};
+
+export type FormatDocumentResult =
+	| { readonly status: "formatted"; readonly linesAdded: number; readonly linesRemoved: number }
+	| { readonly status: "unchanged"; readonly linesAdded: 0; readonly linesRemoved: 0 }
+	| { readonly status: "unavailable"; readonly reason: "capability_not_advertised" };
+
+export interface FormatDocumentOptions {
+	readonly formattingOptions?: FormattingOptions;
+	readonly signal?: AbortSignal;
+}
+
+const UNCHANGED: FormatDocumentResult = { status: "unchanged", linesAdded: 0, linesRemoved: 0 };
+
+/**
+ * Formats one document through the resident language server and commits the result to disk.
+ *
+ * The server is asked for whole-document edits rather than formatted text, so the edits are
+ * validated and applied with the same normalization the workspace-edit path uses. Nothing is
+ * written when the server declines the capability or when the edits reproduce the current bytes,
+ * which keeps a no-op format from touching file mtimes that downstream tooling watches.
+ */
+export async function formatDocumentWithClient(
+	client: LspClient,
+	filePath: string,
+	options: FormatDocumentOptions = {},
+): Promise<FormatDocumentResult> {
+	const edits = await client.formatDocument(
+		filePath,
+		options.formattingOptions ?? DEFAULT_FORMATTING_OPTIONS,
+		options.signal,
+	);
+	if (edits === null) return { status: "unavailable", reason: "capability_not_advertised" };
+	if (edits.length === 0) return UNCHANGED;
+
+	const before = readFileSync(filePath, "utf-8");
+	const normalized = normalizeTextEdits(before, edits, 0);
+	if (normalized.text === before) return UNCHANGED;
+
+	writeAtomically(filePath, normalized.text);
+	await client.openFile(filePath);
+
+	return {
+		status: "formatted",
+		...lineDelta(normalized.edits),
+	};
+}
+
+/**
+ * Counts the lines each edit removes from the document and adds back in its place.
+ *
+ * Counting from the edits instead of diffing the whole file keeps the report proportional to what
+ * the server actually rewrote, so an unrelated line never shows up as churn.
+ */
+function lineDelta(edits: readonly TextEdit[]): { readonly linesAdded: number; readonly linesRemoved: number } {
+	let linesAdded = 0;
+	let linesRemoved = 0;
+	for (const edit of edits) {
+		linesRemoved += edit.range.end.line - edit.range.start.line + 1;
+		linesAdded += edit.newText.split("\n").length;
+	}
+	return { linesAdded, linesRemoved };
+}
+
+function writeAtomically(filePath: string, content: string): void {
+	const tempPath = `${filePath}.omo-format.tmp`;
+	writeFileSync(tempPath, content, "utf-8");
+	try {
+		renameSync(tempPath, filePath);
+	} catch (error) {
+		try {
+			unlinkSync(tempPath);
+		} catch {
+			// The rename failure is the actionable error; a leftover temp file must not mask it.
+		}
+		throw error;
+	}
+}

package/packages/lsp-core/src/lsp/json-rpc-connection.ts

@@ -73,6 +73,7 @@ export class JsonRpcConnection {
 		let settled = false;
 		const writeCancel = (): Promise<void> => this.writeMessage({ jsonrpc: "2.0", method: "$/cancelRequest", params: { id } });
 
+		let rejectAfterCancelWrite: (() => Promise<void>) | undefined;
 		const responsePromise = new Promise<T>((resolve, reject) => {
 			const cleanup = (): void => {
 				options.signal?.removeEventListener("abort", onAbort);
@@ -83,15 +84,19 @@ export class JsonRpcConnection {
 				this.pendingRequests.delete(key);
 				cleanup();
 				const rejectCancelled = (): void => reject(abortError(options.signal));
+				rejectAfterCancelWrite = async (): Promise<void> => {
+					try {
+						await writeCancel();
+					} catch (error) {
+						this.emitError(toError(error));
+					}
+					rejectCancelled();
+				};
 				if (!requestWritten) {
 					cancelAfterWrite = true;
-					rejectCancelled();
 					return;
 				}
-				void writeCancel().then(rejectCancelled, (error) => {
-					this.emitError(toError(error));
-					rejectCancelled();
-				});
+				void rejectAfterCancelWrite();
 			};
 			const onAbort = (): void => settleCancel();
 			this.pendingRequests.set(key, {
@@ -118,7 +123,7 @@ export class JsonRpcConnection {
 		try {
 			await this.writeMessage(message);
 			requestWritten = true;
-			if (cancelAfterWrite) await writeCancel();
+			if (cancelAfterWrite) await rejectAfterCancelWrite?.();
 		} catch (error) {
 			if (settled) return responsePromise;
 			const pending = this.pendingRequests.get(key);

package/packages/lsp-core/src/lsp/manager-lifecycle.test.ts

@@ -0,0 +1,262 @@
+import { describe, expect, it } from "bun:test";
+
+import type { LspClient } from "./client.js";
+import { LspManager } from "./manager.js";
+import type { ResolvedServer } from "./types.js";
+
+const SERVER: ResolvedServer = {
+	id: "typescript",
+	command: ["typescript-language-server", "--stdio"],
+	extensions: [".ts"],
+	priority: 1,
+};
+
+/** Simulates a stop that lingers (production stops reach ~21s), which is the behavior under test. */
+const SLOW_STOP_MS = 50;
+
+interface RecordingClientOptions {
+	readonly stopDelayMs: number;
+	readonly initializeError?: Error;
+	readonly alive: boolean;
+}
+
+class RecordingFakeClient {
+	alive: boolean;
+	stopCallCount = 0;
+
+	constructor(
+		readonly root: string,
+		private readonly log: EventLog,
+		private readonly options: RecordingClientOptions,
+	) {
+		this.alive = options.alive;
+	}
+
+	async start(): Promise<void> {
+		this.log.push(`spawn:${this.root}`);
+	}
+
+	async initialize(): Promise<void> {
+		if (this.options.initializeError !== undefined) {
+			throw this.options.initializeError;
+		}
+	}
+
+	isAlive(): boolean {
+		return this.alive;
+	}
+
+	command(): string[] {
+		return [...SERVER.command];
+	}
+
+	async stop(): Promise<void> {
+		this.stopCallCount += 1;
+		this.log.push(`stop-start:${this.root}`);
+		await delay(this.options.stopDelayMs);
+		this.alive = false;
+		this.log.push(`stop-end:${this.root}`);
+	}
+}
+
+class EventLog {
+	readonly entries: string[] = [];
+	private waiters: Array<{ readonly event: string; readonly resolve: () => void }> = [];
+
+	push(event: string): void {
+		this.entries.push(event);
+		const due = this.waiters.filter((waiter) => waiter.event === event);
+		if (due.length === 0) return;
+		this.waiters = this.waiters.filter((waiter) => waiter.event !== event);
+		for (const waiter of due) waiter.resolve();
+	}
+
+	waitFor(event: string): Promise<void> {
+		if (this.entries.includes(event)) return Promise.resolve();
+		return new Promise<void>((resolve) => {
+			this.waiters.push({ event, resolve });
+		});
+	}
+}
+
+function delay(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+interface HarnessOptions {
+	readonly maxResidentClients?: number;
+	readonly stopDelayMs?: number;
+	/** Leading generations whose initialize() rejects; later generations start cleanly. */
+	readonly initializeFailures?: number;
+	/** Generations at or above this index report alive; earlier ones emulate a crash-looping server. */
+	readonly aliveFromGeneration?: number;
+}
+
+function managerHarness(options: HarnessOptions = {}): {
+	readonly manager: LspManager;
+	readonly clients: RecordingFakeClient[];
+	readonly log: EventLog;
+	readonly clock: { value: number };
+} {
+	const clients: RecordingFakeClient[] = [];
+	const log = new EventLog();
+	const clock = { value: 1_000 };
+	const generations = new Map<string, number>();
+	const aliveFrom = options.aliveFromGeneration ?? 1;
+	const stopDelayMs = options.stopDelayMs ?? 0;
+	const initializeFailures = options.initializeFailures ?? 0;
+	const manager = new LspManager({
+		maxResidentClients: options.maxResidentClients ?? 6,
+		reaperIntervalMs: 60_000,
+		idleTimeoutMs: 60_000_000,
+		now: () => clock.value,
+		clientFactory: (root) => {
+			const generation = (generations.get(root) ?? 0) + 1;
+			generations.set(root, generation);
+			const client = new RecordingFakeClient(root, log, {
+				stopDelayMs,
+				alive: generation >= aliveFrom,
+				...(generation <= initializeFailures ? { initializeError: new Error("initialize boom") } : {}),
+			});
+			clients.push(client);
+			return client as unknown as LspClient;
+		},
+	});
+	return { manager, clients, log, clock };
+}
+
+describe("LspManager pending-stop tombstones", () => {
+	it("#given a slow-stopping client evicted at the resident cap #when the evicted root is re-acquired while the stop is pending #then the replacement spawns only after the old client stopped", async () => {
+		// given
+		const { manager, log } = managerHarness({ maxResidentClients: 1, stopDelayMs: SLOW_STOP_MS });
+		try {
+			await manager.getClient("/root-a", SERVER);
+			manager.releaseClient("/root-a", SERVER.id);
+
+			// when: admitting root-b synchronously evicts idle root-a (void stop), and root-a is
+			// re-acquired in the same window while the evicted client's stop is still pending.
+			const rootBPromise = manager.getClient("/root-b", SERVER);
+			const rootAPromise = manager.getClient("/root-a", SERVER);
+			await Promise.all([rootBPromise, rootAPromise]);
+			manager.releaseClient("/root-b", SERVER.id);
+			manager.releaseClient("/root-a", SERVER.id);
+
+			// then
+			expect(log.entries.filter((event) => event.endsWith("/root-a"))).toEqual([
+				"spawn:/root-a",
+				"stop-start:/root-a",
+				"stop-end:/root-a",
+				"spawn:/root-a",
+			]);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+
+	it("#given an invalidated client whose stop is still pending #when the same root is re-acquired #then the replacement spawns only after the invalidated client stopped", async () => {
+		// given
+		const { manager, log } = managerHarness({ stopDelayMs: SLOW_STOP_MS });
+		try {
+			await manager.getClient("/root-a", SERVER);
+			manager.releaseClient("/root-a", SERVER.id);
+
+			// when
+			manager.invalidateClient("/root-a", SERVER.id);
+			await manager.getClient("/root-a", SERVER);
+
+			// then
+			expect(log.entries).toEqual([
+				"spawn:/root-a",
+				"stop-start:/root-a",
+				"stop-end:/root-a",
+				"spawn:/root-a",
+			]);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+
+	it("#given a client whose initialization failed while its stop is pending #when another getClient races the stop #then the replacement spawns only after the failed client stopped", async () => {
+		// given
+		const { manager, log } = managerHarness({
+			stopDelayMs: SLOW_STOP_MS,
+			initializeFailures: 1,
+		});
+		try {
+			const failing = manager.getClient("/root-fail", SERVER);
+
+			// when: the second acquisition starts once the first one's stop is provably in flight.
+			await log.waitFor("stop-start:/root-fail");
+			const racing = manager.getClient("/root-fail", SERVER);
+			await expect(failing).rejects.toThrow("initialize boom");
+			await racing;
+			manager.releaseClient("/root-fail", SERVER.id);
+
+			// then
+			expect(log.entries).toEqual([
+				"spawn:/root-fail",
+				"stop-start:/root-fail",
+				"stop-end:/root-fail",
+				"spawn:/root-fail",
+			]);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+});
+
+describe("LspManager dead-client respawn budget", () => {
+	it("#given a crash-looping client that is dead on every acquisition #when getClient is called repeatedly #then respawn attempts stay within the retry budget", async () => {
+		// given
+		const { manager, clients } = managerHarness({ aliveFromGeneration: Number.POSITIVE_INFINITY });
+		try {
+			// when
+			for (let attempt = 0; attempt < 8; attempt += 1) {
+				await manager
+					.getClient("/root-crash", SERVER)
+					.then(
+						() => manager.releaseClient("/root-crash", SERVER.id),
+						() => undefined,
+					);
+			}
+
+			// then: one initial generation plus at most two budgeted respawns.
+			expect(clients.length).toBeLessThanOrEqual(3);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+
+	it("#given the respawn budget is exhausted #when getClient is called again #then it rejects with a budget error instead of respawning", async () => {
+		// given
+		const { manager } = managerHarness({ aliveFromGeneration: Number.POSITIVE_INFINITY });
+		try {
+			await expect(manager.getClient("/root-crash", SERVER)).rejects.toThrow(/respawn budget/);
+
+			// when / then
+			await expect(manager.getClient("/root-crash", SERVER)).rejects.toThrow(/respawn budget/);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+
+	it("#given the budget was exhausted but the cooldown elapsed and the server recovered #when getClient is called #then it spawns again and resets the budget on a healthy generation", async () => {
+		// given: generations one and two crash, the third is healthy.
+		const { manager, clients, clock } = managerHarness({ aliveFromGeneration: 3 });
+		try {
+			await expect(manager.getClient("/root-crash", SERVER)).rejects.toThrow(/respawn budget/);
+
+			// when: cooldown elapses (60s default) and the server stays alive again.
+			clock.value += 60_001;
+			const recovered = await manager.getClient("/root-crash", SERVER);
+			manager.releaseClient("/root-crash", SERVER.id);
+
+			// then
+			expect(clients.length).toBe(3);
+			await expect(manager.getClient("/root-crash", SERVER)).resolves.toBe(recovered);
+			manager.releaseClient("/root-crash", SERVER.id);
+		} finally {
+			await manager.stopAll();
+		}
+	});
+});