@serverkgg/bridge 0.0.15 → 0.0.17

package/src/runtime/bridgeRegistry.ts

@@ -31,6 +31,18 @@ const requiresRunning = (module: BridgePanelModule) => {
 	return false;
 };
 
+export const protectedActionsOf = (module: BridgePanelModule): string[] => {
+	const protectedActions = "protectedActions" in module ? (module.protectedActions ?? []) : [];
+	const declared = actionsOf(module);
+	if (
+		protectedActions.some((action) => isReadAction(action) || !declared.includes(action))
+		|| (protectedActions.length > 0 && requiresRunning(module))
+	) {
+		throw new BridgeDriverError("protected actions must be declared mutations that work while stopped");
+	}
+	return protectedActions;
+};
+
 const refreshOf = (module: BridgePanelModule) => {
 	if (module.kind === BridgeKind.Collection || module.kind === BridgeKind.Detail) {
 		return module.refreshSeconds ?? null;
@@ -358,6 +370,12 @@ export const createRegistry = (
 			}
 
 			case BridgeKind.Catalog: {
+				if (action === BridgeAction.Releases && module.releases) {
+					return await module.releases(context, asString(args, "id"));
+				}
+				if (action === BridgeAction.Preview && module.preview) {
+					return await module.preview(context, asString(args, "id"), asOptional(args, "releaseId") ?? undefined);
+				}
 				if (action === BridgeAction.Search) {
 					return await module.search(context, {
 						query: asText(args, "query"),
@@ -373,7 +391,7 @@ export const createRegistry = (
 				}
 
 				if (action === BridgeAction.Install) {
-					return await module.install(context, asString(args, "id"));
+					return await module.install(context, asString(args, "id"), asOptional(args, "releaseId") ?? undefined);
 				}
 
 				if (action === BridgeAction.Remove) {
@@ -398,6 +416,7 @@ export const createRegistry = (
 	return {
 		manifest(): BridgeManifest {
 			const entries: BridgeModuleManifest[] = Object.entries(modules).map(([id, module]) => {
+				const protectedActions = protectedActionsOf(module);
 				return {
 					id,
 					kind: module.kind,
@@ -406,6 +425,11 @@ export const createRegistry = (
 					pageSize: pageSizeOf(module),
 					dependsOn: dependsOf(module),
 					actions: actionsOf(module),
+					...(protectedActions.length > 0
+						? {
+								protectedActions,
+							}
+						: {}),
 				};
 			});
 
@@ -419,13 +443,17 @@ export const createRegistry = (
 			};
 		},
 
-		async invoke(name: string, action: string, args: BridgeValues = {}) {
+		async invoke(name: string, action: string, args: BridgeValues = {}, recoveryOperationId?: string) {
 			const module = modules[name];
 
 			if (!module) {
 				throw new BridgeDriverError(`no module named "${name}"`);
 			}
 
+			if (protectedActionsOf(module).includes(action) && (!recoveryOperationId || supervisor.running)) {
+				throw new BridgeDriverError("this action needs a recovery operation with the game stopped");
+			}
+
 			if (requiresRunning(module) && !supervisor.running) {
 				throw new BridgeDriverError(`module "${name}" needs the game to be running`);
 			}

package/src/runtime/bridgeServer.test.ts

@@ -40,6 +40,7 @@ const baseOptions = (socket: string): ServerOptions => ({
 		exitCode: null,
 		outOfMemory: false,
 		restarts: 0,
+		crashRecords: true,
 		installError: null,
 		installErrorText: null,
 		installHold: null,
@@ -596,3 +597,60 @@ describe("error serialisation", () => {
 		});
 	});
 });
+
+describe("backup ownership dispatch", () => {
+	let dir = "";
+	beforeEach(async () => {
+		dir = await mkdtemp(join(tmpdir(), "bridge-lease-"));
+	});
+	afterEach(async () => {
+		await rm(dir, {
+			force: true,
+			recursive: true,
+		});
+	});
+	test("carries the lease token through renew and release frames", async () => {
+		const socket = join(dir, "bridge.sock");
+		const renewals: (string | undefined)[] = [];
+		const releases: (string | undefined)[] = [];
+		const server = createServer({
+			...baseOptions(socket),
+			onQuiesce: async (supportsSnapshot, leaseId) => {
+				expect(supportsSnapshot).toBe(true);
+				renewals.push(leaseId);
+				return {
+					supported: true,
+					quiesced: true,
+					settleSeconds: 0,
+					leaseId,
+				};
+			},
+			onRelease: async (leaseId) => {
+				releases.push(leaseId);
+			},
+		});
+		try {
+			await call(socket, {
+				id: "renew",
+				token: TOKEN,
+				method: BridgeMethod.Quiesce,
+				supportsSnapshot: true,
+				leaseId: "owner",
+			});
+			await call(socket, {
+				id: "release",
+				token: TOKEN,
+				method: BridgeMethod.Release,
+				leaseId: "owner",
+			});
+			expect(renewals).toEqual([
+				"owner",
+			]);
+			expect(releases).toEqual([
+				"owner",
+			]);
+		} finally {
+			server.stop();
+		}
+	});
+});

package/src/runtime/bridgeServer.ts

@@ -25,11 +25,11 @@ export interface ServerOptions {
 	onStart: () => Promise<void>;
 	onStop: () => Promise<void>;
 	onCommand: (input: string, expect: string | undefined, timeoutMs: number | undefined) => Promise<BridgeCommandResult>;
-	onInvoke: (module: string, action: string, args: BridgeValues) => Promise<unknown>;
+	onInvoke: (module: string, action: string, args: BridgeValues, recoveryOperationId?: string) => Promise<unknown>;
 	onEvents: (cursor: number) => BridgeEventsResult;
 	onSample: () => Promise<BridgeSampleResult>;
-	onQuiesce: () => Promise<BridgeQuiesceResult>;
-	onRelease: () => Promise<void>;
+	onQuiesce: (supportsSnapshot: boolean, leaseId?: string) => Promise<BridgeQuiesceResult>;
+	onRelease: (leaseId?: string) => Promise<void>;
 	onDescribe: () => Promise<BridgeDescribeResult>;
 	onAnnounce: (message: string) => Promise<BridgeAnnounceResult>;
 	onPendingList: () => Promise<BridgePendingResult>;
@@ -40,6 +40,9 @@ export interface ServerOptions {
 }
 
 interface IncomingFrame {
+	leaseId?: unknown;
+	supportsSnapshot?: unknown;
+	recoveryOperationId?: unknown;
 	id?: unknown;
 	token?: unknown;
 	method?: unknown;
@@ -116,7 +119,12 @@ export const createServer = (options: ServerOptions) => {
 			}
 
 			case BridgeMethod.Invoke: {
-				return await options.onInvoke(String(request.module ?? ""), String(request.action ?? ""), asArgs(request.args));
+				return await options.onInvoke(
+					String(request.module ?? ""),
+					String(request.action ?? ""),
+					asArgs(request.args),
+					asText(request.recoveryOperationId) ?? undefined,
+				);
 			}
 
 			case BridgeMethod.Events: {
@@ -128,11 +136,11 @@ export const createServer = (options: ServerOptions) => {
 			}
 
 			case BridgeMethod.Quiesce: {
-				return await options.onQuiesce();
+				return await options.onQuiesce(request.supportsSnapshot === true, asText(request.leaseId) ?? undefined);
 			}
 
 			case BridgeMethod.Release: {
-				await options.onRelease();
+				await options.onRelease(asText(request.leaseId) ?? undefined);
 
 				return null;
 			}

package/src/runtime/bridgeSupervisor.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import { tmpdir } from "node:os";
 import { BridgeRuntime } from "../core";
+import { BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { readProcessGroup } from "./bridgeSignal";
 import { createSupervisor } from "./bridgeSupervisor";
@@ -11,6 +12,26 @@ const WRAPPER = "sleep 300 & echo child=$!; wait";
 
 const STUBBORN_WRAPPER = 'trap "" TERM; sleep 300 & echo child=$!; wait';
 
+const NEVER_READY = /this game never says it is ready/;
+
+const READY = /^ready$/;
+
+const BOOT_CRASH = "sleep 0.3; exit 3";
+
+const OOM_CRASH = "sleep 0.3; kill -9 $$";
+
+const RUNNING_CRASH = "echo ready; sleep 0.3; exit 4";
+
+const LATE_CRASH = "sleep 1; exit 5";
+
+const LAST_LINE = "boom, the game gave up here";
+
+const LAST_LINE_CRASH = `echo "${LAST_LINE}"; sleep 0.5; exit 7`;
+
+const TIGHT_LAST_LINE_CRASH = `sleep 0.2; echo "${LAST_LINE}"; exit 3`;
+
+const INSTANT_EXIT = "false";
+
 const silent: BridgeLeveledLogger = Object.assign(() => {}, {
 	warn: () => {},
 	error: () => {},
@@ -55,6 +76,46 @@ const wrapperSupervisor = (script: string, stopTimeoutSeconds: number) => {
 	});
 };
 
+const argvSupervisor = (resolveCommand: () => Promise<string[]>, ready: RegExp) => {
+	const records: BridgeCrashRecord[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		onLog: silent,
+		resolveCommand,
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+	};
+};
+
+const crashSupervisor = (command: () => string, ready: RegExp) => {
+	return argvSupervisor(async () => {
+		return [
+			"bash",
+			"-c",
+			command(),
+		];
+	}, ready);
+};
+
+const waitFor = async (settled: () => boolean, timeoutMs: number) => {
+	for (let waited = 0; waited < timeoutMs && !settled(); waited += 50) {
+		await Bun.sleep(50);
+	}
+
+	return settled();
+};
+
 const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	const supervisor = wrapperSupervisor(script, stopTimeoutSeconds);
 	const started = supervisor.watch(CHILD_PATTERN, 10_000);
@@ -73,6 +134,41 @@ const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	};
 };
 
+const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: number) => {
+	const records: BridgeCrashRecord[] = [];
+
+	let stables = 0;
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		stableAfterMs,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				command(),
+			];
+		},
+		onStable() {
+			stables += 1;
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+		stables: () => stables,
+	};
+};
+
 describe("createSupervisor", () => {
 	test("runs the game outside the bridge's own process group", async () => {
 		const { supervisor, pid } = await startWrapper(WRAPPER, 1);
@@ -108,4 +204,197 @@ describe("createSupervisor", () => {
 		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
 		expect(await waitGone(pid, 5000), "the game outlived the SIGKILL escalation").toBe(true);
 	}, 30_000);
+
+	test("does not restart a game that exited before it ever reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => BOOT_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(3);
+		expect(record?.outOfMemory).toBe(false);
+		expect(record?.restarts).toBe(0);
+
+		await Bun.sleep(3000);
+
+		expect(records, "the booting crash was retried").toHaveLength(1);
+		expect(supervisor.running).toBe(false);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(0);
+	}, 30_000);
+
+	test("restarts a game that crashed after it reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(4);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("flags a game killed with SIGKILL as out of memory", async () => {
+		const { supervisor, records } = crashSupervisor(() => OOM_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.outOfMemory).toBe(true);
+		expect(record?.exitCode).toBe(137);
+		expect(supervisor.outOfMemory).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("trips the breaker on the third running crash inside the window", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+
+		const record = records.at(2);
+
+		expect(record?.looping).toBe(true);
+		expect(record?.restarting).toBe(false);
+		expect(record?.restarts, "the record carries the count the supervisor reached").toBe(3);
+		expect(records.map((entry) => entry.restarts)).toEqual([
+			1,
+			2,
+			3,
+		]);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(3);
+
+		await supervisor.stop();
+	}, 40_000);
+
+	test("launches again when start is asked after a booting crash", async () => {
+		let script = BOOT_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+	}, 30_000);
+
+	test("carries the last line the game printed into the crash record", async () => {
+		const { supervisor, records } = crashSupervisor(() => LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine).toBe(LAST_LINE);
+	}, 30_000);
+
+	test("carries the last line even when the game exits with no pause after printing it", async () => {
+		const { supervisor, records } = crashSupervisor(() => TIGHT_LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine, "the crash record was built before the pumps drained").toBe(LAST_LINE);
+		expect(records.at(0)?.exitCode).toBe(3);
+	}, 30_000);
+
+	test("leaves no dead child behind when the game exits the instant it launches", async () => {
+		let adopted = 0;
+
+		for (let attempt = 0; attempt < 40 && adopted < 20; attempt += 1) {
+			const { supervisor, records } = argvSupervisor(async () => {
+				return [
+					INSTANT_EXIT,
+				];
+			}, NEVER_READY);
+
+			const launched = await supervisor.start().then(
+				() => true,
+				() => false,
+			);
+
+			if (!launched) {
+				continue;
+			}
+
+			adopted += 1;
+
+			expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+			await Bun.sleep(100);
+
+			expect(records, "the instant exit was reported more than once").toHaveLength(1);
+			expect(records.at(0)?.exitCode).toBe(1);
+			expect(records.at(0)?.outOfMemory).toBe(false);
+			expect(supervisor.running, "the supervisor kept a child that had already exited").toBe(false);
+			expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		}
+
+		expect(adopted, "the game never survived long enough to be adopted as the child").toBeGreaterThan(0);
+	}, 30_000);
+
+	test("treats a run that outlived the stable window as running even when the ready line never came", async () => {
+		const { supervisor, records, stables } = timedSupervisor(() => LATE_CRASH, NEVER_READY, 300);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.exitCode).toBe(5);
+		expect(stables(), "the run was reported stable more than once").toBe(1);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("opens a new episode when start is asked after the breaker tripped", async () => {
+		let script = RUNNING_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+		expect(supervisor.restarts).toBe(3);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.restarts, "the cumulative count survived an explicit start").toBe(0);
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+	}, 40_000);
 });