@serverkgg/bridge 0.0.15 → 0.0.17

package/src/runtime/bridgeSupervisor.ts

@@ -1,5 +1,6 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { BRIDGE_CRASH_LINE_LIMIT, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { errorText } from "./bridgeLog";
@@ -14,9 +15,11 @@ const KILL_GRACE_MS = 5000;
 
 const RESTART_DELAY_MS = 2000;
 
-const RESTART_WINDOW_MS = 300_000;
+const RESTART_WINDOW_MS = 600_000;
 
-const RESTART_LIMIT = 5;
+const RESTART_LIMIT = 3;
+
+const STABLE_UPTIME_MS = 180_000;
 
 const GAME_OOM_SCORE_ADJ = 500;
 
@@ -35,7 +38,11 @@ export interface SupervisorOptions {
 	ready?: RegExp;
 	resolveCommand: () => Promise<string[]>;
 	gracefulStop?: () => Promise<void>;
+	stableAfterMs?: number;
 	onReady?: () => void;
+	onStable?: () => void;
+	onStopped?: () => void;
+	onExit?: (record: BridgeCrashRecord) => void;
 	onLog: BridgeLeveledLogger;
 }
 
@@ -119,6 +126,15 @@ const splitLines = (buffer: string, chunk: string) => {
 	};
 };
 
+const PUMP_DRAIN_MS = 500;
+
+const drainPumps = async (pumps: readonly Promise<unknown>[]) => {
+	await Promise.race([
+		Promise.all(pumps),
+		Bun.sleep(PUMP_DRAIN_MS),
+	]);
+};
+
 export const createSupervisor = (options: SupervisorOptions) => {
 	const tail: SupervisorLine[] = [];
 	const watchers = new Set<LineWatcher>();
@@ -132,8 +148,21 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	let exitCode: number | null = null;
 	let outOfMemory = false;
 	let restarts = 0;
+	let launchedAt = performance.now();
+	let stable = false;
+	let stableTimer: ReturnType<typeof setTimeout> | null = null;
 	let queue: Promise<unknown> = Promise.resolve();
 
+	const markStable = () => {
+		if (stable) {
+			return;
+		}
+
+		stable = true;
+
+		options.onStable?.();
+	};
+
 	const emit = (entry: SupervisorLine) => {
 		if (entry.stream === BridgeStream.Stderr) {
 			process.stderr.write(`${entry.line}\n`);
@@ -223,6 +252,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		return restartStamps.length >= RESTART_LIMIT;
 	};
 
+	const lastLine = () => {
+		for (let index = tail.length - 1; index >= 0; index -= 1) {
+			const line = stripAnsi(tail.at(index)?.line ?? "").trim();
+
+			if (line.length > 0) {
+				return line.slice(0, BRIDGE_CRASH_LINE_LIMIT);
+			}
+		}
+
+		return null;
+	};
+
 	const launch = async () => {
 		const command = await options.resolveCommand();
 		const [executable, ...rest] = command;
@@ -237,6 +278,34 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		];
 
 		const { argv: spawnArgv, credentials } = await unprivilegedArgv(argv, options.uid, options.gid);
+
+		launchedAt = performance.now();
+		stable = false;
+
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
+		let accepted = false;
+		let reported = false;
+		let pumps: Promise<unknown>[] = [];
+
+		const finish = (code: number) => {
+			if (!accepted || reported) {
+				return;
+			}
+
+			reported = true;
+
+			void handleExit(code, pumps).catch((error: unknown) => {
+				options.onLog.error("the game exit was not reported", {
+					error: errorText(error),
+				});
+			});
+		};
+
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
 			cwd: options.cwd,
@@ -245,7 +314,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			stdin: "pipe",
 			stdout: "pipe",
 			onExit(_subprocess, code, signal) {
-				handleExit(exitCodeOf(code, signal));
+				finish(exitCodeOf(code, signal));
 			},
 		});
 
@@ -281,8 +350,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		childGroup = group;
 		exitCode = null;
 
-		void pump(spawned.stdout, BridgeStream.Stdout);
-		void pump(spawned.stderr, BridgeStream.Stderr);
+		const launchStamp = launchedAt;
+
+		stableTimer = setTimeout(() => {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			markStable();
+		}, options.stableAfterMs ?? STABLE_UPTIME_MS);
+
+		pumps = [
+			pump(spawned.stdout, BridgeStream.Stdout).catch(() => undefined),
+			pump(spawned.stderr, BridgeStream.Stderr).catch(() => undefined),
+		];
+
+		accepted = true;
 
 		options.onLog("game started", {
 			pid: spawned.pid,
@@ -290,6 +373,12 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			uid: options.uid,
 		});
 
+		if (spawned.exitCode !== null || spawned.signalCode !== null) {
+			finish(exitCodeOf(spawned.exitCode, spawned.signalCode));
+
+			return;
+		}
+
 		if (!options.ready) {
 			runtime = BridgeRuntime.Running;
 
@@ -299,12 +388,30 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		runtime = BridgeRuntime.Starting;
 
 		void watch(options.ready, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
-			if (match && runtime === BridgeRuntime.Starting) {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			if (match) {
+				if (runtime === BridgeRuntime.Starting) {
+					runtime = BridgeRuntime.Running;
+
+					options.onLog("game ready");
+					options.onReady?.();
+				}
+
+				markStable();
+
+				return;
+			}
+
+			if (runtime === BridgeRuntime.Starting) {
 				runtime = BridgeRuntime.Running;
 
-				options.onLog("game ready");
-				options.onReady?.();
+				options.onLog.warn("game ready line not seen, treating it as running");
 			}
+
+			markStable();
 		});
 	};
 
@@ -326,12 +433,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		}
 	};
 
-	function handleExit(code: number) {
+	async function handleExit(code: number, pumps: readonly Promise<unknown>[]) {
 		const finished = child;
 
 		child = null;
 		exitCode = code;
 
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
 		if (finished) {
 			sweep(finished.pid, childGroup);
 		}
@@ -350,6 +463,9 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return;
 		}
 
+		const uptimeMs = performance.now() - launchedAt;
+		const phase = stable ? BridgeCrashPhase.Running : BridgeCrashPhase.Booting;
+
 		outOfMemory = code === SIGKILL_EXIT_CODE;
 
 		if (outOfMemory) {
@@ -358,6 +474,38 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			});
 		}
 
+		const report = async (restarting: boolean, looping: boolean) => {
+			const settled = {
+				exitCode: code,
+				outOfMemory,
+				phase,
+				uptimeSeconds: Math.floor(uptimeMs / 1000),
+				restarting,
+				looping,
+				restarts,
+			};
+
+			await drainPumps(pumps);
+
+			options.onExit?.({
+				...settled,
+				lastLine: lastLine(),
+			});
+		};
+
+		if (phase === BridgeCrashPhase.Booting) {
+			runtime = BridgeRuntime.Crashed;
+
+			options.onLog.error("game exited while booting, not restarting", {
+				code,
+				phase,
+			});
+
+			await report(false, false);
+
+			return;
+		}
+
 		restarts += 1;
 		restartStamps.push(performance.now());
 
@@ -370,6 +518,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				restarts,
 			});
 
+			await report(false, true);
+
 			return;
 		}
 
@@ -380,6 +530,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			restarts,
 		});
 
+		await report(true, false);
+
 		void restart();
 	}
 
@@ -429,6 +581,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			shouldRun = true;
 			breaker = false;
 			outOfMemory = false;
+			restarts = 0;
 			restartStamps.length = 0;
 
 			if (child) {
@@ -444,6 +597,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			if (!child) {
 				runtime = BridgeRuntime.Stopped;
 
+				options.onStopped?.();
+
 				return;
 			}
 
@@ -488,6 +643,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			sweep(active.pid, group);
 
 			runtime = BridgeRuntime.Stopped;
+
+			options.onStopped?.();
 		},
 
 		async command(input: string, commandOptions: BridgeCommandOptions = {}): Promise<BridgeReply> {

package/src/runtime/bridgeZip.ts

@@ -1,4 +1,4 @@
-import { inflateRawSync } from "node:zlib";
+import { crc32, inflateRawSync } from "node:zlib";
 
 const EOCD_SIGNATURE = 0x06_05_4b_50;
 
@@ -32,11 +32,14 @@ const DEFLATED = 8;
 
 const ENCRYPTED_FLAG = 0x1;
 
+const DATA_DESCRIPTOR_FLAG = 0x8;
+
 export const ZIP_ENTRY_LIMIT = 2048;
 
 export const ZIP_TREE_ENTRY_LIMIT = 16_384;
 
 const MAX_ENTRY_BYTES = 128 * 1024 * 1024;
+const MAX_CONFIGURED_ENTRY_BYTES = 512 * 1024 * 1024;
 
 const UNIX_MODE_FORMAT_MASK = 0xf0_00;
 
@@ -48,6 +51,8 @@ export interface ZipEntry {
 	sizeBytes: number;
 	compressedSize: number;
 	method: number;
+	flags: number;
+	checksum: number;
 	localOffset: number;
 }
 
@@ -155,7 +160,14 @@ const readZip64Extra = (extra: Buffer, sizeBytes: number, compressedSize: number
 	return extended;
 };
 
-export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT): Promise<ZipArchive> => {
+export const openZip = async (
+	path: string,
+	maxEntries: number = ZIP_ENTRY_LIMIT,
+	maxEntryBytes: number = MAX_ENTRY_BYTES,
+): Promise<ZipArchive> => {
+	if (!Number.isSafeInteger(maxEntryBytes) || maxEntryBytes <= 0 || maxEntryBytes > MAX_CONFIGURED_ENTRY_BYTES) {
+		throw new Error("archive entry byte limit is invalid");
+	}
 	const size = Bun.file(path).size;
 	const tailStart = Math.max(0, size - (EOCD_SEARCH_LIMIT + ZIP64_LOCATOR_LENGTH));
 	const tail = await readRange(path, tailStart, size);
@@ -174,6 +186,9 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 	if (directory.total > maxEntries) {
 		throw new Error(`archive has too many entries (${directory.total})`);
 	}
+	if (directory.sizeBytes > 16 * 1024 * 1024 || directory.offset + directory.sizeBytes > size) {
+		throw new Error("archive central directory exceeds its bounds");
+	}
 
 	const central = await readRange(path, directory.offset, directory.offset + directory.sizeBytes);
 	const entries: ZipEntry[] = [];
@@ -187,6 +202,7 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 
 		const flags = central.readUInt16LE(cursor + 8);
 		const method = central.readUInt16LE(cursor + 10);
+		const checksum = central.readUInt32LE(cursor + 16);
 		const compressedSize = central.readUInt32LE(cursor + 20);
 		const sizeBytes = central.readUInt32LE(cursor + 24);
 		const nameLength = central.readUInt16LE(cursor + 28);
@@ -200,8 +216,14 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 		const extended = readZip64Extra(extra, sizeBytes, compressedSize, localOffset);
 
 		cursor = nameStart + nameLength + extraLength + commentLength;
+		if (cursor > central.length) {
+			throw new Error("archive central entry exceeds its directory");
+		}
 
-		if (name.endsWith("/") || (flags & ENCRYPTED_FLAG) !== 0) {
+		if ((flags & ENCRYPTED_FLAG) !== 0) {
+			throw new Error("encrypted archive entries are not supported");
+		}
+		if (name.endsWith("/")) {
 			continue;
 		}
 
@@ -211,6 +233,8 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 			sizeBytes: extended.sizeBytes ?? sizeBytes,
 			compressedSize: extended.compressedSize ?? compressedSize,
 			method,
+			flags,
+			checksum,
 			localOffset: extended.localOffset ?? localOffset,
 		});
 	}
@@ -219,20 +243,59 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 		entries,
 
 		async read(entry) {
-			if (entry.sizeBytes > MAX_ENTRY_BYTES || entry.compressedSize > MAX_ENTRY_BYTES) {
+			if (entry.sizeBytes > maxEntryBytes || entry.compressedSize > maxEntryBytes) {
 				throw new Error(`archive entry "${entry.name}" is too large`);
 			}
 
+			if (entry.localOffset + LOCAL_HEADER_LENGTH > directory.offset) {
+				throw new Error(`archive entry "${entry.name}" has an out-of-bounds local header`);
+			}
+
 			const header = await readRange(path, entry.localOffset, entry.localOffset + LOCAL_HEADER_LENGTH);
 
 			if (header.length < LOCAL_HEADER_LENGTH || header.readUInt32LE(0) !== LOCAL_SIGNATURE) {
 				throw new Error(`archive entry "${entry.name}" has a corrupt local header`);
 			}
 
-			const dataOffset = entry.localOffset + LOCAL_HEADER_LENGTH + header.readUInt16LE(26) + header.readUInt16LE(28);
+			if (header.readUInt16LE(8) !== entry.method || header.readUInt16LE(6) !== entry.flags) {
+				throw new Error(`archive entry "${entry.name}" has inconsistent local compression flags or method`);
+			}
+			if ((entry.flags & DATA_DESCRIPTOR_FLAG) === 0 && header.readUInt32LE(14) !== entry.checksum) {
+				throw new Error(`archive entry "${entry.name}" has inconsistent local checksum`);
+			}
+			if ((entry.flags & DATA_DESCRIPTOR_FLAG) === 0) {
+				const localCompressed = header.readUInt32LE(18);
+				const localSize = header.readUInt32LE(22);
+				if (
+					(localCompressed !== SIZE_SENTINEL && localCompressed !== entry.compressedSize)
+					|| (localSize !== SIZE_SENTINEL && localSize !== entry.sizeBytes)
+				) {
+					throw new Error(`archive entry "${entry.name}" has inconsistent local sizes`);
+				}
+			}
+
+			const nameOffset = entry.localOffset + LOCAL_HEADER_LENGTH;
+			const nameLength = header.readUInt16LE(26);
+			const dataOffset = nameOffset + nameLength + header.readUInt16LE(28);
+			if (dataOffset > directory.offset || entry.compressedSize > directory.offset - dataOffset) {
+				throw new Error(`archive entry "${entry.name}" data exceeds its local bounds`);
+			}
+			const name = await readRange(path, nameOffset, nameOffset + nameLength);
+			if (name.toString("utf8") !== entry.name) {
+				throw new Error(`archive entry "${entry.name}" has an inconsistent local name`);
+			}
 			const compressed = await readRange(path, dataOffset, dataOffset + entry.compressedSize);
+			if (compressed.length !== entry.compressedSize) {
+				throw new Error(`archive entry "${entry.name}" has truncated compressed data`);
+			}
 
 			if (entry.method === STORED) {
+				if (compressed.length !== entry.sizeBytes || compressed.length !== entry.compressedSize) {
+					throw new Error(`archive entry "${entry.name}" does not match its declared size`);
+				}
+				if (crc32(compressed) !== entry.checksum) {
+					throw new Error(`archive entry "${entry.name}" failed its CRC32 checksum`);
+				}
 				return compressed;
 			}
 
@@ -241,13 +304,17 @@ export const openZip = async (path: string, maxEntries: number = ZIP_ENTRY_LIMIT
 			}
 
 			const data = inflateRawSync(compressed, {
-				maxOutputLength: MAX_ENTRY_BYTES,
+				maxOutputLength: maxEntryBytes,
 			});
 
 			if (data.length !== entry.sizeBytes) {
 				throw new Error(`archive entry "${entry.name}" did not inflate to its declared size`);
 			}
 
+			if (crc32(data) !== entry.checksum) {
+				throw new Error(`archive entry "${entry.name}" failed its CRC32 checksum`);
+			}
+
 			return data;
 		},
 	};

package/src/runtime/serve.ts

@@ -9,6 +9,7 @@ import {
 	type BridgeSampleResult,
 	type BridgeStatus,
 } from "../protocol";
+import { createBackupLease } from "./bridgeBackupLease";
 import { configPath, readConfig, socketPath, statePath } from "./bridgeConfigReader";
 import { createConsoleHandler } from "./bridgeConsoleHandler";
 import { createContext } from "./bridgeContext";
@@ -96,6 +97,15 @@ export const serve = async (driver: BridgeDriver) => {
 				});
 			});
 		},
+		onStable() {
+			ring.discard();
+		},
+		onStopped() {
+			ring.discard();
+		},
+		onExit(record) {
+			ring.crash(record);
+		},
 	});
 
 	const ring = createEventRing(driver.events, supervisor);
@@ -111,6 +121,18 @@ export const serve = async (driver: BridgeDriver) => {
 	});
 
 	const registry = createRegistry(driver, context, supervisor, ring);
+	const backupLease = createBackupLease(async () => {
+		if (supervisor.running) {
+			await driver.backup?.release?.(context);
+		}
+	});
+	setInterval(() => {
+		void backupLease.expire().catch((error: unknown) => {
+			log.error("could not resume saves after the backup lease expired", {
+				error: errorText(error),
+			});
+		});
+	}, 10_000).unref();
 
 	const runConsole = createConsoleHandler({
 		driver,
@@ -130,6 +152,7 @@ export const serve = async (driver: BridgeDriver) => {
 			exitCode: supervisor.exitCode,
 			outOfMemory: supervisor.outOfMemory,
 			restarts: supervisor.restarts,
+			crashRecords: true,
 			installError,
 			installErrorText,
 			installHold,
@@ -219,8 +242,8 @@ export const serve = async (driver: BridgeDriver) => {
 				timeoutMs,
 			});
 		},
-		async onInvoke(module, action, args) {
-			return await registry.invoke(module, action, args);
+		async onInvoke(module, action, args, recoveryOperationId) {
+			return await registry.invoke(module, action, args, recoveryOperationId);
 		},
 		onEvents(cursor) {
 			return ring.drain(cursor);
@@ -228,7 +251,18 @@ export const serve = async (driver: BridgeDriver) => {
 		async onSample() {
 			return await sample();
 		},
-		async onQuiesce() {
+		async onQuiesce(supportsSnapshot, leaseId) {
+			if (leaseId) {
+				return {
+					supported: true,
+					quiesced: true,
+					settleSeconds: 0,
+					leaseId: backupLease.renew(leaseId),
+				};
+			}
+			if (backupLease.active) {
+				throw new Error("a backup already owns the save hold");
+			}
 			const backup = driver.backup;
 
 			if (!backup?.quiesce) {
@@ -247,19 +281,29 @@ export const serve = async (driver: BridgeDriver) => {
 				};
 			}
 
-			await backup.quiesce(context);
-
-			return {
-				supported: true,
-				quiesced: true,
-				settleSeconds: backup.settleSeconds ?? 0,
-			};
-		},
-		async onRelease() {
-			if (supervisor.running) {
-				await driver.backup?.release?.(context);
+			const quiesce = backup.quiesce;
+			const owner = backupLease.begin(!supportsSnapshot);
+			try {
+				const prepared = await backupLease.prepare(owner, () => quiesce(context));
+				if (prepared?.snapshot && !supportsSnapshot) {
+					throw new Error("this backup requires an agent that supports bounded snapshots");
+				}
+				backupLease.renew(owner);
+				return {
+					supported: true,
+					quiesced: true,
+					settleSeconds: backup.settleSeconds ?? 0,
+					snapshot: prepared?.snapshot,
+					leaseId: supportsSnapshot ? owner : undefined,
+				};
+			} catch (error) {
+				await backupLease.finish(owner);
+				throw error;
 			}
 		},
+		async onRelease(leaseId) {
+			await backupLease.finish(leaseId);
+		},
 		async onDescribe() {
 			const describe = driver.install.describe;
 

package/src/wire/panel.ts

@@ -337,6 +337,7 @@ export const bridgeModuleManifestSchema = z.object({
 	pageSize: z.int().positive().nullable().default(null),
 	dependsOn: z.array(zKey).max(8),
 	actions: z.array(z.string().min(1).max(64)).max(16),
+	protectedActions: z.array(z.string().min(1).max(64)).max(16).optional(),
 });
 
 export const bridgeManifestSchema: z.ZodType<BridgeManifest> = z.object({