@serverkgg/bridge 0.0.21 → 0.1.2

package/src/runtime/bridgeSupervisor.ts

@@ -1,5 +1,7 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { BridgeLifecycleSubject } from "../modules/bridgeLifecycleSubject";
+import { BRIDGE_STOP_SIGNALS, type BridgeStopSignal } from "../modules/bridgeStopSignal";
 import { BRIDGE_CRASH_LINE_LIMIT, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
@@ -21,10 +23,14 @@ const RESTART_LIMIT = 3;
 
 const STABLE_UPTIME_MS = 180_000;
 
+const READY_TIMEOUT_STOP_FACTOR = 10;
+
 const GAME_OOM_SCORE_ADJ = 500;
 
 const SIGKILL_EXIT_CODE = 128 + 9;
 
+const ENVIRONMENT_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
 export interface SupervisorLine {
 	stream: BridgeStream;
 	line: string;
@@ -35,9 +41,14 @@ export interface SupervisorOptions {
 	uid: number;
 	gid: number;
 	stopTimeoutSeconds: number;
+	readyTimeoutSeconds?: number;
+	stopSignal?: BridgeStopSignal;
+	subject?: BridgeLifecycleSubject;
 	ready?: RegExp;
 	failed?: RegExp;
 	resolveCommand: () => Promise<string[]>;
+	resolveEnvironment?: () => Promise<Record<string, string>>;
+	redact?: (text: string) => string;
 	gracefulStop?: () => Promise<void>;
 	stableAfterMs?: number;
 	onReady?: () => void;
@@ -79,6 +90,27 @@ const SIGNAL_NUMBERS: Record<string, number> = {
 	SIGTERM: 15,
 };
 
+const spawnEnvironment = (subject: BridgeLifecycleSubject, resolved: Record<string, string>) => {
+	for (const [key, value] of Object.entries(resolved)) {
+		if (!ENVIRONMENT_KEY.test(key)) {
+			throw new Error(`the ${subject} environment names "${key}", which is not a valid variable name`);
+		}
+
+		if (typeof value !== "string") {
+			throw new Error(`the ${subject} environment value for "${key}" is not a string`);
+		}
+
+		if (value.includes("\0")) {
+			throw new Error(`the ${subject} environment value for "${key}" contains a NUL byte`);
+		}
+	}
+
+	return {
+		...process.env,
+		...resolved,
+	};
+};
+
 const EARLY_OUTPUT_LIMIT = 2000;
 
 const earlyOutput = async (spawned: { stdout: unknown; stderr: unknown }) => {
@@ -152,6 +184,7 @@ const chain = () => {
 };
 
 export const createSupervisor = (options: SupervisorOptions) => {
+	const subject = options.subject ?? BridgeLifecycleSubject.Game;
 	const tail: SupervisorLine[] = [];
 	const watchers = new Set<LineWatcher>();
 	const restartStamps: number[] = [];
@@ -168,10 +201,36 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	let stable = false;
 	let stableTimer: ReturnType<typeof setTimeout> | null = null;
 	let bootFailure: string | null = null;
+	let replacing: SpawnedGame | null = null;
 
 	const serialize = chain();
 	const transition = chain();
 
+	const readyTimeoutMs = (options.readyTimeoutSeconds ?? options.stopTimeoutSeconds * READY_TIMEOUT_STOP_FACTOR) * 1000;
+
+	const awaitingReady = () => {
+		return options.readyTimeoutSeconds !== undefined && runtime === BridgeRuntime.Starting;
+	};
+
+	const failBoot = (line: string) => {
+		const failing = child;
+		const failingGroup = childGroup;
+
+		if (failing === null) {
+			return;
+		}
+
+		bootFailure = line.trim().slice(0, BRIDGE_CRASH_LINE_LIMIT);
+
+		terminate(failing, failingGroup, "SIGTERM");
+
+		void Bun.sleep(KILL_GRACE_MS).then(() => {
+			if (child === failing) {
+				terminate(failing, failingGroup, "SIGKILL");
+			}
+		});
+	};
+
 	const markStable = () => {
 		if (stable) {
 			return;
@@ -182,7 +241,14 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		options.onStable?.();
 	};
 
-	const emit = (entry: SupervisorLine) => {
+	const emit = (raw: SupervisorLine) => {
+		const entry = options.redact
+			? {
+					stream: raw.stream,
+					line: options.redact(raw.line),
+				}
+			: raw;
+
 		if (entry.stream === BridgeStream.Stderr) {
 			process.stderr.write(`${entry.line}\n`);
 		} else {
@@ -288,7 +354,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const [executable, ...rest] = command;
 
 		if (!executable) {
-			throw new Error("the game start command is empty");
+			throw new Error(`the ${subject} start command is empty`);
 		}
 
 		const argv = [
@@ -296,6 +362,10 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			...rest,
 		];
 
+		const environment = options.resolveEnvironment
+			? spawnEnvironment(subject, await options.resolveEnvironment())
+			: null;
+
 		const { argv: spawnArgv, credentials } = await unprivilegedArgv(argv, options.uid, options.gid);
 
 		launchedAt = performance.now();
@@ -320,7 +390,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			reported = true;
 
 			void handleExit(spawned, code, pumps).catch((error: unknown) => {
-				options.onLog.error("the game exit was not reported", {
+				options.onLog.error(`the ${subject} exit was not reported`, {
 					error: errorText(error),
 				});
 			});
@@ -328,6 +398,9 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
+			...(environment && {
+				env: environment,
+			}),
 			cwd: options.cwd,
 			detached: true,
 			stderr: "pipe",
@@ -349,7 +422,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 			if (observed === null && finished) {
 				throw new Error(
-					`the game exited immediately with code ${exitCodeOf(spawned.exitCode, spawned.signalCode)}${await earlyOutput(spawned)}`,
+					`the ${subject} exited immediately with code ${exitCodeOf(spawned.exitCode, spawned.signalCode)}${await earlyOutput(spawned)}`,
 				);
 			}
 
@@ -360,7 +433,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			sweep(spawned.pid, group);
 
 			throw new Error(
-				`the game process runs as uid ${String(observed)}, expected ${options.uid} — refusing to run it as the bridge`,
+				`the ${subject} process runs as uid ${String(observed)}, expected ${options.uid} — refusing to run it as the bridge`,
 			);
 		}
 
@@ -373,7 +446,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const launchStamp = launchedAt;
 
 		stableTimer = setTimeout(() => {
-			if (launchStamp !== launchedAt || child === null) {
+			if (launchStamp !== launchedAt || child === null || awaitingReady()) {
 				return;
 			}
 
@@ -387,7 +460,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		accepted = true;
 
-		options.onLog("game started", {
+		options.onLog(`${subject} started`, {
 			pid: spawned.pid,
 			group,
 			uid: options.uid,
@@ -400,27 +473,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		}
 
 		if (options.failed) {
-			void watch(options.failed, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
+			void watch(options.failed, readyTimeoutMs).then((match) => {
 				if (!match || launchStamp !== launchedAt || child === null || stable) {
 					return;
 				}
 
-				bootFailure = (match.input ?? match[0]).trim().slice(0, BRIDGE_CRASH_LINE_LIMIT);
+				const line = match.input ?? match[0];
 
-				options.onLog.error("the game reported a failed boot, stopping it", {
-					line: bootFailure,
+				options.onLog.error(`the ${subject} reported a failed boot, stopping it`, {
+					line: line.trim().slice(0, BRIDGE_CRASH_LINE_LIMIT),
 				});
 
-				const failing = child;
-				const failingGroup = childGroup;
-
-				terminate(failing, failingGroup, "SIGTERM");
-
-				void Bun.sleep(KILL_GRACE_MS).then(() => {
-					if (child === failing) {
-						terminate(failing, failingGroup, "SIGKILL");
-					}
-				});
+				failBoot(line);
 			});
 		}
 
@@ -432,7 +496,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		runtime = BridgeRuntime.Starting;
 
-		void watch(options.ready, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
+		void watch(options.ready, readyTimeoutMs).then((match) => {
 			if (launchStamp !== launchedAt || child === null) {
 				return;
 			}
@@ -441,7 +505,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				if (runtime === BridgeRuntime.Starting) {
 					runtime = BridgeRuntime.Running;
 
-					options.onLog("game ready");
+					options.onLog(`${subject} ready`);
 					options.onReady?.();
 				}
 
@@ -450,17 +514,27 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				return;
 			}
 
+			if (runtime === BridgeRuntime.Starting && options.readyTimeoutSeconds !== undefined) {
+				options.onLog.error(`the ${subject} did not report ready in time, stopping it`, {
+					readyTimeoutSeconds: options.readyTimeoutSeconds,
+				});
+
+				failBoot(`the ${subject} did not report ready within ${options.readyTimeoutSeconds} seconds`);
+
+				return;
+			}
+
 			if (runtime === BridgeRuntime.Starting) {
 				runtime = BridgeRuntime.Running;
 
-				options.onLog.warn("game ready line not seen, treating it as running");
+				options.onLog.warn(`${subject} ready line not seen, treating it as running`);
 			}
 
 			markStable();
 		});
 	};
 
-	const restart = async () => {
+	const restartAfterCrash = async () => {
 		await Bun.sleep(RESTART_DELAY_MS);
 
 		await transition(async () => {
@@ -473,7 +547,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			} catch (error) {
 				runtime = BridgeRuntime.Crashed;
 
-				options.onLog.error("game failed to restart", {
+				options.onLog.error(`${subject} failed to restart`, {
 					error: errorText(error),
 				});
 			}
@@ -502,12 +576,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		reap(finished, code);
 
+		if (finished === replacing) {
+			outOfMemory = false;
+
+			options.onLog(`${subject} stopped for a restart`, {
+				code,
+			});
+
+			return;
+		}
+
 		if (!shouldRun) {
 			outOfMemory = false;
 
 			runtime = BridgeRuntime.Stopped;
 
-			options.onLog("game stopped", {
+			options.onLog(`${subject} stopped`, {
 				code,
 			});
 
@@ -520,7 +604,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		outOfMemory = code === SIGKILL_EXIT_CODE && bootFailure === null;
 
 		if (outOfMemory) {
-			options.onLog.error("the game was killed for running out of memory", {
+			options.onLog.error(`the ${subject} was killed for running out of memory`, {
 				code,
 			});
 		}
@@ -538,16 +622,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 			await drainPumps(pumps);
 
+			const line = bootFailure ?? lastLine();
+
 			options.onExit?.({
 				...settled,
-				lastLine: bootFailure ?? lastLine(),
+				lastLine: line !== null && options.redact ? options.redact(line) : line,
 			});
 		};
 
 		if (phase === BridgeCrashPhase.Booting) {
 			runtime = BridgeRuntime.Crashed;
 
-			options.onLog.error("game exited while booting, not restarting", {
+			options.onLog.error(`${subject} exited while booting, not restarting`, {
 				code,
 				phase,
 			});
@@ -564,7 +650,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			breaker = true;
 			runtime = BridgeRuntime.Crashed;
 
-			options.onLog.error("game crash loop detected, not restarting", {
+			options.onLog.error(`${subject} crash loop detected, not restarting`, {
 				code,
 				restarts,
 			});
@@ -576,19 +662,80 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		runtime = BridgeRuntime.Starting;
 
-		options.onLog.warn("game exited, restarting", {
+		options.onLog.warn(`${subject} exited, restarting`, {
 			code,
 			restarts,
 		});
 
 		await report(true, false);
 
-		void restart();
+		void restartAfterCrash();
 	}
 
+	const escalate = async (active: SpawnedGame, group: number | null) => {
+		if (!options.stopSignal) {
+			terminate(active, group, "SIGTERM");
+
+			const afterTerm = await Promise.race([
+				active.exited,
+				Bun.sleep(KILL_GRACE_MS).then(() => null),
+			]);
+
+			if (afterTerm !== null) {
+				return;
+			}
+		}
+
+		terminate(active, group, "SIGKILL");
+
+		await active.exited;
+	};
+
+	const halt = async (active: SpawnedGame, group: number | null) => {
+		if (options.gracefulStop) {
+			const graceful = options.gracefulStop().catch((error: unknown) => {
+				options.onLog.warn("graceful stop failed", {
+					error: errorText(error),
+				});
+			});
+
+			await Promise.race([
+				graceful,
+				active.exited,
+			]);
+		}
+
+		const exited = active.exitCode !== null || active.signalCode !== null;
+
+		if (options.stopSignal && !exited) {
+			terminate(active, group, BRIDGE_STOP_SIGNALS[options.stopSignal]);
+		}
+
+		const settled = await Promise.race([
+			active.exited,
+			Bun.sleep(options.stopTimeoutSeconds * 1000).then(() => null),
+		]);
+
+		if (settled === null) {
+			await escalate(active, group);
+		}
+
+		if (child !== active) {
+			return null;
+		}
+
+		const code = exitCodeOf(active.exitCode, active.signalCode);
+
+		reap(active, code);
+
+		outOfMemory = false;
+
+		return code;
+	};
+
 	const writeLine = async (input: string) => {
 		if (!child) {
-			throw new Error("the game is not running");
+			throw new Error(`the ${subject} is not running`);
 		}
 
 		child.stdin.write(`${input}\n`);
@@ -652,54 +799,59 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 				runtime = BridgeRuntime.Stopping;
 
-				if (options.gracefulStop) {
-					const graceful = options.gracefulStop().catch((error: unknown) => {
-						options.onLog.warn("graceful stop failed", {
-							error: errorText(error),
-						});
+				const code = await halt(active, group);
+
+				if (code !== null) {
+					options.onLog(`${subject} stopped`, {
+						code,
 					});
+				}
+
+				runtime = BridgeRuntime.Stopped;
+
+				options.onStopped?.();
+			});
+		},
 
-					await Promise.race([
-						graceful,
-						active.exited,
-					]);
+		async restart() {
+			await transition(async () => {
+				if (!shouldRun) {
+					throw new Error(`the ${subject} is not meant to be running, so there is nothing to restart`);
 				}
 
-				const settled = await Promise.race([
-					active.exited,
-					Bun.sleep(options.stopTimeoutSeconds * 1000).then(() => null),
-				]);
+				const active = child;
 
-				if (settled === null) {
-					terminate(active, group, "SIGTERM");
+				runtime = BridgeRuntime.Stopping;
 
-					const afterTerm = await Promise.race([
-						active.exited,
-						Bun.sleep(KILL_GRACE_MS).then(() => null),
-					]);
+				if (active) {
+					replacing = active;
 
-					if (afterTerm === null) {
-						terminate(active, group, "SIGKILL");
+					try {
+						const code = await halt(active, childGroup);
 
-						await active.exited;
+						if (code !== null) {
+							options.onLog(`${subject} stopped for a restart`, {
+								code,
+							});
+						}
+					} finally {
+						replacing = null;
 					}
 				}
 
-				if (child === active) {
-					const code = exitCodeOf(active.exitCode, active.signalCode);
+				if (!shouldRun) {
+					return;
+				}
 
-					reap(active, code);
+				breaker = false;
 
-					outOfMemory = false;
+				try {
+					await launch();
+				} catch (error) {
+					runtime = BridgeRuntime.Crashed;
 
-					options.onLog("game stopped", {
-						code,
-					});
+					throw error;
 				}
-
-				runtime = BridgeRuntime.Stopped;
-
-				options.onStopped?.();
 			});
 		},