@serverkgg/bridge 0.0.20 → 0.1.1

package/src/runtime/bridgeSupervisor.ts

@@ -1,5 +1,7 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { BridgeLifecycleSubject } from "../modules/bridgeLifecycleSubject";
+import { BRIDGE_STOP_SIGNALS, type BridgeStopSignal } from "../modules/bridgeStopSignal";
 import { BRIDGE_CRASH_LINE_LIMIT, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
@@ -21,10 +23,14 @@ const RESTART_LIMIT = 3;
 
 const STABLE_UPTIME_MS = 180_000;
 
+const READY_TIMEOUT_STOP_FACTOR = 10;
+
 const GAME_OOM_SCORE_ADJ = 500;
 
 const SIGKILL_EXIT_CODE = 128 + 9;
 
+const ENVIRONMENT_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
 export interface SupervisorLine {
 	stream: BridgeStream;
 	line: string;
@@ -35,8 +41,14 @@ export interface SupervisorOptions {
 	uid: number;
 	gid: number;
 	stopTimeoutSeconds: number;
+	readyTimeoutSeconds?: number;
+	stopSignal?: BridgeStopSignal;
+	subject?: BridgeLifecycleSubject;
 	ready?: RegExp;
+	failed?: RegExp;
 	resolveCommand: () => Promise<string[]>;
+	resolveEnvironment?: () => Promise<Record<string, string>>;
+	redact?: (text: string) => string;
 	gracefulStop?: () => Promise<void>;
 	stableAfterMs?: number;
 	onReady?: () => void;
@@ -78,6 +90,27 @@ const SIGNAL_NUMBERS: Record<string, number> = {
 	SIGTERM: 15,
 };
 
+const spawnEnvironment = (subject: BridgeLifecycleSubject, resolved: Record<string, string>) => {
+	for (const [key, value] of Object.entries(resolved)) {
+		if (!ENVIRONMENT_KEY.test(key)) {
+			throw new Error(`the ${subject} environment names "${key}", which is not a valid variable name`);
+		}
+
+		if (typeof value !== "string") {
+			throw new Error(`the ${subject} environment value for "${key}" is not a string`);
+		}
+
+		if (value.includes("\0")) {
+			throw new Error(`the ${subject} environment value for "${key}" contains a NUL byte`);
+		}
+	}
+
+	return {
+		...process.env,
+		...resolved,
+	};
+};
+
 const EARLY_OUTPUT_LIMIT = 2000;
 
 const earlyOutput = async (spawned: { stdout: unknown; stderr: unknown }) => {
@@ -151,6 +184,7 @@ const chain = () => {
 };
 
 export const createSupervisor = (options: SupervisorOptions) => {
+	const subject = options.subject ?? BridgeLifecycleSubject.Game;
 	const tail: SupervisorLine[] = [];
 	const watchers = new Set<LineWatcher>();
 	const restartStamps: number[] = [];
@@ -166,10 +200,37 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	let launchedAt = performance.now();
 	let stable = false;
 	let stableTimer: ReturnType<typeof setTimeout> | null = null;
+	let bootFailure: string | null = null;
+	let replacing: SpawnedGame | null = null;
 
 	const serialize = chain();
 	const transition = chain();
 
+	const readyTimeoutMs = (options.readyTimeoutSeconds ?? options.stopTimeoutSeconds * READY_TIMEOUT_STOP_FACTOR) * 1000;
+
+	const awaitingReady = () => {
+		return options.readyTimeoutSeconds !== undefined && runtime === BridgeRuntime.Starting;
+	};
+
+	const failBoot = (line: string) => {
+		const failing = child;
+		const failingGroup = childGroup;
+
+		if (failing === null) {
+			return;
+		}
+
+		bootFailure = line.trim().slice(0, BRIDGE_CRASH_LINE_LIMIT);
+
+		terminate(failing, failingGroup, "SIGTERM");
+
+		void Bun.sleep(KILL_GRACE_MS).then(() => {
+			if (child === failing) {
+				terminate(failing, failingGroup, "SIGKILL");
+			}
+		});
+	};
+
 	const markStable = () => {
 		if (stable) {
 			return;
@@ -180,7 +241,14 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		options.onStable?.();
 	};
 
-	const emit = (entry: SupervisorLine) => {
+	const emit = (raw: SupervisorLine) => {
+		const entry = options.redact
+			? {
+					stream: raw.stream,
+					line: options.redact(raw.line),
+				}
+			: raw;
+
 		if (entry.stream === BridgeStream.Stderr) {
 			process.stderr.write(`${entry.line}\n`);
 		} else {
@@ -286,7 +354,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const [executable, ...rest] = command;
 
 		if (!executable) {
-			throw new Error("the game start command is empty");
+			throw new Error(`the ${subject} start command is empty`);
 		}
 
 		const argv = [
@@ -294,10 +362,15 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			...rest,
 		];
 
+		const environment = options.resolveEnvironment
+			? spawnEnvironment(subject, await options.resolveEnvironment())
+			: null;
+
 		const { argv: spawnArgv, credentials } = await unprivilegedArgv(argv, options.uid, options.gid);
 
 		launchedAt = performance.now();
 		stable = false;
+		bootFailure = null;
 
 		if (stableTimer) {
 			clearTimeout(stableTimer);
@@ -317,7 +390,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			reported = true;
 
 			void handleExit(spawned, code, pumps).catch((error: unknown) => {
-				options.onLog.error("the game exit was not reported", {
+				options.onLog.error(`the ${subject} exit was not reported`, {
 					error: errorText(error),
 				});
 			});
@@ -325,6 +398,9 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
+			...(environment && {
+				env: environment,
+			}),
 			cwd: options.cwd,
 			detached: true,
 			stderr: "pipe",
@@ -346,7 +422,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 			if (observed === null && finished) {
 				throw new Error(
-					`the game exited immediately with code ${exitCodeOf(spawned.exitCode, spawned.signalCode)}${await earlyOutput(spawned)}`,
+					`the ${subject} exited immediately with code ${exitCodeOf(spawned.exitCode, spawned.signalCode)}${await earlyOutput(spawned)}`,
 				);
 			}
 
@@ -357,7 +433,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			sweep(spawned.pid, group);
 
 			throw new Error(
-				`the game process runs as uid ${String(observed)}, expected ${options.uid} — refusing to run it as the bridge`,
+				`the ${subject} process runs as uid ${String(observed)}, expected ${options.uid} — refusing to run it as the bridge`,
 			);
 		}
 
@@ -370,7 +446,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const launchStamp = launchedAt;
 
 		stableTimer = setTimeout(() => {
-			if (launchStamp !== launchedAt || child === null) {
+			if (launchStamp !== launchedAt || child === null || awaitingReady()) {
 				return;
 			}
 
@@ -384,7 +460,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		accepted = true;
 
-		options.onLog("game started", {
+		options.onLog(`${subject} started`, {
 			pid: spawned.pid,
 			group,
 			uid: options.uid,
@@ -396,6 +472,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return;
 		}
 
+		if (options.failed) {
+			void watch(options.failed, readyTimeoutMs).then((match) => {
+				if (!match || launchStamp !== launchedAt || child === null || stable) {
+					return;
+				}
+
+				const line = match.input ?? match[0];
+
+				options.onLog.error(`the ${subject} reported a failed boot, stopping it`, {
+					line: line.trim().slice(0, BRIDGE_CRASH_LINE_LIMIT),
+				});
+
+				failBoot(line);
+			});
+		}
+
 		if (!options.ready) {
 			runtime = BridgeRuntime.Running;
 
@@ -404,7 +496,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		runtime = BridgeRuntime.Starting;
 
-		void watch(options.ready, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
+		void watch(options.ready, readyTimeoutMs).then((match) => {
 			if (launchStamp !== launchedAt || child === null) {
 				return;
 			}
@@ -413,7 +505,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				if (runtime === BridgeRuntime.Starting) {
 					runtime = BridgeRuntime.Running;
 
-					options.onLog("game ready");
+					options.onLog(`${subject} ready`);
 					options.onReady?.();
 				}
 
@@ -422,17 +514,27 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				return;
 			}
 
+			if (runtime === BridgeRuntime.Starting && options.readyTimeoutSeconds !== undefined) {
+				options.onLog.error(`the ${subject} did not report ready in time, stopping it`, {
+					readyTimeoutSeconds: options.readyTimeoutSeconds,
+				});
+
+				failBoot(`the ${subject} did not report ready within ${options.readyTimeoutSeconds} seconds`);
+
+				return;
+			}
+
 			if (runtime === BridgeRuntime.Starting) {
 				runtime = BridgeRuntime.Running;
 
-				options.onLog.warn("game ready line not seen, treating it as running");
+				options.onLog.warn(`${subject} ready line not seen, treating it as running`);
 			}
 
 			markStable();
 		});
 	};
 
-	const restart = async () => {
+	const restartAfterCrash = async () => {
 		await Bun.sleep(RESTART_DELAY_MS);
 
 		await transition(async () => {
@@ -445,7 +547,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			} catch (error) {
 				runtime = BridgeRuntime.Crashed;
 
-				options.onLog.error("game failed to restart", {
+				options.onLog.error(`${subject} failed to restart`, {
 					error: errorText(error),
 				});
 			}
@@ -474,12 +576,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		reap(finished, code);
 
+		if (finished === replacing) {
+			outOfMemory = false;
+
+			options.onLog(`${subject} stopped for a restart`, {
+				code,
+			});
+
+			return;
+		}
+
 		if (!shouldRun) {
 			outOfMemory = false;
 
 			runtime = BridgeRuntime.Stopped;
 
-			options.onLog("game stopped", {
+			options.onLog(`${subject} stopped`, {
 				code,
 			});
 
@@ -489,10 +601,10 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const uptimeMs = performance.now() - launchedAt;
 		const phase = stable ? BridgeCrashPhase.Running : BridgeCrashPhase.Booting;
 
-		outOfMemory = code === SIGKILL_EXIT_CODE;
+		outOfMemory = code === SIGKILL_EXIT_CODE && bootFailure === null;
 
 		if (outOfMemory) {
-			options.onLog.error("the game was killed for running out of memory", {
+			options.onLog.error(`the ${subject} was killed for running out of memory`, {
 				code,
 			});
 		}
@@ -510,16 +622,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 			await drainPumps(pumps);
 
+			const line = bootFailure ?? lastLine();
+
 			options.onExit?.({
 				...settled,
-				lastLine: lastLine(),
+				lastLine: line !== null && options.redact ? options.redact(line) : line,
 			});
 		};
 
 		if (phase === BridgeCrashPhase.Booting) {
 			runtime = BridgeRuntime.Crashed;
 
-			options.onLog.error("game exited while booting, not restarting", {
+			options.onLog.error(`${subject} exited while booting, not restarting`, {
 				code,
 				phase,
 			});
@@ -536,7 +650,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			breaker = true;
 			runtime = BridgeRuntime.Crashed;
 
-			options.onLog.error("game crash loop detected, not restarting", {
+			options.onLog.error(`${subject} crash loop detected, not restarting`, {
 				code,
 				restarts,
 			});
@@ -548,19 +662,80 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		runtime = BridgeRuntime.Starting;
 
-		options.onLog.warn("game exited, restarting", {
+		options.onLog.warn(`${subject} exited, restarting`, {
 			code,
 			restarts,
 		});
 
 		await report(true, false);
 
-		void restart();
+		void restartAfterCrash();
 	}
 
+	const escalate = async (active: SpawnedGame, group: number | null) => {
+		if (!options.stopSignal) {
+			terminate(active, group, "SIGTERM");
+
+			const afterTerm = await Promise.race([
+				active.exited,
+				Bun.sleep(KILL_GRACE_MS).then(() => null),
+			]);
+
+			if (afterTerm !== null) {
+				return;
+			}
+		}
+
+		terminate(active, group, "SIGKILL");
+
+		await active.exited;
+	};
+
+	const halt = async (active: SpawnedGame, group: number | null) => {
+		if (options.gracefulStop) {
+			const graceful = options.gracefulStop().catch((error: unknown) => {
+				options.onLog.warn("graceful stop failed", {
+					error: errorText(error),
+				});
+			});
+
+			await Promise.race([
+				graceful,
+				active.exited,
+			]);
+		}
+
+		const exited = active.exitCode !== null || active.signalCode !== null;
+
+		if (options.stopSignal && !exited) {
+			terminate(active, group, BRIDGE_STOP_SIGNALS[options.stopSignal]);
+		}
+
+		const settled = await Promise.race([
+			active.exited,
+			Bun.sleep(options.stopTimeoutSeconds * 1000).then(() => null),
+		]);
+
+		if (settled === null) {
+			await escalate(active, group);
+		}
+
+		if (child !== active) {
+			return null;
+		}
+
+		const code = exitCodeOf(active.exitCode, active.signalCode);
+
+		reap(active, code);
+
+		outOfMemory = false;
+
+		return code;
+	};
+
 	const writeLine = async (input: string) => {
 		if (!child) {
-			throw new Error("the game is not running");
+			throw new Error(`the ${subject} is not running`);
 		}
 
 		child.stdin.write(`${input}\n`);
@@ -624,54 +799,59 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 				runtime = BridgeRuntime.Stopping;
 
-				if (options.gracefulStop) {
-					const graceful = options.gracefulStop().catch((error: unknown) => {
-						options.onLog.warn("graceful stop failed", {
-							error: errorText(error),
-						});
+				const code = await halt(active, group);
+
+				if (code !== null) {
+					options.onLog(`${subject} stopped`, {
+						code,
 					});
+				}
+
+				runtime = BridgeRuntime.Stopped;
+
+				options.onStopped?.();
+			});
+		},
 
-					await Promise.race([
-						graceful,
-						active.exited,
-					]);
+		async restart() {
+			await transition(async () => {
+				if (!shouldRun) {
+					throw new Error(`the ${subject} is not meant to be running, so there is nothing to restart`);
 				}
 
-				const settled = await Promise.race([
-					active.exited,
-					Bun.sleep(options.stopTimeoutSeconds * 1000).then(() => null),
-				]);
+				const active = child;
 
-				if (settled === null) {
-					terminate(active, group, "SIGTERM");
+				runtime = BridgeRuntime.Stopping;
 
-					const afterTerm = await Promise.race([
-						active.exited,
-						Bun.sleep(KILL_GRACE_MS).then(() => null),
-					]);
+				if (active) {
+					replacing = active;
 
-					if (afterTerm === null) {
-						terminate(active, group, "SIGKILL");
+					try {
+						const code = await halt(active, childGroup);
 
-						await active.exited;
+						if (code !== null) {
+							options.onLog(`${subject} stopped for a restart`, {
+								code,
+							});
+						}
+					} finally {
+						replacing = null;
 					}
 				}
 
-				if (child === active) {
-					const code = exitCodeOf(active.exitCode, active.signalCode);
+				if (!shouldRun) {
+					return;
+				}
 
-					reap(active, code);
+				breaker = false;
 
-					outOfMemory = false;
+				try {
+					await launch();
+				} catch (error) {
+					runtime = BridgeRuntime.Crashed;
 
-					options.onLog("game stopped", {
-						code,
-					});
+					throw error;
 				}
-
-				runtime = BridgeRuntime.Stopped;
-
-				options.onStopped?.();
 			});
 		},
 

package/src/runtime/serve.ts

@@ -1,6 +1,6 @@
 import { chmod, chown, mkdir, rm } from "node:fs/promises";
 import { BridgeDesired, type BridgeInstallHold, BridgeRuntime, BridgeUserError, type LocalizedText } from "../core";
-import type { BridgeDriver } from "../modules";
+import { type BridgeDriver, BridgeLifecycleSubject } from "../modules";
 import {
 	BRIDGE_INSTALL_ERROR_LIMIT,
 	BRIDGE_VERSION,
@@ -16,8 +16,10 @@ import { createContext } from "./bridgeContext";
 import { validateDriver } from "./bridgeDriver";
 import { createEventRing } from "./bridgeEvents";
 import { decideInstall, recountHold } from "./bridgeInstall";
-import { errorText, log } from "./bridgeLog";
+import { createBridgeLiveConfig } from "./bridgeLiveConfig";
+import { errorText, log as stdoutLog } from "./bridgeLog";
 import { pendingResult, rejectedPending, unsupportedPending } from "./bridgePending";
+import { createBridgeRedaction, redactLocalized, redactLogger, secretFieldKeys } from "./bridgeRedaction";
 import { createRegistry } from "./bridgeRegistry";
 import { createServer } from "./bridgeServer";
 import { setupSummaryOf } from "./bridgeSetup";
@@ -69,24 +71,42 @@ export const serve = async (driver: BridgeDriver) => {
 
 	const desired = await readDesired(statePath(config));
 
+	const live = createBridgeLiveConfig(configPath(), config);
+
+	const redact = createBridgeRedaction(secretFieldKeys(driver), live);
+
+	const log = redactLogger(stdoutLog, redact);
+
 	let installError: string | null = null;
 
 	let installErrorText: LocalizedText | null = null;
 
 	let installHold: BridgeInstallHold | null = null;
 
-	const setup = createSetupHandlers(driver, log);
+	const setup = createSetupHandlers(driver, log, redact);
 
 	const supervisor = createSupervisor({
 		cwd: config.cwd,
 		uid: config.gameUid,
 		gid: config.gameGid,
 		stopTimeoutSeconds: driver.lifecycle.stopTimeoutSeconds ?? DEFAULT_STOP_TIMEOUT_SECONDS,
+		readyTimeoutSeconds: driver.lifecycle.readyTimeoutSeconds,
+		stopSignal: driver.lifecycle.stopSignal,
+		subject: driver.lifecycle.subject,
 		ready: driver.lifecycle.ready,
+		failed: driver.lifecycle.failed,
+		redact,
 		onLog: log,
 		async resolveCommand() {
+			live.reload();
+
 			return await driver.lifecycle.command(context);
 		},
+		resolveEnvironment: driver.lifecycle.environment
+			? async () => {
+					return (await driver.lifecycle.environment?.(context)) ?? {};
+				}
+			: undefined,
 		async gracefulStop() {
 			await driver.lifecycle.stop?.(context);
 		},
@@ -110,15 +130,29 @@ export const serve = async (driver: BridgeDriver) => {
 
 	const ring = createEventRing(driver.events, supervisor);
 
-	const context = createContext(config, supervisor, {
-		onLog: log,
-		onEmit(event, payload) {
-			ring.emit(event, payload);
-		},
-		onSetup(runtime) {
-			setup.onSetup(runtime);
+	const context = createContext(
+		config,
+		supervisor,
+		{
+			onLog: log,
+			onEmit(event, payload) {
+				ring.emit(event, payload);
+			},
+			onSetup(runtime) {
+				setup.onSetup(runtime);
+			},
+			async onRestart() {
+				if (current !== BridgeDesired.Running) {
+					throw new Error(
+						`the ${driver.lifecycle.subject ?? BridgeLifecycleSubject.Game} is not meant to be running, so there is nothing to restart`,
+					);
+				}
+
+				await supervisor.restart();
+			},
 		},
-	});
+		live,
+	);
 
 	const registry = createRegistry(driver, context, supervisor, ring);
 	const backupLease = createBackupLease(async () => {
@@ -219,6 +253,7 @@ export const serve = async (driver: BridgeDriver) => {
 	const server = createServer({
 		socket: socketPath(config),
 		token: config.token,
+		redact,
 		onLog: log,
 		onStatus: status,
 		onManifest() {
@@ -427,8 +462,8 @@ export const serve = async (driver: BridgeDriver) => {
 
 		log("install complete");
 	} catch (error) {
-		installError = errorText(error).slice(0, BRIDGE_INSTALL_ERROR_LIMIT);
-		installErrorText = error instanceof BridgeUserError ? error.text : null;
+		installError = redact(errorText(error)).slice(0, BRIDGE_INSTALL_ERROR_LIMIT);
+		installErrorText = error instanceof BridgeUserError ? redactLocalized(error.text, redact) : null;
 
 		supervisor.setRuntime(BridgeRuntime.Crashed);