@serverkgg/bridge 0.0.21 → 0.1.2

package/src/runtime/bridgeSupervisor.test.ts

@@ -1,8 +1,10 @@
-import { describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
 import { tmpdir } from "node:os";
 import { BridgeRuntime } from "../core";
+import { BridgeLifecycleSubject, BridgeStopSignal } from "../modules";
 import { BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import type { BridgeLeveledLogger } from "./bridgeLog";
+import { createBridgeRedaction, REDACTED } from "./bridgeRedaction";
 import { readProcessGroup } from "./bridgeSignal";
 import { type BridgeSupervisor, createSupervisor } from "./bridgeSupervisor";
 
@@ -142,7 +144,7 @@ const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	};
 };
 
-const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: number) => {
+const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: number, readyTimeoutSeconds?: number) => {
 	const records: BridgeCrashRecord[] = [];
 
 	let stables = 0;
@@ -152,6 +154,7 @@ const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: nu
 		uid: process.getuid?.() ?? 0,
 		gid: process.getgid?.() ?? 0,
 		stopTimeoutSeconds: 1,
+		readyTimeoutSeconds,
 		ready,
 		stableAfterMs,
 		onLog: silent,
@@ -435,6 +438,56 @@ describe("createSupervisor", () => {
 		await supervisor.stop();
 	}, 30_000);
 
+	test("keeps a slow boot starting past the stable window when the driver declares a ready timeout", async () => {
+		const { supervisor, records, stables } = timedSupervisor(() => LATE_CRASH, NEVER_READY, 300, 30);
+
+		await supervisor.start();
+
+		await Bun.sleep(600);
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(stables()).toBe(0);
+	}, 30_000);
+
+	test("fails the boot when the declared ready timeout passes, never flipping it to running", async () => {
+		const { supervisor, records, stables } = timedSupervisor(() => "exec sleep 300", NEVER_READY, 60_000, 1);
+
+		await supervisor.start();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(record?.outOfMemory).toBe(false);
+		expect(record?.lastLine).toBe("the game did not report ready within 1 seconds");
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.running).toBe(false);
+		expect(stables()).toBe(0);
+	}, 30_000);
+
+	test("still treats a game with no declared ready timeout as running once the default wait passes", async () => {
+		const { supervisor, stables } = timedSupervisor(() => "exec sleep 300", NEVER_READY, 60_000);
+
+		await supervisor.start();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+		expect(await waitFor(() => supervisor.runtime === BridgeRuntime.Running, 15_000)).toBe(true);
+		expect(stables()).toBe(1);
+
+		await supervisor.stop();
+	}, 30_000);
+
 	test("opens a new episode when start is asked after the breaker tripped", async () => {
 		let script = RUNNING_CRASH;
 
@@ -609,3 +662,618 @@ describe("createSupervisor", () => {
 		await supervisor.stop();
 	}, 30_000);
 });
+
+const PID_LINE = /^pid=(?<pid>\d+)$/;
+
+const READY_THEN_WAIT = "echo pid=$$; echo ready; exec sleep 300";
+
+const restartSupervisor = (script: () => string, resolveEnvironment?: () => Promise<Record<string, string>>) => {
+	const records: BridgeCrashRecord[] = [];
+	const pids: number[] = [];
+
+	let stopped = 0;
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready: READY,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script(),
+			];
+		},
+		resolveEnvironment,
+		onStopped() {
+			stopped += 1;
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	supervisor.subscribe((entry) => {
+		const pid = Number(entry.line.match(PID_LINE)?.groups?.pid);
+
+		if (Number.isSafeInteger(pid)) {
+			pids.push(pid);
+		}
+	});
+
+	return {
+		supervisor,
+		records,
+		pids,
+		stopped: () => stopped,
+	};
+};
+
+describe("the resolved environment", () => {
+	test("reaches the game beside what the bridge inherited", async () => {
+		const { supervisor } = restartSupervisor(
+			() => 'echo "probe=$SERVERK_PROBE"; echo "path=$PATH"; echo ready; exec sleep 300',
+			async () => ({
+				SERVERK_PROBE: "from the driver",
+			}),
+		);
+		const probe = supervisor.watch(/^probe=(?<value>.*)$/, 10_000);
+		const path = supervisor.watch(/^path=(?<value>.*)$/, 10_000);
+
+		await supervisor.start();
+
+		expect((await probe)?.groups?.value).toBe("from the driver");
+		expect((await path)?.groups?.value).toBe(process.env.PATH ?? "");
+
+		await supervisor.stop();
+	}, 20_000);
+
+	test("leaves the inherited environment alone when the driver resolves none", async () => {
+		const { supervisor } = restartSupervisor(() => 'echo "path=$PATH"; echo ready; exec sleep 300');
+		const path = supervisor.watch(/^path=(?<value>.*)$/, 10_000);
+
+		await supervisor.start();
+
+		const value = (await path)?.groups?.value ?? "";
+
+		expect(value.length).toBeGreaterThan(0);
+		expect(value).toBe(process.env.PATH ?? "");
+
+		await supervisor.stop();
+	}, 20_000);
+
+	test("refuses a key that is not a variable name as a start failure", async () => {
+		const { supervisor } = restartSupervisor(
+			() => READY_THEN_WAIT,
+			async () => ({
+				"BAD-KEY": "value",
+			}),
+		);
+
+		await expect(supervisor.start()).rejects.toThrow('the game environment names "BAD-KEY"');
+		expect(supervisor.running).toBe(false);
+	});
+
+	test("refuses a value carrying a NUL byte without echoing it", async () => {
+		const { supervisor } = restartSupervisor(
+			() => READY_THEN_WAIT,
+			async () => ({
+				BOT_TOKEN: "secret\0tail",
+			}),
+		);
+
+		const failure = await supervisor.start().then(
+			() => null,
+			(error: unknown) => (error instanceof Error ? error.message : String(error)),
+		);
+
+		expect(failure).toBe('the game environment value for "BOT_TOKEN" contains a NUL byte');
+		expect(supervisor.running).toBe(false);
+	});
+});
+
+describe("restart", () => {
+	test("replaces the process without ever reporting it stopped", async () => {
+		const { supervisor, records, pids, stopped } = restartSupervisor(() => READY_THEN_WAIT);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => pids.length === 1 && supervisor.runtime === BridgeRuntime.Running, 10_000)).toBe(true);
+
+		const seen = new Set<BridgeRuntime>();
+
+		let sampling = true;
+
+		const sampler = (async () => {
+			while (sampling) {
+				seen.add(supervisor.runtime);
+
+				await Bun.sleep(5);
+			}
+		})();
+
+		await supervisor.restart();
+
+		expect(await waitFor(() => pids.length === 2 && supervisor.runtime === BridgeRuntime.Running, 10_000)).toBe(true);
+
+		sampling = false;
+
+		await sampler;
+
+		const [first, second] = pids;
+
+		expect(second).not.toBe(first);
+		expect(await waitGone(first ?? 0, 5000), "the replaced process outlived the restart").toBe(true);
+		expect(seen.has(BridgeRuntime.Stopping)).toBe(true);
+		expect(seen.has(BridgeRuntime.Stopped), "a restart reported the game stopped").toBe(false);
+		expect(stopped(), "a restart ran the stopped hook").toBe(0);
+		expect(records, "a restart was recorded as a crash").toHaveLength(0);
+		expect(supervisor.restarts).toBe(0);
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+
+		expect(stopped()).toBe(1);
+	}, 30_000);
+
+	test("counts a quick exit after a restart as a boot crash, not a running one", async () => {
+		let script = READY_THEN_WAIT;
+
+		const { supervisor, records, pids } = restartSupervisor(() => script);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => pids.length === 1 && supervisor.runtime === BridgeRuntime.Running, 10_000)).toBe(true);
+
+		script = BOOT_CRASH;
+
+		await supervisor.restart();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(record?.exitCode).toBe(3);
+		expect(record?.restarts).toBe(0);
+
+		await Bun.sleep(3000);
+
+		expect(records, "the boot crash after a restart was retried").toHaveLength(1);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(0);
+	}, 30_000);
+
+	test("refuses to restart a game that is meant to be stopped", async () => {
+		const { supervisor } = restartSupervisor(() => READY_THEN_WAIT);
+
+		await expect(supervisor.restart()).rejects.toThrow("the game is not meant to be running");
+		expect(supervisor.running).toBe(false);
+	});
+});
+
+const SIGNAL_EXITS = "echo pid=$$; echo ready; exec sleep 300";
+
+const SIGNAL_IGNORED = 'trap "" TERM INT; sleep 300 & echo child=$!; echo ready; wait';
+
+const signalSupervisor = (script: string, stopTimeoutSeconds: number, stopSignal?: BridgeStopSignal) => {
+	const pids: number[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds,
+		stopSignal,
+		ready: READY,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script,
+			];
+		},
+	});
+
+	supervisor.subscribe((entry) => {
+		const pid = Number((entry.line.match(PID_LINE) ?? entry.line.match(CHILD_PATTERN))?.groups?.pid);
+
+		if (Number.isSafeInteger(pid)) {
+			pids.push(pid);
+		}
+	});
+
+	return {
+		supervisor,
+		pids,
+	};
+};
+
+const timed = async (run: () => Promise<void>) => {
+	const started = performance.now();
+
+	await run();
+
+	return performance.now() - started;
+};
+
+const runningWith = async (supervisor: BridgeSupervisor, pids: number[], count: number) => {
+	return await waitFor(() => pids.length === count && supervisor.runtime === BridgeRuntime.Running, 10_000);
+};
+
+describe("the stop signal", () => {
+	test("stops a game that exits on SIGTERM well inside the timeout", async () => {
+		const { supervisor, pids } = signalSupervisor(SIGNAL_EXITS, 10, BridgeStopSignal.Term);
+
+		await supervisor.start();
+
+		expect(await runningWith(supervisor, pids, 1)).toBe(true);
+
+		const elapsed = await timed(() => supervisor.stop());
+
+		expect(elapsed, "the stop waited for the timeout instead of signalling first").toBeLessThan(3000);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+		expect(await waitGone(pids[0] ?? 0, 5000)).toBe(true);
+	}, 30_000);
+
+	test("stops a game that exits on SIGINT when the driver asks for it", async () => {
+		const { supervisor, pids } = signalSupervisor(SIGNAL_EXITS, 10, BridgeStopSignal.Interrupt);
+
+		await supervisor.start();
+
+		expect(await runningWith(supervisor, pids, 1)).toBe(true);
+
+		const elapsed = await timed(() => supervisor.stop());
+
+		expect(elapsed).toBeLessThan(3000);
+		expect(await waitGone(pids[0] ?? 0, 5000)).toBe(true);
+	}, 30_000);
+
+	test("still kills a game that ignores the signal once the timeout passes", async () => {
+		const { supervisor, pids } = signalSupervisor(SIGNAL_IGNORED, 1, BridgeStopSignal.Term);
+
+		await supervisor.start();
+
+		expect(await runningWith(supervisor, pids, 1)).toBe(true);
+
+		const elapsed = await timed(() => supervisor.stop());
+
+		expect(elapsed, "the kill came before the timeout").toBeGreaterThanOrEqual(900);
+		expect(elapsed, "the kill waited for the SIGTERM grace a signalled stop does not take").toBeLessThan(4000);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+		expect(await waitGone(pids[0] ?? 0, 5000), "the ignoring child outlived the kill").toBe(true);
+	}, 30_000);
+
+	test("keeps waiting the timeout before SIGTERM when the driver declares no signal", async () => {
+		const { supervisor, pids } = signalSupervisor(SIGNAL_EXITS, 2);
+
+		await supervisor.start();
+
+		expect(await runningWith(supervisor, pids, 1)).toBe(true);
+
+		const elapsed = await timed(() => supervisor.stop());
+
+		expect(elapsed, "the stop signalled before the timeout without being asked to").toBeGreaterThanOrEqual(1900);
+		expect(elapsed).toBeLessThan(6000);
+		expect(await waitGone(pids[0] ?? 0, 5000)).toBe(true);
+	}, 30_000);
+
+	test("restarts through the same signal-first path", async () => {
+		const { supervisor, pids } = signalSupervisor(SIGNAL_EXITS, 10, BridgeStopSignal.Term);
+
+		await supervisor.start();
+
+		expect(await runningWith(supervisor, pids, 1)).toBe(true);
+
+		const elapsed = await timed(() => supervisor.restart());
+
+		expect(elapsed, "the restart waited for the timeout instead of signalling first").toBeLessThan(3000);
+		expect(await runningWith(supervisor, pids, 2)).toBe(true);
+		expect(pids[1]).not.toBe(pids[0]);
+		expect(await waitGone(pids[0] ?? 0, 5000)).toBe(true);
+		expect(supervisor.restarts).toBe(0);
+
+		await supervisor.stop();
+	}, 30_000);
+});
+
+const LEAKED = "tok-1234567890-secret";
+
+const LEAK_AND_CRASH = `echo "out ${LEAKED}"; echo "err ${LEAKED}" >&2; sleep 0.2; echo "fatal: bad token ${LEAKED}"; exit 3`;
+
+const TICKING = `while true; do echo "tick ${LEAKED}"; sleep 0.1; done`;
+
+const redactingSupervisor = (script: string, redact?: (text: string) => string) => {
+	const records: BridgeCrashRecord[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		stopSignal: BridgeStopSignal.Term,
+		onLog: silent,
+		redact,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script,
+			];
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+	};
+};
+
+const variablesSource = (initial: Record<string, string>, secrets: Record<string, string> = {}) => {
+	let variables: Readonly<Record<string, string>> = initial;
+
+	return {
+		source: {
+			variables: () => variables,
+			secrets: () => secrets,
+		},
+		save(next: Record<string, string>) {
+			variables = next;
+		},
+	};
+};
+
+describe("secret redaction", () => {
+	let written: string[] = [];
+	let restore = () => {};
+
+	beforeEach(() => {
+		written = [];
+
+		const capture = (chunk: unknown) => {
+			written.push(String(chunk));
+
+			return true;
+		};
+		const out = spyOn(process.stdout, "write").mockImplementation(capture);
+		const err = spyOn(process.stderr, "write").mockImplementation(capture);
+
+		restore = () => {
+			out.mockRestore();
+			err.mockRestore();
+		};
+	});
+
+	afterEach(() => {
+		restore();
+	});
+
+	test("hides a secret value from stdout, stderr, the stored lines and the crash last line", async () => {
+		const redact = createBridgeRedaction(
+			[
+				"BOT_TOKEN",
+			],
+			variablesSource({
+				BOT_TOKEN: LEAKED,
+			}).source,
+		);
+		const { supervisor, records } = redactingSupervisor(LEAK_AND_CRASH, redact);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const lines = supervisor.tail(10);
+
+		expect(lines).toContain(`out ${REDACTED}`);
+		expect(lines).toContain(`err ${REDACTED}`);
+		expect(records.at(0)?.lastLine).toBe(`fatal: bad token ${REDACTED}`);
+		expect(lines.join("\n")).not.toContain(LEAKED);
+		expect(written.join("")).not.toContain(LEAKED);
+		expect(written.join("")).toContain(REDACTED);
+	}, 30_000);
+
+	test("hides a platform secret from the manifest that the game prints", async () => {
+		const redact = createBridgeRedaction(
+			[],
+			variablesSource(
+				{},
+				{
+					CURSEFORGE_API_KEY: LEAKED,
+				},
+			).source,
+		);
+		const { supervisor, records } = redactingSupervisor(LEAK_AND_CRASH, redact);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const lines = supervisor.tail(10);
+
+		expect(lines).toContain(`out ${REDACTED}`);
+		expect(lines).toContain(`err ${REDACTED}`);
+		expect(lines.join("\n")).not.toContain(LEAKED);
+		expect(records.at(0)?.lastLine).toBe(`fatal: bad token ${REDACTED}`);
+	}, 30_000);
+
+	test("leaves a value shorter than eight characters in the lines", async () => {
+		const redact = createBridgeRedaction(
+			[
+				"RCON_PASSWORD",
+			],
+			variablesSource({
+				RCON_PASSWORD: "tok-123",
+			}).source,
+		);
+		const { supervisor, records } = redactingSupervisor(LEAK_AND_CRASH, redact);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine).toBe(`fatal: bad token ${LEAKED}`);
+	}, 30_000);
+
+	test("hides a value saved after the start from the lines that follow", async () => {
+		const { source, save } = variablesSource({});
+		const redact = createBridgeRedaction(
+			[
+				"BOT_TOKEN",
+			],
+			source,
+		);
+		const { supervisor } = redactingSupervisor(TICKING, redact);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => supervisor.tail(1).at(0) === `tick ${LEAKED}`, 10_000)).toBe(true);
+
+		save({
+			BOT_TOKEN: LEAKED,
+		});
+
+		const seen = supervisor.tail(500).length;
+
+		expect(await waitFor(() => supervisor.tail(500).length >= seen + 3, 10_000)).toBe(true);
+
+		const following = supervisor.tail(500).slice(seen);
+
+		expect(following.length).toBeGreaterThan(0);
+		expect(following.every((line) => line === `tick ${REDACTED}`)).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("runs a driver with no secret fields and no secrets without touching a line", async () => {
+		const redact = createBridgeRedaction([], variablesSource({}).source);
+
+		const { supervisor, records } = redactingSupervisor(LEAK_AND_CRASH, redact);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(supervisor.tail(10)).toContain(`out ${LEAKED}`);
+		expect(records.at(0)?.lastLine).toBe(`fatal: bad token ${LEAKED}`);
+	}, 30_000);
+});
+
+const recordingLogger = () => {
+	const messages: string[] = [];
+
+	const logger: BridgeLeveledLogger = Object.assign(
+		(message: string) => {
+			messages.push(message);
+		},
+		{
+			warn: (message: string) => {
+				messages.push(message);
+			},
+			error: (message: string) => {
+				messages.push(message);
+			},
+		},
+	);
+
+	return {
+		logger,
+		messages,
+	};
+};
+
+const subjectSupervisor = (script: string, subject?: BridgeLifecycleSubject) => {
+	const { logger, messages } = recordingLogger();
+	const records: BridgeCrashRecord[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready: READY,
+		subject,
+		onLog: logger,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script,
+			];
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		messages,
+		records,
+	};
+};
+
+describe("the lifecycle subject", () => {
+	test("names the game in the lifecycle lines when the driver declares no subject", async () => {
+		const { supervisor, messages } = subjectSupervisor(READY_THEN_WAIT);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => messages.includes("game ready"), 10_000)).toBe(true);
+
+		await supervisor.stop();
+
+		expect(messages).toEqual([
+			"game started",
+			"game ready",
+			"game stopped",
+		]);
+	}, 20_000);
+
+	test("names the bot in every lifecycle line of a bot, the crash loop included", async () => {
+		const { supervisor, messages, records } = subjectSupervisor(RUNNING_CRASH, BridgeLifecycleSubject.Bot);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+
+		await supervisor.stop();
+
+		expect(messages).toContain("bot started");
+		expect(messages).toContain("bot ready");
+		expect(messages).toContain("bot exited, restarting");
+		expect(messages).toContain("bot crash loop detected, not restarting");
+		expect(messages.filter((message) => message.includes("game"))).toEqual([]);
+	}, 40_000);
+
+	test("names the bot when it refuses an environment it cannot pass on", async () => {
+		const { logger } = recordingLogger();
+
+		const supervisor = createSupervisor({
+			cwd: tmpdir(),
+			uid: process.getuid?.() ?? 0,
+			gid: process.getgid?.() ?? 0,
+			stopTimeoutSeconds: 1,
+			subject: BridgeLifecycleSubject.Bot,
+			onLog: logger,
+			resolveCommand: async () => {
+				return [
+					"bash",
+					"-c",
+					READY_THEN_WAIT,
+				];
+			},
+			resolveEnvironment: async () => ({
+				"BAD-KEY": "value",
+			}),
+		});
+
+		await expect(supervisor.start()).rejects.toThrow('the bot environment names "BAD-KEY"');
+	});
+});