@serverkgg/bridge 0.0.8 → 0.0.10

package/src/runtime/bridgeSetup.test.ts

@@ -0,0 +1,189 @@
+import { describe, expect, test } from "bun:test";
+import { BridgeSetupPromptKind, BridgeSetupStepState } from "../core";
+import type { BridgeSetupRuntime } from "../protocol";
+import { failedSetup, setupSummaryOf, unsupportedSetup } from "./bridgeSetup";
+
+const text = (value: string) => ({
+	ar: value,
+	en: value,
+});
+
+const runtimeWith = (prompt: BridgeSetupRuntime["prompt"]): BridgeSetupRuntime => ({
+	steps: [
+		{
+			id: "sign-in",
+			state: BridgeSetupStepState.Active,
+		},
+		{
+			id: "download",
+			state: BridgeSetupStepState.Pending,
+		},
+	],
+	prompt,
+});
+
+describe("the setup summary", () => {
+	test("has nothing to summarise while the driver reported no runtime", () => {
+		expect(setupSummaryOf(null)).toBeNull();
+	});
+
+	test("names the active driver step", () => {
+		expect(
+			setupSummaryOf(
+				runtimeWith({
+					kind: BridgeSetupPromptKind.Wait,
+					message: text("starting your server"),
+				}),
+			),
+		).toEqual({
+			step: "sign-in",
+			attention: false,
+		});
+	});
+
+	test("names no step once every driver step is done", () => {
+		expect(
+			setupSummaryOf({
+				steps: [
+					{
+						id: "sign-in",
+						state: BridgeSetupStepState.Done,
+					},
+				],
+				prompt: null,
+			}),
+		).toEqual({
+			step: null,
+			attention: false,
+		});
+	});
+
+	test("asks for attention while a link prompt waits on the customer", () => {
+		expect(
+			setupSummaryOf(
+				runtimeWith({
+					kind: BridgeSetupPromptKind.Link,
+					message: text("sign in with your account"),
+					url: "https://hytale.com/device",
+					code: "ABCD-EFGH",
+					expiresAt: null,
+					actions: [],
+				}),
+			)?.attention,
+		).toBe(true);
+	});
+
+	test("asks for attention on an action prompt and on a failed one", () => {
+		expect(
+			setupSummaryOf(
+				runtimeWith({
+					kind: BridgeSetupPromptKind.Action,
+					message: text("get a new code"),
+					actions: [],
+				}),
+			)?.attention,
+		).toBe(true);
+
+		expect(
+			setupSummaryOf(
+				runtimeWith({
+					kind: BridgeSetupPromptKind.Failed,
+					message: text("the sign-in expired"),
+					actions: [],
+				}),
+			)?.attention,
+		).toBe(true);
+	});
+
+	test("waits quietly while a progress prompt runs", () => {
+		expect(
+			setupSummaryOf(
+				runtimeWith({
+					kind: BridgeSetupPromptKind.Progress,
+					message: text("downloading the game"),
+					percent: 10,
+					detail: "146.6 MiB / 1.4 GiB",
+				}),
+			)?.attention,
+		).toBe(false);
+	});
+});
+
+describe("the unsupported sentinel", () => {
+	test("answers a driver that declares no setup", () => {
+		expect(unsupportedSetup()).toEqual({
+			supported: false,
+			runtime: null,
+		});
+	});
+});
+
+describe("a rejected submit", () => {
+	test("turns the user error into a failed prompt keeping the previous actions", () => {
+		const result = failedSetup(
+			runtimeWith({
+				kind: BridgeSetupPromptKind.Link,
+				message: text("sign in with your account"),
+				url: "https://hytale.com/device",
+				code: "ABCD-EFGH",
+				expiresAt: null,
+				actions: [
+					{
+						id: "renew",
+						label: text("Get a new code"),
+					},
+				],
+			}),
+			text("this code cannot be renewed any more"),
+		);
+
+		expect(result).toEqual({
+			supported: true,
+			runtime: {
+				steps: [
+					{
+						id: "sign-in",
+						state: BridgeSetupStepState.Active,
+					},
+					{
+						id: "download",
+						state: BridgeSetupStepState.Pending,
+					},
+				],
+				prompt: {
+					kind: BridgeSetupPromptKind.Failed,
+					message: text("this code cannot be renewed any more"),
+					actions: [
+						{
+							id: "renew",
+							label: text("Get a new code"),
+						},
+					],
+				},
+			},
+		});
+	});
+
+	test("carries no actions over from a prompt that had none", () => {
+		const result = failedSetup(
+			runtimeWith({
+				kind: BridgeSetupPromptKind.Wait,
+				message: text("starting your server"),
+			}),
+			text("the server never came up"),
+		);
+
+		expect(result.runtime?.prompt).toEqual({
+			kind: BridgeSetupPromptKind.Failed,
+			message: text("the server never came up"),
+			actions: [],
+		});
+	});
+
+	test("stays empty when the driver never reported a runtime", () => {
+		expect(failedSetup(null, text("nothing to fail"))).toEqual({
+			supported: true,
+			runtime: null,
+		});
+	});
+});

package/src/runtime/bridgeSetup.ts

@@ -0,0 +1,63 @@
+import { BridgeSetupPromptKind, BridgeSetupStepState, type LocalizedText } from "../core";
+import type {
+	BridgeSetupAction,
+	BridgeSetupPrompt,
+	BridgeSetupResult,
+	BridgeSetupRuntime,
+	BridgeSetupSummary,
+} from "../protocol";
+
+const ATTENTION_KINDS: ReadonlySet<BridgeSetupPromptKind> = new Set([
+	BridgeSetupPromptKind.Link,
+	BridgeSetupPromptKind.Action,
+	BridgeSetupPromptKind.Failed,
+]);
+
+const actionsOf = (prompt: BridgeSetupPrompt | null): BridgeSetupAction[] => {
+	if (!prompt || prompt.kind === BridgeSetupPromptKind.Wait || prompt.kind === BridgeSetupPromptKind.Progress) {
+		return [];
+	}
+
+	return prompt.actions;
+};
+
+export const setupSummaryOf = (runtime: BridgeSetupRuntime | null): BridgeSetupSummary | null => {
+	if (!runtime) {
+		return null;
+	}
+
+	const active = runtime.steps.find((step) => step.state === BridgeSetupStepState.Active);
+
+	return {
+		step: active?.id ?? null,
+		attention: runtime.prompt !== null && ATTENTION_KINDS.has(runtime.prompt.kind),
+	};
+};
+
+export const unsupportedSetup = (): BridgeSetupResult => {
+	return {
+		supported: false,
+		runtime: null,
+	};
+};
+
+export const failedSetup = (runtime: BridgeSetupRuntime | null, text: LocalizedText): BridgeSetupResult => {
+	if (!runtime) {
+		return {
+			supported: true,
+			runtime: null,
+		};
+	}
+
+	return {
+		supported: true,
+		runtime: {
+			steps: runtime.steps,
+			prompt: {
+				kind: BridgeSetupPromptKind.Failed,
+				message: text,
+				actions: actionsOf(runtime.prompt),
+			},
+		},
+	};
+};

package/src/runtime/bridgeSignal.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+import { leadsGroup, readProcessGroup, signalGroup } from "./bridgeSignal";
+
+const MISSING_PID = 2 ** 22 - 1;
+
+describe("readProcessGroup", () => {
+	test("reads the process group of this process", async () => {
+		const group = await readProcessGroup(process.pid);
+
+		expect(group).not.toBeNull();
+		expect(group).toBeGreaterThan(0);
+	});
+
+	test("puts a detached child in a group of its own and an attached one in ours", async () => {
+		const detached = Bun.spawn(
+			[
+				"sleep",
+				"5",
+			],
+			{
+				detached: true,
+				stdout: "ignore",
+				stderr: "ignore",
+			},
+		);
+
+		const attached = Bun.spawn(
+			[
+				"sleep",
+				"5",
+			],
+			{
+				stdout: "ignore",
+				stderr: "ignore",
+			},
+		);
+
+		try {
+			expect(await readProcessGroup(detached.pid)).toBe(detached.pid);
+			expect(await readProcessGroup(attached.pid)).toBe(await readProcessGroup(process.pid));
+		} finally {
+			detached.kill("SIGKILL");
+			attached.kill("SIGKILL");
+
+			await Promise.all([
+				detached.exited,
+				attached.exited,
+			]);
+		}
+	});
+
+	test("answers null for a pid that does not exist", async () => {
+		expect(await readProcessGroup(MISSING_PID)).toBeNull();
+	});
+});
+
+describe("leadsGroup", () => {
+	test("only a pid equal to its own group leads it", () => {
+		expect(leadsGroup(1234, 1234)).toBe(true);
+		expect(leadsGroup(1234, 1200)).toBe(false);
+		expect(leadsGroup(1234, null)).toBe(false);
+	});
+});
+
+describe("signalGroup", () => {
+	test("signals every process sharing the leader's group", async () => {
+		const leader = Bun.spawn(
+			[
+				"bash",
+				"-c",
+				"sleep 30 & echo $!; wait",
+			],
+			{
+				detached: true,
+				stdout: "pipe",
+				stderr: "ignore",
+			},
+		);
+
+		const first = await leader.stdout.getReader().read();
+		const child = Number(new TextDecoder().decode(first.value).trim());
+
+		expect(Number.isSafeInteger(child)).toBe(true);
+
+		expect(signalGroup(leader.pid, await readProcessGroup(leader.pid), "SIGKILL")).toBe(true);
+
+		await leader.exited;
+		await Bun.sleep(200);
+
+		expect(() => process.kill(child, 0)).toThrow();
+	}, 20_000);
+
+	test("refuses to signal a group the pid does not lead", async () => {
+		const attached = Bun.spawn(
+			[
+				"sleep",
+				"5",
+			],
+			{
+				stdout: "ignore",
+				stderr: "ignore",
+			},
+		);
+
+		try {
+			expect(signalGroup(attached.pid, await readProcessGroup(attached.pid), "SIGKILL")).toBe(false);
+			expect(attached.killed).toBe(false);
+		} finally {
+			attached.kill("SIGKILL");
+
+			await attached.exited;
+		}
+	});
+});

package/src/runtime/bridgeSignal.ts

@@ -0,0 +1,31 @@
+const STAT_PGID_INDEX = 2;
+
+export const readProcessGroup = async (pid: number) => {
+	try {
+		const stat = await Bun.file(`/proc/${pid}/stat`).text();
+		const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+		const group = Number(fields.at(STAT_PGID_INDEX));
+
+		return Number.isSafeInteger(group) && group > 0 ? group : null;
+	} catch {
+		return null;
+	}
+};
+
+export const leadsGroup = (pid: number, group: number | null) => {
+	return group !== null && group === pid;
+};
+
+export const signalGroup = (pid: number, group: number | null, signal: NodeJS.Signals) => {
+	if (!leadsGroup(pid, group)) {
+		return false;
+	}
+
+	try {
+		process.kill(-pid, signal);
+
+		return true;
+	} catch {
+		return false;
+	}
+};

package/src/runtime/bridgeSupervisor.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, test } from "bun:test";
+import { tmpdir } from "node:os";
+import { BridgeRuntime } from "../core";
+import type { BridgeLeveledLogger } from "./bridgeLog";
+import { readProcessGroup } from "./bridgeSignal";
+import { createSupervisor } from "./bridgeSupervisor";
+
+const CHILD_PATTERN = /child=(?<pid>\d+)/;
+
+const WRAPPER = "sleep 300 & echo child=$!; wait";
+
+const STUBBORN_WRAPPER = 'trap "" TERM; sleep 300 & echo child=$!; wait';
+
+const silent: BridgeLeveledLogger = Object.assign(() => {}, {
+	warn: () => {},
+	error: () => {},
+});
+
+const alive = (pid: number) => {
+	try {
+		process.kill(pid, 0);
+
+		return true;
+	} catch {
+		return false;
+	}
+};
+
+const waitGone = async (pid: number, timeoutMs: number) => {
+	for (let waited = 0; waited < timeoutMs; waited += 50) {
+		if (!alive(pid)) {
+			return true;
+		}
+
+		await Bun.sleep(50);
+	}
+
+	return !alive(pid);
+};
+
+const wrapperSupervisor = (script: string, stopTimeoutSeconds: number) => {
+	return createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script,
+			];
+		},
+		onLog: silent,
+	});
+};
+
+const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
+	const supervisor = wrapperSupervisor(script, stopTimeoutSeconds);
+	const started = supervisor.watch(CHILD_PATTERN, 10_000);
+
+	await supervisor.start();
+
+	const match = await started;
+	const pid = Number(match?.groups?.pid);
+
+	expect(match, "the wrapper never reported the pid it spawned").not.toBeNull();
+	expect(Number.isSafeInteger(pid)).toBe(true);
+
+	return {
+		supervisor,
+		pid,
+	};
+};
+
+describe("createSupervisor", () => {
+	test("runs the game outside the bridge's own process group", async () => {
+		const { supervisor, pid } = await startWrapper(WRAPPER, 1);
+
+		try {
+			const gameGroup = await readProcessGroup(pid);
+			const bridgeGroup = await readProcessGroup(process.pid);
+
+			expect(gameGroup, "the game must land in a group of its own or a group kill would hit the bridge").not.toBe(
+				bridgeGroup,
+			);
+		} finally {
+			await supervisor.stop();
+		}
+
+		expect(await waitGone(pid, 5000)).toBe(true);
+	}, 20_000);
+
+	test("takes the wrapper's child down with it when the stop times out", async () => {
+		const { supervisor, pid } = await startWrapper(WRAPPER, 1);
+
+		await supervisor.stop();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+		expect(await waitGone(pid, 5000), "the game was orphaned when the wrapper was signalled").toBe(true);
+	}, 20_000);
+
+	test("kills the wrapper's child through the escalation when the wrapper ignores SIGTERM", async () => {
+		const { supervisor, pid } = await startWrapper(STUBBORN_WRAPPER, 1);
+
+		await supervisor.stop();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+		expect(await waitGone(pid, 5000), "the game outlived the SIGKILL escalation").toBe(true);
+	}, 30_000);
+});

package/src/runtime/bridgeSupervisor.ts

@@ -1,8 +1,10 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { errorText } from "./bridgeLog";
 import { settleUid, unprivilegedArgv } from "./bridgePrivilege";
+import { readProcessGroup, signalGroup } from "./bridgeSignal";
 
 const TAIL_LIMIT = 500;
 
@@ -18,6 +20,8 @@ const RESTART_LIMIT = 5;
 
 const GAME_OOM_SCORE_ADJ = 500;
 
+const SIGKILL_EXIT_CODE = 128 + 9;
+
 export interface SupervisorLine {
 	stream: BridgeStream;
 	line: string;
@@ -37,6 +41,18 @@ export interface SupervisorOptions {
 
 type LineWatcher = (entry: SupervisorLine) => void;
 
+type SpawnedGame = Bun.Subprocess<"pipe", "pipe", "pipe">;
+
+const terminate = (spawned: SpawnedGame, group: number | null, signal: NodeJS.Signals) => {
+	if (!signalGroup(spawned.pid, group, signal)) {
+		spawned.kill(signal);
+	}
+};
+
+const sweep = (pid: number, group: number | null) => {
+	signalGroup(pid, group, "SIGKILL");
+};
+
 const preferOomVictim = async (pid: number) => {
 	try {
 		await Bun.write(`/proc/${pid}/oom_score_adj`, `${GAME_OOM_SCORE_ADJ}\n`);
@@ -108,11 +124,13 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	const watchers = new Set<LineWatcher>();
 	const restartStamps: number[] = [];
 
-	let child: Bun.Subprocess<"pipe", "pipe", "pipe"> | null = null;
+	let child: SpawnedGame | null = null;
+	let childGroup: number | null = null;
 	let runtime: BridgeRuntime = BridgeRuntime.Stopped;
 	let shouldRun = false;
 	let breaker = false;
 	let exitCode: number | null = null;
+	let outOfMemory = false;
 	let restarts = 0;
 	let queue: Promise<unknown> = Promise.resolve();
 
@@ -180,7 +198,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			}, timeoutMs);
 
 			const watcher: LineWatcher = (entry) => {
-				const match = entry.line.match(pattern);
+				const match = stripAnsi(entry.line).match(pattern);
 
 				if (!match) {
 					return;
@@ -222,6 +240,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
 			cwd: options.cwd,
+			detached: true,
 			stderr: "pipe",
 			stdin: "pipe",
 			stdout: "pipe",
@@ -230,6 +249,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			},
 		});
 
+		const group = await readProcessGroup(spawned.pid);
 		const observed = await settleUid(spawned.pid, options.uid);
 
 		if (observed !== options.uid) {
@@ -244,10 +264,12 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				);
 			}
 
-			spawned.kill("SIGKILL");
+			terminate(spawned, group, "SIGKILL");
 
 			await spawned.exited;
 
+			sweep(spawned.pid, group);
+
 			throw new Error(
 				`the game process runs as uid ${String(observed)}, expected ${options.uid} — refusing to run it as the bridge`,
 			);
@@ -256,6 +278,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		await preferOomVictim(spawned.pid);
 
 		child = spawned;
+		childGroup = group;
 		exitCode = null;
 
 		void pump(spawned.stdout, BridgeStream.Stdout);
@@ -263,6 +286,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 		options.onLog("game started", {
 			pid: spawned.pid,
+			group,
 			uid: options.uid,
 		});
 
@@ -303,10 +327,20 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	};
 
 	function handleExit(code: number) {
+		const finished = child;
+
 		child = null;
 		exitCode = code;
 
+		if (finished) {
+			sweep(finished.pid, childGroup);
+		}
+
+		childGroup = null;
+
 		if (!shouldRun) {
+			outOfMemory = false;
+
 			runtime = BridgeRuntime.Stopped;
 
 			options.onLog("game stopped", {
@@ -316,6 +350,14 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return;
 		}
 
+		outOfMemory = code === SIGKILL_EXIT_CODE;
+
+		if (outOfMemory) {
+			options.onLog.error("the game was killed for running out of memory", {
+				code,
+			});
+		}
+
 		restarts += 1;
 		restartStamps.push(performance.now());
 
@@ -375,6 +417,10 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return exitCode;
 		},
 
+		get outOfMemory() {
+			return outOfMemory;
+		},
+
 		get restarts() {
 			return restarts;
 		},
@@ -382,6 +428,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		async start() {
 			shouldRun = true;
 			breaker = false;
+			outOfMemory = false;
 			restartStamps.length = 0;
 
 			if (child) {
@@ -401,6 +448,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			}
 
 			const active = child;
+			const group = childGroup;
 
 			runtime = BridgeRuntime.Stopping;
 
@@ -423,7 +471,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			]);
 
 			if (settled === null) {
-				active.kill("SIGTERM");
+				terminate(active, group, "SIGTERM");
 
 				const afterTerm = await Promise.race([
 					active.exited,
@@ -431,12 +479,14 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				]);
 
 				if (afterTerm === null) {
-					active.kill("SIGKILL");
+					terminate(active, group, "SIGKILL");
 
 					await active.exited;
 				}
 			}
 
+			sweep(active.pid, group);
+
 			runtime = BridgeRuntime.Stopped;
 		},