@serverkgg/bridge 0.0.14 → 0.0.16

package/src/runtime/bridgeSupervisor.ts

@@ -1,5 +1,6 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { BRIDGE_CRASH_LINE_LIMIT, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { errorText } from "./bridgeLog";
@@ -14,9 +15,11 @@ const KILL_GRACE_MS = 5000;
 
 const RESTART_DELAY_MS = 2000;
 
-const RESTART_WINDOW_MS = 300_000;
+const RESTART_WINDOW_MS = 600_000;
 
-const RESTART_LIMIT = 5;
+const RESTART_LIMIT = 3;
+
+const STABLE_UPTIME_MS = 180_000;
 
 const GAME_OOM_SCORE_ADJ = 500;
 
@@ -35,7 +38,11 @@ export interface SupervisorOptions {
 	ready?: RegExp;
 	resolveCommand: () => Promise<string[]>;
 	gracefulStop?: () => Promise<void>;
+	stableAfterMs?: number;
 	onReady?: () => void;
+	onStable?: () => void;
+	onStopped?: () => void;
+	onExit?: (record: BridgeCrashRecord) => void;
 	onLog: BridgeLeveledLogger;
 }
 
@@ -119,6 +126,15 @@ const splitLines = (buffer: string, chunk: string) => {
 	};
 };
 
+const PUMP_DRAIN_MS = 500;
+
+const drainPumps = async (pumps: readonly Promise<unknown>[]) => {
+	await Promise.race([
+		Promise.all(pumps),
+		Bun.sleep(PUMP_DRAIN_MS),
+	]);
+};
+
 export const createSupervisor = (options: SupervisorOptions) => {
 	const tail: SupervisorLine[] = [];
 	const watchers = new Set<LineWatcher>();
@@ -132,8 +148,21 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	let exitCode: number | null = null;
 	let outOfMemory = false;
 	let restarts = 0;
+	let launchedAt = performance.now();
+	let stable = false;
+	let stableTimer: ReturnType<typeof setTimeout> | null = null;
 	let queue: Promise<unknown> = Promise.resolve();
 
+	const markStable = () => {
+		if (stable) {
+			return;
+		}
+
+		stable = true;
+
+		options.onStable?.();
+	};
+
 	const emit = (entry: SupervisorLine) => {
 		if (entry.stream === BridgeStream.Stderr) {
 			process.stderr.write(`${entry.line}\n`);
@@ -223,6 +252,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		return restartStamps.length >= RESTART_LIMIT;
 	};
 
+	const lastLine = () => {
+		for (let index = tail.length - 1; index >= 0; index -= 1) {
+			const line = stripAnsi(tail.at(index)?.line ?? "").trim();
+
+			if (line.length > 0) {
+				return line.slice(0, BRIDGE_CRASH_LINE_LIMIT);
+			}
+		}
+
+		return null;
+	};
+
 	const launch = async () => {
 		const command = await options.resolveCommand();
 		const [executable, ...rest] = command;
@@ -237,6 +278,34 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		];
 
 		const { argv: spawnArgv, credentials } = await unprivilegedArgv(argv, options.uid, options.gid);
+
+		launchedAt = performance.now();
+		stable = false;
+
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
+		let accepted = false;
+		let reported = false;
+		let pumps: Promise<unknown>[] = [];
+
+		const finish = (code: number) => {
+			if (!accepted || reported) {
+				return;
+			}
+
+			reported = true;
+
+			void handleExit(code, pumps).catch((error: unknown) => {
+				options.onLog.error("the game exit was not reported", {
+					error: errorText(error),
+				});
+			});
+		};
+
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
 			cwd: options.cwd,
@@ -245,7 +314,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			stdin: "pipe",
 			stdout: "pipe",
 			onExit(_subprocess, code, signal) {
-				handleExit(exitCodeOf(code, signal));
+				finish(exitCodeOf(code, signal));
 			},
 		});
 
@@ -281,8 +350,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		childGroup = group;
 		exitCode = null;
 
-		void pump(spawned.stdout, BridgeStream.Stdout);
-		void pump(spawned.stderr, BridgeStream.Stderr);
+		const launchStamp = launchedAt;
+
+		stableTimer = setTimeout(() => {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			markStable();
+		}, options.stableAfterMs ?? STABLE_UPTIME_MS);
+
+		pumps = [
+			pump(spawned.stdout, BridgeStream.Stdout).catch(() => undefined),
+			pump(spawned.stderr, BridgeStream.Stderr).catch(() => undefined),
+		];
+
+		accepted = true;
 
 		options.onLog("game started", {
 			pid: spawned.pid,
@@ -290,6 +373,12 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			uid: options.uid,
 		});
 
+		if (spawned.exitCode !== null || spawned.signalCode !== null) {
+			finish(exitCodeOf(spawned.exitCode, spawned.signalCode));
+
+			return;
+		}
+
 		if (!options.ready) {
 			runtime = BridgeRuntime.Running;
 
@@ -299,12 +388,30 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		runtime = BridgeRuntime.Starting;
 
 		void watch(options.ready, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
-			if (match && runtime === BridgeRuntime.Starting) {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			if (match) {
+				if (runtime === BridgeRuntime.Starting) {
+					runtime = BridgeRuntime.Running;
+
+					options.onLog("game ready");
+					options.onReady?.();
+				}
+
+				markStable();
+
+				return;
+			}
+
+			if (runtime === BridgeRuntime.Starting) {
 				runtime = BridgeRuntime.Running;
 
-				options.onLog("game ready");
-				options.onReady?.();
+				options.onLog.warn("game ready line not seen, treating it as running");
 			}
+
+			markStable();
 		});
 	};
 
@@ -326,12 +433,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		}
 	};
 
-	function handleExit(code: number) {
+	async function handleExit(code: number, pumps: readonly Promise<unknown>[]) {
 		const finished = child;
 
 		child = null;
 		exitCode = code;
 
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
 		if (finished) {
 			sweep(finished.pid, childGroup);
 		}
@@ -350,6 +463,9 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return;
 		}
 
+		const uptimeMs = performance.now() - launchedAt;
+		const phase = stable ? BridgeCrashPhase.Running : BridgeCrashPhase.Booting;
+
 		outOfMemory = code === SIGKILL_EXIT_CODE;
 
 		if (outOfMemory) {
@@ -358,6 +474,38 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			});
 		}
 
+		const report = async (restarting: boolean, looping: boolean) => {
+			const settled = {
+				exitCode: code,
+				outOfMemory,
+				phase,
+				uptimeSeconds: Math.floor(uptimeMs / 1000),
+				restarting,
+				looping,
+				restarts,
+			};
+
+			await drainPumps(pumps);
+
+			options.onExit?.({
+				...settled,
+				lastLine: lastLine(),
+			});
+		};
+
+		if (phase === BridgeCrashPhase.Booting) {
+			runtime = BridgeRuntime.Crashed;
+
+			options.onLog.error("game exited while booting, not restarting", {
+				code,
+				phase,
+			});
+
+			await report(false, false);
+
+			return;
+		}
+
 		restarts += 1;
 		restartStamps.push(performance.now());
 
@@ -370,6 +518,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				restarts,
 			});
 
+			await report(false, true);
+
 			return;
 		}
 
@@ -380,6 +530,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			restarts,
 		});
 
+		await report(true, false);
+
 		void restart();
 	}
 
@@ -429,6 +581,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			shouldRun = true;
 			breaker = false;
 			outOfMemory = false;
+			restarts = 0;
 			restartStamps.length = 0;
 
 			if (child) {
@@ -444,6 +597,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			if (!child) {
 				runtime = BridgeRuntime.Stopped;
 
+				options.onStopped?.();
+
 				return;
 			}
 
@@ -488,6 +643,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			sweep(active.pid, group);
 
 			runtime = BridgeRuntime.Stopped;
+
+			options.onStopped?.();
 		},
 
 		async command(input: string, commandOptions: BridgeCommandOptions = {}): Promise<BridgeReply> {

package/src/runtime/serve.ts

@@ -96,6 +96,15 @@ export const serve = async (driver: BridgeDriver) => {
 				});
 			});
 		},
+		onStable() {
+			ring.discard();
+		},
+		onStopped() {
+			ring.discard();
+		},
+		onExit(record) {
+			ring.crash(record);
+		},
 	});
 
 	const ring = createEventRing(driver.events, supervisor);
@@ -130,6 +139,7 @@ export const serve = async (driver: BridgeDriver) => {
 			exitCode: supervisor.exitCode,
 			outOfMemory: supervisor.outOfMemory,
 			restarts: supervisor.restarts,
+			crashRecords: true,
 			installError,
 			installErrorText,
 			installHold,

package/src/steam/avatars.test.ts

@@ -0,0 +1,306 @@
+import { afterEach, describe, expect, setSystemTime, test } from "bun:test";
+import type { BridgeContext } from "../context";
+import type { BridgeValues } from "../core";
+import { createSteamAvatars, STEAM_AVATAR_CACHE_TTL_MS, steamAvatarHashOf } from "./avatars";
+
+const ALICE_HASH = "a".repeat(40);
+
+const BOB_HASH = "b".repeat(40);
+
+const ALICE = "76561198000000001";
+
+const BOB = "76561198000000002";
+
+const identity = (id: string) => (id.startsWith("7656") ? id : null);
+
+interface Warning {
+	message: string;
+	fields: BridgeValues | undefined;
+}
+
+interface Summary {
+	steamid: string;
+	avatarfull: string;
+}
+
+interface FakeSteam {
+	context: BridgeContext;
+	calls: string[][];
+	warnings: Warning[];
+}
+
+const summaryOf = (steamId: string, hash: string): Summary => {
+	return {
+		steamid: steamId,
+		avatarfull: `https://avatars.steamstatic.com/${hash}_full.jpg`,
+	};
+};
+
+const fakeSteam = (options: { key?: string | null; answer?: (steamIds: string[]) => Summary[] }): FakeSteam => {
+	const calls: string[][] = [];
+	const warnings: Warning[] = [];
+
+	const context = {
+		secret(key: string) {
+			return key === "STEAM_WEB_API_KEY" ? (options.key === undefined ? "secret-key" : options.key) : null;
+		},
+		net: {
+			async json(url: string) {
+				const steamIds = (new URL(url).searchParams.get("steamids") ?? "").split(",").filter((id) => id.length > 0);
+
+				calls.push(steamIds);
+
+				return {
+					response: {
+						players: options.answer?.(steamIds) ?? [],
+					},
+				};
+			},
+		},
+		log: {
+			warn(message: string, fields?: BridgeValues) {
+				warnings.push({
+					message,
+					fields,
+				});
+			},
+		},
+	} as unknown as BridgeContext;
+
+	return {
+		context,
+		calls,
+		warnings,
+	};
+};
+
+afterEach(() => {
+	setSystemTime();
+});
+
+describe("reading the avatar hash out of a steam profile picture", () => {
+	test("takes the hash out of the full-size url steam answers with", () => {
+		expect(steamAvatarHashOf(`https://avatars.steamstatic.com/${ALICE_HASH}_full.jpg`)).toBe(ALICE_HASH);
+	});
+
+	test("takes the hash out of a url that carries no size suffix", () => {
+		expect(steamAvatarHashOf(`https://avatars.steamstatic.com/${BOB_HASH}.jpg`)).toBe(BOB_HASH);
+	});
+
+	test("reports no hash for a url that is not a steam avatar", () => {
+		expect(steamAvatarHashOf("https://example.com/avatar.png")).toBeNull();
+		expect(steamAvatarHashOf("")).toBeNull();
+	});
+});
+
+describe("keeping the steam avatars a roster shows", () => {
+	test("asks steam for a hash it has never seen, and answers with it", async () => {
+		const steam = fakeSteam({
+			answer: () => [
+				summaryOf(ALICE, ALICE_HASH),
+			],
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		expect(
+			await avatars.resolve(steam.context, [
+				ALICE,
+			]),
+		).toEqual(
+			new Map([
+				[
+					ALICE,
+					ALICE_HASH,
+				],
+			]),
+		);
+		expect(steam.calls).toEqual([
+			[
+				ALICE,
+			],
+		]);
+	});
+
+	test("answers the second time without asking steam again", async () => {
+		const steam = fakeSteam({
+			answer: () => [
+				summaryOf(ALICE, ALICE_HASH),
+			],
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		expect(
+			(
+				await avatars.resolve(steam.context, [
+					ALICE,
+				])
+			).get(ALICE),
+		).toBe(ALICE_HASH);
+		expect(steam.calls.length).toBe(1);
+	});
+
+	test("asks only for the ids it is missing, and still answers for the cached one", async () => {
+		const steam = fakeSteam({
+			answer: (steamIds) => {
+				return steamIds.map((steamId) => {
+					return summaryOf(steamId, steamId.endsWith("1") ? ALICE_HASH : BOB_HASH);
+				});
+			},
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		const hashes = await avatars.resolve(steam.context, [
+			ALICE,
+			BOB,
+		]);
+
+		expect(steam.calls.at(1)).toEqual([
+			BOB,
+		]);
+		expect(hashes.get(ALICE)).toBe(ALICE_HASH);
+		expect(hashes.get(BOB)).toBe(BOB_HASH);
+	});
+
+	test("asks steam again once the cached hash is six hours old", async () => {
+		setSystemTime(new Date(0));
+
+		const steam = fakeSteam({
+			answer: () => [
+				summaryOf(ALICE, ALICE_HASH),
+			],
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		setSystemTime(new Date(STEAM_AVATAR_CACHE_TTL_MS - 1));
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		expect(steam.calls.length).toBe(1);
+
+		setSystemTime(new Date(STEAM_AVATAR_CACHE_TTL_MS));
+
+		expect(
+			(
+				await avatars.resolve(steam.context, [
+					ALICE,
+				])
+			).get(ALICE),
+		).toBe(ALICE_HASH);
+		expect(steam.calls.length).toBe(2);
+	});
+
+	test("never asks steam about a player the game does not map to a steam id", async () => {
+		const steam = fakeSteam({});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		expect(
+			await avatars.resolve(steam.context, [
+				"gdk_2535000000000001",
+			]),
+		).toEqual(new Map());
+		expect(steam.calls).toEqual([]);
+	});
+
+	test("answers with nothing while the platform has no steam key", async () => {
+		const steam = fakeSteam({
+			key: null,
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		expect(
+			await avatars.resolve(steam.context, [
+				ALICE,
+			]),
+		).toEqual(new Map());
+		expect(steam.calls).toEqual([]);
+		expect(steam.warnings).toEqual([]);
+	});
+
+	test("answers with nothing and warns once when steam refuses", async () => {
+		const steam = fakeSteam({
+			answer: () => {
+				throw new Error("429 too many requests");
+			},
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		expect(
+			await avatars.resolve(steam.context, [
+				ALICE,
+			]),
+		).toEqual(new Map());
+		expect(steam.warnings.length).toBe(1);
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		expect(steam.warnings.length).toBe(1);
+	});
+
+	test("never writes the api key into the warning it logs", async () => {
+		const steam = fakeSteam({
+			key: "top-secret-key",
+			answer: () => {
+				throw new Error("steam said no");
+			},
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		await avatars.resolve(steam.context, [
+			ALICE,
+		]);
+
+		expect(JSON.stringify(steam.warnings)).not.toContain("top-secret-key");
+	});
+
+	test("keeps the roster moving when steam sends a profile with no usable picture", async () => {
+		const steam = fakeSteam({
+			answer: () => [
+				{
+					steamid: ALICE,
+					avatarfull: "https://example.com/none.png",
+				},
+			],
+		});
+		const avatars = createSteamAvatars({
+			steamIdOf: identity,
+		});
+
+		expect(
+			await avatars.resolve(steam.context, [
+				ALICE,
+			]),
+		).toEqual(new Map());
+	});
+});