@serverkgg/bridge 0.0.14 → 0.0.16

package/src/runtime/bridgeEvents.ts

@@ -1,6 +1,16 @@
 import type { BridgeValues } from "../core";
 import type { BridgeEventsModule } from "../modules";
-import type { BridgeEvent } from "../protocol";
+import {
+	BRIDGE_CRASH_EVIDENCE,
+	BRIDGE_CRASH_FLAG_NO,
+	BRIDGE_CRASH_FLAG_YES,
+	BRIDGE_CRASH_KEYS,
+	BridgeCrashCause,
+	type BridgeCrashRecord,
+	type BridgeEvent,
+	BridgeEventName,
+	isCrashEvidence,
+} from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeSupervisor } from "./bridgeSupervisor";
 
@@ -8,6 +18,7 @@ const EVENT_BUFFER = 500;
 
 export const createEventRing = (events: BridgeEventsModule | undefined, supervisor: BridgeSupervisor) => {
 	const ring: BridgeEvent[] = [];
+	const evidence = new Map<string, BridgeValues | null>();
 
 	let sequence = 0;
 
@@ -25,6 +36,16 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 		}
 	};
 
+	const observe = (event: string, payload: BridgeValues | null) => {
+		if (isCrashEvidence(event)) {
+			evidence.set(event, payload);
+
+			return;
+		}
+
+		push(event, payload);
+	};
+
 	if (events) {
 		supervisor.subscribe((entry) => {
 			const line = stripAnsi(entry.line);
@@ -48,7 +69,7 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 							}
 						: null;
 
-				push(pattern.emit, groups);
+				observe(pattern.emit, groups);
 			}
 		});
 	}
@@ -64,7 +85,36 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 		},
 
 		emit(event: string, payload?: BridgeValues) {
-			push(event, payload ?? null);
+			observe(event, payload ?? null);
+		},
+
+		crash(record: BridgeCrashRecord) {
+			const winner = record.outOfMemory ? undefined : BRIDGE_CRASH_EVIDENCE.find((entry) => evidence.has(entry.name));
+
+			const cause = record.outOfMemory ? BridgeCrashCause.OomKill : (winner?.cause ?? BridgeCrashCause.Unknown);
+
+			const payload: BridgeValues = {
+				...(winner ? evidence.get(winner.name) : null),
+				[BRIDGE_CRASH_KEYS.cause]: cause,
+				[BRIDGE_CRASH_KEYS.phase]: record.phase,
+				[BRIDGE_CRASH_KEYS.exitCode]: String(record.exitCode),
+				[BRIDGE_CRASH_KEYS.restarting]: record.restarting ? BRIDGE_CRASH_FLAG_YES : BRIDGE_CRASH_FLAG_NO,
+				[BRIDGE_CRASH_KEYS.looping]: record.looping ? BRIDGE_CRASH_FLAG_YES : BRIDGE_CRASH_FLAG_NO,
+				[BRIDGE_CRASH_KEYS.restarts]: String(record.restarts),
+				[BRIDGE_CRASH_KEYS.uptimeSeconds]: String(record.uptimeSeconds),
+			};
+
+			if (cause === BridgeCrashCause.Unknown && record.lastLine !== null) {
+				payload[BRIDGE_CRASH_KEYS.lastLine] = record.lastLine;
+			}
+
+			push(BridgeEventName.ServerCrashed, payload);
+
+			evidence.clear();
+		},
+
+		discard() {
+			evidence.clear();
 		},
 
 		drain(cursor: number) {

package/src/runtime/bridgeRegistry.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 import { tmpdir } from "node:os";
 import { BridgeAction, BridgeKind, type BridgeRow, BridgeSetupStepKind, type BridgeValues } from "../core";
 import type { BridgeDriver } from "../modules";
+import { BridgeLayout } from "../panel";
 import { type BridgeConfig, BridgeConfirm, BridgeEventName } from "../protocol";
 import { createContext } from "./bridgeContext";
 import { createEventRing } from "./bridgeEvents";
@@ -296,6 +297,101 @@ describe("collection row actions", () => {
 	});
 });
 
+describe("offline row actions", () => {
+	const rosterDriver = (
+		offline: boolean,
+		list: () => Promise<BridgeRow[]>,
+		ban: (row: BridgeRow) => void,
+	): BridgeDriver => ({
+		...baseDriver(),
+		panel: {
+			tabs: [
+				{
+					id: "players",
+					title: {
+						ar: "اللاعبين",
+						en: "Players",
+					},
+					sections: [
+						{
+							layout: BridgeLayout.Table,
+							id: "online",
+							module: "players",
+							columns: [
+								{
+									key: "name",
+									label: {
+										ar: "الاسم",
+										en: "Name",
+									},
+								},
+							],
+							actions: [
+								{
+									id: "ban",
+									label: {
+										ar: "حظر",
+										en: "Ban",
+									},
+									offline,
+								},
+							],
+						},
+					],
+				},
+			],
+		},
+		modules: {
+			players: {
+				kind: BridgeKind.Collection,
+				list,
+				actions: {
+					ban: async (_context, row) => {
+						ban(row);
+					},
+				},
+			},
+		},
+	});
+
+	test("hands an offline action the id alone when the row is not listed", async () => {
+		const seen: BridgeRow[] = [];
+		const registry = registryFor(
+			rosterDriver(
+				true,
+				async () => [],
+				(row) => seen.push(row),
+			),
+		);
+
+		await registry.invoke("players", "ban", {
+			id: "steam_1",
+		});
+
+		expect(seen).toEqual([
+			{
+				id: "steam_1",
+			},
+		]);
+	});
+
+	test("still refuses an unlisted row for an action that is not offline", async () => {
+		const registry = registryFor(
+			rosterDriver(
+				false,
+				async () => [],
+				() => undefined,
+			),
+		);
+
+		await expect(
+			registry.invoke("players", "ban", {
+				id: "steam_1",
+			}),
+		).rejects.toThrow('module "players" has no row "steam_1"');
+	});
+});
+
 describe("preview modules", () => {
 	test("lists the preview action in the manifest", () => {
 		const registry = registryFor({

package/src/runtime/bridgeRegistry.ts

@@ -1,6 +1,7 @@
 import type { BridgeContext } from "../context";
 import { BridgeAction, BridgeKind, type BridgeRow, type BridgeValues, isReadAction } from "../core";
 import type { BridgeDriver, BridgePanelModule, BridgeSetupModule, BridgeTerminalModule } from "../modules";
+import { BridgeLayout } from "../panel";
 import {
 	BRIDGE_VERSION,
 	type BridgeManifest,
@@ -148,6 +149,29 @@ const asPage = (args: BridgeValues) => {
 	return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
 };
 
+const offlineActionsOf = (driver: BridgeDriver): Map<string, Set<string>> => {
+	const offline = new Map<string, Set<string>>();
+
+	for (const tab of driver.panel?.tabs ?? []) {
+		for (const section of tab.sections) {
+			if (section.layout !== BridgeLayout.Table && section.layout !== BridgeLayout.Cards) {
+				continue;
+			}
+
+			for (const action of section.actions ?? []) {
+				if (action.offline === true) {
+					const actions = offline.get(section.module) ?? new Set<string>();
+
+					actions.add(action.id);
+					offline.set(section.module, actions);
+				}
+			}
+		}
+	}
+
+	return offline;
+};
+
 export const createRegistry = (
 	driver: BridgeDriver,
 	context: BridgeContext,
@@ -157,6 +181,7 @@ export const createRegistry = (
 	const cache = new Map<string, CacheEntry>();
 	const inflight = new Map<string, Promise<unknown>>();
 	const modules = driver.modules ?? {};
+	const offlineActions = offlineActionsOf(driver);
 
 	for (const [id, module] of Object.entries(modules)) {
 		if (module.kind !== BridgeKind.Actions && module.kind !== BridgeKind.Detail) {
@@ -297,7 +322,14 @@ export const createRegistry = (
 				if (rowAction) {
 					const rows = await module.list(context);
 					const id = asString(args, "id");
-					const row = rows.find((entry) => entry.id === id);
+					const listed = rows.find((entry) => entry.id === id);
+					const row =
+						listed
+						?? (offlineActions.get(name)?.has(action)
+							? {
+									id,
+								}
+							: undefined);
 
 					if (!row) {
 						throw new BridgeDriverError(`module "${name}" has no row "${id}"`);

package/src/runtime/bridgeServer.test.ts

@@ -40,6 +40,7 @@ const baseOptions = (socket: string): ServerOptions => ({
 		exitCode: null,
 		outOfMemory: false,
 		restarts: 0,
+		crashRecords: true,
 		installError: null,
 		installErrorText: null,
 		installHold: null,

package/src/runtime/bridgeSupervisor.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import { tmpdir } from "node:os";
 import { BridgeRuntime } from "../core";
+import { BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { readProcessGroup } from "./bridgeSignal";
 import { createSupervisor } from "./bridgeSupervisor";
@@ -11,6 +12,26 @@ const WRAPPER = "sleep 300 & echo child=$!; wait";
 
 const STUBBORN_WRAPPER = 'trap "" TERM; sleep 300 & echo child=$!; wait';
 
+const NEVER_READY = /this game never says it is ready/;
+
+const READY = /^ready$/;
+
+const BOOT_CRASH = "sleep 0.3; exit 3";
+
+const OOM_CRASH = "sleep 0.3; kill -9 $$";
+
+const RUNNING_CRASH = "echo ready; sleep 0.3; exit 4";
+
+const LATE_CRASH = "sleep 1; exit 5";
+
+const LAST_LINE = "boom, the game gave up here";
+
+const LAST_LINE_CRASH = `echo "${LAST_LINE}"; sleep 0.5; exit 7`;
+
+const TIGHT_LAST_LINE_CRASH = `sleep 0.2; echo "${LAST_LINE}"; exit 3`;
+
+const INSTANT_EXIT = "false";
+
 const silent: BridgeLeveledLogger = Object.assign(() => {}, {
 	warn: () => {},
 	error: () => {},
@@ -55,6 +76,46 @@ const wrapperSupervisor = (script: string, stopTimeoutSeconds: number) => {
 	});
 };
 
+const argvSupervisor = (resolveCommand: () => Promise<string[]>, ready: RegExp) => {
+	const records: BridgeCrashRecord[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		onLog: silent,
+		resolveCommand,
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+	};
+};
+
+const crashSupervisor = (command: () => string, ready: RegExp) => {
+	return argvSupervisor(async () => {
+		return [
+			"bash",
+			"-c",
+			command(),
+		];
+	}, ready);
+};
+
+const waitFor = async (settled: () => boolean, timeoutMs: number) => {
+	for (let waited = 0; waited < timeoutMs && !settled(); waited += 50) {
+		await Bun.sleep(50);
+	}
+
+	return settled();
+};
+
 const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	const supervisor = wrapperSupervisor(script, stopTimeoutSeconds);
 	const started = supervisor.watch(CHILD_PATTERN, 10_000);
@@ -73,6 +134,41 @@ const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	};
 };
 
+const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: number) => {
+	const records: BridgeCrashRecord[] = [];
+
+	let stables = 0;
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		stableAfterMs,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				command(),
+			];
+		},
+		onStable() {
+			stables += 1;
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+		stables: () => stables,
+	};
+};
+
 describe("createSupervisor", () => {
 	test("runs the game outside the bridge's own process group", async () => {
 		const { supervisor, pid } = await startWrapper(WRAPPER, 1);
@@ -108,4 +204,197 @@ describe("createSupervisor", () => {
 		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
 		expect(await waitGone(pid, 5000), "the game outlived the SIGKILL escalation").toBe(true);
 	}, 30_000);
+
+	test("does not restart a game that exited before it ever reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => BOOT_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(3);
+		expect(record?.outOfMemory).toBe(false);
+		expect(record?.restarts).toBe(0);
+
+		await Bun.sleep(3000);
+
+		expect(records, "the booting crash was retried").toHaveLength(1);
+		expect(supervisor.running).toBe(false);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(0);
+	}, 30_000);
+
+	test("restarts a game that crashed after it reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(4);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("flags a game killed with SIGKILL as out of memory", async () => {
+		const { supervisor, records } = crashSupervisor(() => OOM_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.outOfMemory).toBe(true);
+		expect(record?.exitCode).toBe(137);
+		expect(supervisor.outOfMemory).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("trips the breaker on the third running crash inside the window", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+
+		const record = records.at(2);
+
+		expect(record?.looping).toBe(true);
+		expect(record?.restarting).toBe(false);
+		expect(record?.restarts, "the record carries the count the supervisor reached").toBe(3);
+		expect(records.map((entry) => entry.restarts)).toEqual([
+			1,
+			2,
+			3,
+		]);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(3);
+
+		await supervisor.stop();
+	}, 40_000);
+
+	test("launches again when start is asked after a booting crash", async () => {
+		let script = BOOT_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+	}, 30_000);
+
+	test("carries the last line the game printed into the crash record", async () => {
+		const { supervisor, records } = crashSupervisor(() => LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine).toBe(LAST_LINE);
+	}, 30_000);
+
+	test("carries the last line even when the game exits with no pause after printing it", async () => {
+		const { supervisor, records } = crashSupervisor(() => TIGHT_LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine, "the crash record was built before the pumps drained").toBe(LAST_LINE);
+		expect(records.at(0)?.exitCode).toBe(3);
+	}, 30_000);
+
+	test("leaves no dead child behind when the game exits the instant it launches", async () => {
+		let adopted = 0;
+
+		for (let attempt = 0; attempt < 40 && adopted < 20; attempt += 1) {
+			const { supervisor, records } = argvSupervisor(async () => {
+				return [
+					INSTANT_EXIT,
+				];
+			}, NEVER_READY);
+
+			const launched = await supervisor.start().then(
+				() => true,
+				() => false,
+			);
+
+			if (!launched) {
+				continue;
+			}
+
+			adopted += 1;
+
+			expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+			await Bun.sleep(100);
+
+			expect(records, "the instant exit was reported more than once").toHaveLength(1);
+			expect(records.at(0)?.exitCode).toBe(1);
+			expect(records.at(0)?.outOfMemory).toBe(false);
+			expect(supervisor.running, "the supervisor kept a child that had already exited").toBe(false);
+			expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		}
+
+		expect(adopted, "the game never survived long enough to be adopted as the child").toBeGreaterThan(0);
+	}, 30_000);
+
+	test("treats a run that outlived the stable window as running even when the ready line never came", async () => {
+		const { supervisor, records, stables } = timedSupervisor(() => LATE_CRASH, NEVER_READY, 300);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.exitCode).toBe(5);
+		expect(stables(), "the run was reported stable more than once").toBe(1);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("opens a new episode when start is asked after the breaker tripped", async () => {
+		let script = RUNNING_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+		expect(supervisor.restarts).toBe(3);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.restarts, "the cumulative count survived an explicit start").toBe(0);
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+	}, 40_000);
 });