@serverkgg/bridge 0.0.15 → 0.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@serverkgg/bridge",
-	"version": "0.0.15",
+	"version": "0.0.16",
 	"description": "Serverk game bridge and driver runtime, wire protocol, manifest schema, and maintainer CLI",
 	"homepage": "https://serverk.gg",
 	"license": "SEE LICENSE IN LICENSE.md",

package/src/protocol/bridgeCrash.ts

@@ -0,0 +1,83 @@
+import { BridgeEventName } from "./bridgeEvents";
+
+export enum BridgeCrashCause {
+	OomKill = "oom-kill",
+	OutOfMemory = "out-of-memory",
+	WrongJavaVersion = "wrong-java",
+	PortBusy = "port-busy",
+	WorldCorrupt = "world-corrupt",
+	MissingDependency = "missing-dependency",
+	ModCrashed = "mod-crashed",
+	Unknown = "unknown",
+}
+
+export enum BridgeCrashPhase {
+	Booting = "booting",
+	Running = "running",
+}
+
+export interface BridgeCrashEvidence {
+	name: BridgeEventName;
+	cause: BridgeCrashCause;
+}
+
+export const BRIDGE_CRASH_EVIDENCE: readonly BridgeCrashEvidence[] = [
+	{
+		name: BridgeEventName.ServerCrashed,
+		cause: BridgeCrashCause.OutOfMemory,
+	},
+	{
+		name: BridgeEventName.WrongJavaVersion,
+		cause: BridgeCrashCause.WrongJavaVersion,
+	},
+	{
+		name: BridgeEventName.PortBindFailed,
+		cause: BridgeCrashCause.PortBusy,
+	},
+	{
+		name: BridgeEventName.WorldCorrupt,
+		cause: BridgeCrashCause.WorldCorrupt,
+	},
+	{
+		name: BridgeEventName.MissingDependency,
+		cause: BridgeCrashCause.MissingDependency,
+	},
+	{
+		name: BridgeEventName.ModCrashed,
+		cause: BridgeCrashCause.ModCrashed,
+	},
+];
+
+const BRIDGE_CRASH_EVIDENCE_NAMES: ReadonlySet<string> = new Set(BRIDGE_CRASH_EVIDENCE.map((entry) => entry.name));
+
+export const isCrashEvidence = (name: string): boolean => {
+	return BRIDGE_CRASH_EVIDENCE_NAMES.has(name);
+};
+
+export const BRIDGE_CRASH_KEYS = {
+	cause: "cause",
+	phase: "phase",
+	exitCode: "exitCode",
+	restarting: "restarting",
+	looping: "looping",
+	restarts: "restarts",
+	uptimeSeconds: "uptimeSeconds",
+	lastLine: "lastLine",
+} as const;
+
+export const BRIDGE_CRASH_FLAG_YES = "yes";
+
+export const BRIDGE_CRASH_FLAG_NO = "no";
+
+export const BRIDGE_CRASH_LINE_LIMIT = 256;
+
+export interface BridgeCrashRecord {
+	exitCode: number;
+	outOfMemory: boolean;
+	phase: BridgeCrashPhase;
+	uptimeSeconds: number;
+	restarting: boolean;
+	looping: boolean;
+	restarts: number;
+	lastLine: string | null;
+}

package/src/protocol/bridgeFrame.ts

@@ -76,6 +76,7 @@ export interface BridgeStatus {
 	exitCode: number | null;
 	outOfMemory: boolean;
 	restarts: number;
+	crashRecords: boolean;
 	installError: string | null;
 	installErrorText: LocalizedText | null;
 	installHold: BridgeInstallHold | null;

package/src/protocol/index.ts

@@ -3,6 +3,7 @@ export * from "../panel";
 export * from "../terminal";
 export * from "./bridgeCatalog";
 export * from "./bridgeConfig";
+export * from "./bridgeCrash";
 export * from "./bridgeEgress";
 export * from "./bridgeEvents";
 export * from "./bridgeFrame";

package/src/runtime/bridgeEvents.test.ts

@@ -4,8 +4,8 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { BridgeKind } from "../core";
 import type { BridgeEventsModule } from "../modules";
-import { BridgeEventName } from "../protocol";
-import { createEventRing } from "./bridgeEvents";
+import { BridgeCrashPhase, type BridgeCrashRecord, BridgeEventName } from "../protocol";
+import { type BridgeEventRing, createEventRing } from "./bridgeEvents";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { createSupervisor } from "./bridgeSupervisor";
 
@@ -13,6 +13,10 @@ const RESET = "\u001B[m";
 
 const PLAYER_JOINED = /^\[World\|(?<world>[^\]]+)\] Player '(?<player>[^']+)' joined world (?<target>\S+)$/;
 
+const MOD_CRASHED = /^Exception in mod (?<mod>\S+) from (?<jar>\S+)$/;
+
+const READY_LINE = /^ready$/;
+
 const silent: BridgeLeveledLogger = Object.assign(() => {}, {
 	warn: () => {},
 	error: () => {},
@@ -25,16 +29,18 @@ const events: BridgeEventsModule = {
 			match: PLAYER_JOINED,
 			emit: BridgeEventName.PlayerJoined,
 		},
+		{
+			match: MOD_CRASHED,
+			emit: BridgeEventName.ModCrashed,
+		},
+	],
+	emits: [
+		BridgeEventName.WorldCorrupt,
 	],
 };
 
-const emittedFor = async (line: string) => {
-	const dir = await mkdtemp(join(tmpdir(), "bridge-events-"));
-	const path = join(dir, "line.txt");
-
-	await Bun.write(path, `${line}\n`);
-
-	const supervisor = createSupervisor({
+const lineSupervisor = (dir: string, path: string) => {
+	return createSupervisor({
 		cwd: dir,
 		uid: process.getuid?.() ?? 0,
 		gid: process.getgid?.() ?? 0,
@@ -47,7 +53,15 @@ const emittedFor = async (line: string) => {
 			path,
 		],
 	});
+};
+
+const emittedFor = async (line: string) => {
+	const dir = await mkdtemp(join(tmpdir(), "bridge-events-"));
+	const path = join(dir, "line.txt");
 
+	await Bun.write(path, `${line}\n`);
+
+	const supervisor = lineSupervisor(dir, path);
 	const ring = createEventRing(events, supervisor);
 
 	await supervisor.start();
@@ -71,6 +85,72 @@ const emittedFor = async (line: string) => {
 	}
 };
 
+const crashAfter = async (line: string, record: BridgeCrashRecord) => {
+	const dir = await mkdtemp(join(tmpdir(), "bridge-events-"));
+	const path = join(dir, "line.txt");
+
+	await Bun.write(path, `${line}\n`);
+
+	const supervisor = lineSupervisor(dir, path);
+	const ring = createEventRing(events, supervisor);
+
+	await supervisor.start();
+
+	try {
+		for (let waited = 0; waited < 5000; waited += 50) {
+			if (supervisor.tail(50).includes(line)) {
+				break;
+			}
+
+			await Bun.sleep(50);
+		}
+
+		const held = ring.drain(0).events;
+
+		ring.crash(record);
+
+		return {
+			held,
+			events: ring.drain(0).events,
+		};
+	} finally {
+		await supervisor.stop();
+		await rm(dir, {
+			force: true,
+			recursive: true,
+		});
+	}
+};
+
+const idleRing = () => {
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		onLog: silent,
+		resolveCommand: async () => [
+			"true",
+		],
+	});
+
+	return createEventRing(events, supervisor);
+};
+
+const crashRecord = (overrides: Partial<BridgeCrashRecord> = {}): BridgeCrashRecord => {
+	return {
+		exitCode: 1,
+		outOfMemory: false,
+		phase: BridgeCrashPhase.Running,
+		uptimeSeconds: 12,
+		restarting: false,
+		looping: false,
+		restarts: 0,
+		lastLine: null,
+		...overrides,
+	};
+};
+
 const joined = {
 	seq: 1,
 	event: BridgeEventName.PlayerJoined,
@@ -93,4 +173,326 @@ describe("matching a game line against the driver patterns", () => {
 			joined,
 		]);
 	});
+
+	test("holds a matched diagnostic line as evidence and spends it on the crash record", async () => {
+		const { held, events: drained } = await crashAfter(
+			"Exception in mod journeymap from journeymap-5.9.jar",
+			crashRecord({
+				exitCode: 1,
+				phase: BridgeCrashPhase.Booting,
+			}),
+		);
+
+		expect(held, "the diagnostic line reached the ring as a standalone event").toEqual([]);
+		expect(drained).toEqual([
+			{
+				seq: 1,
+				event: BridgeEventName.ServerCrashed,
+				payload: {
+					mod: "journeymap",
+					jar: "journeymap-5.9.jar",
+					cause: "mod-crashed",
+					phase: "booting",
+					exitCode: "1",
+					restarting: "no",
+					looping: "no",
+					restarts: "0",
+					uptimeSeconds: "12",
+				},
+			},
+		]);
+	}, 20_000);
+});
+
+describe("the crash record the ring builds", () => {
+	test("holds evidence out of the ring and lets every other event drain", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			jar: "broken.jar",
+		});
+
+		ring.emit(BridgeEventName.TickLagging, {
+			ms: 900,
+		});
+
+		expect(ring.drain(0).events.map((entry) => entry.event)).toEqual([
+			BridgeEventName.TickLagging,
+		]);
+	});
+
+	test("prefers the oom kill over any evidence it holds", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			jar: "broken.jar",
+		});
+
+		ring.crash(
+			crashRecord({
+				exitCode: 137,
+				outOfMemory: true,
+			}),
+		);
+
+		expect(ring.drain(0).events.at(0)?.payload).toEqual({
+			cause: "oom-kill",
+			phase: "running",
+			exitCode: "137",
+			restarting: "no",
+			looping: "no",
+			restarts: "0",
+			uptimeSeconds: "12",
+		});
+	});
+
+	test("follows the evidence precedence when more than one name is held", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			jar: "broken.jar",
+		});
+
+		ring.emit(BridgeEventName.WrongJavaVersion, {
+			detail: "class file version 65",
+		});
+
+		ring.crash(crashRecord());
+
+		expect(ring.drain(0).events.at(0)?.payload).toEqual({
+			detail: "class file version 65",
+			cause: "wrong-java",
+			phase: "running",
+			exitCode: "1",
+			restarting: "no",
+			looping: "no",
+			restarts: "0",
+			uptimeSeconds: "12",
+		});
+	});
+
+	test("lets the out of memory log pattern outrank the java version", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.WrongJavaVersion, {
+			detail: "class file version 65",
+		});
+
+		ring.emit(BridgeEventName.ServerCrashed, {
+			error: "java.lang.OutOfMemoryError",
+		});
+
+		ring.crash(crashRecord());
+
+		expect(ring.drain(0).events.at(0)?.payload).toMatchObject({
+			cause: "out-of-memory",
+			error: "java.lang.OutOfMemoryError",
+		});
+	});
+
+	test("never lets the evidence overwrite the record keys", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			cause: "not this one",
+			phase: "not this one either",
+		});
+
+		ring.crash(
+			crashRecord({
+				phase: BridgeCrashPhase.Booting,
+			}),
+		);
+
+		expect(ring.drain(0).events.at(0)?.payload).toMatchObject({
+			cause: "mod-crashed",
+			phase: "booting",
+		});
+	});
+
+	test("clears the evidence once it is spent", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			jar: "broken.jar",
+		});
+
+		ring.crash(crashRecord());
+
+		const first = ring.drain(0);
+
+		ring.crash(crashRecord());
+
+		expect(first.events.at(0)?.payload).toMatchObject({
+			cause: "mod-crashed",
+		});
+
+		expect(ring.drain(first.cursor).events.at(0)?.payload).toMatchObject({
+			cause: "unknown",
+		});
+	});
+
+	test("carries the last line only when nothing explains the crash", () => {
+		const ring = idleRing();
+
+		ring.emit(BridgeEventName.ModCrashed, {
+			jar: "broken.jar",
+		});
+
+		ring.crash(
+			crashRecord({
+				lastLine: "explained, so this line is noise",
+			}),
+		);
+
+		const explained = ring.drain(0);
+
+		ring.crash(
+			crashRecord({
+				lastLine: "Segmentation fault",
+			}),
+		);
+
+		expect(explained.events.at(0)?.payload).not.toHaveProperty("lastLine");
+		expect(ring.drain(explained.cursor).events.at(0)?.payload).toMatchObject({
+			cause: "unknown",
+			lastLine: "Segmentation fault",
+		});
+	});
+
+	test("keeps listing the evidence names the driver declares", () => {
+		expect(idleRing().names()).toEqual([
+			BridgeEventName.PlayerJoined,
+			BridgeEventName.ModCrashed,
+			BridgeEventName.WorldCorrupt,
+		]);
+	});
+});
+
+const EVIDENCE_LINE = "Exception in mod journeymap from journeymap-5.9.jar";
+
+const EVIDENCE_BEFORE_READY = `echo "${EVIDENCE_LINE}"; sleep 0.3; echo ready; sleep 0.5; exit 4`;
+
+const EVIDENCE_AFTER_READY = `echo ready; sleep 0.3; echo "${EVIDENCE_LINE}"; sleep 0.5; exit 4`;
+
+const EVIDENCE_THEN_IDLE = `echo "${EVIDENCE_LINE}"; sleep 300`;
+
+const wired = async (script: string, ready?: RegExp) => {
+	const dir = await mkdtemp(join(tmpdir(), "bridge-events-"));
+
+	const supervisor = createSupervisor({
+		cwd: dir,
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		onLog: silent,
+		resolveCommand: async () => [
+			"bash",
+			"-c",
+			script,
+		],
+		onStable() {
+			ring.discard();
+		},
+		onStopped() {
+			ring.discard();
+		},
+		onExit(record) {
+			ring.crash(record);
+		},
+	});
+
+	const ring = createEventRing(events, supervisor);
+
+	return {
+		supervisor,
+		ring,
+		async cleanup() {
+			await supervisor.stop();
+			await rm(dir, {
+				force: true,
+				recursive: true,
+			});
+		},
+	};
+};
+
+const crashedEvent = async (ring: BridgeEventRing) => {
+	for (let waited = 0; waited < 10_000; waited += 50) {
+		const found = ring.drain(0).events.find((entry) => entry.event === BridgeEventName.ServerCrashed);
+
+		if (found) {
+			return found;
+		}
+
+		await Bun.sleep(50);
+	}
+
+	return null;
+};
+
+describe("the evidence a run may spend", () => {
+	test("spends nothing printed before the run became stable on a running crash", async () => {
+		const wiring = await wired(EVIDENCE_BEFORE_READY, READY_LINE);
+
+		await wiring.supervisor.start();
+
+		try {
+			const crashed = await crashedEvent(wiring.ring);
+
+			expect(crashed?.payload).toMatchObject({
+				cause: "unknown",
+				phase: "running",
+			});
+
+			expect(crashed?.payload).not.toHaveProperty("jar");
+		} finally {
+			await wiring.cleanup();
+		}
+	}, 20_000);
+
+	test("spends what the run printed after it became stable", async () => {
+		const wiring = await wired(EVIDENCE_AFTER_READY, READY_LINE);
+
+		await wiring.supervisor.start();
+
+		try {
+			const crashed = await crashedEvent(wiring.ring);
+
+			expect(crashed?.payload).toMatchObject({
+				cause: "mod-crashed",
+				phase: "running",
+				jar: "journeymap-5.9.jar",
+			});
+		} finally {
+			await wiring.cleanup();
+		}
+	}, 20_000);
+
+	test("drops what it holds when the stop was requested", async () => {
+		const wiring = await wired(EVIDENCE_THEN_IDLE);
+
+		await wiring.supervisor.start();
+
+		try {
+			for (let waited = 0; waited < 5000; waited += 50) {
+				if (wiring.supervisor.tail(50).includes(EVIDENCE_LINE)) {
+					break;
+				}
+
+				await Bun.sleep(50);
+			}
+
+			await wiring.supervisor.stop();
+
+			wiring.ring.crash(crashRecord());
+
+			expect(wiring.ring.drain(0).events.at(0)?.payload).toMatchObject({
+				cause: "unknown",
+			});
+		} finally {
+			await wiring.cleanup();
+		}
+	}, 20_000);
 });

package/src/runtime/bridgeEvents.ts

@@ -1,6 +1,16 @@
 import type { BridgeValues } from "../core";
 import type { BridgeEventsModule } from "../modules";
-import type { BridgeEvent } from "../protocol";
+import {
+	BRIDGE_CRASH_EVIDENCE,
+	BRIDGE_CRASH_FLAG_NO,
+	BRIDGE_CRASH_FLAG_YES,
+	BRIDGE_CRASH_KEYS,
+	BridgeCrashCause,
+	type BridgeCrashRecord,
+	type BridgeEvent,
+	BridgeEventName,
+	isCrashEvidence,
+} from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeSupervisor } from "./bridgeSupervisor";
 
@@ -8,6 +18,7 @@ const EVENT_BUFFER = 500;
 
 export const createEventRing = (events: BridgeEventsModule | undefined, supervisor: BridgeSupervisor) => {
 	const ring: BridgeEvent[] = [];
+	const evidence = new Map<string, BridgeValues | null>();
 
 	let sequence = 0;
 
@@ -25,6 +36,16 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 		}
 	};
 
+	const observe = (event: string, payload: BridgeValues | null) => {
+		if (isCrashEvidence(event)) {
+			evidence.set(event, payload);
+
+			return;
+		}
+
+		push(event, payload);
+	};
+
 	if (events) {
 		supervisor.subscribe((entry) => {
 			const line = stripAnsi(entry.line);
@@ -48,7 +69,7 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 							}
 						: null;
 
-				push(pattern.emit, groups);
+				observe(pattern.emit, groups);
 			}
 		});
 	}
@@ -64,7 +85,36 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 		},
 
 		emit(event: string, payload?: BridgeValues) {
-			push(event, payload ?? null);
+			observe(event, payload ?? null);
+		},
+
+		crash(record: BridgeCrashRecord) {
+			const winner = record.outOfMemory ? undefined : BRIDGE_CRASH_EVIDENCE.find((entry) => evidence.has(entry.name));
+
+			const cause = record.outOfMemory ? BridgeCrashCause.OomKill : (winner?.cause ?? BridgeCrashCause.Unknown);
+
+			const payload: BridgeValues = {
+				...(winner ? evidence.get(winner.name) : null),
+				[BRIDGE_CRASH_KEYS.cause]: cause,
+				[BRIDGE_CRASH_KEYS.phase]: record.phase,
+				[BRIDGE_CRASH_KEYS.exitCode]: String(record.exitCode),
+				[BRIDGE_CRASH_KEYS.restarting]: record.restarting ? BRIDGE_CRASH_FLAG_YES : BRIDGE_CRASH_FLAG_NO,
+				[BRIDGE_CRASH_KEYS.looping]: record.looping ? BRIDGE_CRASH_FLAG_YES : BRIDGE_CRASH_FLAG_NO,
+				[BRIDGE_CRASH_KEYS.restarts]: String(record.restarts),
+				[BRIDGE_CRASH_KEYS.uptimeSeconds]: String(record.uptimeSeconds),
+			};
+
+			if (cause === BridgeCrashCause.Unknown && record.lastLine !== null) {
+				payload[BRIDGE_CRASH_KEYS.lastLine] = record.lastLine;
+			}
+
+			push(BridgeEventName.ServerCrashed, payload);
+
+			evidence.clear();
+		},
+
+		discard() {
+			evidence.clear();
 		},
 
 		drain(cursor: number) {

package/src/runtime/bridgeServer.test.ts

@@ -40,6 +40,7 @@ const baseOptions = (socket: string): ServerOptions => ({
 		exitCode: null,
 		outOfMemory: false,
 		restarts: 0,
+		crashRecords: true,
 		installError: null,
 		installErrorText: null,
 		installHold: null,

package/src/runtime/bridgeSupervisor.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import { tmpdir } from "node:os";
 import { BridgeRuntime } from "../core";
+import { BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { readProcessGroup } from "./bridgeSignal";
 import { createSupervisor } from "./bridgeSupervisor";
@@ -11,6 +12,26 @@ const WRAPPER = "sleep 300 & echo child=$!; wait";
 
 const STUBBORN_WRAPPER = 'trap "" TERM; sleep 300 & echo child=$!; wait';
 
+const NEVER_READY = /this game never says it is ready/;
+
+const READY = /^ready$/;
+
+const BOOT_CRASH = "sleep 0.3; exit 3";
+
+const OOM_CRASH = "sleep 0.3; kill -9 $$";
+
+const RUNNING_CRASH = "echo ready; sleep 0.3; exit 4";
+
+const LATE_CRASH = "sleep 1; exit 5";
+
+const LAST_LINE = "boom, the game gave up here";
+
+const LAST_LINE_CRASH = `echo "${LAST_LINE}"; sleep 0.5; exit 7`;
+
+const TIGHT_LAST_LINE_CRASH = `sleep 0.2; echo "${LAST_LINE}"; exit 3`;
+
+const INSTANT_EXIT = "false";
+
 const silent: BridgeLeveledLogger = Object.assign(() => {}, {
 	warn: () => {},
 	error: () => {},
@@ -55,6 +76,46 @@ const wrapperSupervisor = (script: string, stopTimeoutSeconds: number) => {
 	});
 };
 
+const argvSupervisor = (resolveCommand: () => Promise<string[]>, ready: RegExp) => {
+	const records: BridgeCrashRecord[] = [];
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		onLog: silent,
+		resolveCommand,
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+	};
+};
+
+const crashSupervisor = (command: () => string, ready: RegExp) => {
+	return argvSupervisor(async () => {
+		return [
+			"bash",
+			"-c",
+			command(),
+		];
+	}, ready);
+};
+
+const waitFor = async (settled: () => boolean, timeoutMs: number) => {
+	for (let waited = 0; waited < timeoutMs && !settled(); waited += 50) {
+		await Bun.sleep(50);
+	}
+
+	return settled();
+};
+
 const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	const supervisor = wrapperSupervisor(script, stopTimeoutSeconds);
 	const started = supervisor.watch(CHILD_PATTERN, 10_000);
@@ -73,6 +134,41 @@ const startWrapper = async (script: string, stopTimeoutSeconds: number) => {
 	};
 };
 
+const timedSupervisor = (command: () => string, ready: RegExp, stableAfterMs: number) => {
+	const records: BridgeCrashRecord[] = [];
+
+	let stables = 0;
+
+	const supervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready,
+		stableAfterMs,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				command(),
+			];
+		},
+		onStable() {
+			stables += 1;
+		},
+		onExit(record) {
+			records.push(record);
+		},
+	});
+
+	return {
+		supervisor,
+		records,
+		stables: () => stables,
+	};
+};
+
 describe("createSupervisor", () => {
 	test("runs the game outside the bridge's own process group", async () => {
 		const { supervisor, pid } = await startWrapper(WRAPPER, 1);
@@ -108,4 +204,197 @@ describe("createSupervisor", () => {
 		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
 		expect(await waitGone(pid, 5000), "the game outlived the SIGKILL escalation").toBe(true);
 	}, 30_000);
+
+	test("does not restart a game that exited before it ever reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => BOOT_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(record?.restarting).toBe(false);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(3);
+		expect(record?.outOfMemory).toBe(false);
+		expect(record?.restarts).toBe(0);
+
+		await Bun.sleep(3000);
+
+		expect(records, "the booting crash was retried").toHaveLength(1);
+		expect(supervisor.running).toBe(false);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(0);
+	}, 30_000);
+
+	test("restarts a game that crashed after it reached its ready line", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.looping).toBe(false);
+		expect(record?.exitCode).toBe(4);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Starting);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("flags a game killed with SIGKILL as out of memory", async () => {
+		const { supervisor, records } = crashSupervisor(() => OOM_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.outOfMemory).toBe(true);
+		expect(record?.exitCode).toBe(137);
+		expect(supervisor.outOfMemory).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("trips the breaker on the third running crash inside the window", async () => {
+		const { supervisor, records } = crashSupervisor(() => RUNNING_CRASH, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+
+		const record = records.at(2);
+
+		expect(record?.looping).toBe(true);
+		expect(record?.restarting).toBe(false);
+		expect(record?.restarts, "the record carries the count the supervisor reached").toBe(3);
+		expect(records.map((entry) => entry.restarts)).toEqual([
+			1,
+			2,
+			3,
+		]);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		expect(supervisor.restarts).toBe(3);
+
+		await supervisor.stop();
+	}, 40_000);
+
+	test("launches again when start is asked after a booting crash", async () => {
+		let script = BOOT_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+
+		expect(supervisor.runtime).toBe(BridgeRuntime.Stopped);
+	}, 30_000);
+
+	test("carries the last line the game printed into the crash record", async () => {
+		const { supervisor, records } = crashSupervisor(() => LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine).toBe(LAST_LINE);
+	}, 30_000);
+
+	test("carries the last line even when the game exits with no pause after printing it", async () => {
+		const { supervisor, records } = crashSupervisor(() => TIGHT_LAST_LINE_CRASH, NEVER_READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.lastLine, "the crash record was built before the pumps drained").toBe(LAST_LINE);
+		expect(records.at(0)?.exitCode).toBe(3);
+	}, 30_000);
+
+	test("leaves no dead child behind when the game exits the instant it launches", async () => {
+		let adopted = 0;
+
+		for (let attempt = 0; attempt < 40 && adopted < 20; attempt += 1) {
+			const { supervisor, records } = argvSupervisor(async () => {
+				return [
+					INSTANT_EXIT,
+				];
+			}, NEVER_READY);
+
+			const launched = await supervisor.start().then(
+				() => true,
+				() => false,
+			);
+
+			if (!launched) {
+				continue;
+			}
+
+			adopted += 1;
+
+			expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+			await Bun.sleep(100);
+
+			expect(records, "the instant exit was reported more than once").toHaveLength(1);
+			expect(records.at(0)?.exitCode).toBe(1);
+			expect(records.at(0)?.outOfMemory).toBe(false);
+			expect(supervisor.running, "the supervisor kept a child that had already exited").toBe(false);
+			expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+		}
+
+		expect(adopted, "the game never survived long enough to be adopted as the child").toBeGreaterThan(0);
+	}, 30_000);
+
+	test("treats a run that outlived the stable window as running even when the ready line never came", async () => {
+		const { supervisor, records, stables } = timedSupervisor(() => LATE_CRASH, NEVER_READY, 300);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+
+		const record = records.at(0);
+
+		expect(record?.phase).toBe(BridgeCrashPhase.Running);
+		expect(record?.restarting).toBe(true);
+		expect(record?.exitCode).toBe(5);
+		expect(stables(), "the run was reported stable more than once").toBe(1);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	test("opens a new episode when start is asked after the breaker tripped", async () => {
+		let script = RUNNING_CRASH;
+
+		const { supervisor, records } = crashSupervisor(() => script, READY);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length >= 3, 25_000)).toBe(true);
+		expect(supervisor.restarts).toBe(3);
+
+		script = WRAPPER;
+
+		await supervisor.start();
+
+		expect(supervisor.restarts, "the cumulative count survived an explicit start").toBe(0);
+		expect(supervisor.running).toBe(true);
+
+		await supervisor.stop();
+	}, 40_000);
 });

package/src/runtime/bridgeSupervisor.ts

@@ -1,5 +1,6 @@
 import type { BridgeCommandOptions, BridgeReply } from "../context";
 import { BridgeRuntime, BridgeStream } from "../core";
+import { BRIDGE_CRASH_LINE_LIMIT, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
 import { stripAnsi } from "../utils";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { errorText } from "./bridgeLog";
@@ -14,9 +15,11 @@ const KILL_GRACE_MS = 5000;
 
 const RESTART_DELAY_MS = 2000;
 
-const RESTART_WINDOW_MS = 300_000;
+const RESTART_WINDOW_MS = 600_000;
 
-const RESTART_LIMIT = 5;
+const RESTART_LIMIT = 3;
+
+const STABLE_UPTIME_MS = 180_000;
 
 const GAME_OOM_SCORE_ADJ = 500;
 
@@ -35,7 +38,11 @@ export interface SupervisorOptions {
 	ready?: RegExp;
 	resolveCommand: () => Promise<string[]>;
 	gracefulStop?: () => Promise<void>;
+	stableAfterMs?: number;
 	onReady?: () => void;
+	onStable?: () => void;
+	onStopped?: () => void;
+	onExit?: (record: BridgeCrashRecord) => void;
 	onLog: BridgeLeveledLogger;
 }
 
@@ -119,6 +126,15 @@ const splitLines = (buffer: string, chunk: string) => {
 	};
 };
 
+const PUMP_DRAIN_MS = 500;
+
+const drainPumps = async (pumps: readonly Promise<unknown>[]) => {
+	await Promise.race([
+		Promise.all(pumps),
+		Bun.sleep(PUMP_DRAIN_MS),
+	]);
+};
+
 export const createSupervisor = (options: SupervisorOptions) => {
 	const tail: SupervisorLine[] = [];
 	const watchers = new Set<LineWatcher>();
@@ -132,8 +148,21 @@ export const createSupervisor = (options: SupervisorOptions) => {
 	let exitCode: number | null = null;
 	let outOfMemory = false;
 	let restarts = 0;
+	let launchedAt = performance.now();
+	let stable = false;
+	let stableTimer: ReturnType<typeof setTimeout> | null = null;
 	let queue: Promise<unknown> = Promise.resolve();
 
+	const markStable = () => {
+		if (stable) {
+			return;
+		}
+
+		stable = true;
+
+		options.onStable?.();
+	};
+
 	const emit = (entry: SupervisorLine) => {
 		if (entry.stream === BridgeStream.Stderr) {
 			process.stderr.write(`${entry.line}\n`);
@@ -223,6 +252,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		return restartStamps.length >= RESTART_LIMIT;
 	};
 
+	const lastLine = () => {
+		for (let index = tail.length - 1; index >= 0; index -= 1) {
+			const line = stripAnsi(tail.at(index)?.line ?? "").trim();
+
+			if (line.length > 0) {
+				return line.slice(0, BRIDGE_CRASH_LINE_LIMIT);
+			}
+		}
+
+		return null;
+	};
+
 	const launch = async () => {
 		const command = await options.resolveCommand();
 		const [executable, ...rest] = command;
@@ -237,6 +278,34 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		];
 
 		const { argv: spawnArgv, credentials } = await unprivilegedArgv(argv, options.uid, options.gid);
+
+		launchedAt = performance.now();
+		stable = false;
+
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
+		let accepted = false;
+		let reported = false;
+		let pumps: Promise<unknown>[] = [];
+
+		const finish = (code: number) => {
+			if (!accepted || reported) {
+				return;
+			}
+
+			reported = true;
+
+			void handleExit(code, pumps).catch((error: unknown) => {
+				options.onLog.error("the game exit was not reported", {
+					error: errorText(error),
+				});
+			});
+		};
+
 		const spawned = Bun.spawn(spawnArgv, {
 			...credentials,
 			cwd: options.cwd,
@@ -245,7 +314,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			stdin: "pipe",
 			stdout: "pipe",
 			onExit(_subprocess, code, signal) {
-				handleExit(exitCodeOf(code, signal));
+				finish(exitCodeOf(code, signal));
 			},
 		});
 
@@ -281,8 +350,22 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		childGroup = group;
 		exitCode = null;
 
-		void pump(spawned.stdout, BridgeStream.Stdout);
-		void pump(spawned.stderr, BridgeStream.Stderr);
+		const launchStamp = launchedAt;
+
+		stableTimer = setTimeout(() => {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			markStable();
+		}, options.stableAfterMs ?? STABLE_UPTIME_MS);
+
+		pumps = [
+			pump(spawned.stdout, BridgeStream.Stdout).catch(() => undefined),
+			pump(spawned.stderr, BridgeStream.Stderr).catch(() => undefined),
+		];
+
+		accepted = true;
 
 		options.onLog("game started", {
 			pid: spawned.pid,
@@ -290,6 +373,12 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			uid: options.uid,
 		});
 
+		if (spawned.exitCode !== null || spawned.signalCode !== null) {
+			finish(exitCodeOf(spawned.exitCode, spawned.signalCode));
+
+			return;
+		}
+
 		if (!options.ready) {
 			runtime = BridgeRuntime.Running;
 
@@ -299,12 +388,30 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		runtime = BridgeRuntime.Starting;
 
 		void watch(options.ready, options.stopTimeoutSeconds * 1000 * 10).then((match) => {
-			if (match && runtime === BridgeRuntime.Starting) {
+			if (launchStamp !== launchedAt || child === null) {
+				return;
+			}
+
+			if (match) {
+				if (runtime === BridgeRuntime.Starting) {
+					runtime = BridgeRuntime.Running;
+
+					options.onLog("game ready");
+					options.onReady?.();
+				}
+
+				markStable();
+
+				return;
+			}
+
+			if (runtime === BridgeRuntime.Starting) {
 				runtime = BridgeRuntime.Running;
 
-				options.onLog("game ready");
-				options.onReady?.();
+				options.onLog.warn("game ready line not seen, treating it as running");
 			}
+
+			markStable();
 		});
 	};
 
@@ -326,12 +433,18 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		}
 	};
 
-	function handleExit(code: number) {
+	async function handleExit(code: number, pumps: readonly Promise<unknown>[]) {
 		const finished = child;
 
 		child = null;
 		exitCode = code;
 
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+
+			stableTimer = null;
+		}
+
 		if (finished) {
 			sweep(finished.pid, childGroup);
 		}
@@ -350,6 +463,9 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			return;
 		}
 
+		const uptimeMs = performance.now() - launchedAt;
+		const phase = stable ? BridgeCrashPhase.Running : BridgeCrashPhase.Booting;
+
 		outOfMemory = code === SIGKILL_EXIT_CODE;
 
 		if (outOfMemory) {
@@ -358,6 +474,38 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			});
 		}
 
+		const report = async (restarting: boolean, looping: boolean) => {
+			const settled = {
+				exitCode: code,
+				outOfMemory,
+				phase,
+				uptimeSeconds: Math.floor(uptimeMs / 1000),
+				restarting,
+				looping,
+				restarts,
+			};
+
+			await drainPumps(pumps);
+
+			options.onExit?.({
+				...settled,
+				lastLine: lastLine(),
+			});
+		};
+
+		if (phase === BridgeCrashPhase.Booting) {
+			runtime = BridgeRuntime.Crashed;
+
+			options.onLog.error("game exited while booting, not restarting", {
+				code,
+				phase,
+			});
+
+			await report(false, false);
+
+			return;
+		}
+
 		restarts += 1;
 		restartStamps.push(performance.now());
 
@@ -370,6 +518,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 				restarts,
 			});
 
+			await report(false, true);
+
 			return;
 		}
 
@@ -380,6 +530,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			restarts,
 		});
 
+		await report(true, false);
+
 		void restart();
 	}
 
@@ -429,6 +581,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			shouldRun = true;
 			breaker = false;
 			outOfMemory = false;
+			restarts = 0;
 			restartStamps.length = 0;
 
 			if (child) {
@@ -444,6 +597,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			if (!child) {
 				runtime = BridgeRuntime.Stopped;
 
+				options.onStopped?.();
+
 				return;
 			}
 
@@ -488,6 +643,8 @@ export const createSupervisor = (options: SupervisorOptions) => {
 			sweep(active.pid, group);
 
 			runtime = BridgeRuntime.Stopped;
+
+			options.onStopped?.();
 		},
 
 		async command(input: string, commandOptions: BridgeCommandOptions = {}): Promise<BridgeReply> {

package/src/runtime/serve.ts

@@ -96,6 +96,15 @@ export const serve = async (driver: BridgeDriver) => {
 				});
 			});
 		},
+		onStable() {
+			ring.discard();
+		},
+		onStopped() {
+			ring.discard();
+		},
+		onExit(record) {
+			ring.crash(record);
+		},
 	});
 
 	const ring = createEventRing(driver.events, supervisor);
@@ -130,6 +139,7 @@ export const serve = async (driver: BridgeDriver) => {
 			exitCode: supervisor.exitCode,
 			outOfMemory: supervisor.outOfMemory,
 			restarts: supervisor.restarts,
+			crashRecords: true,
 			installError,
 			installErrorText,
 			installHold,