@serverkgg/bridge 0.1.4 → 0.1.6

package/src/rcon/rconClient.test.ts

@@ -19,6 +19,7 @@ import { createRconWriter, type RconWriter } from "./rconWriter";
 const HOST = "127.0.0.1";
 
 const CLOSED_PORT = 59_433;
+const CLOSED_HOST = "::1";
 
 const PASSWORD = "kX7-forbidden-in-errors";
 
@@ -1005,7 +1006,7 @@ describe("RconClient failures", () => {
 
 	it("reports a port nothing listens on", async () => {
 		const client = new RconClient({
-			host: HOST,
+			host: CLOSED_HOST,
 			port: CLOSED_PORT,
 			password: PASSWORD,
 			timeoutMs: 1000,
@@ -1475,7 +1476,7 @@ describe("RconClient secrecy", () => {
 		});
 
 		const unreachable = new RconClient({
-			host: HOST,
+			host: CLOSED_HOST,
 			port: CLOSED_PORT,
 			password: PASSWORD,
 			timeoutMs: 500,

package/src/runtime/bridgeDriver.test.ts

@@ -1899,6 +1899,51 @@ describe("the lifecycle module", () => {
 		}
 	});
 
+	it("accepts a grace from one second to ten minutes and a stable line", () => {
+		for (const graceSeconds of [
+			1,
+			20,
+			600,
+		]) {
+			expect(() =>
+				validateDriver(
+					lifecycleDriver({
+						graceSeconds,
+						stableLine: /logged in as/i,
+					}),
+				),
+			).not.toThrow();
+		}
+	});
+
+	it("refuses a grace outside one second to ten minutes", () => {
+		for (const graceSeconds of [
+			0,
+			601,
+			2.5,
+		]) {
+			expect(() =>
+				validateDriver(
+					lifecycleDriver({
+						graceSeconds,
+					}),
+				),
+			).toThrow(
+				`the lifecycle module declares a grace of ${graceSeconds} seconds — it must be a whole number from 1 to 600`,
+			);
+		}
+	});
+
+	it("refuses a stable line that is not a regular expression", () => {
+		expect(() =>
+			validateDriver(
+				lifecycleDriver({
+					stableLine: "logged in as",
+				}),
+			),
+		).toThrow("the lifecycle module declares a stable line that is not a regular expression");
+	});
+
 	it("accepts each lifecycle subject the bridge names", () => {
 		for (const subject of Object.values(BridgeLifecycleSubject)) {
 			expect(() =>

package/src/runtime/bridgeDriver.ts

@@ -62,6 +62,10 @@ const READY_TIMEOUT_MIN_SECONDS = 1;
 
 const READY_TIMEOUT_MAX_SECONDS = 3600;
 
+const GRACE_MIN_SECONDS = 1;
+
+const GRACE_MAX_SECONDS = 600;
+
 const RESERVED_MODULE_IDS = new Set([
 	"announce",
 	"backup",
@@ -635,6 +639,27 @@ const validateLifecycle = (driver: BridgeDriver) => {
 		);
 	}
 
+	const graceSeconds: unknown = driver.lifecycle.graceSeconds;
+
+	if (
+		graceSeconds !== undefined
+		&& !(
+			Number.isInteger(graceSeconds)
+			&& (graceSeconds as number) >= GRACE_MIN_SECONDS
+			&& (graceSeconds as number) <= GRACE_MAX_SECONDS
+		)
+	) {
+		fail(
+			`the lifecycle module declares a grace of ${String(graceSeconds)} seconds — it must be a whole number from ${GRACE_MIN_SECONDS} to ${GRACE_MAX_SECONDS}`,
+		);
+	}
+
+	const stableLine: unknown = driver.lifecycle.stableLine;
+
+	if (stableLine !== undefined && !(stableLine instanceof RegExp)) {
+		fail("the lifecycle module declares a stable line that is not a regular expression");
+	}
+
 	const subject: unknown = driver.lifecycle.subject;
 
 	if (subject !== undefined && !isBridgeLifecycleSubject(subject)) {

package/src/runtime/bridgeEvents.ts

@@ -111,6 +111,8 @@ export const createEventRing = (events: BridgeEventsModule | undefined, supervis
 			push(BridgeEventName.ServerCrashed, payload);
 
 			evidence.clear();
+
+			return cause;
 		},
 
 		discard() {

package/src/runtime/bridgeSupervisor.test.ts

@@ -1,12 +1,14 @@
 import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test";
 import { tmpdir } from "node:os";
-import { BridgeRuntime } from "../core";
+import { BOT_CRASH_EVIDENCE, LOGIN_LINE } from "../bot";
+import { BridgeKind, BridgeRuntime } from "../core";
 import { BridgeLifecycleSubject, BridgeStopSignal } from "../modules";
-import { BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
+import { BridgeCrashCause, BridgeCrashPhase, type BridgeCrashRecord } from "../protocol";
+import { createEventRing } from "./bridgeEvents";
 import type { BridgeLeveledLogger } from "./bridgeLog";
 import { createBridgeRedaction, REDACTED } from "./bridgeRedaction";
 import { readProcessGroup } from "./bridgeSignal";
-import { type BridgeSupervisor, createSupervisor } from "./bridgeSupervisor";
+import { type BridgeSupervisor, createSupervisor, type SupervisorOptions } from "./bridgeSupervisor";
 
 const CHILD_PATTERN = /child=(?<pid>\d+)/;
 
@@ -1277,3 +1279,161 @@ describe("the lifecycle subject", () => {
 		await expect(supervisor.start()).rejects.toThrow('the bot environment names "BAD-KEY"');
 	});
 });
+
+const GRACE_SECONDS = 1;
+
+const READY_THEN_ALIVE = "echo ready; exec sleep 300";
+
+const READY_THEN_LOGIN = "echo ready; sleep 0.2; echo 'Logged in as serverk-bot#0001'; exec sleep 300";
+
+const READY_THEN_TOKEN_CRASH =
+	"echo ready; sleep 0.3; echo 'Error [TokenInvalid]: An invalid token was provided.' >&2; exit 1";
+
+const graceSupervisor = (script: string, grace: Partial<Pick<SupervisorOptions, "graceSeconds" | "stableLine">>) => {
+	const records: BridgeCrashRecord[] = [];
+	const causes: BridgeCrashCause[] = [];
+
+	let stables = 0;
+
+	const supervisor: BridgeSupervisor = createSupervisor({
+		cwd: tmpdir(),
+		uid: process.getuid?.() ?? 0,
+		gid: process.getgid?.() ?? 0,
+		stopTimeoutSeconds: 1,
+		ready: READY,
+		...grace,
+		onLog: silent,
+		resolveCommand: async () => {
+			return [
+				"bash",
+				"-c",
+				script,
+			];
+		},
+		onReady() {
+			ring.discard();
+		},
+		onStable() {
+			stables += 1;
+		},
+		onExit(record) {
+			records.push(record);
+			causes.push(ring.crash(record));
+		},
+	});
+
+	const ring = createEventRing(
+		{
+			kind: BridgeKind.Events,
+			patterns: [
+				...BOT_CRASH_EVIDENCE,
+			],
+		},
+		supervisor,
+	);
+
+	return {
+		supervisor,
+		records,
+		causes,
+		stables: () => stables,
+	};
+};
+
+describe("the grace window", () => {
+	it("marks a driver from an older bridge that declares no grace stable at its ready line", async () => {
+		const { supervisor, stables } = graceSupervisor(READY_THEN_ALIVE, {});
+
+		await supervisor.start();
+
+		expect(await waitFor(() => stables() === 1, 5000)).toBe(true);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Running);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	it("holds a bot that printed no login line at booting until the grace passes", async () => {
+		const { supervisor, stables } = graceSupervisor(READY_THEN_ALIVE, {
+			graceSeconds: GRACE_SECONDS,
+			stableLine: LOGIN_LINE,
+		});
+
+		await supervisor.start();
+
+		expect(await waitFor(() => supervisor.runtime === BridgeRuntime.Running, 5000)).toBe(true);
+
+		await Bun.sleep(400);
+
+		expect(stables(), "the bot was stable before the grace passed").toBe(0);
+		expect(await waitFor(() => stables() === 1, 5000)).toBe(true);
+
+		await Bun.sleep(GRACE_SECONDS * 1000);
+
+		expect(stables(), "the bot was reported stable twice").toBe(1);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	it("marks a bot stable at its login line, before the grace passes", async () => {
+		const { supervisor, stables } = graceSupervisor(READY_THEN_LOGIN, {
+			graceSeconds: 30,
+			stableLine: LOGIN_LINE,
+		});
+
+		await supervisor.start();
+
+		expect(await waitFor(() => stables() === 1, 5000)).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+
+	it("counts an exit inside the grace as a boot failure with the cause the bot printed", async () => {
+		const { supervisor, records, causes, stables } = graceSupervisor(READY_THEN_TOKEN_CRASH, {
+			graceSeconds: 30,
+			stableLine: LOGIN_LINE,
+		});
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(records.at(0)?.restarting).toBe(false);
+		expect(causes).toEqual([
+			BridgeCrashCause.TokenInvalid,
+		]);
+		expect(stables()).toBe(0);
+		expect(supervisor.runtime).toBe(BridgeRuntime.Crashed);
+	}, 30_000);
+
+	it("forgets what the install printed once the launcher hands over, so only the bot's own lines name the cause", async () => {
+		const { supervisor, records, causes } = graceSupervisor(
+			"echo 'gyp ERR! stack Error: not found: make'; echo ready; sleep 0.3; exit 1",
+			{
+				graceSeconds: 30,
+				stableLine: LOGIN_LINE,
+			},
+		);
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.phase).toBe(BridgeCrashPhase.Booting);
+		expect(causes).toEqual([
+			BridgeCrashCause.Unknown,
+		]);
+	}, 30_000);
+
+	it("restarts the same exit when it comes after the grace", async () => {
+		const { supervisor, records } = graceSupervisor("echo ready; sleep 0.5; exit 1", {
+			graceSeconds: 0.2,
+		});
+
+		await supervisor.start();
+
+		expect(await waitFor(() => records.length > 0, 10_000)).toBe(true);
+		expect(records.at(0)?.phase).toBe(BridgeCrashPhase.Running);
+		expect(records.at(0)?.restarting).toBe(true);
+
+		await supervisor.stop();
+	}, 30_000);
+});

package/src/runtime/bridgeSupervisor.ts

@@ -46,6 +46,8 @@ export interface SupervisorOptions {
 	subject?: BridgeLifecycleSubject;
 	ready?: RegExp;
 	failed?: RegExp;
+	graceSeconds?: number;
+	stableLine?: RegExp;
 	resolveCommand: () => Promise<string[]>;
 	resolveEnvironment?: () => Promise<Record<string, string>>;
 	redact?: (text: string) => string;
@@ -208,6 +210,11 @@ export const createSupervisor = (options: SupervisorOptions) => {
 
 	const readyTimeoutMs = (options.readyTimeoutSeconds ?? options.stopTimeoutSeconds * READY_TIMEOUT_STOP_FACTOR) * 1000;
 
+	const graceMs =
+		options.graceSeconds === undefined ? (options.stableAfterMs ?? STABLE_UPTIME_MS) : options.graceSeconds * 1000;
+
+	const deferred = options.graceSeconds !== undefined || options.stableLine !== undefined;
+
 	const awaitingReady = () => {
 		return options.readyTimeoutSeconds !== undefined && runtime === BridgeRuntime.Starting;
 	};
@@ -241,6 +248,36 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		options.onStable?.();
 	};
 
+	const settle = (launchStamp: number) => {
+		if (!deferred) {
+			markStable();
+
+			return;
+		}
+
+		const current = () => launchStamp === launchedAt && child !== null;
+
+		if (stableTimer) {
+			clearTimeout(stableTimer);
+		}
+
+		stableTimer = setTimeout(() => {
+			if (current()) {
+				markStable();
+			}
+		}, graceMs);
+
+		if (!options.stableLine) {
+			return;
+		}
+
+		void watch(options.stableLine, graceMs).then((match) => {
+			if (match && current()) {
+				markStable();
+			}
+		});
+	};
+
 	const emit = (raw: SupervisorLine) => {
 		const entry = options.redact
 			? {
@@ -491,6 +528,10 @@ export const createSupervisor = (options: SupervisorOptions) => {
 		if (!options.ready) {
 			runtime = BridgeRuntime.Running;
 
+			if (deferred) {
+				settle(launchStamp);
+			}
+
 			return;
 		}
 
@@ -509,7 +550,7 @@ export const createSupervisor = (options: SupervisorOptions) => {
 					options.onReady?.();
 				}
 
-				markStable();
+				settle(launchStamp);
 
 				return;
 			}

package/src/runtime/serve.ts

@@ -98,6 +98,14 @@ export const serve = async (driver: BridgeDriver) => {
 
 	const setup = createSetupHandlers(driver, log, redact);
 
+	const runHook = (hook: string, run: () => Promise<void>) => {
+		void run().catch((error: unknown) => {
+			log.warn(`lifecycle ${hook} hook failed`, {
+				error: errorText(error),
+			});
+		});
+	};
+
 	const supervisor = createSupervisor({
 		cwd: config.cwd,
 		uid: config.gameUid,
@@ -108,6 +116,8 @@ export const serve = async (driver: BridgeDriver) => {
 		subject: driver.lifecycle.subject,
 		ready: driver.lifecycle.ready,
 		failed: driver.lifecycle.failed,
+		graceSeconds: driver.lifecycle.graceSeconds,
+		stableLine: driver.lifecycle.stableLine,
 		redact,
 		onLog: log,
 		async resolveCommand() {
@@ -124,20 +134,31 @@ export const serve = async (driver: BridgeDriver) => {
 			await driver.lifecycle.stop?.(context);
 		},
 		onReady() {
-			void Promise.resolve(driver.lifecycle.onReady?.(context)).catch((error: unknown) => {
-				log.warn("lifecycle ready hook failed", {
-					error: errorText(error),
-				});
+			ring.discard();
+
+			runHook("ready", async () => {
+				await driver.lifecycle.onReady?.(context);
 			});
 		},
 		onStable() {
 			ring.discard();
+
+			runHook("stable", async () => {
+				await driver.lifecycle.onStable?.(context);
+			});
 		},
 		onStopped() {
 			ring.discard();
 		},
 		onExit(record) {
-			ring.crash(record);
+			const cause = ring.crash(record);
+
+			runHook("exit", async () => {
+				await driver.lifecycle.onExit?.(context, {
+					...record,
+					cause,
+				});
+			});
 		},
 	});