@babylonjs-toolkit/server 0.0.0-stage → 9.35.1

package/lib/admin/RolloutCoordinator.js

@@ -0,0 +1,321 @@
+// Rolling updates (game-server-admin-console D24, DR §7.8): the fleet leader drains one instance at a time and waits for
+// a replacement (a new instance, not in the rollout's starting set, ready and at least the starting version) before the
+// next step; a step without a replacement within perInstanceTimeoutMinutes fails and pauses the rollout. The state lives
+// in the "rollouts" collection, so a new leader continues where the old one stopped. Machines are never started or
+// stopped here: the platform replaces drained instances.
+import { randomUUID } from "node:crypto";
+import { AdminRequestError } from "./AdminApiRouter.js";
+import { AdminConflictError } from "./AdminStore.js";
+/** HTTP statuses of the refusals. */
+const HTTP_BAD_REQUEST = 400;
+const HTTP_NOT_FOUND = 404;
+const HTTP_CONFLICT = 409;
+/** Milliseconds per second and per minute. */
+const MILLISECONDS_PER_SECOND = 1000;
+const MILLISECONDS_PER_MINUTE = 60000;
+/** The refusal of a rollout on a single instance (DR §7.8). */
+const NEEDS_TWO_INSTANCES_MESSAGE = "a rolling update needs at least two instances; use maintenance and drain instead";
+/** Rollout states that still hold the fleet. */
+const UNFINISHED_STATES = new Set(["running", "paused"]);
+/** Step states that are finished. */
+const FINISHED_STEP_STATES = new Set(["replaced", "failed"]);
+/** Starts, steers and drives the fleet's rolling updates. */
+export class RolloutCoordinator {
+    /** The shared core services. */
+    services;
+    /** The fleet directory (instances and the drain command). */
+    fleet;
+    /** The activity feed's server events. */
+    serverEvents;
+    /**
+     * Creates the coordinator; nothing runs until the leader ticks.
+     * @param services The shared core services.
+     * @param fleet The fleet directory.
+     * @param serverEvents The activity feed's server events.
+     */
+    constructor(services, fleet, serverEvents) {
+        this.services = services;
+        this.fleet = fleet;
+        this.serverEvents = serverEvents;
+    }
+    /**
+     * Every rollout, newest first.
+     * @returns The rollouts.
+     */
+    async list() {
+        const records = (await this.storedRollouts()).map((stored) => stored.value);
+        return records.sort((left, right) => right.startedAt.localeCompare(left.startedAt)).map(rolloutView);
+    }
+    /**
+     * Starts a rollout over the live instances (POST rollouts). The instance that starts it goes last, so it keeps
+     * answering the console for as long as possible.
+     * @param request The notice and timeouts.
+     * @param startedBy Who starts it.
+     * @returns The rollout.
+     * @throws AdminRequestError 409 with fewer than two live instances or while another rollout is unfinished, 400 for
+     *     negative times.
+     */
+    async start(request, startedBy) {
+        if (request.noticeLeadSeconds < 0 || request.perInstanceTimeoutMinutes <= 0)
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "noticeLeadSeconds may not be negative and perInstanceTimeoutMinutes must be above 0");
+        const upInstances = (await this.fleet.listInstances()).filter((instance) => instance.status === "up");
+        if (upInstances.length < 2)
+            throw new AdminRequestError(HTTP_CONFLICT, NEEDS_TWO_INSTANCES_MESSAGE);
+        const unfinished = (await this.list()).some((rollout) => UNFINISHED_STATES.has(rollout.state));
+        if (unfinished)
+            throw new AdminRequestError(HTTP_CONFLICT, "a rolling update is already running; pause, resume or cancel it first");
+        const ordered = upInstances.filter((instance) => !instance.isThisInstance).concat(upInstances.filter((instance) => instance.isThisInstance));
+        const record = {
+            rolloutId: randomUUID(),
+            state: "running",
+            startedAt: new Date(this.services.clock.now()).toISOString(),
+            startedBy: startedBy,
+            startingProcessIds: ordered.map((instance) => instance.processId),
+            startingVersion: lowestVersion(ordered.map((instance) => instance.serverVersion)),
+            noticeText: request.noticeText,
+            noticeLeadSeconds: request.noticeLeadSeconds,
+            perInstanceTimeoutMinutes: request.perInstanceTimeoutMinutes,
+            steps: ordered.map((instance) => ({ processId: instance.processId, state: "waiting", replacementProcessId: null, message: null })),
+            stepStartedAt: null,
+            message: null
+        };
+        await this.services.store.putDocument("rollouts", record.rolloutId, record, 0);
+        return rolloutView(record);
+    }
+    /**
+     * Pauses, resumes or cancels a rollout. Resuming a failed step gives it another perInstanceTimeoutMinutes.
+     * @param rolloutId The rollout.
+     * @param command The command.
+     * @returns The rollout.
+     * @throws AdminRequestError 404 for an unknown rollout or command, 409 when the rollout cannot take the command.
+     */
+    async command(rolloutId, command) {
+        const stored = await this.services.store.getDocument("rollouts", rolloutId);
+        if (stored == null)
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such rolling update");
+        const record = structuredClone(stored.value);
+        if (command === "pause")
+            pauseRollout(record);
+        else if (command === "resume")
+            this.resumeRollout(record);
+        else if (command === "cancel")
+            cancelRollout(record);
+        else
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such rolling update command");
+        await this.services.store.putDocument("rollouts", rolloutId, record, stored.revision);
+        return rolloutView(record);
+    }
+    /** The leader tick: moves every running rollout on by one decision. */
+    async tick() {
+        for (const stored of await this.storedRollouts()) {
+            if (stored.value.state !== "running")
+                continue;
+            const record = structuredClone(stored.value);
+            await this.advance(record);
+            await this.saveIfChanged(stored, record);
+        }
+    }
+    /**
+     * Runs the current step of a running rollout (DR §7.8 steps 1–4), or finishes the rollout.
+     * @param record The rollout, changed in place.
+     */
+    async advance(record) {
+        const step = record.steps.find((candidate) => !FINISHED_STEP_STATES.has(candidate.state));
+        if (step == null) {
+            record.state = "done";
+            record.stepStartedAt = null;
+            this.serverEvents.record("rolling update finished", null);
+            return;
+        }
+        const now = this.services.clock.now();
+        if (step.state === "waiting") {
+            await this.startDrain(record, step, now);
+            return;
+        }
+        const stepStartedAt = Date.parse(record.stepStartedAt ?? new Date(now).toISOString());
+        if (step.state === "noticeSent" && now >= stepStartedAt + record.noticeLeadSeconds * MILLISECONDS_PER_SECOND)
+            step.state = "draining";
+        await this.checkReplacement(record, step, now, stepStartedAt);
+    }
+    /**
+     * Sends a step's drain (with the rollout's notice) to its instance.
+     * @param record The rollout.
+     * @param step The step.
+     * @param now The clock time.
+     */
+    async startDrain(record, step, now) {
+        record.stepStartedAt = new Date(now).toISOString();
+        try {
+            await this.fleet.runOn(step.processId, "drain", { noticeText: record.noticeText, noticeLeadSeconds: record.noticeLeadSeconds });
+        }
+        catch (error) {
+            // An instance that is already gone needs no drain: its replacement is all the step waits for.
+            if (!(error instanceof AdminRequestError && error.status === HTTP_NOT_FOUND)) {
+                failStep(record, step, "the drain could not start: " + ((error instanceof Error) ? error.message : String(error)));
+                return;
+            }
+        }
+        const hasNotice = record.noticeText != null && record.noticeText !== "" && record.noticeLeadSeconds > 0;
+        step.state = hasNotice ? "noticeSent" : "draining";
+        this.serverEvents.record("rolling update drains " + step.processId, step.processId);
+    }
+    /**
+     * Marks a step replaced once its instance is gone and a replacement is ready, or fails it after the timeout.
+     * @param record The rollout.
+     * @param step The step.
+     * @param now The clock time.
+     * @param stepStartedAt When the step's drain started, in milliseconds.
+     */
+    async checkReplacement(record, step, now, stepStartedAt) {
+        const instances = await this.fleet.listInstances();
+        const target = instances.find((instance) => instance.processId === step.processId);
+        const isGone = target == null || target.status === "lost";
+        const replacement = isGone ? this.findReplacement(record, instances) : undefined;
+        if (replacement != null) {
+            step.state = "replaced";
+            step.replacementProcessId = replacement.processId;
+            record.stepStartedAt = null;
+            return;
+        }
+        if (now - stepStartedAt > record.perInstanceTimeoutMinutes * MILLISECONDS_PER_MINUTE) {
+            failStep(record, step, "no replacement for " + step.processId + " within " + record.perInstanceTimeoutMinutes + " minutes");
+            this.serverEvents.record("rolling update paused: " + (step.message ?? ""), step.processId);
+        }
+    }
+    /**
+     * A ready instance that joined after the rollout started, runs at least its starting version and replaces no other
+     * step yet.
+     * @param record The rollout.
+     * @param instances The fleet now.
+     * @returns The replacement, or undefined.
+     */
+    findReplacement(record, instances) {
+        const usedReplacements = new Set(record.steps.map((step) => step.replacementProcessId));
+        return instances.find((instance) => instance.status === "up" && instance.ready && !record.startingProcessIds.includes(instance.processId) && !usedReplacements.has(instance.processId)
+            && compareVersions(instance.serverVersion, record.startingVersion) >= 0);
+    }
+    /**
+     * Resumes a paused rollout; its failed step waits for a replacement again.
+     * @param record The rollout, changed in place.
+     * @throws AdminRequestError 409 unless it is paused.
+     */
+    resumeRollout(record) {
+        if (record.state !== "paused")
+            throw new AdminRequestError(HTTP_CONFLICT, "only a paused rolling update can be resumed");
+        record.state = "running";
+        record.message = null;
+        const failed = record.steps.find((step) => step.state === "failed");
+        if (failed == null)
+            return;
+        failed.state = "draining";
+        failed.message = null;
+        record.stepStartedAt = new Date(this.services.clock.now()).toISOString();
+    }
+    /**
+     * Stores a rollout the tick changed; a pause or cancel stored meanwhile wins and the change is dropped.
+     * @param stored The rollout as read.
+     * @param record The rollout after the tick.
+     */
+    async saveIfChanged(stored, record) {
+        if (JSON.stringify(record) === JSON.stringify(stored.value))
+            return;
+        try {
+            await this.services.store.putDocument("rollouts", stored.id, record, stored.revision);
+        }
+        catch (error) {
+            if (!(error instanceof AdminConflictError))
+                throw error;
+        }
+    }
+    /**
+     * Every stored rollout.
+     * @returns The documents.
+     */
+    storedRollouts() {
+        return this.services.store.listDocuments("rollouts");
+    }
+}
+/**
+ * Fails a step and pauses its rollout.
+ * @param record The rollout, changed in place.
+ * @param step The step.
+ * @param message Why.
+ */
+function failStep(record, step, message) {
+    step.state = "failed";
+    step.message = message;
+    record.state = "paused";
+    record.message = message;
+}
+/**
+ * Pauses a running rollout.
+ * @param record The rollout, changed in place.
+ * @throws AdminRequestError 409 unless it is running.
+ */
+function pauseRollout(record) {
+    if (record.state !== "running")
+        throw new AdminRequestError(HTTP_CONFLICT, "only a running rolling update can be paused");
+    record.state = "paused";
+}
+/**
+ * Cancels an unfinished rollout; instances already draining keep draining.
+ * @param record The rollout, changed in place.
+ * @throws AdminRequestError 409 when it is already finished.
+ */
+function cancelRollout(record) {
+    if (!UNFINISHED_STATES.has(record.state))
+        throw new AdminRequestError(HTTP_CONFLICT, "this rolling update is already finished");
+    record.state = "cancelled";
+    record.stepStartedAt = null;
+}
+/**
+ * A rollout as the console sees it.
+ * @param record The stored rollout.
+ * @returns The view.
+ */
+function rolloutView(record) {
+    return {
+        rolloutId: record.rolloutId,
+        state: record.state,
+        startedAt: record.startedAt,
+        startedBy: record.startedBy,
+        steps: record.steps.map((step) => Object.assign({}, step)),
+        message: record.message
+    };
+}
+/**
+ * Compares two dotted versions by their numeric parts ("9.30.1" < "9.31.0"); a non-numeric part counts as 0.
+ * @param left One version.
+ * @param right The other.
+ * @returns A negative number, 0 or a positive number.
+ */
+export function compareVersions(left, right) {
+    const leftParts = versionParts(left);
+    const rightParts = versionParts(right);
+    for (let i = 0; i < Math.max(leftParts.length, rightParts.length); i++) {
+        const difference = (leftParts[i] ?? 0) - (rightParts[i] ?? 0);
+        if (difference !== 0)
+            return difference;
+    }
+    return 0;
+}
+/**
+ * The numeric parts of a version (anything after "-" or "+" is ignored).
+ * @param version The version.
+ * @returns The parts.
+ */
+function versionParts(version) {
+    const core = version.split(/[-+]/)[0];
+    return core.split(".").map((part) => {
+        const value = Number.parseInt(part, 10);
+        return Number.isFinite(value) ? value : 0;
+    });
+}
+/**
+ * The lowest of some versions.
+ * @param versions At least one version.
+ * @returns The lowest.
+ */
+function lowestVersion(versions) {
+    return versions.reduce((lowest, version) => (compareVersions(version, lowest) < 0) ? version : lowest);
+}

package/lib/admin/ServerEvents.d.ts

@@ -0,0 +1,57 @@
+import type { AdminAccounts } from "./AdminAccounts.js";
+import type { InstanceView } from "./AdminApiTypes.js";
+import type { AdminClock } from "./AdminClock.js";
+import type { AdminStore } from "./AdminStore.js";
+/** Appends server events to the activity feed, under the record-keeping rule of R3. */
+export declare class ServerEvents {
+    /** The admin data store. */
+    private readonly store;
+    /** The clock of each event's time. */
+    private readonly clock;
+    /** The accounts, which decide whether a file store keeps records yet. */
+    private readonly accounts;
+    /**
+     * Creates the recorder.
+     * @param store The admin data store.
+     * @param clock The clock.
+     * @param accounts The admin accounts.
+     */
+    constructor(store: AdminStore, clock: AdminClock, accounts: AdminAccounts);
+    /**
+     * Whether the console writes records nobody asked for (legacy call audits, player history, server events): always
+     * on Redis, and on the file store once an admin account exists, so token-only deployments never create the data
+     * folder (R3).
+     * @returns True when such records are kept.
+     */
+    isRecordKeepingOn(): Promise<boolean>;
+    /**
+     * Appends a server event in the background; a store failure is logged, never thrown.
+     * @param text What happened, e.g. "instance abc lost".
+     * @param processId The instance it concerns, or null for the fleet.
+     */
+    record(text: string, processId: string | null): void;
+    /**
+     * Appends a server event when record keeping is on.
+     * @param text What happened.
+     * @param processId The instance it concerns, or null.
+     */
+    private append;
+}
+/** Records instances joining, getting lost and leaving, from the leader's view of the heartbeats. */
+export declare class InstanceEventTracker {
+    /** The activity feed's server events. */
+    private readonly serverEvents;
+    /** Each known instance's status at the last observation, or null before the first one. */
+    private lastStatuses;
+    /**
+     * Creates the tracker.
+     * @param serverEvents The activity feed's server events.
+     */
+    constructor(serverEvents: ServerEvents);
+    /**
+     * Compares the instances with the last observation and records what changed. The first observation only notes the
+     * fleet as it is, so a new leader never reports instances that were already there.
+     * @param instances The fleet now.
+     */
+    observe(instances: InstanceView[]): void;
+}

package/lib/admin/ServerEvents.js

@@ -0,0 +1,89 @@
+// Server events of the activity feed (game-server-admin-console T14 step 5): instance joined or lost, drain started,
+// alert fired or resolved, certificate reloaded and maintenance on or off are appended to the "events" stream with kind
+// "server" by whichever instance observes them. Admin actions reach the same stream through AdminAudit.
+import { getLogger } from "../GameServerLogger.js";
+/** Appends server events to the activity feed, under the record-keeping rule of R3. */
+export class ServerEvents {
+    /** The admin data store. */
+    store;
+    /** The clock of each event's time. */
+    clock;
+    /** The accounts, which decide whether a file store keeps records yet. */
+    accounts;
+    /**
+     * Creates the recorder.
+     * @param store The admin data store.
+     * @param clock The clock.
+     * @param accounts The admin accounts.
+     */
+    constructor(store, clock, accounts) {
+        this.store = store;
+        this.clock = clock;
+        this.accounts = accounts;
+    }
+    /**
+     * Whether the console writes records nobody asked for (legacy call audits, player history, server events): always
+     * on Redis, and on the file store once an admin account exists, so token-only deployments never create the data
+     * folder (R3).
+     * @returns True when such records are kept.
+     */
+    async isRecordKeepingOn() {
+        return this.store.kind === "redis" || await this.accounts.hasUsers();
+    }
+    /**
+     * Appends a server event in the background; a store failure is logged, never thrown.
+     * @param text What happened, e.g. "instance abc lost".
+     * @param processId The instance it concerns, or null for the fleet.
+     */
+    record(text, processId) {
+        this.append(text, processId).catch((error) => getLogger().warn("a server event could not be recorded", { text: text, error: String(error) }));
+    }
+    /**
+     * Appends a server event when record keeping is on.
+     * @param text What happened.
+     * @param processId The instance it concerns, or null.
+     */
+    async append(text, processId) {
+        if (!(await this.isRecordKeepingOn()))
+            return;
+        const item = { at: new Date(this.clock.now()).toISOString(), kind: "server", text: text, processId: processId };
+        await this.store.appendRecord("events", item);
+    }
+}
+/** Records instances joining, getting lost and leaving, from the leader's view of the heartbeats. */
+export class InstanceEventTracker {
+    /** The activity feed's server events. */
+    serverEvents;
+    /** Each known instance's status at the last observation, or null before the first one. */
+    lastStatuses = null;
+    /**
+     * Creates the tracker.
+     * @param serverEvents The activity feed's server events.
+     */
+    constructor(serverEvents) {
+        this.serverEvents = serverEvents;
+    }
+    /**
+     * Compares the instances with the last observation and records what changed. The first observation only notes the
+     * fleet as it is, so a new leader never reports instances that were already there.
+     * @param instances The fleet now.
+     */
+    observe(instances) {
+        const statuses = new Map(instances.map((instance) => [instance.processId, instance.status]));
+        const previous = this.lastStatuses;
+        this.lastStatuses = statuses;
+        if (previous == null)
+            return;
+        for (const [processId, status] of statuses) {
+            const before = previous.get(processId);
+            if (status === "up" && before !== "up")
+                this.serverEvents.record("instance " + processId + " joined", processId);
+            if (status === "lost" && before === "up")
+                this.serverEvents.record("instance " + processId + " lost", processId);
+        }
+        for (const [processId, before] of previous) {
+            if (!statuses.has(processId) && before === "up")
+                this.serverEvents.record("instance " + processId + " left", processId);
+        }
+    }
+}

package/lib/admin/SessionRoutes.d.ts

@@ -0,0 +1,113 @@
+import { type AdminApiRouter, type AdminRequest } from "./AdminApiRouter.js";
+import type { AdminNoticePayload, DedicatedRestartOptions, PageView, SessionDetailView, SessionInspectorView, SessionSummaryView } from "./AdminApiTypes.js";
+import type { AdminServices } from "./AdminServices.js";
+/** The only SessionRoom methods the admin API may call through remoteRoomCall (D11). */
+export declare const SESSION_ROOM_ADMIN_METHODS: {
+    readonly describe: "adminDescribe";
+    readonly inspect: "adminInspect";
+    readonly kick: "adminKickPlayer";
+    readonly close: "adminClose";
+    readonly lock: "adminSetLocked";
+    readonly notice: "adminSendNotice";
+    readonly stop: "adminStopDedicated";
+    readonly disconnectBanned: "adminDisconnectBanned";
+};
+/** Every room action of the API, on whichever instance owns the room (R17). Only fixed method names are passed (D11). */
+export declare class RoomAdminClient {
+    /**
+     * A session's page.
+     * @param roomId The room.
+     * @param includeAddresses False hides player addresses.
+     * @returns The page.
+     */
+    describe(roomId: string, includeAddresses: boolean): Promise<SessionDetailView>;
+    /**
+     * A session's network objects.
+     * @param roomId The room.
+     * @returns The inspector view.
+     */
+    inspect(roomId: string): Promise<SessionInspectorView>;
+    /**
+     * Kicks a player (D12).
+     * @param roomId The room.
+     * @param playerId The player.
+     * @param message Shown to the player.
+     * @param allowRejoin True to skip the kick block.
+     * @returns False when the room has no connected player with that id.
+     */
+    kick(roomId: string, playerId: number, message: string, allowRejoin: boolean): Promise<boolean>;
+    /**
+     * Closes a session, telling its players GameClosed (D12).
+     * @param roomId The room.
+     * @param message Shown to the players.
+     */
+    close(roomId: string, message: string): Promise<void>;
+    /**
+     * Locks or unlocks a session.
+     * @param roomId The room.
+     * @param locked True to lock.
+     */
+    setLocked(roomId: string, locked: boolean): Promise<void>;
+    /**
+     * Sends a notice to one player of a session or to all of them.
+     * @param roomId The room.
+     * @param notice The notice.
+     * @param playerId The player, or 0 for everyone.
+     * @returns How many players were targeted.
+     */
+    sendNotice(roomId: string, notice: AdminNoticePayload, playerId: number): Promise<number>;
+    /**
+     * Stops a dedicated session, telling its players GameClosed (D23). The room answers before it disconnects (R17).
+     * @param roomId The room.
+     * @param message Shown to the players.
+     * @returns What a restart under the same name needs.
+     * @throws AdminRequestError 404 when the room runs no dedicated runner.
+     */
+    stopDedicated(roomId: string, message: string): Promise<DedicatedRestartOptions>;
+    /**
+     * Calls a fixed admin method of a room on its instance.
+     * @param roomId The room.
+     * @param method The method (from SESSION_ROOM_ADMIN_METHODS).
+     * @param methodArguments Its arguments.
+     * @param timeoutMs How long its instance has to answer (ROOM_CALL_TIMEOUT_MS unless the call says).
+     * @returns The method's result.
+     * @throws AdminRequestError 404 for an unknown room, 504 when its instance does not answer.
+     */
+    private call;
+}
+/**
+ * Registers the sessions/* and notices routes.
+ * @param router The admin API router.
+ * @param services The shared services.
+ */
+export declare function registerSessionRoutes(router: AdminApiRouter, services: AdminServices): void;
+/**
+ * Registers this instance's describeSessions and sendNotice fleet commands (R15).
+ * @param services The shared services.
+ */
+export declare function registerSessionCommands(services: AdminServices): void;
+/**
+ * Every session of the fleet (R18), described by each instance in-process.
+ * @param services The shared services.
+ * @returns The sessions, unsorted.
+ */
+export declare function describeFleetSessions(services: AdminServices): Promise<SessionSummaryView[]>;
+/**
+ * Whether the caller may see player addresses.
+ * @param request The request.
+ * @returns True with players.viewAddresses.
+ */
+export declare function canViewAddresses(request: AdminRequest): boolean;
+/**
+ * The page number of a list query (1-based).
+ * @param request The request.
+ * @returns The page, 1 when missing or invalid.
+ */
+export declare function pageOf(request: AdminRequest): number;
+/**
+ * One page of a list (ADMIN_PAGE_SIZE rows).
+ * @param items Every row, sorted.
+ * @param page The 1-based page.
+ * @returns The page and the total.
+ */
+export declare function pageItems<Item>(items: Item[], page: number): PageView<Item>;