@babylonjs-toolkit/server 0.0.0-stage → 9.35.1

package/lib/admin/FleetDirectory.js

@@ -0,0 +1,489 @@
+// The fleet as one instance sees it (game-server-admin-console D16, DR §6.5, §13 R15): each instance publishes a
+// heartbeat to a presence hash, and instances ask each other to run fixed, typed commands over presence topics. Without
+// Redis the presence is local, so the fleet is this instance alone and every command runs in-process.
+import { matchMaker } from "colyseus";
+import { randomUUID, X509Certificate } from "node:crypto";
+import { hostname } from "node:os";
+import { monitorEventLoopDelay } from "node:perf_hooks";
+import { getLogger } from "../GameServerLogger.js";
+import { getToolkitRuntime } from "../HeadlessRuntime.js";
+import { readinessReasons, SERVER_VERSION } from "../ServerEndpoints.js";
+import { AdminRequestError } from "./AdminApiRouter.js";
+import { ADMIN_CHANGED_TOPIC, EVENT_LOOP_MONITOR_RESOLUTION_MS, FLEET_COMMAND_TIMEOUT_MS, FLEET_COMMAND_TOPIC_PREFIX, FLEET_FORGET_AFTER_MS, FLEET_INSTANCES_KEY, FLEET_LOST_AFTER_MS, FLEET_REPLY_TOPIC_PREFIX } from "./AdminConstants.js";
+/** HTTP status of an unknown or lost instance. */
+const HTTP_NOT_FOUND = 404;
+/** HTTP status of a command no handler is registered for. */
+const HTTP_NOT_IMPLEMENTED = 501;
+/** HTTP status of a command that failed on another instance without a status of its own. */
+const HTTP_BAD_GATEWAY = 502;
+/** HTTP status of a command that got no reply in time. */
+const HTTP_GATEWAY_TIMEOUT = 504;
+/** Least time between two "heartbeat failed" warnings, in milliseconds (DR §9: once per minute). */
+const HEARTBEAT_WARNING_INTERVAL_MS = 60000;
+/** Milliseconds per day, for certificate expiry. */
+const MILLISECONDS_PER_DAY = 86400000;
+/** Nanoseconds per millisecond (the event-loop histogram reports nanoseconds). */
+const NANOSECONDS_PER_MILLISECOND = 1e6;
+/** Nanoseconds per microsecond (process.cpuUsage reports microseconds). */
+const NANOSECONDS_PER_MICROSECOND = 1000n;
+/** Percent of one fully used CPU core. */
+const FULL_CORE_PERCENT = 100;
+/** Heartbeats, the command bus and change announcements of this instance. */
+export class FleetDirectory {
+    /** The shared core services. */
+    services;
+    /** Command handlers by name (each name once). */
+    handlers = new Map();
+    /** Listeners of change announcements. */
+    changeListeners = [];
+    /** The event-loop delay histogram, while started. */
+    eventLoopMonitor = null;
+    /** The heartbeat timer, while started. */
+    heartbeatTimer = null;
+    /** True while a heartbeat is being written, so a slow store never stacks them. */
+    isWritingHeartbeat = false;
+    /** The newest heartbeat this instance wrote, or null. */
+    lastHeartbeat = null;
+    /** The heartbeat the counter rates are measured from, or null. */
+    rateBaseHeartbeat = null;
+    /** The counter snapshot of the newest heartbeat (the metrics cache hands the same object until it renews). */
+    lastSnapshot = null;
+    /** CPU time used up to the last heartbeat. */
+    lastCpuUsage = process.cpuUsage();
+    /** High-resolution time of the last CPU sample, in nanoseconds. */
+    lastCpuSampleAt = process.hrtime.bigint();
+    /** Real time of the last "heartbeat failed" warning, in milliseconds. */
+    lastHeartbeatWarningAt = 0;
+    /** The subscribed command listener, while started. */
+    commandListener = (command) => void this.receiveCommand(command);
+    /** The subscribed change listener, while started. */
+    changeListener = (announcement) => this.notifyChange(announcement.collection);
+    /**
+     * Prepares the directory; nothing is published until start().
+     * @param services The shared core services.
+     */
+    constructor(services) {
+        this.services = services;
+    }
+    /** This instance's process id (matchMaker.processId, R5). */
+    get localProcessId() {
+        return this.services.processId();
+    }
+    /** True when Redis connects the fleet; false for a single instance. */
+    get isScaledOut() {
+        return this.services.config.redis.url != null;
+    }
+    /** Starts heartbeats, command handling and change announcements; the first heartbeat is written at once. */
+    async start() {
+        this.eventLoopMonitor = monitorEventLoopDelay({ resolution: EVENT_LOOP_MONITOR_RESOLUTION_MS });
+        this.eventLoopMonitor.enable();
+        await this.presence().subscribe(FLEET_COMMAND_TOPIC_PREFIX + this.localProcessId, this.commandListener);
+        await this.presence().subscribe(ADMIN_CHANGED_TOPIC, this.changeListener);
+        await this.writeHeartbeat();
+        this.heartbeatTimer = this.services.clock.setInterval(() => void this.writeHeartbeat(), this.services.heartbeatIntervalMs);
+    }
+    /** Stops heartbeats and command handling and removes this instance's heartbeat (R4). */
+    async stop() {
+        if (this.heartbeatTimer != null)
+            this.services.clock.clearInterval(this.heartbeatTimer);
+        this.heartbeatTimer = null;
+        this.eventLoopMonitor?.disable();
+        this.eventLoopMonitor = null;
+        try {
+            this.presence().unsubscribe(FLEET_COMMAND_TOPIC_PREFIX + this.localProcessId, this.commandListener);
+            this.presence().unsubscribe(ADMIN_CHANGED_TOPIC, this.changeListener);
+            await this.presence().hdel(FLEET_INSTANCES_KEY, this.localProcessId);
+        }
+        catch (error) {
+            getLogger().warn("the fleet heartbeat could not be removed", { error: String(error) });
+        }
+    }
+    /**
+     * Every known instance with its status, newest heartbeat first.
+     * @returns The instance rows.
+     */
+    async listInstances() {
+        const records = await this.listInstanceRecords();
+        return records.map((record) => record.view);
+    }
+    /**
+     * Every known instance with the heartbeat its rates are measured from, newest heartbeat first. Instances silent
+     * for FLEET_LOST_AFTER_MS are "lost"; those silent for FLEET_FORGET_AFTER_MS are removed from the hash.
+     * @returns The records.
+     */
+    async listInstanceRecords() {
+        const entries = await this.presence().hgetall(FLEET_INSTANCES_KEY);
+        const now = this.services.clock.now();
+        const records = [];
+        for (const [processId, text] of Object.entries(entries ?? {})) {
+            const stored = parseStoredHeartbeat(text);
+            if (stored == null)
+                continue;
+            const silentFor = now - Date.parse(stored.heartbeat.sentAt);
+            if (silentFor > FLEET_FORGET_AFTER_MS) {
+                void this.forgetInstance(processId);
+                continue;
+            }
+            const status = (silentFor > FLEET_LOST_AFTER_MS) ? "lost" : "up";
+            const view = Object.assign({}, stored.heartbeat, { status: status, isThisInstance: processId === this.localProcessId });
+            records.push({ view: view, previous: stored.previous });
+        }
+        records.sort((left, right) => right.view.sentAt.localeCompare(left.view.sentAt));
+        return this.withoutRestartedInstances(records);
+    }
+    /**
+     * Drops (and forgets) each lost instance whose public address a newer live instance now serves: that instance was
+     * restarted in its place, so it is no longer lost (T14: restarting a lost instance resolves its alert).
+     * @param records The instances, newest heartbeat first.
+     * @returns The records without the restarted instances.
+     */
+    withoutRestartedInstances(records) {
+        const upRecords = records.filter((record) => record.view.status === "up");
+        const isRestarted = (record) => record.view.status === "lost" && record.view.publicAddress != null
+            && upRecords.some((upRecord) => upRecord.view.publicAddress === record.view.publicAddress && upRecord.view.startedAt > record.view.startedAt);
+        for (const record of records) {
+            if (isRestarted(record))
+                void this.forgetInstance(record.view.processId);
+        }
+        return records.filter((record) => !isRestarted(record));
+    }
+    /**
+     * Runs a command on one instance: in-process when it is this one, else over the bus.
+     * @param processId The instance.
+     * @param name The command.
+     * @param commandArguments Its arguments.
+     * @returns The command's result.
+     * @throws AdminRequestError 404 for an unknown or lost instance, 504 when no reply came in time, or the status the
+     *     handler refused with.
+     */
+    async runOn(processId, name, commandArguments) {
+        if (processId === this.localProcessId)
+            return this.runLocally(name, commandArguments);
+        const instances = await this.listInstances();
+        const target = instances.find((instance) => instance.processId === processId);
+        if (target == null || target.status === "lost")
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such instance");
+        return this.sendCommand(processId, name, commandArguments);
+    }
+    /**
+     * Runs a command on every live instance (this one always included).
+     * @param name The command.
+     * @param commandArguments Its arguments.
+     * @returns Each instance's result, and the instances that failed.
+     */
+    async runOnAll(name, commandArguments) {
+        const instances = await this.listInstances();
+        const processIds = new Set([this.localProcessId]);
+        for (const instance of instances)
+            if (instance.status === "up")
+                processIds.add(instance.processId);
+        const outcome = { results: [], failures: [] };
+        await Promise.all(Array.from(processIds).map(async (processId) => {
+            try {
+                const result = await this.runOn(processId, name, commandArguments);
+                outcome.results.push({ processId: processId, result: result });
+            }
+            catch (error) {
+                outcome.failures.push({ processId: processId, error: errorMessageOf(error) });
+            }
+        }));
+        return outcome;
+    }
+    /**
+     * Registers this instance's handler of a command.
+     * @param name The command.
+     * @param handler Runs it; its arguments and result are copies (structured clones).
+     * @throws Error when the command already has a handler.
+     */
+    handle(name, handler) {
+        if (this.handlers.has(name))
+            throw new Error("the fleet command " + name + " already has a handler");
+        this.handlers.set(name, handler);
+    }
+    /**
+     * Publishes that a store collection changed, so every instance (this one included) reloads it.
+     * @param collection The collection.
+     */
+    async announceChange(collection) {
+        const announcement = { collection: collection };
+        await this.presence().publish(ADMIN_CHANGED_TOPIC, announcement);
+    }
+    /**
+     * Calls back on every announced change (any instance, this one included).
+     * @param listener The listener.
+     */
+    onChange(listener) {
+        this.changeListeners.push(listener);
+    }
+    /**
+     * The presence every instance shares (Redis when scaled out). Colyseus sets it when the server listens.
+     * @returns The presence.
+     */
+    presence() {
+        return matchMaker.presence;
+    }
+    /**
+     * Runs a registered handler on this instance with copies of the arguments and the result, since local presence
+     * hands objects over by reference.
+     * @param name The command.
+     * @param commandArguments Its arguments.
+     * @returns A copy of the result.
+     * @throws AdminRequestError 501 when no handler is registered.
+     */
+    async runLocally(name, commandArguments) {
+        const handler = this.handlers.get(name);
+        if (handler == null)
+            throw new AdminRequestError(HTTP_NOT_IMPLEMENTED, "this instance cannot run " + name);
+        const result = await handler(structuredClone(commandArguments));
+        return structuredClone(result ?? null);
+    }
+    /**
+     * Sends a command to another instance and waits for its reply on a per-command topic.
+     * @param processId The instance.
+     * @param name The command.
+     * @param commandArguments Its arguments.
+     * @returns The result.
+     * @throws AdminRequestError on a failed reply or after FLEET_COMMAND_TIMEOUT_MS.
+     */
+    async sendCommand(processId, name, commandArguments) {
+        const commandId = randomUUID();
+        const command = { commandId: commandId, name: name, arguments: commandArguments, replyTopic: FLEET_REPLY_TOPIC_PREFIX + commandId, fromProcessId: this.localProcessId };
+        const reply = await this.awaitReply(command, processId);
+        if (reply.ok)
+            return reply.result;
+        throw new AdminRequestError(reply.status ?? HTTP_BAD_GATEWAY, reply.error ?? "the instance could not run " + name);
+    }
+    /**
+     * Subscribes to a command's reply topic, publishes the command and resolves with the reply.
+     * @param command The command.
+     * @param processId The instance it goes to.
+     * @returns The reply.
+     * @throws AdminRequestError 504 after FLEET_COMMAND_TIMEOUT_MS.
+     */
+    awaitReply(command, processId) {
+        return new Promise((resolve, reject) => {
+            const onReply = (reply) => {
+                finish();
+                resolve(reply);
+            };
+            const finish = () => {
+                this.services.clock.clearTimeout(timeout);
+                this.presence().unsubscribe(command.replyTopic, onReply);
+            };
+            const timeout = this.services.clock.setTimeout(() => {
+                finish();
+                reject(new AdminRequestError(HTTP_GATEWAY_TIMEOUT, "the instance " + processId + " did not answer"));
+            }, FLEET_COMMAND_TIMEOUT_MS);
+            const publish = async () => {
+                await this.presence().subscribe(command.replyTopic, onReply);
+                await this.presence().publish(FLEET_COMMAND_TOPIC_PREFIX + processId, command);
+            };
+            publish().catch((error) => {
+                finish();
+                reject(error);
+            });
+        });
+    }
+    /**
+     * Runs a command another instance sent and publishes the reply.
+     * @param command The command.
+     */
+    async receiveCommand(command) {
+        const reply = await this.answerCommand(command);
+        try {
+            await this.presence().publish(command.replyTopic, reply);
+        }
+        catch (error) {
+            getLogger().warn("a fleet command reply could not be sent", { command: command.name, error: String(error) });
+        }
+    }
+    /**
+     * Runs a received command and turns its outcome into a reply.
+     * @param command The command.
+     * @returns The reply.
+     */
+    async answerCommand(command) {
+        const reply = { commandId: command.commandId, processId: this.localProcessId, ok: true, result: null, error: null, status: null };
+        try {
+            reply.result = await this.runLocally(command.name, command.arguments);
+        }
+        catch (error) {
+            reply.ok = false;
+            reply.error = errorMessageOf(error);
+            reply.status = (error instanceof AdminRequestError) ? error.status : null;
+        }
+        return reply;
+    }
+    /**
+     * Calls every change listener; a listener that throws is logged and the others still run.
+     * @param collection The changed collection.
+     */
+    notifyChange(collection) {
+        for (const listener of this.changeListeners) {
+            try {
+                listener(collection);
+            }
+            catch (error) {
+                getLogger().error("an admin change listener failed", { collection: collection, error: String(error) });
+            }
+        }
+    }
+    /**
+     * Removes a long-lost instance from the heartbeat hash.
+     * @param processId The instance.
+     */
+    async forgetInstance(processId) {
+        try {
+            await this.presence().hdel(FLEET_INSTANCES_KEY, processId);
+        }
+        catch (error) {
+            getLogger().warn("a lost instance could not be forgotten", { processId: processId, error: String(error) });
+        }
+    }
+    /** Builds and writes this instance's heartbeat; a failure is logged at warn once per minute (DR §9). */
+    async writeHeartbeat() {
+        if (this.isWritingHeartbeat)
+            return;
+        this.isWritingHeartbeat = true;
+        try {
+            const stored = await this.nextStoredHeartbeat();
+            await this.presence().hset(FLEET_INSTANCES_KEY, this.localProcessId, JSON.stringify(stored));
+        }
+        catch (error) {
+            this.warnHeartbeatFailure(error);
+        }
+        finally {
+            this.isWritingHeartbeat = false;
+        }
+    }
+    /**
+     * Builds the next heartbeat entry, moving the rate base forward whenever the counters were renewed.
+     * @returns The entry to store.
+     */
+    async nextStoredHeartbeat() {
+        const metrics = this.services.routeContext.metrics();
+        const snapshot = (metrics != null) ? await metrics.snapshot() : null;
+        const heartbeat = await this.buildHeartbeat(snapshot);
+        if (snapshot !== this.lastSnapshot)
+            this.rateBaseHeartbeat = this.lastHeartbeat;
+        this.lastSnapshot = snapshot;
+        this.lastHeartbeat = heartbeat;
+        return { heartbeat: heartbeat, previous: this.rateBaseHeartbeat };
+    }
+    /**
+     * This instance's heartbeat now (DR §6.5).
+     * @param snapshot The metrics counters, or null when metrics are off (the counters are then 0).
+     * @returns The heartbeat.
+     */
+    async buildHeartbeat(snapshot) {
+        const state = this.services.state;
+        const reasons = await readinessReasons(this.services.routeContext);
+        const rooms = Array.from(state.sessionRooms);
+        const memory = process.memoryUsage();
+        return {
+            processId: this.localProcessId,
+            publicAddress: state.config.publicAddress,
+            hostName: hostname(),
+            instanceLabel: state.config.admin.instanceLabel,
+            serverVersion: SERVER_VERSION,
+            protocolVersion: getToolkitRuntime().TOOLKIT.NETWORK_PROTOCOL_VERSION,
+            startedAt: state.startedAt,
+            sentAt: new Date(this.services.clock.now()).toISOString(),
+            ready: reasons.length === 0,
+            readinessReasons: reasons,
+            draining: state.draining,
+            drainDeadline: this.services.routeContext.drainDeadline(),
+            maintenance: state.maintenance,
+            players: rooms.reduce((total, room) => total + room.playerCount, 0),
+            sessions: rooms.length,
+            dedicatedSessions: rooms.filter((room) => room.isDedicated).length,
+            cpuPercent: this.sampleCpuPercent(),
+            residentMemoryBytes: memory.rss,
+            heapUsedBytes: memory.heapUsed,
+            eventLoopLagMilliseconds: this.sampleEventLoopLag(),
+            certificateDaysLeft: certificateDaysLeft(state.tlsMaterial, this.services.clock.now()),
+            logLevel: getLogger().currentLevel,
+            joinsAccepted: snapshot?.joinsAccepted ?? 0,
+            joinsRefused: snapshot?.joinsRefused ?? 0,
+            refusalsByReason: Object.assign({}, snapshot?.refusalsByReason ?? {}),
+            authFailures: snapshot?.authFailures ?? 0,
+            framesWebsocket: snapshot?.framesByTransport.websocket ?? 0,
+            framesWebtransport: snapshot?.framesByTransport.webtransport ?? 0
+        };
+    }
+    /**
+     * CPU use since the last heartbeat, as a percent of one core (user plus system time over elapsed time).
+     * @returns The percent.
+     */
+    sampleCpuPercent() {
+        const now = process.hrtime.bigint();
+        const used = process.cpuUsage(this.lastCpuUsage);
+        const elapsedMicroseconds = Number((now - this.lastCpuSampleAt) / NANOSECONDS_PER_MICROSECOND);
+        this.lastCpuUsage = process.cpuUsage();
+        this.lastCpuSampleAt = now;
+        if (elapsedMicroseconds <= 0)
+            return 0;
+        return ((used.user + used.system) / elapsedMicroseconds) * FULL_CORE_PERCENT;
+    }
+    /**
+     * The mean event-loop delay since the last heartbeat; the histogram restarts afterwards.
+     * @returns The delay, in milliseconds (0 before any sample).
+     */
+    sampleEventLoopLag() {
+        if (this.eventLoopMonitor == null)
+            return 0;
+        const meanNanoseconds = this.eventLoopMonitor.mean;
+        this.eventLoopMonitor.reset();
+        return Number.isFinite(meanNanoseconds) ? meanNanoseconds / NANOSECONDS_PER_MILLISECOND : 0;
+    }
+    /**
+     * Logs a failed heartbeat at warn, at most once per HEARTBEAT_WARNING_INTERVAL_MS.
+     * @param error What failed.
+     */
+    warnHeartbeatFailure(error) {
+        const now = Date.now();
+        if (now - this.lastHeartbeatWarningAt < HEARTBEAT_WARNING_INTERVAL_MS)
+            return;
+        this.lastHeartbeatWarningAt = now;
+        getLogger().warn("the fleet heartbeat failed", { error: String(error) });
+    }
+}
+/**
+ * Days until the served certificate expires (DR §8), rounded down.
+ * @param material The TLS material served now, or null without TLS.
+ * @param nowMilliseconds The current time.
+ * @returns The days left (negative once expired), or null without TLS or for an unreadable certificate.
+ */
+export function certificateDaysLeft(material, nowMilliseconds) {
+    if (material == null)
+        return null;
+    try {
+        const validTo = new Date(new X509Certificate(material.cert).validTo).getTime();
+        return Math.floor((validTo - nowMilliseconds) / MILLISECONDS_PER_DAY);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Parses one heartbeat hash entry.
+ * @param text The stored JSON.
+ * @returns The entry, or null when it is not a heartbeat entry.
+ */
+function parseStoredHeartbeat(text) {
+    try {
+        const parsed = JSON.parse(text);
+        return (parsed != null && parsed.heartbeat != null && typeof parsed.heartbeat.sentAt === "string") ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * The readable message of an error.
+ * @param error What was thrown.
+ * @returns Its message.
+ */
+function errorMessageOf(error) {
+    return (error instanceof Error) ? error.message : String(error);
+}

package/lib/admin/FleetLeader.d.ts

@@ -0,0 +1,43 @@
+import type { AdminCoreServices } from "./AdminServices.js";
+/** One piece of leader work, run on every leader tick. */
+export type LeaderTickHandler = () => Promise<void>;
+/** Takes and renews the leader lease and runs the tick handlers while it holds it. */
+export declare class FleetLeader {
+    /** The shared core services. */
+    private readonly services;
+    /** The work run on every tick, in registration order. */
+    private readonly tickHandlers;
+    /** True while this instance holds the lease. */
+    private holdsLease;
+    /** The lease renewal timer, while started. */
+    private renewTimer;
+    /** The tick timer, while started. */
+    private tickTimer;
+    /** True while a tick runs, so a slow tick never overlaps the next one. */
+    private isTicking;
+    /**
+     * Prepares the leader; nothing runs until start().
+     * @param services The shared core services.
+     */
+    constructor(services: AdminCoreServices);
+    /** True while this instance holds the leader lease. */
+    get isLeader(): boolean;
+    /**
+     * Adds work to every leader tick. Handlers run one after another; a failing handler is logged and the others run.
+     * @param handler The work.
+     */
+    addTickHandler(handler: LeaderTickHandler): void;
+    /** Takes the lease if it is free, then renews it every LEADER_RENEW_INTERVAL_MS and ticks every leaderTickMs. */
+    start(): Promise<void>;
+    /** Stops renewing and ticking; the lease runs out by itself after LEADER_LEASE_TTL_MS. */
+    stop(): void;
+    /** Runs every tick handler once, when this instance is the leader and no tick is running. */
+    tick(): Promise<void>;
+    /**
+     * Runs one tick handler; a throw is logged.
+     * @param handler The handler.
+     */
+    private runHandler;
+    /** Takes or renews the lease; a store failure counts as not holding it. */
+    private renewLease;
+}

package/lib/admin/FleetLeader.js

@@ -0,0 +1,89 @@
+// The fleet leader (game-server-admin-console D24): the instance holding the store lease "fleet-leader" evaluates
+// alert rules, runs maintenance windows and drives rolling updates. Without Redis the file store grants the lease to
+// its one instance, so a single server is always the leader.
+import { getLogger } from "../GameServerLogger.js";
+import { LEADER_LEASE_NAME, LEADER_LEASE_TTL_MS, LEADER_RENEW_INTERVAL_MS } from "./AdminConstants.js";
+/** Takes and renews the leader lease and runs the tick handlers while it holds it. */
+export class FleetLeader {
+    /** The shared core services. */
+    services;
+    /** The work run on every tick, in registration order. */
+    tickHandlers = [];
+    /** True while this instance holds the lease. */
+    holdsLease = false;
+    /** The lease renewal timer, while started. */
+    renewTimer = null;
+    /** The tick timer, while started. */
+    tickTimer = null;
+    /** True while a tick runs, so a slow tick never overlaps the next one. */
+    isTicking = false;
+    /**
+     * Prepares the leader; nothing runs until start().
+     * @param services The shared core services.
+     */
+    constructor(services) {
+        this.services = services;
+    }
+    /** True while this instance holds the leader lease. */
+    get isLeader() {
+        return this.holdsLease;
+    }
+    /**
+     * Adds work to every leader tick. Handlers run one after another; a failing handler is logged and the others run.
+     * @param handler The work.
+     */
+    addTickHandler(handler) {
+        this.tickHandlers.push(handler);
+    }
+    /** Takes the lease if it is free, then renews it every LEADER_RENEW_INTERVAL_MS and ticks every leaderTickMs. */
+    async start() {
+        await this.renewLease();
+        this.renewTimer = this.services.clock.setInterval(() => void this.renewLease(), LEADER_RENEW_INTERVAL_MS);
+        this.tickTimer = this.services.clock.setInterval(() => void this.tick(), this.services.leaderTickMs);
+    }
+    /** Stops renewing and ticking; the lease runs out by itself after LEADER_LEASE_TTL_MS. */
+    stop() {
+        if (this.renewTimer != null)
+            this.services.clock.clearInterval(this.renewTimer);
+        if (this.tickTimer != null)
+            this.services.clock.clearInterval(this.tickTimer);
+        this.renewTimer = null;
+        this.tickTimer = null;
+        this.holdsLease = false;
+    }
+    /** Runs every tick handler once, when this instance is the leader and no tick is running. */
+    async tick() {
+        if (!this.holdsLease || this.isTicking)
+            return;
+        this.isTicking = true;
+        try {
+            for (const handler of this.tickHandlers)
+                await this.runHandler(handler);
+        }
+        finally {
+            this.isTicking = false;
+        }
+    }
+    /**
+     * Runs one tick handler; a throw is logged.
+     * @param handler The handler.
+     */
+    async runHandler(handler) {
+        try {
+            await handler();
+        }
+        catch (error) {
+            getLogger().warn("a fleet leader task failed", { error: String(error) });
+        }
+    }
+    /** Takes or renews the lease; a store failure counts as not holding it. */
+    async renewLease() {
+        try {
+            this.holdsLease = await this.services.store.tryAcquireLease(LEADER_LEASE_NAME, this.services.processId(), LEADER_LEASE_TTL_MS);
+        }
+        catch (error) {
+            this.holdsLease = false;
+            getLogger().warn("the fleet leader lease could not be renewed", { error: String(error) });
+        }
+    }
+}

package/lib/admin/FleetRoutes.d.ts

@@ -0,0 +1,55 @@
+import type { GameServerStateValues } from "../GameServerState.js";
+import { type AdminApiRouter } from "./AdminApiRouter.js";
+import type { AdminNoticePayload, AdminNoticeSeverity } from "./AdminApiTypes.js";
+import type { AdminCoreServices, AdminServices } from "./AdminServices.js";
+/**
+ * Registers the overview, instances/* and maintenance routes.
+ * @param router The admin API router.
+ * @param services The shared services.
+ */
+export declare function registerFleetRoutes(router: AdminApiRouter, services: AdminServices): void;
+/**
+ * Registers this instance's handlers of the setMaintenance, clearMaintenance and drain fleet commands (R15).
+ * @param services The shared services.
+ */
+export declare function registerFleetCommands(services: AdminServices): void;
+/**
+ * A notice to players (DR §6.9) with a fresh id and the server's time.
+ * @param services The shared services (the clock).
+ * @param text The text (at most NOTICE_TEXT_MAX_LENGTH characters).
+ * @param severity How prominent it is.
+ * @param countdownSeconds Seconds until the announced event, or null.
+ * @returns The notice.
+ */
+export declare function createNoticePayload(services: Pick<AdminCoreServices, "clock">, text: string, severity: AdminNoticeSeverity, countdownSeconds: number | null): AdminNoticePayload;
+/**
+ * Sends a notice to the sessions of this instance.
+ * @param state The server state.
+ * @param roomId One room, or null for every room of this instance.
+ * @param playerId One player of that room, or 0 for everyone.
+ * @param notice The notice.
+ * @returns How many players were targeted.
+ */
+export declare function sendLocalNotice(state: GameServerStateValues, roomId: string | null, playerId: number, notice: AdminNoticePayload): number;
+/**
+ * Refuses a fleet-only action on a single instance (DR §9, R33).
+ * @param services The shared services.
+ * @throws AdminRequestError 409 without Redis.
+ */
+export declare function requireScaledOut(services: AdminServices): void;
+/**
+ * An optional notice or maintenance text, at most NOTICE_TEXT_MAX_LENGTH characters.
+ * @param body The body.
+ * @param name The field.
+ * @returns The text, or null.
+ * @throws AdminRequestError 400 when it is not text or too long.
+ */
+export declare function noticeTextField(body: Record<string, unknown>, name: string): string | null;
+/**
+ * A number field that may not be negative.
+ * @param body The body.
+ * @param name The field.
+ * @returns The number.
+ * @throws AdminRequestError 400 when it is not a number or negative.
+ */
+export declare function nonNegativeNumberField(body: Record<string, unknown>, name: string): number;