@babylonjs-toolkit/server 0.0.0-stage → 9.35.1

package/lib/admin/AlertEngine.js

@@ -0,0 +1,681 @@
+// The alert engine (game-server-admin-console D24, DR §7.9, §8 rule table): built-in rules with stored settings, their
+// evaluation on every fleet leader tick, delivery to the rules' channels, and the alert history with acknowledge and
+// silence. Only the leader evaluates, so each alert fires once per fleet. Alerts live in the "alertHistory" stream as
+// snapshots; the newest snapshot of an alertId is the alert's current state.
+import { randomUUID } from "node:crypto";
+import { getLogger } from "../GameServerLogger.js";
+import { AdminRequestError } from "./AdminApiRouter.js";
+import { ADMIN_PAGE_SIZE, NEW_ADMIN_ADDRESS_LOOKBACK_DAYS } from "./AdminConstants.js";
+import { AdminConflictError, RECORD_QUERY_MAX_LIMIT } from "./AdminStore.js";
+import { AlertChannels, alertChannelView } from "./AlertChannels.js";
+import { metricsStreamOf } from "./MetricsSampler.js";
+/** HTTP statuses of the engine's refusals. */
+const HTTP_BAD_REQUEST = 400;
+const HTTP_NOT_FOUND = 404;
+/** Milliseconds per second, minute and day. */
+const MILLISECONDS_PER_SECOND = 1000;
+const MILLISECONDS_PER_MINUTE = 60000;
+const MILLISECONDS_PER_DAY = 86400000;
+/** Seconds per minute, for per-minute rates. */
+const SECONDS_PER_MINUTE = 60;
+/** Bytes per megabyte (the memoryHigh threshold unit). */
+const BYTES_PER_MEGABYTE = 1048576;
+/** Percent of a whole. */
+const FULL_PERCENT = 100;
+/** The readiness reason of an instance whose Redis does not answer. */
+const REDIS_UNREACHABLE_REASON = "redis unreachable";
+/** The audit actions the event rules read. */
+const SECOND_FACTOR_ACTION = "auth.secondFactor";
+const LOCKOUT_ACTION = "auth.lockout";
+/** Rules that fire once per event (an audit record) and are stored resolved at once, so no resolve is sent. */
+const EVENT_RULES = new Set(["newAdminAddress", "accountLockout"]);
+/** The rule id a channel test alert carries. */
+const TEST_ALERT_RULE_ID = "instanceLost";
+/** Alert channel kinds. */
+const ALERT_CHANNEL_KINDS = new Set(["webhook", "slack", "discord", "teams", "pagerduty"]);
+/** Alert severities. */
+const ALERT_SEVERITIES = new Set(["info", "warning", "critical"]);
+/** The built-in rules (DR §8). Every rule starts enabled. */
+export const ALERT_RULE_DEFINITIONS = {
+    instanceLost: { label: "An instance is lost", threshold: null, thresholdUnit: null, forMinutes: 0, severity: "critical" },
+    instanceNotReady: { label: "An instance is not ready", threshold: null, thresholdUnit: null, forMinutes: 5, severity: "warning" },
+    redisUnreachable: { label: "Redis is unreachable", threshold: null, thresholdUnit: null, forMinutes: 0, severity: "critical" },
+    refusalSpike: { label: "Join refusals spike", threshold: 60, thresholdUnit: "refusals per minute", forMinutes: 5, severity: "warning" },
+    authFailureSpike: { label: "Player auth failures spike", threshold: 30, thresholdUnit: "failures per minute", forMinutes: 5, severity: "warning" },
+    eventLoopLag: { label: "Event loop lag", threshold: 200, thresholdUnit: "milliseconds", forMinutes: 5, severity: "warning" },
+    memoryHigh: { label: "Memory use is high", threshold: 1536, thresholdUnit: "MB", forMinutes: 5, severity: "warning" },
+    dedicatedOverruns: { label: "Dedicated tick overruns", threshold: 30, thresholdUnit: "overruns per minute", forMinutes: 5, severity: "warning" },
+    certificateExpiring: { label: "Certificate expiring", threshold: 14, thresholdUnit: "days", forMinutes: 0, severity: "warning" },
+    versionMismatch: { label: "Instances run different versions", threshold: null, thresholdUnit: null, forMinutes: 30, severity: "warning" },
+    nearLimits: { label: "Near the session limit", threshold: 90, thresholdUnit: "% of limits.maxSessions", forMinutes: 5, severity: "warning" },
+    newAdminAddress: { label: "Admin sign-in from a new address", threshold: null, thresholdUnit: null, forMinutes: 0, severity: "info" },
+    accountLockout: { label: "Admin account locked", threshold: null, thresholdUnit: null, forMinutes: 0, severity: "warning" }
+};
+/** Every rule id, in the table's order. */
+const ALERT_RULE_IDS = Object.keys(ALERT_RULE_DEFINITIONS);
+/** The rules, channels and alert history of the fleet. */
+export class AlertEngine {
+    /** The shared core services. */
+    services;
+    /** The fleet directory (heartbeats). */
+    fleet;
+    /** The activity feed's server events. */
+    serverEvents;
+    /** This instance's metrics history (dedicated overruns without Redis). */
+    metricsSampler;
+    /** The channel deliverer. */
+    channels = new AlertChannels();
+    /** When each rule started breaching on this leader, in milliseconds. */
+    breachingSince = new Map();
+    /** Listeners of fired alerts. */
+    firedListeners = [];
+    /** The newest audit sequence already read, or null before the first evaluation. */
+    lastAuditSequence = null;
+    /** True once the default rules were written to the store. */
+    hasSeededRules = false;
+    /**
+     * Creates the engine; nothing runs until the leader calls evaluate().
+     * @param services The shared core services.
+     * @param fleet The fleet directory.
+     * @param serverEvents The activity feed's server events.
+     * @param metricsSampler This instance's metrics history.
+     */
+    constructor(services, fleet, serverEvents, metricsSampler) {
+        this.services = services;
+        this.fleet = fleet;
+        this.serverEvents = serverEvents;
+        this.metricsSampler = metricsSampler;
+    }
+    /**
+     * Calls back whenever an alert fires.
+     * @param listener The listener.
+     */
+    onAlertFired(listener) {
+        this.firedListeners.push(listener);
+    }
+    /** One leader evaluation (DR §7.9): every enabled, unsilenced rule is checked, fired or resolved. */
+    async evaluate() {
+        await this.seedRulesOnce();
+        const context = await this.buildContext();
+        const firing = await this.firingAlertsByRule();
+        for (const rule of await this.ruleRecords()) {
+            await this.evaluateRule(rule, context, firing.get(rule.ruleId) ?? null);
+        }
+    }
+    /**
+     * Every rule with its settings (stored settings over the defaults).
+     * @returns The rules, in the table's order.
+     */
+    async listRules() {
+        const stored = await this.storedRules();
+        return ALERT_RULE_IDS.map((ruleId) => ruleView(ruleId, stored.get(ruleId) ?? null));
+    }
+    /**
+     * Changes a rule's settings (PUT alerts/rules/:ruleId).
+     * @param ruleId The rule.
+     * @param request The new settings.
+     * @returns The rule.
+     * @throws AdminRequestError 404 for an unknown rule, 400 for an unknown channel or severity.
+     */
+    async updateRule(ruleId, request) {
+        const current = await this.requireRule(ruleId);
+        if (!ALERT_SEVERITIES.has(request.severity))
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "severity must be info, warning or critical");
+        if (request.forMinutes < 0)
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "forMinutes may not be negative");
+        await this.requireChannels(request.channelIds);
+        const base = current?.value ?? defaultRuleRecord(ruleId);
+        const record = Object.assign({}, base, {
+            enabled: request.enabled,
+            threshold: request.threshold,
+            forMinutes: request.forMinutes,
+            severity: request.severity,
+            channelIds: request.channelIds.slice()
+        });
+        const stored = await this.services.store.putDocument("alertRules", ruleId, record, null);
+        return ruleView(record.ruleId, stored);
+    }
+    /**
+     * Silences a rule for some minutes (0 ends a silence).
+     * @param ruleId The rule.
+     * @param minutes How long.
+     * @returns The rule.
+     * @throws AdminRequestError 404 for an unknown rule, 400 for a negative duration.
+     */
+    async silenceRule(ruleId, minutes) {
+        const current = await this.requireRule(ruleId);
+        if (minutes < 0)
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "minutes may not be negative");
+        const base = current?.value ?? defaultRuleRecord(ruleId);
+        const silencedUntil = (minutes > 0) ? new Date(this.services.clock.now() + minutes * MILLISECONDS_PER_MINUTE).toISOString() : null;
+        const record = Object.assign({}, base, { silencedUntil: silencedUntil });
+        const stored = await this.services.store.putDocument("alertRules", ruleId, record, null);
+        return ruleView(record.ruleId, stored);
+    }
+    /**
+     * Every channel, secrets masked.
+     * @returns The channels, oldest first.
+     */
+    async listChannels() {
+        const records = await this.channelRecords();
+        return records.sort((left, right) => left.createdAt.localeCompare(right.createdAt)).map(alertChannelView);
+    }
+    /**
+     * Adds a channel (POST alerts/channels).
+     * @param request The channel.
+     * @param createdBy Who adds it.
+     * @returns The channel, secrets masked.
+     * @throws AdminRequestError 400 for an unknown kind, a missing name, URL or routing key.
+     */
+    async createChannel(request, createdBy) {
+        validateChannelRequest(request);
+        const record = {
+            channelId: randomUUID(),
+            kind: request.kind,
+            name: request.name,
+            url: (request.kind === "pagerduty") ? null : request.url,
+            routingKey: (request.kind === "pagerduty") ? request.routingKey : null,
+            createdBy: createdBy,
+            createdAt: new Date(this.services.clock.now()).toISOString()
+        };
+        await this.services.store.putDocument("alertChannels", record.channelId, record, 0);
+        return alertChannelView(record);
+    }
+    /**
+     * Removes a channel and takes it off every rule.
+     * @param channelId The channel.
+     * @throws AdminRequestError 404 for an unknown channel.
+     */
+    async deleteChannel(channelId) {
+        if (!(await this.services.store.deleteDocument("alertChannels", channelId)))
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such alert channel");
+        for (const stored of (await this.storedRules()).values()) {
+            if (!stored.value.channelIds.includes(channelId))
+                continue;
+            const channelIds = stored.value.channelIds.filter((candidate) => candidate !== channelId);
+            await this.services.store.putDocument("alertRules", stored.id, Object.assign({}, stored.value, { channelIds: channelIds }), null);
+        }
+    }
+    /**
+     * Sends a test alert to a channel (POST alerts/channels/:channelId/test).
+     * @param channelId The channel.
+     * @returns Whether it was delivered.
+     * @throws AdminRequestError 404 for an unknown channel.
+     */
+    async testChannel(channelId) {
+        const stored = await this.services.store.getDocument("alertChannels", channelId);
+        if (stored == null)
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such alert channel");
+        const now = new Date(this.services.clock.now()).toISOString();
+        const alert = {
+            alertId: "test-" + randomUUID(),
+            ruleId: TEST_ALERT_RULE_ID,
+            severity: "info",
+            summary: "Test alert from the Babylon Game Server console",
+            processId: this.services.processId(),
+            state: "firing",
+            firedAt: now,
+            resolvedAt: null,
+            acknowledgedBy: null,
+            acknowledgedAt: null,
+            deliveries: []
+        };
+        const outcome = await this.channels.deliver(stored.value, alert, "firing", this.services.pagerDutyUrl);
+        return { delivered: outcome.ok, error: outcome.error };
+    }
+    /**
+     * One page of the alert history, newest first.
+     * @param state Only "firing" or "resolved" alerts, or null for both.
+     * @param page The page, from 1.
+     * @returns The page and the number of matches.
+     */
+    async listAlerts(state, page) {
+        const matching = (await this.currentAlerts()).filter((alert) => state == null || alert.state === state);
+        const firstIndex = (Math.max(1, Math.floor(page)) - 1) * ADMIN_PAGE_SIZE;
+        return { items: matching.slice(firstIndex, firstIndex + ADMIN_PAGE_SIZE), total: matching.length };
+    }
+    /**
+     * Acknowledges an alert: records who saw it and when.
+     * @param alertId The alert.
+     * @param acknowledgedBy The admin's name.
+     * @returns The alert.
+     * @throws AdminRequestError 404 for an unknown alert.
+     */
+    async acknowledge(alertId, acknowledgedBy) {
+        const alert = (await this.currentAlerts()).find((candidate) => candidate.alertId === alertId);
+        if (alert == null)
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such alert");
+        const acknowledged = Object.assign({}, alert, { acknowledgedBy: acknowledgedBy, acknowledgedAt: new Date(this.services.clock.now()).toISOString() });
+        await this.services.store.appendRecord("alertHistory", acknowledged);
+        return acknowledged;
+    }
+    /** Writes the default rules the store does not have yet, once record keeping is on (R3). */
+    async seedRulesOnce() {
+        if (this.hasSeededRules || !(await this.serverEvents.isRecordKeepingOn()))
+            return;
+        const stored = await this.storedRules();
+        for (const ruleId of ALERT_RULE_IDS) {
+            if (stored.has(ruleId))
+                continue;
+            try {
+                await this.services.store.putDocument("alertRules", ruleId, defaultRuleRecord(ruleId), 0);
+            }
+            catch (error) {
+                // Another instance seeded it first.
+                if (!(error instanceof AdminConflictError))
+                    throw error;
+            }
+        }
+        this.hasSeededRules = true;
+    }
+    /**
+     * Reads what every rule needs once: the instances, the new audit records and the dedicated overrun rate.
+     * @returns The context.
+     */
+    async buildContext() {
+        const records = await this.fleet.listInstanceRecords();
+        const upInstances = records.filter((record) => record.view.status === "up").map((record) => record.view);
+        return {
+            now: this.services.clock.now(),
+            records: records,
+            upInstances: upInstances,
+            newAuditRecords: await this.readNewAuditRecords(),
+            dedicatedOverrunsPerMinute: await this.dedicatedOverrunsPerMinute(upInstances)
+        };
+    }
+    /**
+     * Checks one rule and fires or resolves its alert (DR §7.9).
+     * @param rule The rule's settings.
+     * @param context The evaluation context.
+     * @param firing The rule's firing alert, or null.
+     */
+    async evaluateRule(rule, context, firing) {
+        if (!rule.enabled) {
+            this.breachingSince.delete(rule.ruleId);
+            if (firing != null)
+                await this.resolve(rule, firing);
+            return;
+        }
+        if (rule.silencedUntil != null && Date.parse(rule.silencedUntil) > context.now)
+            return;
+        const breach = await this.breachOf(rule, context);
+        if (EVENT_RULES.has(rule.ruleId)) {
+            if (breach != null)
+                await this.fire(rule, breach, context.now);
+            return;
+        }
+        if (breach == null) {
+            this.breachingSince.delete(rule.ruleId);
+            if (firing != null)
+                await this.resolve(rule, firing);
+            return;
+        }
+        const since = this.breachingSince.get(rule.ruleId) ?? context.now;
+        this.breachingSince.set(rule.ruleId, since);
+        if (firing != null || context.now - since < rule.forMinutes * MILLISECONDS_PER_MINUTE)
+            return;
+        await this.fire(rule, breach, context.now);
+    }
+    /**
+     * Whether a rule breaches now (DR §8 conditions).
+     * @param rule The rule's settings.
+     * @param context The evaluation context.
+     * @returns The breach, or null.
+     */
+    async breachOf(rule, context) {
+        const threshold = rule.threshold ?? ALERT_RULE_DEFINITIONS[rule.ruleId].threshold ?? 0;
+        switch (rule.ruleId) {
+            case "instanceLost":
+                return instancesBreach(context.records.filter((record) => record.view.status === "lost").map((record) => record.view), "is lost", "are lost");
+            case "instanceNotReady":
+                return instancesBreach(context.upInstances.filter((instance) => !instance.ready && !instance.draining), "is not ready", "are not ready");
+            case "redisUnreachable":
+                return instancesBreach(context.upInstances.filter((instance) => instance.readinessReasons.includes(REDIS_UNREACHABLE_REASON)), "cannot reach Redis", "cannot reach Redis");
+            case "refusalSpike":
+                return rateBreach(sumRate(context.records, (heartbeat) => heartbeat.joinsRefused), threshold, "join refusals per minute");
+            case "authFailureSpike":
+                return rateBreach(sumRate(context.records, (heartbeat) => heartbeat.authFailures), threshold, "player auth failures per minute");
+            case "eventLoopLag":
+                return instancesBreach(context.upInstances.filter((instance) => instance.eventLoopLagMilliseconds > threshold), "has event loop lag above " + threshold + " ms", "have event loop lag above " + threshold + " ms");
+            case "memoryHigh":
+                return instancesBreach(context.upInstances.filter((instance) => instance.residentMemoryBytes / BYTES_PER_MEGABYTE > threshold), "uses more than " + threshold + " MB", "use more than " + threshold + " MB");
+            case "dedicatedOverruns":
+                return rateBreach(context.dedicatedOverrunsPerMinute, threshold, "dedicated tick overruns per minute");
+            case "certificateExpiring":
+                return instancesBreach(context.upInstances.filter((instance) => instance.certificateDaysLeft != null && instance.certificateDaysLeft < threshold), "has a certificate expiring within " + threshold + " days", "have certificates expiring within " + threshold + " days");
+            case "versionMismatch":
+                return versionBreach(context.upInstances);
+            case "nearLimits":
+                return this.nearLimitsBreach(context.upInstances, threshold);
+            case "newAdminAddress":
+                return this.newAdminAddressBreach(context);
+            case "accountLockout":
+                return lockoutBreach(context.newAuditRecords);
+            default:
+                return null;
+        }
+    }
+    /**
+     * nearLimits: an instance hosts at least threshold percent of limits.maxSessions.
+     * @param upInstances The live instances.
+     * @param thresholdPercent The threshold, in percent.
+     * @returns The breach, or null.
+     */
+    nearLimitsBreach(upInstances, thresholdPercent) {
+        const maxSessions = this.services.config.limits.maxSessions;
+        if (maxSessions <= 0)
+            return null;
+        const near = upInstances.filter((instance) => (instance.sessions / maxSessions) * FULL_PERCENT >= thresholdPercent);
+        return instancesBreach(near, "hosts at least " + thresholdPercent + "% of its session limit", "host at least " + thresholdPercent + "% of their session limit");
+    }
+    /**
+     * newAdminAddress: a new second-factor sign-in from an address the account did not use in the lookback period. An
+     * account's first sign-in never counts.
+     * @param context The evaluation context.
+     * @returns The breach, or null.
+     */
+    async newAdminAddressBreach(context) {
+        const signIns = context.newAuditRecords.filter((record) => record.action === SECOND_FACTOR_ACTION && record.result.ok && record.target != null);
+        if (signIns.length === 0)
+            return null;
+        const history = await this.services.store.readRecords("audit", { limit: RECORD_QUERY_MAX_LIMIT, newestFirst: true });
+        const lookbackStart = context.now - NEW_ADMIN_ADDRESS_LOOKBACK_DAYS * MILLISECONDS_PER_DAY;
+        for (const signIn of signIns) {
+            const earlier = history.map((stored) => stored.value).filter((record) => record.sequence < signIn.sequence && record.action === SECOND_FACTOR_ACTION && record.result.ok && record.target === signIn.target && Date.parse(record.at) >= lookbackStart);
+            const isNewAddress = earlier.length > 0 && !earlier.some((record) => record.clientAddress === signIn.clientAddress);
+            if (isNewAddress)
+                return { summary: "Admin " + signIn.target + " signed in from a new address " + (signIn.clientAddress ?? "unknown"), processId: signIn.processId };
+        }
+        return null;
+    }
+    /**
+     * Audit records appended since the last evaluation. The first evaluation only notes where the stream ends.
+     * @returns The new records, oldest first.
+     */
+    async readNewAuditRecords() {
+        if (this.lastAuditSequence == null) {
+            const newest = await this.services.store.readRecords("audit", { limit: 1, newestFirst: true });
+            this.lastAuditSequence = newest[0]?.sequence ?? 0;
+            return [];
+        }
+        const records = await this.services.store.readRecords("audit", { afterSequence: this.lastAuditSequence, limit: RECORD_QUERY_MAX_LIMIT, newestFirst: false });
+        if (records.length > 0)
+            this.lastAuditSequence = records[records.length - 1].sequence;
+        return records.map((record) => record.value);
+    }
+    /**
+     * The fleet's dedicated tick overruns per minute between each live instance's last two metrics samples.
+     * @param upInstances The live instances.
+     * @returns The rate.
+     */
+    async dedicatedOverrunsPerMinute(upInstances) {
+        let total = 0;
+        for (const instance of upInstances) {
+            const samples = await this.lastTwoSamples(instance);
+            if (samples.length < 2)
+                continue;
+            const [older, newer] = samples;
+            const seconds = (Date.parse(newer.at) - Date.parse(older.at)) / MILLISECONDS_PER_SECOND;
+            const growth = newer.dedicatedTickOverruns - older.dedicatedTickOverruns;
+            if (seconds > 0 && growth > 0)
+                total += (growth / seconds) * SECONDS_PER_MINUTE;
+        }
+        return total;
+    }
+    /**
+     * An instance's last two metrics samples: from this instance's ring, or from the store stream with Redis.
+     * @param instance The instance.
+     * @returns At most two samples, oldest first.
+     */
+    async lastTwoSamples(instance) {
+        if (instance.isThisInstance)
+            return this.metricsSampler.samples(Number.POSITIVE_INFINITY).slice(-2);
+        if (this.services.store.kind !== "redis")
+            return [];
+        const stored = await this.services.store.readRecords(metricsStreamOf(instance.processId), { limit: 2, newestFirst: true });
+        return stored.map((record) => record.value).reverse();
+    }
+    /**
+     * Fires an alert: stores it, delivers it to the rule's channels, records a server event and calls the listeners.
+     * Event rules store it resolved at once (one alert per event).
+     * @param rule The rule.
+     * @param breach What breaches.
+     * @param now The evaluation time.
+     */
+    async fire(rule, breach, now) {
+        const firedAt = new Date(now).toISOString();
+        const isOneShot = EVENT_RULES.has(rule.ruleId);
+        const alert = {
+            alertId: randomUUID(),
+            ruleId: rule.ruleId,
+            severity: rule.severity,
+            summary: breach.summary,
+            processId: breach.processId,
+            state: isOneShot ? "resolved" : "firing",
+            firedAt: firedAt,
+            resolvedAt: isOneShot ? firedAt : null,
+            acknowledgedBy: null,
+            acknowledgedAt: null,
+            deliveries: []
+        };
+        alert.deliveries = await this.deliverToRule(rule, alert, "firing");
+        await this.services.store.appendRecord("alertHistory", alert);
+        this.serverEvents.record("alert fired: " + alert.summary, alert.processId);
+        for (const listener of this.firedListeners)
+            listener(alert);
+    }
+    /**
+     * Resolves a firing alert and delivers the resolve.
+     * @param rule The rule.
+     * @param firing The firing alert.
+     */
+    async resolve(rule, firing) {
+        const resolved = Object.assign({}, firing, { state: "resolved", resolvedAt: new Date(this.services.clock.now()).toISOString() });
+        resolved.deliveries = firing.deliveries.concat(await this.deliverToRule(rule, resolved, "resolved"));
+        await this.services.store.appendRecord("alertHistory", resolved);
+        this.serverEvents.record("alert resolved: " + resolved.summary, resolved.processId);
+    }
+    /**
+     * Delivers an alert to every channel of its rule (missing channels are skipped).
+     * @param rule The rule.
+     * @param alert The alert.
+     * @param state Fired or resolved.
+     * @returns One delivery entry per channel.
+     */
+    async deliverToRule(rule, alert, state) {
+        const channels = (await this.channelRecords()).filter((channel) => rule.channelIds.includes(channel.channelId));
+        return Promise.all(channels.map(async (channel) => {
+            const outcome = await this.channels.deliver(channel, alert, state, this.services.pagerDutyUrl);
+            if (!outcome.ok)
+                getLogger().warn("an alert could not be delivered", { channel: channel.name, error: outcome.error });
+            return { channelId: channel.channelId, ok: outcome.ok, error: outcome.error, at: new Date(this.services.clock.now()).toISOString() };
+        }));
+    }
+    /**
+     * Every alert in its newest state, newest fired first.
+     * @returns The alerts.
+     */
+    async currentAlerts() {
+        const records = await this.services.store.readRecords("alertHistory", { limit: RECORD_QUERY_MAX_LIMIT, newestFirst: true });
+        const newestById = new Map();
+        for (const record of records) {
+            if (!newestById.has(record.value.alertId))
+                newestById.set(record.value.alertId, record.value);
+        }
+        return Array.from(newestById.values()).sort((left, right) => right.firedAt.localeCompare(left.firedAt));
+    }
+    /**
+     * The firing alert of each rule.
+     * @returns Rule id → firing alert.
+     */
+    async firingAlertsByRule() {
+        const firing = new Map();
+        for (const alert of await this.currentAlerts()) {
+            if (alert.state === "firing" && !firing.has(alert.ruleId))
+                firing.set(alert.ruleId, alert);
+        }
+        return firing;
+    }
+    /**
+     * Every rule's settings, stored or default.
+     * @returns The records, in the table's order.
+     */
+    async ruleRecords() {
+        const stored = await this.storedRules();
+        return ALERT_RULE_IDS.map((ruleId) => stored.get(ruleId)?.value ?? defaultRuleRecord(ruleId));
+    }
+    /**
+     * The stored rule documents.
+     * @returns Rule id → document.
+     */
+    async storedRules() {
+        const documents = await this.services.store.listDocuments("alertRules");
+        return new Map(documents.map((document) => [document.id, document]));
+    }
+    /**
+     * The stored channels.
+     * @returns The channels.
+     */
+    async channelRecords() {
+        const documents = await this.services.store.listDocuments("alertChannels");
+        return documents.map((document) => document.value);
+    }
+    /**
+     * A known rule's stored document.
+     * @param ruleId The rule.
+     * @returns The document, or null while the rule has its defaults.
+     * @throws AdminRequestError 404 for an unknown rule.
+     */
+    async requireRule(ruleId) {
+        if (!Object.hasOwn(ALERT_RULE_DEFINITIONS, ruleId))
+            throw new AdminRequestError(HTTP_NOT_FOUND, "no such alert rule");
+        return this.services.store.getDocument("alertRules", ruleId);
+    }
+    /**
+     * Checks that every channel id exists.
+     * @param channelIds The channel ids.
+     * @throws AdminRequestError 400 naming the first unknown channel.
+     */
+    async requireChannels(channelIds) {
+        const known = new Set((await this.channelRecords()).map((channel) => channel.channelId));
+        const unknown = channelIds.find((channelId) => !known.has(channelId));
+        if (unknown != null)
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "no such alert channel: " + unknown);
+    }
+}
+/**
+ * A rule's default settings.
+ * @param ruleId The rule.
+ * @returns The record.
+ */
+function defaultRuleRecord(ruleId) {
+    const definition = ALERT_RULE_DEFINITIONS[ruleId];
+    return { ruleId: ruleId, enabled: true, threshold: definition.threshold, forMinutes: definition.forMinutes, severity: definition.severity, channelIds: [], silencedUntil: null };
+}
+/**
+ * A rule as the console sees it.
+ * @param ruleId The rule.
+ * @param stored Its stored document, or null for the defaults (revision 0).
+ * @returns The view.
+ */
+function ruleView(ruleId, stored) {
+    const record = stored?.value ?? defaultRuleRecord(ruleId);
+    const definition = ALERT_RULE_DEFINITIONS[ruleId];
+    return {
+        ruleId: ruleId,
+        label: definition.label,
+        enabled: record.enabled,
+        threshold: record.threshold,
+        thresholdUnit: definition.thresholdUnit,
+        forMinutes: record.forMinutes,
+        severity: record.severity,
+        channelIds: record.channelIds.slice(),
+        silencedUntil: record.silencedUntil,
+        revision: stored?.revision ?? 0
+    };
+}
+/**
+ * A breach naming the instances it concerns.
+ * @param instances The breaching instances.
+ * @param singularText What one instance does ("is lost").
+ * @param pluralText What several do ("are lost").
+ * @returns The breach, or null when no instance breaches.
+ */
+function instancesBreach(instances, singularText, pluralText) {
+    if (instances.length === 0)
+        return null;
+    if (instances.length === 1)
+        return { summary: "Instance " + instances[0].processId + " " + singularText, processId: instances[0].processId };
+    return { summary: instances.length + " instances " + pluralText, processId: null };
+}
+/**
+ * A breach of a fleet rate above its threshold.
+ * @param rate The rate.
+ * @param threshold The threshold.
+ * @param unit What the rate counts.
+ * @returns The breach, or null at or below the threshold.
+ */
+function rateBreach(rate, threshold, unit) {
+    if (rate <= threshold)
+        return null;
+    return { summary: Math.round(rate) + " " + unit + " (threshold " + threshold + ")", processId: null };
+}
+/**
+ * The fleet's per-minute rate of a heartbeat counter between each live instance's last two heartbeats.
+ * @param records The instances with their earlier heartbeats.
+ * @param read Reads the counter.
+ * @returns The summed rate, never negative per instance.
+ */
+function sumRate(records, read) {
+    let total = 0;
+    for (const record of records) {
+        if (record.view.status !== "up" || record.previous == null)
+            continue;
+        const seconds = (Date.parse(record.view.sentAt) - Date.parse(record.previous.sentAt)) / MILLISECONDS_PER_SECOND;
+        const growth = read(record.view) - read(record.previous);
+        if (seconds > 0 && growth > 0)
+            total += (growth / seconds) * SECONDS_PER_MINUTE;
+    }
+    return total;
+}
+/**
+ * versionMismatch: the live instances run more than one server or protocol version.
+ * @param upInstances The live instances.
+ * @returns The breach, or null.
+ */
+function versionBreach(upInstances) {
+    const versions = new Set(upInstances.map((instance) => instance.serverVersion + " (protocol " + instance.protocolVersion + ")"));
+    if (versions.size <= 1)
+        return null;
+    return { summary: "Instances run different versions: " + Array.from(versions).sort().join(", "), processId: null };
+}
+/**
+ * accountLockout: a new auth.lockout audit record.
+ * @param newAuditRecords The audit records since the last evaluation.
+ * @returns The breach, or null.
+ */
+function lockoutBreach(newAuditRecords) {
+    const lockouts = newAuditRecords.filter((record) => record.action === LOCKOUT_ACTION);
+    if (lockouts.length === 0)
+        return null;
+    const names = Array.from(new Set(lockouts.map((record) => record.target ?? "unknown")));
+    return { summary: "Admin account locked after failed sign-ins: " + names.join(", "), processId: lockouts[0].processId };
+}
+/**
+ * Checks a channel request.
+ * @param request The request.
+ * @throws AdminRequestError 400 for an unknown kind, a missing name, a missing or non-http(s) URL or routing key.
+ */
+function validateChannelRequest(request) {
+    if (!ALERT_CHANNEL_KINDS.has(request.kind))
+        throw new AdminRequestError(HTTP_BAD_REQUEST, "kind must be webhook, slack, discord, teams or pagerduty");
+    if (request.name.trim() === "")
+        throw new AdminRequestError(HTTP_BAD_REQUEST, "name must be text");
+    if (request.kind === "pagerduty") {
+        if (request.routingKey == null || request.routingKey === "")
+            throw new AdminRequestError(HTTP_BAD_REQUEST, "a PagerDuty channel needs a routingKey");
+        return;
+    }
+    if (request.url == null || !/^https?:\/\//.test(request.url))
+        throw new AdminRequestError(HTTP_BAD_REQUEST, "url must be an http:// or https:// URL");
+}