@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/services/alert.service.js

@@ -1,20 +1,46 @@
 /**
- * Alert Service — Rule-based alerting engine
+ * Alert Service — rule-based alerting.
  *
- * Evaluates alert rules at configurable intervals and fires/resolves alerts.
- * Supports three rule types:
- *   - metric: threshold-based (e.g., CPU > 80%)
- *   - log: pattern matching (e.g., level=error AND app=main count > 10/5min)
- *   - health: health status changes (e.g., app goes unhealthy)
+ * The daemon scheduler's `alert-evaluation` job calls `evaluate()` every
+ * `monitoring.healthCheck.interval` (15 s by default). A rule is an
+ * expression of `shared/alert-expression.ts` — an app's status, cpu or
+ * memory, a container's health, or the space free on this host's disk — and
+ * what it watches (`health` or `metric`) is read off it.
  *
- * Alert lifecycle: pending → firing → resolved (or silenced/acknowledged)
+ * Lifecycle: a condition that holds becomes PENDING for the rule's
+ * `forDuration` (kept in this process: a restart starts the wait again), then
+ * FIRING, then RESOLVED when it stops holding. Acknowledging a firing alert
+ * records who and when; it stays firing until the condition clears.
  *
- * Stores alert rules and events in omnitron-pg.
+ * Rules and events are stored in omnitron-pg; the defaults a fresh master
+ * starts with are seeded once by migrations 011 and 012, and are the
+ * operator's after.
  */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { sql } from 'kysely';
+import { Injectable, Inject, Optional } from '@omnitron-dev/titan/decorators';
+import { LOGGER_SERVICE_TOKEN } from '@omnitron-dev/titan/module/logger';
+import { ALERT_EXPRESSION_FORMS, ALERT_SEVERITIES, alertRuleTypeOf, isAlertExpressionParseable, } from '../shared/alert-expression.js';
+export { ALERT_EXPRESSION_FORMS, isAlertExpressionParseable };
+import { OMNITRON_DB_TOKEN, ORCHESTRATOR_TOKEN, PROJECT_SERVICE_TOKEN } from '../shared/tokens.js';
+import { diskAtHome } from '../monitoring/host-disk.js';
+import { notifyDesktop } from '../monitoring/desktop-notifier.js';
+import { callExposedService } from '../orchestrator/call-exposed-service.js';
 function evaluateExpression(expr, ctx) {
     const trimmed = expr.trim();
     // app.<name>.status != online
-    const appStatusMatch = trimmed.match(/^app\.(\*|[\w-]+)\.status\s*(!=|==)\s*(\w+)$/);
+    const appStatusMatch = trimmed.match(ALERT_EXPRESSION_FORMS[0]);
     if (appStatusMatch) {
         const [, appName, op, expected] = appStatusMatch;
         const targets = appName === '*' ? ctx.apps : ctx.apps.filter((a) => a.name === appName);
@@ -25,7 +51,7 @@ function evaluateExpression(expr, ctx) {
         };
     }
     // app.<name>.cpu > N
-    const appMetricMatch = trimmed.match(/^app\.(\*|[\w-]+)\.(cpu|memory)\s*(>|<|>=|<=)\s*(\d+)$/);
+    const appMetricMatch = trimmed.match(ALERT_EXPRESSION_FORMS[1]);
     if (appMetricMatch) {
         const [, appName, metric, op, thresholdStr] = appMetricMatch;
         const threshold = Number(thresholdStr);
@@ -46,7 +72,7 @@ function evaluateExpression(expr, ctx) {
         };
     }
     // infra.<service>.health != healthy
-    const infraMatch = trimmed.match(/^infra\.(\*|[\w-]+)\.health\s*(!=|==)\s*(\w+)$/);
+    const infraMatch = trimmed.match(ALERT_EXPRESSION_FORMS[2]);
     if (infraMatch) {
         const [, svcName = '*', op = '!=', expected = 'healthy'] = infraMatch;
         const entries = svcName === '*'
@@ -58,44 +84,112 @@ function evaluateExpression(expr, ctx) {
             value: violations.map(([k, v]) => `${k}=${v.health}`).join(', '),
         };
     }
-    // Fallback: always false for unparseable expressions
-    return { firing: false, value: 'unparseable expression' };
+    // host.disk.free < N — bytes free on this host's disk.
+    const diskMatch = trimmed.match(ALERT_EXPRESSION_FORMS[3]);
+    if (diskMatch) {
+        const [, op, thresholdStr] = diskMatch;
+        const threshold = Number(thresholdStr);
+        const free = ctx.diskFree ?? null;
+        if (free === null)
+            return { firing: false, value: 'disk unreadable', unreadable: true };
+        const holds = op === '<'
+            ? free < threshold
+            : op === '<='
+                ? free <= threshold
+                : op === '>'
+                    ? free > threshold
+                    : free >= threshold;
+        return { firing: holds, value: `${(free / 1024 ** 3).toFixed(1)} GiB free` };
+    }
+    // Nothing matched. `firing: false` is what every caller acts on, and it is
+    // the same answer a healthy platform gives — so a rule the evaluator
+    // cannot read used to sit in the UI enabled, green, and permanently
+    // silent. The rule is returned as unparseable and the caller decides;
+    // `firing` stays false because firing on a syntax error would page
+    // somebody about the wrong thing.
+    return { firing: false, value: 'unparseable expression', unparseable: true };
+}
+/** How long a critical alert is still worth delivering: older, the sink never hears of it. */
+const DELIVERY_WINDOW_MS = 24 * 60 * 60 * 1000;
+/** How long a project's alert sink is taken from its config before it is read again. */
+const SINK_TTL_MS = 60_000;
+/** An undelivered critical alert on a daemon that runs no project naming a sink. */
+export const NO_SINK = 'no alert sink: no project this daemon runs names one (monitoring.alertSink)';
+/** A rule's summary, from its annotations as the database hands them back — a jsonb object or its text. */
+function summaryOf(annotations) {
+    let value = annotations;
+    if (typeof value === 'string') {
+        try {
+            value = JSON.parse(value);
+        }
+        catch {
+            return null;
+        }
+    }
+    const summary = value && typeof value === 'object' ? value.summary : null;
+    return typeof summary === 'string' && summary.length > 0 ? summary : null;
+}
+/**
+ * One word for a container's health, as an `infra.<name>.health` rule reads
+ * it: `healthy`, `starting`, `unhealthy` or `stopped`.
+ *
+ * A container without a healthcheck reports `none`; running is then the only
+ * sign there is, and it reads `healthy` — otherwise `infra.*.health !=
+ * healthy` would fire for ever on tor and every other image that declares no
+ * check. One that is running but detached from every network serves nothing
+ * and reads `unhealthy`.
+ */
+export function containerHealth(state) {
+    if (state.status !== 'running')
+        return 'stopped';
+    if (state.networkAttached === false)
+        return 'unhealthy';
+    if (state.health === 'unhealthy')
+        return 'unhealthy';
+    if (state.health === 'starting')
+        return 'starting';
+    return 'healthy';
 }
 // =============================================================================
 // Service
 // =============================================================================
-export class AlertService {
+let AlertService = class AlertService {
     db;
     orchestrator;
-    infraState;
-    evaluationTimer = null;
-    constructor(db, orchestrator, infraState) {
-        this.db = db;
-        this.orchestrator = orchestrator;
-        this.infraState = infraState;
-    }
+    projects;
+    logger;
     /**
-     * Start the alert evaluation loop.
-     * Runs every `intervalMs` (default 15s).
+     * When each rule's condition began to hold, for a rule that must hold for
+     * `forDuration` before it fires. In memory: a daemon restart starts every
+     * wait again, which errs toward firing later rather than at once.
      */
-    start(intervalMs = 15_000) {
-        if (this.evaluationTimer)
-            return;
-        this.evaluationTimer = setInterval(() => {
-            this.evaluate().catch(() => { });
-        }, intervalMs);
-        this.evaluationTimer.unref();
+    pendingSince = new Map();
+    constructor(loggerModule, db, orchestrator, 
+    // The containers are the stacks', and the stacks are the project
+    // service's. This was an accessor token wired to `() => ({})`: every
+    // `infra.…` rule — which the console's form accepts as supported — was
+    // evaluated against no containers at all and could never fire.
+    projects) {
+        this.db = db;
+        this.orchestrator = orchestrator;
+        this.projects = projects;
+        this.logger = loggerModule.logger;
     }
-    stop() {
-        if (this.evaluationTimer) {
-            clearInterval(this.evaluationTimer);
-            this.evaluationTimer = null;
+    /** Every container of every stack on this machine, by container name. */
+    infraState() {
+        const out = {};
+        for (const { infra } of this.projects?.getInfraManager?.()?.listInstances() ?? []) {
+            for (const [name, state] of Object.entries(infra.getState().services)) {
+                out[name] = { status: state.status, health: containerHealth(state) };
+            }
         }
+        return out;
     }
     /**
-     * Evaluate all enabled alert rules.
+     * Evaluate all enabled alert rules — once per scheduler tick. `now` is a
+     * parameter for the court that walks a rule through its wait.
      */
-    async evaluate() {
+    async evaluate(now = Date.now()) {
         const rules = await this.db
             .selectFrom('alert_rules')
             .selectAll()
@@ -109,9 +203,26 @@ export class AlertService {
         const ctx = {
             apps: apps.map((a) => ({ name: a.name, status: a.status, cpu: a.cpu, memory: a.memory })),
             infra,
+            // Read only when a rule asks: one statfs a tick, for the rules that watch it.
+            diskFree: rules.some((r) => ALERT_EXPRESSION_FORMS[3].test(r.expression.trim()))
+                ? ((await diskAtHome())?.free ?? null)
+                : null,
         };
         for (const rule of rules) {
-            const { firing, value } = evaluateExpression(rule.expression, ctx);
+            const { firing, value, unparseable, unreadable } = evaluateExpression(rule.expression, ctx);
+            if (unparseable) {
+                // Said once per cycle per rule rather than swallowed. An operator
+                // wrote this rule to catch something; the platform is not catching
+                // it, and no other surface would ever say so.
+                this.logger.warn({ ruleId: rule.id, ruleName: rule.name, expression: rule.expression }, 'Alert rule expression cannot be evaluated — this rule will never fire');
+            }
+            if (unreadable) {
+                // The reading failed, not the condition: «unknown» is neither «full»
+                // nor «fine». Resolving here would close an alert on a full disk as
+                // all clear, so the rule — its wait, its event — is left as it was.
+                this.logger.warn({ ruleId: rule.id, ruleName: rule.name, expression: rule.expression }, 'Alert rule could not read what it watches — left as it was');
+                continue;
+            }
             // Get current firing alert for this rule (if any)
             const currentAlert = await this.db
                 .selectFrom('alert_events')
@@ -119,15 +230,33 @@ export class AlertService {
                 .where('ruleId', '=', rule.id)
                 .where('status', '=', 'firing')
                 .executeTakeFirst();
+            if (!firing)
+                this.pendingSince.delete(rule.id);
             if (firing && !currentAlert) {
-                // New alert — create firing event
+                // Held long enough? A rule with a wait fires only once its condition
+                // has held for all of it, so a spike shorter than the wait is not paged.
+                const waitMs = (rule.forDuration ?? 0) * 1000;
+                if (waitMs > 0) {
+                    const since = this.pendingSince.get(rule.id) ?? now;
+                    this.pendingSince.set(rule.id, since);
+                    if (now - since < waitMs)
+                        continue;
+                }
+                this.pendingSince.delete(rule.id);
                 await this.db.insertInto('alert_events').values({
                     ruleId: rule.id,
                     status: 'firing',
                     value,
-                    annotations: rule.annotations ? JSON.stringify(rule.annotations) : null,
+                    // Postgres hands `jsonb` back as an object, the node's SQLite as the
+                    // text it holds; stringifying that text again stored a JSON string of
+                    // JSON, which `summaryOf` reads as no summary at all.
+                    annotations: rule.annotations
+                        ? (typeof rule.annotations === 'string' ? rule.annotations : JSON.stringify(rule.annotations))
+                        : null,
                     firedAt: new Date(),
                 }).execute();
+                if (rule.severity === 'critical')
+                    this.sayOnDesktop(rule, 'firing', value);
             }
             else if (!firing && currentAlert) {
                 // Alert resolved
@@ -136,6 +265,8 @@ export class AlertService {
                     .set({ status: 'resolved', resolvedAt: new Date() })
                     .where('id', '=', currentAlert.id)
                     .execute();
+                if (rule.severity === 'critical')
+                    this.sayOnDesktop(rule, 'resolved', null);
             }
             // Update last evaluated timestamp
             await this.db
@@ -144,6 +275,153 @@ export class AlertService {
                 .where('id', '=', rule.id)
                 .execute();
         }
+        // After the rules, and never in their way: a sink that cannot be reached
+        // this tick is tried again on the next.
+        try {
+            await this.deliverPending(now);
+        }
+        catch (err) {
+            this.logger.warn({ err }, 'Alert sink delivery failed this tick — tried again on the next');
+        }
+    }
+    /** Each project's alert sink, read from its config at most once a minute. */
+    sinkCache = null;
+    async alertSinks(now) {
+        if (this.sinkCache && now - this.sinkCache.at < SINK_TTL_MS)
+            return this.sinkCache.sinks;
+        const sinks = [];
+        for (const project of this.projects?.listProjects?.() ?? []) {
+            try {
+                const config = await this.projects.loadProjectConfig(project.name);
+                const sink = config.monitoring?.alertSink;
+                if (sink)
+                    sinks.push({ project: project.name, ...sink });
+            }
+            catch (err) {
+                this.logger.warn({ err, project: project.name }, 'A project config could not be read for its alert sink');
+            }
+        }
+        this.sinkCache = { at: now, sinks };
+        return sinks;
+    }
+    /**
+     * The critical alerts the platform's alert sink has not taken yet, handed to
+     * it — every tick until it takes them (migration 013). The sink is an app of
+     * each stack of the sink's project this daemon runs (`daos/dev/main` on the
+     * master, `daos/test/main` on its node): a daemon delivers its own alerts to
+     * the stacks it runs, and a node's never wait for the master to be awake.
+     * The call takes `exec`'s road — the supervisor's socket into the process —
+     * and only into an app in bootstrap mode.
+     */
+    async deliverPending(now) {
+        const sinks = await this.alertSinks(now);
+        const targets = sinks.flatMap((sink) => this.sinkTargets(sink).map((name) => ({ sink, name })));
+        if (targets.length === 0) {
+            // Said on the event, once, rather than returned from in silence. The test
+            // node ran without a sink for a release: its alerts fired, resolved, and
+            // kept `deliveryError` empty — nothing anywhere said they went nowhere.
+            await this.db
+                .updateTable('alert_events')
+                .set({ deliveryError: NO_SINK })
+                .where('deliveredAt', 'is', null)
+                .where('deliveryError', 'is', null)
+                .where('firedAt', '>', new Date(now - DELIVERY_WINDOW_MS))
+                .where('ruleId', 'in', (eb) => eb.selectFrom('alert_rules').select('id').where('severity', '=', 'critical'))
+                .execute();
+            return;
+        }
+        const pending = await this.db
+            .selectFrom('alert_events')
+            .innerJoin('alert_rules', 'alert_rules.id', 'alert_events.ruleId')
+            .select([
+            'alert_events.id as id',
+            'alert_events.status as status',
+            'alert_events.value as value',
+            'alert_events.deliveredAt as deliveredAt',
+            'alert_events.resolveDeliveredAt as resolveDeliveredAt',
+            'alert_rules.name as ruleName',
+            'alert_rules.annotations as annotations',
+        ])
+            .where('alert_rules.severity', '=', 'critical')
+            .where('alert_events.firedAt', '>', new Date(now - DELIVERY_WINDOW_MS))
+            .where((eb) => eb.or([
+            eb('alert_events.deliveredAt', 'is', null),
+            eb.and([eb('alert_events.status', '=', 'resolved'), eb('alert_events.resolveDeliveredAt', 'is', null)]),
+        ]))
+            .orderBy('alert_events.firedAt', 'asc')
+            .limit(20)
+            .execute();
+        for (const event of pending) {
+            // The firing before its end: a resolve nobody saw fire says nothing.
+            const transitions = [];
+            if (!event.deliveredAt)
+                transitions.push('firing');
+            if (event.status === 'resolved' && !event.resolveDeliveredAt)
+                transitions.push('resolved');
+            for (const status of transitions) {
+                const failure = await this.deliverTo(targets, event, status);
+                await this.db
+                    .updateTable('alert_events')
+                    .set((failure
+                    ? { deliveryError: failure }
+                    : {
+                        [status === 'firing' ? 'deliveredAt' : 'resolveDeliveredAt']: new Date(now),
+                        deliveryError: null,
+                    }))
+                    .where('id', '=', event.id)
+                    .execute();
+                if (failure)
+                    break;
+            }
+        }
+    }
+    /** The apps a sink names on the stacks of its project this daemon runs. */
+    sinkTargets(sink) {
+        return this.orchestrator
+            .list()
+            .map((app) => app.name)
+            .filter((name) => {
+            const [project, stack, app, ...rest] = name.split('/');
+            return rest.length === 0 && Boolean(stack) && project === sink.project && app === sink.app;
+        });
+    }
+    /** One transition to every target: the first failure, or null when every one took it. */
+    async deliverTo(targets, event, status) {
+        for (const { sink, name } of targets) {
+            const handle = this.orchestrator.getHandle(name);
+            if (!handle || handle.status !== 'online')
+                return `${name} is not online`;
+            if (handle.mode !== 'bootstrap' || !handle.supervisor)
+                return `${name} is not a bootstrap app — its alert sink is refused`;
+            const [project, stack] = name.split('/');
+            const host = sink.host ?? `${project}/${stack}`;
+            try {
+                await callExposedService(handle.supervisor, name, sink.service, sink.method, [
+                    {
+                        eventId: event.id,
+                        status,
+                        ruleName: event.ruleName,
+                        value: event.value ?? '',
+                        summary: summaryOf(event.annotations),
+                        host,
+                    },
+                ]);
+            }
+            catch (err) {
+                return `${name}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 500);
+            }
+        }
+        return null;
+    }
+    /**
+     * A critical alert's fire and its end, on the master's desktop — at the
+     * transition, once, never per tick (monitoring/desktop-notifier.ts).
+     */
+    sayOnDesktop(rule, what, value) {
+        const summary = summaryOf(rule.annotations);
+        notifyDesktop(what === 'firing'
+            ? { title: 'omnitron — critical', subtitle: rule.name, body: [value, summary].filter(Boolean).join(' · ') || rule.name }
+            : { title: 'omnitron — resolved', subtitle: rule.name, body: 'The condition no longer holds.' }, (err) => this.logger.warn({ err, rule: rule.name }, 'A critical alert could not be said on the desktop'));
     }
     // ===========================================================================
     // CRUD
@@ -152,32 +430,40 @@ export class AlertService {
         const rows = await this.db.selectFrom('alert_rules').selectAll().orderBy('createdAt', 'desc').execute();
         return rows.map(mapRule);
     }
+    /** A rule from fields `readAlertRuleFields` checked; what it watches is read off its expression. */
     async createRule(rule) {
         const row = await this.db.insertInto('alert_rules').values({
             name: rule.name,
             expression: rule.expression,
-            type: rule.type,
+            type: alertRuleTypeOf(rule.expression),
             severity: rule.severity,
-            forDuration: rule.forDuration ?? null,
-            annotations: rule.annotations ? JSON.stringify(rule.annotations) : null,
-            labels: rule.labels ? JSON.stringify(rule.labels) : null,
+            forDuration: rule.forDuration,
+            annotations: rule.summary ? JSON.stringify({ summary: rule.summary }) : null,
             enabled: rule.enabled,
         }).returningAll().executeTakeFirstOrThrow();
         return mapRule(row);
     }
+    /** Whatever the update carries, checked the same way; a new expression is a new type. */
     async updateRule(id, updates) {
         const set = { updatedAt: new Date() };
         if (updates.name !== undefined)
             set['name'] = updates.name;
-        if (updates.expression !== undefined)
+        if (updates.expression !== undefined) {
             set['expression'] = updates.expression;
+            set['type'] = alertRuleTypeOf(updates.expression);
+        }
         if (updates.severity !== undefined)
             set['severity'] = updates.severity;
         if (updates.enabled !== undefined)
             set['enabled'] = updates.enabled;
         if (updates.forDuration !== undefined)
             set['forDuration'] = updates.forDuration;
+        if (updates.summary !== undefined) {
+            set['annotations'] = updates.summary ? JSON.stringify({ summary: updates.summary }) : null;
+        }
         const row = await this.db.updateTable('alert_rules').set(set).where('id', '=', id).returningAll().executeTakeFirstOrThrow();
+        // A condition that waited under the old rule proves nothing about the new one.
+        this.pendingSince.delete(id);
         return mapRule(row);
     }
     async deleteRule(id) {
@@ -192,42 +478,125 @@ export class AlertService {
         const rows = await query.execute();
         return rows.map(mapEvent);
     }
-    async acknowledgeAlert(alertId, acknowledgedBy) {
-        await this.db.updateTable('alert_events').set({
-            acknowledgedAt: new Date(),
-            acknowledgedBy,
-        }).where('id', '=', alertId).execute();
+    /**
+     * Firing and acknowledged alerts, joined with their rule.
+     *
+     * Resolved events are excluded: this answers "what needs attention now".
+     * The message prefers the rule's `summary` annotation and falls back to the
+     * expression and the observed value, so an alert always renders as
+     * something a human can act on.
+     */
+    async getActiveAlerts(limit = 100) {
+        const rows = await this.db
+            .selectFrom('alert_events')
+            .innerJoin('alert_rules', 'alert_rules.id', 'alert_events.ruleId')
+            .select([
+            'alert_events.id as id',
+            'alert_events.ruleId as ruleId',
+            'alert_events.status as status',
+            'alert_events.value as value',
+            'alert_events.annotations as eventAnnotations',
+            'alert_events.firedAt as firedAt',
+            'alert_events.resolvedAt as resolvedAt',
+            'alert_events.acknowledgedAt as acknowledgedAt',
+            'alert_rules.name as ruleName',
+            'alert_rules.severity as severity',
+            'alert_rules.expression as expression',
+            'alert_rules.annotations as ruleAnnotations',
+        ])
+            .where('alert_events.status', '=', 'firing')
+            .orderBy('alert_events.firedAt', 'desc')
+            .limit(limit)
+            .execute();
+        return rows.map((row) => {
+            // Merged, not chosen.
+            //
+            // This was `eventAnnotations ?? ruleAnnotations`, so an event with ANY
+            // annotations of its own lost every annotation its rule carried —
+            // including `summary`, which is the sentence this list shows. An event
+            // annotated `{node: …}` and nothing else would have rendered as the
+            // raw expression, which is exactly what a replicated alert now looks
+            // like: the master stamps the node it came from onto every one.
+            //
+            // The event's own values win where both have a key: the rule's
+            // annotations are the template and the event's are what happened.
+            const annotations = {
+                ...(row.ruleAnnotations ?? {}),
+                ...(row.eventAnnotations ?? {}),
+            };
+            const summary = typeof annotations['summary'] === 'string' ? annotations['summary'] : null;
+            const node = typeof annotations['node'] === 'string' ? annotations['node'] : null;
+            const value = row.value !== null && row.value !== undefined ? String(row.value) : null;
+            return {
+                id: String(row.id),
+                ruleId: String(row.ruleId),
+                ruleName: String(row.ruleName),
+                severity: row.severity,
+                message: summary ?? (value ? `${row.expression} (value: ${value})` : String(row.expression)),
+                firedAt: new Date(row.firedAt).toISOString(),
+                resolvedAt: row.resolvedAt ? new Date(row.resolvedAt).toISOString() : null,
+                acknowledged: row.acknowledgedAt !== null && row.acknowledgedAt !== undefined,
+                // Null for an alert this master raised about itself. Present for one
+                // replicated from a node, which is the case where "disk above 90%"
+                // is unactionable without it.
+                node,
+            };
+        });
+    }
+    /**
+     * Record who acknowledged a firing alert, and when. Only a firing one: a
+     * resolved alert has nobody left to answer. And only the first time: the
+     * moment it was taken, by whom, is the fact — a second click is not a
+     * second taking.
+     *
+     * The acknowledger is the caller the RPC read from the session — it was a
+     * field of the request, so anyone could sign an acknowledgement with
+     * another operator's name. A console session is recorded by its account's
+     * username; the CLI, admitted on the unix socket as `omnitron-local`, as
+     * that — the audit trail's convention. Looked up by text, because
+     * `omnitron-local` is no uuid and a uuid comparison would refuse the whole
+     * acknowledgement over it.
+     */
+    async acknowledgeAlert(alertId, userId) {
+        const user = await this.db
+            .selectFrom('omnitron_users')
+            .select('username')
+            .where(sql `${sql.ref('id')}::text`, '=', userId)
+            .executeTakeFirst();
+        const result = await this.db
+            .updateTable('alert_events')
+            .set({ acknowledgedAt: new Date(), acknowledgedBy: user?.username ?? userId })
+            .where('id', '=', alertId)
+            .where('status', '=', 'firing')
+            .where('acknowledgedAt', 'is', null)
+            .executeTakeFirst();
+        return Number(result.numUpdatedRows ?? 0) > 0;
     }
     async getSummary() {
-        // Count events by status using SQL GROUP BY (avoids loading all rows)
         const statusCounts = await this.db
             .selectFrom('alert_events')
             .select(['status'])
             .select((eb) => eb.fn.count('id').as('count'))
+            .select((eb) => eb.fn.count('acknowledgedAt').as('acknowledged'))
             .groupBy('status')
             .execute();
         let firing = 0;
+        let acknowledged = 0;
         let resolved = 0;
-        let silenced = 0;
         let total = 0;
         for (const row of statusCounts) {
             const count = Number(row.count);
             total += count;
-            switch (row.status) {
-                case 'firing':
-                    firing = count;
-                    break;
-                case 'resolved':
-                    resolved = count;
-                    break;
-                case 'silenced':
-                    silenced = count;
-                    break;
-                default: break;
+            if (row.status === 'firing') {
+                firing = count;
+                acknowledged = Number(row.acknowledged);
+            }
+            else if (row.status === 'resolved') {
+                resolved = count;
             }
         }
-        // Count firing alerts by severity using SQL GROUP BY + JOIN
-        const bySeverity = {};
+        // Firing alerts by severity — every severity named, zero included.
+        const bySeverity = Object.fromEntries(ALERT_SEVERITIES.map((s) => [s, 0]));
         const severityCounts = await this.db
             .selectFrom('alert_events')
             .innerJoin('alert_rules', 'alert_rules.id', 'alert_events.ruleId')
@@ -237,24 +606,40 @@ export class AlertService {
             .groupBy('alert_rules.severity')
             .execute();
         for (const row of severityCounts) {
-            bySeverity[row.severity] = Number(row.count);
+            if (row.severity in bySeverity)
+                bySeverity[row.severity] = Number(row.count);
         }
-        return { firing, resolved, silenced, total, bySeverity };
+        return { firing, acknowledged, resolved, total, bySeverity };
     }
-}
+};
+AlertService = __decorate([
+    Injectable(),
+    __param(0, Inject(LOGGER_SERVICE_TOKEN)),
+    __param(1, Inject(OMNITRON_DB_TOKEN)),
+    __param(2, Inject(ORCHESTRATOR_TOKEN)),
+    __param(3, Optional()),
+    __param(3, Inject(PROJECT_SERVICE_TOKEN)),
+    __metadata("design:paramtypes", [Object, Function, Function, Object])
+], AlertService);
+export { AlertService };
 // =============================================================================
 // Mappers
 // =============================================================================
 function mapRule(row) {
+    const annotations = row.annotations
+        ? typeof row.annotations === 'string'
+            ? JSON.parse(row.annotations)
+            : row.annotations
+        : null;
     return {
         id: row.id,
         name: row.name,
         expression: row.expression,
-        type: row.type,
+        // Read off the expression: a row filed as `log` by the old form watched nothing.
+        type: alertRuleTypeOf(row.expression) ?? row.type,
         severity: row.severity,
         forDuration: row.forDuration ?? null,
-        annotations: row.annotations ? (typeof row.annotations === 'string' ? JSON.parse(row.annotations) : row.annotations) : null,
-        labels: row.labels ? (typeof row.labels === 'string' ? JSON.parse(row.labels) : row.labels) : null,
+        summary: typeof annotations?.summary === 'string' ? annotations.summary : null,
         enabled: row.enabled,
         lastEvaluatedAt: row.lastEvaluatedAt ? (row.lastEvaluatedAt instanceof Date ? row.lastEvaluatedAt.toISOString() : String(row.lastEvaluatedAt)) : null,
         createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : String(row.createdAt),