@celilo/cli 0.13.3 → 0.14.1

package/src/services/alerting/store.ts

@@ -0,0 +1,289 @@
+/**
+ * Alert persistence — applying reconciler decisions to the alerts table.
+ *
+ * Kept separate from `reconcile.ts` on purpose: the decisions are pure and
+ * exhaustively tested without a database, and this layer only has to get the
+ * writes right. The one piece of judgement here is `activeKey`, which must
+ * move in lockstep with `state` or the "one live alert per key" index either
+ * rejects legitimate re-fires or permits duplicates.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { and, eq, inArray, isNotNull, lte } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type Alert, type AlertSeverity, alerts } from '../../db/schema';
+import type { LiveAlert, ReconcileAction } from './reconcile';
+
+/** Alert states that are still "live" — i.e. not resolved. */
+const LIVE_STATES = ['pending', 'firing', 'acked', 'suppressed'] as const;
+
+/** Load the alerts a monitor currently owns that have not resolved. */
+export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] {
+  return db
+    .select({ id: alerts.id, key: alerts.key })
+    .from(alerts)
+    .where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey)))
+    .all();
+}
+
+export interface ApplyContext {
+  monitorId: string;
+  escalationPolicyId: string | null;
+  now: Date;
+}
+
+/**
+ * Apply reconciler actions.
+ *
+ * Returns the ids of alerts created and resolved, so the caller can drive
+ * notification (a resolve fans out an all-clear only to routes that were
+ * actually told about the alert).
+ */
+export function applyReconcileActions(
+  db: DbClient,
+  actions: ReconcileAction[],
+  context: ApplyContext,
+): { createdIds: string[]; resolvedIds: string[] } {
+  const createdIds: string[] = [];
+  const resolvedIds: string[] = [];
+
+  for (const action of actions) {
+    if (action.type === 'create') {
+      const id = randomUUID();
+      db.insert(alerts)
+        .values({
+          id,
+          key: action.key,
+          // Mirrors `key` while live; NULLed on resolve. This is what the
+          // unique index keys off — SQLite treats NULLs as distinct, so any
+          // number of resolved rows may share a key while at most one live
+          // row may hold it.
+          activeKey: action.key,
+          monitorId: context.monitorId,
+          state: 'pending',
+          severity: action.severity,
+          firstFiredAt: context.now,
+          lastSeenAt: context.now,
+          graceUntil: action.graceUntil,
+          escalationStep: 0,
+          escalationPolicyId: context.escalationPolicyId,
+          message: action.message,
+          details: action.details,
+        })
+        .run();
+      createdIds.push(id);
+      continue;
+    }
+
+    if (action.type === 'refresh') {
+      // Severity and message can change between runs (a warning becoming a
+      // failure, a percentage climbing); the alert's identity does not.
+      db.update(alerts)
+        .set({
+          lastSeenAt: context.now,
+          severity: action.severity,
+          message: action.message,
+          details: action.details,
+        })
+        .where(eq(alerts.id, action.alertId))
+        .run();
+      continue;
+    }
+
+    db.update(alerts)
+      .set({ state: 'resolved', activeKey: null, resolvedAt: context.now })
+      .where(eq(alerts.id, action.alertId))
+      .run();
+    resolvedIds.push(action.alertId);
+  }
+
+  return { createdIds, resolvedIds };
+}
+
+/**
+ * Record that an alert has become suppressed by an ancestor alert or a deploy
+ * window. Idempotent: re-suppressing an already-suppressed alert is a no-op
+ * rather than resetting its bookkeeping.
+ */
+export function markSuppressed(
+  db: DbClient,
+  alertId: string,
+  by: { alertId?: string; windowId?: string },
+): void {
+  db.update(alerts)
+    .set({
+      state: 'suppressed',
+      suppressedByAlertId: by.alertId ?? null,
+      suppressedByWindowId: by.windowId ?? null,
+    })
+    .where(eq(alerts.id, alertId))
+    .run();
+}
+
+/**
+ * Lift suppression.
+ *
+ * Two things happen together, and both matter:
+ *
+ *  - `awaitingConfirmation` is set, so the alert does NOT notify until a
+ *    subsequent run confirms it is still failing. When a machine comes back,
+ *    its modules almost always came back with it; paging for them would
+ *    recreate the storm suppression just prevented.
+ *  - the escalation clock is restarted from now. Running it from the original
+ *    fire time would fire every step whose delay had nominally elapsed during
+ *    the suppression, all at once.
+ */
+export function markUnsuppressed(db: DbClient, alertId: string, now: Date): void {
+  db.update(alerts)
+    .set({
+      state: 'firing',
+      suppressedByAlertId: null,
+      suppressedByWindowId: null,
+      unsuppressedAt: now,
+      awaitingConfirmation: true,
+      escalationStep: 0,
+      nextEscalationAt: now,
+    })
+    .where(eq(alerts.id, alertId))
+    .run();
+}
+
+/**
+ * Clear `awaitingConfirmation` for alerts a SUCCESSFUL run has just confirmed
+ * are still failing. Only a run that executed may clear it — that is the same
+ * asymmetry the reconciler rests on, applied to a different column.
+ */
+export function confirmStillFailing(db: DbClient, alertIds: string[], now: Date): void {
+  if (alertIds.length === 0) return;
+  db.update(alerts)
+    .set({ awaitingConfirmation: false, nextEscalationAt: now })
+    .where(inArray(alerts.id, alertIds))
+    .run();
+}
+
+/**
+ * Promote `pending` alerts whose grace window has elapsed to `firing`.
+ *
+ * Without this an alert never leaves `pending`, because nothing else writes
+ * that transition — `reconcile` only creates and resolves. Escalation would
+ * still behave correctly (it compares `graceUntil` directly), but every alert
+ * would read as `pending` forever in `celilo alerts` and in the health column,
+ * which is a lie about the system's state even though nothing downstream acts
+ * on it.
+ *
+ * Suppressed alerts are left alone: they are already accounted for, and moving
+ * them to `firing` would lose the record of what explains them.
+ */
+export function promoteReadyAlerts(db: DbClient, now: Date): number {
+  const ready = db
+    .select({ id: alerts.id })
+    .from(alerts)
+    .where(and(eq(alerts.state, 'pending'), lte(alerts.graceUntil, now)))
+    .all();
+
+  if (ready.length === 0) return 0;
+  db.update(alerts)
+    .set({ state: 'firing' })
+    .where(
+      inArray(
+        alerts.id,
+        ready.map((r) => r.id),
+      ),
+    )
+    .run();
+  return ready.length;
+}
+
+/**
+ * Record that an escalation step was taken.
+ *
+ * Called for a delivered message AND for one deferred by quiet hours, because
+ * the step is what advances, not the message (D13). Without this the sweep
+ * would re-decide the same step every five minutes and never reach the
+ * secondary.
+ */
+export function recordStepTaken(
+  db: DbClient,
+  alertId: string,
+  taken: { stepIndex: number; nextStepDueAt: Date | null },
+): void {
+  db.update(alerts)
+    .set({ escalationStep: taken.stepIndex + 1, nextEscalationAt: taken.nextStepDueAt })
+    .where(eq(alerts.id, alertId))
+    .run();
+}
+
+/**
+ * Hold a message until someone's quiet hours end.
+ *
+ * Keeps the EARLIEST pending deferral rather than the latest: if a second step
+ * defers behind a longer window, the operator still hears about it as soon as
+ * anyone is reachable.
+ */
+export function deferNotification(
+  db: DbClient,
+  alertId: string,
+  deferral: { routeId: string; until: Date },
+): void {
+  const current = db
+    .select({ until: alerts.deferredUntil })
+    .from(alerts)
+    .where(eq(alerts.id, alertId))
+    .get();
+  if (current?.until && current.until <= deferral.until) return;
+
+  db.update(alerts)
+    .set({ deferredUntil: deferral.until, deferredRouteId: deferral.routeId })
+    .where(eq(alerts.id, alertId))
+    .run();
+}
+
+/** Alerts whose deferral window has ended and are owed a message. */
+export function dueDeferrals(db: DbClient, now: Date): Alert[] {
+  return db
+    .select()
+    .from(alerts)
+    .where(and(inArray(alerts.state, [...LIVE_STATES]), lte(alerts.deferredUntil, now)))
+    .all();
+}
+
+export function clearDeferral(db: DbClient, alertId: string): void {
+  db.update(alerts)
+    .set({ deferredUntil: null, deferredRouteId: null })
+    .where(eq(alerts.id, alertId))
+    .run();
+}
+
+/** Every live alert, for the notify sweep and the health column. */
+export function loadAllLiveAlerts(db: DbClient): Alert[] {
+  return db
+    .select()
+    .from(alerts)
+    .where(inArray(alerts.state, [...LIVE_STATES]))
+    .all();
+}
+
+export interface AlertSummary {
+  key: string;
+  severity: AlertSeverity;
+  suppressed: boolean;
+}
+
+/** Live alerts grouped by module id, for the `celilo module list` column. */
+export function summariseByModule(liveAlerts: Alert[]): Map<string, AlertSummary[]> {
+  const byModule = new Map<string, AlertSummary[]>();
+  for (const alert of liveAlerts) {
+    if (!alert.key.startsWith('module:')) continue;
+    const rest = alert.key.slice('module:'.length);
+    const moduleId = rest.split('/')[0];
+    if (!moduleId) continue;
+    const list = byModule.get(moduleId) ?? [];
+    list.push({
+      key: alert.key,
+      severity: alert.severity,
+      suppressed: alert.state === 'suppressed',
+    });
+    byModule.set(moduleId, list);
+  }
+  return byModule;
+}

package/src/services/alerting/suppression.test.ts

@@ -0,0 +1,228 @@
+import { describe, expect, test } from 'bun:test';
+import { moduleAlertKey, moduleCheckAlertKey } from './keys';
+import {
+  type SuppressionTopology,
+  ancestorKeysFor,
+  findSuppressor,
+  machineAlertKey,
+} from './suppression';
+
+/**
+ * Six modules on machine `iot`, plus a resolver providing dns_internal for the
+ * `internal` zone, plus caddy off in the dmz on a container.
+ */
+const TOPOLOGY: SuppressionTopology = {
+  moduleSystems: [
+    ...['homebridge', 'module-b', 'module-c', 'module-d', 'module-e', 'module-f'].map(
+      (moduleId) => ({
+        moduleId,
+        hostname: 'iot',
+        zone: 'internal',
+        infraType: 'machine' as const,
+      }),
+    ),
+    { moduleId: 'technitium', hostname: 'dns', zone: 'internal', infraType: 'machine' as const },
+    {
+      moduleId: 'caddy',
+      hostname: 'caddy-1',
+      zone: 'dmz',
+      infraType: 'container_service' as const,
+    },
+  ],
+  zoneProviders: [{ moduleId: 'technitium', capabilityName: 'dns_internal', zones: ['internal'] }],
+};
+
+const noSuppression = {
+  suppressible: true,
+  modulesInDeployWindow: new Set<string>(),
+  topology: TOPOLOGY,
+};
+
+describe('ancestorKeysFor', () => {
+  test('a check item is owned by its module, nearest first', () => {
+    const ancestors = ancestorKeysFor(moduleCheckAlertKey('homebridge', 'port'), TOPOLOGY);
+    expect(ancestors[0]).toBe(moduleAlertKey('homebridge'));
+  });
+
+  test('a module is owned by the machine it runs on', () => {
+    expect(ancestorKeysFor(moduleAlertKey('homebridge'), TOPOLOGY)).toContain(
+      machineAlertKey('iot'),
+    );
+  });
+
+  test('a module is owned by a zone-capability provider covering its zone', () => {
+    expect(ancestorKeysFor(moduleAlertKey('homebridge'), TOPOLOGY)).toContain(
+      moduleAlertKey('technitium'),
+    );
+  });
+
+  // Without this the resolver, whose own check fails BECAUSE dns is down,
+  // silences its own alert — and the one thing naming the cause vanishes.
+  test('a zone provider is not its own ancestor', () => {
+    expect(ancestorKeysFor(moduleAlertKey('technitium'), TOPOLOGY)).not.toContain(
+      moduleAlertKey('technitium'),
+    );
+  });
+
+  test('a module in another zone is unaffected by that zone provider', () => {
+    expect(ancestorKeysFor(moduleAlertKey('caddy'), TOPOLOGY)).not.toContain(
+      moduleAlertKey('technitium'),
+    );
+  });
+
+  // machines_reachable probes the machine POOL, so a container has no
+  // corresponding alert. A real gap, documented rather than papered over.
+  test('a container-hosted module gets no machine ancestor', () => {
+    const ancestors = ancestorKeysFor(moduleAlertKey('caddy'), TOPOLOGY);
+    expect(ancestors.filter((k) => k.startsWith('builtin:machines_reachable'))).toEqual([]);
+  });
+
+  test('a built-in alert is a root cause with no ancestors', () => {
+    expect(ancestorKeysFor(machineAlertKey('iot'), TOPOLOGY)).toEqual([]);
+  });
+
+  test('an unparseable key yields no ancestors rather than throwing', () => {
+    expect(ancestorKeysFor('not-a-key', TOPOLOGY)).toEqual([]);
+  });
+});
+
+describe('findSuppressor — six pages become one', () => {
+  test('every module on a downed machine is suppressed', () => {
+    const firingKeys = new Set([machineAlertKey('iot')]);
+    const moduleIds = ['homebridge', 'module-b', 'module-c', 'module-d', 'module-e', 'module-f'];
+    for (const moduleId of moduleIds) {
+      expect(
+        findSuppressor({ key: moduleAlertKey(moduleId), firingKeys, ...noSuppression }),
+      ).toEqual({
+        kind: 'alert',
+        key: machineAlertKey('iot'),
+      });
+    }
+  });
+
+  test('their check items are suppressed too', () => {
+    const firingKeys = new Set([machineAlertKey('iot')]);
+    const suppressor = findSuppressor({
+      key: moduleCheckAlertKey('homebridge', 'port'),
+      firingKeys,
+      ...noSuppression,
+    });
+    expect(suppressor).toEqual({ kind: 'alert', key: machineAlertKey('iot') });
+  });
+
+  test('the machine alert itself is never suppressed — it is the page you get', () => {
+    const firingKeys = new Set([machineAlertKey('iot')]);
+    expect(
+      findSuppressor({ key: machineAlertKey('iot'), firingKeys, ...noSuppression }),
+    ).toBeNull();
+  });
+
+  test('a module on a DIFFERENT machine is not suppressed', () => {
+    const firingKeys = new Set([machineAlertKey('iot')]);
+    expect(
+      findSuppressor({ key: moduleAlertKey('caddy'), firingKeys, ...noSuppression }),
+    ).toBeNull();
+  });
+
+  test('the module-level alert suppresses its own check items', () => {
+    const firingKeys = new Set([moduleAlertKey('homebridge')]);
+    expect(
+      findSuppressor({
+        key: moduleCheckAlertKey('homebridge', 'port'),
+        firingKeys,
+        ...noSuppression,
+      }),
+    ).toEqual({ kind: 'alert', key: moduleAlertKey('homebridge') });
+  });
+
+  test('nothing firing means nothing suppressed', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('homebridge'),
+        firingKeys: new Set(),
+        ...noSuppression,
+      }),
+    ).toBeNull();
+  });
+
+  test('the nearest ancestor is reported as the cause', () => {
+    // Both the module-level alert and the machine alert are firing; the module
+    // one is the more specific explanation.
+    const firingKeys = new Set([moduleAlertKey('homebridge'), machineAlertKey('iot')]);
+    expect(
+      findSuppressor({
+        key: moduleCheckAlertKey('homebridge', 'port'),
+        firingKeys,
+        ...noSuppression,
+      }),
+    ).toEqual({ kind: 'alert', key: moduleAlertKey('homebridge') });
+  });
+});
+
+describe('findSuppressor — dead resolver', () => {
+  test('a firing zone provider suppresses modules in its zone', () => {
+    const firingKeys = new Set([moduleAlertKey('technitium')]);
+    expect(
+      findSuppressor({ key: moduleAlertKey('homebridge'), firingKeys, ...noSuppression }),
+    ).toEqual({ kind: 'alert', key: moduleAlertKey('technitium') });
+  });
+
+  test('but not itself', () => {
+    const firingKeys = new Set([moduleAlertKey('technitium')]);
+    expect(
+      findSuppressor({ key: moduleAlertKey('technitium'), firingKeys, ...noSuppression }),
+    ).toBeNull();
+  });
+});
+
+describe('findSuppressor — guards', () => {
+  // A cascading failure must never silence the component reporting it.
+  test('an unsuppressible monitor is never suppressed, whatever is firing', () => {
+    const firingKeys = new Set([machineAlertKey('iot'), moduleAlertKey('technitium')]);
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('homebridge'),
+        firingKeys,
+        suppressible: false,
+        modulesInDeployWindow: new Set(),
+        topology: TOPOLOGY,
+      }),
+    ).toBeNull();
+  });
+
+  test('a deploy window suppresses the module being deployed', () => {
+    expect(
+      findSuppressor({
+        key: moduleCheckAlertKey('forgejo', 'http'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(['forgejo']),
+        topology: TOPOLOGY,
+      }),
+    ).toEqual({ kind: 'deploy_window', moduleId: 'forgejo' });
+  });
+
+  test('a deploy window does not suppress other modules', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('homebridge'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(['forgejo']),
+        topology: TOPOLOGY,
+      }),
+    ).toBeNull();
+  });
+
+  test('a deploy window outranks an ancestor alert as the explanation', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('homebridge'),
+        firingKeys: new Set([machineAlertKey('iot')]),
+        suppressible: true,
+        modulesInDeployWindow: new Set(['homebridge']),
+        topology: TOPOLOGY,
+      }),
+    ).toEqual({ kind: 'deploy_window', moduleId: 'homebridge' });
+  });
+});

package/src/services/alerting/suppression.ts

@@ -0,0 +1,142 @@
+/**
+ * Suppression — deciding which alerts explain other alerts.
+ *
+ * When a machine drops off the network, every module on it fails its health
+ * check. Reporting all of them is not thoroughness, it is six pages at 3am for
+ * one fact. Suppression collapses that to the page that names the cause.
+ *
+ * The edges are DERIVED from deployment state, never configured. celilo already
+ * records which modules run on which systems, in which zones, and which modules
+ * provide zone-scoped capabilities; asking an operator to restate that would
+ * guarantee it drifts out of date, and a stale suppression graph is worse than
+ * none — it hides real failures.
+ *
+ *   zone ─ (a module providing a zone-scoped capability, e.g. dns_internal)
+ *     └── system ─ (builtin:machines_reachable/machine:<hostname>)
+ *           └── module ─ (module:<id>)
+ *                 └── check item ─ (module:<id>/check:<name>)
+ *
+ * See openspec/changes/add-alerting/design.md D7.
+ */
+
+import { builtinAlertKey, moduleAlertKey, parseAlertKey } from './keys';
+
+/** The audit category whose findings represent an unreachable system. */
+export const MACHINES_REACHABLE_CHECK = 'machines_reachable';
+
+export interface ModuleSystemRow {
+  moduleId: string;
+  /** User-facing hostname — the identifier a machine alert is keyed by. */
+  hostname: string;
+  zone: string;
+  infraType: 'machine' | 'container_service';
+}
+
+export interface ZoneCapabilityProviderRow {
+  moduleId: string;
+  capabilityName: string;
+  /** Zones this capability is scoped to. Empty means zone-agnostic. */
+  zones: string[];
+}
+
+export interface SuppressionTopology {
+  moduleSystems: ModuleSystemRow[];
+  zoneProviders: ZoneCapabilityProviderRow[];
+}
+
+/** The machine-level key for a system, by its user-facing hostname. */
+export function machineAlertKey(hostname: string): string {
+  return builtinAlertKey(MACHINES_REACHABLE_CHECK, 'machine', hostname);
+}
+
+/**
+ * The keys that, if firing, would explain `key` — nearest ancestor first.
+ *
+ * Order matters: the nearest firing ancestor is the one reported as the cause,
+ * and "caddy's hook could not run" is a better explanation than "something in
+ * the dmz is broken" when both are true.
+ */
+export function ancestorKeysFor(key: string, topology: SuppressionTopology): string[] {
+  const parsed = parseAlertKey(key);
+
+  // A built-in alert (an unreachable machine, a coverage gap) is already a
+  // root cause in this model. Nothing above it explains it.
+  if (!parsed || parsed.source !== 'module') return [];
+
+  const ancestors: string[] = [];
+  const moduleId = parsed.moduleId;
+
+  // 1. The module-level key owns its check items.
+  if (parsed.check) ancestors.push(moduleAlertKey(moduleId));
+
+  const systems = topology.moduleSystems.filter((s) => s.moduleId === moduleId);
+
+  // 2. The systems the module runs on.
+  //
+  // Only machine-pool systems: `machines_reachable` probes the machine pool,
+  // so a container_service instance has no corresponding alert to be suppressed
+  // by. A module hosted only in containers therefore gets no system-level
+  // suppression — a real gap, not an oversight, and one that wants a
+  // container-reachability check before it can close.
+  for (const system of systems) {
+    if (system.infraType === 'machine') ancestors.push(machineAlertKey(system.hostname));
+  }
+
+  // 3. Modules providing a zone-scoped capability covering a zone this module
+  //    sits in — a dead resolver or firewall explains everything behind it.
+  const zones = new Set(systems.map((s) => s.zone));
+  for (const provider of topology.zoneProviders) {
+    // A provider never suppresses itself through its own zone edge. Without
+    // this, a resolver whose health check fails BECAUSE dns is down would
+    // silence its own alert, and the one thing naming the cause disappears.
+    if (provider.moduleId === moduleId) continue;
+    if (provider.zones.some((zone) => zones.has(zone))) {
+      ancestors.push(moduleAlertKey(provider.moduleId));
+    }
+  }
+
+  return ancestors;
+}
+
+export interface SuppressorLookup {
+  key: string;
+  /** Keys of alerts currently firing (candidate suppressors). */
+  firingKeys: ReadonlySet<string>;
+  /** False for monitors watching the alerting system itself. */
+  suppressible: boolean;
+  /** Modules currently inside a deploy window. */
+  modulesInDeployWindow: ReadonlySet<string>;
+  topology: SuppressionTopology;
+}
+
+export type Suppressor =
+  | { kind: 'alert'; key: string }
+  | { kind: 'deploy_window'; moduleId: string };
+
+/**
+ * Find what is suppressing `key`, or null if it should be reported.
+ *
+ * Evaluated at NOTIFY time rather than when the alert fires: monitors do not
+ * run in a guaranteed order, so a module's check can fail seconds before the
+ * machine check that explains it. Deciding at fire time pages for the symptom
+ * moments before the cause arrives.
+ */
+export function findSuppressor(lookup: SuppressorLookup): Suppressor | null {
+  // A self-monitor is never suppressed. A cascading failure must not silence
+  // the component reporting the cascade.
+  if (!lookup.suppressible) return null;
+
+  const parsed = parseAlertKey(lookup.key);
+
+  // A deploy is the same mechanism with a window as the source instead of an
+  // ancestor alert — which is why deploy auto-silencing is not a second feature.
+  if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) {
+    return { kind: 'deploy_window', moduleId: parsed.moduleId };
+  }
+
+  for (const ancestor of ancestorKeysFor(lookup.key, lookup.topology)) {
+    if (lookup.firingKeys.has(ancestor)) return { kind: 'alert', key: ancestor };
+  }
+
+  return null;
+}