@celilo/cli 0.13.2 → 0.14.0

package/src/services/alerting/run-monitor.test.ts

@@ -0,0 +1,185 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type Monitor, alerts, monitorRuns, monitors } from '../../db/schema';
+import { setupTestDatabase } from '../../test-utils/setup-test-db';
+import type { HealthCheckResult } from '../health-runner';
+import { moduleAlertKey, moduleCheckAlertKey } from './keys';
+import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
+import { loadLiveAlerts, markUnsuppressed } from './store';
+
+const NOW = new Date('2026-07-28T03:00:00Z');
+const LATER = new Date('2026-07-28T03:15:00Z');
+const MODULE = 'caddy';
+const DISK = moduleCheckAlertKey(MODULE, 'disk-space');
+const MODULE_KEY = moduleAlertKey(MODULE);
+
+const healthy: HealthCheckResult = { moduleId: MODULE, status: 'healthy', checks: [] };
+const diskFailing: HealthCheckResult = {
+  moduleId: MODULE,
+  status: 'unhealthy',
+  checks: [{ name: 'disk-space', status: 'fail', message: '/var 94% used' }],
+};
+const unreachable: HealthCheckResult = {
+  moduleId: MODULE,
+  status: 'error',
+  checks: [],
+  error: 'ssh: connect to host caddy.dmz port 22: timed out',
+};
+
+describe('runOneMonitor', () => {
+  let dir: string;
+  let db: DbClient;
+  let monitor: Monitor;
+
+  function deps(result: HealthCheckResult, now: Date = NOW): MonitorRunDeps {
+    return {
+      runModuleCheck: async () => result,
+      runBuiltinCheck: async () => [],
+      loadModuleCoverage: () => [],
+      now: () => now,
+      graceMs: 60_000,
+    };
+  }
+
+  const liveKeys = () =>
+    loadLiveAlerts(db, monitor.id)
+      .map((a) => a.key)
+      .sort();
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'run-monitor-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    db.insert(monitors)
+      .values({
+        id: 'mon-1',
+        kind: 'module_hook',
+        target: MODULE,
+        intervalMinutes: 15,
+        severity: 'critical',
+      })
+      .run();
+    monitor = db.select().from(monitors).where(eq(monitors.id, 'mon-1')).get() as Monitor;
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  test('a failing check creates an alert and records a successful run', async () => {
+    const summary = await runOneMonitor(db, monitor, deps(diskFailing));
+    expect(summary.outcome).toBe('success');
+    expect(liveKeys()).toEqual([DISK]);
+
+    const runs = db.select().from(monitorRuns).all();
+    expect(runs).toHaveLength(1);
+    expect(runs[0].outcome).toBe('success');
+  });
+
+  test('a clean check resolves what it no longer reports', async () => {
+    await runOneMonitor(db, monitor, deps(diskFailing));
+    await runOneMonitor(db, monitor, deps(healthy, LATER));
+    expect(liveKeys()).toEqual([]);
+  });
+
+  test('lastRunAt advances so the sweep will not immediately re-run it', async () => {
+    await runOneMonitor(db, monitor, deps(diskFailing));
+    const row = db.select().from(monitors).where(eq(monitors.id, 'mon-1')).get();
+    expect(row?.lastRunAt).toEqual(NOW);
+  });
+
+  // The end-to-end version of the guard. A problem is already firing when the
+  // box drops off the network.
+  describe('when the system becomes unreachable', () => {
+    test('the firing alert is NOT resolved', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+
+      const disk = db.select().from(alerts).where(eq(alerts.key, DISK)).get();
+      expect(disk?.state).not.toBe('resolved');
+      expect(disk?.resolvedAt).toBeNull();
+    });
+
+    test('the monitor-level alert fires carrying the real error', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+
+      const moduleAlert = db.select().from(alerts).where(eq(alerts.key, MODULE_KEY)).get();
+      expect(moduleAlert).toBeDefined();
+      expect(moduleAlert?.message).toContain('timed out');
+    });
+
+    test('the run is recorded as errored, not as a clean pass', async () => {
+      await runOneMonitor(db, monitor, deps(unreachable));
+      const run = db.select().from(monitorRuns).all()[0];
+      expect(run.outcome).toBe('error');
+      expect(run.errorMessage).toContain('timed out');
+    });
+
+    test('both alerts stand until a run actually executes again', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+      expect(liveKeys()).toEqual([DISK, MODULE_KEY].sort());
+    });
+  });
+
+  describe('recovery', () => {
+    test('a successful run clears the monitor-level alert but keeps a real failure', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+      await runOneMonitor(db, monitor, deps(diskFailing, new Date('2026-07-28T03:30:00Z')));
+
+      expect(liveKeys()).toEqual([DISK]);
+    });
+
+    test('a successful clean run clears everything the outage froze', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+      await runOneMonitor(db, monitor, deps(healthy, new Date('2026-07-28T03:30:00Z')));
+
+      expect(liveKeys()).toEqual([]);
+    });
+  });
+
+  describe('confirmation after un-suppression', () => {
+    test('a still-failing key clears awaitingConfirmation', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      const [alert] = loadLiveAlerts(db, monitor.id);
+      markUnsuppressed(db, alert.id, LATER);
+
+      await runOneMonitor(db, monitor, deps(diskFailing, LATER));
+      const row = db.select().from(alerts).where(eq(alerts.id, alert.id)).get();
+      expect(row?.awaitingConfirmation).toBe(false);
+    });
+
+    // An errored run confirms nothing — the alert must keep waiting rather
+    // than becoming eligible to page on the strength of a run that never ran.
+    test('an errored run does NOT clear awaitingConfirmation', async () => {
+      await runOneMonitor(db, monitor, deps(diskFailing));
+      const [alert] = loadLiveAlerts(db, monitor.id);
+      markUnsuppressed(db, alert.id, LATER);
+
+      await runOneMonitor(db, monitor, deps(unreachable, LATER));
+      const row = db.select().from(alerts).where(eq(alerts.id, alert.id)).get();
+      expect(row?.awaitingConfirmation).toBe(true);
+    });
+  });
+
+  test('a module with no health_check hook is a clean run, not a fault', async () => {
+    const noChecks: HealthCheckResult = { moduleId: MODULE, status: 'no-checks', checks: [] };
+    const summary = await runOneMonitor(db, monitor, deps(noChecks));
+    expect(summary.outcome).toBe('success');
+    expect(liveKeys()).toEqual([]);
+  });
+});

package/src/services/alerting/run-monitor.ts

@@ -0,0 +1,177 @@
+/**
+ * Running one monitor: check → reconcile → persist.
+ *
+ * This is the Execution half of the monitoring loop. Everything it decides was
+ * decided elsewhere — which monitors are due (`sweep.ts`), what a result means
+ * (`keys.ts`, `builtin-monitors.ts`), and what should change (`reconcile.ts`) —
+ * so this function's only job is to sequence those and write the outcome down.
+ *
+ * The checks themselves are injected rather than imported. A monitor run
+ * otherwise reaches SSH, the module hook executor, and the audit subsystem,
+ * which would make the loop untestable without a live fleet.
+ */
+
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type AlertSeverity, type Monitor, monitorRuns, monitors } from '../../db/schema';
+import type { DriftCategory, DriftFinding } from '../audit/types';
+import type { HealthCheckResult } from '../health-runner';
+import { failingKeysFromFindings } from './builtin-monitors';
+import { HEALTH_COVERAGE_CHECK, healthCoverageFailingKeys } from './health-coverage';
+import type { ModuleCoverageInput } from './health-coverage';
+import {
+  type FailingKey,
+  builtinMonitorKey,
+  failingKeysFromHealthItems,
+  moduleAlertKey,
+} from './keys';
+import { reconcile } from './reconcile';
+import { applyReconcileActions, confirmStillFailing, loadLiveAlerts } from './store';
+
+export interface MonitorRunDeps {
+  /** Run a module's health_check hook unattended. */
+  runModuleCheck(moduleId: string): Promise<HealthCheckResult>;
+  /** Run one audit category. */
+  runBuiltinCheck(category: DriftCategory): Promise<DriftFinding[]>;
+  /** Module roster for the health-coverage check. */
+  loadModuleCoverage(): ModuleCoverageInput[];
+  now(): Date;
+  /** Grace window before a new alert may notify. */
+  graceMs: number;
+}
+
+export interface MonitorRunOutcomeSummary {
+  monitorId: string;
+  outcome: 'success' | 'error';
+  errorMessage?: string;
+  failingKeyCount: number;
+  createdIds: string[];
+  resolvedIds: string[];
+}
+
+/**
+ * What the check produced, before it becomes alert state.
+ *
+ * `outcome` is the load-bearing field: `error` means the check could not run,
+ * which says nothing about any individual key and must never resolve anything.
+ */
+interface CheckProduct {
+  outcome: 'success' | 'error';
+  failingKeys: FailingKey[];
+  errorMessage?: string;
+}
+
+async function runCheck(
+  monitor: Monitor,
+  deps: MonitorRunDeps,
+  severity: AlertSeverity,
+): Promise<CheckProduct> {
+  if (monitor.kind === 'module_hook') {
+    const result = await deps.runModuleCheck(monitor.target);
+
+    // `error` is the hook failing to execute. `no-checks` means the module
+    // declares no health_check hook — nothing ran, but nothing was expected to,
+    // so it is a successful run with an empty failing set rather than a fault.
+    if (result.status === 'error') {
+      return { outcome: 'error', failingKeys: [], errorMessage: result.error };
+    }
+    return {
+      outcome: 'success',
+      failingKeys: failingKeysFromHealthItems(monitor.target, result.checks, severity),
+    };
+  }
+
+  // The coverage check reads local state only, so it has no failure mode worth
+  // distinguishing from an empty result.
+  if (monitor.target === HEALTH_COVERAGE_CHECK) {
+    return {
+      outcome: 'success',
+      failingKeys: healthCoverageFailingKeys(deps.loadModuleCoverage()),
+    };
+  }
+
+  const category = monitor.target as DriftCategory;
+  try {
+    const findings = await deps.runBuiltinCheck(category);
+    return {
+      outcome: 'success',
+      failingKeys: failingKeysFromFindings(category, findings, severity),
+    };
+  } catch (error) {
+    return {
+      outcome: 'error',
+      failingKeys: [],
+      errorMessage: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+/** The key meaning "this monitor could not run". */
+export function monitorLevelKeyFor(monitor: Monitor): string {
+  return monitor.kind === 'module_hook'
+    ? moduleAlertKey(monitor.target)
+    : builtinMonitorKey(monitor.target);
+}
+
+/**
+ * Run one monitor and apply its consequences.
+ *
+ * Records the run's outcome before anything else derived from it, so that a
+ * crash between checking and reconciling leaves evidence of what happened
+ * rather than a silent gap.
+ */
+export async function runOneMonitor(
+  db: DbClient,
+  monitor: Monitor,
+  deps: MonitorRunDeps,
+): Promise<MonitorRunOutcomeSummary> {
+  const now = deps.now();
+  const product = await runCheck(monitor, deps, monitor.severity);
+
+  db.insert(monitorRuns)
+    .values({
+      monitorId: monitor.id,
+      ranAt: now,
+      outcome: product.outcome,
+      errorMessage: product.errorMessage,
+    })
+    .run();
+
+  const liveAlerts = loadLiveAlerts(db, monitor.id);
+  const actions = reconcile({
+    liveAlerts,
+    outcome: product.outcome,
+    failingKeys: product.failingKeys,
+    monitorLevelKey: monitorLevelKeyFor(monitor),
+    errorMessage: product.errorMessage,
+    monitorSeverity: monitor.severity,
+    now,
+    graceMs: deps.graceMs,
+  });
+
+  const { createdIds, resolvedIds } = applyReconcileActions(db, actions, {
+    monitorId: monitor.id,
+    escalationPolicyId: monitor.escalationPolicyId,
+    now,
+  });
+
+  // A successful run that still reports a key confirms any alert waiting on
+  // confirmation after un-suppression. Only a run that EXECUTED may do this —
+  // an errored run has confirmed nothing.
+  if (product.outcome === 'success') {
+    const stillFailing = new Set(product.failingKeys.map((f) => f.key));
+    const confirmable = liveAlerts.filter((a) => stillFailing.has(a.key)).map((a) => a.id);
+    confirmStillFailing(db, confirmable, now);
+  }
+
+  db.update(monitors).set({ lastRunAt: now }).where(eq(monitors.id, monitor.id)).run();
+
+  return {
+    monitorId: monitor.id,
+    outcome: product.outcome,
+    errorMessage: product.errorMessage,
+    failingKeyCount: product.failingKeys.length,
+    createdIds,
+    resolvedIds,
+  };
+}

package/src/services/alerting/store.test.ts

@@ -0,0 +1,222 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { alerts, monitors } from '../../db/schema';
+import { setupTestDatabase } from '../../test-utils/setup-test-db';
+import { moduleAlertKey, moduleCheckAlertKey } from './keys';
+import type { ReconcileAction } from './reconcile';
+import {
+  applyReconcileActions,
+  confirmStillFailing,
+  loadLiveAlerts,
+  markSuppressed,
+  markUnsuppressed,
+  promoteReadyAlerts,
+  summariseByModule,
+} from './store';
+
+const NOW = new Date('2026-07-28T03:00:00Z');
+const LATER = new Date('2026-07-28T03:20:00Z');
+const MONITOR = 'mon-1';
+const DISK = moduleCheckAlertKey('caddy', 'disk-space');
+
+describe('alert store', () => {
+  let dir: string;
+  let db: DbClient;
+
+  const context = { monitorId: MONITOR, escalationPolicyId: null, now: NOW };
+
+  const create = (key: string): ReconcileAction => ({
+    type: 'create',
+    key,
+    severity: 'critical',
+    message: 'bad',
+    graceUntil: NOW,
+  });
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'alert-store-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    db.insert(monitors)
+      .values({
+        id: MONITOR,
+        kind: 'module_hook',
+        target: 'caddy',
+        intervalMinutes: 15,
+        severity: 'critical',
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  test('create inserts a live alert discoverable by loadLiveAlerts', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    expect(createdIds).toHaveLength(1);
+    expect(loadLiveAlerts(db, MONITOR)).toEqual([{ id: createdIds[0], key: DISK }]);
+  });
+
+  test('refresh updates message and severity without changing identity', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    applyReconcileActions(
+      db,
+      [
+        {
+          type: 'refresh',
+          alertId: createdIds[0],
+          key: DISK,
+          severity: 'warning',
+          message: '/var 80% used',
+        },
+      ],
+      { ...context, now: LATER },
+    );
+    const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+    expect(row?.message).toBe('/var 80% used');
+    expect(row?.severity).toBe('warning');
+    expect(row?.lastSeenAt).toEqual(LATER);
+    expect(row?.firstFiredAt).toEqual(NOW);
+  });
+
+  test('resolve clears activeKey so the alert leaves the live set', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    applyReconcileActions(db, [{ type: 'resolve', alertId: createdIds[0], key: DISK }], context);
+    expect(loadLiveAlerts(db, MONITOR)).toEqual([]);
+    const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+    expect(row?.state).toBe('resolved');
+    expect(row?.activeKey).toBeNull();
+    expect(row?.resolvedAt).toEqual(NOW);
+  });
+
+  // The whole point of the activeKey/index pairing: history accumulates, but
+  // only one row per key is ever live.
+  test('a key can re-fire after resolving, and both rows persist', () => {
+    const first = applyReconcileActions(db, [create(DISK)], context);
+    applyReconcileActions(
+      db,
+      [{ type: 'resolve', alertId: first.createdIds[0], key: DISK }],
+      context,
+    );
+    const second = applyReconcileActions(db, [create(DISK)], { ...context, now: LATER });
+
+    expect(second.createdIds[0]).not.toBe(first.createdIds[0]);
+    expect(loadLiveAlerts(db, MONITOR)).toEqual([{ id: second.createdIds[0], key: DISK }]);
+    const all = db.select().from(alerts).where(eq(alerts.key, DISK)).all();
+    expect(all).toHaveLength(2);
+  });
+
+  test('two live alerts for one key are rejected by the index', () => {
+    applyReconcileActions(db, [create(DISK)], context);
+    expect(() => applyReconcileActions(db, [create(DISK)], context)).toThrow();
+  });
+
+  test('suppression is recorded with its cause', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    markSuppressed(db, createdIds[0], { alertId: 'ancestor-1' });
+    const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+    expect(row?.state).toBe('suppressed');
+    expect(row?.suppressedByAlertId).toBe('ancestor-1');
+  });
+
+  // Un-suppression must not page immediately, and must not fire every step
+  // whose delay elapsed while suppressed.
+  test('un-suppression sets awaitingConfirmation and restarts the clock', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    markSuppressed(db, createdIds[0], { alertId: 'ancestor-1' });
+    markUnsuppressed(db, createdIds[0], LATER);
+
+    const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+    expect(row?.state).toBe('firing');
+    expect(row?.suppressedByAlertId).toBeNull();
+    expect(row?.awaitingConfirmation).toBe(true);
+    expect(row?.unsuppressedAt).toEqual(LATER);
+    expect(row?.escalationStep).toBe(0);
+  });
+
+  test('a confirming run clears awaitingConfirmation', () => {
+    const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+    markUnsuppressed(db, createdIds[0], LATER);
+    confirmStillFailing(db, createdIds, LATER);
+    const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+    expect(row?.awaitingConfirmation).toBe(false);
+  });
+
+  test('confirming an empty list is a no-op', () => {
+    expect(() => confirmStillFailing(db, [], NOW)).not.toThrow();
+  });
+
+  describe('promoteReadyAlerts', () => {
+    // Nothing else writes pending -> firing, so without this every alert reads
+    // as pending forever.
+    test('a pending alert past its grace window becomes firing', () => {
+      applyReconcileActions(db, [create(DISK)], context);
+      expect(promoteReadyAlerts(db, LATER)).toBe(1);
+      const row = db.select().from(alerts).where(eq(alerts.key, DISK)).get();
+      expect(row?.state).toBe('firing');
+    });
+
+    test('a pending alert inside its grace window is left alone', () => {
+      applyReconcileActions(
+        db,
+        [{ type: 'create', key: DISK, severity: 'critical', message: 'bad', graceUntil: LATER }],
+        context,
+      );
+      expect(promoteReadyAlerts(db, NOW)).toBe(0);
+      const row = db.select().from(alerts).where(eq(alerts.key, DISK)).get();
+      expect(row?.state).toBe('pending');
+    });
+
+    // Promoting these would lose the record of what explains them.
+    test('a suppressed alert is not promoted', () => {
+      const { createdIds } = applyReconcileActions(db, [create(DISK)], context);
+      markSuppressed(db, createdIds[0], { alertId: 'ancestor-1' });
+      expect(promoteReadyAlerts(db, LATER)).toBe(0);
+      const row = db.select().from(alerts).where(eq(alerts.id, createdIds[0])).get();
+      expect(row?.state).toBe('suppressed');
+    });
+
+    test('promotion is idempotent', () => {
+      applyReconcileActions(db, [create(DISK)], context);
+      expect(promoteReadyAlerts(db, LATER)).toBe(1);
+      expect(promoteReadyAlerts(db, LATER)).toBe(0);
+    });
+  });
+});
+
+describe('summariseByModule', () => {
+  const row = (key: string, state: 'firing' | 'suppressed') =>
+    ({ key, severity: 'critical', state }) as never;
+
+  test('groups item and module keys under the module', () => {
+    const summary = summariseByModule([
+      row(moduleCheckAlertKey('caddy', 'disk'), 'firing'),
+      row(moduleAlertKey('caddy'), 'firing'),
+      row(moduleAlertKey('forgejo'), 'firing'),
+    ]);
+    expect(summary.get('caddy')).toHaveLength(2);
+    expect(summary.get('forgejo')).toHaveLength(1);
+  });
+
+  test('marks suppressed alerts so the column can say so', () => {
+    const summary = summariseByModule([row(moduleAlertKey('caddy'), 'suppressed')]);
+    expect(summary.get('caddy')?.[0].suppressed).toBe(true);
+  });
+
+  test('ignores built-in alerts, which are not module-scoped', () => {
+    const summary = summariseByModule([row('builtin:machines_reachable/machine:iot', 'firing')]);
+    expect(summary.size).toBe(0);
+  });
+});