@celilo/cli 0.13.2 → 0.14.0

package/src/services/alerting/format.ts

@@ -0,0 +1,150 @@
+/**
+ * Rendering alerts for the operator.
+ *
+ * A flat table rather than a TUI: alert triage is a list-scan, not an
+ * exploration, and a table is greppable, pipeable, and renders unchanged over
+ * the Remote API. See design D14.
+ *
+ * Pure — takes rows, returns strings — so the wording and the column that says
+ * "explained by something upstream" can be asserted without a database.
+ */
+
+import type { Alert, AlertState } from '../../db/schema';
+
+export interface AlertRow {
+  key: string;
+  /** Display label, not the stored state — `firing` renders as `FIRING`. */
+  state: string;
+  severity: string;
+  age: string;
+  detail: string;
+}
+
+const STATE_LABEL: Record<AlertState, string> = {
+  pending: 'pending',
+  firing: 'FIRING',
+  acked: 'acked',
+  suppressed: 'suppressed',
+  resolved: 'resolved',
+};
+
+/**
+ * Compact humanised age. Deliberately coarse — an operator scanning a list
+ * needs "is this new or has it been broken all day", not seconds.
+ */
+export function humaniseAge(from: Date, now: Date): string {
+  const seconds = Math.max(0, Math.floor((now.getTime() - from.getTime()) / 1000));
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  return `${Math.floor(hours / 24)}d`;
+}
+
+/**
+ * Time REMAINING, rounded up.
+ *
+ * `humaniseAge` floors, which is right for elapsed time and wrong for a
+ * countdown: with 1h59m left, "1h" reads as though a 2h silence were nearly
+ * over. Rounding up never overstates how soon an alert comes back.
+ */
+export function humaniseRemaining(until: Date, now: Date): string {
+  const seconds = Math.max(0, Math.ceil((until.getTime() - now.getTime()) / 1000));
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.ceil(seconds / 60);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.ceil(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  return `${Math.ceil(hours / 24)}d`;
+}
+
+/**
+ * The most useful thing to say about an alert beyond its state.
+ *
+ * Ordering is by what an operator would act on: an acknowledgement tells them
+ * someone already has it, a suppression tells them not to chase this one, and
+ * a silence tells them it is deliberate. Only when none apply does the alert's
+ * own message earn the column.
+ */
+export function alertDetail(alert: Alert, now: Date): string {
+  if (alert.state === 'acked') return 'acked';
+  if (alert.state === 'suppressed') {
+    return alert.suppressedByWindowId ? 'deploy in progress' : 'explained by an upstream alert';
+  }
+  if (alert.silencedUntil && alert.silencedUntil > now) {
+    return `silenced for ${humaniseRemaining(alert.silencedUntil, now)}`;
+  }
+  if (alert.awaitingConfirmation) return 'awaiting confirmation';
+  if (alert.state === 'pending' && alert.graceUntil > now) return 'within grace window';
+  return alert.message;
+}
+
+export function toAlertRow(alert: Alert, now: Date): AlertRow {
+  return {
+    key: alert.key,
+    state: STATE_LABEL[alert.state],
+    severity: alert.severity,
+    age: humaniseAge(alert.firstFiredAt, now),
+    detail: alertDetail(alert, now),
+  };
+}
+
+/**
+ * Render rows as a fixed-width table.
+ *
+ * Sorted most-actionable first: firing alerts an operator must deal with, then
+ * pending, then everything already accounted for. Within a group, oldest first
+ * — a problem that has been burning for a day outranks one from a minute ago.
+ */
+const STATE_ORDER: Record<string, number> = {
+  FIRING: 0,
+  pending: 1,
+  acked: 2,
+  suppressed: 3,
+  resolved: 4,
+};
+
+export function sortAlertRows(rows: AlertRow[]): AlertRow[] {
+  return [...rows].sort((a, b) => {
+    const byState = (STATE_ORDER[a.state] ?? 9) - (STATE_ORDER[b.state] ?? 9);
+    return byState !== 0 ? byState : a.key.localeCompare(b.key);
+  });
+}
+
+export function renderAlertTable(rows: AlertRow[]): string {
+  if (rows.length === 0) return 'No alerts.';
+
+  const sorted = sortAlertRows(rows);
+  const headers = ['KEY', 'STATE', 'SEVERITY', 'AGE', 'DETAIL'];
+  const columns: (keyof AlertRow)[] = ['key', 'state', 'severity', 'age', 'detail'];
+  const widths = columns.map((column, i) =>
+    Math.max(headers[i].length, ...sorted.map((row) => row[column].length)),
+  );
+
+  const line = (cells: string[]) =>
+    cells
+      .map((cell, i) => (i === cells.length - 1 ? cell : cell.padEnd(widths[i])))
+      .join('  ')
+      .trimEnd();
+
+  return [line(headers), ...sorted.map((row) => line(columns.map((c) => row[c])))].join('\n');
+}
+
+/**
+ * The observed-health cell for `celilo module list`.
+ *
+ * "Not observed" is a finding, not a blank: a module nothing is watching is
+ * unverified infrastructure, and rendering it as healthy is the lie this column
+ * exists to stop telling. See design D15.
+ */
+export function moduleHealthCell(input: {
+  monitored: boolean;
+  firingCount: number;
+  suppressed: boolean;
+}): string {
+  if (!input.monitored) return 'not observed';
+  if (input.suppressed) return 'suppressed';
+  if (input.firingCount > 0) return `${input.firingCount} firing`;
+  return 'ok';
+}

package/src/services/alerting/health-coverage.ts

@@ -0,0 +1,81 @@
+/**
+ * Health coverage — which deployed modules nothing is watching.
+ *
+ * A module that has never been successfully checked is not a neutral absence
+ * of information. It is unverified infrastructure, and it fails in exactly the
+ * way a broken module does, only later and with less warning. Once checks are
+ * scheduled this becomes a durable condition rather than a transient one: a
+ * monitored module resolves to healthy or failing within one interval, so
+ * anything still unobserved is structurally blind.
+ *
+ * Two ways that happens, and both are worth saying out loud:
+ *   - the module ships no `health_check` hook, so it can never be verified;
+ *   - it has one, but no enabled monitor schedules it.
+ *
+ * Deliberately cheap: this reads the module roster against the monitor set and
+ * contacts nothing. It is safe to run on every sweep.
+ *
+ * See openspec/changes/add-alerting/design.md D15.
+ */
+
+import type { ModuleState } from '../../db/schema';
+import { type FailingKey, builtinAlertKey } from './keys';
+
+export const HEALTH_COVERAGE_CHECK = 'health_coverage';
+
+/**
+ * States in which a module is live enough to be worth watching. A module still
+ * being imported or configured has nothing deployed to observe yet, so an
+ * absent monitor there is expected rather than a gap.
+ */
+const DEPLOYED_STATES: ReadonlySet<ModuleState> = new Set<ModuleState>(['INSTALLED', 'VERIFIED']);
+
+export interface ModuleCoverageInput {
+  id: string;
+  state: ModuleState;
+  /** Whether the manifest declares a `health_check` hook at all. */
+  hasHealthCheckHook: boolean;
+  /** Whether an ENABLED monitor schedules that hook. */
+  hasEnabledMonitor: boolean;
+}
+
+/**
+ * Findings are `warning` severity: a coverage gap is real, but it is not an
+ * outage and must not wake anyone. It shows up in `celilo alerts` and in the
+ * `celilo module list` health column, where it reads as "not observed" rather
+ * than being silently rendered as healthy.
+ */
+export function healthCoverageFailingKeys(modules: ModuleCoverageInput[]): FailingKey[] {
+  const failing: FailingKey[] = [];
+  for (const module of modules) {
+    if (!DEPLOYED_STATES.has(module.state)) continue;
+
+    const key = builtinAlertKey(HEALTH_COVERAGE_CHECK, 'module', module.id);
+
+    if (!module.hasHealthCheckHook) {
+      failing.push({
+        key,
+        severity: 'warning',
+        message: `${module.id} declares no health_check hook — it can never be verified`,
+        details:
+          'celilo has no way to confirm this module works. Add a health_check\n' +
+          'hook to its manifest, or accept that failures here surface only when\n' +
+          'something downstream breaks.',
+      });
+      continue;
+    }
+
+    if (!module.hasEnabledMonitor) {
+      failing.push({
+        key,
+        severity: 'warning',
+        message: `${module.id} has a health_check hook but nothing schedules it`,
+        details:
+          'The module can be verified on demand but is not being watched.\n' +
+          'Create a monitor for it, or set hooks.health_check.interval in the\n' +
+          'module manifest so one is created on deploy.',
+      });
+    }
+  }
+  return failing;
+}

package/src/services/alerting/inbound-poller.test.ts

@@ -0,0 +1,298 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { InboundMessage } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
+import { setupTestDatabase } from '../../test-utils/setup-test-db';
+import { type InboundPollDeps, pollInbound, transportsWithRoutes } from './inbound-poller';
+import { moduleCheckAlertKey } from './keys';
+import { createPerson, createRoute } from './people';
+import { mintDelivery } from './tokens';
+
+const NOW = new Date('2026-07-28T03:00:00Z');
+const KEY = moduleCheckAlertKey('caddy', 'disk-space');
+const PETER = '+15550001';
+const WIFE = '+15550002';
+const STRANGER = '+15559999';
+
+describe('pollInbound', () => {
+  let dir: string;
+  let db: DbClient;
+  let peterRoute: Route;
+  let wifeRoute: Route;
+  const cursors = new Map<string, string | null>();
+
+  function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
+    return {
+      receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
+      readCursor: (t) => cursors.get(t) ?? null,
+      writeCursor: (t, c) => {
+        cursors.set(t, c);
+      },
+      now: () => NOW,
+      ...over,
+    };
+  }
+
+  const inbound = (from: string, body: string): InboundMessage => ({
+    senderAddress: from,
+    body,
+    receivedAt: NOW.getTime(),
+    messageId: `${from}:${NOW.getTime()}`,
+  });
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'inbound-poll-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    cursors.clear();
+
+    db.insert(modules)
+      .values({
+        id: 'signal',
+        name: 'Signal',
+        version: '0.1.0',
+        manifestData: {},
+        sourcePath: '/tmp/signal',
+      })
+      .run();
+    db.insert(monitors)
+      .values({
+        id: 'mon-1',
+        kind: 'module_hook',
+        target: 'caddy',
+        intervalMinutes: 15,
+        severity: 'critical',
+      })
+      .run();
+    db.insert(alerts)
+      .values({
+        id: 'alert-1',
+        key: KEY,
+        activeKey: KEY,
+        monitorId: 'mon-1',
+        state: 'firing',
+        severity: 'critical',
+        graceUntil: NOW,
+        message: '/var 94% used',
+      })
+      .run();
+
+    const peter = createPerson(db, { name: 'peter', timezone: 'UTC' });
+    const wife = createPerson(db, { name: 'wife', timezone: 'UTC' });
+    peterRoute = createRoute(db, {
+      personId: peter.id,
+      transportModuleId: 'signal',
+      address: PETER,
+      canAck: true,
+    });
+    wifeRoute = createRoute(db, {
+      personId: wife.id,
+      transportModuleId: 'signal',
+      address: WIFE,
+      canAck: true,
+    });
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  const page = (routeId: string) =>
+    mintDelivery(db, {
+      kind: 'alert',
+      targetId: 'alert-1',
+      routeId,
+      now: NOW,
+      ttlMs: 86_400_000,
+    });
+
+  const alertState = () => db.select().from(alerts).where(eq(alerts.id, 'alert-1')).get()?.state;
+
+  test('only transports with routes are polled', () => {
+    expect(transportsWithRoutes(db)).toEqual(['signal']);
+  });
+
+  test('a valid token from the right sender acknowledges the alert', async () => {
+    const delivery = page(peterRoute.id);
+    const report = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+    expect(report.acked).toBe(1);
+    expect(alertState()).toBe('acked');
+  });
+
+  // 7.5 — the secondary acks, and the primary is told so they can stand down.
+  test('an ack is broadcast to everyone else who was paged', async () => {
+    const sent: { address: string; body: string }[] = [];
+    page(peterRoute.id);
+    const wifeDelivery = page(wifeRoute.id);
+
+    const report = await pollInbound(
+      db,
+      deps([inbound(WIFE, wifeDelivery.token)], {
+        transportFor: () => ({
+          send: async (request: { address: string; body: string }) => {
+            sent.push(request);
+            return { messageId: 'm-1' };
+          },
+        }),
+      }),
+    );
+
+    expect(report.acked).toBe(1);
+    expect(report.broadcast).toBe(1);
+    expect(sent).toHaveLength(1);
+    expect(sent[0].address).toBe(PETER);
+    expect(sent[0].body).toContain('wife');
+  });
+
+  test('the acknowledger is not told about their own ack', async () => {
+    const sent: string[] = [];
+    const delivery = page(peterRoute.id);
+
+    await pollInbound(
+      db,
+      deps([inbound(PETER, delivery.token)], {
+        transportFor: () => ({
+          send: async (request: { address: string; body: string }) => {
+            sent.push(request.address);
+            return { messageId: 'm-1' };
+          },
+        }),
+      }),
+    );
+
+    expect(sent).toEqual([]);
+  });
+
+  // An installation with no send path still acks — it just cannot tell anyone.
+  test('a missing transport does not stop the ack', async () => {
+    page(peterRoute.id);
+    const wifeDelivery = page(wifeRoute.id);
+    const report = await pollInbound(db, deps([inbound(WIFE, wifeDelivery.token)]));
+
+    expect(report.acked).toBe(1);
+    expect(report.broadcast).toBe(0);
+    expect(alertState()).toBe('acked');
+  });
+
+  test('a token from the wrong sender is rejected and changes nothing', async () => {
+    const delivery = page(peterRoute.id);
+    const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
+
+    expect(report.rejected).toBe(1);
+    expect(report.acked).toBe(0);
+    expect(alertState()).toBe('firing');
+  });
+
+  // Silence, not a reply — answering confirms the number is live.
+  test('an unknown sender is ignored', async () => {
+    const delivery = page(peterRoute.id);
+    const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
+
+    expect(report.ignored).toBe(1);
+    expect(alertState()).toBe('firing');
+  });
+
+  test('the token is consumed so it cannot be replayed', async () => {
+    const delivery = page(peterRoute.id);
+    await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+    const row = db
+      .select()
+      .from(notificationDeliveries)
+      .where(eq(notificationDeliveries.id, delivery.id))
+      .get();
+    expect(row?.consumedAt).not.toBeNull();
+  });
+
+  // At-least-once is the contract: the same batch replayed after a crash must
+  // not do anything twice that matters.
+  test('replaying the same message is harmless', async () => {
+    const delivery = page(peterRoute.id);
+    const message = inbound(PETER, delivery.token);
+
+    const first = await pollInbound(db, deps([message]));
+    const second = await pollInbound(db, deps([message]));
+
+    expect(first.acked).toBe(1);
+    // The token is spent, so the replay is refused rather than acking again.
+    expect(second.acked).toBe(0);
+    expect(second.rejected).toBe(1);
+    expect(alertState()).toBe('acked');
+  });
+
+  test('a bare ack works when exactly one delivery is outstanding', async () => {
+    page(peterRoute.id);
+    const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
+    expect(report.acked).toBe(1);
+    expect(alertState()).toBe('acked');
+  });
+
+  test('a bare ack is refused when several are outstanding', async () => {
+    page(peterRoute.id);
+    db.insert(alerts)
+      .values({
+        id: 'alert-2',
+        key: 'module:caddy/check:cert',
+        activeKey: 'module:caddy/check:cert',
+        monitorId: 'mon-1',
+        state: 'firing',
+        severity: 'critical',
+        graceUntil: NOW,
+        message: 'expiring',
+      })
+      .run();
+    mintDelivery(db, {
+      kind: 'alert',
+      targetId: 'alert-2',
+      routeId: peterRoute.id,
+      now: NOW,
+      ttlMs: 86_400_000,
+    });
+
+    const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
+    expect(report.rejected).toBe(1);
+    expect(alertState()).toBe('firing');
+  });
+
+  // One dead transport must not stop the others being read.
+  test('an unreachable transport is skipped without throwing', async () => {
+    const report = await pollInbound(db, deps([], { receiveFrom: async () => null }));
+    expect(report.transportsPolled).toBe(1);
+    expect(report.messagesRead).toBe(0);
+  });
+
+  test('the cursor is persisted after a successful poll', async () => {
+    await pollInbound(db, deps([]));
+    expect(cursors.get('signal')).toBe('cursor-1');
+  });
+
+  test('the cursor is NOT advanced when the transport could not be reached', async () => {
+    await pollInbound(db, deps([], { receiveFrom: async () => null }));
+    expect(cursors.get('signal')).toBeUndefined();
+  });
+
+  test('gibberish is rejected without touching the alert', async () => {
+    page(peterRoute.id);
+    const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
+    expect(report.rejected).toBe(1);
+    expect(alertState()).toBe('firing');
+  });
+
+  test('a route pointing at a transport nobody uses is not polled', () => {
+    db.delete(alerts).run();
+    expect(transportsWithRoutes(db)).toEqual(['signal']);
+    expect(wifeRoute.transportModuleId).toBe('signal');
+  });
+});