@celilo/cli 0.13.2 → 0.14.0

package/src/services/alerting/monitors.ts

@@ -0,0 +1,185 @@
+/**
+ * Monitor CRUD.
+ *
+ * A monitor binds a thing to check to a cadence, a severity, and an escalation
+ * policy. Operators address them by their target (`caddy`,
+ * `machines_reachable`) rather than by id — per CLAUDE.md, a UUID never
+ * reaches the operator.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { and, eq, isNotNull } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import {
+  type AlertSeverity,
+  type Monitor,
+  type MonitorKind,
+  alerts,
+  monitors,
+} from '../../db/schema';
+import { parseIntervalMinutes } from '../../manifest/schema';
+
+export interface CreateMonitorInput {
+  kind: MonitorKind;
+  target: string;
+  intervalMinutes: number;
+  severity?: AlertSeverity;
+  suppressible?: boolean;
+  escalationPolicyId?: string | null;
+}
+
+export function listMonitors(db: DbClient): Monitor[] {
+  return db.select().from(monitors).all();
+}
+
+export function findMonitor(db: DbClient, kind: MonitorKind, target: string): Monitor | undefined {
+  return db
+    .select()
+    .from(monitors)
+    .where(and(eq(monitors.kind, kind), eq(monitors.target, target)))
+    .get();
+}
+
+/**
+ * Find a monitor by target alone.
+ *
+ * Targets are unique in practice — a module id and an audit category cannot
+ * collide, since audit categories are a fixed snake_case set and module ids are
+ * kebab-case. Resolving by target alone is what lets the operator type
+ * `celilo monitor run caddy` without also naming the kind.
+ */
+export function findMonitorByTarget(db: DbClient, target: string): Monitor | undefined {
+  return db.select().from(monitors).where(eq(monitors.target, target)).get();
+}
+
+export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor {
+  const id = randomUUID();
+  db.insert(monitors)
+    .values({
+      id,
+      kind: input.kind,
+      target: input.target,
+      intervalMinutes: input.intervalMinutes,
+      severity: input.severity ?? 'critical',
+      suppressible: input.suppressible ?? true,
+      escalationPolicyId: input.escalationPolicyId ?? null,
+    })
+    .run();
+  return db.select().from(monitors).where(eq(monitors.id, id)).get() as Monitor;
+}
+
+/**
+ * Ensure a monitor exists for a module that declares a suggested interval.
+ *
+ * Called on deploy. Never overwrites an existing monitor: the manifest's
+ * interval is the author's SUGGESTION and the operator's setting must survive
+ * module upgrades, so a second call is a no-op rather than a reset.
+ *
+ * Returns the monitor when one was created, null when one already existed or
+ * the manifest declares no interval.
+ */
+export function ensureMonitorForModule(
+  db: DbClient,
+  moduleId: string,
+  suggestedInterval: string | undefined,
+): Monitor | null {
+  if (!suggestedInterval) return null;
+  if (findMonitor(db, 'module_hook', moduleId)) return null;
+
+  const intervalMinutes = parseIntervalMinutes(suggestedInterval);
+  if (intervalMinutes === null) return null;
+
+  return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
+}
+
+/**
+ * Enable or disable a monitor.
+ *
+ * Disabling resolves its alerts rather than leaving them hanging: a disabled
+ * monitor will never report on them again, so anything still live would be
+ * frozen forever with no way to clear.
+ */
+export function setMonitorEnabled(
+  db: DbClient,
+  monitorId: string,
+  enabled: boolean,
+  now: Date,
+): void {
+  db.update(monitors).set({ enabled }).where(eq(monitors.id, monitorId)).run();
+  if (!enabled) resolveMonitorAlerts(db, monitorId, now);
+}
+
+/** Resolve every live alert owned by a monitor. */
+export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date): number {
+  const live = db
+    .select({ id: alerts.id })
+    .from(alerts)
+    .where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey)))
+    .all();
+
+  for (const row of live) {
+    db.update(alerts)
+      .set({ state: 'resolved', activeKey: null, resolvedAt: now })
+      .where(eq(alerts.id, row.id))
+      .run();
+  }
+  return live.length;
+}
+
+/**
+ * The bus subscriber that makes alerting run by itself.
+ *
+ * Registered as an ordinary subscriber with a shell handler, exactly like a
+ * module subscription — the dispatcher spawns `celilo alerts sweep` on every
+ * five-minute tick. No new scheduling mechanism: `timer.tick.5m` already
+ * exists, already survives restarts, and already has retry and dedup.
+ *
+ * Idempotent: `bus.subscribe` upserts by name, so calling this on every
+ * monitor creation is safe and means switching on the first monitor is also
+ * what switches on the sweep.
+ */
+export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
+export const ALERTING_SWEEP_PATTERN = 'timer.tick.5m';
+
+export interface SubscriberRegistrar {
+  subscribe(options: {
+    name: string;
+    pattern: string;
+    handler: string;
+    registeredBy?: string;
+  }): unknown;
+}
+
+export const ALERTING_POLL_SUBSCRIBER = 'celilo-alerting-inbound';
+
+export function ensureSweepSubscriber(bus: SubscriberRegistrar): void {
+  bus.subscribe({
+    name: ALERTING_SWEEP_SUBSCRIBER,
+    pattern: ALERTING_SWEEP_PATTERN,
+    handler: 'celilo alerts sweep',
+    registeredBy: 'celilo-alerting',
+  });
+}
+
+/**
+ * Read inbound replies on the same five-minute tick.
+ *
+ * Five minutes is coarse for an ack — the design wants seconds — but the bus
+ * timer menu is deliberately fixed, and a coarse ack that works beats a
+ * bespoke scheduler that has to be supervised. An operator wanting faster
+ * turnaround runs `celilo alerts poll`, and a tighter loop can be added later
+ * without changing anything else.
+ */
+export function ensureInboundSubscriber(bus: SubscriberRegistrar): void {
+  bus.subscribe({
+    name: ALERTING_POLL_SUBSCRIBER,
+    pattern: ALERTING_SWEEP_PATTERN,
+    handler: 'celilo alerts poll',
+    registeredBy: 'celilo-alerting',
+  });
+}
+
+export function deleteMonitor(db: DbClient, monitorId: string, now: Date): void {
+  resolveMonitorAlerts(db, monitorId, now);
+  db.delete(monitors).where(eq(monitors.id, monitorId)).run();
+}

package/src/services/alerting/notification-responder.test.ts

@@ -0,0 +1,290 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defineEvents, openBus } from '@celilo/event-bus';
+import type { DbClient } from '../../db/client';
+import { type Route, modules } from '../../db/schema';
+import { setupTestDatabase } from '../../test-utils/setup-test-db';
+import {
+  type NotificationResponderHandle,
+  startNotificationResponder,
+} from './notification-responder';
+import { createPerson, createRoute } from './people';
+import { consumeDelivery, findLiveDelivery } from './tokens';
+
+const NO_SCHEMAS = defineEvents({});
+const NOW = new Date('2026-07-28T12:00:00Z');
+
+describe('notification responder', () => {
+  let dir: string;
+  let db: DbClient;
+  let busPath: string;
+  let route: Route;
+  let sent: { address: string; body: string }[];
+  let declined: { type: string; reason: string }[];
+  let sendFailures: string[];
+  let sendFails: boolean;
+  let handle: NotificationResponderHandle | undefined;
+
+  function start(hasTty = false, routes?: Route[]) {
+    handle = startNotificationResponder({
+      db,
+      busDbPath: busPath,
+      routes: routes ?? [route],
+      transportFor: () => ({
+        send: async (r) => {
+          if (sendFails) throw new Error('unlinked account cannot send');
+          sent.push({ address: r.address, body: r.body });
+          return { messageId: 'm1' };
+        },
+      }),
+      hasTty: () => hasTty,
+      // Real time, not the fixed NOW: the bus stamps `emitted_at` with the
+      // wall clock, so a fake clock in the past makes every question look
+      // like it has not been emitted yet.
+      now: () => new Date(),
+      ttlMs: 86_400_000,
+      // Most tests are about the delivery decision, not the grace — a
+      // question emitted microseconds ago is otherwise always too fresh.
+      // The grace itself gets its own test below.
+      askAfterMs: 0,
+      onDeclined: (type, reason) => declined.push({ type, reason }),
+      onSendFailed: (type, error) => sendFailures.push(`${type}: ${error}`),
+    });
+    return handle;
+  }
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'nresp-'));
+    const dbPath = join(dir, 'celilo.db');
+    busPath = join(dir, 'events.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    sent = [];
+    declined = [];
+    sendFailures = [];
+    sendFails = false;
+
+    db.insert(modules)
+      .values({
+        id: 'signal',
+        name: 'Signal',
+        version: '0.1.0',
+        manifestData: {},
+        sourcePath: '/tmp/signal',
+      })
+      .run();
+    const peter = createPerson(db, { name: 'peter', timezone: 'UTC' });
+    route = createRoute(db, {
+      personId: peter.id,
+      transportModuleId: 'signal',
+      address: '+15550001',
+      canAck: true,
+    });
+  });
+
+  afterEach(() => {
+    handle?.stop();
+    handle = undefined;
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  const emit = (type: string, payload: Record<string, unknown>) => {
+    const bus = openBus({ dbPath: busPath, events: NO_SCHEMAS });
+    try {
+      return bus.emitRaw(type, payload, { emittedBy: 'test' }).id;
+    } finally {
+      bus.close();
+    }
+  };
+
+  const repliesFor = (eventId: number) => {
+    const bus = openBus({ dbPath: busPath, events: NO_SCHEMAS });
+    try {
+      return bus.db.query('SELECT type, payload FROM events WHERE reply_for = ?').all(eventId) as {
+        type: string;
+        payload: string;
+      }[];
+    } finally {
+      bus.close();
+    }
+  };
+
+  test('a non-secret question is delivered to a route', async () => {
+    const responder = start();
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    expect(sent).toHaveLength(1);
+    expect(sent[0].address).toBe('+15550001');
+    expect(sent[0].body).toContain('caddy needs: hostname');
+    expect(sent[0].body).toMatch(/reply [0-9A-Z]{6} <value>/);
+  });
+
+  // THE security property. A secret must never reach a chat, even headlessly.
+  test('a secret question is never delivered', async () => {
+    const responder = start();
+    emit('secret.required.forgejo.admin_token', { scope: 'forgejo', key: 'admin_token' });
+    await responder.poll();
+
+    expect(sent).toEqual([]);
+    expect(declined[0].type).toBe('secret.required.forgejo.admin_token');
+    expect(declined[0].reason).toContain('terminal');
+  });
+
+  test('nothing is delivered when a terminal is attached', async () => {
+    const responder = start(true);
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    expect(sent).toEqual([]);
+    expect(declined[0].reason).toContain('terminal responder');
+  });
+
+  test('nothing is delivered when no route can receive replies', async () => {
+    const responder = start(false, []);
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    expect(sent).toEqual([]);
+    expect(declined[0].reason).toContain('never be answered');
+  });
+
+  /**
+   * Resolve a token the way the inbound poller does — through the DELIVERY
+   * row, not through anything the responder remembers. That is what lets a
+   * reply arriving minutes later, in a different process, still land.
+   */
+  function eventIdForToken(token: string): string | null {
+    const delivery = findLiveDelivery(db, token, NOW);
+    return delivery?.kind === 'interview' ? delivery.targetId : null;
+  }
+
+  const tokenIn = (body: string) => /reply ([0-9A-Z]{6})/.exec(body)?.[1] as string;
+
+  // The unification: a reply from a phone unblocks a waiting deploy.
+  test('an inbound reply publishes the answer against the waiting question', async () => {
+    const responder = start();
+    const eventId = emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    const resolved = eventIdForToken(tokenIn(sent[0].body)) as string;
+    expect(resolved).toBe(String(eventId));
+    responder.answer(resolved, 'www.example.com');
+
+    const replies = repliesFor(eventId);
+    expect(replies).toHaveLength(1);
+    expect(replies[0].type).toBe('config.required.caddy.hostname.reply');
+    expect(JSON.parse(replies[0].payload).value).toBe('www.example.com');
+  });
+
+  /**
+   * A responder that is actually attached — a terminal, or the wire bridge in
+   * api-serve — answers within seconds. Paging a phone about a question
+   * somebody is already looking at is noise, and both responders would reply
+   * to the same question.
+   */
+  test('a fresh question is left for whatever responder is attached', async () => {
+    const responder = startNotificationResponder({
+      db,
+      busDbPath: busPath,
+      routes: [route],
+      transportFor: () => ({
+        send: async (r) => {
+          sent.push({ address: r.address, body: r.body });
+          return { messageId: 'm1' };
+        },
+      }),
+      hasTty: () => false,
+      now: () => new Date(),
+      ttlMs: 86_400_000,
+      askAfterMs: 90_000,
+    });
+    handle = responder;
+
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    expect((await responder.poll()).asked).toBe(0);
+    expect(sent).toEqual([]);
+  });
+
+  /**
+   * The CLI runs one process per invocation and polls every few seconds, so
+   * "asked already" cannot live in memory — it has to be the delivery row.
+   */
+  test('a question is not re-asked by a fresh responder process', async () => {
+    const first = start();
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    expect((await first.poll()).asked).toBe(1);
+    first.stop();
+
+    const second = start();
+    expect((await second.poll()).asked).toBe(0);
+    expect(sent).toHaveLength(1);
+  });
+
+  // Found by the e2e: the delivery row is minted BEFORE the send (the token has
+  // to be in the body), so a failed send would otherwise leave a record saying
+  // the question was asked — suppressing every retry while the deploy waits
+  // forever on a question nobody received.
+  test('a failed send is reported and does NOT count as asked', async () => {
+    sendFails = true;
+    const responder = start();
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+
+    const first = await responder.poll();
+    expect(first.asked).toBe(0);
+    expect(first.failed).toBe(1);
+    expect(sendFailures[0]).toContain('config.required.caddy.hostname');
+
+    // The question is still outstanding, so a later poll retries it.
+    sendFails = false;
+    expect((await responder.poll()).asked).toBe(1);
+  });
+
+  test('a token nobody issued resolves to nothing', async () => {
+    const responder = start();
+    const eventId = emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    expect(eventIdForToken('ZZZZZZ')).toBeNull();
+    expect(repliesFor(eventId)).toEqual([]);
+  });
+
+  // Two questions outstanding must not cross-answer.
+  test('each question is answered by its own token', async () => {
+    const responder = start();
+    const first = emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+    const second = emit('config.required.forgejo.domain', { scope: 'forgejo', key: 'domain' });
+    await responder.poll();
+
+    expect(sent).toHaveLength(2);
+    responder.answer(eventIdForToken(tokenIn(sent[1].body)) as string, 'git.example.com');
+
+    expect(repliesFor(first)).toEqual([]);
+    expect(JSON.parse(repliesFor(second)[0].payload).value).toBe('git.example.com');
+  });
+
+  // Single-use lives in the token layer, which is where the alert path
+  // enforces it too — one mechanism, not two.
+  test('a consumed token cannot be replayed', async () => {
+    const responder = start();
+    emit('config.required.caddy.hostname', { scope: 'caddy', key: 'hostname' });
+    await responder.poll();
+
+    const token = tokenIn(sent[0].body);
+    const delivery = findLiveDelivery(db, token, NOW);
+    expect(delivery).toBeDefined();
+    consumeDelivery(db, (delivery as { id: string }).id, NOW);
+
+    expect(eventIdForToken(token)).toBeNull();
+    expect(responder).toBeDefined();
+  });
+});