@celilo/cli 0.13.3 → 0.14.0

package/src/services/alerting/notification-responder.ts

@@ -0,0 +1,260 @@
+/**
+ * The notification-transport interview responder.
+ *
+ * Watches `interview.required.*` (and the config/ensure/aspect families), sends
+ * the question to a route, and publishes the operator's reply back to the
+ * waiting deploy via `bus.reply`.
+ *
+ * QUERIES the events table rather than using `bus.watch`, for two reasons —
+ * neither of which is that watch cannot see other processes. (It can: each
+ * watch runs a 250ms polling loop alongside its in-process callback.)
+ *
+ *   1. A watch seeds its high-water mark to MAX(id) at registration, so it
+ *      only ever sees events emitted AFTER it starts. A deploy that raised its
+ *      question thirty seconds before this poll fires would be invisible —
+ *      and that is the normal case, not the edge one.
+ *   2. `celilo alerts poll` is a one-shot process. A watch needs the process
+ *      to stay alive to be worth anything; this one exits immediately.
+ *
+ * So "what is still unanswered" has to be a question asked of the table, not a
+ * subscription to the future.
+ *
+ * The "ask" takes minutes on someone's phone rather than milliseconds on a
+ * TTY. That is fine: the interview flow has no timeout by design ("the deploy
+ * waits indefinitely for a responder"), which is what makes a phone a viable
+ * answering device at all.
+ *
+ * Additive, never exclusive: a terminal responder wins when stdin is a TTY, so
+ * a deploy started from a laptop is answered there rather than appearing to
+ * hang while waiting for a text.
+ */
+
+import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
+import { and, eq, gt, isNull } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type Route, notificationDeliveries } from '../../db/schema';
+import {
+  composeInterviewBody,
+  decideInterviewDelivery,
+  describeQuestion,
+} from './interview-responder';
+import type { NotificationTransport } from './notifier';
+import { consumeDelivery, mintDelivery } from './tokens';
+
+const NO_SCHEMAS = defineEvents({});
+
+/**
+ * How long a question must go unanswered before it is worth someone's phone.
+ *
+ * An attached responder — a terminal, or the wire bridge in `api-serve` —
+ * answers within seconds. Paging a phone about a question somebody is already
+ * looking at is noise, and worse, both responders would reply to the same
+ * question. Waiting lets the responder that is actually present win, and only
+ * escalates to a messaging transport when nobody is there.
+ *
+ * This is the same rule as "a terminal wins when stdin is a TTY" (D12), applied
+ * to responders this process cannot see.
+ */
+const ASK_AFTER_MS = 90_000;
+
+/** Families a deploy can block on. `secret.*` is watched so it can be REFUSED. */
+const INTERVIEW_PATTERNS = [
+  'interview.required.*.*',
+  'config.required.*.*',
+  'ensure.required.*.*',
+  'aspect.required.*.*',
+  'secret.required.*.*',
+] as const;
+
+export interface NotificationResponderOptions {
+  db: DbClient;
+  busDbPath: string;
+  /** Ack-capable routes, in the order questions should be offered. */
+  routes: Route[];
+  transportFor(route: Route): NotificationTransport;
+  /** True when a terminal responder is available (D12). */
+  hasTty(): boolean;
+  now(): Date;
+  ttlMs: number;
+  /**
+   * Grace before a question is escalated to a messaging transport, so a
+   * responder that is actually attached answers first. Defaults to 90s.
+   */
+  askAfterMs?: number;
+  /** Called when a question is declined, so the reason is not lost. */
+  onDeclined?(eventType: string, reason: string): void;
+  /** Called when a question could not be DELIVERED — never silent. */
+  onSendFailed?(eventType: string, error: string): void;
+}
+
+export interface AskReport {
+  /** Questions delivered to someone who can answer them. */
+  asked: number;
+  /**
+   * Questions whose delivery FAILED. Counted separately and never silently:
+   * a deploy blocked on a question nobody received looks identical to a deploy
+   * that is merely slow.
+   */
+  failed: number;
+}
+
+export interface NotificationResponderHandle {
+  stop(): void;
+  /** Deliver any unanswered questions. */
+  poll(): Promise<AskReport>;
+  /**
+   * Publish an answer against a waiting question.
+   *
+   * Takes the bus event id from the DELIVERY rather than from memory, so a
+   * reply that arrives in a later process still lands — the CLI runs one
+   * command per invocation, and an in-memory pending list would not survive
+   * the gap between asking and being answered.
+   */
+  answer(eventId: string, value: string): void;
+}
+
+interface PendingRow {
+  id: number;
+  type: string;
+  payload: string;
+}
+
+/**
+ * Attach the responder to the bus.
+ *
+ * Holds no state of its own. What has been asked is recorded as a delivery row
+ * (`kind: 'interview'`, `targetId` = the bus event id), which is also what the
+ * inbound path looks a reply up by — so asking and answering can happen in
+ * different processes, minutes apart, which is the normal case.
+ */
+export function startNotificationResponder(
+  opts: NotificationResponderOptions,
+): NotificationResponderHandle {
+  const bus: Bus = openBus({ dbPath: opts.busDbPath, events: NO_SCHEMAS });
+  const askAfterMs = opts.askAfterMs ?? ASK_AFTER_MS;
+
+  /**
+   * Bus events already asked about, and still awaiting a reply.
+   *
+   * Read from the deliveries table rather than kept in memory. The CLI runs
+   * one command per invocation and the poll runs every few seconds, so an
+   * in-memory guard would re-send the same question forever. A delivery that
+   * has EXPIRED is deliberately absent here — nobody answered in a day, so
+   * asking again is the right move.
+   */
+  function alreadyAsked(now: Date): Set<string> {
+    return new Set(
+      opts.db
+        .select({ targetId: notificationDeliveries.targetId })
+        .from(notificationDeliveries)
+        .where(
+          and(
+            eq(notificationDeliveries.kind, 'interview'),
+            isNull(notificationDeliveries.consumedAt),
+            gt(notificationDeliveries.expiresAt, now),
+          ),
+        )
+        .all()
+        .map((row) => row.targetId),
+    );
+  }
+
+  /**
+   * Questions with no reply yet.
+   *
+   * `reply_for` on a reply event points at the question, so "unanswered" is
+   * simply the absence of such a row. Matching on the family prefixes keeps
+   * this from picking up unrelated traffic.
+   */
+  function unanswered(now: Date): PendingRow[] {
+    return bus.db
+      .query(
+        `SELECT id, type, payload FROM events
+          WHERE reply_for IS NULL
+            AND emitted_at <= ?
+            AND (${INTERVIEW_PATTERNS.map(() => 'type LIKE ?').join(' OR ')})
+            AND NOT EXISTS (SELECT 1 FROM events r WHERE r.reply_for = events.id)
+          ORDER BY id ASC`,
+      )
+      .all(
+        now.getTime() - askAfterMs,
+        ...INTERVIEW_PATTERNS.map((p) => `${p.replace(/\*\.\*$/, '')}%`),
+      ) as PendingRow[];
+  }
+
+  return {
+    stop() {
+      bus.close();
+    },
+
+    async poll(): Promise<AskReport> {
+      const now = opts.now();
+      const asked = alreadyAsked(now);
+      const report: AskReport = { asked: 0, failed: 0 };
+      for (const row of unanswered(now)) {
+        if (asked.has(String(row.id))) continue;
+
+        const decision = decideInterviewDelivery({
+          eventType: row.type,
+          hasTty: opts.hasTty(),
+          hasBidirectionalRoute: opts.routes.length > 0,
+        });
+
+        if (!decision.deliver) {
+          // No delivery row is written for a declined question, so this fires
+          // once per poll. That is intentional for `secret.*`: the operator
+          // needs to keep seeing why their deploy is stuck.
+          opts.onDeclined?.(row.type, decision.reason ?? 'declined');
+          continue;
+        }
+
+        const route = opts.routes[0];
+        const question = describeQuestion(
+          row.type,
+          JSON.parse(row.payload) as Record<string, unknown>,
+        );
+
+        const delivery = mintDelivery(opts.db, {
+          kind: 'interview',
+          // A BUS event id, not an alerts row — the two share this table and
+          // differ only in what targetId points at.
+          targetId: String(row.id),
+          routeId: route.id,
+          now,
+          ttlMs: opts.ttlMs,
+        });
+
+        try {
+          await opts.transportFor(route).send({
+            address: route.address,
+            body: composeInterviewBody({ ...question, token: delivery.token }),
+          });
+          report.asked++;
+        } catch (error) {
+          // The delivery row is what records "this was asked", and it was
+          // minted BEFORE the send because the token has to be in the body.
+          // A send that failed must not leave that record behind: it would
+          // suppress every retry, and the deploy would wait forever on a
+          // question nobody ever received.
+          consumeDelivery(opts.db, delivery.id, now);
+          report.failed++;
+          opts.onSendFailed?.(row.type, error instanceof Error ? error.message : String(error));
+        }
+      }
+      return report;
+    },
+
+    answer(eventId: string, value: string): void {
+      const question = bus.db
+        .query('SELECT type FROM events WHERE id = ?')
+        .get(Number(eventId)) as { type: string } | null;
+      if (!question) return;
+
+      bus.emitRaw(
+        `${question.type}.reply`,
+        { value },
+        { replyFor: Number(eventId), emittedBy: 'celilo-notification-responder' },
+      );
+    },
+  };
+}

package/src/services/alerting/notifier.ts

@@ -0,0 +1,219 @@
+/**
+ * The notifier — turning a firing alert into a message someone receives.
+ *
+ * This is the seam where every earlier decision finally has consequences:
+ * escalation says whether and whom, quiet hours say when, suppression says
+ * nothing at all. The transport is injected so the whole loop can be driven
+ * against a simulator, or against nothing.
+ *
+ * Composition, not new policy: this function makes no decisions of its own. If
+ * it looks like it is deciding something, that logic belongs in escalation.ts
+ * or quiet-hours.ts where it can be tested without a database.
+ */
+
+import type { Alert, Route } from '../../db/schema';
+import { type EscalationStep, type RouteForEscalation, decideEscalation } from './escalation';
+import { type QuietHoursWindow, isWithinQuietHours, quietHoursEndAfter } from './quiet-hours';
+
+export interface NotificationTransport {
+  send(request: { address: string; body: string; token?: string }): Promise<{ messageId: string }>;
+}
+
+export interface PersonWindow extends QuietHoursWindow {
+  personId: string;
+}
+
+export interface NotifyDeps {
+  /** Steps of the alert's escalation policy, in order. */
+  steps: EscalationStep[];
+  routes: Map<string, RouteForEscalation>;
+  /** Full route rows, for address and person lookup. */
+  routeDetails: Map<string, Route>;
+  quietHoursByPerson: Map<string, PersonWindow>;
+  /** Whether the alert's policy is allowed to page through quiet hours. */
+  bypassQuietHours: boolean;
+  transportFor(route: Route): NotificationTransport;
+  mintToken(alertId: string, routeId: string): string | null;
+  now: Date;
+}
+
+export type NotifyOutcome =
+  | {
+      result: 'sent';
+      routeId: string;
+      stepIndex: number;
+      messageId: string;
+      nextStepDueAt: Date | null;
+    }
+  | { result: 'deferred'; routeId: string; until: Date; reason: 'quiet_hours' }
+  | { result: 'skipped'; reason: string }
+  | { result: 'failed'; routeId: string; error: string };
+
+/**
+ * Format the message body.
+ *
+ * Key first, because on a phone the first line is what shows in the
+ * notification — an operator half-awake needs to know WHAT before WHY.
+ */
+export function composeAlertBody(alert: Pick<Alert, 'key' | 'severity' | 'message'>): string {
+  const marker = alert.severity === 'critical' ? '🔴' : '⚠️';
+  return `${marker} ${alert.key}\n${alert.message}`;
+}
+
+export function composeResolvedBody(alert: Pick<Alert, 'key'>): string {
+  return `✅ RESOLVED — ${alert.key}`;
+}
+
+/**
+ * Tell the others that someone has it.
+ *
+ * Names the acknowledger rather than just saying "acknowledged": the question
+ * a woken secondary actually has is "do I still need to get up", and only a
+ * name answers it.
+ */
+export function composeAckBroadcastBody(alert: Pick<Alert, 'key'>, ackedBy: string): string {
+  return `👍 ${ackedBy} has ${alert.key} — no action needed.`;
+}
+
+/**
+ * Notify for one alert, if it is due.
+ *
+ * Returns what happened rather than throwing, because "nobody was told, and
+ * here is why" is information the caller must be able to record — a silent
+ * skip is indistinguishable from a bug.
+ */
+export async function notifyAlert(alert: Alert, deps: NotifyDeps): Promise<NotifyOutcome> {
+  const decision = decideEscalation({
+    alert: {
+      id: alert.id,
+      severity: alert.severity,
+      escalationStep: alert.escalationStep,
+      acked: alert.state === 'acked',
+      resolved: alert.state === 'resolved',
+      suppressed: alert.state === 'suppressed',
+      awaitingConfirmation: alert.awaitingConfirmation,
+      silencedUntil: alert.silencedUntil,
+      graceUntil: alert.graceUntil,
+      escalationStartedAt: alert.unsuppressedAt ?? alert.firstFiredAt,
+    },
+    steps: deps.steps,
+    routes: deps.routes,
+    now: deps.now,
+  });
+
+  if (decision.type === 'skip') return { result: 'skipped', reason: decision.reason };
+
+  const route = deps.routeDetails.get(decision.routeId);
+  if (!route) return { result: 'skipped', reason: 'no_eligible_route' };
+
+  // Quiet hours defer delivery of EVERY severity (D13). The escalation step is
+  // still considered taken — steps advance on schedule and only delivery waits,
+  // so nothing is skipped, it arrives when the window opens.
+  if (!deps.bypassQuietHours) {
+    const window = deps.quietHoursByPerson.get(route.personId);
+    if (window && isWithinQuietHours(window, deps.now)) {
+      const until = quietHoursEndAfter(window, deps.now);
+      if (until) return { result: 'deferred', routeId: route.id, until, reason: 'quiet_hours' };
+    }
+  }
+
+  // A route that cannot receive replies gets no token — a reply instruction
+  // nobody can follow is worse than none.
+  const token = route.canAck ? (deps.mintToken(alert.id, route.id) ?? undefined) : undefined;
+
+  try {
+    const { messageId } = await deps.transportFor(route).send({
+      address: route.address,
+      body: composeAlertBody(alert),
+      token,
+    });
+    return {
+      result: 'sent',
+      routeId: route.id,
+      stepIndex: decision.stepIndex,
+      messageId,
+      nextStepDueAt: decision.nextStepDueAt,
+    };
+  } catch (error) {
+    // A transport failure must not be silent: the operator believes they are
+    // covered, and the only evidence otherwise is this record.
+    return {
+      result: 'failed',
+      routeId: route.id,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+/**
+ * Deliver a message held over someone's quiet hours.
+ *
+ * The state re-check is the point of doing this at window end rather than
+ * scheduling the send in advance: an alert that resolved at 03:00 must not
+ * arrive at 07:00 announcing a problem that no longer exists. Ack and silence
+ * count too — someone who dealt with it overnight has already been told.
+ */
+export async function deliverDeferred(
+  alert: Alert,
+  route: Route,
+  deps: Pick<NotifyDeps, 'transportFor' | 'mintToken' | 'now'>,
+): Promise<NotifyOutcome> {
+  if (alert.state === 'resolved') return { result: 'skipped', reason: 'resolved' };
+  if (alert.state === 'acked') return { result: 'skipped', reason: 'acked' };
+  if (alert.state === 'suppressed') return { result: 'skipped', reason: 'suppressed' };
+  if (alert.silencedUntil && alert.silencedUntil > deps.now) {
+    return { result: 'skipped', reason: 'silenced' };
+  }
+
+  const token = route.canAck ? (deps.mintToken(alert.id, route.id) ?? undefined) : undefined;
+  try {
+    const { messageId } = await deps.transportFor(route).send({
+      address: route.address,
+      body: composeAlertBody(alert),
+      token,
+    });
+    return {
+      result: 'sent',
+      routeId: route.id,
+      stepIndex: alert.escalationStep,
+      messageId,
+      nextStepDueAt: alert.nextEscalationAt,
+    };
+  } catch (error) {
+    return {
+      result: 'failed',
+      routeId: route.id,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+/**
+ * Send an all-clear to every route that was told about the alert.
+ *
+ * Only to those routes: someone who was never paged does not need to be told
+ * it is over, and telling them trains them to ignore the channel.
+ */
+export async function notifyResolved(
+  alert: Pick<Alert, 'id' | 'key'>,
+  notifiedRoutes: Route[],
+  deps: Pick<NotifyDeps, 'transportFor'>,
+): Promise<{ routeId: string; ok: boolean; error?: string }[]> {
+  const results: { routeId: string; ok: boolean; error?: string }[] = [];
+  for (const route of notifiedRoutes) {
+    try {
+      await deps.transportFor(route).send({
+        address: route.address,
+        body: composeResolvedBody(alert),
+      });
+      results.push({ routeId: route.id, ok: true });
+    } catch (error) {
+      results.push({
+        routeId: route.id,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  return results;
+}

package/src/services/alerting/people.ts

@@ -0,0 +1,178 @@
+/**
+ * People, routes, and escalation policies — who celilo can reach, how, and in
+ * what order.
+ *
+ * The three are deliberately separate tables (design D8):
+ *
+ *   person  who they are, and when they may be disturbed
+ *   route   a person's address ON a transport, plus the policy for using it
+ *   policy  an ordered list of routes with delays
+ *
+ * Recipient addresses live on ROUTES, never in a transport module's config.
+ * signal-cli holds exactly one credential — its own registration — while
+ * recipients are just addresses. Putting them in module config would make
+ * adding a person a module redeploy.
+ *
+ * Escalation steps reference routes rather than people, because "page Peter"
+ * is ambiguous about which transport to use.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { and, asc, eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import {
+  type AlertSeverity,
+  type EscalationPolicy,
+  type EscalationStep,
+  type Person,
+  type Route,
+  escalationPolicies,
+  escalationSteps,
+  people,
+  routes,
+} from '../../db/schema';
+
+// ── People ──────────────────────────────────────────────────────────────────
+
+export interface CreatePersonInput {
+  name: string;
+  timezone: string;
+  quietHoursStart?: string | null;
+  quietHoursEnd?: string | null;
+}
+
+export function listPeople(db: DbClient): Person[] {
+  return db.select().from(people).orderBy(asc(people.name)).all();
+}
+
+export function findPerson(db: DbClient, name: string): Person | undefined {
+  return db.select().from(people).where(eq(people.name, name)).get();
+}
+
+export function createPerson(db: DbClient, input: CreatePersonInput): Person {
+  const id = randomUUID();
+  db.insert(people)
+    .values({
+      id,
+      name: input.name,
+      timezone: input.timezone,
+      quietHoursStart: input.quietHoursStart ?? null,
+      quietHoursEnd: input.quietHoursEnd ?? null,
+    })
+    .run();
+  return db.select().from(people).where(eq(people.id, id)).get() as Person;
+}
+
+/**
+ * Remove a person and everything addressed to them.
+ *
+ * Routes cascade in the schema. Escalation steps referencing those routes
+ * cascade too — a policy step pointing at a deleted route would otherwise
+ * silently skip at page time, which is the worst moment to discover it.
+ */
+export function deletePerson(db: DbClient, personId: string): void {
+  db.delete(people).where(eq(people.id, personId)).run();
+}
+
+// ── Routes ──────────────────────────────────────────────────────────────────
+
+export interface CreateRouteInput {
+  personId: string;
+  transportModuleId: string;
+  address: string;
+  severityFloor?: AlertSeverity;
+  /** Whether replies can arrive back over this transport. */
+  canAck?: boolean;
+}
+
+export function listRoutes(db: DbClient): Route[] {
+  return db.select().from(routes).all();
+}
+
+export function listRoutesForPerson(db: DbClient, personId: string): Route[] {
+  return db.select().from(routes).where(eq(routes.personId, personId)).all();
+}
+
+export function findRoute(
+  db: DbClient,
+  personId: string,
+  transportModuleId: string,
+): Route | undefined {
+  return db
+    .select()
+    .from(routes)
+    .where(and(eq(routes.personId, personId), eq(routes.transportModuleId, transportModuleId)))
+    .get();
+}
+
+export function createRoute(db: DbClient, input: CreateRouteInput): Route {
+  const id = randomUUID();
+  db.insert(routes)
+    .values({
+      id,
+      personId: input.personId,
+      transportModuleId: input.transportModuleId,
+      address: input.address,
+      severityFloor: input.severityFloor ?? 'warning',
+      canAck: input.canAck ?? false,
+    })
+    .run();
+  return db.select().from(routes).where(eq(routes.id, id)).get() as Route;
+}
+
+export function deleteRoute(db: DbClient, routeId: string): void {
+  db.delete(routes).where(eq(routes.id, routeId)).run();
+}
+
+// ── Escalation policies ─────────────────────────────────────────────────────
+
+export function listPolicies(db: DbClient): EscalationPolicy[] {
+  return db.select().from(escalationPolicies).orderBy(asc(escalationPolicies.name)).all();
+}
+
+export function findPolicy(db: DbClient, name: string): EscalationPolicy | undefined {
+  return db.select().from(escalationPolicies).where(eq(escalationPolicies.name, name)).get();
+}
+
+export function createPolicy(db: DbClient, name: string): EscalationPolicy {
+  const id = randomUUID();
+  db.insert(escalationPolicies).values({ id, name }).run();
+  return db
+    .select()
+    .from(escalationPolicies)
+    .where(eq(escalationPolicies.id, id))
+    .get() as EscalationPolicy;
+}
+
+/** Steps of a policy, in the order escalation walks them. */
+export function listPolicySteps(db: DbClient, policyId: string): EscalationStep[] {
+  return db
+    .select()
+    .from(escalationSteps)
+    .where(eq(escalationSteps.policyId, policyId))
+    .orderBy(asc(escalationSteps.stepIndex))
+    .all();
+}
+
+/**
+ * Append a step.
+ *
+ * The index is assigned rather than supplied: an operator adding steps in order
+ * should not have to think about numbering, and a hand-picked index that
+ * collided would violate the composite primary key at the least helpful moment.
+ */
+export function addPolicyStep(
+  db: DbClient,
+  policyId: string,
+  routeId: string,
+  delayMinutes: number,
+): EscalationStep {
+  const existing = listPolicySteps(db, policyId);
+  const stepIndex = existing.length === 0 ? 0 : existing[existing.length - 1].stepIndex + 1;
+  db.insert(escalationSteps).values({ policyId, stepIndex, routeId, delayMinutes }).run();
+  return { policyId, stepIndex, routeId, delayMinutes };
+}
+
+export function deletePolicy(db: DbClient, policyId: string): void {
+  db.delete(escalationPolicies).where(eq(escalationPolicies.id, policyId)).run();
+}