@celilo/cli 0.13.3 → 0.14.1

package/src/services/alerting/interview-responder.test.ts

@@ -0,0 +1,169 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  composeInterviewBody,
+  decideInterviewDelivery,
+  describeQuestion,
+  isRefusedFamily,
+  parseInterviewAnswer,
+} from './interview-responder';
+
+const headless = { hasTty: false, hasBidirectionalRoute: true };
+
+describe('isRefusedFamily', () => {
+  test.each(['secret.required.namecheap.ddns_passwords', 'secret.required.forgejo.admin_token'])(
+    '%p is refused',
+    (type) => {
+      expect(isRefusedFamily(type)).toBe(true);
+    },
+  );
+
+  test.each([
+    'config.required.caddy.hostname',
+    'ensure.required.namecheap.managed_domain',
+    'interview.required.service:proxmox.node',
+    'aspect.required.technitium.dns',
+  ])('%p is not refused', (type) => {
+    expect(isRefusedFamily(type)).toBe(false);
+  });
+});
+
+describe('decideInterviewDelivery', () => {
+  // The refusal is checked FIRST, so a secret is declined even on a headless
+  // box with a working transport. The deploy waits for a terminal, which is
+  // correct — the alternative is leaking a credential for convenience.
+  test('a secret question is refused even when nothing else could answer it', () => {
+    const decision = decideInterviewDelivery({
+      eventType: 'secret.required.forgejo.admin_token',
+      ...headless,
+    });
+    expect(decision.deliver).toBe(false);
+    expect(decision.reason).toContain('terminal');
+  });
+
+  test('a secret question is refused with a TTY too', () => {
+    const decision = decideInterviewDelivery({
+      eventType: 'secret.required.forgejo.admin_token',
+      hasTty: true,
+      hasBidirectionalRoute: true,
+    });
+    expect(decision.deliver).toBe(false);
+  });
+
+  // Without this a deploy started from a laptop appears to hang while
+  // silently waiting for a text message.
+  test('a terminal wins when one is attached', () => {
+    const decision = decideInterviewDelivery({
+      eventType: 'config.required.caddy.hostname',
+      hasTty: true,
+      hasBidirectionalRoute: true,
+    });
+    expect(decision.deliver).toBe(false);
+    expect(decision.reason).toContain('terminal responder');
+  });
+
+  test('a non-secret question is delivered headlessly', () => {
+    expect(
+      decideInterviewDelivery({ eventType: 'config.required.caddy.hostname', ...headless }),
+    ).toEqual({ deliver: true });
+  });
+
+  // Delivering to a route that cannot reply produces a question nobody can
+  // answer, which reads to the operator as the deploy having hung.
+  test('nothing is delivered when no route can receive a reply', () => {
+    const decision = decideInterviewDelivery({
+      eventType: 'config.required.caddy.hostname',
+      hasTty: false,
+      hasBidirectionalRoute: false,
+    });
+    expect(decision.deliver).toBe(false);
+    expect(decision.reason).toContain('never be answered');
+  });
+});
+
+describe('composeInterviewBody', () => {
+  test('leads with the question, ends with a bare reply instruction', () => {
+    const body = composeInterviewBody({
+      scope: 'caddy',
+      key: 'hostname',
+      description: 'Public hostname for the site',
+      token: 'K7QM2X',
+    });
+    expect(body.split('\n')[0]).toContain('caddy needs: hostname');
+    expect(body.endsWith('reply K7QM2X <value>')).toBe(true);
+  });
+
+  test('works without a description', () => {
+    const body = composeInterviewBody({ scope: 'caddy', key: 'hostname', token: 'K7QM2X' });
+    expect(body).toContain('caddy needs: hostname');
+  });
+});
+
+describe('parseInterviewAnswer', () => {
+  test('takes everything after the token as the answer', () => {
+    expect(parseInterviewAnswer('K7QM2X www.example.com', 'K7QM2X')).toBe('www.example.com');
+  });
+
+  test('is case-insensitive on the token', () => {
+    expect(parseInterviewAnswer('k7qm2x www.example.com', 'K7QM2X')).toBe('www.example.com');
+  });
+
+  // An answer may legitimately contain spaces and punctuation. Second-guessing
+  // it would corrupt exactly the values that are painful to retype.
+  test('preserves an answer containing spaces and punctuation verbatim', () => {
+    expect(parseInterviewAnswer('K7QM2X my value: with, punctuation', 'K7QM2X')).toBe(
+      'my value: with, punctuation',
+    );
+  });
+
+  test('an answer that looks like a verb is still an answer', () => {
+    expect(parseInterviewAnswer('K7QM2X resolve', 'K7QM2X')).toBe('resolve');
+  });
+
+  test('a token with no value is not an answer', () => {
+    expect(parseInterviewAnswer('K7QM2X', 'K7QM2X')).toBeNull();
+    expect(parseInterviewAnswer('K7QM2X   ', 'K7QM2X')).toBeNull();
+  });
+
+  test('a different token does not match', () => {
+    expect(parseInterviewAnswer('ZZZZZZ value', 'K7QM2X')).toBeNull();
+  });
+});
+
+/**
+ * The families disagree on payload shape, and reading the wrong field means an
+ * operator gets the raw event type instead of the question.
+ */
+describe('describeQuestion', () => {
+  test("uses the interview family's message — it IS the question", () => {
+    const q = describeQuestion('interview.required.service.node', {
+      scope: 'service:proxmox-home-lab',
+      key: 'default_target_node',
+      message: 'Which Proxmox node should new containers land on?',
+    });
+
+    expect(q.scope).toBe('service:proxmox-home-lab');
+    expect(q.key).toBe('Which Proxmox node should new containers land on?');
+  });
+
+  // config.required carries `module`, not `scope`, and has no prose at all.
+  test('composes from module + key when there is no message', () => {
+    const q = describeQuestion('config.required.caddy.acme_email', {
+      module: 'caddy',
+      key: 'acme_email',
+      description: 'Address ACME registers with',
+      type: 'string',
+      required: true,
+    });
+
+    expect(q.scope).toBe('caddy');
+    expect(q.key).toBe('acme_email');
+    expect(q.description).toBe('Address ACME registers with');
+  });
+
+  // Never render "undefined needs: value" at 3am.
+  test('falls back to the event type rather than undefined', () => {
+    const q = describeQuestion('ensure.required.forgejo.runner', {});
+    expect(q.scope).toBe('ensure.required.forgejo.runner');
+    expect(q.key).toBe('value');
+  });
+});

package/src/services/alerting/interview-responder.ts

@@ -0,0 +1,158 @@
+/**
+ * Answering deploy interviews over a notification transport.
+ *
+ * The payoff the whole design was shaped around: because an alert and an
+ * interview question are the same message with different reply semantics, one
+ * transport gives both. The send path, the token table, and the inbound path
+ * are shared; only what `targetId` points at differs.
+ *
+ *   alert     targetId -> alerts.id           reply acknowledges
+ *   interview targetId -> a BUS event id      reply IS the answer
+ *
+ * Two rules here are safety properties rather than preferences, and both are
+ * enforced before anything is delivered.
+ */
+
+import type { AlertSeverity } from '../../db/schema';
+
+/**
+ * Interview families a messaging transport must never carry.
+ *
+ * `headless-cli-interview` already requires that secrets never travel in a
+ * reply payload — a responder writes plaintext out-of-band to the encrypted
+ * store and replies only `{ acknowledged: true }`. A messaging transport
+ * cannot satisfy that: the operator would have to type the secret into a chat,
+ * putting it in the transport's message store, the device's history, and every
+ * other route on the policy, all before the responder ever sees it. The
+ * out-of-band write is meaningless when the inbound channel is itself the leak.
+ *
+ * So this is compliance with an existing requirement, not a new rule. See
+ * design D11.
+ */
+const REFUSED_PREFIXES = ['secret.required.'] as const;
+
+export function isRefusedFamily(eventType: string): boolean {
+  return REFUSED_PREFIXES.some((prefix) => eventType.startsWith(prefix));
+}
+
+export interface InterviewDeliveryDecision {
+  deliver: boolean;
+  /** Why not, when `deliver` is false — shown to the operator. */
+  reason?: string;
+}
+
+export interface InterviewContext {
+  eventType: string;
+  /** True when a terminal responder is available to answer instead. */
+  hasTty: boolean;
+  /** True when at least one ack-capable route exists. */
+  hasBidirectionalRoute: boolean;
+}
+
+/**
+ * Whether a notification transport should carry this interview question.
+ *
+ * Order matters: the refusal is checked FIRST, so a secret question is
+ * declined even on a headless box with a perfectly good transport. Declining
+ * it there leaves the deploy waiting for a terminal, which is correct — the
+ * alternative is leaking a credential to make a deploy convenient.
+ */
+export function decideInterviewDelivery(context: InterviewContext): InterviewDeliveryDecision {
+  if (isRefusedFamily(context.eventType)) {
+    return {
+      deliver: false,
+      reason:
+        'This question asks for a secret, which must not travel over a messaging transport. ' +
+        'Answer it at a terminal — see `celilo events list-pending`.',
+    };
+  }
+
+  // Terminal wins when someone is sitting there (D12). Without this, a deploy
+  // started from a laptop appears to hang while silently waiting for a text.
+  if (context.hasTty) {
+    return { deliver: false, reason: 'A terminal responder is attached and will answer.' };
+  }
+
+  if (!context.hasBidirectionalRoute) {
+    return {
+      deliver: false,
+      reason: 'No route can receive replies, so a delivered question could never be answered.',
+    };
+  }
+
+  return { deliver: true };
+}
+
+/**
+ * The interview families do not share a payload shape, and the differences
+ * matter for what a human sees on a phone:
+ *
+ *   interview.required.*   { scope, key, message, ... }   `message` IS the question
+ *   config.required.*      { module, key, type, ... }     no prose; compose from module+key
+ *
+ * Reading only `scope`/`key` drops the actual question text on the family that
+ * has it, and renders the raw event type on the family that does not. Neither
+ * is something you want to read at 3am.
+ */
+export function describeQuestion(
+  eventType: string,
+  payload: Record<string, unknown>,
+): { scope: string; key: string; description?: string } {
+  const key = typeof payload.key === 'string' ? payload.key : 'value';
+  const description = typeof payload.description === 'string' ? payload.description : undefined;
+
+  // The interview family's `message` is the operator-facing question — prefer
+  // it over anything we would compose ourselves.
+  if (typeof payload.message === 'string' && payload.message.length > 0) {
+    const scope = typeof payload.scope === 'string' ? payload.scope : eventType;
+    return { scope, key: payload.message, description };
+  }
+
+  const owner =
+    typeof payload.module === 'string'
+      ? payload.module
+      : typeof payload.scope === 'string'
+        ? payload.scope
+        : eventType;
+  return { scope: owner, key, description };
+}
+
+/**
+ * Render an interview question for a phone.
+ *
+ * The question leads, because that is the notification preview. The token
+ * instruction is last and bare — someone answering a deploy prompt from a
+ * phone should not have to parse a menu.
+ */
+export function composeInterviewBody(input: {
+  scope: string;
+  key: string;
+  description?: string;
+  token: string;
+}): string {
+  const lines = [`❓ ${input.scope} needs: ${input.key}`];
+  if (input.description) lines.push(input.description);
+  lines.push('', `reply ${input.token} <value>`);
+  return lines.join('\n');
+}
+
+/**
+ * Extract the answer from an inbound reply.
+ *
+ * Everything after the token is the value, verbatim and unparsed — an answer
+ * may legitimately contain spaces, punctuation, or something that looks like a
+ * verb, and second-guessing it would corrupt exactly the values an operator
+ * cannot easily retype.
+ */
+export function parseInterviewAnswer(body: string, token: string): string | null {
+  const trimmed = body.trim();
+  const upper = trimmed.toUpperCase();
+  const normalisedToken = token.toUpperCase();
+  if (!upper.startsWith(normalisedToken)) return null;
+
+  const answer = trimmed.slice(token.length).trim();
+  return answer.length > 0 ? answer : null;
+}
+
+/** Severity an interview question is delivered at — never a page-worthy one. */
+export const INTERVIEW_SEVERITY: AlertSeverity = 'warning';

package/src/services/alerting/keys.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  builtinAlertKey,
+  failingKeysFromHealthItems,
+  moduleAlertKey,
+  moduleCheckAlertKey,
+  parentModuleKey,
+  parseAlertKey,
+  severityForItemStatus,
+} from './keys';
+
+describe('alert key construction', () => {
+  test('module-level key', () => {
+    expect(moduleAlertKey('caddy')).toBe('module:caddy');
+  });
+
+  test('module check key', () => {
+    expect(moduleCheckAlertKey('caddy', 'cert-validity')).toBe('module:caddy/check:cert-validity');
+  });
+
+  test('builtin key', () => {
+    expect(builtinAlertKey('machines_reachable', 'machine', 'iot')).toBe(
+      'builtin:machines_reachable/machine:iot',
+    );
+  });
+});
+
+describe('parseAlertKey', () => {
+  test('round-trips a module-level key', () => {
+    expect(parseAlertKey('module:caddy')).toEqual({ source: 'module', moduleId: 'caddy' });
+  });
+
+  test('round-trips a module check key', () => {
+    expect(parseAlertKey('module:caddy/check:cert-validity')).toEqual({
+      source: 'module',
+      moduleId: 'caddy',
+      check: 'cert-validity',
+    });
+  });
+
+  test('round-trips a builtin key', () => {
+    expect(parseAlertKey('builtin:machines_reachable/machine:iot')).toEqual({
+      source: 'builtin',
+      check: 'machines_reachable',
+      targetKind: 'machine',
+      targetId: 'iot',
+    });
+  });
+
+  // Check ITEM names are module-authored and unconstrained. A key must survive
+  // an author choosing punctuation, or alert identity silently corrupts.
+  test('check name containing a colon survives', () => {
+    const key = moduleCheckAlertKey('caddy', 'tls:handshake');
+    expect(parseAlertKey(key)).toEqual({
+      source: 'module',
+      moduleId: 'caddy',
+      check: 'tls:handshake',
+    });
+  });
+
+  test('check name containing a slash survives', () => {
+    const key = moduleCheckAlertKey('caddy', 'disk:/var');
+    expect(parseAlertKey(key)).toEqual({
+      source: 'module',
+      moduleId: 'caddy',
+      check: 'disk:/var',
+    });
+  });
+
+  // The bare form is the built-in counterpart of the module-level key:
+  // "the check itself could not run", so it has no target.
+  test('a bare builtin key parses as the monitor-level form', () => {
+    expect(parseAlertKey('builtin:machines_reachable')).toEqual({
+      source: 'builtin',
+      check: 'machines_reachable',
+    });
+  });
+
+  test.each([
+    ['', 'empty'],
+    ['caddy', 'no source prefix'],
+    ['module:', 'empty module id'],
+    ['module:caddy/check:', 'empty check name'],
+    ['builtin:', 'empty check name'],
+    ['builtin:machines_reachable/iot', 'builtin target without a kind'],
+    ['unknown:thing', 'unrecognized source'],
+  ])('rejects %p (%s)', (key) => {
+    expect(parseAlertKey(key)).toBeNull();
+  });
+});
+
+describe('parentModuleKey', () => {
+  test('an item key is owned by its module key', () => {
+    expect(parentModuleKey('module:caddy/check:cert-validity')).toBe('module:caddy');
+  });
+
+  test('a module-level key has no parent', () => {
+    expect(parentModuleKey('module:caddy')).toBeNull();
+  });
+
+  test('a builtin key has no module parent', () => {
+    expect(parentModuleKey('builtin:machines_reachable/machine:iot')).toBeNull();
+  });
+});
+
+describe('severityForItemStatus', () => {
+  test('pass produces no alert', () => {
+    expect(severityForItemStatus('pass', 'critical')).toBeNull();
+  });
+
+  test('warn is always warning, never the monitor severity', () => {
+    expect(severityForItemStatus('warn', 'critical')).toBe('warning');
+  });
+
+  test('fail takes the monitor severity', () => {
+    expect(severityForItemStatus('fail', 'critical')).toBe('critical');
+    expect(severityForItemStatus('fail', 'warning')).toBe('warning');
+  });
+});
+
+describe('failingKeysFromHealthItems', () => {
+  const items = [
+    { name: 'disk-space', status: 'fail' as const, message: '/var 94% used' },
+    { name: 'cert-validity', status: 'pass' as const, message: 'expires in 62d' },
+    { name: 'upstream', status: 'warn' as const, message: 'slow', details: '900ms' },
+  ];
+
+  test('only non-passing items produce keys', () => {
+    const failing = failingKeysFromHealthItems('caddy', items, 'critical');
+    expect(failing.map((f) => f.key)).toEqual([
+      'module:caddy/check:disk-space',
+      'module:caddy/check:upstream',
+    ]);
+  });
+
+  test('severity follows the item status, not the monitor alone', () => {
+    const failing = failingKeysFromHealthItems('caddy', items, 'critical');
+    expect(failing.find((f) => f.key.endsWith('disk-space'))?.severity).toBe('critical');
+    expect(failing.find((f) => f.key.endsWith('upstream'))?.severity).toBe('warning');
+  });
+
+  test('message and details are carried through', () => {
+    const failing = failingKeysFromHealthItems('caddy', items, 'critical');
+    const upstream = failing.find((f) => f.key.endsWith('upstream'));
+    expect(upstream?.message).toBe('slow');
+    expect(upstream?.details).toBe('900ms');
+  });
+
+  // The empty set here means "ran, found nothing wrong" — trustworthy.
+  // The dangerous empty set (could not run) never reaches this function.
+  test('all-passing produces an empty failing set', () => {
+    const allPass = [{ name: 'x', status: 'pass' as const, message: 'ok' }];
+    expect(failingKeysFromHealthItems('caddy', allPass, 'critical')).toEqual([]);
+  });
+});

package/src/services/alerting/keys.ts

@@ -0,0 +1,190 @@
+/**
+ * Alert keys — the stable identity every alert is tracked by.
+ *
+ * A key is derived from the monitor and the check result; it is never
+ * operator-supplied. Everything downstream depends on it being stable across
+ * firings: dedup, acknowledgement, suppression, and resolution-by-absence all
+ * key off this string.
+ *
+ *   module:<id>                      the hook itself could not run
+ *   module:<id>/check:<name>         one item within a health_check result
+ *   builtin:<check>/<kind>:<target>  one target of a built-in audit check
+ *
+ * The module-level form is not a degenerate case. "Could not run at all" is the
+ * most common real failure — an unreachable system — and it is the key that
+ * machine- and zone-level suppression acts upon, so it must be addressable
+ * separately from the items underneath it.
+ *
+ * See openspec/changes/add-alerting/design.md D4.
+ */
+
+import type { AlertSeverity } from '../../db/schema';
+
+export interface ModuleAlertKey {
+  source: 'module';
+  moduleId: string;
+  /** Absent for the module-level key ("the hook could not run"). */
+  check?: string;
+}
+
+export interface BuiltinAlertKey {
+  source: 'builtin';
+  /** Audit check name, e.g. `machines_reachable`. */
+  check: string;
+  /**
+   * What kind of thing the target is, e.g. `machine`, `module`. Absent on the
+   * bare `builtin:<check>` form, which means "the check itself could not run"
+   * — the built-in counterpart of the module-level key.
+   */
+  targetKind?: string;
+  targetId?: string;
+}
+
+export type ParsedAlertKey = ModuleAlertKey | BuiltinAlertKey;
+
+const CHECK_SEPARATOR = '/check:';
+
+/** `module:<id>` — the module-level key. */
+export function moduleAlertKey(moduleId: string): string {
+  return `module:${moduleId}`;
+}
+
+/** `module:<id>/check:<name>` — one item of a health_check result. */
+export function moduleCheckAlertKey(moduleId: string, check: string): string {
+  return `${moduleAlertKey(moduleId)}${CHECK_SEPARATOR}${check}`;
+}
+
+/** `builtin:<check>/<kind>:<target>` — one target of a built-in check. */
+export function builtinAlertKey(check: string, targetKind: string, targetId: string): string {
+  return `builtin:${check}/${targetKind}:${targetId}`;
+}
+
+/**
+ * `builtin:<check>` — the check itself could not run.
+ *
+ * The built-in counterpart of the module-level key. Without it, a built-in
+ * check that throws would produce no failing keys at all, which the reconciler
+ * would read as "nothing is wrong" — the same false-all-clear this design
+ * exists to prevent, just on a different code path.
+ */
+export function builtinMonitorKey(check: string): string {
+  return `builtin:${check}`;
+}
+
+/**
+ * Parse a key back into its parts, or `null` if it is not a well-formed key.
+ *
+ * Module ids and built-in check names are kebab-case by construction, so they
+ * contain neither `:` nor `/`. Check ITEM names come from module authors and
+ * are deliberately unconstrained: everything after the first `/check:` is taken
+ * verbatim, so an item may contain colons and slashes without breaking the key.
+ * That keeps a module author's naming choice from corrupting alert identity.
+ */
+export function parseAlertKey(key: string): ParsedAlertKey | null {
+  if (key.startsWith('module:')) {
+    const rest = key.slice('module:'.length);
+    const sepIndex = rest.indexOf(CHECK_SEPARATOR);
+    if (sepIndex === -1) {
+      return rest.length > 0 ? { source: 'module', moduleId: rest } : null;
+    }
+    const moduleId = rest.slice(0, sepIndex);
+    const check = rest.slice(sepIndex + CHECK_SEPARATOR.length);
+    if (!moduleId || !check) return null;
+    return { source: 'module', moduleId, check };
+  }
+
+  if (key.startsWith('builtin:')) {
+    const rest = key.slice('builtin:'.length);
+    const slash = rest.indexOf('/');
+    if (slash === -1) {
+      // Bare `builtin:<check>` — the check itself could not run.
+      return rest.length > 0 ? { source: 'builtin', check: rest } : null;
+    }
+    const check = rest.slice(0, slash);
+    const target = rest.slice(slash + 1);
+    const colon = target.indexOf(':');
+    if (colon === -1) return null;
+    const targetKind = target.slice(0, colon);
+    const targetId = target.slice(colon + 1);
+    if (!check || !targetKind || !targetId) return null;
+    return { source: 'builtin', check, targetKind, targetId };
+  }
+
+  return null;
+}
+
+/**
+ * The module-level key that owns a given key, or `null` if the key IS a
+ * module-level key (or is not module-scoped at all).
+ *
+ * Used by suppression: `module:caddy` firing suppresses every
+ * `module:caddy/check:*` beneath it.
+ */
+export function parentModuleKey(key: string): string | null {
+  const parsed = parseAlertKey(key);
+  if (!parsed || parsed.source !== 'module' || !parsed.check) return null;
+  return moduleAlertKey(parsed.moduleId);
+}
+
+/**
+ * Map a health_check item status to the severity its alert carries, or `null`
+ * when the item produces no alert at all.
+ *
+ * `fail` takes the monitor's configured severity; `warn` is always `warning`
+ * and never pages. That gives module authors somewhere to put "cert expires in
+ * 20 days" without anyone having to decide whether it is a 3am problem.
+ *
+ * See design D6.
+ */
+export function severityForItemStatus(
+  itemStatus: 'pass' | 'warn' | 'fail',
+  monitorSeverity: AlertSeverity,
+): AlertSeverity | null {
+  if (itemStatus === 'pass') return null;
+  if (itemStatus === 'warn') return 'warning';
+  return monitorSeverity;
+}
+
+/** One currently-failing key produced by a monitor run. */
+export interface FailingKey {
+  key: string;
+  severity: AlertSeverity;
+  message: string;
+  details?: string;
+}
+
+export interface HealthCheckItemLike {
+  name: string;
+  status: 'pass' | 'warn' | 'fail';
+  message: string;
+  details?: string;
+}
+
+/**
+ * Project a SUCCESSFUL health_check result into the complete set of currently
+ * failing keys.
+ *
+ * Only call this for a run that actually executed. A run that could not execute
+ * yields an empty item list for reasons that say nothing about the checks, and
+ * feeding that here would produce an empty failing set — which the reconciler
+ * would read as "everything recovered" at the exact moment a system became
+ * unreachable. That branch belongs to the caller (design D5).
+ */
+export function failingKeysFromHealthItems(
+  moduleId: string,
+  items: HealthCheckItemLike[],
+  monitorSeverity: AlertSeverity,
+): FailingKey[] {
+  const failing: FailingKey[] = [];
+  for (const item of items) {
+    const severity = severityForItemStatus(item.status, monitorSeverity);
+    if (!severity) continue;
+    failing.push({
+      key: moduleCheckAlertKey(moduleId, item.name),
+      severity,
+      message: item.message,
+      details: item.details,
+    });
+  }
+  return failing;
+}