@celilo/cli 0.13.3 → 0.14.0

package/src/services/alerting/tokens.ts

@@ -0,0 +1,119 @@
+/**
+ * Reply tokens — how an inbound message identifies what it is answering.
+ *
+ * Minted per DELIVERY, not per alert. The token therefore identifies WHO
+ * replied as well as what they replied to, which is what "an ack from the
+ * secondary is broadcast to everyone already paged" needs, and it doubles as
+ * the audit trail. A per-alert token could not distinguish two people
+ * answering the same page.
+ *
+ * The same table serves alert acknowledgements and interview answers; only
+ * `targetId` differs in meaning. That is the unification the design is built
+ * around (D10).
+ */
+
+import { randomInt, randomUUID } from 'node:crypto';
+import { and, eq, gt, isNull } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { type NotificationDelivery, notificationDeliveries } from '../../db/schema';
+
+/**
+ * Crockford-style base32 without I, L, O, U — the characters people misread or
+ * mistype when copying a code off a phone screen. Six of these is ~1e9
+ * possibilities, which combined with expiry and the sender check is ample: the
+ * token is one of two factors, not a secret on its own.
+ */
+const TOKEN_ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
+const TOKEN_LENGTH = 6;
+
+export function generateToken(): string {
+  let token = '';
+  for (let i = 0; i < TOKEN_LENGTH; i++) {
+    token += TOKEN_ALPHABET[randomInt(TOKEN_ALPHABET.length)];
+  }
+  return token;
+}
+
+/** Normalise operator input: case-insensitive, and forgiving of separators. */
+export function normaliseToken(raw: string): string {
+  return raw.trim().toUpperCase().replace(/[\s-]/g, '');
+}
+
+export interface MintTokenInput {
+  kind: 'alert' | 'interview';
+  /** `alerts.id` for an alert; a BUS event id for an interview. */
+  targetId: string;
+  routeId: string;
+  now: Date;
+  ttlMs: number;
+}
+
+/**
+ * Mint a delivery record with a unique token.
+ *
+ * Retries on collision rather than trusting randomness: a six-character token
+ * WILL collide eventually, and a collision that silently reassigned an
+ * outstanding token would route someone's ack to the wrong alert.
+ */
+export function mintDelivery(db: DbClient, input: MintTokenInput): NotificationDelivery {
+  for (let attempt = 0; attempt < 8; attempt++) {
+    const token = generateToken();
+    if (findLiveDelivery(db, token, input.now)) continue;
+
+    const id = randomUUID();
+    db.insert(notificationDeliveries)
+      .values({
+        id,
+        token,
+        kind: input.kind,
+        targetId: input.targetId,
+        routeId: input.routeId,
+        sentAt: input.now,
+        expiresAt: new Date(input.now.getTime() + input.ttlMs),
+      })
+      .run();
+    return db
+      .select()
+      .from(notificationDeliveries)
+      .where(eq(notificationDeliveries.id, id))
+      .get() as NotificationDelivery;
+  }
+  throw new Error('Could not mint a unique reply token after 8 attempts');
+}
+
+/** An unexpired, unconsumed delivery for this token. */
+export function findLiveDelivery(
+  db: DbClient,
+  token: string,
+  now: Date,
+): NotificationDelivery | undefined {
+  return db
+    .select()
+    .from(notificationDeliveries)
+    .where(
+      and(
+        eq(notificationDeliveries.token, normaliseToken(token)),
+        isNull(notificationDeliveries.consumedAt),
+        gt(notificationDeliveries.expiresAt, now),
+      ),
+    )
+    .get();
+}
+
+export function consumeDelivery(db: DbClient, deliveryId: string, now: Date): void {
+  db.update(notificationDeliveries)
+    .set({ consumedAt: now })
+    .where(eq(notificationDeliveries.id, deliveryId))
+    .run();
+}
+
+/** Every delivery made for an alert — who to send the all-clear to. */
+export function deliveriesForAlert(db: DbClient, alertId: string): NotificationDelivery[] {
+  return db
+    .select()
+    .from(notificationDeliveries)
+    .where(
+      and(eq(notificationDeliveries.kind, 'alert'), eq(notificationDeliveries.targetId, alertId)),
+    )
+    .all();
+}

package/src/services/alerting/transport-loader.ts

@@ -0,0 +1,48 @@
+/**
+ * Loading a transport's `send` from the module that provides it.
+ *
+ * A notification transport is an ordinary module providing the `notification`
+ * capability, so reaching it goes through the same capability-loader every
+ * other cross-module call uses. Nothing here knows what Signal is.
+ *
+ * A transport that cannot be loaded throws at send time rather than returning
+ * a no-op. Silently swallowing it would mean the sweep records "notified"
+ * for a page that never left the building — the one outcome an alerting
+ * system must never produce.
+ */
+
+import type { NotificationCapability } from '@celilo/capabilities';
+import type { DbClient } from '../../db/client';
+import { loadCapabilityFunctions } from '../../hooks/capability-loader';
+import { createCapturingLogger } from '../../hooks/logger';
+import type { NotificationTransport } from './notifier';
+
+/**
+ * A transport bound to one module.
+ *
+ * Resolution is deferred to the first `send` because a sweep may evaluate many
+ * alerts and page for none of them — loading a module's capability functions
+ * for every alert would be work done to reach nobody.
+ */
+export function loadNotificationTransport(
+  db: DbClient,
+  transportModuleId: string,
+): NotificationTransport {
+  return {
+    async send(request) {
+      const { logger } = createCapturingLogger();
+      const capabilities = await loadCapabilityFunctions(transportModuleId, db, logger);
+      const notification = (capabilities as Record<string, unknown>).notification as
+        | NotificationCapability
+        | undefined;
+
+      if (!notification) {
+        throw new Error(
+          `Module "${transportModuleId}" does not provide the notification capability. Deploy it, or point the route at a module that does.`,
+        );
+      }
+
+      return notification.send(request);
+    },
+  };
+}

package/src/services/audit/index.test.ts

@@ -32,6 +32,7 @@ const emptyDeps = {
   secretsDecryptable: { results: [] },
   servicesReachable: { results: [] },
   machinesReachable: { results: [] },
+  trustedSources: { firewalls: [] },
 };
 
 describe('runAudit', () => {

package/src/services/audit/index.ts

@@ -25,6 +25,7 @@ import {
 } from './services-credentials';
 import { type ServicesReachableAuditDeps, auditServicesReachable } from './services-reachable';
 import { type TerraformPlanAuditDeps, auditTerraformPlan } from './terraform-plan';
+import { type TrustedSourcesAuditDeps, auditTrustedSources } from './trusted-sources';
 import {
   type DriftCategory,
   type DriftFinding,
@@ -52,6 +53,7 @@ export interface AuditDeps {
   secretsDecryptable: SecretsDecryptableAuditDeps;
   servicesReachable: ServicesReachableAuditDeps;
   machinesReachable: MachinesReachableAuditDeps;
+  trustedSources: TrustedSourcesAuditDeps;
   /** Defaults to `Date.now()`-based ISO string. */
   now?: () => Date;
 }
@@ -101,6 +103,7 @@ export async function runAudit(
     wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
     wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
     wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
+    wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
   ]);
 
   const findings: DriftFinding[] = groups.flat();

package/src/services/audit/trusted-sources.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, it } from 'bun:test';
+import { auditTrustedSources, parseLiveReachRules } from './trusted-sources';
+
+const ZONES = ['10.0.10.0/24', '10.0.20.0/24', '10.0.30.0/24'];
+const CONTROL_PLANE = '192.168.0.0/24';
+const VPN = '10.255.255.0/24';
+
+/** A live `iptables-save -t filter` dump with a hand-added VPN reach rule. */
+const LIVE_WITH_UNOWNED = [
+  '*filter',
+  ':INPUT ACCEPT [0:0]',
+  ':FORWARD DROP [0:0]',
+  ':OUTPUT ACCEPT [0:0]',
+  '-A FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT',
+  '-A FORWARD -o eth1 -j ACCEPT',
+  `-A FORWARD -s ${CONTROL_PLANE} -d 10.0.10.0/24 -j ACCEPT`,
+  `-A FORWARD -s ${CONTROL_PLANE} -d 10.0.20.0/24 -j ACCEPT`,
+  `-A FORWARD -s ${CONTROL_PLANE} -d 10.0.30.0/24 -j ACCEPT`,
+  `-A FORWARD -s ${VPN} -d 10.0.10.0/24 -j ACCEPT`,
+  'COMMIT',
+].join('\n');
+
+describe('parsing reach rules from a live ruleset', () => {
+  it('finds subnet-scoped reach into a managed zone', () => {
+    expect(parseLiveReachRules(LIVE_WITH_UNOWNED, ZONES)).toContainEqual({
+      origin: { kind: 'subnet', value: VPN },
+      destSubnet: '10.0.10.0/24',
+    });
+  });
+
+  it('finds interface-scoped reach, which celilo cannot model at all', () => {
+    // The shape fw-keeper.sh actually maintained: `-i wg0 -d <zone>`.
+    const rules = parseLiveReachRules(
+      '*filter\n-A FORWARD -i wg0 -d 10.0.20.0/24 -j ACCEPT\nCOMMIT',
+      ZONES,
+    );
+    expect(rules).toEqual([
+      { origin: { kind: 'interface', value: 'wg0' }, destSubnet: '10.0.20.0/24' },
+    ]);
+  });
+
+  it('ignores rules that do not reach a managed zone', () => {
+    const rules = parseLiveReachRules(
+      [
+        '*filter',
+        '-A FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT',
+        '-A FORWARD -o eth1 -j ACCEPT',
+        '-A FORWARD -s 10.99.0.0/24 -d 172.16.0.0/24 -j ACCEPT',
+        '-A INPUT -s 10.99.0.0/24 -d 10.0.10.0/24 -j ACCEPT',
+        '-A FORWARD -s 10.99.0.0/24 -d 10.0.10.0/24 -j DROP',
+        'COMMIT',
+      ].join('\n'),
+      ZONES,
+    );
+    expect(rules).toEqual([]);
+  });
+});
+
+describe('unowned-trusted-network audit', () => {
+  it('reports a network celilo does not recognise', async () => {
+    const findings = await auditTrustedSources({
+      firewalls: [
+        {
+          firewallIp: '192.168.0.254',
+          live: parseLiveReachRules(LIVE_WITH_UNOWNED, ZONES),
+          known: [CONTROL_PLANE],
+        },
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].code).toBe('unowned_trusted_network');
+    expect(findings[0].message).toContain(VPN);
+    expect(findings[0].severity).toBe('drift');
+  });
+
+  it('is silent once that network is registered', async () => {
+    // The point of the change: once a module owns the VPN and registers its
+    // subnet, the rule IS the rendered output and there is nothing to report.
+    const findings = await auditTrustedSources({
+      firewalls: [
+        {
+          firewallIp: '192.168.0.254',
+          live: parseLiveReachRules(LIVE_WITH_UNOWNED, ZONES),
+          known: [CONTROL_PLANE, VPN],
+        },
+      ],
+    });
+
+    expect(findings).toEqual([]);
+  });
+
+  it('reports an unrecognised network once, not once per zone', async () => {
+    const live = parseLiveReachRules(
+      ZONES.map((z) => `-A FORWARD -s ${VPN} -d ${z} -j ACCEPT`).join('\n'),
+      ZONES,
+    );
+    const findings = await auditTrustedSources({
+      firewalls: [{ firewallIp: '192.168.0.254', live, known: [CONTROL_PLANE] }],
+    });
+
+    expect(findings).toHaveLength(1);
+  });
+
+  it('names the interface when trust is granted by interface, and says why it cannot be registered as-is', async () => {
+    const findings = await auditTrustedSources({
+      firewalls: [
+        {
+          firewallIp: '192.168.0.254',
+          live: [{ origin: { kind: 'interface', value: 'wg0' }, destSubnet: '10.0.10.0/24' }],
+          known: [CONTROL_PLANE],
+        },
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].message).toContain('wg0');
+    expect(findings[0].details).toContain('SUBNET');
+  });
+
+  it('a firewall with no live reach beyond what celilo composes is clean', async () => {
+    const findings = await auditTrustedSources({
+      firewalls: [
+        {
+          firewallIp: '192.168.0.254',
+          live: parseLiveReachRules(
+            ZONES.map((z) => `-A FORWARD -s ${CONTROL_PLANE} -d ${z} -j ACCEPT`).join('\n'),
+            ZONES,
+          ),
+          known: [CONTROL_PLANE],
+        },
+      ],
+    });
+
+    expect(findings).toEqual([]);
+  });
+});

package/src/services/audit/trusted-sources.ts

@@ -0,0 +1,124 @@
+/**
+ * Unowned-trusted-network check.
+ *
+ * A converge renders the firewall's whole ruleset from the registry and applies
+ * it atomically, so a rule nobody registered is a rule nobody owns — and it is
+ * removed. That is correct behaviour, but it is silent, and silence is what
+ * produced `/tmp/fw-keeper.sh`: a shell script re-adding the admin VPN's rules
+ * every second because the VPN had no module to register on its behalf, and
+ * nothing anywhere said so.
+ *
+ * This check reports reach granted on the live firewall to a network celilo does
+ * not recognise, so the next unowned thing surfaces as an actionable gap instead
+ * of as an unexplained outage.
+ *
+ * Pure and dependency-injected: the CLI adapter reads the live ruleset off each
+ * firewall and hands the parsed result in.
+ */
+
+import type { DriftFinding } from './types';
+
+/** How a live FORWARD rule grants reach into a managed zone. */
+export type ReachOrigin = { kind: 'subnet'; value: string } | { kind: 'interface'; value: string };
+
+export interface LiveReachRule {
+  origin: ReachOrigin;
+  /** The managed zone subnet the rule permits reaching. */
+  destSubnet: string;
+}
+
+export interface FirewallReachState {
+  firewallIp: string;
+  /** Reach rules parsed from the firewall's LIVE ruleset (`iptables-save`). */
+  live: LiveReachRule[];
+  /** The subnets celilo composes for this firewall (derived + registered + override). */
+  known: string[];
+}
+
+export interface TrustedSourcesAuditDeps {
+  firewalls: FirewallReachState[];
+}
+
+/**
+ * Extract reach-granting FORWARD rules from an `iptables-save` dump: any ACCEPT
+ * into one of the managed zone subnets, keyed by what grants it.
+ *
+ * Both shapes matter. celilo renders `-s <subnet> -d <zone>`; a hand-added rule
+ * (or a keeper script) is as likely to be `-i <iface> -d <zone>`, which celilo
+ * cannot model at all — trust in celilo is by subnet, never by interface.
+ */
+export function parseLiveReachRules(rulesetText: string, zoneSubnets: string[]): LiveReachRule[] {
+  const zones = new Set(zoneSubnets);
+  const rules: LiveReachRule[] = [];
+
+  for (const line of rulesetText.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith('-A FORWARD')) continue;
+    if (!/-j\s+ACCEPT\b/.test(trimmed)) continue;
+
+    const dest = trimmed.match(/-d\s+(\S+)/)?.[1];
+    if (!dest || !zones.has(dest)) continue;
+
+    const source = trimmed.match(/-s\s+(\S+)/)?.[1];
+    if (source) {
+      rules.push({ origin: { kind: 'subnet', value: source }, destSubnet: dest });
+      continue;
+    }
+    const iface = trimmed.match(/-i\s+(\S+)/)?.[1];
+    if (iface) {
+      rules.push({ origin: { kind: 'interface', value: iface }, destSubnet: dest });
+    }
+  }
+  return rules;
+}
+
+function originKey(origin: ReachOrigin): string {
+  return `${origin.kind}:${origin.value}`;
+}
+
+export async function auditTrustedSources(deps: TrustedSourcesAuditDeps): Promise<DriftFinding[]> {
+  const findings: DriftFinding[] = [];
+
+  for (const firewall of deps.firewalls) {
+    const known = new Set(firewall.known);
+    const reported = new Set<string>();
+
+    for (const rule of firewall.live) {
+      // A subnet celilo composes is owned by definition — that rule IS the
+      // rendered output.
+      if (rule.origin.kind === 'subnet' && known.has(rule.origin.value)) continue;
+      if (reported.has(originKey(rule.origin))) continue;
+      reported.add(originKey(rule.origin));
+
+      const byInterface = rule.origin.kind === 'interface';
+      findings.push({
+        category: 'trusted_sources',
+        severity: 'drift',
+        code: 'unowned_trusted_network',
+        message: byInterface
+          ? `${firewall.firewallIp}: interface ${rule.origin.value} reaches managed zones, but celilo models no such trust`
+          : `${firewall.firewallIp}: ${rule.origin.value} reaches managed zones, but celilo does not recognise that network`,
+        details: [
+          'The live ruleset permits this reach; celilo does not render it.',
+          'The next converge rebuilds the ruleset from the registry and will',
+          'remove it, with no error and no explanation at the point of failure.',
+          ...(byInterface
+            ? [
+                '',
+                'celilo expresses trust by SUBNET, never by interface — an',
+                'interface-scoped rule cannot be registered as-is. Find the',
+                'network behind that interface and register THAT.',
+              ]
+            : []),
+        ].join('\n'),
+        remediation: byInterface
+          ? 'Deploy a module that owns this network and registers its client subnet as a trusted source (the wireguard module does this for the admin VPN), or record it with `celilo system config set firewall.trusted_subnets <cidr>`'
+          : `Deploy a module that registers ${rule.origin.value} as a trusted source, or record it with \`celilo system config set firewall.trusted_subnets ${rule.origin.value}\``,
+        actionable: false,
+        subject: firewall.firewallIp,
+      });
+    }
+  }
+
+  return findings;
+}

package/src/services/audit/types.ts

@@ -35,7 +35,8 @@ export type DriftCategory =
   | 'services_credentials'
   | 'secrets_decryptable'
   | 'services_reachable'
-  | 'machines_reachable';
+  | 'machines_reachable'
+  | 'trusted_sources';
 
 export type DriftSeverity = 'todo' | 'drift' | 'blocked';
 

package/src/services/firewall-reach.ts

@@ -0,0 +1,83 @@
+/**
+ * Collect what each firewall's LIVE ruleset permits to reach the managed zones,
+ * alongside what celilo composes for it — the input to the unowned-trusted-
+ * network audit check.
+ *
+ * The live read is `iptables-save`, not celilo's own `rules.v4`: a rule added
+ * out of band (the `fw-keeper.sh` case) exists in the running table and not in
+ * the file celilo renders, and that difference is exactly the finding.
+ */
+
+import { type Runner, execRunner, runAppCommand } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { capabilities, moduleConfigs, systemConfig } from '../db/schema';
+import { loadTrustedSubnets } from '../hooks/capability-loader';
+import type { FirewallReachState } from './audit/trusted-sources';
+import { parseLiveReachRules } from './audit/trusted-sources';
+
+/** The segmented tiers the renderer builds its zone matrix from. */
+const ZONE_TIERS = ['dmz', 'app', 'secure'] as const;
+
+const IPTABLES_SAVE_TIMEOUT_MS = 15_000;
+
+/** Firewall hosts celilo knows about, by the `firewall_ip` in their module config. */
+export function listFirewallIps(db: DbClient): string[] {
+  const providers = db
+    .select()
+    .from(capabilities)
+    .where(eq(capabilities.capabilityName, 'firewall'))
+    .all();
+
+  const ips: string[] = [];
+  for (const provider of providers) {
+    const row = db
+      .select()
+      .from(moduleConfigs)
+      .where(eq(moduleConfigs.moduleId, provider.moduleId))
+      .all()
+      .find((c) => c.key === 'firewall_ip');
+    if (row?.value) ips.push(row.value);
+  }
+  return [...new Set(ips)];
+}
+
+export function loadZoneSubnets(db: DbClient): string[] {
+  const subnets: string[] = [];
+  for (const zone of ZONE_TIERS) {
+    const row = db
+      .select()
+      .from(systemConfig)
+      .where(eq(systemConfig.key, `network.${zone}.subnet`))
+      .get();
+    if (row?.value) subnets.push(row.value);
+  }
+  return subnets;
+}
+
+/**
+ * Read each firewall's live ruleset and pair it with celilo's composed trusted
+ * set. A firewall that can't be reached contributes nothing rather than a false
+ * "no unowned reach" — an unreadable box is unknown, not clean.
+ */
+export function collectFirewallReach(db: DbClient, run: Runner = execRunner): FirewallReachState[] {
+  const zoneSubnets = loadZoneSubnets(db);
+  if (zoneSubnets.length === 0) return [];
+
+  const states: FirewallReachState[] = [];
+  for (const firewallIp of listFirewallIps(db)) {
+    // escape-hatch: reads the running filter table (`iptables-save`), which is
+    // the only place an out-of-band rule appears. Read-only, no capability.
+    const result = runAppCommand({ ipv4_address: firewallIp }, 'iptables-save -t filter', run, {
+      timeoutMs: IPTABLES_SAVE_TIMEOUT_MS,
+    });
+    if (!result.ok) continue;
+
+    states.push({
+      firewallIp,
+      live: parseLiveReachRules(result.stdout, zoneSubnets),
+      known: loadTrustedSubnets(db, firewallIp).map((e) => e.subnet),
+    });
+  }
+  return states;
+}

package/src/services/health-runner.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from 'bun:test';
+import { nextModuleState } from './health-runner';
+
+/**
+ * The D15 rule: a scheduled (monitor-driven) health check observes only and
+ * never moves module lifecycle state; an operator-invoked one still does.
+ *
+ * These are the cases that regress invisibly — a wrong boolean here does not
+ * throw, it just makes `celilo module list` flicker between INSTALLED and
+ * VERIFIED every monitor interval.
+ */
+describe('nextModuleState', () => {
+  describe('operator-invoked run (unattended = false)', () => {
+    test('passing check verifies an INSTALLED module', () => {
+      expect(nextModuleState('INSTALLED', 'healthy', false)).toBe('VERIFIED');
+    });
+
+    test('a warning still verifies — degraded is not failure', () => {
+      expect(nextModuleState('INSTALLED', 'degraded', false)).toBe('VERIFIED');
+    });
+
+    test('failing check un-verifies a VERIFIED module', () => {
+      expect(nextModuleState('VERIFIED', 'unhealthy', false)).toBe('INSTALLED');
+    });
+
+    test('failing check on an already-INSTALLED module changes nothing', () => {
+      expect(nextModuleState('INSTALLED', 'unhealthy', false)).toBeNull();
+    });
+  });
+
+  describe('scheduled run (unattended = true)', () => {
+    test('failing check does NOT un-verify a VERIFIED module', () => {
+      expect(nextModuleState('VERIFIED', 'unhealthy', true)).toBeNull();
+    });
+
+    test('passing check does NOT verify an INSTALLED module', () => {
+      expect(nextModuleState('INSTALLED', 'healthy', true)).toBeNull();
+    });
+
+    test('no status moves state, whatever the module is currently in', () => {
+      const states = ['INSTALLED', 'VERIFIED', 'ERROR', 'CONFIGURED'] as const;
+      const statuses = ['healthy', 'degraded', 'unhealthy'] as const;
+      for (const state of states) {
+        for (const status of statuses) {
+          expect(nextModuleState(state, status, true)).toBeNull();
+        }
+      }
+    });
+  });
+});