@celilo/cli 0.14.0 → 0.14.2

package/CELILO_CORE_MODULES.md

@@ -39,7 +39,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
 - **celilo-mgmt** — the celilo management server itself, deployed as a module (replaces install.sh + `system init`; ships daemon, runs migrations, self-registers). **provides:** `celilo_event_bus`, `celilo_module_deploy_worker`. **requires:** `cross_module_read`. See `openspec/specs/management-as-module/spec.md`.
 - **celilo-registry** — module registry server (Cargo sparse protocol); stores `.netapp` files, serves index + search/download API. On install it provisions a confidential introspection OIDC client via `idp.create_oidc_client` (SECURE_MODULE_PUBLISH.md §5[D-A]) and converges its issuer + introspection endpoint + creds onto the box for RFC 7662 token verification. **provides:** `registry_publish`. **requires:** `public_web`, `dns_registrar`, `idp`.
 - **celilo-apt-repo** — Debian apt repository (reprepro + Bun HTTP server) serving the celilo `.deb` at apt.celilo.computer. **provides:** `apt_publish`. **requires:** `public_web`, `dns_registrar`.
-- **signal** — bidirectional Signal transport for alerts and deploy-interview questions; runs signal-cli in daemon mode with its JSON-RPC socket bound to the host's own address (never public). Enrolled as a SECONDARY DEVICE of an existing Signal account rather than registering its own number — Signal blocks most VOIP ranges and bans bot-ish registrations. Recipient addresses live on celilo routes, not in module config, so adding a person never requires a redeploy. Runs on x86_64 and aarch64. `libsignal-client` ships no linux-aarch64 native, so celilo builds one (`modules/signal/build/`) and installs it as a `libsignal-jni` .deb on ARM hosts; x86_64 uses the JAR's bundled native. **provides:** `notification` (`send`, `receive`). **requires:** nothing — a transport that depended on the proxy, registrar or firewall could not tell you those were broken. See `openspec/changes/add-alerting/`.
+- **signal** — bidirectional Signal transport for alerts and deploy-interview questions; runs signal-cli in daemon mode with its JSON-RPC socket bound to the host's own address (never public). Enrolled as a SECONDARY DEVICE of an existing Signal account rather than registering its own number — Signal blocks most VOIP ranges and bans bot-ish registrations. Recipient addresses live on celilo routes, not in module config, so adding a person never requires a redeploy. Runs on x86_64 and aarch64. `libsignal-client` ships no linux-aarch64 native, so celilo builds one (`modules/signal/build/`) and installs it as a `libsignal-jni` .deb on ARM hosts; x86_64 uses the JAR's bundled native. **provides:** `notification` (`send`, `receive`). **requires:** no capabilities — a transport that depended on the proxy, registrar or firewall could not tell you those were broken — and a system in the **`secure-mgmt`** zone: it holds a linked Signal account (the operator's own messaging identity and keys), and its job is to observe every tier while depending on none, which is what the control-plane zone is for. See `openspec/changes/add-alerting/`.
 - **celilo-website** — public docs site (static Astro) served via Caddy on celilo.computer. **requires:** `public_web`, `dns_registrar`.
 
 ## Git forge & CI pipeline

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.14.0",
+  "version": "0.14.2",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {

package/src/capabilities/well-known.ts

@@ -19,10 +19,20 @@
  *   instead. Modules placed here inherit that reach, so placement is a privilege
  *   decision. celilo-mgmt may equally run in `internal`.
  *
+ * Semi-trusted LAN:
+ * - internal: Behind the firewall (NAT outbound, port-forward inbound), shielded
+ *   from the uncontrolled outside internet. Not firewall-segmented.
+ *
  * External Zone (Cloud/VPS):
  * - external: Services hosted outside home network (no VLAN, e.g., VPS on internet)
+ *
+ * Re-exported from db/schema rather than redeclared: this file used to carry its
+ * own hand-written copy of the union, and a second hand-maintained copy is what
+ * let `secure-mgmt` go missing in other places (see NETWORK_ZONES' comment).
  */
-export type NetworkZone = 'internal' | 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'external';
+import type { NetworkZone } from '../db/schema';
+
+export type { NetworkZone };
 
 export interface WellKnownCapability {
   canonical_hostname: string;

package/src/cli/commands/alerts-sweep.ts

@@ -152,5 +152,35 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
     `${report.unsuppressed} unsuppressed`,
     `${report.notified} notified`,
   ];
-  return { success: true, message: `alert sweep: ${parts.join(', ')}` };
+  // Only shown when non-zero: a quiet sweep should stay quiet. But a delivery
+  // that failed, was deferred, or was declined must never render as `0 notified`
+  // and nothing else — that is indistinguishable from "nothing needed sending",
+  // which is exactly how a transport that has stopped paging looks like a quiet
+  // night (#450).
+  if (report.deferred > 0) parts.push(`${report.deferred} deferred`);
+  if (report.deferredDelivered > 0) parts.push(`${report.deferredDelivered} deferred-delivered`);
+  if (report.failed > 0) parts.push(`${report.failed} FAILED`);
+  if (report.noPolicy > 0) parts.push(`${report.noPolicy} no-policy`);
+
+  const lines = [`alert sweep: ${parts.join(', ')}`];
+
+  // The reason escalation declined is the single most useful fact when someone
+  // asks "why was I not paged", so name it rather than aggregating it away.
+  const skipped = Object.entries(report.skipped).sort(([, a], [, b]) => b - a);
+  if (skipped.length > 0) {
+    lines.push(`  not delivered: ${skipped.map(([r, n]) => `${r}×${n}`).join(', ')}`);
+  }
+  // The error itself, not just a count: the transport is loaded lazily inside
+  // the send, so a capability that will not load produces no other record
+  // anywhere — nothing ever reaches the transport's own logs.
+  for (const failure of report.failures) {
+    lines.push(`  FAILED ${failure}`);
+  }
+  if (report.noPolicy > 0) {
+    lines.push(
+      `  ${report.noPolicy} live alert(s) have no escalation policy — assign one with:\n    celilo escalation-policy assign <policy> <monitor>`,
+    );
+  }
+
+  return { success: true, message: lines.join('\n') };
 }

package/src/cli/commands/module-show.ts

@@ -5,7 +5,7 @@
 
 import { eq } from 'drizzle-orm';
 import { getDb } from '../../db/client';
-import { modules } from '../../db/schema';
+import { type NetworkZone, modules } from '../../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../../manifest/schema';
 import { buildResolutionContext } from '../../variables/context';
 import { getArg, validateRequiredArgs } from '../parser';
@@ -161,14 +161,22 @@ export async function handleModuleShowZone(args: string[]): Promise<CommandResul
       };
     }
 
-    const zoneDescriptions: Record<string, string> = {
+    // Keyed by NetworkZone, not string: a new zone added to NETWORK_ZONES becomes a
+    // compile error here rather than silently rendering as "Unknown zone". Both
+    // `internal` and `secure-mgmt` were missing from the previous string-keyed map.
+    const zoneDescriptions: Record<NetworkZone, string> = {
+      internal: 'Internal (Semi-trusted network behind the firewall)',
       dmz: 'DMZ (Public-facing services)',
       app: 'Application (Internal services)',
       secure: 'Secure (Authentication/Database)',
+      'secure-mgmt': "Secure-Mgmt (celilo's own control plane)",
       external: 'External (VPS/Cloud)',
     };
 
-    const description = zoneDescriptions[zone] || 'Unknown zone';
+    // Cast at the lookup, not the declaration: `zone` comes from config and may be
+    // any string, so the runtime fallback stays — but the map above still has to
+    // cover every NetworkZone.
+    const description = zoneDescriptions[zone as NetworkZone] ?? 'Unknown zone';
 
     const lines = [`Module: ${moduleId}`, `Zone: ${zone} - ${description}`, ''];
 

package/src/db/schema.ts

@@ -198,7 +198,7 @@ export const ipAllocations = sqliteTable('ip_allocations', {
     .references(() => modules.id, { onDelete: 'cascade' }),
   vmid: integer('vmid').notNull().unique(),
   containerIp: text('container_ip').notNull().unique(), // CIDR format (e.g., "10.0.10.10/24")
-  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'>().notNull(),
+  zone: text('zone').$type<AllocatableZone>().notNull(),
   allocatedAt: integer('allocated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
 });
 
@@ -211,7 +211,7 @@ export const ipReservations = sqliteTable('ip_reservations', {
   id: integer('id').primaryKey({ autoIncrement: true }),
   ipStart: text('ip_start').notNull(), // Single IP or range start
   ipEnd: text('ip_end'), // NULL for single IP, end IP for range
-  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'>().notNull(),
+  zone: text('zone').$type<AllocatableZone>().notNull(),
   reason: text('reason').notNull(),
   reservedAt: integer('reserved_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
 });
@@ -275,6 +275,17 @@ export const NETWORK_ZONES = [
  */
 export type NetworkZone = (typeof NETWORK_ZONES)[number];
 
+/**
+ * Zones an IP allocation or reservation can name: every NetworkZone except
+ * `external`, whose systems are addressed by the provider, not by our IPAM.
+ *
+ * Derived rather than hand-written for the same reason as NetworkZone above —
+ * the previous hand-written union was copied into two column definitions and a
+ * cast in machine-pool.ts, and the cast had already drifted (it was missing
+ * `secure-mgmt`, and its comment claimed the only difference was `external`).
+ */
+export type AllocatableZone = Exclude<NetworkZone, 'external'>;
+
 /**
  * Container services table
  * Stores container service providers (Proxmox, Digital Ocean, etc.)

package/src/manifest/schema.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { NETWORK_ZONES } from '../db/schema';
 import { SUPPORTED_CONTRACT_VERSIONS } from './contracts';
 
 /**
@@ -309,9 +310,19 @@ export function parseIntervalMinutes(value: string): number | null {
  * operator creates a monitor by hand.
  */
 export const HealthCheckHookSchema = LifecycleHookSchema.extend({
+  // `.regex` duplicates the well-formedness half of the `superRefine` below on
+  // purpose: only `.regex` survives the export to JSON Schema, and that export is
+  // what validates `modules/*/manifest.yml` in the editor. The 5-minute floor
+  // cannot be expressed in JSON Schema at all, so it stays a refinement — which
+  // is why both exist rather than one. Keeping them in sync is the point of
+  // sharing DURATION_PATTERN.
   interval: z
     .string()
+    .regex(DURATION_PATTERN)
     .optional()
+    .describe(
+      'Suggested monitoring cadence, e.g. "15m", "1h", "1d". Must be 5m or longer — the monitor sweep runs on a 5-minute grid.',
+    )
     .superRefine((value, ctx) => {
       if (value === undefined) return;
       const minutes = parseIntervalMinutes(value);
@@ -379,9 +390,10 @@ export const SystemResourceSchema = z.object({
       'Proxmox provisioning type: lxc (default) or vm (qemu, for Docker / kernel-module workloads). ' +
         'Modules declare this explicitly; celilo never infers it. Moot for machine-pool / external infra.',
     ),
-  zone: z
-    .enum(['internal', 'dmz', 'app', 'secure', 'secure-mgmt', 'external'])
-    .describe('Required security zone for this module'),
+  // Derived from NETWORK_ZONES, not re-listed: a hand-written copy here would be
+  // a fourth place a new zone has to be remembered, and the copies in
+  // module-show and machine-pool had both already drifted.
+  zone: z.enum(NETWORK_ZONES).describe('Required security zone for this module'),
 });
 
 /**
@@ -594,7 +606,9 @@ export const ModuleManifestSchema = z
         container_created: LifecycleHookSchema.optional(),
         on_install: LifecycleHookSchema.optional(),
         on_uninstall: LifecycleHookSchema.optional(),
-        health_check: HealthCheckHookSchema.optional(),
+        health_check: HealthCheckHookSchema.optional().describe(
+          "Health check hook. `interval` is the module's SUGGESTED monitoring cadence; the operator's monitor row is the effective schedule and always wins.",
+        ),
         validate_config: LifecycleHookSchema.optional(),
         on_backup: LifecycleHookSchema.optional(),
         on_backup_analyze: LifecycleHookSchema.optional(),

package/src/services/alerting/sweep-runner.test.ts

@@ -226,4 +226,94 @@ describe('runSweep', () => {
     expect(db.select().from(monitors).get()?.lastRunAt).toEqual(NOW);
     expect(monitor.lastRunAt).toBeNull();
   });
+
+  // A delivery that never happened must be distinguishable from one that was
+  // never needed. Both used to render as `notified: 0` and nothing else, which
+  // is how a firing-but-undelivered alert became undebuggable (#450).
+  describe('undelivered alerts are accounted for, not silently dropped', () => {
+    test('an alert with no escalation policy is counted, not skipped in silence', async () => {
+      // `notifyDepsFor` returning null IS "nobody is configured to be told" —
+      // the default in every other test here, which is why this went unnoticed.
+      const report = await runSweep(db, currentMonitors(), deps());
+
+      expect(liveAlerts()).toHaveLength(1);
+      expect(report.notified).toBe(0);
+      expect(report.noPolicy).toBe(1);
+    });
+
+    test('escalation declining to notify records WHICH reason', async () => {
+      // A fresh alert is inside its grace window, so escalation declines with
+      // `within_grace` — a real skip reason reached through the real code path
+      // rather than a stubbed outcome.
+      const notifyDeps = {
+        steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }],
+        routes: new Map([['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }]]),
+        routeDetails: new Map([
+          [
+            'route-1',
+            { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never,
+          ],
+        ]),
+        quietHoursByPerson: new Map(),
+        bypassQuietHours: false,
+        transportFor: () => {
+          throw new Error('transport must not be reached for a skipped delivery');
+        },
+        mintToken: () => 'tok',
+        now: NOW,
+      } as never;
+
+      const report = await runSweep(
+        db,
+        currentMonitors(),
+        deps({ notifyDepsFor: () => notifyDeps }),
+      );
+
+      expect(report.notified).toBe(0);
+      expect(report.noPolicy).toBe(0);
+      expect(report.skipped.within_grace).toBe(1);
+    });
+
+    test('a transport that cannot be loaded records the error, not just a count', async () => {
+      // The transport is resolved lazily INSIDE the send, so a capability that
+      // will not load never reaches the transport's own logs. If the sweep does
+      // not carry the message, it exists nowhere.
+      const notifyDeps = (now: Date) =>
+        ({
+          steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }],
+          routes: new Map([
+            ['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }],
+          ]),
+          routeDetails: new Map([
+            [
+              'route-1',
+              { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never,
+            ],
+          ]),
+          quietHoursByPerson: new Map(),
+          bypassQuietHours: false,
+          transportFor: () => {
+            throw new Error('does not provide the notification capability');
+          },
+          mintToken: () => 'tok',
+          now,
+        }) as never;
+
+      // First sweep creates the alert; the second is past the grace window, so
+      // escalation actually reaches the transport.
+      await runSweep(db, currentMonitors(), deps());
+      const at = later(20);
+      const report = await runSweep(
+        db,
+        currentMonitors(),
+        deps({ notifyDepsFor: () => notifyDeps(at) }, failing, at),
+      );
+
+      expect(report.notified).toBe(0);
+      expect(report.failed).toBe(1);
+      expect(report.failures).toHaveLength(1);
+      expect(report.failures[0]).toContain(PORT_CHECK);
+      expect(report.failures[0]).toContain('does not provide the notification capability');
+    });
+  });
 });

package/src/services/alerting/sweep-runner.ts

@@ -61,6 +61,34 @@ export interface SweepReport {
   /** Messages held over quiet hours and delivered now that the window ended. */
   deferredDelivered: number;
   failed: number;
+  /**
+   * Live alerts nobody is configured to be told about — no escalation policy on
+   * the monitor, so `notifyDepsFor` returns null.
+   *
+   * Counted rather than skipped in silence: "nothing needed sending" and "an
+   * alert is firing and no policy points at anyone" are opposite situations that
+   * previously rendered identically as `0 notified`.
+   */
+  noPolicy: number;
+  /**
+   * Deliveries escalation declined, keyed by its reason (`within_grace`,
+   * `no_eligible_route`, …).
+   *
+   * `notifyAlert` returns the reason precisely so the caller can record it — its
+   * own contract says a silent skip is indistinguishable from a bug. Dropping it
+   * here is what made a firing-but-undelivered alert undebuggable (#450).
+   */
+  skipped: Record<string, number>;
+  /**
+   * Why each failed delivery failed, as `<alert key>: <error>`.
+   *
+   * A count alone does not answer the only question that matters after a page
+   * did not arrive. The transport is loaded lazily *inside* `notifyAlert`'s try
+   * block, so "the capability would not load" and "Signal rejected the message"
+   * both surface here and nowhere else — there is no daemon-side log for the
+   * former, because nothing ever reached the daemon.
+   */
+  failures: string[];
 }
 
 /**
@@ -85,6 +113,9 @@ export async function runSweep(
     deferred: 0,
     deferredDelivered: 0,
     failed: 0,
+    noPolicy: 0,
+    skipped: {},
+    failures: [],
   };
 
   // 1. Run due monitors.
@@ -162,24 +193,36 @@ export async function runSweep(
     try {
       const outcome = await deliverDeferred(alert, route, notifyDeps);
       if (outcome.result === 'sent') report.deferredDelivered++;
-      else if (outcome.result === 'failed') report.failed++;
-    } catch {
+      else if (outcome.result === 'failed') {
+        report.failed++;
+        report.failures.push(`${alert.key} (deferred): ${outcome.error}`);
+      }
+    } catch (error) {
       report.failed++;
+      report.failures.push(
+        `${alert.key} (deferred): ${error instanceof Error ? error.message : String(error)}`,
+      );
     }
   }
 
   // 5. Notify. Re-read: the steps above changed state under us.
   for (const alert of loadAllLiveAlerts(db)) {
     const notifyDeps = deps.notifyDepsFor(alert);
-    if (!notifyDeps) continue;
+    if (!notifyDeps) {
+      report.noPolicy++;
+      continue;
+    }
 
     let outcome: NotifyOutcome;
     try {
       outcome = await notifyAlert(alert, notifyDeps);
-    } catch {
+    } catch (error) {
       // notifyAlert already converts transport errors into a `failed` outcome;
       // reaching here means something above the transport broke.
       report.failed++;
+      report.failures.push(
+        `${alert.key}: ${error instanceof Error ? error.message : String(error)}`,
+      );
       continue;
     }
 
@@ -197,6 +240,9 @@ export async function runSweep(
       report.deferred++;
     } else if (outcome.result === 'failed') {
       report.failed++;
+      report.failures.push(`${alert.key}: ${outcome.error}`);
+    } else if (outcome.result === 'skipped') {
+      report.skipped[outcome.reason] = (report.skipped[outcome.reason] ?? 0) + 1;
     }
   }
 

package/src/services/machine-pool.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { and, eq, inArray } from 'drizzle-orm';
 import { getDb } from '../db/client';
 import {
+  type AllocatableZone,
   type NetworkZone,
   containerServices,
   ipAllocations,
@@ -355,7 +356,7 @@ export async function getContainerSystemsByZone(
         eq(moduleInfrastructure.infrastructureType, 'container_service'),
         // ip_allocations.zone is narrower than NetworkZone (no 'external');
         // safe to cast — any 'external' input would just match zero rows.
-        inArray(ipAllocations.zone, zones as Array<'dmz' | 'app' | 'secure' | 'internal'>),
+        inArray(ipAllocations.zone, zones as AllocatableZone[]),
       ),
     );