@celilo/cli 0.21.0 → 0.22.0

package/CELILO_CORE_MODULES.md

@@ -14,19 +14,20 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
 > the `@celilo/cli` npm package). Re-grep manifests if an entry looks stale.
 
 > How the graph closes: a public app (`requires: public_web`) is served by **caddy**, which
-> `requires: dns_registrar` (**namecheap**) to publish its name and `firewall` (**greenwave** /
-> **iptables**) to open the port. Internal name resolution comes from a **dns_internal** provider
+> `requires: dns_registrar` (**namecheap**) to publish its name and `firewall` (**axon** /
+> **greenwave** / **iptables**) to open the port. Internal name resolution comes from a **dns_internal** provider
 > (**knot-unbound-internal** / **technitium**). Identity comes from **authentik** (`idp`). That's
 > the whole edge: DNS + firewall + ingress + identity, each a swappable provider module.
 
 ## Network fabric (DNS / firewall / DHCP)
 
-- **greenwave** — GreenWave C4000XG ISP router driver; port-forwarding + public-IP discovery via REST. **provides:** `firewall`, `dhcp_server`.
+- **axon** — Axon Networks Q1000K ISP router driver (Brightspeed-branded); port-forwarding + public-IP discovery + DHCP DNS via the TR-181 CGI API. **provides:** `firewall`, `dhcp_server`. Fork of **greenwave** — identical protocol, differing only in the vendor extension prefix (`X_AXON_` vs `X_GWS_`/`X_LANTIQ_COM_`). Pick by device: Q1000K → **axon**, C4000XG → **greenwave**.
+- **greenwave** — GreenWave C4000XG ISP router driver; port-forwarding + public-IP discovery via REST. **provides:** `firewall`, `dhcp_server`. Legacy device; new deployments on Axon hardware want **axon**.
 - **iptables** — iptables firewall + NAT; cross-VLAN port exposure with recursive upstream delegation. Converge model: `exposeService` registers into the shared-core port-forward registry, then renders the complete ruleset and applies it atomically via `iptables-restore` (default-DROP FORWARD + coarse zone-tier matrix; SSH-free). **provides:** `firewall`.
 - **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
 - **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
 - **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). A caller supplies a NAME and nothing else: the address is the source IP of celilo's own update, re-derived on every assert. Registering `<domain>` also claims `www.<domain>` and vice versa (best effort, reported back as `outputs.companion_fqdn` so the framework's `public_dns` check watches it — Namecheap answers `ErrCount 0` for `www` updates it does not apply). DDNS passwords are keyed by the **registrable domain**, never the FQDN. **provides:** `dns_registrar`.
-- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
+- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does; the ISP-router drivers `greenwave` and `axon` do not).
 
 ## Public edge (ingress / identity)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -57,10 +57,10 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1024.0",
-    "@celilo/capabilities": "^1.0.0",
+    "@celilo/capabilities": "^1.0.1",
     "@celilo/cli-display": "^0.1.10",
-    "@celilo/core": "^0.5.0",
-    "@celilo/event-bus": "^0.3.0",
+    "@celilo/core": "^0.6.0",
+    "@celilo/event-bus": "^0.4.0",
     "@clack/prompts": "^1.1.0",
     "ajv": "^8.18.0",
     "drizzle-orm": "^0.36.4",

package/src/cli/commands/events.test.ts

@@ -4,10 +4,12 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { defineEvents, openBus } from '@celilo/event-bus';
 import {
+  type FailedDeliveryReport,
   handleEventsAck,
   handleEventsDrain,
   handleEventsEmit,
   handleEventsFail,
+  handleEventsListFailed,
   handleEventsListPending,
   handleEventsListSubscribers,
   handleEventsRepair,
@@ -262,3 +264,67 @@ describe('celilo events command handlers', () => {
     expect(replies[0].payload).toEqual({ value: 'a.net' });
   });
 });
+
+describe('celilo events list-failed', () => {
+  let dir: string;
+  let dbPath: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'events-failed-test-'));
+    dbPath = join(dir, 'events.db');
+    process.env.EVENT_BUS_DB = dbPath;
+    const bus = openBus({ dbPath, events: defineEvents({}) });
+    const sub = bus.subscribe({ name: 'namecheap.ddns', pattern: 'ddns.*', handler: 'echo' });
+    for (let i = 0; i < 60; i++) {
+      const event = bus.emitRaw('ddns.refresh', { n: i });
+      bus.markFailed({ eventId: event.id, subscriberId: sub.id }, new Error('boom'), {
+        abandoned: true,
+      });
+    }
+    bus.close();
+  });
+  afterEach(() => {
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  // The whole point of the command: `total` is a COUNT, `shown` is the limit.
+  it('separates the true total from the capped sample', async () => {
+    const result = await handleEventsListFailed([], { limit: '5' });
+    expect(result.success).toBe(true);
+    if (!result.success) throw new Error('expected success');
+    const report = result.data as FailedDeliveryReport;
+    expect(report.total).toBe(60);
+    expect(report.shown).toBe(5);
+    expect(report.deliveries).toHaveLength(5);
+    expect(report.bySubscriber).toEqual([
+      { subscriber: 'namecheap.ddns', count: 60, latestFinishedAt: expect.any(Number) },
+    ]);
+  });
+
+  it('names the subscriber and event type on every row', async () => {
+    const result = await handleEventsListFailed([], { limit: '1' });
+    if (!result.success) throw new Error('expected success');
+    const row = (result.data as FailedDeliveryReport).deliveries[0];
+    expect(row.subscriber).toBe('namecheap.ddns');
+    expect(row.eventType).toBe('ddns.refresh');
+    expect(row.status).toBe('abandoned');
+    expect(row.error).toBe('boom');
+    expect(row.finishedAt).toBeGreaterThan(0);
+  });
+
+  it('--subscriber scopes both the rows and the total', async () => {
+    const mine = await handleEventsListFailed([], { subscriber: 'namecheap.ddns', limit: '3' });
+    if (!mine.success) throw new Error('expected success');
+    expect((mine.data as FailedDeliveryReport).total).toBe(60);
+
+    const other = await handleEventsListFailed([], { subscriber: 'nobody' });
+    if (!other.success) throw new Error('expected success');
+    expect((other.data as FailedDeliveryReport).total).toBe(0);
+    expect((other.data as FailedDeliveryReport).deliveries).toHaveLength(0);
+  });
+});

package/src/cli/commands/events.ts

@@ -10,6 +10,7 @@
  *   tail                 recent events
  *   list-subscribers     persistent subscribers
  *   list-pending         pending deliveries (subscriber fan-out — NOT questions)
+ *   list-failed          failed/abandoned deliveries + a TRUE total
  *   list-unanswered      interview questions nobody has answered yet
  *   drain                process pending deliveries once
  *   run                  long-running dispatcher (foreground; SIGINT to stop)
@@ -24,7 +25,9 @@
 import { spawnSync } from 'node:child_process';
 import {
   BUS_VERSION,
+  type FailedBySubscriber,
   defineEvents,
+  describeError,
   drainOnce,
   openBus,
   recoverFromCrash,
@@ -246,6 +249,75 @@ export async function handleEventsListPending(
   }
 }
 
+/** One failed/abandoned delivery, as `events list-failed` reports it. */
+export interface FailedDelivery {
+  eventId: number;
+  eventType: string | null;
+  subscriber: string | null;
+  status: string;
+  attempts: number;
+  finishedAt: number | null;
+  error: string | null;
+}
+
+/** What `events list-failed` returns: the true total, the shape, and a sample. */
+export interface FailedDeliveryReport {
+  total: number;
+  bySubscriber: FailedBySubscriber[];
+  shown: number;
+  deliveries: FailedDelivery[];
+}
+
+/**
+ * `celilo events list-failed` — what has failed or been abandoned, who it was
+ * for, and why.
+ *
+ * The instrument celilo#623 lacked: `failedDeliveries` had exactly one caller
+ * (the doctor) and no operator surface at all, so the only signal was a count
+ * that was really a LIMIT. `total` here is a real COUNT — `deliveries` is the
+ * sample, capped by `--limit`, and `shown` says so.
+ *
+ * This is NOT `list-pending`, which reads deliveries still queued.
+ */
+export async function handleEventsListFailed(
+  _args: string[],
+  flags: Record<string, string | boolean>,
+): Promise<CommandResult> {
+  const bus = openCliBus();
+  try {
+    const subscriber = typeof flags.subscriber === 'string' ? flags.subscriber : undefined;
+    const limit = flags.limit ? Number(flags.limit) : 50;
+    const { total, bySubscriber } = bus.failedDeliveryTotals();
+    // Subscriber/event names live in other tables; the delivery row has ids.
+    const named = new Map(
+      bus.db
+        .query<{ id: number; name: string }, []>('SELECT id, name FROM subscribers')
+        .all()
+        .map((r) => [r.id, r.name]),
+    );
+    const deliveries: FailedDelivery[] = bus.failedDeliveries({ limit, subscriber }).map((d) => ({
+      eventId: d.eventId,
+      eventType: bus.getEvent(d.eventId)?.type ?? null,
+      subscriber: named.get(d.subscriberId) ?? null,
+      status: d.status,
+      attempts: d.attempts,
+      finishedAt: d.finishedAt,
+      error: describeError(d.lastError),
+    }));
+    const report: FailedDeliveryReport = {
+      total: subscriber
+        ? (bySubscriber.find((s) => s.subscriber === subscriber)?.count ?? 0)
+        : total,
+      bySubscriber,
+      shown: deliveries.length,
+      deliveries,
+    };
+    return jsonResult(report);
+  } finally {
+    bus.close();
+  }
+}
+
 /** One unanswered interview question, as `events list-unanswered` reports it. */
 export interface UnansweredInterview {
   eventId: number;

package/src/cli/completion.ts

@@ -113,6 +113,7 @@ export async function getCompletions(words: string[], current: number): Promise<
       'list-subscribers',
       'resync-subscriptions',
       'list-pending',
+      'list-failed',
       'list-unanswered',
       'drain',
       'run',

package/src/cli/index.ts

@@ -32,6 +32,7 @@ import {
   handleEventsEmit,
   handleEventsFail,
   handleEventsInstallDaemon,
+  handleEventsListFailed,
   handleEventsListPending,
   handleEventsListSubscribers,
   handleEventsListUnanswered,
@@ -300,6 +301,7 @@ Subcommands:
   list-subscribers             List persistent bus subscribers
   resync-subscriptions         Rebuild subscribers from deployed modules' manifests (after a restore/migration)
   list-pending [--subscriber]  List pending deliveries
+  list-failed [--subscriber]   List failed/abandoned deliveries with a true total
   drain [--concurrency N]      Process pending deliveries once and return
   run [--poll-ms N]            Run the long-running dispatcher (foreground)
   emit <type> [<payload>]      Emit an event (operator/test path)
@@ -1425,6 +1427,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
         return handleEventsResyncSubscriptions();
       case 'list-pending':
         return handleEventsListPending(parsed.args, parsed.flags);
+      case 'list-failed':
+        return handleEventsListFailed(parsed.args, parsed.flags);
       case 'list-unanswered':
         return handleEventsListUnanswered(parsed.args, parsed.flags);
       case 'drain':

package/src/services/dns-registrations.ts

@@ -40,8 +40,20 @@ export interface DnsRegistrationRow {
  * triggers for FK-cascaded deletes only when `recursive_triggers` is on,
  * and reads are the only thing that consumes the ledger. Move it into a
  * trigger if something ever reads these rows without going through here.
+ *
+ * It CHECKS before deleting, so the overwhelmingly common case (nothing
+ * orphaned) stays a pure read. The first version ran the DELETE
+ * unconditionally, which took a write lock on every list — including the
+ * refresh hook's, and `celilo dns registrations`. A read that quietly writes
+ * is a surprise on its own, and on SQLite it is a surprise that serialises
+ * against every other writer for no benefit.
  */
 function pruneOrphanedRegistrations(db: DbClient): void {
+  const orphaned = sql`SELECT 1 FROM dns_registrations WHERE NOT EXISTS (
+      SELECT 1 FROM dns_registration_consumers c WHERE c.registration_id = dns_registrations.id
+    ) LIMIT 1`;
+  if (!db.get(orphaned)) return;
+
   db.run(
     sql`DELETE FROM dns_registrations WHERE NOT EXISTS (
       SELECT 1 FROM dns_registration_consumers c WHERE c.registration_id = dns_registrations.id

package/src/services/fleet-checks.test.ts

@@ -42,6 +42,19 @@ function seedHeartbeat(
   );
 }
 
+/** Abandon `count` deliveries to one subscriber, oldest first. */
+function seedFailed(bus: Bus, count: number): void {
+  const sub = bus.subscribe({ name: 'namecheap.ddns', pattern: 'ddns.refresh', handler: 'echo' });
+  for (let i = 0; i < count; i++) {
+    const event = bus.emitRaw('ddns.refresh', { n: i });
+    bus.markFailed(
+      { eventId: event.id, subscriberId: sub.id },
+      new Error('handler timed out after 30000ms'),
+      { abandoned: true },
+    );
+  }
+}
+
 /** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
 function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
   const path = getDaemonUnitPath('linux', home, scope, systemRoot);
@@ -199,6 +212,39 @@ describe('checkDispatcher', () => {
     expect(f.status).toBe('warn');
     expect(f.detail.join(' ')).toContain('not emitting on schedule');
   });
+
+  // celilo#623 — the old check read `failedDeliveries({ limit: 50 }).length`,
+  // so on celilo-mgr it printed a literal `50` that meant "at least 50" and
+  // read as an exact count. 137 > any limit anyone would pick.
+  it('reports the TRUE total of failed deliveries, not the read limit', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 137);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('137 failed/abandoned delivery(ies) total');
+    expect(f.detail.join(' ')).not.toContain('50 failed');
+  });
+
+  // The stored error is double-wrapped for every row written before the
+  // serializeError fix; those live 90 days, so doctor must unwrap them.
+  it('renders the sample error text, not {"message":"[object Object]"}', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 1);
+    bus.db.run('UPDATE deliveries SET last_error = ?', [
+      JSON.stringify({
+        message: '[object Object]',
+        value: { message: 'handler exited with code 1' },
+      }),
+    ]);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    const detail = f.detail.join(' ');
+    expect(detail).toContain('handler exited with code 1');
+    expect(detail).not.toContain('[object Object]');
+  });
 });
 
 describe('checkSubscribers + checkCapabilityProviders', () => {

package/src/services/fleet-checks.ts

@@ -17,7 +17,7 @@
  * rendering + `--fix` orchestration lives in the doctor command.
  */
 
-import type { Bus } from '@celilo/event-bus';
+import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
@@ -343,12 +343,21 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     }
   }
 
-  const failed = bus.failedDeliveries({ limit: 50 });
-  if (failed.length > 0) {
+  // A TRUE total, not `failedDeliveries().length` — that saturates at its LIMIT
+  // and printed a literal `50` on celilo-mgr that read as a count (celilo#623).
+  // The newest row dates the backlog: a big total whose newest entry is days old
+  // is drained history, not active bleeding.
+  const { total: failedTotal } = bus.failedDeliveryTotals();
+  if (failedTotal > 0) {
     statuses.push('warn');
-    const sample = failed[0]?.lastError ? ` (e.g. ${failed[0].lastError.split('\n')[0]})` : '';
-    detail.push(`${failed.length} failed/abandoned delivery(ies)${sample}`);
-    remediations.push('inspect failed deliveries and re-emit/repair as needed');
+    const newest = bus.failedDeliveries({ limit: 1 })[0];
+    const age = newest?.finishedAt
+      ? `, most recent ${Math.round((now - newest.finishedAt) / 60000)}min ago`
+      : '';
+    detail.push(`${failedTotal} failed/abandoned delivery(ies) total${age}`);
+    const sample = describeError(newest?.lastError ?? null)?.split('\n')[0];
+    if (sample) detail.push(`  example (newest, not the only one): ${sample}`);
+    remediations.push('`celilo events list-failed` to see them; re-emit/repair as needed');
   }
 
   const status = worst(statuses);