@celilo/cli 0.20.0 → 0.22.0

package/src/services/fleet-checks.test.ts

@@ -14,6 +14,8 @@ import {
 } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monitors';
+import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
   checkCapabilityProviders,
@@ -25,6 +27,7 @@ import {
   describeCapabilityProblem,
   findBrokenCapabilityDerivations,
 } from './fleet-checks';
+import { ensureOperationsSweepSubscriber } from './module-operations';
 
 const MINUTE = 60_000;
 
@@ -39,6 +42,19 @@ function seedHeartbeat(
   );
 }
 
+/** Abandon `count` deliveries to one subscriber, oldest first. */
+function seedFailed(bus: Bus, count: number): void {
+  const sub = bus.subscribe({ name: 'namecheap.ddns', pattern: 'ddns.refresh', handler: 'echo' });
+  for (let i = 0; i < count; i++) {
+    const event = bus.emitRaw('ddns.refresh', { n: i });
+    bus.markFailed(
+      { eventId: event.id, subscriberId: sub.id },
+      new Error('handler timed out after 30000ms'),
+      { abandoned: true },
+    );
+  }
+}
+
 /** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
 function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
   const path = getDaemonUnitPath('linux', home, scope, systemRoot);
@@ -196,6 +212,39 @@ describe('checkDispatcher', () => {
     expect(f.status).toBe('warn');
     expect(f.detail.join(' ')).toContain('not emitting on schedule');
   });
+
+  // celilo#623 — the old check read `failedDeliveries({ limit: 50 }).length`,
+  // so on celilo-mgr it printed a literal `50` that meant "at least 50" and
+  // read as an exact count. 137 > any limit anyone would pick.
+  it('reports the TRUE total of failed deliveries, not the read limit', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 137);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('137 failed/abandoned delivery(ies) total');
+    expect(f.detail.join(' ')).not.toContain('50 failed');
+  });
+
+  // The stored error is double-wrapped for every row written before the
+  // serializeError fix; those live 90 days, so doctor must unwrap them.
+  it('renders the sample error text, not {"message":"[object Object]"}', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 1);
+    bus.db.run('UPDATE deliveries SET last_error = ?', [
+      JSON.stringify({
+        message: '[object Object]',
+        value: { message: 'handler exited with code 1' },
+      }),
+    ]);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    const detail = f.detail.join(' ');
+    expect(detail).toContain('handler exited with code 1');
+    expect(detail).not.toContain('[object Object]');
+  });
 });
 
 describe('checkSubscribers + checkCapabilityProviders', () => {
@@ -274,10 +323,53 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
 
     it('warns about a stale subscriber with no deployed module', () => {
       // No modules deployed, but a leftover subscriber lingers.
-      bus.subscribe({ name: 'ghost.sub', pattern: 'x', handler: 'echo' });
+      bus.subscribe({ name: 'ghost.sub', pattern: 'x', handler: 'echo', registeredBy: 'ghost' });
       const f = checkSubscribers(bus, db);
       expect(f.status).toBe('warn');
       expect(f.detail.join(' ')).toContain('ghost.sub');
+      // `resync-subscriptions` never deletes, so it cannot clear this (#624).
+      expect(f.autoFixable).toBe(false);
+      expect(f.remediation).toContain('subscribers remove');
+    });
+
+    it('ignores core-registered subscribers that no manifest declares (#624)', () => {
+      ensureSweepSubscriber(bus);
+      ensureInboundSubscriber(bus);
+      ensureBackupSweepSubscriber(bus);
+      ensureOperationsSweepSubscriber(bus);
+      const f = checkSubscribers(bus, db);
+      expect(f.status).toBe('ok');
+      expect(f.detail.join(' ')).not.toContain('celilo-');
+    });
+
+    // The guard for the invariant `checkSubscribers` classifies by, asserted
+    // at the sites that ESTABLISH it rather than the one that consumes it —
+    // so it keeps holding if the predicate is ever rewritten. Goes red the
+    // moment a core registrar names its row under its own registrar id
+    // (`celilo-alerting` registering `celilo-alerting.digest`), which would
+    // read as a module row, find no module of that name, and be reported
+    // stale — #624 again, and baffling to whoever hit it.
+    //
+    // ponytail: enumerates the registrars by hand because celilo has no
+    // registry of them. A fifth one is not covered until it is added here;
+    // if that ever bites, the upgrade is a shared `registerCoreSubscriber`
+    // helper that every core site goes through, tested once.
+    it('no core registrar produces a module-shaped subscriber name', () => {
+      ensureSweepSubscriber(bus);
+      ensureInboundSubscriber(bus);
+      ensureBackupSweepSubscriber(bus);
+      ensureOperationsSweepSubscriber(bus);
+
+      const rows = bus.db
+        .query<{ name: string; registered_by: string | null }, []>(
+          'SELECT name, registered_by FROM subscribers',
+        )
+        .all();
+      expect(rows.length).toBeGreaterThan(0);
+      for (const row of rows) {
+        expect(row.registered_by).not.toBeNull();
+        expect(row.name.startsWith(`${row.registered_by}.`)).toBe(false);
+      }
     });
   });
 

package/src/services/fleet-checks.ts

@@ -17,7 +17,7 @@
  * rendering + `--fix` orchestration lives in the doctor command.
  */
 
-import type { Bus } from '@celilo/event-bus';
+import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
@@ -343,12 +343,21 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     }
   }
 
-  const failed = bus.failedDeliveries({ limit: 50 });
-  if (failed.length > 0) {
+  // A TRUE total, not `failedDeliveries().length` — that saturates at its LIMIT
+  // and printed a literal `50` on celilo-mgr that read as a count (celilo#623).
+  // The newest row dates the backlog: a big total whose newest entry is days old
+  // is drained history, not active bleeding.
+  const { total: failedTotal } = bus.failedDeliveryTotals();
+  if (failedTotal > 0) {
     statuses.push('warn');
-    const sample = failed[0]?.lastError ? ` (e.g. ${failed[0].lastError.split('\n')[0]})` : '';
-    detail.push(`${failed.length} failed/abandoned delivery(ies)${sample}`);
-    remediations.push('inspect failed deliveries and re-emit/repair as needed');
+    const newest = bus.failedDeliveries({ limit: 1 })[0];
+    const age = newest?.finishedAt
+      ? `, most recent ${Math.round((now - newest.finishedAt) / 60000)}min ago`
+      : '';
+    detail.push(`${failedTotal} failed/abandoned delivery(ies) total${age}`);
+    const sample = describeError(newest?.lastError ?? null)?.split('\n')[0];
+    if (sample) detail.push(`  example (newest, not the only one): ${sample}`);
+    remediations.push('`celilo events list-failed` to see them; re-emit/repair as needed');
   }
 
   const status = worst(statuses);
@@ -386,6 +395,9 @@ function loadDeployedModules(db: DbClient): DeployedModule[] {
  * manifests declare. A restore/migration starts it EMPTY (ISS-0088), so
  * every reactive subscription silently vanishes until a resync or a
  * redeploy. Missing rows fail; stale rows (a since-removed module) warn.
+ *
+ * Scoped to MODULE-owned rows — core's own subscribers share this bus and are
+ * declared by no manifest.
  */
 export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
   const deployed = loadDeployedModules(db);
@@ -402,7 +414,9 @@ export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
   }
 
   const actualRows = bus.db
-    .query<{ name: string; pattern: string }, []>('SELECT name, pattern FROM subscribers')
+    .query<{ name: string; pattern: string; registered_by: string | null }, []>(
+      'SELECT name, pattern, registered_by FROM subscribers',
+    )
     .all();
   const actual = new Map(actualRows.map((r) => [r.name, r.pattern]));
 
@@ -413,7 +427,25 @@ export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
     if (have === undefined) missing.push(name);
     else if (have !== pattern) mismatched.push(`${name} (manifest: ${pattern}, bus: ${have})`);
   }
-  const stale = actualRows.map((r) => r.name).filter((name) => !expected.has(name));
+  // Only module-owned rows can be stale. celilo core registers subscribers on
+  // the same bus (the alerting/backup/operations sweeps) that no manifest
+  // declares, so judging every row against the manifest-derived map reported
+  // them as drift on every install, forever (#624).
+  //
+  // INVARIANT this relies on: a module row is named `<registered_by>.<sub>` —
+  // `resolveSubscription` (services/module-subscriptions.ts) derives both the
+  // scoped name and registered_by from the same module id. A core row is NOT
+  // scoped under its registrar's id (`celilo-alerting` registers
+  // `celilo-alerting-sweep`, not `celilo-alerting.sweep`). That's what lets
+  // provenance separate them with no allowlist to keep in sync.
+  //
+  // To break it: have a core registrar name a row under its own id — a future
+  // `celilo-alerting.digest` would read as module-owned, find no deployed
+  // module named `celilo-alerting`, and be reported stale. Which is #624 again.
+  const stale = actualRows
+    .filter((r) => r.registered_by !== null && r.name.startsWith(`${r.registered_by}.`))
+    .map((r) => r.name)
+    .filter((name) => !expected.has(name));
 
   const detail: string[] = [];
   const statuses: FleetFindingStatus[] = [];
@@ -436,6 +468,15 @@ export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
     );
   }
 
+  // `resync-subscriptions` only upserts from manifests — it never deletes, so
+  // it cannot clear a stale row. Advertising that as the remedy (and as
+  // auto-fixable) made `--fix` run a guaranteed no-op and call it a fix.
+  const resyncable = missing.length > 0 || mismatched.length > 0;
+  const remediations: string[] = [];
+  if (resyncable) remediations.push('`celilo events resync-subscriptions` (safe, idempotent)');
+  if (stale.length > 0)
+    remediations.push(`\`celilo subscribers remove <name>\` for: ${stale.join(', ')}`);
+
   const status = worst(statuses);
   return {
     id: 'subscribers',
@@ -446,8 +487,8 @@ export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
         ? `${expected.size} subscription(s) match the deployed fleet`
         : 'bus subscribers drifted from the deployed fleet',
     detail,
-    remediation: status === 'ok' ? null : '`celilo events resync-subscriptions` (safe, idempotent)',
-    autoFixable: status !== 'ok',
+    remediation: remediations.length > 0 ? remediations.join('; ') : null,
+    autoFixable: resyncable,
   };
 }
 

package/src/services/module-subscriptions.test.ts

@@ -46,6 +46,15 @@ describe('resolveSubscription', () => {
     expect(resolved.registeredBy).toBe('lunacycle');
   });
 
+  // The other half of the invariant `checkSubscribers` classifies by (#624):
+  // core rows are not scoped under their registrar id, module rows always are.
+  // Stop scoping the name here and every module row reads as core-owned — the
+  // stale check would go silently blind rather than noisily wrong.
+  it('always scopes the name under the id it stamps in registeredBy', () => {
+    const resolved = resolveSubscription({ name: 'a', pattern: 'x', handler: 'echo' }, 'foo', '/p');
+    expect(resolved.name.startsWith(`${resolved.registeredBy}.`)).toBe(true);
+  });
+
   it('only substitutes $self when followed by . or end-of-string', () => {
     // `$selfish` would not be a real pattern but we want to ensure the
     // substitution doesn't accidentally rewrite identifier-like names.

package/src/services/public-dns-probe.test.ts

@@ -0,0 +1,81 @@
+/**
+ * The gate for the original bug, one layer up.
+ *
+ * A `public_dns` check that quietly resolved through the fleet's own resolver
+ * would pass forever whatever the internet sees — exactly like caddy's `dig`
+ * check does today, and exactly the blindness that let celilo#626 run for nine
+ * days. So "the resolver is off-fleet" is asserted rather than left to a
+ * comment, and the assertion lives where the probe is built.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import {
+  DEFAULT_PUBLIC_RESOLVER,
+  FleetResolverAsPublicVantageError,
+  assertOffFleetResolver,
+  parseFleetResolvers,
+} from './public-dns-probe';
+
+const FLEET = [
+  { role: 'dns.primary', ip: '10.0.20.5' },
+  { role: 'dns.fallback', ip: '8.8.8.8' },
+];
+
+describe('assertOffFleetResolver', () => {
+  test("refuses the fleet's own resolver", () => {
+    expect(() => assertOffFleetResolver('10.0.20.5', FLEET)).toThrow(
+      FleetResolverAsPublicVantageError,
+    );
+  });
+
+  test('refuses it even when the fleet resolver is a public address', () => {
+    // The failure is "the fleet asks this resolver too", not "the address is
+    // private": a fleet configured to forward to 8.8.8.8 gets the same
+    // split-horizon answers back through it.
+    expect(() => assertOffFleetResolver('8.8.8.8', FLEET)).toThrow(
+      FleetResolverAsPublicVantageError,
+    );
+  });
+
+  test('accepts a resolver the fleet does not use', () => {
+    expect(() => assertOffFleetResolver(DEFAULT_PUBLIC_RESOLVER, FLEET)).not.toThrow();
+  });
+
+  test('the refusal says how to fix it', () => {
+    try {
+      assertOffFleetResolver('10.0.20.5', FLEET);
+      throw new Error('expected a refusal');
+    } catch (error) {
+      expect((error as Error).message).toContain('celilo system config set public_dns.resolver');
+    }
+  });
+});
+
+describe('parseFleetResolvers', () => {
+  test('a comma-separated dns.fallback yields one entry per address', () => {
+    // `system init` writes `dns.fallback=1.0.0.1,8.8.8.8`. Compared whole, the
+    // guard matched neither address and would have accepted 8.8.8.8 as the
+    // "off-fleet" vantage point of a fleet that forwards to 8.8.8.8.
+    expect(
+      parseFleetResolvers([
+        { role: 'dns.primary', value: '10.0.20.5' },
+        { role: 'dns.fallback', value: '1.0.0.1, 8.8.8.8' },
+      ]),
+    ).toEqual([
+      { role: 'dns.primary', ip: '10.0.20.5' },
+      { role: 'dns.fallback', ip: '1.0.0.1' },
+      { role: 'dns.fallback', ip: '8.8.8.8' },
+    ]);
+  });
+
+  test('an unset key contributes nothing', () => {
+    expect(parseFleetResolvers([{ role: 'dns.primary', value: undefined }])).toEqual([]);
+  });
+
+  test('a multi-entry fallback is caught by the guard', () => {
+    const fleet = parseFleetResolvers([{ role: 'dns.fallback', value: '1.0.0.1,8.8.8.8' }]);
+    expect(() => assertOffFleetResolver('8.8.8.8', fleet)).toThrow(
+      FleetResolverAsPublicVantageError,
+    );
+  });
+});

package/src/services/public-dns-probe.ts

@@ -0,0 +1,156 @@
+/**
+ * The off-fleet vantage point for the `public_dns` check.
+ *
+ * Two independent third parties, deliberately:
+ *
+ *  - a **resolver that is not the fleet's**, asked what the internet resolves
+ *    for each name. The fleet's own resolver runs split-horizon and answers
+ *    with an address that is reachable in-zone — correct for its purpose, and
+ *    not evidence about the public internet. A `public_dns` check that quietly
+ *    used it would pass forever, which is the original defect one layer up, so
+ *    `assertOffFleetResolver` refuses rather than trusting a code comment.
+ *  - an **echo service**, for what the fleet's public ingress address actually
+ *    is. Not the registrar's response: comparing what was published against
+ *    what we asked to publish is self-agreement, and Namecheap answers
+ *    `ErrCount 0` for updates it does not apply (design.md D2/D3).
+ *
+ * Both are configurable, and both are named in every finding they produce.
+ */
+
+import { Resolver } from 'node:dns/promises';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { systemConfig } from '../db/schema';
+import type { IngressObservation, PublicDnsProbe, PublicResolution } from './audit/public-dns';
+
+/** Cloudflare. Overridable — the requirement is that it is not the fleet's. */
+export const DEFAULT_PUBLIC_RESOLVER = '1.1.1.1';
+export const DEFAULT_ECHO_URL = 'https://api.ipify.org';
+
+const PROBE_TIMEOUT_MS = 5_000;
+
+export class FleetResolverAsPublicVantageError extends Error {
+  constructor(resolver: string, role: string) {
+    super(
+      `Refusing to check public DNS through ${resolver}: it is the fleet's own resolver (${role}).\nThe fleet resolver runs split-horizon and answers with an in-zone address, so a\ncheck that used it would pass whatever the internet sees — which is exactly how\ncelilo#626 stayed invisible for nine days.\n\nSet an off-fleet resolver:\n  celilo system config set public_dns.resolver 1.1.1.1`,
+    );
+    this.name = 'FleetResolverAsPublicVantageError';
+  }
+}
+
+/**
+ * Reject a resolver the fleet itself uses. Pure so the gate is unit-testable
+ * without a database — it is the assertion §5.2 asks for.
+ */
+export function assertOffFleetResolver(
+  resolver: string,
+  fleetResolvers: { role: string; ip: string }[],
+): void {
+  const match = fleetResolvers.find((r) => r.ip === resolver);
+  if (match) throw new FleetResolverAsPublicVantageError(resolver, match.role);
+}
+
+function configValue(db: DbClient, key: string): string | undefined {
+  const row = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
+  const value = row?.value?.trim();
+  return value && value.length > 0 ? value : undefined;
+}
+
+/**
+ * Every resolver address the fleet is configured to use for its own lookups.
+ *
+ * `dns.fallback` holds a COMMA-SEPARATED list (`1.0.0.1,8.8.8.8` is what
+ * `system init` writes), so it is split rather than compared whole. Treating it
+ * as one string made the guard below miss every fallback but a single-entry
+ * one — a fleet forwarding to 8.8.8.8 could have been handed 8.8.8.8 as its
+ * "off-fleet" vantage point and the check would have agreed with itself
+ * forever, which is precisely the failure this guard exists to prevent.
+ */
+export function fleetResolvers(db: DbClient): { role: string; ip: string }[] {
+  return parseFleetResolvers(
+    ['dns.primary', 'dns.fallback'].map((role) => ({ role, value: configValue(db, role) })),
+  );
+}
+
+/** The parse, split from the read so the comma handling is testable on its own. */
+export function parseFleetResolvers(
+  entries: { role: string; value: string | undefined }[],
+): { role: string; ip: string }[] {
+  const resolvers: { role: string; ip: string }[] = [];
+  for (const { role, value } of entries) {
+    for (const ip of (value ?? '').split(',')) {
+      const trimmed = ip.trim();
+      if (trimmed) resolvers.push({ role, ip: trimmed });
+    }
+  }
+  return resolvers;
+}
+
+export interface PublicDnsProbeSettings {
+  resolver: string;
+  echoUrl: string;
+}
+
+export function loadPublicDnsProbeSettings(db: DbClient): PublicDnsProbeSettings {
+  const resolver = configValue(db, 'public_dns.resolver') ?? DEFAULT_PUBLIC_RESOLVER;
+  assertOffFleetResolver(resolver, fleetResolvers(db));
+  return { resolver, echoUrl: configValue(db, 'public_dns.echo_url') ?? DEFAULT_ECHO_URL };
+}
+
+/**
+ * `fetch` with a bound, so an unanswered echo request cannot hang a scheduled
+ * check (the shape celilo#622 fixed for DDNS).
+ */
+async function fetchIngress(echoUrl: string): Promise<IngressObservation> {
+  try {
+    const response = await fetch(echoUrl, {
+      signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+    });
+    if (!response.ok) {
+      return { kind: 'undetermined', reason: `HTTP ${response.status}` };
+    }
+    const ip = (await response.text()).trim();
+    if (!/^\d{1,3}(\.\d{1,3}){3}$/.test(ip)) {
+      return { kind: 'undetermined', reason: `unparseable answer: ${ip.slice(0, 40)}` };
+    }
+    return { kind: 'observed', ip };
+  } catch (error) {
+    return {
+      kind: 'undetermined',
+      reason: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export function createPublicDnsProbe(settings: PublicDnsProbeSettings): PublicDnsProbe {
+  const resolver = new Resolver({ timeout: PROBE_TIMEOUT_MS, tries: 2 });
+  resolver.setServers([settings.resolver]);
+
+  return {
+    resolver: settings.resolver,
+    echoService: settings.echoUrl,
+
+    observeIngress: () => fetchIngress(settings.echoUrl),
+
+    async resolve(fqdn: string): Promise<PublicResolution> {
+      try {
+        // `ttl: true` is why this uses node:dns rather than shelling out to
+        // dig: the TTL is what the hysteresis window is measured in.
+        const answers = await resolver.resolve4(fqdn, { ttl: true });
+        const first = answers[0];
+        if (!first) return { kind: 'no_record' };
+        return { kind: 'answer', ip: first.address, ttlSeconds: first.ttl };
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+        // NXDOMAIN / NODATA are authoritative answers: the name really has no
+        // A record. Everything else (timeout, SERVFAIL, refused) means the
+        // probe could not look, which is never a pass.
+        if (code === 'ENOTFOUND' || code === 'ENODATA') return { kind: 'no_record' };
+        return {
+          kind: 'undetermined',
+          reason: code ?? (error instanceof Error ? error.message : String(error)),
+        };
+      }
+    },
+  };
+}

package/src/services/update/orchestrator.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test';
 import type { DbClient } from '../../db/client';
 import type { ModuleManifest } from '../../manifest/schema';
 import type { AuditDeps } from '../audit';
+import { unusedPublicDnsProbe } from '../audit/public-dns';
 import { buildModuleGraph } from './dep-graph';
 import {
   type ModuleSnapshot,
@@ -85,6 +86,7 @@ const cleanAudit: AuditDeps = {
   machinesReachable: { results: [] },
   transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
   trustedSources: { firewalls: [] },
+  publicDns: { records: [], probe: unusedPublicDnsProbe },
 };
 
 describe('consumerSkipReason', () => {