@celilo/cli 0.20.0 → 0.21.0

package/src/services/audit/index.ts

@@ -21,6 +21,7 @@ import { type HealthAuditDeps, auditHealth } from './health';
 import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
 import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
 import { type ModuleVersionsAuditDeps, auditModuleVersions } from './module-versions';
+import { type PublicDnsAuditDeps, auditPublicDns } from './public-dns';
 import { type SchemaAuditDeps, auditSchema } from './schema';
 import { type SecretsDecryptableAuditDeps, auditSecretsDecryptable } from './secrets-decryptable';
 import {
@@ -59,6 +60,7 @@ export interface AuditDeps {
   secretsDecryptable: SecretsDecryptableAuditDeps;
   servicesReachable: ServicesReachableAuditDeps;
   machinesReachable: MachinesReachableAuditDeps;
+  publicDns: PublicDnsAuditDeps;
   transportReads: TransportReadsAuditDeps;
   trustedSources: TrustedSourcesAuditDeps;
   /** Defaults to `Date.now()`-based ISO string. */
@@ -114,6 +116,7 @@ export async function runAudit(
     wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
     wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
     wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
+    wrap('public_dns', auditPublicDns(deps.publicDns)),
     wrap('transport_reads', auditTransportReads(deps.transportReads)),
     wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
   ]);

package/src/services/audit/public-dns-source.ts

@@ -0,0 +1,55 @@
+/**
+ * DB-facing half of the `public_dns` check: the ledger names it watches and
+ * the consecutive-undetermined counters it carries between runs.
+ *
+ * Split from `public-dns.ts` so the audit itself stays a pure function over an
+ * injected probe — the counters are the only state it has, and a check that
+ * owned its own storage could not be tested without one.
+ */
+
+import type { DbClient } from '../../db/client';
+import { publicDnsEvidence } from '../../db/schema';
+import { listDnsRegistrations } from '../dns-registrations';
+import type { PublicDnsEvidence, PublicDnsRecord } from './public-dns';
+
+/**
+ * Every name celilo has asserted publicly. `lastAssertedAt` is when the fleet
+ * last told the registrar about it — the point the record's own TTL is
+ * measured from, so a divergence inside one TTL reads as propagation.
+ */
+export function loadPublicDnsRecords(db: DbClient): PublicDnsRecord[] {
+  return listDnsRegistrations(db).map((row) => ({
+    fqdn: row.fqdn,
+    companion: row.companion,
+    lastAssertedAt: row.refreshedAt ?? row.registeredAt,
+  }));
+}
+
+export function loadPublicDnsEvidence(db: DbClient): PublicDnsEvidence[] {
+  return db
+    .select({
+      subject: publicDnsEvidence.subject,
+      undeterminedRuns: publicDnsEvidence.undeterminedRuns,
+    })
+    .from(publicDnsEvidence)
+    .all();
+}
+
+/**
+ * Replace the whole set. A subject that produced evidence this run is absent
+ * from `evidence` and its counter goes with it — that is what makes the count
+ * consecutive rather than cumulative.
+ */
+export function savePublicDnsEvidence(db: DbClient, evidence: PublicDnsEvidence[]): void {
+  db.delete(publicDnsEvidence).run();
+  if (evidence.length === 0) return;
+  db.insert(publicDnsEvidence)
+    .values(
+      evidence.map((e) => ({
+        subject: e.subject,
+        undeterminedRuns: e.undeterminedRuns,
+        lastCheckedAt: new Date(),
+      })),
+    )
+    .run();
+}

package/src/services/audit/public-dns.test.ts

@@ -0,0 +1,209 @@
+/**
+ * The recurrence gate celilo#626 asks for, at the layer it can actually be
+ * asserted: feed the check a public record that diverges from where the fleet
+ * really is, and it must say so.
+ *
+ * Every check celilo had answered "healthy" for the nine days five public
+ * names were dark. This is the one that could not have.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import {
+  type IngressObservation,
+  type PublicDnsEvidence,
+  type PublicDnsProbe,
+  type PublicResolution,
+  auditPublicDns,
+} from './public-dns';
+
+const FLEET_INGRESS = '71.36.123.107';
+/** The address the fleet moved off, and kept publishing for nine days. */
+const DEAD_ADDRESS = '71.36.112.98';
+
+const LONG_AGO = new Date('2026-08-01T00:00:00Z');
+const NOW = new Date('2026-08-06T00:00:00Z');
+
+function probe(options: {
+  ingress?: IngressObservation;
+  answers?: Record<string, PublicResolution>;
+  fallback?: PublicResolution;
+}): PublicDnsProbe {
+  return {
+    resolver: '1.1.1.1',
+    echoService: 'https://echo.example',
+    async observeIngress() {
+      return options.ingress ?? { kind: 'observed', ip: FLEET_INGRESS };
+    },
+    async resolve(fqdn) {
+      return (
+        options.answers?.[fqdn] ??
+        options.fallback ?? { kind: 'answer', ip: FLEET_INGRESS, ttlSeconds: 180 }
+      );
+    },
+  };
+}
+
+function declared(fqdn: string, lastAssertedAt = LONG_AGO) {
+  return { fqdn, companion: false, lastAssertedAt };
+}
+
+describe('auditPublicDns', () => {
+  test('a record pointing at an address the fleet no longer holds is a finding', async () => {
+    const findings = await auditPublicDns({
+      records: [declared('apt.celilo.computer')],
+      probe: probe({
+        answers: {
+          'apt.celilo.computer': { kind: 'answer', ip: DEAD_ADDRESS, ttlSeconds: 180 },
+        },
+      }),
+      now: NOW,
+    });
+
+    expect(findings.length).toBe(1);
+    expect(findings[0].code).toBe('public_dns_stale');
+    expect(findings[0].subject).toBe('apt.celilo.computer');
+    // The finding has to name both addresses; "DNS is wrong" is not actionable.
+    expect(findings[0].message).toContain(DEAD_ADDRESS);
+    expect(findings[0].message).toContain(FLEET_INGRESS);
+  });
+
+  test('a record that agrees with the fleet is silent', async () => {
+    const findings = await auditPublicDns({
+      records: [declared('git.celilo.computer')],
+      probe: probe({}),
+      now: NOW,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  // ── Hysteresis: propagation is not a fault ────────────────────────────────
+  test('divergence inside the record TTL is propagation, not a finding', async () => {
+    const assertedSecondsAgo = 60;
+    const findings = await auditPublicDns({
+      records: [
+        declared('git.celilo.computer', new Date(NOW.getTime() - assertedSecondsAgo * 1000)),
+      ],
+      probe: probe({
+        answers: {
+          'git.celilo.computer': { kind: 'answer', ip: DEAD_ADDRESS, ttlSeconds: 180 },
+        },
+      }),
+      now: NOW,
+    });
+    // An ISP re-lease produces exactly this for up to one TTL. Alerting here
+    // would page on every re-lease, and a check nobody trusts is not a check.
+    expect(findings).toEqual([]);
+  });
+
+  test('the same divergence once the TTL has elapsed IS a finding', async () => {
+    const findings = await auditPublicDns({
+      records: [declared('git.celilo.computer', new Date(NOW.getTime() - 181 * 1000))],
+      probe: probe({
+        answers: {
+          'git.celilo.computer': { kind: 'answer', ip: DEAD_ADDRESS, ttlSeconds: 180 },
+        },
+      }),
+      now: NOW,
+    });
+    expect(findings.map((f) => f.code)).toEqual(['public_dns_stale']);
+  });
+
+  // ── Missing evidence is louder than absent evidence ───────────────────────
+  test('one undetermined run stays quiet', async () => {
+    const findings = await auditPublicDns({
+      records: [declared('git.celilo.computer')],
+      probe: probe({ ingress: { kind: 'undetermined', reason: 'ECONNREFUSED' } }),
+      now: NOW,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  test('consecutive undetermined runs become their own finding', async () => {
+    let evidence: PublicDnsEvidence[] = [];
+    const run = () =>
+      auditPublicDns({
+        records: [declared('git.celilo.computer')],
+        probe: probe({ ingress: { kind: 'undetermined', reason: 'ECONNREFUSED' } }),
+        evidence,
+        saveEvidence: (next) => {
+          evidence = next;
+        },
+        now: NOW,
+      });
+
+    expect(await run()).toEqual([]);
+    expect(await run()).toEqual([]);
+    const findings = await run();
+
+    expect(findings.map((f) => f.code)).toEqual(['public_dns_unverifiable']);
+    // Distinguishable from "the site is unreachable" — this says nothing is
+    // currently checking, which is the state the outage hid behind.
+    expect(findings[0].message).toContain('unverifiable');
+  });
+
+  test('a run that obtained evidence resets the count', async () => {
+    let evidence: PublicDnsEvidence[] = [];
+    const save = (next: PublicDnsEvidence[]) => {
+      evidence = next;
+    };
+    const args = (ingress?: IngressObservation) => ({
+      records: [declared('git.celilo.computer')],
+      probe: probe({ ingress }),
+      evidence,
+      saveEvidence: save,
+      now: NOW,
+    });
+
+    await auditPublicDns(args({ kind: 'undetermined', reason: 'ECONNREFUSED' }));
+    await auditPublicDns(args({ kind: 'undetermined', reason: 'ECONNREFUSED' }));
+    await auditPublicDns(args());
+    expect(evidence).toEqual([]);
+
+    expect(await auditPublicDns(args({ kind: 'undetermined', reason: 'ECONNREFUSED' }))).toEqual(
+      [],
+    );
+  });
+
+  // ── Companions: best effort must not mean silent ──────────────────────────
+  test('an unclaimed companion is reported with the manual registrar step', async () => {
+    const findings = await auditPublicDns({
+      records: [{ fqdn: 'www.peterbanka.org', companion: true, lastAssertedAt: LONG_AGO }],
+      probe: probe({
+        // The parked placeholder Namecheap keeps serving after reporting
+        // ErrCount 0 for an update it never applied (design.md D3).
+        answers: { 'www.peterbanka.org': { kind: 'answer', ip: '1.2.23.4', ttlSeconds: 1799 } },
+      }),
+      now: NOW,
+    });
+
+    expect(findings.map((f) => f.code)).toEqual(['public_dns_companion_unclaimed']);
+    expect(findings[0].remediation).toContain('Dynamic DNS');
+  });
+
+  test('a name with no public record at all is a finding', async () => {
+    const findings = await auditPublicDns({
+      records: [declared('nexus.lunacycle.net')],
+      probe: probe({ fallback: { kind: 'no_record' } }),
+      now: NOW,
+    });
+    expect(findings.map((f) => f.code)).toEqual(['public_dns_missing']);
+  });
+
+  test('an empty ledger never touches the network', async () => {
+    const findings = await auditPublicDns({
+      records: [],
+      probe: {
+        resolver: 'x',
+        echoService: 'y',
+        observeIngress() {
+          throw new Error('probed with nothing to check');
+        },
+        resolve() {
+          throw new Error('probed with nothing to check');
+        },
+      },
+      now: NOW,
+    });
+    expect(findings).toEqual([]);
+  });
+});

package/src/services/audit/public-dns.ts

@@ -0,0 +1,286 @@
+/**
+ * Public-DNS reachability check.
+ *
+ * Every check celilo had reported healthy throughout celilo#626, during which
+ * five public names — including the apt repo and the module registry — resolved
+ * to an address that answered nothing for nine days. None of them was wrong:
+ * they all looked from *inside*, where the split-horizon resolver deliberately
+ * answers with an address that is reachable in-zone. celilo simply had no
+ * notion of a check whose vantage point is outside itself.
+ *
+ * This is that check. For every name in the DNS registration ledger it asks an
+ * OFF-FLEET resolver what the internet resolves, and compares that against the
+ * address the fleet actually appears to come from according to an independent
+ * echo service.
+ *
+ * Three properties are load-bearing, and each exists because of a specific way
+ * this check could quietly become useless:
+ *
+ *  - **The resolver must not be the fleet's.** A `public_dns` check that used
+ *    the configured resolver would pass forever, exactly like caddy's `dig`
+ *    does today. That is the original bug one layer up, so the probe asserts
+ *    it rather than leaving it to a comment (see `public-dns-probe.ts`).
+ *  - **The expectation must not come from the registrar.** Comparing what was
+ *    published against what we asked to publish is self-agreement — and worse
+ *    than useless here, because Namecheap's DDNS API returns `ErrCount 0` with
+ *    the requested address echoed back for a `www` update it silently does not
+ *    apply (design.md D3).
+ *  - **Missing evidence is not a pass.** The one genuine external probe the
+ *    fleet had (isitup.org, in celilo-website's health check) was itself
+ *    unreachable during the outage, and recorded that as *undetermined* with no
+ *    check item at all. So a real outage produced complete silence. A single
+ *    undetermined result stays quiet — a prober blip must not page — but
+ *    consecutive ones become their own finding, distinct from "unreachable".
+ *
+ * Pure over an injected probe and the previous run's evidence counters: no
+ * network, no DB. See design.md D2.
+ */
+
+import type { DriftFinding } from './types';
+
+/** What an off-fleet resolver answered for one name. */
+export type PublicResolution =
+  | { kind: 'answer'; ip: string; ttlSeconds: number }
+  /** The resolver answered authoritatively that there is no A record. */
+  | { kind: 'no_record' }
+  /** The resolver could not be reached, or failed. Not a pass, not a failure. */
+  | { kind: 'undetermined'; reason: string };
+
+/** The address the fleet appears to come from, per an independent third party. */
+export type IngressObservation =
+  | { kind: 'observed'; ip: string }
+  | { kind: 'undetermined'; reason: string };
+
+export interface PublicDnsProbe {
+  /** Named so findings can say where the answer came from, and so a test can assert it is off-fleet. */
+  readonly resolver: string;
+  /** Named for the same reason — an echo service outage must be attributable. */
+  readonly echoService: string;
+  observeIngress(): Promise<IngressObservation>;
+  resolve(fqdn: string): Promise<PublicResolution>;
+}
+
+/** One ledger name to check. */
+export interface PublicDnsRecord {
+  fqdn: string;
+  /**
+   * Claimed by celilo as the companion of a declared name rather than asked
+   * for by a module. Best effort at claim time, so its remediation is the
+   * manual registrar step rather than "fix the registrar and redeploy".
+   */
+  companion: boolean;
+  /**
+   * When the fleet last asserted this name (the provider's last refresh, or
+   * the registration). Divergence inside one TTL of an assert is propagation,
+   * not a fault — see the hysteresis note below.
+   */
+  lastAssertedAt: Date;
+}
+
+/**
+ * Per-subject counter of consecutive runs that produced no evidence. Carried
+ * across runs by the caller; the audit itself neither reads nor writes storage.
+ */
+export interface PublicDnsEvidence {
+  subject: string;
+  undeterminedRuns: number;
+}
+
+export interface PublicDnsAuditDeps {
+  records: PublicDnsRecord[];
+  probe: PublicDnsProbe;
+  /** Counters from the previous run. Empty on a first run. */
+  evidence?: PublicDnsEvidence[];
+  /** Sink for the updated counters. Omitted in unit tests. */
+  saveEvidence?: (evidence: PublicDnsEvidence[]) => void;
+  now?: Date;
+  /** Consecutive undetermined runs before the absence of evidence is itself a finding. */
+  undeterminedThreshold?: number;
+}
+
+/** The echo service's own subject, so its outage is counted like any other. */
+const ECHO_SUBJECT = 'system';
+
+const DEFAULT_UNDETERMINED_THRESHOLD = 3;
+
+/**
+ * A record that diverges within one TTL of the last assert is still
+ * propagating. Fleet records measure 180s, so alerting on first divergence
+ * would page on every ISP re-lease. Used when the resolver gave no TTL.
+ */
+const FALLBACK_TTL_SECONDS = 300;
+
+function companionRemediation(fqdn: string): string {
+  const label = fqdn.startsWith('www.') ? 'www' : '@';
+  return [
+    `celilo claims ${fqdn} alongside the name the module declared, but the`,
+    'registrar could not publish it. A DDNS API can update an existing record',
+    'and not create one, and Namecheap reports success for a `www` update it',
+    'silently does not apply — so this has to be fixed at the registrar:',
+    '',
+    `  1. Registrar → DNS for this domain → add an A record with Host="${label}"`,
+    '     (any value — celilo overwrites it), and enable Dynamic DNS.',
+    '  2. celilo claims it on the next assert; no redeploy needed.',
+  ].join('\n');
+}
+
+/**
+ * For callers that legitimately have nothing to check — `system update` runs a
+ * partial audit over what it already has and does not reach the network. It
+ * throws rather than answering, so a caller that grows records later cannot
+ * silently keep an inert vantage point.
+ */
+export const unusedPublicDnsProbe: PublicDnsProbe = {
+  resolver: '(not used)',
+  echoService: '(not used)',
+  observeIngress() {
+    throw new Error('public_dns probe used by a caller that declared no records');
+  },
+  resolve() {
+    throw new Error('public_dns probe used by a caller that declared no records');
+  },
+};
+
+export async function auditPublicDns(deps: PublicDnsAuditDeps): Promise<DriftFinding[]> {
+  // No names asserted publicly means nothing to verify — and no reason to
+  // reach the network or advance an undetermined counter.
+  if (deps.records.length === 0) return [];
+
+  const now = deps.now ?? new Date();
+  const threshold = deps.undeterminedThreshold ?? DEFAULT_UNDETERMINED_THRESHOLD;
+  const previous = new Map((deps.evidence ?? []).map((e) => [e.subject, e.undeterminedRuns]));
+  const next = new Map<string, number>();
+  const findings: DriftFinding[] = [];
+
+  /**
+   * Count an absence of evidence. The first few are silent — one prober blip
+   * must not page — but they are never forgotten, which is the difference
+   * between "we were not told" and "we could not have been told".
+   */
+  const countUndetermined = (subject: string): number => {
+    const runs = (previous.get(subject) ?? 0) + 1;
+    next.set(subject, runs);
+    return runs;
+  };
+
+  const ingress = await deps.probe.observeIngress();
+
+  if (ingress.kind === 'undetermined') {
+    const runs = countUndetermined(ECHO_SUBJECT);
+    // Every name is unverifiable when the expectation is, so the records are
+    // left uncounted rather than each accruing a duplicate of the same outage.
+    if (runs >= threshold) {
+      findings.push({
+        category: 'public_dns',
+        severity: 'drift',
+        code: 'public_dns_unverifiable',
+        message: `Public DNS has been unverifiable for ${runs} consecutive checks`,
+        details: `The echo service (${deps.probe.echoService}) could not be reached, so there is no\nexpectation to compare public DNS against. Latest reason: ${ingress.reason}.\n\nThis is NOT a report that the fleet is reachable, and not a report that it\nis unreachable — it is a report that nothing is currently checking. The\noutage this check exists for ran for nine days behind exactly this silence.`,
+        remediation: [
+          'Check outbound connectivity from the management host, then:',
+          `  celilo system config set public_dns.echo_url <url>   # current: ${deps.probe.echoService}`,
+        ].join('\n'),
+        actionable: false,
+        subject: ECHO_SUBJECT,
+      });
+    }
+    deps.saveEvidence?.(toEvidence(next));
+    return findings;
+  }
+
+  for (const record of deps.records) {
+    const resolution = await deps.probe.resolve(record.fqdn);
+
+    if (resolution.kind === 'undetermined') {
+      const runs = countUndetermined(record.fqdn);
+      if (runs >= threshold) {
+        findings.push({
+          category: 'public_dns',
+          severity: 'drift',
+          code: 'public_dns_unverifiable',
+          message: `${record.fqdn}: public reachability unverifiable for ${runs} consecutive checks`,
+          details: `${deps.probe.resolver} did not answer for this name. Latest reason: ${resolution.reason}.\nWhether the name resolves publicly is currently unknown — which is not the\nsame as it being fine.`,
+          remediation: `Resolve it by hand to see what the internet gets:\n  dig @${deps.probe.resolver} ${record.fqdn} A`,
+          actionable: false,
+          subject: record.fqdn,
+        });
+      }
+      continue;
+    }
+
+    if (resolution.kind === 'no_record') {
+      findings.push(
+        record.companion
+          ? {
+              category: 'public_dns',
+              severity: 'drift',
+              code: 'public_dns_companion_unclaimed',
+              message: `${record.fqdn}: not published (companion of a name celilo serves)`,
+              details:
+                'This name has no public A record, so it does not reach the fleet at all.\n' +
+                'A working www beside a dead apex (or the reverse) is the same defect as\n' +
+                'celilo#626 in miniature: publicly broken, locally invisible.',
+              remediation: companionRemediation(record.fqdn),
+              actionable: false,
+              subject: record.fqdn,
+            }
+          : {
+              category: 'public_dns',
+              severity: 'drift',
+              code: 'public_dns_missing',
+              message: `${record.fqdn}: no public A record`,
+              details: `celilo registered this name but ${deps.probe.resolver} resolves no A record for\nit. The registrar reported success; a provider's success response is a claim\nabout an API call, not evidence that the record was published.`,
+              remediation: `Re-assert it, then re-check:\n  celilo module run-hook <registrar> refresh_registrations\n  dig @${deps.probe.resolver} ${record.fqdn} A`,
+              actionable: false,
+              subject: record.fqdn,
+            },
+      );
+      continue;
+    }
+
+    if (resolution.ip === ingress.ip) continue;
+
+    // Hysteresis. A legitimate address change produces a genuinely divergent
+    // public record until the old answer's TTL expires, so a divergence is only
+    // a finding once it has outlived the TTL of the record we last asserted.
+    // Without this the check pages on every ISP re-lease.
+    const ttlSeconds = resolution.ttlSeconds > 0 ? resolution.ttlSeconds : FALLBACK_TTL_SECONDS;
+    const settledAt = record.lastAssertedAt.getTime() + ttlSeconds * 1000;
+    if (now.getTime() < settledAt) continue;
+
+    findings.push(
+      record.companion
+        ? {
+            category: 'public_dns',
+            severity: 'drift',
+            code: 'public_dns_companion_unclaimed',
+            message: `${record.fqdn}: publicly resolves to ${resolution.ip}, not ${ingress.ip}`,
+            details: `${deps.probe.resolver} serves ${resolution.ip} for this companion name while the fleet is\nreachable at ${ingress.ip} (per ${deps.probe.echoService}). A parked or redirect record is\nthe usual cause: Namecheap renders those as A records the DDNS endpoint\naccepts updates for and does not apply, reporting ErrCount 0 (design.md D3).`,
+            remediation: companionRemediation(record.fqdn),
+            actionable: false,
+            subject: record.fqdn,
+          }
+        : {
+            category: 'public_dns',
+            severity: 'drift',
+            code: 'public_dns_stale',
+            message: `${record.fqdn}: publicly resolves to ${resolution.ip}, not ${ingress.ip}`,
+            details: `${deps.probe.resolver} serves ${resolution.ip} for this name while the fleet is reachable at\n${ingress.ip} (per ${deps.probe.echoService}). The divergence has outlived the record's own\nTTL (${ttlSeconds}s since the last assert), so this is not propagation.\n\nNothing inside the fleet can see this: the split-horizon resolver answers\nwith an address that IS reachable in-zone, which is correct for its purpose\nand says nothing about the public internet.`,
+            remediation: `Re-assert the record, then re-check:\n  celilo module run-hook <registrar> refresh_registrations\n  dig @${deps.probe.resolver} ${record.fqdn} A`,
+            actionable: false,
+            subject: record.fqdn,
+          },
+    );
+  }
+
+  deps.saveEvidence?.(toEvidence(next));
+  return findings;
+}
+
+/**
+ * Only subjects that were undetermined THIS run survive. A subject that
+ * answered has no counter, which is what makes the count "consecutive".
+ */
+function toEvidence(next: Map<string, number>): PublicDnsEvidence[] {
+  return [...next].map(([subject, undeterminedRuns]) => ({ subject, undeterminedRuns }));
+}

package/src/services/audit/types.ts

@@ -37,6 +37,7 @@ export type DriftCategory =
   | 'secrets_decryptable'
   | 'services_reachable'
   | 'machines_reachable'
+  | 'public_dns'
   | 'disk_space'
   | 'transport_reads'
   | 'trusted_sources';