@celilo/cli 2.3.0 → 3.0.0

package/src/services/audit/health.ts

@@ -17,9 +17,15 @@
  * pure for testing. The aggregator wires `runAllHealthChecks` to it.
  */
 
+import type { HealthWaiver } from '../../manifest/schema';
 import type { HealthCheckResult } from '../health-runner';
 import type { DriftFinding } from './types';
 
+/** Render a waiver as the annotation carried on the `unmeasured` finding. */
+export function waiverAnnotation(waiver: HealthWaiver): string {
+  return `WAIVED by ${waiver.by} (${waiver.at}): ${waiver.reason}`;
+}
+
 export interface HealthAuditDeps {
   results: HealthCheckResult[];
 }
@@ -49,6 +55,12 @@ export async function auditHealth(deps: HealthAuditDeps): Promise<DriftFinding[]
     const skipped = r.checks.filter((c) => c.status === 'skip');
     const measured = r.checks.filter((c) => c.status !== 'skip');
 
+    // A waiver annotates an `unmeasured` finding; it never replaces it.
+    // The severity stays `unmeasured` so the fleet verdict stays UNKNOWN
+    // (D3) — zero unmeasured findings after a waiver would mean the
+    // waiver re-greened the board, which is the failure it guards.
+    const waiverNote = r.waiver ? ` — ${waiverAnnotation(r.waiver)}` : '';
+
     if (r.status === 'no-checks') {
       // Either the module declares no health_check hook, or the hook ran
       // and every check reported skip. Neither measured anything.
@@ -57,13 +69,13 @@ export async function auditHealth(deps: HealthAuditDeps): Promise<DriftFinding[]
           ? skippedChecksFinding(
               r,
               'health_unmeasured',
-              `${r.moduleId}: all ${skipped.length} health checks skipped — nothing measured`,
+              `${r.moduleId}: all ${skipped.length} health checks skipped — nothing measured${waiverNote}`,
             )
           : {
               category: 'health',
               severity: 'unmeasured',
               code: 'health_unmeasured',
-              message: `${r.moduleId}: no health_check hook — never measured`,
+              message: `${r.moduleId}: no health_check hook — never measured${waiverNote}`,
               remediation: `Add a health_check hook to the ${r.moduleId} module manifest`,
               actionable: false,
               subject: r.moduleId,
@@ -77,7 +89,7 @@ export async function auditHealth(deps: HealthAuditDeps): Promise<DriftFinding[]
         skippedChecksFinding(
           r,
           'health_checks_skipped',
-          `${r.moduleId}: ${skipped.length} of ${r.checks.length} health checks skipped`,
+          `${r.moduleId}: ${skipped.length} of ${r.checks.length} health checks skipped${waiverNote}`,
         ),
       );
     }

package/src/services/audit/index.test.ts

@@ -44,7 +44,7 @@ const emptyDeps = {
   machinesReachable: { results: [] },
   diskSpace: { results: [] },
   transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
-  trustedSources: { firewalls: [] },
+  trustedSources: { firewalls: [], unreachableFirewalls: [] },
   publicDns: { records: [], probe: unusedPublicDnsProbe },
 };
 

package/src/services/audit/interface-classification.test.ts

@@ -207,14 +207,25 @@ describe('describeClassification — the picture, not a finding', () => {
 
 describe('auditInterfaceClassification — every firewall, flattened', () => {
   test('reports across several firewalls, each subjected on its own hostname', () => {
-    const findings = auditInterfaceClassification([
-      liveFleetShape(),
-      liveFleetShape({ hostname: 'fw-branch' }),
-    ]);
+    const findings = auditInterfaceClassification({
+      unreachableFirewalls: [],
+      views: [liveFleetShape(), liveFleetShape({ hostname: 'fw-branch' })],
+    });
     expect(findings.map((f) => f.subject).sort()).toEqual(['fw-branch', 'fw-main']);
   });
 
   test('no firewalls means no findings', () => {
-    expect(auditInterfaceClassification([])).toEqual([]);
+    expect(auditInterfaceClassification({ views: [], unreachableFirewalls: [] })).toEqual([]);
+  });
+
+  test('a firewall whose interfaces could not be read is unmeasured, not silent (D7)', () => {
+    const findings = auditInterfaceClassification({
+      views: [liveFleetShape()],
+      unreachableFirewalls: ['fw-branch'],
+    });
+    const unmeasured = findings.filter((f) => f.severity === 'unmeasured');
+    expect(unmeasured).toHaveLength(1);
+    expect(unmeasured[0].subject).toBe('fw-branch');
+    expect(unmeasured[0].code).toBe('interface_classification_unmeasured');
   });
 });

package/src/services/audit/interface-classification.ts

@@ -142,8 +142,31 @@ export function auditFirewallInterfaces(view: FirewallInterfaceView): DriftFindi
 }
 
 /** Every firewall's findings, flattened. */
-export function auditInterfaceClassification(views: FirewallInterfaceView[]): DriftFinding[] {
-  return views.flatMap(auditFirewallInterfaces);
+export interface InterfaceClassificationAuditDeps {
+  views: FirewallInterfaceView[];
+  /**
+   * Firewalls celilo knows about but whose interface state could not be read
+   * over SSH. Each is an `unmeasured` finding: an interface survey that
+   * cannot see a firewall has measured nothing about it (D7).
+   */
+  unreachableFirewalls: string[];
+}
+
+export function auditInterfaceClassification(
+  deps: InterfaceClassificationAuditDeps,
+): DriftFinding[] {
+  const unreachable = deps.unreachableFirewalls.map((firewallIp): DriftFinding => {
+    return {
+      category: 'interface_classification',
+      severity: 'unmeasured',
+      code: 'interface_classification_unmeasured',
+      subject: firewallIp,
+      message: `${firewallIp}: interfaces could not be read, so classification is unknown`,
+      remediation: `Check SSH reachability of ${firewallIp}, then re-audit. This finding records that the interfaces were not read, not that they are classified.`,
+      actionable: true,
+    };
+  });
+  return [...unreachable, ...deps.views.flatMap(auditFirewallInterfaces)];
 }
 
 /**

package/src/services/audit/module-versions.ts

@@ -84,7 +84,11 @@ export async function auditModuleVersions(deps: ModuleVersionsAuditDeps): Promis
       const [reason] = errorBuckets.keys();
       findings.push({
         category: 'module_versions',
-        severity: 'drift',
+        // Same fact as the per-module rejection below: the registry did not
+        // answer, so nothing was measured. D7 severity, not drift — this
+        // aggregate used to say "drift" and rendered the fleet DRIFT when
+        // the truth was "I could not ask".
+        severity: 'unmeasured',
         code: 'registry_unreachable',
         message: `Registry unreachable — ${rejected.length} module${rejected.length === 1 ? '' : 's'} unchecked (${reason})`,
         remediation: 'Check network connectivity to the registry, then press R to re-audit.',

package/src/services/audit/public-dns.test.ts

@@ -139,6 +139,9 @@ describe('auditPublicDns', () => {
     // Distinguishable from "the site is unreachable" — this says nothing is
     // currently checking, which is the state the outage hid behind.
     expect(findings[0].message).toContain('unverifiable');
+    // D7: "could not tell" is unmeasured, not drift. It must stop the verdict
+    // at UNKNOWN, never render as an ordinary divergence.
+    expect(findings[0].severity).toBe('unmeasured');
   });
 
   test('a run that obtained evidence resets the count', async () => {
@@ -189,6 +192,23 @@ describe('auditPublicDns', () => {
     expect(findings.map((f) => f.code)).toEqual(['public_dns_missing']);
   });
 
+  test('a name the resolver will not answer for is unmeasured once it stops being a blip (D7)', async () => {
+    const args = () => ({
+      records: [declared('nexus.lunacycle.net')],
+      probe: probe({
+        answers: { 'nexus.lunacycle.net': { kind: 'undetermined' as const, reason: 'timeout' } },
+      }),
+      undeterminedThreshold: 1,
+      now: NOW,
+    });
+
+    const findings = await auditPublicDns(args());
+    expect(findings.map((f) => f.code)).toEqual(['public_dns_unverifiable']);
+    // The resolver did not answer, so the truth about this name is unknown —
+    // not fine, not broken. It must stop the verdict at UNKNOWN.
+    expect(findings[0].severity).toBe('unmeasured');
+  });
+
   test('an empty ledger never touches the network', async () => {
     const findings = await auditPublicDns({
       records: [],

package/src/services/audit/public-dns.ts

@@ -172,7 +172,10 @@ export async function auditPublicDns(deps: PublicDnsAuditDeps): Promise<DriftFin
     if (runs >= threshold) {
       findings.push({
         category: 'public_dns',
-        severity: 'drift',
+        // The vantage point is gone: nothing was measured this run, and D7
+        // says that is `unmeasured`, not `drift`. This finding was `drift`
+        // before D7 existed; its own text already said what unmeasured means.
+        severity: 'unmeasured',
         code: 'public_dns_unverifiable',
         message: `Public DNS has been unverifiable for ${runs} consecutive checks`,
         details: `The echo service (${deps.probe.echoService}) could not be reached, so there is no\nexpectation to compare public DNS against. Latest reason: ${ingress.reason}.\n\nThis is NOT a report that the fleet is reachable, and not a report that it\nis unreachable — it is a report that nothing is currently checking. The\noutage this check exists for ran for nine days behind exactly this silence.`,
@@ -196,7 +199,9 @@ export async function auditPublicDns(deps: PublicDnsAuditDeps): Promise<DriftFin
       if (runs >= threshold) {
         findings.push({
           category: 'public_dns',
-          severity: 'drift',
+          // The resolver would not answer for this name, so whether it resolves
+          // publicly is unknown — not fine, not broken. Unmeasured per D7.
+          severity: 'unmeasured',
           code: 'public_dns_unverifiable',
           message: `${record.fqdn}: public reachability unverifiable for ${runs} consecutive checks`,
           details: `${deps.probe.resolver} did not answer for this name. Latest reason: ${resolution.reason}.\nWhether the name resolves publicly is currently unknown — which is not the\nsame as it being fine.`,

package/src/services/audit/recurrence-gate.test.ts

@@ -0,0 +1,225 @@
+/**
+ * The recurrence gate (celilo#1326, part two).
+ *
+ * D7's rule: every claim celilo makes about the fleet is measured against the
+ * thing it describes, or reported as `unmeasured`. There is no third option,
+ * because contributing nothing renders the category green and the verdict
+ * READY — the defect shape #1326 exists to close.
+ *
+ * A category can silently violate that rule three ways: someone adds a new
+ * DriftCategory with no unmeasured path, someone rewires a category's
+ * "could not reach" case to invent a finding of the wrong severity, or a
+ * refactor drops the path entirely. This gate catches all three by walking
+ * EVERY category — computed from the type, not from a list someone maintains
+ * by hand — and demanding each either:
+ *
+ *   - `probed`: its audit function, given deps that cannot reach their
+ *     subject, produces at least one `unmeasured` finding; or
+ *   - `allow-listed`: the category structurally always reaches its subject,
+ *     with the reason recorded here.
+ *
+ * `Record<DriftCategory, GateEntry>` is the enforcement: a category added to
+ * the union without a gate entry is a type error, and the author has to
+ * answer the question this test exists to ask.
+ *
+ * WATCHING IT FAIL (Rule 7.6): revert ce-prr5's `health.ts` change (the
+ * `no-checks` branch in auditHealth) and the `health` entry fails — health
+ * produces nothing for a module with no hook, which is exactly how six
+ * modules rendered READY while never having been measured.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { ALL_CATEGORIES } from '../../cli/tui/audit-state';
+import type { DbClient } from '../../db/client';
+import { auditCliVersion } from './cli-version';
+import { auditDiskSpace } from './disk-space';
+import { auditHealth } from './health';
+import { auditInterfaceClassification } from './interface-classification';
+import { auditModuleIntegrity } from './module-integrity';
+import { auditModuleVersions } from './module-versions';
+import { auditPublicDns } from './public-dns';
+import { auditSchema } from './schema';
+import { auditTerraformPlan } from './terraform-plan';
+import { auditTrustedSources } from './trusted-sources';
+import type { DriftCategory, DriftFinding, DriftSeverity } from './types';
+
+type GateEntry = {
+  /** The probe: call the category's audit with deps that cannot reach their subject. */
+  readonly probe?: () => Promise<DriftFinding[]> | DriftFinding[];
+  /** Why the category structurally always reaches its subject. */
+  readonly because?: string;
+};
+
+const GATE: Record<DriftCategory, GateEntry> = {
+  cli_version: {
+    probe: () => auditCliVersion({ installedVersion: '9.9.9', fetcher: async () => null }),
+    because: 'probed: the npm registry can be unreachable, and that is unmeasured',
+  },
+  schema: {
+    probe: () =>
+      auditSchema({
+        journal: () => null,
+        applied: () => [],
+        db: {} as DbClient,
+      }),
+    because: 'probed: a missing migration journal means nothing was compared',
+  },
+  capability_abi: {
+    because:
+      'compares capability versions claimed in installed module manifests against the framework registry and deployed providers, all read from celilo own DB and local manifests — the subject is celilo itself and is always reachable',
+  },
+  browser_pin: {
+    because:
+      'compares a bundled client playwright revision against the locally provisioned browser binary, both on this machine — always reachable',
+  },
+  terraform_plan: {
+    probe: () =>
+      auditTerraformPlan({
+        modules: [{ id: 'm', terraformDir: '/tmp/does-not-matter', envVars: {} }],
+        run: async () => ({ exitCode: 0, stdout: '', stderr: '' }),
+      }),
+    because: 'probed: terraform exiting zero with an unreadable summary is not a clean plan',
+  },
+  module_versions: {
+    probe: () =>
+      auditModuleVersions({
+        installed: [{ id: 'm', version: '1.0.0' }],
+        fetcher: () => Promise.reject(new Error('registry unreachable')),
+      }),
+    because: 'probed: a registry that did not answer is not a version difference',
+  },
+  module_configs: {
+    because:
+      'walks manifest variables against config rows in celilo DB — the subject is celilo itself and is always reachable',
+  },
+  health: {
+    probe: () => auditHealth({ results: [{ moduleId: 'm', status: 'no-checks', checks: [] }] }),
+    because: 'probed: a module with no hook, or whose checks all skip, has measured nothing',
+  },
+  backups: {
+    because:
+      'reads recorded backup outcomes and effective cadences from celilo DB — the subject is celilo itself and is always reachable; a missing backup is drift, not silence',
+  },
+  abandoned_operations: {
+    because: 'counts reclaimed operation rows in celilo DB — always reachable',
+  },
+  undeployed_modules: {
+    because: 'reads module lifecycle states from celilo DB — always reachable',
+  },
+  unconfigured_modules: {
+    because: 'reads config-row counts from celilo DB — always reachable',
+  },
+  services_credentials: {
+    because:
+      'reads credential envelopes from celilo DB and decrypts with the local master key; a failed decrypt is itself a measurement (blocked), never silence',
+  },
+  secrets_decryptable: {
+    because:
+      'walks encrypted rows in celilo DB and decrypts; a failed decrypt is itself a measurement (blocked), never silence',
+  },
+  services_reachable: {
+    because:
+      'the probe outcome IS the measurement: an unreachable service is reachable=false and a finding, so every probe measures its subject',
+  },
+  machines_reachable: {
+    because:
+      'the SSH probe outcome IS the measurement: an unreachable machine is reachable=false and a finding, so every probe measures its subject',
+  },
+  public_dns: {
+    probe: () =>
+      auditPublicDns({
+        records: [{ fqdn: 'git.celilo.computer', companion: false, lastAssertedAt: new Date(0) }],
+        probe: {
+          resolver: '1.1.1.1',
+          echoService: 'https://echo.example',
+          observeIngress: async () => ({ kind: 'undetermined', reason: 'ECONNREFUSED' }),
+          resolve: async () => {
+            throw new Error('not reached: the vantage is down');
+          },
+        },
+        undeterminedThreshold: 1,
+        now: new Date(0),
+      }),
+    because: 'probed: the off-fleet echo vantage is external and can be down; unmeasured',
+  },
+  disk_space: {
+    probe: () =>
+      auditDiskSpace({
+        results: [
+          { hostname: 'box', ipAddress: '10.0.0.1', usedPercent: null, message: 'df failed' },
+        ],
+      }),
+    because: 'probed: a df that could not run measured nothing about the disk',
+  },
+  transport_reads: {
+    because:
+      'reads the notification poller recorded read outcomes from celilo DB — always reachable; not happening is itself the drift finding',
+  },
+  trusted_sources: {
+    probe: () => auditTrustedSources({ firewalls: [], unreachableFirewalls: ['10.0.0.254'] }),
+    because: 'probed: a firewall whose live ruleset cannot be read is unknown, not clean',
+  },
+  interface_classification: {
+    probe: () => auditInterfaceClassification({ views: [], unreachableFirewalls: ['10.0.0.254'] }),
+    because: 'probed: a firewall whose interfaces cannot be read is unknown, not clean',
+  },
+  module_integrity: {
+    probe: () =>
+      auditModuleIntegrity({
+        results: [
+          { success: false, moduleId: 'm', violations: [], error: 'No integrity data found.' },
+        ],
+      }),
+    because: 'probed: no baseline row, or a stale one, means nothing was verified',
+  },
+  detect_without_converge: {
+    because:
+      'static analysis of module manifests and system-config rows in celilo DB — always reachable',
+  },
+  jail_exemptions: {
+    because:
+      'reads recorded per-module jail exemptions from celilo DB — the subject is celilo itself and is always reachable; no exemptions is the normal state, not silence',
+  },
+};
+
+const ALLOW_LISTED = Object.entries(GATE).filter(([, e]) => !e.probe);
+
+describe('recurrence gate: every DriftCategory measures its subject or says why it cannot fail to', () => {
+  test('the gate covers every category in the union (compile-time) and matches the TUI list (run-time)', async () => {
+    // `GATE` is `Record<DriftCategory, GateEntry>`, so the compiler rejects a
+    // new category until it gets an entry. This run-time line additionally
+    // catches the reverse: an entry for a category no longer in the union.
+    const gateCategories = Object.keys(GATE) as DriftCategory[];
+    expect(gateCategories.sort()).toEqual([...ALL_CATEGORIES].sort());
+  });
+
+  for (const [category, entry] of Object.entries(GATE) as [DriftCategory, GateEntry][]) {
+    if (!entry.probe) {
+      test(`${category}: allow-listed — ${entry.because}`, () => {
+        expect(entry.because).toBeTruthy();
+      });
+      continue;
+    }
+
+    test(`${category}: cannot reach its subject and still reports unmeasured`, async () => {
+      const probe = entry.probe;
+      if (!probe) expect.unreachable();
+      const findings = await probe();
+      const severities: DriftSeverity[] = findings.map((f) => f.severity);
+      expect(severities).toContain('unmeasured');
+      // The gate exists because silence renders as READY. If an unmeasured
+      // path ever degrades back to drift, the verdict distinction D7 added
+      // is gone and this is the line that says so.
+      const unmeasured = findings.find((f) => f.severity === 'unmeasured') as DriftFinding;
+      expect(unmeasured.category).toBe(category);
+    });
+  }
+
+  test('the allow-list is not a dumping ground: 14 categories, each individually justified', () => {
+    // Not a magic number to defend — a tripwire. If this count changes, the
+    // diff is the review: someone moved a category in or out of the set of
+    // "structurally always reaches its subject", and the reason strings
+    // above are where they have to argue it.
+    expect(ALLOW_LISTED).toHaveLength(14);
+  });
+});

package/src/services/audit/trusted-sources.test.ts

@@ -59,6 +59,7 @@ describe('parsing reach rules from a live ruleset', () => {
 describe('unowned-trusted-network audit', () => {
   it('reports a network celilo does not recognise', async () => {
     const findings = await auditTrustedSources({
+      unreachableFirewalls: [],
       firewalls: [
         {
           firewallIp: '192.168.0.254',
@@ -78,6 +79,7 @@ describe('unowned-trusted-network audit', () => {
     // The point of the change: once a module owns the VPN and registers its
     // subnet, the rule IS the rendered output and there is nothing to report.
     const findings = await auditTrustedSources({
+      unreachableFirewalls: [],
       firewalls: [
         {
           firewallIp: '192.168.0.254',
@@ -96,6 +98,7 @@ describe('unowned-trusted-network audit', () => {
       ZONES,
     );
     const findings = await auditTrustedSources({
+      unreachableFirewalls: [],
       firewalls: [{ firewallIp: '192.168.0.254', live, known: [CONTROL_PLANE] }],
     });
 
@@ -104,6 +107,7 @@ describe('unowned-trusted-network audit', () => {
 
   it('names the interface when trust is granted by interface, and says why it cannot be registered as-is', async () => {
     const findings = await auditTrustedSources({
+      unreachableFirewalls: [],
       firewalls: [
         {
           firewallIp: '192.168.0.254',
@@ -120,6 +124,7 @@ describe('unowned-trusted-network audit', () => {
 
   it('a firewall with no live reach beyond what celilo composes is clean', async () => {
     const findings = await auditTrustedSources({
+      unreachableFirewalls: [],
       firewalls: [
         {
           firewallIp: '192.168.0.254',
@@ -134,4 +139,16 @@ describe('unowned-trusted-network audit', () => {
 
     expect(findings).toEqual([]);
   });
+
+  it('a firewall whose ruleset could not be read is unmeasured, not silent (D7)', async () => {
+    const findings = await auditTrustedSources({
+      unreachableFirewalls: ['192.168.0.254'],
+      firewalls: [],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].severity).toBe('unmeasured');
+    expect(findings[0].code).toBe('trusted_sources_unmeasured');
+    expect(findings[0].subject).toBe('192.168.0.254');
+  });
 });

package/src/services/audit/trusted-sources.ts

@@ -37,6 +37,13 @@ export interface FirewallReachState {
 
 export interface TrustedSourcesAuditDeps {
   firewalls: FirewallReachState[];
+  /**
+   * Firewalls celilo knows about but whose live ruleset could not be read
+   * (`iptables-save` failed over SSH). Each is an `unmeasured` finding, not
+   * silence: a check that cannot see a firewall has not measured that
+   * firewall, and silence renders as READY (D7).
+   */
+  unreachableFirewalls: string[];
 }
 
 /**
@@ -79,6 +86,20 @@ function originKey(origin: ReachOrigin): string {
 export async function auditTrustedSources(deps: TrustedSourcesAuditDeps): Promise<DriftFinding[]> {
   const findings: DriftFinding[] = [];
 
+  // Unreachable firewalls first, so the report leads with what could not be
+  // measured at all before anything it did measure.
+  for (const firewallIp of deps.unreachableFirewalls) {
+    findings.push({
+      category: 'trusted_sources',
+      severity: 'unmeasured',
+      code: 'trusted_sources_unmeasured',
+      message: `${firewallIp}: live ruleset could not be read, so trusted-source drift is unknown`,
+      remediation: `Check SSH reachability of ${firewallIp}, then re-audit. This finding records that the ruleset was not read, not that it is clean.`,
+      actionable: true,
+      subject: firewallIp,
+    });
+  }
+
   for (const firewall of deps.firewalls) {
     const known = new Set(firewall.known);
     const reported = new Set<string>();