@celilo/cli 0.25.1 → 0.26.1

package/src/services/audit/machines-reachable.test.ts

@@ -1,12 +1,14 @@
 import { describe, expect, test } from 'bun:test';
+import { failingKeysFromFindings } from '../alerting/builtin-monitors';
+import { ancestorKeysFor, machineAlertKey } from '../alerting/suppression';
 import { auditMachinesReachable } from './machines-reachable';
 
 describe('auditMachinesReachable', () => {
   test('no findings when every machine is reachable', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
-        { id: 'm2', hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
       ],
     });
     expect(result).toEqual([]);
@@ -15,9 +17,8 @@ describe('auditMachinesReachable', () => {
   test('per-machine drift finding for one unreachable host', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -30,7 +31,8 @@ describe('auditMachinesReachable', () => {
       category: 'machines_reachable',
       severity: 'drift',
       code: 'machine_unreachable',
-      subject: 'm2',
+      // Hostname, not the DB UUID — this assertion encoded the #596 bug.
+      subject: 'dns-ext',
       actionable: false,
     });
     expect(result[0].message).toContain('dns-ext');
@@ -42,14 +44,12 @@ describe('auditMachinesReachable', () => {
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,
           message: 'host down',
         },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -68,12 +68,71 @@ describe('auditMachinesReachable', () => {
     expect(result[0].message).toContain('All 2 machines unreachable');
   });
 
+  /**
+   * celilo#596. The finding's subject becomes the alert key, and suppression
+   * resolves a machine's ancestor key from its HOSTNAME. Subjecting on the DB
+   * UUID produced a key nothing could ever match, so an unreachable machine
+   * suppressed nothing and every module on it paged independently — the exact
+   * cascade suppression exists to prevent.
+   *
+   * The prefix-only assertion in `e2e/tests/alert-ack-return-leg.test.ts`
+   * (`toContain('builtin:machines_reachable/machine:')`) passes for either
+   * value, which is why this survived. These assert the WHOLE key.
+   */
+  describe('the alert key is one suppression can match (#596)', () => {
+    test('producer and consumer derive the same key', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          {
+            hostname: 'dns-ext',
+            ipAddress: '203.0.113.5',
+            reachable: false,
+            message: 'Connection timed out',
+          },
+        ],
+      });
+
+      const keys = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      expect(keys).toEqual(['builtin:machines_reachable/machine:dns-ext']);
+      // The identity that actually matters: asserted against the consumer's own
+      // constructor rather than a second literal, so the two cannot drift apart
+      // while both still look right.
+      expect(keys[0]).toBe(machineAlertKey('dns-ext'));
+    });
+
+    test('an unreachable machine suppresses a module deployed on it', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: false, message: 'down' },
+        ],
+      });
+      const firing = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      const ancestors = ancestorKeysFor('module:homebridge/check:service_running', {
+        moduleSystems: [
+          { moduleId: 'homebridge', hostname: 'dns-ext', zone: 'internal', infraType: 'machine' },
+        ],
+        zoneProviders: [],
+      });
+
+      // The behaviour suppression.ts documents: the machine's own alert is what
+      // explains the module's. Before the fix the intersection was empty.
+      expect(ancestors.some((a) => firing.includes(a))).toBe(true);
+    });
+  });
+
   test('does NOT collapse when only one machine is in the pool', async () => {
     // A single machine failing is per-machine, not a system-wide signal.
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,

package/src/services/audit/machines-reachable.ts

@@ -19,9 +19,21 @@
 import type { DriftFinding } from './types';
 
 export interface MachineReachableResult {
-  /** Stable machine ID (UUID in the DB). */
-  id: string;
-  /** Hostname for the finding message. */
+  /**
+   * User-facing hostname, and the identifier every finding here is keyed by.
+   *
+   * NOT the machine's UUID. Suppression resolves a machine's ancestor key from
+   * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding
+   * subjected on the UUID produces an alert key suppression can never match —
+   * an unreachable machine then suppresses nothing and every module on it pages
+   * independently, which is the cascade suppression exists to prevent. That was
+   * celilo#596, filed against this check and fixed here; `disk-space.ts` cites
+   * it as the reason it keys on hostname too.
+   *
+   * The UUID used to be carried alongside as `id`. It is deleted rather than
+   * left unused (Rule 3.9): its only reader was the defect, and a field kept
+   * "just in case" is what the next subject line would reach for.
+   */
   hostname: string;
   ipAddress: string;
   /** True if SSH probe succeeded. */
@@ -79,7 +91,9 @@ export async function auditMachinesReachable(
       ].join('\n'),
       // Multi-step / interactive; not a one-shot.
       actionable: false,
-      subject: r.id,
+      // Hostname, so `machineAlertKey` can match this — see the note on
+      // MachineReachableResult.hostname (celilo#596).
+      subject: r.hostname,
     });
   }
 

package/src/services/deployed-systems.ts

@@ -99,6 +99,37 @@ export function getProvisionedSystems(db: DbClient): ProvisionedSystem[] {
     .sort((a, b) => (a.vmid ?? 0) - (b.vmid ?? 0));
 }
 
+/** One deployed system, with the module on it — the whole fleet, both infra types. */
+export interface ModulePlacementRow {
+  moduleId: string;
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  vmid: number | null;
+}
+
+/**
+ * Every module deployment across the fleet, machine-pool and container alike.
+ *
+ * The complement to `getModuleSystems` (one module) and `getProvisionedSystems`
+ * (containers only). Added for the doctor's host-liveness check (celilo#728),
+ * which has to ask about EVERY host something is running on — the defect there
+ * was precisely that no fleet-wide view of "what runs where" was being
+ * consulted.
+ */
+export function listAllModuleSystems(db: DbClient): ModulePlacementRow[] {
+  return db
+    .select()
+    .from(moduleSystems)
+    .all()
+    .map((r) => ({
+      moduleId: r.moduleId,
+      hostname: r.hostname,
+      infraType: r.infraType,
+      vmid: r.vmid ?? null,
+    }))
+    .sort((a, b) => a.moduleId.localeCompare(b.moduleId));
+}
+
 /**
  * All container_service systems (Proxmox LXCs, droplets, …) whose zone is in
  * `zones`, across every module — the LXC complement to machine-pool's

package/src/services/fleet-checks.test.ts

@@ -18,14 +18,17 @@ import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monit
 import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
+  type HostLivenessInputs,
   checkCapabilityProviders,
   checkControlPlaneNetwork,
   checkDispatcher,
+  checkHostLiveness,
   checkSchemaDrift,
   checkServiceDns,
   checkSubscribers,
   describeCapabilityProblem,
   findBrokenCapabilityDerivations,
+  runFleetChecks,
 } from './fleet-checks';
 import { ensureOperationsSweepSubscriber } from './module-operations';
 
@@ -766,3 +769,232 @@ describe('checkControlPlaneNetwork', () => {
     expect(finding.summary).toContain('192.168.0.0/24');
   });
 });
+
+/**
+ * celilo#728. `system doctor` reported "OK with warnings" while a Proxmox node
+ * was OFFLINE with `celilo-apt-repo` and `lunacycle` deployed on it. Its two
+ * warnings were about the dispatcher and a false-positive subscriber drift —
+ * neither related. The node's status was already in `proxmox node list`; doctor
+ * never consulted it, and the condition surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there.
+ */
+describe('checkHostLiveness', () => {
+  const inputs = (over: Partial<HostLivenessInputs> = {}): HostLivenessInputs => ({
+    placements: [],
+    machines: [],
+    nodes: [],
+    guestNodes: [],
+    ...over,
+  });
+
+  it('reproduces #728: an offline node hosting modules is a FAILURE naming both', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        nodes: [
+          { node: 'node2', online: false },
+          { node: 'node3', online: true },
+        ],
+        guestNodes: [
+          { vmid: 205, node: 'node2' },
+          { vmid: 202, node: 'node2' },
+          { vmid: 301, node: 'node3' },
+        ],
+      }),
+    );
+
+    expect(finding.status).toBe('fail');
+    expect(finding.summary).toContain('node2');
+    expect(finding.summary).toContain('2 module(s)');
+    // Both the host AND what it takes down with it — the thing doctor could not say.
+    expect(finding.detail.join('\n')).toContain('DOWN node2: celilo-apt-repo, lunacycle');
+    // The healthy node is not implicated.
+    expect(finding.summary).not.toContain('node3');
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('covers the machine pool, not only container-service nodes', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+  });
+
+  it('stays quiet when every host is up — no new permanent warning', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [{ node: 'node3', online: true }],
+        guestNodes: [{ vmid: 301, node: 'node3' }],
+      }),
+    );
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+    expect(finding.summary).toBe('2 host(s) up');
+  });
+
+  /**
+   * The absent-vs-empty rule, one level down: "the cluster did not answer" must
+   * never read as "every node is healthy".
+   *
+   * It WARNS rather than passing quietly. A cluster that will not answer its
+   * own API is not evidence of health, and the failure to answer may be the
+   * outage this check exists to catch — so reporting it as ok-with-a-note would
+   * rebuild #728 one level down. The warning is safe to have precisely because
+   * it is not permanent: a machine-only fleet produces no unverified hosts at
+   * all, every placement resolving through the probe.
+   */
+  it('a host celilo tried and failed to verify WARNS, with the reason', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        // The cluster was unreachable, so nothing came back about vmid 202.
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('lunacycle');
+    expect(finding.detail.join('\n')).toContain("not present in the cluster's resources");
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('an unprobed machine warns, never assumed reachable', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'somethingelse', reachable: true }],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('no probe result for this machine');
+  });
+
+  /**
+   * The reason is per host, not one blanket sentence: "could not verify" tells
+   * an operator nothing about whether to go and look at a cluster, a machine,
+   * or a stale row.
+   */
+  it('names a different reason for each way verification can fail', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'droplet-app', hostname: 'vps', infraType: 'container_service', vmid: null },
+        ],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    const detail = finding.detail.join('\n');
+    expect(detail).toContain('no probe result for this machine');
+    // The shape a non-Proxmox provider takes today, until celilo can read it.
+    expect(detail).toContain('no liveness source for this provider');
+  });
+
+  it('a down host still fails when a sibling host is unverified', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+    expect(finding.detail.join('\n')).toContain('unverified');
+  });
+
+  it('is silent on a fleet with nothing deployed', () => {
+    const finding = checkHostLiveness(inputs());
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+  });
+});
+
+/**
+ * The wiring, which is what actually fixes celilo#728. A pure check nothing
+ * calls changes nothing: on `main` today `runFleetChecks` returns no
+ * host-liveness finding at all, which is precisely why doctor said
+ * "OK with warnings" over an offline node.
+ */
+describe('runFleetChecks includes host liveness (#728)', () => {
+  let dir: string;
+  let db: DbClient;
+  let bus: Bus;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'fleet-liveness-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.EVENT_BUS_DB = join(dir, 'events.db');
+    db = await setupTestDatabase(join(dir, 'celilo.db'));
+    bus = openBus({ dbPath: join(dir, 'events.db'), events: defineEvents({}) });
+  });
+
+  afterEach(() => {
+    bus.close();
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('surfaces an offline node through the doctor findings, not just the checker', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+        ],
+        machines: [],
+        nodes: [{ node: 'node2', online: false }],
+        guestNodes: [{ vmid: 205, node: 'node2' }],
+      }),
+    });
+
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    // Against main this is `undefined` — the check does not exist in the list.
+    expect(liveness).toBeDefined();
+    expect(liveness?.status).toBe('fail');
+    expect(liveness?.summary).toContain('node2');
+  });
+
+  it('does not add a standing warning to a healthy fleet', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [],
+        guestNodes: [],
+      }),
+    });
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    expect(liveness?.status).toBe('ok');
+  });
+});