@celilo/cli 0.26.0 → 0.27.0

package/src/services/alerting/inbound.test.ts

@@ -410,3 +410,69 @@ describe('reply verbs', () => {
     expect(parseInbound('K7QM2X akc')).toEqual({ kind: 'unrecognised' });
   });
 });
+
+/**
+ * celilo#500. The accepted grammar, in one table, because it was previously
+ * knowable only by reading the parser.
+ *
+ * The headline case is `reply K7QM2X`: every page ends with that literal line,
+ * and an operator who did exactly what it said was told `unrecognised` while
+ * the alert kept firing. Two consecutive real operator replies were lost this
+ * way. The instruction the system gives was the one input it refused.
+ *
+ * The trailing-period case is the same shape of unkindness. iOS inserts a full
+ * stop on a double space by default, and a one-word message is precisely where
+ * that fires.
+ *
+ * Being liberal here costs nothing: authentication is the token plus the sender
+ * check (see the header of inbound.ts), never punctuation strictness.
+ */
+describe('the grammar accepts what a phone actually sends (#500)', () => {
+  const ACKNOWLEDGES: Array<[string, string]> = [
+    ['K7QM2X', 'the bare token'],
+    ['k7qm2x', 'lower case'],
+    ['K7-QM2X', 'a separator the operator kept'],
+    ['K7QM2X ack', 'a trailing verb'],
+    ['ack K7QM2X', 'the natural spoken order'],
+    ['reply K7QM2X', 'WHAT THE PAGE ITSELF INSTRUCTS'],
+    ['K7QM2X.', 'iOS double-space autocorrect'],
+    ['Ack K7QM2X.', 'both at once, capitalised'],
+    ['reply k7-qm2x.', 'everything at once'],
+  ];
+
+  test.each(ACKNOWLEDGES)('%p acknowledges (%s)', (body) => {
+    expect(parseInbound(body)).toEqual({ kind: 'ack', token: 'K7QM2X' });
+  });
+
+  /**
+   * The guard that must survive. `<token> resolve` silently acknowledging an
+   * alert the operator meant to escalate is the worst outcome available here,
+   * so widening the grammar must not widen THIS.
+   */
+  const REFUSED: Array<[string, string]> = [
+    ['K7QM2X resolve', 'a verb celilo does not implement'],
+    ['K7QM2X silence 2h', 'a request with an argument'],
+    ['K7QM2X akc', 'a typo, not guessed at'],
+    ['reply K7QM2X resolve', 'the new verb does not smuggle a sentence through'],
+    ['what is going on', 'no token-shaped word anywhere'],
+    ['K7QM2XY', 'too long to be a token'],
+    ['reply', 'the verb alone names nothing'],
+  ];
+
+  test.each(REFUSED)('%p is refused (%s)', (body) => {
+    expect(parseInbound(body)).toEqual({ kind: 'unrecognised' });
+  });
+
+  /**
+   * The instruction and the parser are pinned to the same literal from both
+   * sides, so they cannot drift apart again. `composeBody` in
+   * `modules/signal/scripts/notification.ts` emits `\n\nreply <token>`, and
+   * `signal-rpc.test.ts:273` asserts that exact output. This asserts the parser
+   * accepts it. Change the wording and one of the two goes red.
+   */
+  test('the exact line composeBody emits is accepted', () => {
+    const asSent = 'caddy is down\n\nreply K7QM2X'.split('\n\n')[1];
+    expect(asSent).toBe('reply K7QM2X');
+    expect(parseInbound(asSent)).toEqual({ kind: 'ack', token: 'K7QM2X' });
+  });
+});

package/src/services/alerting/inbound.ts

@@ -65,6 +65,33 @@ export type InboundIntent =
   | { kind: 'unrecognised' };
 
 const ACK_VERB = /^(ack|ok|k|👍)$/iu;
+
+/**
+ * A verb that may only LEAD, and does not itself acknowledge.
+ *
+ * Every page ends with the literal line `reply <TOKEN>` (`composeBody` in
+ * modules/signal/scripts/notification.ts), and an operator who did exactly that
+ * was told `unrecognised` while the alert kept firing — the instruction the
+ * system gives was the one input it refused (#500). Two consecutive real
+ * operator replies were lost to it.
+ *
+ * Distinct from `ACK_VERB` on purpose: an ack synonym standing alone IS an
+ * acknowledgement (`bare_ack`), whereas `reply` alone is someone echoing the
+ * instruction without the token, which names nothing and stays unrecognised.
+ */
+const LEADING_VERB = /^reply$/i;
+
+/**
+ * Punctuation a phone added that the operator did not type.
+ *
+ * iOS turns a double space into a full stop by default, and a one-word reply is
+ * exactly where that fires. Stripping it costs nothing: authentication here is
+ * the token plus the sender check (see the header), never punctuation
+ * strictness.
+ */
+function stripTrailingPunctuation(word: string): string {
+  return word.replace(/[.,!?;:]+$/u, '');
+}
 /**
  * One to six characters of the token alphabet. Not anchored to the full
  * length: a prefix is legal input, and whether it identifies something is a
@@ -76,9 +103,15 @@ export function parseInbound(body: string): InboundIntent {
   const words = body.trim().split(/\s+/).filter(Boolean);
   if (words.length === 0) return { kind: 'unrecognised' };
 
+  // Drop a leading `reply` — instruction-echo, not content. Removed before
+  // anything else so the rest of the grammar is entirely unaffected by whether
+  // the operator included it.
+  if (LEADING_VERB.test(words[0])) words.shift();
+  if (words.length === 0) return { kind: 'unrecognised' };
+
   // Strip ack synonyms wherever they appear. What remains must be the token,
   // or nothing at all.
-  const remainder = words.filter((word) => !ACK_VERB.test(word));
+  const remainder = words.filter((word) => !ACK_VERB.test(stripTrailingPunctuation(word)));
   if (remainder.length === 0) return { kind: 'bare_ack' };
 
   // More than one non-verb word is not a token with politeness around it, it
@@ -89,7 +122,7 @@ export function parseInbound(body: string): InboundIntent {
   // so is strictly better than doing the wrong thing quietly.
   if (remainder.length > 1) return { kind: 'unrecognised' };
 
-  const token = normaliseToken(remainder[0]);
+  const token = normaliseToken(stripTrailingPunctuation(remainder[0]));
   if (!TOKEN_SHAPE.test(token)) return { kind: 'unrecognised' };
 
   return { kind: 'ack', token };

package/src/services/alerting/sweep-runner.test.ts

@@ -329,7 +329,11 @@ describe('runSweep', () => {
 
       expect(report.notified).toBe(0);
       expect(report.noPolicy).toEqual([]);
-      expect(report.skipped.within_grace).toBe(1);
+      // The alert is named, not just counted — an operator asking "why was I
+      // not paged" is asking about a specific alert (#450).
+      expect(report.skipped).toEqual([
+        { alertKey: 'module:homebridge/check:port', reason: 'within_grace' },
+      ]);
     });
 
     test('a transport that cannot be loaded records the error, not just a count', async () => {

package/src/services/alerting/sweep-runner.ts

@@ -78,14 +78,20 @@ export interface SweepReport {
    */
   noPolicy: { alertKey: string; monitor: string }[];
   /**
-   * Deliveries escalation declined, keyed by its reason (`within_grace`,
-   * `no_eligible_route`, …).
+   * Deliveries escalation declined — the reason AND the alert it applies to.
    *
-   * `notifyAlert` returns the reason precisely so the caller can record it — its
-   * own contract says a silent skip is indistinguishable from a bug. Dropping it
-   * here is what made a firing-but-undelivered alert undebuggable (#450).
+   * `notifyAlert` returns the reason precisely so the caller can record it: its
+   * own contract says a silent skip is indistinguishable from a bug, and
+   * dropping it here is what made a firing-but-undelivered alert undebuggable
+   * (#450).
+   *
+   * The alert key is carried too, because a bare `within_grace×2` still does not
+   * answer "why was I not paged" for the alert the operator is actually looking
+   * at — they cannot tell which of their live alerts each count refers to. Same
+   * reasoning `noPolicy` already applies, and the same failure it was fixing.
+   * Counts are derived at render time so there is one source for both.
    */
-  skipped: Record<string, number>;
+  skipped: { alertKey: string; reason: string }[];
   /**
    * Why each failed delivery failed, as `<alert key>: <error>`.
    *
@@ -121,7 +127,7 @@ export async function runSweep(
     deferredDelivered: 0,
     failed: 0,
     noPolicy: [],
-    skipped: {},
+    skipped: [],
     failures: [],
   };
 
@@ -282,7 +288,7 @@ export async function runSweep(
       report.failed++;
       report.failures.push(`${alert.key}: ${outcome.error}`);
     } else if (outcome.result === 'skipped') {
-      report.skipped[outcome.reason] = (report.skipped[outcome.reason] ?? 0) + 1;
+      report.skipped.push({ alertKey: alert.key, reason: outcome.reason });
     }
   }
 

package/src/services/aspect-runner.test.ts

@@ -54,7 +54,6 @@ async function seedMachine(opts: {
     sshUser: 'root',
     sshKey: 'ssh-key-placeholder',
     hardware: { cpu_cores: 1, memory_mb: 512, disk_gb: 5, arch: 'amd64' },
-    assignedModuleIds: [],
     earmarkedModule: null,
   });
   if (opts.apiOnly) {

package/src/services/audit/machines-reachable.test.ts

@@ -1,12 +1,14 @@
 import { describe, expect, test } from 'bun:test';
+import { failingKeysFromFindings } from '../alerting/builtin-monitors';
+import { ancestorKeysFor, machineAlertKey } from '../alerting/suppression';
 import { auditMachinesReachable } from './machines-reachable';
 
 describe('auditMachinesReachable', () => {
   test('no findings when every machine is reachable', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
-        { id: 'm2', hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
       ],
     });
     expect(result).toEqual([]);
@@ -15,9 +17,8 @@ describe('auditMachinesReachable', () => {
   test('per-machine drift finding for one unreachable host', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -30,7 +31,8 @@ describe('auditMachinesReachable', () => {
       category: 'machines_reachable',
       severity: 'drift',
       code: 'machine_unreachable',
-      subject: 'm2',
+      // Hostname, not the DB UUID — this assertion encoded the #596 bug.
+      subject: 'dns-ext',
       actionable: false,
     });
     expect(result[0].message).toContain('dns-ext');
@@ -42,14 +44,12 @@ describe('auditMachinesReachable', () => {
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,
           message: 'host down',
         },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -68,12 +68,71 @@ describe('auditMachinesReachable', () => {
     expect(result[0].message).toContain('All 2 machines unreachable');
   });
 
+  /**
+   * celilo#596. The finding's subject becomes the alert key, and suppression
+   * resolves a machine's ancestor key from its HOSTNAME. Subjecting on the DB
+   * UUID produced a key nothing could ever match, so an unreachable machine
+   * suppressed nothing and every module on it paged independently — the exact
+   * cascade suppression exists to prevent.
+   *
+   * The prefix-only assertion in `e2e/tests/alert-ack-return-leg.test.ts`
+   * (`toContain('builtin:machines_reachable/machine:')`) passes for either
+   * value, which is why this survived. These assert the WHOLE key.
+   */
+  describe('the alert key is one suppression can match (#596)', () => {
+    test('producer and consumer derive the same key', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          {
+            hostname: 'dns-ext',
+            ipAddress: '203.0.113.5',
+            reachable: false,
+            message: 'Connection timed out',
+          },
+        ],
+      });
+
+      const keys = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      expect(keys).toEqual(['builtin:machines_reachable/machine:dns-ext']);
+      // The identity that actually matters: asserted against the consumer's own
+      // constructor rather than a second literal, so the two cannot drift apart
+      // while both still look right.
+      expect(keys[0]).toBe(machineAlertKey('dns-ext'));
+    });
+
+    test('an unreachable machine suppresses a module deployed on it', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: false, message: 'down' },
+        ],
+      });
+      const firing = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      const ancestors = ancestorKeysFor('module:homebridge/check:service_running', {
+        moduleSystems: [
+          { moduleId: 'homebridge', hostname: 'dns-ext', zone: 'internal', infraType: 'machine' },
+        ],
+        zoneProviders: [],
+      });
+
+      // The behaviour suppression.ts documents: the machine's own alert is what
+      // explains the module's. Before the fix the intersection was empty.
+      expect(ancestors.some((a) => firing.includes(a))).toBe(true);
+    });
+  });
+
   test('does NOT collapse when only one machine is in the pool', async () => {
     // A single machine failing is per-machine, not a system-wide signal.
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,

package/src/services/audit/machines-reachable.ts

@@ -19,9 +19,21 @@
 import type { DriftFinding } from './types';
 
 export interface MachineReachableResult {
-  /** Stable machine ID (UUID in the DB). */
-  id: string;
-  /** Hostname for the finding message. */
+  /**
+   * User-facing hostname, and the identifier every finding here is keyed by.
+   *
+   * NOT the machine's UUID. Suppression resolves a machine's ancestor key from
+   * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding
+   * subjected on the UUID produces an alert key suppression can never match —
+   * an unreachable machine then suppresses nothing and every module on it pages
+   * independently, which is the cascade suppression exists to prevent. That was
+   * celilo#596, filed against this check and fixed here; `disk-space.ts` cites
+   * it as the reason it keys on hostname too.
+   *
+   * The UUID used to be carried alongside as `id`. It is deleted rather than
+   * left unused (Rule 3.9): its only reader was the defect, and a field kept
+   * "just in case" is what the next subject line would reach for.
+   */
   hostname: string;
   ipAddress: string;
   /** True if SSH probe succeeded. */
@@ -79,7 +91,9 @@ export async function auditMachinesReachable(
       ].join('\n'),
       // Multi-step / interactive; not a one-shot.
       actionable: false,
-      subject: r.id,
+      // Hostname, so `machineAlertKey` can match this — see the note on
+      // MachineReachableResult.hostname (celilo#596).
+      subject: r.hostname,
     });
   }
 

package/src/services/deployed-systems.ts

@@ -99,6 +99,37 @@ export function getProvisionedSystems(db: DbClient): ProvisionedSystem[] {
     .sort((a, b) => (a.vmid ?? 0) - (b.vmid ?? 0));
 }
 
+/** One deployed system, with the module on it — the whole fleet, both infra types. */
+export interface ModulePlacementRow {
+  moduleId: string;
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  vmid: number | null;
+}
+
+/**
+ * Every module deployment across the fleet, machine-pool and container alike.
+ *
+ * The complement to `getModuleSystems` (one module) and `getProvisionedSystems`
+ * (containers only). Added for the doctor's host-liveness check (celilo#728),
+ * which has to ask about EVERY host something is running on — the defect there
+ * was precisely that no fleet-wide view of "what runs where" was being
+ * consulted.
+ */
+export function listAllModuleSystems(db: DbClient): ModulePlacementRow[] {
+  return db
+    .select()
+    .from(moduleSystems)
+    .all()
+    .map((r) => ({
+      moduleId: r.moduleId,
+      hostname: r.hostname,
+      infraType: r.infraType,
+      vmid: r.vmid ?? null,
+    }))
+    .sort((a, b) => a.moduleId.localeCompare(b.moduleId));
+}
+
 /**
  * All container_service systems (Proxmox LXCs, droplets, …) whose zone is in
  * `zones`, across every module — the LXC complement to machine-pool's

package/src/services/fleet-checks.test.ts

@@ -18,14 +18,17 @@ import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monit
 import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
+  type HostLivenessInputs,
   checkCapabilityProviders,
   checkControlPlaneNetwork,
   checkDispatcher,
+  checkHostLiveness,
   checkSchemaDrift,
   checkServiceDns,
   checkSubscribers,
   describeCapabilityProblem,
   findBrokenCapabilityDerivations,
+  runFleetChecks,
 } from './fleet-checks';
 import { ensureOperationsSweepSubscriber } from './module-operations';
 
@@ -766,3 +769,232 @@ describe('checkControlPlaneNetwork', () => {
     expect(finding.summary).toContain('192.168.0.0/24');
   });
 });
+
+/**
+ * celilo#728. `system doctor` reported "OK with warnings" while a Proxmox node
+ * was OFFLINE with `celilo-apt-repo` and `lunacycle` deployed on it. Its two
+ * warnings were about the dispatcher and a false-positive subscriber drift —
+ * neither related. The node's status was already in `proxmox node list`; doctor
+ * never consulted it, and the condition surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there.
+ */
+describe('checkHostLiveness', () => {
+  const inputs = (over: Partial<HostLivenessInputs> = {}): HostLivenessInputs => ({
+    placements: [],
+    machines: [],
+    nodes: [],
+    guestNodes: [],
+    ...over,
+  });
+
+  it('reproduces #728: an offline node hosting modules is a FAILURE naming both', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        nodes: [
+          { node: 'node2', online: false },
+          { node: 'node3', online: true },
+        ],
+        guestNodes: [
+          { vmid: 205, node: 'node2' },
+          { vmid: 202, node: 'node2' },
+          { vmid: 301, node: 'node3' },
+        ],
+      }),
+    );
+
+    expect(finding.status).toBe('fail');
+    expect(finding.summary).toContain('node2');
+    expect(finding.summary).toContain('2 module(s)');
+    // Both the host AND what it takes down with it — the thing doctor could not say.
+    expect(finding.detail.join('\n')).toContain('DOWN node2: celilo-apt-repo, lunacycle');
+    // The healthy node is not implicated.
+    expect(finding.summary).not.toContain('node3');
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('covers the machine pool, not only container-service nodes', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+  });
+
+  it('stays quiet when every host is up — no new permanent warning', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [{ node: 'node3', online: true }],
+        guestNodes: [{ vmid: 301, node: 'node3' }],
+      }),
+    );
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+    expect(finding.summary).toBe('2 host(s) up');
+  });
+
+  /**
+   * The absent-vs-empty rule, one level down: "the cluster did not answer" must
+   * never read as "every node is healthy".
+   *
+   * It WARNS rather than passing quietly. A cluster that will not answer its
+   * own API is not evidence of health, and the failure to answer may be the
+   * outage this check exists to catch — so reporting it as ok-with-a-note would
+   * rebuild #728 one level down. The warning is safe to have precisely because
+   * it is not permanent: a machine-only fleet produces no unverified hosts at
+   * all, every placement resolving through the probe.
+   */
+  it('a host celilo tried and failed to verify WARNS, with the reason', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        // The cluster was unreachable, so nothing came back about vmid 202.
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('lunacycle');
+    expect(finding.detail.join('\n')).toContain("not present in the cluster's resources");
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('an unprobed machine warns, never assumed reachable', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'somethingelse', reachable: true }],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('no probe result for this machine');
+  });
+
+  /**
+   * The reason is per host, not one blanket sentence: "could not verify" tells
+   * an operator nothing about whether to go and look at a cluster, a machine,
+   * or a stale row.
+   */
+  it('names a different reason for each way verification can fail', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'droplet-app', hostname: 'vps', infraType: 'container_service', vmid: null },
+        ],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    const detail = finding.detail.join('\n');
+    expect(detail).toContain('no probe result for this machine');
+    // The shape a non-Proxmox provider takes today, until celilo can read it.
+    expect(detail).toContain('no liveness source for this provider');
+  });
+
+  it('a down host still fails when a sibling host is unverified', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+    expect(finding.detail.join('\n')).toContain('unverified');
+  });
+
+  it('is silent on a fleet with nothing deployed', () => {
+    const finding = checkHostLiveness(inputs());
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+  });
+});
+
+/**
+ * The wiring, which is what actually fixes celilo#728. A pure check nothing
+ * calls changes nothing: on `main` today `runFleetChecks` returns no
+ * host-liveness finding at all, which is precisely why doctor said
+ * "OK with warnings" over an offline node.
+ */
+describe('runFleetChecks includes host liveness (#728)', () => {
+  let dir: string;
+  let db: DbClient;
+  let bus: Bus;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'fleet-liveness-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.EVENT_BUS_DB = join(dir, 'events.db');
+    db = await setupTestDatabase(join(dir, 'celilo.db'));
+    bus = openBus({ dbPath: join(dir, 'events.db'), events: defineEvents({}) });
+  });
+
+  afterEach(() => {
+    bus.close();
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('surfaces an offline node through the doctor findings, not just the checker', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+        ],
+        machines: [],
+        nodes: [{ node: 'node2', online: false }],
+        guestNodes: [{ vmid: 205, node: 'node2' }],
+      }),
+    });
+
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    // Against main this is `undefined` — the check does not exist in the list.
+    expect(liveness).toBeDefined();
+    expect(liveness?.status).toBe('fail');
+    expect(liveness?.summary).toContain('node2');
+  });
+
+  it('does not add a standing warning to a healthy fleet', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [],
+        guestNodes: [],
+      }),
+    });
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    expect(liveness?.status).toBe('ok');
+  });
+});