@celilo/cli 0.26.0 → 0.26.1

package/CELILO_CORE_MODULES.md

@@ -27,7 +27,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
 - **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
 - **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
 - **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). A caller supplies a NAME and nothing else: the address is the source IP of celilo's own update, re-derived on every assert. Registering `<domain>` also claims `www.<domain>` and vice versa (best effort, reported back as `outputs.companion_fqdn` so the framework's `public_dns` check watches it — Namecheap answers `ErrCount 0` for `www` updates it does not apply). DDNS passwords are keyed by the **registrable domain**, never the FQDN. **provides:** `dns_registrar`.
-- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. **REQUIRES the `control-plane-vpn` network and READS its range** (`requires.networks`; `client_subnet` is a `source: system` derive of `network.control-plane-vpn.subnet`). It does not write that network and has no way to — celilo owns the namespace, and the deploy will not reach any hook until the network is defined, asking for a range if one is missing. So `wg0` is attributable the moment it exists. This replaces a declare-before-you-create ordering inside `on_install`, which could only narrow the window and not close it: a consumer that captured config before the hook started could not see a value the hook wrote, whatever order it wrote it in, and that is exactly what left `wg0` unattributable (celilo#759). `health_check` still asserts the declaration matches what the tunnel serves — what it catches now is divergence, celilo's network having changed since the module resolved its config. That same key is what the internal resolver's split-horizon view consumes. Adopts a running tunnel in place (existing key retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **Adoption is a one-time IMPORT and it ENDS.** The marker is `registered_peers` — celilo's own config key, separate from the operator's `peers` so a machine can never rewrite what an operator typed. While that key has never been written the tunnel is not yet celilo's; the deploy that writes it (even as `[]`) CLAIMS the tunnel, importing whatever `[Peer]` blocks are running, and from then on the render is closed-world (`peers` ∪ `registered_peers`) with a `[Peer]` on the box in neither reported as drift by `on_install` and by `health_check`'s `unknown_peers`. An ABSENT `peers` declaration and a declared-EMPTY one are deliberately different (no `default: []` on either variable): they used to be the same value, so the module read every empty list as "adopt what is running", the last peer could not be revoked, and a hand-added peer rode along on every deploy unreported (celilo#765). On a tunnel celilo has not yet claimed, a declared-empty `peers` is REFUSED rather than obeyed — celilo's own variable-default seeding wrote a real `peers = []` row for every install of the previous version that never set one, so on the installed base an empty list cannot be told from a stored default, and obeying it would `wg syncconf` the admin tunnel down to zero peers. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does; the ISP-router drivers `greenwave` and `axon` do not).
+- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. **REQUIRES the `control-plane-vpn` network and READS its range** (`requires.networks`; `client_subnet` is a `source: system` derive of `network.control-plane-vpn.subnet`). It does not write that network and has no way to — celilo owns the namespace, and the deploy will not reach any hook until the network is defined, asking for a range if one is missing. So `wg0` is attributable the moment it exists. This replaces a declare-before-you-create ordering inside `on_install`, which could only narrow the window and not close it: a consumer that captured config before the hook started could not see a value the hook wrote, whatever order it wrote it in, and that is exactly what left `wg0` unattributable (celilo#759). `health_check` still asserts the declaration matches what the tunnel serves — what it catches now is divergence, celilo's network having changed since the module resolved its config. That same key is what the internal resolver's split-horizon view consumes. Adopts a running tunnel in place (existing key retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **Adoption is a one-time IMPORT and it ENDS.** The marker is `registered_peers` — celilo's own config key, separate from the operator's `peers` so a machine can never rewrite what an operator typed. While that key has never been written the tunnel is not yet celilo's; the deploy that writes it (even as `[]`) CLAIMS the tunnel, importing whatever `[Peer]` blocks are running, and from then on the render is closed-world (`peers` ∪ `registered_peers`) with a `[Peer]` on the box in neither reported as drift by `on_install` and by `health_check`'s `unknown_peers`. An ABSENT `peers` declaration and a declared-EMPTY one are deliberately different (no `default: []` on either variable): they used to be the same value, so the module read every empty list as "adopt what is running", the last peer could not be revoked, and a hand-added peer rode along on every deploy unreported (celilo#765). On a tunnel celilo has not yet claimed, a declared-empty `peers` is REFUSED rather than obeyed — celilo's own variable-default seeding wrote a real `peers = []` row for every install of the previous version that never set one, so on the installed base an empty list cannot be told from a stored default, and obeying it would `wg syncconf` the admin tunnel down to zero peers. **PROVIDES `control_plane_vpn`** so another module can enrol clients without an operator editing YAML: `registerClient` / `revokeClient` write to `registered_peers` — the same key the adoption claim uses — and `getEndpoint` reads the server's public key LIVE off the host rather than storing a copy that could be republished after the key changed. `client_pool` is the range a consumer may allocate from, a strict subset of the client subnet, and `validate_config` REFUSES any operator-declared peer inside it: the two allocators cannot see each other, so the range is divided rather than negotiated, and that check is the only place a bad division is caught. ⚠️ Granting this capability grants fleet-wide reach — the tunnel's client subnet is a registered trusted source, so every client enrolled through it reaches every managed zone. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does; the ISP-router drivers `greenwave` and `axon` do not).
 
 ## Public edge (ingress / identity)
 

package/CELILO_SUBSYSTEMS.md

@@ -60,6 +60,8 @@ see `openspec/specs/`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MOD
 | `registry_publish` | `packages/capabilities/src/registry-publish.ts` | celilo-registry |
 | `notification` | `packages/capabilities/src/notification.ts` (`send`, optional `receive`) | signal (planned) — transports are ordinary modules, both self-hosted and credential-only |
 | `cross_module_read` | `packages/capabilities/src/cross-module-read.ts` | framework (read other modules' capability data) |
+| `control_plane_vpn` | `packages/capabilities/src/control-plane-vpn.ts` (`registerClient`, `revokeClient`, `listClients`, `getEndpoint`). **Its state is NOT in a celilo table** — registrations live in the PROVIDER's own module config under `registered_peers`, kept separate from the operator's `peers` so a machine can never rewrite what an operator typed, and so `revokeClient` structurally cannot reach a declared peer. Do not go looking for a `vpn_clients` table; there isn't one and that is deliberate. Addresses are allocated by the CONSUMER from `client_pool` — celilo's IPAM deliberately does not cover VPN clients. ⚠️ Every client registered here reaches every managed zone: the tunnel's client subnet is a registered trusted source. | wireguard |
+| `private_web` | `packages/capabilities/src/private-web.ts` (`publishStaticSite`, `registerReverseProxy`, `unregisterRoutes`, `getCaCertificate`). **Fleet-only HTTP ingress** — internal DNS, an internally-issued certificate, and no public exposure. A SIBLING of `public_web` rather than a flag on it, because `public_web` treats an unreachable route as a deploy failure and publishes a public record to prevent one. Reuses `public_web`'s request types so a consumer can write `private_web ?? public_web` and call through the union without branching. | caddy-internal (planned) |
 
 ### In-fleet (`public_web`) vs off-fleet (`external_web`)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -58,7 +58,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1109.0",
     "@aws-sdk/lib-storage": "^3.1101.0",
-    "@celilo/capabilities": "^1.3.0",
+    "@celilo/capabilities": "^1.4.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/core": "^0.8.0",
     "@celilo/event-bus": "^0.6.0",

package/src/hooks/types.ts

@@ -121,7 +121,8 @@ export type HookName =
   | 'on_restore'
   | 'on_system_event'
   | 'refresh_registrations'
-  | 'reassert_dhcp_dns';
+  | 'reassert_dhcp_dns'
+  | 'reconcile_clients';
 
 /**
  * Hook manifest section - maps hook names to definitions

package/src/manifest/contracts/v1.ts

@@ -210,6 +210,25 @@ export const V1_HOOKS: ContractHooks = {
     inputs: {},
     outputs: {},
   },
+  /**
+   * Reconcile the clients a self-service app has enrolled against what the
+   * VPN provider actually carries (openspec/changes/wireguard-manager, D1).
+   *
+   * The hook, rather than a bus `handler` subscription, because it needs
+   * CAPABILITY INJECTION: it calls `control_plane_vpn.registerClient` and
+   * `revokeClient`, and a handler gets no capabilities. No framework inputs —
+   * it reads the app's address from its own `systems` and holds a token it
+   * minted for itself at install.
+   *
+   * Typically driven by a `timer.tick.*` subscription, and the tick interval IS
+   * the window in which a revoked device still has reach — which is why the
+   * design requires the UI to show the pending state rather than imply
+   * revocation is instant (D12).
+   */
+  reconcile_clients: {
+    inputs: {},
+    outputs: {},
+  },
   /**
    * Build-bus upstream publish hook. The executor passes the
    * PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,

package/src/manifest/schema.ts

@@ -674,6 +674,7 @@ export const ModuleManifestSchema = z
          * client can be handed the wrong resolver. See celilo#739.
          */
         reassert_dhcp_dns: LifecycleHookSchema.optional(),
+        reconcile_clients: LifecycleHookSchema.optional(),
         /**
          * Build-bus upstream publish hooks. Array (a module can react
          * to multiple upstream packages with different actions). See

package/src/services/alerting/inbound.test.ts

@@ -410,3 +410,69 @@ describe('reply verbs', () => {
     expect(parseInbound('K7QM2X akc')).toEqual({ kind: 'unrecognised' });
   });
 });
+
+/**
+ * celilo#500. The accepted grammar, in one table, because it was previously
+ * knowable only by reading the parser.
+ *
+ * The headline case is `reply K7QM2X`: every page ends with that literal line,
+ * and an operator who did exactly what it said was told `unrecognised` while
+ * the alert kept firing. Two consecutive real operator replies were lost this
+ * way. The instruction the system gives was the one input it refused.
+ *
+ * The trailing-period case is the same shape of unkindness. iOS inserts a full
+ * stop on a double space by default, and a one-word message is precisely where
+ * that fires.
+ *
+ * Being liberal here costs nothing: authentication is the token plus the sender
+ * check (see the header of inbound.ts), never punctuation strictness.
+ */
+describe('the grammar accepts what a phone actually sends (#500)', () => {
+  const ACKNOWLEDGES: Array<[string, string]> = [
+    ['K7QM2X', 'the bare token'],
+    ['k7qm2x', 'lower case'],
+    ['K7-QM2X', 'a separator the operator kept'],
+    ['K7QM2X ack', 'a trailing verb'],
+    ['ack K7QM2X', 'the natural spoken order'],
+    ['reply K7QM2X', 'WHAT THE PAGE ITSELF INSTRUCTS'],
+    ['K7QM2X.', 'iOS double-space autocorrect'],
+    ['Ack K7QM2X.', 'both at once, capitalised'],
+    ['reply k7-qm2x.', 'everything at once'],
+  ];
+
+  test.each(ACKNOWLEDGES)('%p acknowledges (%s)', (body) => {
+    expect(parseInbound(body)).toEqual({ kind: 'ack', token: 'K7QM2X' });
+  });
+
+  /**
+   * The guard that must survive. `<token> resolve` silently acknowledging an
+   * alert the operator meant to escalate is the worst outcome available here,
+   * so widening the grammar must not widen THIS.
+   */
+  const REFUSED: Array<[string, string]> = [
+    ['K7QM2X resolve', 'a verb celilo does not implement'],
+    ['K7QM2X silence 2h', 'a request with an argument'],
+    ['K7QM2X akc', 'a typo, not guessed at'],
+    ['reply K7QM2X resolve', 'the new verb does not smuggle a sentence through'],
+    ['what is going on', 'no token-shaped word anywhere'],
+    ['K7QM2XY', 'too long to be a token'],
+    ['reply', 'the verb alone names nothing'],
+  ];
+
+  test.each(REFUSED)('%p is refused (%s)', (body) => {
+    expect(parseInbound(body)).toEqual({ kind: 'unrecognised' });
+  });
+
+  /**
+   * The instruction and the parser are pinned to the same literal from both
+   * sides, so they cannot drift apart again. `composeBody` in
+   * `modules/signal/scripts/notification.ts` emits `\n\nreply <token>`, and
+   * `signal-rpc.test.ts:273` asserts that exact output. This asserts the parser
+   * accepts it. Change the wording and one of the two goes red.
+   */
+  test('the exact line composeBody emits is accepted', () => {
+    const asSent = 'caddy is down\n\nreply K7QM2X'.split('\n\n')[1];
+    expect(asSent).toBe('reply K7QM2X');
+    expect(parseInbound(asSent)).toEqual({ kind: 'ack', token: 'K7QM2X' });
+  });
+});

package/src/services/alerting/inbound.ts

@@ -65,6 +65,33 @@ export type InboundIntent =
   | { kind: 'unrecognised' };
 
 const ACK_VERB = /^(ack|ok|k|👍)$/iu;
+
+/**
+ * A verb that may only LEAD, and does not itself acknowledge.
+ *
+ * Every page ends with the literal line `reply <TOKEN>` (`composeBody` in
+ * modules/signal/scripts/notification.ts), and an operator who did exactly that
+ * was told `unrecognised` while the alert kept firing — the instruction the
+ * system gives was the one input it refused (#500). Two consecutive real
+ * operator replies were lost to it.
+ *
+ * Distinct from `ACK_VERB` on purpose: an ack synonym standing alone IS an
+ * acknowledgement (`bare_ack`), whereas `reply` alone is someone echoing the
+ * instruction without the token, which names nothing and stays unrecognised.
+ */
+const LEADING_VERB = /^reply$/i;
+
+/**
+ * Punctuation a phone added that the operator did not type.
+ *
+ * iOS turns a double space into a full stop by default, and a one-word reply is
+ * exactly where that fires. Stripping it costs nothing: authentication here is
+ * the token plus the sender check (see the header), never punctuation
+ * strictness.
+ */
+function stripTrailingPunctuation(word: string): string {
+  return word.replace(/[.,!?;:]+$/u, '');
+}
 /**
  * One to six characters of the token alphabet. Not anchored to the full
  * length: a prefix is legal input, and whether it identifies something is a
@@ -76,9 +103,15 @@ export function parseInbound(body: string): InboundIntent {
   const words = body.trim().split(/\s+/).filter(Boolean);
   if (words.length === 0) return { kind: 'unrecognised' };
 
+  // Drop a leading `reply` — instruction-echo, not content. Removed before
+  // anything else so the rest of the grammar is entirely unaffected by whether
+  // the operator included it.
+  if (LEADING_VERB.test(words[0])) words.shift();
+  if (words.length === 0) return { kind: 'unrecognised' };
+
   // Strip ack synonyms wherever they appear. What remains must be the token,
   // or nothing at all.
-  const remainder = words.filter((word) => !ACK_VERB.test(word));
+  const remainder = words.filter((word) => !ACK_VERB.test(stripTrailingPunctuation(word)));
   if (remainder.length === 0) return { kind: 'bare_ack' };
 
   // More than one non-verb word is not a token with politeness around it, it
@@ -89,7 +122,7 @@ export function parseInbound(body: string): InboundIntent {
   // so is strictly better than doing the wrong thing quietly.
   if (remainder.length > 1) return { kind: 'unrecognised' };
 
-  const token = normaliseToken(remainder[0]);
+  const token = normaliseToken(stripTrailingPunctuation(remainder[0]));
   if (!TOKEN_SHAPE.test(token)) return { kind: 'unrecognised' };
 
   return { kind: 'ack', token };

package/src/services/audit/machines-reachable.test.ts

@@ -1,12 +1,14 @@
 import { describe, expect, test } from 'bun:test';
+import { failingKeysFromFindings } from '../alerting/builtin-monitors';
+import { ancestorKeysFor, machineAlertKey } from '../alerting/suppression';
 import { auditMachinesReachable } from './machines-reachable';
 
 describe('auditMachinesReachable', () => {
   test('no findings when every machine is reachable', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
-        { id: 'm2', hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
       ],
     });
     expect(result).toEqual([]);
@@ -15,9 +17,8 @@ describe('auditMachinesReachable', () => {
   test('per-machine drift finding for one unreachable host', async () => {
     const result = await auditMachinesReachable({
       results: [
-        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -30,7 +31,8 @@ describe('auditMachinesReachable', () => {
       category: 'machines_reachable',
       severity: 'drift',
       code: 'machine_unreachable',
-      subject: 'm2',
+      // Hostname, not the DB UUID — this assertion encoded the #596 bug.
+      subject: 'dns-ext',
       actionable: false,
     });
     expect(result[0].message).toContain('dns-ext');
@@ -42,14 +44,12 @@ describe('auditMachinesReachable', () => {
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,
           message: 'host down',
         },
         {
-          id: 'm2',
           hostname: 'dns-ext',
           ipAddress: '203.0.113.5',
           reachable: false,
@@ -68,12 +68,71 @@ describe('auditMachinesReachable', () => {
     expect(result[0].message).toContain('All 2 machines unreachable');
   });
 
+  /**
+   * celilo#596. The finding's subject becomes the alert key, and suppression
+   * resolves a machine's ancestor key from its HOSTNAME. Subjecting on the DB
+   * UUID produced a key nothing could ever match, so an unreachable machine
+   * suppressed nothing and every module on it paged independently — the exact
+   * cascade suppression exists to prevent.
+   *
+   * The prefix-only assertion in `e2e/tests/alert-ack-return-leg.test.ts`
+   * (`toContain('builtin:machines_reachable/machine:')`) passes for either
+   * value, which is why this survived. These assert the WHOLE key.
+   */
+  describe('the alert key is one suppression can match (#596)', () => {
+    test('producer and consumer derive the same key', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          {
+            hostname: 'dns-ext',
+            ipAddress: '203.0.113.5',
+            reachable: false,
+            message: 'Connection timed out',
+          },
+        ],
+      });
+
+      const keys = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      expect(keys).toEqual(['builtin:machines_reachable/machine:dns-ext']);
+      // The identity that actually matters: asserted against the consumer's own
+      // constructor rather than a second literal, so the two cannot drift apart
+      // while both still look right.
+      expect(keys[0]).toBe(machineAlertKey('dns-ext'));
+    });
+
+    test('an unreachable machine suppresses a module deployed on it', async () => {
+      const findings = await auditMachinesReachable({
+        results: [
+          { hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+          { hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: false, message: 'down' },
+        ],
+      });
+      const firing = failingKeysFromFindings('machines_reachable', findings, 'warning').map(
+        (k) => k.key,
+      );
+
+      const ancestors = ancestorKeysFor('module:homebridge/check:service_running', {
+        moduleSystems: [
+          { moduleId: 'homebridge', hostname: 'dns-ext', zone: 'internal', infraType: 'machine' },
+        ],
+        zoneProviders: [],
+      });
+
+      // The behaviour suppression.ts documents: the machine's own alert is what
+      // explains the module's. Before the fix the intersection was empty.
+      expect(ancestors.some((a) => firing.includes(a))).toBe(true);
+    });
+  });
+
   test('does NOT collapse when only one machine is in the pool', async () => {
     // A single machine failing is per-machine, not a system-wide signal.
     const result = await auditMachinesReachable({
       results: [
         {
-          id: 'm1',
           hostname: 'iot',
           ipAddress: '10.0.0.10',
           reachable: false,

package/src/services/audit/machines-reachable.ts

@@ -19,9 +19,21 @@
 import type { DriftFinding } from './types';
 
 export interface MachineReachableResult {
-  /** Stable machine ID (UUID in the DB). */
-  id: string;
-  /** Hostname for the finding message. */
+  /**
+   * User-facing hostname, and the identifier every finding here is keyed by.
+   *
+   * NOT the machine's UUID. Suppression resolves a machine's ancestor key from
+   * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding
+   * subjected on the UUID produces an alert key suppression can never match —
+   * an unreachable machine then suppresses nothing and every module on it pages
+   * independently, which is the cascade suppression exists to prevent. That was
+   * celilo#596, filed against this check and fixed here; `disk-space.ts` cites
+   * it as the reason it keys on hostname too.
+   *
+   * The UUID used to be carried alongside as `id`. It is deleted rather than
+   * left unused (Rule 3.9): its only reader was the defect, and a field kept
+   * "just in case" is what the next subject line would reach for.
+   */
   hostname: string;
   ipAddress: string;
   /** True if SSH probe succeeded. */
@@ -79,7 +91,9 @@ export async function auditMachinesReachable(
       ].join('\n'),
       // Multi-step / interactive; not a one-shot.
       actionable: false,
-      subject: r.id,
+      // Hostname, so `machineAlertKey` can match this — see the note on
+      // MachineReachableResult.hostname (celilo#596).
+      subject: r.hostname,
     });
   }
 

package/src/services/deployed-systems.ts

@@ -99,6 +99,37 @@ export function getProvisionedSystems(db: DbClient): ProvisionedSystem[] {
     .sort((a, b) => (a.vmid ?? 0) - (b.vmid ?? 0));
 }
 
+/** One deployed system, with the module on it — the whole fleet, both infra types. */
+export interface ModulePlacementRow {
+  moduleId: string;
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  vmid: number | null;
+}
+
+/**
+ * Every module deployment across the fleet, machine-pool and container alike.
+ *
+ * The complement to `getModuleSystems` (one module) and `getProvisionedSystems`
+ * (containers only). Added for the doctor's host-liveness check (celilo#728),
+ * which has to ask about EVERY host something is running on — the defect there
+ * was precisely that no fleet-wide view of "what runs where" was being
+ * consulted.
+ */
+export function listAllModuleSystems(db: DbClient): ModulePlacementRow[] {
+  return db
+    .select()
+    .from(moduleSystems)
+    .all()
+    .map((r) => ({
+      moduleId: r.moduleId,
+      hostname: r.hostname,
+      infraType: r.infraType,
+      vmid: r.vmid ?? null,
+    }))
+    .sort((a, b) => a.moduleId.localeCompare(b.moduleId));
+}
+
 /**
  * All container_service systems (Proxmox LXCs, droplets, …) whose zone is in
  * `zones`, across every module — the LXC complement to machine-pool's

package/src/services/fleet-checks.test.ts

@@ -18,14 +18,17 @@ import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monit
 import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
+  type HostLivenessInputs,
   checkCapabilityProviders,
   checkControlPlaneNetwork,
   checkDispatcher,
+  checkHostLiveness,
   checkSchemaDrift,
   checkServiceDns,
   checkSubscribers,
   describeCapabilityProblem,
   findBrokenCapabilityDerivations,
+  runFleetChecks,
 } from './fleet-checks';
 import { ensureOperationsSweepSubscriber } from './module-operations';
 
@@ -766,3 +769,232 @@ describe('checkControlPlaneNetwork', () => {
     expect(finding.summary).toContain('192.168.0.0/24');
   });
 });
+
+/**
+ * celilo#728. `system doctor` reported "OK with warnings" while a Proxmox node
+ * was OFFLINE with `celilo-apt-repo` and `lunacycle` deployed on it. Its two
+ * warnings were about the dispatcher and a false-positive subscriber drift —
+ * neither related. The node's status was already in `proxmox node list`; doctor
+ * never consulted it, and the condition surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there.
+ */
+describe('checkHostLiveness', () => {
+  const inputs = (over: Partial<HostLivenessInputs> = {}): HostLivenessInputs => ({
+    placements: [],
+    machines: [],
+    nodes: [],
+    guestNodes: [],
+    ...over,
+  });
+
+  it('reproduces #728: an offline node hosting modules is a FAILURE naming both', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        nodes: [
+          { node: 'node2', online: false },
+          { node: 'node3', online: true },
+        ],
+        guestNodes: [
+          { vmid: 205, node: 'node2' },
+          { vmid: 202, node: 'node2' },
+          { vmid: 301, node: 'node3' },
+        ],
+      }),
+    );
+
+    expect(finding.status).toBe('fail');
+    expect(finding.summary).toContain('node2');
+    expect(finding.summary).toContain('2 module(s)');
+    // Both the host AND what it takes down with it — the thing doctor could not say.
+    expect(finding.detail.join('\n')).toContain('DOWN node2: celilo-apt-repo, lunacycle');
+    // The healthy node is not implicated.
+    expect(finding.summary).not.toContain('node3');
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('covers the machine pool, not only container-service nodes', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+  });
+
+  it('stays quiet when every host is up — no new permanent warning', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
+        ],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [{ node: 'node3', online: true }],
+        guestNodes: [{ vmid: 301, node: 'node3' }],
+      }),
+    );
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+    expect(finding.summary).toBe('2 host(s) up');
+  });
+
+  /**
+   * The absent-vs-empty rule, one level down: "the cluster did not answer" must
+   * never read as "every node is healthy".
+   *
+   * It WARNS rather than passing quietly. A cluster that will not answer its
+   * own API is not evidence of health, and the failure to answer may be the
+   * outage this check exists to catch — so reporting it as ok-with-a-note would
+   * rebuild #728 one level down. The warning is safe to have precisely because
+   * it is not permanent: a machine-only fleet produces no unverified hosts at
+   * all, every placement resolving through the probe.
+   */
+  it('a host celilo tried and failed to verify WARNS, with the reason', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        // The cluster was unreachable, so nothing came back about vmid 202.
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('lunacycle');
+    expect(finding.detail.join('\n')).toContain("not present in the cluster's resources");
+    expect(finding.remediation).toBeTruthy();
+  });
+
+  it('an unprobed machine warns, never assumed reachable', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'somethingelse', reachable: true }],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join('\n')).toContain('no probe result for this machine');
+  });
+
+  /**
+   * The reason is per host, not one blanket sentence: "could not verify" tells
+   * an operator nothing about whether to go and look at a cluster, a machine,
+   * or a stale row.
+   */
+  it('names a different reason for each way verification can fail', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'droplet-app', hostname: 'vps', infraType: 'container_service', vmid: null },
+        ],
+      }),
+    );
+    expect(finding.status).toBe('warn');
+    const detail = finding.detail.join('\n');
+    expect(detail).toContain('no probe result for this machine');
+    // The shape a non-Proxmox provider takes today, until celilo can read it.
+    expect(detail).toContain('no liveness source for this provider');
+  });
+
+  it('a down host still fails when a sibling host is unverified', () => {
+    const finding = checkHostLiveness(
+      inputs({
+        placements: [
+          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
+          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
+        ],
+        machines: [{ hostname: 'iot', reachable: false }],
+      }),
+    );
+    expect(finding.status).toBe('fail');
+    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
+    expect(finding.detail.join('\n')).toContain('unverified');
+  });
+
+  it('is silent on a fleet with nothing deployed', () => {
+    const finding = checkHostLiveness(inputs());
+    expect(finding.status).toBe('ok');
+    expect(finding.detail).toEqual([]);
+  });
+});
+
+/**
+ * The wiring, which is what actually fixes celilo#728. A pure check nothing
+ * calls changes nothing: on `main` today `runFleetChecks` returns no
+ * host-liveness finding at all, which is precisely why doctor said
+ * "OK with warnings" over an offline node.
+ */
+describe('runFleetChecks includes host liveness (#728)', () => {
+  let dir: string;
+  let db: DbClient;
+  let bus: Bus;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'fleet-liveness-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.EVENT_BUS_DB = join(dir, 'events.db');
+    db = await setupTestDatabase(join(dir, 'celilo.db'));
+    bus = openBus({ dbPath: join(dir, 'events.db'), events: defineEvents({}) });
+  });
+
+  afterEach(() => {
+    bus.close();
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('surfaces an offline node through the doctor findings, not just the checker', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [
+          {
+            moduleId: 'celilo-apt-repo',
+            hostname: 'apt',
+            infraType: 'container_service',
+            vmid: 205,
+          },
+        ],
+        machines: [],
+        nodes: [{ node: 'node2', online: false }],
+        guestNodes: [{ vmid: 205, node: 'node2' }],
+      }),
+    });
+
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    // Against main this is `undefined` — the check does not exist in the list.
+    expect(liveness).toBeDefined();
+    expect(liveness?.status).toBe('fail');
+    expect(liveness?.summary).toContain('node2');
+  });
+
+  it('does not add a standing warning to a healthy fleet', async () => {
+    const findings = await runFleetChecks(bus, db, {
+      hostLiveness: async () => ({
+        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
+        machines: [{ hostname: 'iot', reachable: true }],
+        nodes: [],
+        guestNodes: [],
+      }),
+    });
+    const liveness = findings.find((f) => f.id === 'host-liveness');
+    expect(liveness?.status).toBe('ok');
+  });
+});

package/src/services/fleet-checks.ts

@@ -19,6 +19,7 @@
 
 import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
+import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
 import { getMigrationStatus } from '../db/migration-status';
@@ -27,9 +28,8 @@ import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
 
-/** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
-import { getModuleSystems } from './deployed-systems';
+import { getServiceCredentials, listContainerServices } from './container-service';
+import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -38,9 +38,13 @@ import {
   readInstalledUnit,
   unitMainPid,
 } from './events-daemon';
+import { probeMachines } from './machine-probe';
 import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
+/** The module that IS celilo's control plane. */
+const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other
  * than `internal` is firewall-segmented — an unmanaged LAN device has no
@@ -786,6 +790,12 @@ export async function checkServiceDns(db: DbClient): Promise<FleetFinding> {
 export interface RunFleetChecksOptions {
   now?: number;
   installedCodeMtimeMs?: number | null;
+  /**
+   * Where the host-liveness verdict gets its facts. Injected so the check can
+   * be exercised without SSH or a Proxmox credential; defaults to the live
+   * fleet (`collectHostLiveness`).
+   */
+  hostLiveness?: () => Promise<HostLivenessInputs>;
 }
 
 /**
@@ -901,11 +911,272 @@ export function checkPausedModules(db: DbClient): FleetFinding {
   };
 }
 
+/** One module deployment, and the host it landed on. */
+export interface HostPlacement {
+  moduleId: string;
+  /** The host's user-facing name — a pool hostname, or the container's. */
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  /** Proxmox VMID for a celilo-provisioned container; null for a pool machine. */
+  vmid: number | null;
+}
+
+/**
+ * Everything the liveness verdict is computed from, injected so the check is a
+ * pure function over data and needs neither SSH nor a Proxmox credential to
+ * test.
+ *
+ * ⚠️ Every source here is ALLOWED TO BE ABSENT, and absent is not "fine".
+ * A machine missing from `machines` was not probed; a node missing from `nodes`
+ * was not reported. Neither means the host is up, and neither means it is down.
+ * Conflating "I could not look" with "I looked and it was healthy" is the
+ * failure this whole check exists to end — doctor said OK-with-warnings while a
+ * node hosting two modules was offline.
+ */
+export interface HostLivenessInputs {
+  placements: HostPlacement[];
+  /** Machine-pool SSH probe results. A hostname absent here was NOT probed. */
+  machines: Array<{ hostname: string; reachable: boolean }>;
+  /** Proxmox node status. Empty when no container service is configured. */
+  nodes: Array<{ node: string; online: boolean }>;
+  /** VMID → node name, from `/cluster/resources`. Empty when unqueried. */
+  guestNodes: Array<{ vmid: number; node: string }>;
+}
+
+type HostState = 'up' | 'down' | 'unknown';
+
+interface HostVerdict {
+  host: string;
+  state: HostState;
+  /** Why the state could not be determined. Set only when state is 'unknown'. */
+  reason?: string;
+}
+
+function resolveHostState(placement: HostPlacement, inputs: HostLivenessInputs): HostVerdict {
+  if (placement.infraType === 'machine') {
+    const probe = inputs.machines.find((m) => m.hostname === placement.hostname);
+    if (!probe) {
+      return {
+        host: placement.hostname,
+        state: 'unknown',
+        // Either the SSH probe did not run at all, or this hostname is no
+        // longer in the machine pool — a stale `module_systems` row, which is
+        // its own defect and worth surfacing rather than rounding off.
+        reason: 'no probe result for this machine',
+      };
+    }
+    return { host: placement.hostname, state: probe.reachable ? 'up' : 'down' };
+  }
+
+  // A container's liveness is its NODE's liveness. The guest being stopped is a
+  // different condition with a different owner (`module pause --stop-infra`
+  // stops guests deliberately), so this deliberately reads the node only.
+  if (placement.vmid === null) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: 'no VMID recorded — celilo has no liveness source for this provider',
+    };
+  }
+  const guest = inputs.guestNodes.find((g) => g.vmid === placement.vmid);
+  if (!guest) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: `VMID ${placement.vmid} not present in the cluster's resources`,
+    };
+  }
+  const node = inputs.nodes.find((n) => n.node === guest.node);
+  if (!node) {
+    return { host: guest.node, state: 'unknown', reason: 'the cluster reported no such node' };
+  }
+  return { host: guest.node, state: node.online ? 'up' : 'down' };
+}
+
+/**
+ * Are the hosts this fleet's modules actually run on alive? (celilo#728)
+ *
+ * Every other `Fleet runtime` check is a control-plane concern — the
+ * dispatcher, bus subscribers, capability-derived config, internal DNS. None of
+ * them asked the most basic data-plane question, so `system doctor` reported
+ * "OK with warnings" while a Proxmox node was OFFLINE with `celilo-apt-repo`
+ * and `lunacycle` on it. The information was already in `proxmox node list`;
+ * doctor simply never consulted it. It surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there — had nothing tried
+ * to publish, the node could have stayed down indefinitely.
+ *
+ * A down host is a FAILURE, not a warning: it is strictly worse than the
+ * conditions already reported as failures here, and every module on it is down
+ * with it.
+ *
+ * A host celilo TRIED to verify and could not is a WARNING, and the reason is
+ * named per host. This is not the same as "quiet because it might be fine": a
+ * cluster that will not answer its own API is not obviously healthier than one
+ * reporting a node offline, and the failure to answer may BE the outage this
+ * check exists to catch. Reporting it as ok-with-a-note would rebuild the
+ * defect one level down — a report reading healthy over something unmeasured.
+ *
+ * ⚠️ The thing that makes the warning safe to have is that it is not
+ * permanent. A machine-only fleet produces NO unverified hosts at all: every
+ * placement takes the probe path and resolves. The one standing source would be
+ * a provider celilo cannot interrogate — today a DigitalOcean droplet, whose
+ * client can verify the token but never reads droplet status. That is a gap to
+ * close (its own change), not a reason to soften the signal here. A warning
+ * that fires forever is what trains an operator to skim the whole report
+ * (celilo#723, whose false positive was competing for attention in the very
+ * output that missed the offline node) — so if this one ever becomes standing,
+ * the fix is to teach celilo the missing provider, not to quieten it.
+ */
+export function checkHostLiveness(inputs: HostLivenessInputs): FleetFinding {
+  const base = {
+    id: 'host-liveness',
+    title: 'The hosts running deployed modules are alive',
+    autoFixable: false,
+  } as const;
+
+  if (inputs.placements.length === 0) {
+    return {
+      ...base,
+      status: 'ok',
+      summary: 'no modules are deployed to a host yet',
+      detail: [],
+      remediation: null,
+    };
+  }
+
+  interface HostEntry {
+    state: HostState;
+    modules: Set<string>;
+    reason?: string;
+  }
+
+  const modulesByHost = new Map<string, HostEntry>();
+  for (const placement of inputs.placements) {
+    const { host, state, reason } = resolveHostState(placement, inputs);
+    const entry = modulesByHost.get(host) ?? { state, modules: new Set<string>(), reason };
+    // A host resolved 'down' by any placement stays down — one authoritative
+    // negative outranks an unknown from a sibling placement.
+    if (state === 'down' || entry.state === 'unknown') {
+      entry.state = state;
+      entry.reason = reason;
+    }
+    entry.modules.add(placement.moduleId);
+    modulesByHost.set(host, entry);
+  }
+
+  const describe = (host: string, e: HostEntry) => `${host}: ${[...e.modules].sort().join(', ')}`;
+  const describeUnverified = (host: string, e: HostEntry) =>
+    `unverified — ${describe(host, e)}${e.reason ? ` (${e.reason})` : ''}`;
+
+  const down = [...modulesByHost].filter(([, e]) => e.state === 'down');
+  const unknown = [...modulesByHost].filter(([, e]) => e.state === 'unknown');
+  const up = [...modulesByHost].filter(([, e]) => e.state === 'up');
+
+  if (down.length > 0) {
+    const affected = down.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'fail',
+      summary: `${down.length} host(s) down, ${affected} module(s) unreachable: ${down
+        .map(([host]) => host)
+        .join(', ')}`,
+      detail: [
+        ...down.map(([host, e]) => `DOWN ${describe(host, e)}`),
+        ...unknown.map(([host, e]) => describeUnverified(host, e)),
+        'every module listed against a down host is down with it, whatever its own status says',
+      ],
+      remediation:
+        'bring the host back, then confirm with "celilo proxmox node list" (container services) or "celilo machine status <hostname>" (pool machines)',
+    };
+  }
+
+  if (unknown.length > 0) {
+    // WARN, not ok-with-a-note. celilo tried and could not find out, and the
+    // reason it could not may be the outage itself — a cluster that will not
+    // answer its own API is not evidence of health. Reporting this quietly
+    // would rebuild #728 one level down.
+    const affected = unknown.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'warn',
+      summary: `${up.length} host(s) up, ${unknown.length} could not be verified (${affected} module(s))`,
+      // Named and reasoned, never counted: "1 not verified" tells an operator
+      // neither which host nor what to do about it.
+      detail: unknown.map(([host, e]) => describeUnverified(host, e)),
+      remediation:
+        'check the host directly — "celilo proxmox node list" for a container service, "celilo machine status <hostname>" for a pool machine; a host celilo cannot reach is not a host known to be healthy',
+    };
+  }
+
+  return {
+    ...base,
+    status: 'ok',
+    summary: `${up.length} host(s) up`,
+    detail: [],
+    remediation: null,
+  };
+}
+
+/**
+ * Read the liveness facts off the live fleet.
+ *
+ * Every source degrades to ABSENT rather than to a cheerful default. A Proxmox
+ * cluster that cannot be reached, or a fleet with no container service at all,
+ * contributes no node rows — and `checkHostLiveness` reads that as unverified,
+ * never as healthy. That distinction is the whole point of the check.
+ */
+export async function collectHostLiveness(db: DbClient): Promise<HostLivenessInputs> {
+  const placements: HostPlacement[] = listAllModuleSystems(db).map((s) => ({
+    moduleId: s.moduleId,
+    hostname: s.hostname,
+    infraType: s.infraType,
+    vmid: s.vmid ?? null,
+  }));
+
+  // Nothing deployed — skip the probes entirely rather than SSH a fleet of none.
+  if (placements.length === 0) {
+    return { placements, machines: [], nodes: [], guestNodes: [] };
+  }
+
+  let machines: HostLivenessInputs['machines'] = [];
+  try {
+    machines = (await probeMachines()).map((m) => ({
+      hostname: m.hostname,
+      reachable: m.reachable,
+    }));
+  } catch {
+    // Leave it empty: unprobed, which reports as unverified rather than up.
+  }
+
+  const nodes: HostLivenessInputs['nodes'] = [];
+  const guestNodes: HostLivenessInputs['guestNodes'] = [];
+  try {
+    for (const service of await listContainerServices()) {
+      if (service.providerName !== 'proxmox') continue;
+      const creds = (await getServiceCredentials(service.id)) as ProxmoxCredentials;
+      const result = await new ProxmoxClient(creds).clusterResources();
+      if (!result.success) continue;
+      for (const row of result.data) {
+        if (row.type === 'node' && row.node) {
+          nodes.push({ node: row.node, online: row.status === 'online' });
+        } else if (typeof row.vmid === 'number' && row.node) {
+          guestNodes.push({ vmid: row.vmid, node: row.node });
+        }
+      }
+    }
+  } catch {
+    // Same rule: unreachable is unverified, not healthy.
+  }
+
+  return { placements, machines, nodes, guestNodes };
+}
+
 export async function runFleetChecks(
   bus: Bus,
   db: DbClient,
   opts: RunFleetChecksOptions = {},
 ): Promise<FleetFinding[]> {
+  const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
   return [
     checkSchemaDrift(db),
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
@@ -913,6 +1184,7 @@ export async function runFleetChecks(
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),
     checkPausedModules(db),
+    checkHostLiveness(await hostLiveness()),
     await checkServiceDns(db),
   ];
 }

package/src/services/machine-probe.test.ts

@@ -21,8 +21,8 @@ describe('local machine reachability', () => {
   test('a reachable local box produces no finding', async () => {
     const findings = await auditMachinesReachable({
       results: [
-        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
-        { id: 'briq', hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
+        { hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
       ],
     });
 
@@ -33,9 +33,8 @@ describe('local machine reachability', () => {
     // The skip must not blunt the check for the machines it exists to watch.
     const findings = await auditMachinesReachable({
       results: [
-        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
         {
-          id: 'briq',
           hostname: 'briq',
           ipAddress: '192.168.0.254',
           reachable: false,

package/src/services/machine-probe.ts

@@ -38,7 +38,7 @@ export async function probeMachines(): Promise<MachineReachableResult[]> {
   return Promise.all(
     machines.map(async (m): Promise<MachineReachableResult> => {
       if (m.ipAddress === LOCAL_MACHINE_IP) {
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+        return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
       }
       try {
         await execFileAsync(
@@ -57,11 +57,10 @@ export async function probeMachines(): Promise<MachineReachableResult[]> {
           ],
           { timeout: 8000 },
         );
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+        return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
       } catch (err) {
         const e = err as { stderr?: string; message?: string };
         return {
-          id: m.id,
           hostname: m.hostname,
           ipAddress: m.ipAddress,
           reachable: false,

package/src/services/module-operations.test.ts

@@ -71,6 +71,70 @@ describe('module-operations', () => {
     });
   });
 
+  /**
+   * celilo#737. Recording an outcome is BOOKKEEPING; the caller's error is the
+   * information. On celilo-mgr a `SQLITE_BUSY` inside `failOperation` propagated
+   * out of the catch block that called it and REPLACED the deploy's own error,
+   * so the operator was shown a database-locking problem and never learned what
+   * the deploy did wrong — the original was destroyed and is unrecoverable.
+   *
+   * `breakOperationsTable` stands in for any write failure. The mechanism does
+   * not matter; what matters is that no failure of the write can reach the
+   * caller.
+   */
+  describe('recording an outcome cannot destroy what it records (#737)', () => {
+    function breakOperationsTable(): void {
+      const { getDb } = require('../db/client');
+      getDb().$client.run('DROP TABLE module_operations');
+    }
+
+    it('failOperation does not replace the error it was called to record', () => {
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+
+      const original = new Error('ansible task failed on step 7');
+      let surfaced: unknown;
+
+      // Exactly the shape every call site uses (module-deploy.ts,
+      // module-remove.ts): record the failure, then rethrow the original.
+      try {
+        try {
+          throw original;
+        } catch (err) {
+          failOperation(id, err);
+          throw err;
+        }
+      } catch (err) {
+        surfaced = err;
+      }
+
+      expect(surfaced).toBe(original);
+    });
+
+    it('failOperation does not throw when the write fails', () => {
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+      expect(() => failOperation(id, new Error('original'))).not.toThrow();
+    });
+
+    it('completeOperation does not throw when the write fails', () => {
+      // The mirror bug: a deploy that SUCCEEDED reporting a database error, and
+      // skipping the `emitDeployCompleted` that follows the call.
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+      expect(() => completeOperation(id)).not.toThrow();
+    });
+
+    it('startOperation still throws — its row IS the in-flight lock', () => {
+      // Deliberately NOT swallowed. `checkInFlight` reads this row to refuse a
+      // backup during a deploy, so a silently-missing row would let the two run
+      // together. Failing before any work happens is honest; failing after it
+      // is what #737 is about.
+      breakOperationsTable();
+      expect(() => startOperation('caddy', 'deploy')).toThrow();
+    });
+  });
+
   describe('checkInFlight', () => {
     it('returns empty when no operations are in flight', () => {
       expect(checkInFlight()).toHaveLength(0);
@@ -96,7 +160,14 @@ describe('module-operations', () => {
 
     it('ignores rows whose pid is no longer alive', () => {
       // Spawn a short-lived process, capture its pid, wait for it to exit.
-      const child = spawnSync('node', ['-e', 'process.exit(0)']);
+      //
+      // `process.execPath` (the bun binary running this suite), NOT a bare
+      // `node`: this repo is bun-based and nothing guarantees a node on PATH.
+      // Where there was none, spawnSync returned `pid: undefined` and the
+      // assertion below failed with "Expected and actual values must be numbers
+      // or bigints" — which reads as a broken pid check rather than a missing
+      // interpreter.
+      const child = spawnSync(process.execPath, ['-e', 'process.exit(0)']);
       const deadPid = child.pid;
       expect(deadPid).toBeGreaterThan(0);
       expect(isPidRunnable(deadPid)).toBe(false);

package/src/services/module-operations.ts

@@ -33,6 +33,7 @@
 import { spawnSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { eq } from 'drizzle-orm';
+import { log } from '../cli/prompts';
 import { getDb } from '../db/client';
 import { type ModuleOperation, type ModuleOperationKind, moduleOperations } from '../db/schema';
 
@@ -55,21 +56,59 @@ export function startOperation(moduleId: string, operation: ModuleOperationKind)
   return id;
 }
 
+/**
+ * Write an operation's outcome without ever being able to break the flow that
+ * is reporting it (celilo#737).
+ *
+ * Recording an outcome is BOOKKEEPING; the caller's error is the information.
+ * On celilo-mgr a `SQLITE_BUSY` inside `failOperation` propagated out of the
+ * catch block that called it and REPLACED the deploy's own error, so the
+ * operator was shown a database-locking problem and never learned what the
+ * deploy actually did wrong. That error was destroyed and is unrecoverable —
+ * the whole cost of the bug. It also skipped the `emitDeployFailed` that
+ * follows the call, so the event bus never learned the deploy had failed at
+ * all, and the module was left `INSTALLED` while in fact verified.
+ *
+ * `getDb()` is inside the try on purpose: opening the database is one of the
+ * things that can throw here.
+ *
+ * ⚠️ `startOperation` deliberately does NOT get this treatment. Its row IS the
+ * in-flight lock `checkInFlight` reads to refuse a backup during a deploy, so a
+ * silently-missing row would let the two run together against the same module.
+ * Failing before any work happens is honest; failing after it is what #737 is
+ * about.
+ */
+function recordOutcome(outcome: 'completed' | 'failed', operationId: string, write: () => void) {
+  try {
+    write();
+  } catch (persistError) {
+    // Rule 6.2: never a bare catch. Secondary to whatever the caller is already
+    // reporting, so it is a warning rather than the headline — the caller's own
+    // error is what the operator needs to read.
+    const reason = persistError instanceof Error ? persistError.message : String(persistError);
+    log.warn(`Could not record operation ${operationId} as ${outcome}: ${reason}`);
+  }
+}
+
 export function completeOperation(operationId: string): void {
-  const db = getDb();
-  db.update(moduleOperations)
-    .set({ status: 'completed', completedAt: new Date() })
-    .where(eq(moduleOperations.id, operationId))
-    .run();
+  recordOutcome('completed', operationId, () => {
+    getDb()
+      .update(moduleOperations)
+      .set({ status: 'completed', completedAt: new Date() })
+      .where(eq(moduleOperations.id, operationId))
+      .run();
+  });
 }
 
 export function failOperation(operationId: string, error: unknown): void {
-  const db = getDb();
   const message = error instanceof Error ? error.message : String(error);
-  db.update(moduleOperations)
-    .set({ status: 'failed', completedAt: new Date(), errorMessage: message })
-    .where(eq(moduleOperations.id, operationId))
-    .run();
+  recordOutcome('failed', operationId, () => {
+    getDb()
+      .update(moduleOperations)
+      .set({ status: 'failed', completedAt: new Date(), errorMessage: message })
+      .where(eq(moduleOperations.id, operationId))
+      .run();
+  });
 }
 
 /**