@celilo/cli 0.25.1 → 0.26.1

package/src/services/fleet-checks.ts

@@ -19,6 +19,7 @@
 
 import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
+import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
 import { getMigrationStatus } from '../db/migration-status';
@@ -27,9 +28,8 @@ import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
 
-/** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
-import { getModuleSystems } from './deployed-systems';
+import { getServiceCredentials, listContainerServices } from './container-service';
+import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -38,9 +38,13 @@ import {
   readInstalledUnit,
   unitMainPid,
 } from './events-daemon';
+import { probeMachines } from './machine-probe';
 import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
+/** The module that IS celilo's control plane. */
+const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other
  * than `internal` is firewall-segmented — an unmanaged LAN device has no
@@ -786,6 +790,12 @@ export async function checkServiceDns(db: DbClient): Promise<FleetFinding> {
 export interface RunFleetChecksOptions {
   now?: number;
   installedCodeMtimeMs?: number | null;
+  /**
+   * Where the host-liveness verdict gets its facts. Injected so the check can
+   * be exercised without SSH or a Proxmox credential; defaults to the live
+   * fleet (`collectHostLiveness`).
+   */
+  hostLiveness?: () => Promise<HostLivenessInputs>;
 }
 
 /**
@@ -901,11 +911,272 @@ export function checkPausedModules(db: DbClient): FleetFinding {
   };
 }
 
+/** One module deployment, and the host it landed on. */
+export interface HostPlacement {
+  moduleId: string;
+  /** The host's user-facing name — a pool hostname, or the container's. */
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  /** Proxmox VMID for a celilo-provisioned container; null for a pool machine. */
+  vmid: number | null;
+}
+
+/**
+ * Everything the liveness verdict is computed from, injected so the check is a
+ * pure function over data and needs neither SSH nor a Proxmox credential to
+ * test.
+ *
+ * ⚠️ Every source here is ALLOWED TO BE ABSENT, and absent is not "fine".
+ * A machine missing from `machines` was not probed; a node missing from `nodes`
+ * was not reported. Neither means the host is up, and neither means it is down.
+ * Conflating "I could not look" with "I looked and it was healthy" is the
+ * failure this whole check exists to end — doctor said OK-with-warnings while a
+ * node hosting two modules was offline.
+ */
+export interface HostLivenessInputs {
+  placements: HostPlacement[];
+  /** Machine-pool SSH probe results. A hostname absent here was NOT probed. */
+  machines: Array<{ hostname: string; reachable: boolean }>;
+  /** Proxmox node status. Empty when no container service is configured. */
+  nodes: Array<{ node: string; online: boolean }>;
+  /** VMID → node name, from `/cluster/resources`. Empty when unqueried. */
+  guestNodes: Array<{ vmid: number; node: string }>;
+}
+
+type HostState = 'up' | 'down' | 'unknown';
+
+interface HostVerdict {
+  host: string;
+  state: HostState;
+  /** Why the state could not be determined. Set only when state is 'unknown'. */
+  reason?: string;
+}
+
+function resolveHostState(placement: HostPlacement, inputs: HostLivenessInputs): HostVerdict {
+  if (placement.infraType === 'machine') {
+    const probe = inputs.machines.find((m) => m.hostname === placement.hostname);
+    if (!probe) {
+      return {
+        host: placement.hostname,
+        state: 'unknown',
+        // Either the SSH probe did not run at all, or this hostname is no
+        // longer in the machine pool — a stale `module_systems` row, which is
+        // its own defect and worth surfacing rather than rounding off.
+        reason: 'no probe result for this machine',
+      };
+    }
+    return { host: placement.hostname, state: probe.reachable ? 'up' : 'down' };
+  }
+
+  // A container's liveness is its NODE's liveness. The guest being stopped is a
+  // different condition with a different owner (`module pause --stop-infra`
+  // stops guests deliberately), so this deliberately reads the node only.
+  if (placement.vmid === null) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: 'no VMID recorded — celilo has no liveness source for this provider',
+    };
+  }
+  const guest = inputs.guestNodes.find((g) => g.vmid === placement.vmid);
+  if (!guest) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: `VMID ${placement.vmid} not present in the cluster's resources`,
+    };
+  }
+  const node = inputs.nodes.find((n) => n.node === guest.node);
+  if (!node) {
+    return { host: guest.node, state: 'unknown', reason: 'the cluster reported no such node' };
+  }
+  return { host: guest.node, state: node.online ? 'up' : 'down' };
+}
+
+/**
+ * Are the hosts this fleet's modules actually run on alive? (celilo#728)
+ *
+ * Every other `Fleet runtime` check is a control-plane concern — the
+ * dispatcher, bus subscribers, capability-derived config, internal DNS. None of
+ * them asked the most basic data-plane question, so `system doctor` reported
+ * "OK with warnings" while a Proxmox node was OFFLINE with `celilo-apt-repo`
+ * and `lunacycle` on it. The information was already in `proxmox node list`;
+ * doctor simply never consulted it. It surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there — had nothing tried
+ * to publish, the node could have stayed down indefinitely.
+ *
+ * A down host is a FAILURE, not a warning: it is strictly worse than the
+ * conditions already reported as failures here, and every module on it is down
+ * with it.
+ *
+ * A host celilo TRIED to verify and could not is a WARNING, and the reason is
+ * named per host. This is not the same as "quiet because it might be fine": a
+ * cluster that will not answer its own API is not obviously healthier than one
+ * reporting a node offline, and the failure to answer may BE the outage this
+ * check exists to catch. Reporting it as ok-with-a-note would rebuild the
+ * defect one level down — a report reading healthy over something unmeasured.
+ *
+ * ⚠️ The thing that makes the warning safe to have is that it is not
+ * permanent. A machine-only fleet produces NO unverified hosts at all: every
+ * placement takes the probe path and resolves. The one standing source would be
+ * a provider celilo cannot interrogate — today a DigitalOcean droplet, whose
+ * client can verify the token but never reads droplet status. That is a gap to
+ * close (its own change), not a reason to soften the signal here. A warning
+ * that fires forever is what trains an operator to skim the whole report
+ * (celilo#723, whose false positive was competing for attention in the very
+ * output that missed the offline node) — so if this one ever becomes standing,
+ * the fix is to teach celilo the missing provider, not to quieten it.
+ */
+export function checkHostLiveness(inputs: HostLivenessInputs): FleetFinding {
+  const base = {
+    id: 'host-liveness',
+    title: 'The hosts running deployed modules are alive',
+    autoFixable: false,
+  } as const;
+
+  if (inputs.placements.length === 0) {
+    return {
+      ...base,
+      status: 'ok',
+      summary: 'no modules are deployed to a host yet',
+      detail: [],
+      remediation: null,
+    };
+  }
+
+  interface HostEntry {
+    state: HostState;
+    modules: Set<string>;
+    reason?: string;
+  }
+
+  const modulesByHost = new Map<string, HostEntry>();
+  for (const placement of inputs.placements) {
+    const { host, state, reason } = resolveHostState(placement, inputs);
+    const entry = modulesByHost.get(host) ?? { state, modules: new Set<string>(), reason };
+    // A host resolved 'down' by any placement stays down — one authoritative
+    // negative outranks an unknown from a sibling placement.
+    if (state === 'down' || entry.state === 'unknown') {
+      entry.state = state;
+      entry.reason = reason;
+    }
+    entry.modules.add(placement.moduleId);
+    modulesByHost.set(host, entry);
+  }
+
+  const describe = (host: string, e: HostEntry) => `${host}: ${[...e.modules].sort().join(', ')}`;
+  const describeUnverified = (host: string, e: HostEntry) =>
+    `unverified — ${describe(host, e)}${e.reason ? ` (${e.reason})` : ''}`;
+
+  const down = [...modulesByHost].filter(([, e]) => e.state === 'down');
+  const unknown = [...modulesByHost].filter(([, e]) => e.state === 'unknown');
+  const up = [...modulesByHost].filter(([, e]) => e.state === 'up');
+
+  if (down.length > 0) {
+    const affected = down.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'fail',
+      summary: `${down.length} host(s) down, ${affected} module(s) unreachable: ${down
+        .map(([host]) => host)
+        .join(', ')}`,
+      detail: [
+        ...down.map(([host, e]) => `DOWN ${describe(host, e)}`),
+        ...unknown.map(([host, e]) => describeUnverified(host, e)),
+        'every module listed against a down host is down with it, whatever its own status says',
+      ],
+      remediation:
+        'bring the host back, then confirm with "celilo proxmox node list" (container services) or "celilo machine status <hostname>" (pool machines)',
+    };
+  }
+
+  if (unknown.length > 0) {
+    // WARN, not ok-with-a-note. celilo tried and could not find out, and the
+    // reason it could not may be the outage itself — a cluster that will not
+    // answer its own API is not evidence of health. Reporting this quietly
+    // would rebuild #728 one level down.
+    const affected = unknown.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'warn',
+      summary: `${up.length} host(s) up, ${unknown.length} could not be verified (${affected} module(s))`,
+      // Named and reasoned, never counted: "1 not verified" tells an operator
+      // neither which host nor what to do about it.
+      detail: unknown.map(([host, e]) => describeUnverified(host, e)),
+      remediation:
+        'check the host directly — "celilo proxmox node list" for a container service, "celilo machine status <hostname>" for a pool machine; a host celilo cannot reach is not a host known to be healthy',
+    };
+  }
+
+  return {
+    ...base,
+    status: 'ok',
+    summary: `${up.length} host(s) up`,
+    detail: [],
+    remediation: null,
+  };
+}
+
+/**
+ * Read the liveness facts off the live fleet.
+ *
+ * Every source degrades to ABSENT rather than to a cheerful default. A Proxmox
+ * cluster that cannot be reached, or a fleet with no container service at all,
+ * contributes no node rows — and `checkHostLiveness` reads that as unverified,
+ * never as healthy. That distinction is the whole point of the check.
+ */
+export async function collectHostLiveness(db: DbClient): Promise<HostLivenessInputs> {
+  const placements: HostPlacement[] = listAllModuleSystems(db).map((s) => ({
+    moduleId: s.moduleId,
+    hostname: s.hostname,
+    infraType: s.infraType,
+    vmid: s.vmid ?? null,
+  }));
+
+  // Nothing deployed — skip the probes entirely rather than SSH a fleet of none.
+  if (placements.length === 0) {
+    return { placements, machines: [], nodes: [], guestNodes: [] };
+  }
+
+  let machines: HostLivenessInputs['machines'] = [];
+  try {
+    machines = (await probeMachines()).map((m) => ({
+      hostname: m.hostname,
+      reachable: m.reachable,
+    }));
+  } catch {
+    // Leave it empty: unprobed, which reports as unverified rather than up.
+  }
+
+  const nodes: HostLivenessInputs['nodes'] = [];
+  const guestNodes: HostLivenessInputs['guestNodes'] = [];
+  try {
+    for (const service of await listContainerServices()) {
+      if (service.providerName !== 'proxmox') continue;
+      const creds = (await getServiceCredentials(service.id)) as ProxmoxCredentials;
+      const result = await new ProxmoxClient(creds).clusterResources();
+      if (!result.success) continue;
+      for (const row of result.data) {
+        if (row.type === 'node' && row.node) {
+          nodes.push({ node: row.node, online: row.status === 'online' });
+        } else if (typeof row.vmid === 'number' && row.node) {
+          guestNodes.push({ vmid: row.vmid, node: row.node });
+        }
+      }
+    }
+  } catch {
+    // Same rule: unreachable is unverified, not healthy.
+  }
+
+  return { placements, machines, nodes, guestNodes };
+}
+
 export async function runFleetChecks(
   bus: Bus,
   db: DbClient,
   opts: RunFleetChecksOptions = {},
 ): Promise<FleetFinding[]> {
+  const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
   return [
     checkSchemaDrift(db),
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
@@ -913,6 +1184,7 @@ export async function runFleetChecks(
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),
     checkPausedModules(db),
+    checkHostLiveness(await hostLiveness()),
     await checkServiceDns(db),
   ];
 }

package/src/services/machine-probe.test.ts

@@ -21,8 +21,8 @@ describe('local machine reachability', () => {
   test('a reachable local box produces no finding', async () => {
     const findings = await auditMachinesReachable({
       results: [
-        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
-        { id: 'briq', hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
+        { hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
       ],
     });
 
@@ -33,9 +33,8 @@ describe('local machine reachability', () => {
     // The skip must not blunt the check for the machines it exists to watch.
     const findings = await auditMachinesReachable({
       results: [
-        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
         {
-          id: 'briq',
           hostname: 'briq',
           ipAddress: '192.168.0.254',
           reachable: false,

package/src/services/machine-probe.ts

@@ -38,7 +38,7 @@ export async function probeMachines(): Promise<MachineReachableResult[]> {
   return Promise.all(
     machines.map(async (m): Promise<MachineReachableResult> => {
       if (m.ipAddress === LOCAL_MACHINE_IP) {
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+        return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
       }
       try {
         await execFileAsync(
@@ -57,11 +57,10 @@ export async function probeMachines(): Promise<MachineReachableResult[]> {
           ],
           { timeout: 8000 },
         );
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+        return { hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
       } catch (err) {
         const e = err as { stderr?: string; message?: string };
         return {
-          id: m.id,
           hostname: m.hostname,
           ipAddress: m.ipAddress,
           reachable: false,

package/src/services/module-operations.test.ts

@@ -71,6 +71,70 @@ describe('module-operations', () => {
     });
   });
 
+  /**
+   * celilo#737. Recording an outcome is BOOKKEEPING; the caller's error is the
+   * information. On celilo-mgr a `SQLITE_BUSY` inside `failOperation` propagated
+   * out of the catch block that called it and REPLACED the deploy's own error,
+   * so the operator was shown a database-locking problem and never learned what
+   * the deploy did wrong — the original was destroyed and is unrecoverable.
+   *
+   * `breakOperationsTable` stands in for any write failure. The mechanism does
+   * not matter; what matters is that no failure of the write can reach the
+   * caller.
+   */
+  describe('recording an outcome cannot destroy what it records (#737)', () => {
+    function breakOperationsTable(): void {
+      const { getDb } = require('../db/client');
+      getDb().$client.run('DROP TABLE module_operations');
+    }
+
+    it('failOperation does not replace the error it was called to record', () => {
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+
+      const original = new Error('ansible task failed on step 7');
+      let surfaced: unknown;
+
+      // Exactly the shape every call site uses (module-deploy.ts,
+      // module-remove.ts): record the failure, then rethrow the original.
+      try {
+        try {
+          throw original;
+        } catch (err) {
+          failOperation(id, err);
+          throw err;
+        }
+      } catch (err) {
+        surfaced = err;
+      }
+
+      expect(surfaced).toBe(original);
+    });
+
+    it('failOperation does not throw when the write fails', () => {
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+      expect(() => failOperation(id, new Error('original'))).not.toThrow();
+    });
+
+    it('completeOperation does not throw when the write fails', () => {
+      // The mirror bug: a deploy that SUCCEEDED reporting a database error, and
+      // skipping the `emitDeployCompleted` that follows the call.
+      const id = startOperation('caddy', 'deploy');
+      breakOperationsTable();
+      expect(() => completeOperation(id)).not.toThrow();
+    });
+
+    it('startOperation still throws — its row IS the in-flight lock', () => {
+      // Deliberately NOT swallowed. `checkInFlight` reads this row to refuse a
+      // backup during a deploy, so a silently-missing row would let the two run
+      // together. Failing before any work happens is honest; failing after it
+      // is what #737 is about.
+      breakOperationsTable();
+      expect(() => startOperation('caddy', 'deploy')).toThrow();
+    });
+  });
+
   describe('checkInFlight', () => {
     it('returns empty when no operations are in flight', () => {
       expect(checkInFlight()).toHaveLength(0);
@@ -96,7 +160,14 @@ describe('module-operations', () => {
 
     it('ignores rows whose pid is no longer alive', () => {
       // Spawn a short-lived process, capture its pid, wait for it to exit.
-      const child = spawnSync('node', ['-e', 'process.exit(0)']);
+      //
+      // `process.execPath` (the bun binary running this suite), NOT a bare
+      // `node`: this repo is bun-based and nothing guarantees a node on PATH.
+      // Where there was none, spawnSync returned `pid: undefined` and the
+      // assertion below failed with "Expected and actual values must be numbers
+      // or bigints" — which reads as a broken pid check rather than a missing
+      // interpreter.
+      const child = spawnSync(process.execPath, ['-e', 'process.exit(0)']);
       const deadPid = child.pid;
       expect(deadPid).toBeGreaterThan(0);
       expect(isPidRunnable(deadPid)).toBe(false);

package/src/services/module-operations.ts

@@ -33,6 +33,7 @@
 import { spawnSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { eq } from 'drizzle-orm';
+import { log } from '../cli/prompts';
 import { getDb } from '../db/client';
 import { type ModuleOperation, type ModuleOperationKind, moduleOperations } from '../db/schema';
 
@@ -55,21 +56,59 @@ export function startOperation(moduleId: string, operation: ModuleOperationKind)
   return id;
 }
 
+/**
+ * Write an operation's outcome without ever being able to break the flow that
+ * is reporting it (celilo#737).
+ *
+ * Recording an outcome is BOOKKEEPING; the caller's error is the information.
+ * On celilo-mgr a `SQLITE_BUSY` inside `failOperation` propagated out of the
+ * catch block that called it and REPLACED the deploy's own error, so the
+ * operator was shown a database-locking problem and never learned what the
+ * deploy actually did wrong. That error was destroyed and is unrecoverable —
+ * the whole cost of the bug. It also skipped the `emitDeployFailed` that
+ * follows the call, so the event bus never learned the deploy had failed at
+ * all, and the module was left `INSTALLED` while in fact verified.
+ *
+ * `getDb()` is inside the try on purpose: opening the database is one of the
+ * things that can throw here.
+ *
+ * ⚠️ `startOperation` deliberately does NOT get this treatment. Its row IS the
+ * in-flight lock `checkInFlight` reads to refuse a backup during a deploy, so a
+ * silently-missing row would let the two run together against the same module.
+ * Failing before any work happens is honest; failing after it is what #737 is
+ * about.
+ */
+function recordOutcome(outcome: 'completed' | 'failed', operationId: string, write: () => void) {
+  try {
+    write();
+  } catch (persistError) {
+    // Rule 6.2: never a bare catch. Secondary to whatever the caller is already
+    // reporting, so it is a warning rather than the headline — the caller's own
+    // error is what the operator needs to read.
+    const reason = persistError instanceof Error ? persistError.message : String(persistError);
+    log.warn(`Could not record operation ${operationId} as ${outcome}: ${reason}`);
+  }
+}
+
 export function completeOperation(operationId: string): void {
-  const db = getDb();
-  db.update(moduleOperations)
-    .set({ status: 'completed', completedAt: new Date() })
-    .where(eq(moduleOperations.id, operationId))
-    .run();
+  recordOutcome('completed', operationId, () => {
+    getDb()
+      .update(moduleOperations)
+      .set({ status: 'completed', completedAt: new Date() })
+      .where(eq(moduleOperations.id, operationId))
+      .run();
+  });
 }
 
 export function failOperation(operationId: string, error: unknown): void {
-  const db = getDb();
   const message = error instanceof Error ? error.message : String(error);
-  db.update(moduleOperations)
-    .set({ status: 'failed', completedAt: new Date(), errorMessage: message })
-    .where(eq(moduleOperations.id, operationId))
-    .run();
+  recordOutcome('failed', operationId, () => {
+    getDb()
+      .update(moduleOperations)
+      .set({ status: 'failed', completedAt: new Date(), errorMessage: message })
+      .where(eq(moduleOperations.id, operationId))
+      .run();
+  });
 }
 
 /**