@celilo/cli 0.26.0 → 0.27.0

package/src/services/fleet-checks.ts

@@ -19,6 +19,7 @@
 
 import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
+import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
 import { getMigrationStatus } from '../db/migration-status';
@@ -27,9 +28,8 @@ import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
 
-/** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
-import { getModuleSystems } from './deployed-systems';
+import { getServiceCredentials, listContainerServices } from './container-service';
+import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -38,9 +38,13 @@ import {
   readInstalledUnit,
   unitMainPid,
 } from './events-daemon';
+import { probeMachines } from './machine-probe';
 import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
+/** The module that IS celilo's control plane. */
+const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other
  * than `internal` is firewall-segmented — an unmanaged LAN device has no
@@ -786,6 +790,12 @@ export async function checkServiceDns(db: DbClient): Promise<FleetFinding> {
 export interface RunFleetChecksOptions {
   now?: number;
   installedCodeMtimeMs?: number | null;
+  /**
+   * Where the host-liveness verdict gets its facts. Injected so the check can
+   * be exercised without SSH or a Proxmox credential; defaults to the live
+   * fleet (`collectHostLiveness`).
+   */
+  hostLiveness?: () => Promise<HostLivenessInputs>;
 }
 
 /**
@@ -901,11 +911,272 @@ export function checkPausedModules(db: DbClient): FleetFinding {
   };
 }
 
+/** One module deployment, and the host it landed on. */
+export interface HostPlacement {
+  moduleId: string;
+  /** The host's user-facing name — a pool hostname, or the container's. */
+  hostname: string;
+  infraType: 'machine' | 'container_service';
+  /** Proxmox VMID for a celilo-provisioned container; null for a pool machine. */
+  vmid: number | null;
+}
+
+/**
+ * Everything the liveness verdict is computed from, injected so the check is a
+ * pure function over data and needs neither SSH nor a Proxmox credential to
+ * test.
+ *
+ * ⚠️ Every source here is ALLOWED TO BE ABSENT, and absent is not "fine".
+ * A machine missing from `machines` was not probed; a node missing from `nodes`
+ * was not reported. Neither means the host is up, and neither means it is down.
+ * Conflating "I could not look" with "I looked and it was healthy" is the
+ * failure this whole check exists to end — doctor said OK-with-warnings while a
+ * node hosting two modules was offline.
+ */
+export interface HostLivenessInputs {
+  placements: HostPlacement[];
+  /** Machine-pool SSH probe results. A hostname absent here was NOT probed. */
+  machines: Array<{ hostname: string; reachable: boolean }>;
+  /** Proxmox node status. Empty when no container service is configured. */
+  nodes: Array<{ node: string; online: boolean }>;
+  /** VMID → node name, from `/cluster/resources`. Empty when unqueried. */
+  guestNodes: Array<{ vmid: number; node: string }>;
+}
+
+type HostState = 'up' | 'down' | 'unknown';
+
+interface HostVerdict {
+  host: string;
+  state: HostState;
+  /** Why the state could not be determined. Set only when state is 'unknown'. */
+  reason?: string;
+}
+
+function resolveHostState(placement: HostPlacement, inputs: HostLivenessInputs): HostVerdict {
+  if (placement.infraType === 'machine') {
+    const probe = inputs.machines.find((m) => m.hostname === placement.hostname);
+    if (!probe) {
+      return {
+        host: placement.hostname,
+        state: 'unknown',
+        // Either the SSH probe did not run at all, or this hostname is no
+        // longer in the machine pool — a stale `module_systems` row, which is
+        // its own defect and worth surfacing rather than rounding off.
+        reason: 'no probe result for this machine',
+      };
+    }
+    return { host: placement.hostname, state: probe.reachable ? 'up' : 'down' };
+  }
+
+  // A container's liveness is its NODE's liveness. The guest being stopped is a
+  // different condition with a different owner (`module pause --stop-infra`
+  // stops guests deliberately), so this deliberately reads the node only.
+  if (placement.vmid === null) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: 'no VMID recorded — celilo has no liveness source for this provider',
+    };
+  }
+  const guest = inputs.guestNodes.find((g) => g.vmid === placement.vmid);
+  if (!guest) {
+    return {
+      host: placement.hostname,
+      state: 'unknown',
+      reason: `VMID ${placement.vmid} not present in the cluster's resources`,
+    };
+  }
+  const node = inputs.nodes.find((n) => n.node === guest.node);
+  if (!node) {
+    return { host: guest.node, state: 'unknown', reason: 'the cluster reported no such node' };
+  }
+  return { host: guest.node, state: node.online ? 'up' : 'down' };
+}
+
+/**
+ * Are the hosts this fleet's modules actually run on alive? (celilo#728)
+ *
+ * Every other `Fleet runtime` check is a control-plane concern — the
+ * dispatcher, bus subscribers, capability-derived config, internal DNS. None of
+ * them asked the most basic data-plane question, so `system doctor` reported
+ * "OK with warnings" while a Proxmox node was OFFLINE with `celilo-apt-repo`
+ * and `lunacycle` on it. The information was already in `proxmox node list`;
+ * doctor simply never consulted it. It surfaced only because a release run got
+ * an HTTP 502 from the apt repo that happened to live there — had nothing tried
+ * to publish, the node could have stayed down indefinitely.
+ *
+ * A down host is a FAILURE, not a warning: it is strictly worse than the
+ * conditions already reported as failures here, and every module on it is down
+ * with it.
+ *
+ * A host celilo TRIED to verify and could not is a WARNING, and the reason is
+ * named per host. This is not the same as "quiet because it might be fine": a
+ * cluster that will not answer its own API is not obviously healthier than one
+ * reporting a node offline, and the failure to answer may BE the outage this
+ * check exists to catch. Reporting it as ok-with-a-note would rebuild the
+ * defect one level down — a report reading healthy over something unmeasured.
+ *
+ * ⚠️ The thing that makes the warning safe to have is that it is not
+ * permanent. A machine-only fleet produces NO unverified hosts at all: every
+ * placement takes the probe path and resolves. The one standing source would be
+ * a provider celilo cannot interrogate — today a DigitalOcean droplet, whose
+ * client can verify the token but never reads droplet status. That is a gap to
+ * close (its own change), not a reason to soften the signal here. A warning
+ * that fires forever is what trains an operator to skim the whole report
+ * (celilo#723, whose false positive was competing for attention in the very
+ * output that missed the offline node) — so if this one ever becomes standing,
+ * the fix is to teach celilo the missing provider, not to quieten it.
+ */
+export function checkHostLiveness(inputs: HostLivenessInputs): FleetFinding {
+  const base = {
+    id: 'host-liveness',
+    title: 'The hosts running deployed modules are alive',
+    autoFixable: false,
+  } as const;
+
+  if (inputs.placements.length === 0) {
+    return {
+      ...base,
+      status: 'ok',
+      summary: 'no modules are deployed to a host yet',
+      detail: [],
+      remediation: null,
+    };
+  }
+
+  interface HostEntry {
+    state: HostState;
+    modules: Set<string>;
+    reason?: string;
+  }
+
+  const modulesByHost = new Map<string, HostEntry>();
+  for (const placement of inputs.placements) {
+    const { host, state, reason } = resolveHostState(placement, inputs);
+    const entry = modulesByHost.get(host) ?? { state, modules: new Set<string>(), reason };
+    // A host resolved 'down' by any placement stays down — one authoritative
+    // negative outranks an unknown from a sibling placement.
+    if (state === 'down' || entry.state === 'unknown') {
+      entry.state = state;
+      entry.reason = reason;
+    }
+    entry.modules.add(placement.moduleId);
+    modulesByHost.set(host, entry);
+  }
+
+  const describe = (host: string, e: HostEntry) => `${host}: ${[...e.modules].sort().join(', ')}`;
+  const describeUnverified = (host: string, e: HostEntry) =>
+    `unverified — ${describe(host, e)}${e.reason ? ` (${e.reason})` : ''}`;
+
+  const down = [...modulesByHost].filter(([, e]) => e.state === 'down');
+  const unknown = [...modulesByHost].filter(([, e]) => e.state === 'unknown');
+  const up = [...modulesByHost].filter(([, e]) => e.state === 'up');
+
+  if (down.length > 0) {
+    const affected = down.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'fail',
+      summary: `${down.length} host(s) down, ${affected} module(s) unreachable: ${down
+        .map(([host]) => host)
+        .join(', ')}`,
+      detail: [
+        ...down.map(([host, e]) => `DOWN ${describe(host, e)}`),
+        ...unknown.map(([host, e]) => describeUnverified(host, e)),
+        'every module listed against a down host is down with it, whatever its own status says',
+      ],
+      remediation:
+        'bring the host back, then confirm with "celilo proxmox node list" (container services) or "celilo machine status <hostname>" (pool machines)',
+    };
+  }
+
+  if (unknown.length > 0) {
+    // WARN, not ok-with-a-note. celilo tried and could not find out, and the
+    // reason it could not may be the outage itself — a cluster that will not
+    // answer its own API is not evidence of health. Reporting this quietly
+    // would rebuild #728 one level down.
+    const affected = unknown.reduce((n, [, e]) => n + e.modules.size, 0);
+    return {
+      ...base,
+      status: 'warn',
+      summary: `${up.length} host(s) up, ${unknown.length} could not be verified (${affected} module(s))`,
+      // Named and reasoned, never counted: "1 not verified" tells an operator
+      // neither which host nor what to do about it.
+      detail: unknown.map(([host, e]) => describeUnverified(host, e)),
+      remediation:
+        'check the host directly — "celilo proxmox node list" for a container service, "celilo machine status <hostname>" for a pool machine; a host celilo cannot reach is not a host known to be healthy',
+    };
+  }
+
+  return {
+    ...base,
+    status: 'ok',
+    summary: `${up.length} host(s) up`,
+    detail: [],
+    remediation: null,
+  };
+}
+
+/**
+ * Read the liveness facts off the live fleet.
+ *
+ * Every source degrades to ABSENT rather than to a cheerful default. A Proxmox
+ * cluster that cannot be reached, or a fleet with no container service at all,
+ * contributes no node rows — and `checkHostLiveness` reads that as unverified,
+ * never as healthy. That distinction is the whole point of the check.
+ */
+export async function collectHostLiveness(db: DbClient): Promise<HostLivenessInputs> {
+  const placements: HostPlacement[] = listAllModuleSystems(db).map((s) => ({
+    moduleId: s.moduleId,
+    hostname: s.hostname,
+    infraType: s.infraType,
+    vmid: s.vmid ?? null,
+  }));
+
+  // Nothing deployed — skip the probes entirely rather than SSH a fleet of none.
+  if (placements.length === 0) {
+    return { placements, machines: [], nodes: [], guestNodes: [] };
+  }
+
+  let machines: HostLivenessInputs['machines'] = [];
+  try {
+    machines = (await probeMachines()).map((m) => ({
+      hostname: m.hostname,
+      reachable: m.reachable,
+    }));
+  } catch {
+    // Leave it empty: unprobed, which reports as unverified rather than up.
+  }
+
+  const nodes: HostLivenessInputs['nodes'] = [];
+  const guestNodes: HostLivenessInputs['guestNodes'] = [];
+  try {
+    for (const service of await listContainerServices()) {
+      if (service.providerName !== 'proxmox') continue;
+      const creds = (await getServiceCredentials(service.id)) as ProxmoxCredentials;
+      const result = await new ProxmoxClient(creds).clusterResources();
+      if (!result.success) continue;
+      for (const row of result.data) {
+        if (row.type === 'node' && row.node) {
+          nodes.push({ node: row.node, online: row.status === 'online' });
+        } else if (typeof row.vmid === 'number' && row.node) {
+          guestNodes.push({ vmid: row.vmid, node: row.node });
+        }
+      }
+    }
+  } catch {
+    // Same rule: unreachable is unverified, not healthy.
+  }
+
+  return { placements, machines, nodes, guestNodes };
+}
+
 export async function runFleetChecks(
   bus: Bus,
   db: DbClient,
   opts: RunFleetChecksOptions = {},
 ): Promise<FleetFinding[]> {
+  const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
   return [
     checkSchemaDrift(db),
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
@@ -913,6 +1184,7 @@ export async function runFleetChecks(
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),
     checkPausedModules(db),
+    checkHostLiveness(await hostLiveness()),
     await checkServiceDns(db),
   ];
 }

package/src/services/infrastructure-selector.test.ts

@@ -100,7 +100,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 4, memory_mb: 4096, disk_gb: 128 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       const result = await selectInfrastructure(module);
@@ -143,7 +142,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 4, memory_mb: 4096, disk_gb: 128 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       const result = await selectInfrastructure(module);
@@ -308,7 +306,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 1, memory_mb: 4096, disk_gb: 128 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       // Should throw InfrastructureError with resource details
@@ -350,7 +347,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 4, memory_mb: 1024, disk_gb: 128 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       await expect(selectInfrastructure(module)).rejects.toThrow(InfrastructureError);
@@ -390,7 +386,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 4, memory_mb: 4096, disk_gb: 10 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       await expect(selectInfrastructure(module)).rejects.toThrow(InfrastructureError);
@@ -430,7 +425,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 8, memory_mb: 16384, disk_gb: 256 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       const result = await selectInfrastructure(module);
@@ -473,7 +467,6 @@ describe('infrastructure-selector', () => {
         hardware: { cpu_cores: 4, memory_mb: 4096, disk_gb: 128 },
         role: 'host',
         interfaces: [],
-        assignedModuleIds: [],
       });
 
       const result = await selectInfrastructure(module);

package/src/services/infrastructure-selector.ts

@@ -7,7 +7,7 @@ import type {
   ResourceRequirements,
 } from '../types/infrastructure';
 import { listContainerServices } from './container-service';
-import { getModuleResourcesOnMachine, listMachines } from './machine-pool';
+import { getModulesOnMachine, listMachines } from './machine-pool';
 
 /**
  * Infrastructure selection error
@@ -49,25 +49,24 @@ function getResourceRequirements(module: Module): ResourceRequirements {
 }
 
 /**
- * Check if a machine has capacity for the given requirements
+ * Is this machine big enough for the module at all?
+ *
+ * Deliberately NOT multi-tenant capacity accounting (celilo#773). It used to
+ * subtract an "already allocated" figure that was hard-coded to zero behind a
+ * TODO, so the subtraction never changed an answer and the check has always
+ * been exactly this comparison. Saying so is the honest version; a gate that
+ * cannot fail reads as protection that is not there (Rule 7.6).
+ *
+ * Real accounting needs a decision nobody has made — whether a module's draw is
+ * its manifest MINIMUM (`requires.system`) or its deployed size, which on a
+ * pool machine celilo does not own. Until then the occupancy filter, which IS
+ * now correct, is what keeps two modules off one box.
  */
-async function machineHasCapacity(
-  machine: Machine,
-  requirements: ResourceRequirements,
-): Promise<boolean> {
-  // Get already-allocated resources
-  const allocated = await getModuleResourcesOnMachine(machine.id);
-
-  // Calculate remaining capacity
-  const remainingCpu = machine.hardware.cpu_cores - allocated.cpu;
-  const remainingMemory = machine.hardware.memory_mb - allocated.memory;
-  const remainingDisk = machine.hardware.disk_gb - allocated.disk;
-
-  // Check if machine has enough remaining resources
+function machineHasCapacity(machine: Machine, requirements: ResourceRequirements): boolean {
   return (
-    remainingCpu >= requirements.cpu &&
-    remainingMemory >= requirements.memory &&
-    remainingDisk >= requirements.disk
+    machine.hardware.cpu_cores >= requirements.cpu &&
+    machine.hardware.memory_mb >= requirements.memory &&
+    machine.hardware.disk_gb >= requirements.disk
   );
 }
 
@@ -181,11 +180,12 @@ export async function selectInfrastructure(module: Module): Promise<Infrastructu
       continue;
     }
     // Skip machines already assigned to other modules
-    if (machine.assignedModuleIds.length > 0 && !machine.assignedModuleIds.includes(moduleId)) {
+    const occupants = getModulesOnMachine(machine.id);
+    if (occupants.length > 0 && !occupants.includes(moduleId)) {
       rejections.push({
         hostname: machine.hostname,
         ipAddress: machine.ipAddress,
-        reason: `already assigned to module(s): ${machine.assignedModuleIds.join(', ')}`,
+        reason: `already assigned to module(s): ${occupants.join(', ')}`,
       });
       continue;
     }
@@ -208,14 +208,13 @@ export async function selectInfrastructure(module: Module): Promise<Infrastructu
       });
       continue;
     }
-    if (await machineHasCapacity(machine, requirements)) {
+    if (machineHasCapacity(machine, requirements)) {
       availableMachines.push(machine);
     } else {
-      const allocated = await getModuleResourcesOnMachine(machine.id);
       const shortfalls: string[] = [];
-      const remainingCpu = machine.hardware.cpu_cores - allocated.cpu;
-      const remainingMemory = machine.hardware.memory_mb - allocated.memory;
-      const remainingDisk = machine.hardware.disk_gb - allocated.disk;
+      const remainingCpu = machine.hardware.cpu_cores;
+      const remainingMemory = machine.hardware.memory_mb;
+      const remainingDisk = machine.hardware.disk_gb;
 
       if (remainingCpu < requirements.cpu) {
         shortfalls.push(`CPU: ${remainingCpu} available, ${requirements.cpu} required`);

package/src/services/infrastructure-variable-resolver.test.ts

@@ -96,7 +96,6 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
       sshKeyEncrypted: 'encrypted-key',
       hardware: { cpu_cores: 2, memory_mb: 2048, disk_gb: 20 },
       zone: 'external',
-      assignedModuleIds: [],
     });
 
     // Create module infrastructure selection
@@ -169,7 +168,6 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
       sshKeyEncrypted: 'encrypted',
       hardware: { cpu_cores: 1, memory_mb: 1024, disk_gb: 10 },
       zone: 'internal',
-      assignedModuleIds: [],
     });
 
     await db.insert(moduleInfrastructure).values({
@@ -468,7 +466,6 @@ describe('resolveInfrastructureVariables - User Override', () => {
       sshKeyEncrypted: 'encrypted',
       hardware: { cpu_cores: 2, memory_mb: 2048, disk_gb: 20 },
       zone: 'external',
-      assignedModuleIds: [],
     });
 
     await db.insert(moduleInfrastructure).values({
@@ -526,7 +523,6 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
       sshKeyEncrypted: 'encrypted',
       hardware: { cpu_cores: 2, memory_mb: 2048, disk_gb: 20 },
       zone: 'internal',
-      assignedModuleIds: [],
     });
 
     await db.insert(moduleInfrastructure).values({
@@ -583,7 +579,6 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
       sshKeyEncrypted: 'encrypted',
       hardware: { cpu_cores: 2, memory_mb: 2048, disk_gb: 20 },
       zone: 'internal',
-      assignedModuleIds: [],
     });
 
     await db.insert(moduleInfrastructure).values({
@@ -694,7 +689,6 @@ describe('resolveInfrastructureVariables - Edge Cases', () => {
       sshKeyEncrypted: 'encrypted',
       hardware: { cpu_cores: 2, memory_mb: 2048, disk_gb: 20 },
       zone: 'internal',
-      assignedModuleIds: [],
     });
 
     await db.insert(moduleInfrastructure).values({

package/src/services/infrastructure-variable-resolver.ts

@@ -90,9 +90,6 @@ export async function resolveInfrastructureVariables(
       hardware: machineRow.hardware || { cpu_cores: 0, memory_mb: 0, disk_gb: 0 },
       role: (machineRow.role as Machine['role']) || 'host',
       interfaces: (machineRow.interfaces || []) as Machine['interfaces'],
-      assignedModuleIds: Array.isArray(machineRow.assignedModuleIds)
-        ? machineRow.assignedModuleIds
-        : [],
       earmarkedModule: machineRow.earmarkedModule ?? undefined,
       createdAt: new Date(machineRow.createdAt),
       updatedAt: new Date(machineRow.updatedAt),