@celilo/cli 4.0.0 → 4.1.0

package/src/services/update/orchestrator.test.ts

@@ -3,10 +3,12 @@ import type { DbClient } from '../../db/client';
 import type { ModuleManifest } from '../../manifest/schema';
 import type { AuditDeps } from '../audit';
 import { unusedPublicDnsProbe } from '../audit/public-dns';
+import type { DriftFinding } from '../audit/types';
 import { buildModuleGraph } from './dep-graph';
 import {
   type ModuleSnapshot,
   type OrchestratorOps,
+  blockersOutOfReach,
   consumerSkipReason,
   runSystemUpdate,
 } from './orchestrator';
@@ -73,6 +75,12 @@ const cleanAudit: AuditDeps = {
   schema: { journal: () => null, applied: () => [], db: fakeDb },
   capabilityAbi: { modules: [] },
   browserPin: { consumers: [], provisioned: null },
+  serverBunPin: {
+    pinnedVersion: '1.4.2',
+    runningVersion: '1.4.2',
+    pinPath: '/etc/celilo/mise.pin.toml',
+  },
+  rebootPending: { verdict: { pending: false, unmeasured: false, reasons: [] }, marker: null },
   terraformPlan: { modules: [], run: async () => ({ exitCode: 0, stdout: '', stderr: '' }) },
   moduleVersions: { installed: [], fetcher: async () => ({ latest: null }) },
   moduleConfigs: { modules: [] },
@@ -88,6 +96,7 @@ const cleanAudit: AuditDeps = {
   secretsDecryptable: { results: [] },
   servicesReachable: { results: [] },
   machinesReachable: { results: [] },
+  machineInterfaceZones: { machines: [] },
   diskSpace: { results: [] },
   transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
   trustedSources: { firewalls: [], unreachableFirewalls: [] },
@@ -162,6 +171,8 @@ describe('runSystemUpdate', () => {
     expect(result.audit.verdict).toBe('BLOCKED');
     expect(result.modules).toEqual([]);
     expect(upgradeCalls).toBe(0);
+    // Not `no-network` — the network is fine; the audit is what refused.
+    expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
   });
 
   test('happy path: clean audit + one drifting module → backup, upgrade, deploy, health, done', async () => {
@@ -471,3 +482,156 @@ describe('runSystemUpdate', () => {
     expect([...seenIds]).toEqual(['shared-id']);
   });
 });
+
+describe('blockersOutOfReach', () => {
+  const abiBlocker = (subject: string): DriftFinding => ({
+    category: 'capability_abi',
+    severity: 'blocked',
+    code: 'capability_abi_provider_mismatch',
+    message: `${subject} provides public_web@3.1.0 but framework runtime expects 4.0.0`,
+    subject,
+  });
+
+  test('an unscoped run cannot reach any blocker', () => {
+    expect(blockersOutOfReach([abiBlocker('caddy')], undefined)).toHaveLength(1);
+  });
+
+  test('--module X reaches a blocker about X', () => {
+    expect(blockersOutOfReach([abiBlocker('caddy')], 'caddy')).toEqual([]);
+  });
+
+  test('--module Y does not reach a blocker about X', () => {
+    const out = blockersOutOfReach([abiBlocker('caddy')], 'celilo-mgmt');
+    expect(out.map((f) => f.subject)).toEqual(['caddy']);
+  });
+
+  test('--module X never reaches a system-wide blocker', () => {
+    const systemBlocker: DriftFinding = {
+      category: 'schema',
+      severity: 'blocked',
+      code: 'schema_pending_migrations',
+      message: '1 pending migration',
+      subject: 'system',
+    };
+    const out = blockersOutOfReach([abiBlocker('caddy'), systemBlocker], 'caddy');
+    expect(out.map((f) => f.subject)).toEqual(['system']);
+  });
+
+  test('non-blocking findings are never returned', () => {
+    const drift: DriftFinding = {
+      category: 'module_versions',
+      severity: 'drift',
+      code: 'module_version_drift',
+      message: 'caddy 1.0.0 → 1.1.0',
+      subject: 'caddy',
+    };
+    expect(blockersOutOfReach([drift], undefined)).toEqual([]);
+  });
+});
+
+describe('runSystemUpdate with a module-scoped BLOCKED audit', () => {
+  // caddy is deployed and provides public_web@3.1.0 while the runtime
+  // contract is at 4.0.0 — the celilo#1381 shape. Upgrading caddy is
+  // exactly what the blocking finding asks for.
+  const abiBlockedAudit: AuditDeps = {
+    ...cleanAudit,
+    capabilityAbi: {
+      modules: [
+        {
+          id: 'caddy',
+          state: 'VERIFIED',
+          manifest: makeManifest('caddy', { provides: ['public_web'] }),
+        },
+      ],
+      contractVersions: { public_web: '2.0.0' },
+    },
+  };
+
+  function run(onlyModule: string | undefined, ops: OrchestratorOps) {
+    return runSystemUpdate({
+      audit: abiBlockedAudit,
+      graph: buildModuleGraph([
+        makeManifest('caddy', { provides: ['public_web'] }),
+        makeManifest('celilo-mgmt'),
+      ]),
+      snapshots: new Map([
+        ['caddy', makeSnapshot('caddy', { installed: '1.0.0', latest: '1.1.0' })],
+        ['celilo-mgmt', makeSnapshot('celilo-mgmt', { installed: '1.0.0', latest: '1.1.0' })],
+      ]),
+      ops,
+      selfUpdate: {
+        installedVersion: '0.1.5',
+        fetcher: async () => '0.1.5',
+        updater: async () => ({ ok: true, stderr: '' }),
+      },
+      progress: capturingProgress(),
+      now: () => new Date('2026-04-25T00:00:00Z'),
+      idGen: () => 'fixed-id',
+      onlyModule,
+    });
+  }
+
+  test('--module caddy upgrades caddy, with backup and health still run', async () => {
+    const calls: string[] = [];
+    const ops = makeOps({
+      backup: async (id) => {
+        calls.push(`backup:${id}`);
+        return { ok: true };
+      },
+      upgrade: async (id) => {
+        calls.push(`upgrade:${id}`);
+        return { ok: true };
+      },
+      deploy: async (id) => {
+        calls.push(`deploy:${id}`);
+        return { ok: true };
+      },
+      health: async (id) => {
+        calls.push(`health:${id}`);
+        return { status: 'healthy' };
+      },
+    });
+
+    const result = await run('caddy', ops);
+
+    expect(result.audit.verdict).toBe('BLOCKED');
+    expect(result.modules.map((m) => m.moduleId)).toEqual(['caddy']);
+    expect(result.modules[0]?.step).toBe('done');
+    expect(calls).toEqual(['backup:caddy', 'upgrade:caddy', 'deploy:caddy', 'health:caddy']);
+  });
+
+  test('--module celilo-mgmt refuses, naming the caddy finding, and never says no-network', async () => {
+    let upgrades = 0;
+    const ops = makeOps({
+      upgrade: async () => {
+        upgrades++;
+        return { ok: true };
+      },
+    });
+
+    const result = await run('celilo-mgmt', ops);
+
+    expect(result.ok).toBe(false);
+    expect(result.modules).toEqual([]);
+    expect(upgrades).toBe(0);
+    expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
+    const blocked = result.audit.findings.filter((f) => f.severity === 'blocked');
+    expect(blocked.map((f) => f.subject)).toEqual(['caddy']);
+  });
+
+  test('an unscoped run still refuses', async () => {
+    let upgrades = 0;
+    const ops = makeOps({
+      upgrade: async () => {
+        upgrades++;
+        return { ok: true };
+      },
+    });
+
+    const result = await run(undefined, ops);
+
+    expect(result.ok).toBe(false);
+    expect(result.modules).toEqual([]);
+    expect(upgrades).toBe(0);
+  });
+});

package/src/services/update/orchestrator.ts

@@ -20,6 +20,7 @@
 
 import { runAudit } from '../audit';
 import type { AuditDeps, SystemAuditReport } from '../audit';
+import type { DriftFinding } from '../audit/types';
 import {
   type ModuleGraph,
   type ModuleId,
@@ -133,6 +134,29 @@ export function consumerSkipReason(
   return null; // compatible; consumer can proceed against still-installed provider
 }
 
+/**
+ * The blocking findings a run cannot clear by doing its own work.
+ *
+ * `system update` used to refuse on ANY blocked finding, which made the
+ * command unable to fix the condition that stopped it: when a provider is
+ * behind a capability major, upgrading that provider is the remediation,
+ * and `--module <provider>` was refused too (celilo#1381).
+ *
+ * A finding is within reach only when the run is scoped to one module
+ * (`--module`) and the finding is about that same module — that module's
+ * upgrade is what the finding asks for. Findings about `system` or about
+ * any other module are never within reach: nothing this run does touches
+ * them. An unscoped run therefore still refuses on every blocker, as before.
+ */
+export function blockersOutOfReach(
+  findings: DriftFinding[],
+  onlyModule: string | undefined,
+): DriftFinding[] {
+  const blocked = findings.filter((f) => f.severity === 'blocked');
+  if (!onlyModule) return blocked;
+  return blocked.filter((f) => f.subject !== onlyModule);
+}
+
 /**
  * Drive a single module through the state machine. Stops at the first
  * failed step.
@@ -218,20 +242,27 @@ export async function runSystemUpdate(deps: RunUpdateDeps): Promise<SystemUpdate
 
   deps.progress.emit({ kind: 'plan', modules: deps.snapshots.size });
 
-  // Audit first; bail if BLOCKED.
+  // Audit first; bail if BLOCKED and the selected module can't clear it.
   const audit: SystemAuditReport = await runAudit(deps.audit);
   if (audit.verdict === 'BLOCKED') {
-    return {
-      version: 1,
-      updateId,
-      startedAt,
-      finishedAt: now().toISOString(),
-      audit,
-      selfUpdate: { performed: false, reason: 'no-network' },
-      backupsCreated: false,
-      modules: [],
-      ok: false,
-    };
+    const stillBlocking = blockersOutOfReach(audit.findings, deps.onlyModule);
+    if (stillBlocking.length > 0) {
+      deps.progress.emit({
+        kind: 'self-update-skipped',
+        reason: `audit BLOCKED: ${stillBlocking.map((f) => `${f.subject}: ${f.message}`).join('; ')}`,
+      });
+      return {
+        version: 1,
+        updateId,
+        startedAt,
+        finishedAt: now().toISOString(),
+        audit,
+        selfUpdate: { performed: false, reason: 'audit-blocked' },
+        backupsCreated: false,
+        modules: [],
+        ok: false,
+      };
+    }
   }
 
   // Self-update.

package/src/services/update/types.ts

@@ -43,7 +43,16 @@ export interface ModuleUpdateState {
 }
 
 export type SelfUpdateResult =
-  | { performed: false; reason: 'already-current' | 'dev-mode' | 'no-network' }
+  | {
+      performed: false;
+      /**
+       * `audit-blocked` means the run refused before the self-update was
+       * attempted. It exists because that refusal used to report
+       * `no-network` (celilo#1381), sending operators to debug a network
+       * that was fine.
+       */
+      reason: 'already-current' | 'dev-mode' | 'no-network' | 'audit-blocked';
+    }
   | { performed: true; from: string; to: string };
 
 export interface SystemUpdateResult {

package/src/templates/generator.test.ts

@@ -656,7 +656,7 @@ resource "proxmox_lxc" "container" {
       expect(out).toContain('  nameserver = "$self:lxc_nameserver"');
       expect(out).toContain('  lifecycle {');
       expect(out).toContain(
-        '    ignore_changes = [nameserver, network[0].hwaddr, rootfs[0].volume, ssh_public_keys]',
+        '    ignore_changes = [nameserver, network[0].hwaddr, ostemplate, rootfs[0].storage, ssh_public_keys]',
       );
       // Injected immediately after the opening line, before author attributes.
       const lines = out.split('\n');
@@ -673,10 +673,21 @@ resource "proxmox_lxc" "container" {
       expect(out).not.toContain('nameserver = "$self:lxc_nameserver"');
       expect(out).toContain('  lifecycle {');
       expect(out).toContain(
-        '    ignore_changes = [nameserver, network[0].hwaddr, rootfs[0].volume, ssh_public_keys]',
+        '    ignore_changes = [nameserver, network[0].hwaddr, ostemplate, rootfs[0].storage, ssh_public_keys]',
       );
     });
 
+    // celilo#1402: `ostemplate` and `rootfs[0].storage` are ForceNew and do
+    // NOT round-trip through `terraform import` — Proxmox reports neither. Drop
+    // them from the ignore list and every import of an existing container
+    // computes a REPLACE, which the post-import guard refuses, leaving the
+    // container redeployable by no route at all.
+    test('ignores the ForceNew attributes an import cannot recover', () => {
+      const out = injectProxmoxDns(LXC, true);
+      expect(out).toContain('ostemplate');
+      expect(out).toContain('rootfs[0].storage');
+    });
+
     test('is idempotent — already-injected content is returned unchanged', () => {
       const once = injectProxmoxDns(LXC, true);
       const twice = injectProxmoxDns(once, true);
@@ -737,7 +748,7 @@ resource "proxmox_lxc" "container" {
       ].join('\n');
       const out = injectProxmoxDns(mixed, true);
       expect(out).toContain(
-        'ignore_changes = [nameserver, network[0].hwaddr, rootfs[0].volume, ssh_public_keys]',
+        'ignore_changes = [nameserver, network[0].hwaddr, ostemplate, rootfs[0].storage, ssh_public_keys]',
       );
       expect(out).toContain('ignore_changes = [nameserver, network[0].macaddr, sshkeys, clone]');
     });
@@ -767,7 +778,7 @@ resource "proxmox_lxc" "container" {
       expect(out).toContain('    nameserver = "$self:lxc_nameserver"');
       expect(out).toContain('    lifecycle {');
       expect(out).toContain(
-        '      ignore_changes = [nameserver, network[0].hwaddr, rootfs[0].volume, ssh_public_keys]',
+        '      ignore_changes = [nameserver, network[0].hwaddr, ostemplate, rootfs[0].storage, ssh_public_keys]',
       );
     });
   });

package/src/templates/generator.ts

@@ -199,9 +199,17 @@ export function injectProxmoxDns(content: string, hasNameserver: boolean): strin
     // in-place UPDATE, never a destructive REPLACE. A non-existent attribute
     // here fails `terraform validate`, so each list is exactly the schema
     // attributes the resource actually has.
-    //   LXC (ISS-0055/0089): the create-time MAC + rootfs volume, plus
-    //   `ssh_public_keys` (ForceNew — a fleet-key rotation must not REPLACE the
-    //   container). `rootfs.size` is NOT ignored, so a disk grow still applies.
+    //   LXC (ISS-0055/0089): the create-time MAC, plus `ssh_public_keys`
+    //   (ForceNew — a fleet-key rotation must not REPLACE the container), plus
+    //   the two ForceNew attributes an IMPORT cannot recover: `ostemplate`
+    //   (Proxmox never reports the source template) and `rootfs[0].storage`
+    //   (import records the volume, not the config's storage name). Without
+    //   them every import of an existing container computes a REPLACE, the
+    //   post-import guard refuses it, and the container becomes redeployable
+    //   by no route at all (celilo#1402). `rootfs[0].volume` was listed here
+    //   and terraform reported it "Redundant ... has no effect" — it is
+    //   computed, never configured — so it is gone.
+    //   `rootfs.size` is NOT ignored, so a disk grow still applies.
     //   We do NOT ignore `target_node` — a node change is real (migration,
     //   ISS-0090), never silently dropped.
     //   VM (telmate 3.x proxmox_vm_qemu): `clone` (ForceNew clone source),
@@ -212,7 +220,7 @@ export function injectProxmoxDns(content: string, hasNameserver: boolean): strin
     const ignore =
       resourceType === 'proxmox_vm_qemu'
         ? 'nameserver, network[0].macaddr, sshkeys, clone'
-        : 'nameserver, network[0].hwaddr, rootfs[0].volume, ssh_public_keys';
+        : 'nameserver, network[0].hwaddr, ostemplate, rootfs[0].storage, ssh_public_keys';
     injected.push(`${inner}  ignore_changes = [${ignore}]`);
     injected.push(`${inner}}`);
     return injected.join('\n');

package/src/variables/context.ts

@@ -423,8 +423,21 @@ async function assembleResolutionContext(
         const deployedZone = machineRow.zone ?? zone;
         const machineIdentity = resolveMachineIdentityAddress(machineRow, deployedZone);
         if (!machineIdentity) {
+          // Distinguish "no such interface" from "the interface is there under
+          // a stale zone label" — the second is what a zone-model change
+          // produces, and its remediation is a re-classification, not new
+          // hardware (celilo#1382).
+          const { diffInterfaceZones, explainMissingZoneInterface } = await import(
+            '../services/machine-interface-zones'
+          );
+          const { detectZoneFromIp } = await import('../services/zone-detector');
+          const changes = await diffInterfaceZones(machineRow.interfaces, detectZoneFromIp);
           throw new Error(
-            `Machine '${infraSelection.machineId}' for module '${moduleId}' has no non-loopback interface in zone '${deployedZone}'`,
+            `Machine '${machineRow.hostname}' for module '${moduleId}' ${explainMissingZoneInterface(
+              machineRow.interfaces,
+              deployedZone,
+              changes,
+            )}`,
           );
         }
         upsertDeployedSystem(db, moduleId, {