@celilo/cli 1.6.0 → 1.7.0

package/src/services/container-service.test.ts

@@ -17,6 +17,8 @@ import {
   getServiceCredentials,
   listContainerServices,
   removeContainerService,
+  updateServiceCredentials,
+  updateVerificationStatus,
 } from './container-service';
 
 describe('container-service', () => {
@@ -270,6 +272,38 @@ describe('container-service', () => {
         /Container service not found/,
       );
     });
+
+    it('re-encrypts replacements and clears stale verification state', async () => {
+      const service = await addContainerService({
+        name: 'Moving Proxmox',
+        providerName: 'proxmox',
+        zones: ['internal'],
+        providerConfig: {},
+        apiCredentials: {
+          api_url: 'https://192.168.0.50:8006',
+          api_token_id: 'root@pam!celilo',
+          api_token_secret: 'existing-secret',
+        },
+      });
+      await updateVerificationStatus(service.id, { success: true, message: 'Connected' });
+
+      await updateServiceCredentials(service.id, {
+        api_url: 'https://10.77.20.50:8006',
+        api_token_id: 'root@pam!celilo',
+        api_token_secret: 'existing-secret',
+      });
+
+      const credentials = await getServiceCredentials(service.id);
+      const updated = await getContainerService(service.id);
+      expect(credentials).toEqual({
+        api_url: 'https://10.77.20.50:8006',
+        api_token_id: 'root@pam!celilo',
+        api_token_secret: 'existing-secret',
+      });
+      expect(updated?.verified).toBe(false);
+      expect(updated?.verifiedAt).toBeNull();
+      expect(updated?.verificationError).toBeNull();
+    });
   });
 
   describe('removeContainerService', () => {

package/src/services/container-service.ts

@@ -251,6 +251,50 @@ export async function getServiceCredentials(serviceId: string): Promise<ServiceC
   );
 }
 
+/**
+ * Replace a container service's encrypted API credentials.
+ *
+ * Credentials identify the remote provider endpoint as well as the principal
+ * used there, so changing either invalidates the previous verification result.
+ * Callers should explicitly re-run service verification after this update.
+ */
+export async function updateServiceCredentials(
+  id: string,
+  credentials: ServiceCredentials,
+): Promise<void> {
+  const service = await getContainerService(id);
+  if (!service) {
+    throw new Error(`Container service not found: ${id}`);
+  }
+
+  const validated =
+    service.providerName === 'proxmox'
+      ? ProxmoxCredentialsSchema.parse(credentials)
+      : service.providerName === 'digitalocean'
+        ? DigitalOceanCredentialsSchema.parse(credentials)
+        : null;
+
+  if (!validated) {
+    throw new Error(
+      `Unsupported provider for credential validation: ${service.providerName}. Supported providers: proxmox, digitalocean`,
+    );
+  }
+
+  const masterKey = await getOrCreateMasterKey();
+  const encrypted = encryptSecret(JSON.stringify(validated), masterKey);
+
+  await getDb()
+    .update(containerServices)
+    .set({
+      apiCredentialsEncrypted: JSON.stringify(encrypted),
+      verified: false,
+      verifiedAt: null,
+      verificationError: null,
+      updatedAt: new Date(),
+    })
+    .where(eq(containerServices.id, id));
+}
+
 /**
  * List container services with optional filters
  */

package/src/services/deployed-systems.test.ts

@@ -266,6 +266,57 @@ describe('backfillModuleSystems', () => {
     });
   });
 
+  test('backfill records a local machine by its interface in the resolved zone', () => {
+    db.insert(machines)
+      .values({
+        id: 'm-local-bf',
+        hostname: 'celilo-mgr',
+        // Local-execution sentinel, not the machine's network identity.
+        ipAddress: '127.0.0.1',
+        sshUser: 'jem',
+        sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
+        hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
+        zone: 'secure-mgmt',
+        interfaces: [
+          { name: 'en0', ipAddress: '192.168.0.32', zone: 'external' },
+          { name: 'ens18', ipAddress: '10.77.20.32', zone: 'secure-mgmt' },
+        ],
+      })
+      .run();
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: { requires: { system: { zone: 'internal' } } },
+        sourcePath: '/tmp/celilo-mgmt',
+        state: 'VERIFIED',
+      })
+      .run();
+    db.insert(moduleInfrastructure)
+      .values({
+        id: 'infra-celilo-mgmt-local',
+        moduleId: 'celilo-mgmt',
+        infrastructureType: 'machine',
+        machineId: 'm-local-bf',
+      })
+      .run();
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'celilo-mgmt',
+        key: 'hostname',
+        value: 'celilo-mgr',
+        valueJson: '"celilo-mgr"',
+      })
+      .run();
+
+    expect(backfillModuleSystems(db)).toEqual(['celilo-mgmt']);
+    expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({
+      zone: 'secure-mgmt',
+      ipv4_address: '10.77.20.32',
+    });
+  });
+
   // The control-plane case. celilo-mgmt declares `internal` (it bootstraps before
   // any firewall exists) but in a segmented fleet the operator earmarks a box on
   // `secure-mgmt`. What gets RECORDED must be where the system actually is, since
@@ -318,6 +369,56 @@ describe('backfillModuleSystems', () => {
     expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({ zone: 'secure-mgmt' });
   });
 
+  test('records a local machine by its interface in the resolved zone', async () => {
+    db.insert(machines)
+      .values({
+        id: 'm-local',
+        hostname: 'celilo-mgr',
+        // Local-execution sentinel, not the machine's network identity.
+        ipAddress: '127.0.0.1',
+        sshUser: 'jem',
+        sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
+        hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
+        zone: 'secure-mgmt',
+        earmarkedModule: 'celilo-mgmt',
+        interfaces: [
+          { name: 'en0', ipAddress: '192.168.0.32', zone: 'external' },
+          { name: 'ens18', ipAddress: '10.77.20.32', zone: 'secure-mgmt' },
+        ],
+      })
+      .run();
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: { requires: { system: { zone: 'internal' } } },
+        sourcePath: '/tmp/celilo-mgmt',
+        state: 'VERIFIED',
+      })
+      .run();
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'celilo-mgmt',
+        key: 'hostname',
+        value: 'celilo-mgr',
+        valueJson: '"celilo-mgr"',
+      })
+      .run();
+
+    const recorded = await recordDeployedSystemForModule(
+      'celilo-mgmt',
+      { requires: { system: { zone: 'internal' } } } as ModuleManifest,
+      { type: 'machine', machineId: 'm-local' },
+      db,
+    );
+
+    expect(recorded[0]).toMatchObject({
+      zone: 'secure-mgmt',
+      ipv4_address: '10.77.20.32',
+    });
+  });
+
   test('skips an API-only module (no declared systems)', () => {
     ensureProxmoxService(db);
     db.insert(modules)

package/src/services/deployed-systems.ts

@@ -236,6 +236,35 @@ function asZone(value: string | null | undefined): NetworkZone | null {
     : null;
 }
 
+/** True for an IPv4 loopback address, including CIDR-form input. */
+function isLoopbackIpv4(value: string): boolean {
+  return stripCidr(value).startsWith('127.');
+}
+
+/**
+ * The machine pool uses 127.0.0.1 as a transport sentinel for the machine
+ * running celilo itself: it selects Ansible's local connection rather than
+ * SSH. That value is not the machine's network identity and must never leak
+ * into module_systems (and, downstream, internal DNS).
+ *
+ * For an ordinary machine, its primary address remains canonical. For the
+ * local sentinel, use the catalogued non-loopback interface in the system's
+ * resolved zone. Deliberately do not fall back to an arbitrary interface: a
+ * control plane can also have an upstream/Wi-Fi address, and publishing that
+ * would be a quieter version of the same identity bug.
+ */
+export function resolveMachineIdentityAddress(
+  machine: typeof machines.$inferSelect | undefined,
+  zone: NetworkZone,
+): string | undefined {
+  if (!machine) return undefined;
+  if (!isLoopbackIpv4(machine.ipAddress)) return machine.ipAddress;
+
+  return machine.interfaces.find(
+    (iface) => asZone(iface.zone) === zone && !isLoopbackIpv4(iface.ipAddress),
+  )?.ipAddress;
+}
+
 /**
  * Resolve a module's deployed system(s) from the deploy state and persist them
  * to `module_systems`. Called during deploy after infrastructure variables are
@@ -265,14 +294,9 @@ export async function recordDeployedSystemForModule(
   // No hostname → not yet addressable. A modeled state, not an error.
   if (!hostname) return [];
 
-  // IP: the module's resolved target_ip, else ip.primary, else the assigned
-  // machine's own IP (machine-pool deploys where neither was written).
-  let ip = cfg('target_ip') ?? cfg('ip.primary');
   const machine = infrastructure?.machineId
     ? db.select().from(machines).where(eq(machines.id, infrastructure.machineId)).get()
     : undefined;
-  if (!ip) ip = machine?.ipAddress;
-  if (!ip) return [];
 
   // Single-system transition: take the first declared system's name + zone.
   const decl = declared[0];
@@ -289,6 +313,13 @@ export async function recordDeployedSystemForModule(
     );
   }
 
+  // IP: the module's resolved target_ip, else ip.primary, else the assigned
+  // machine's network identity. A local machine's 127.0.0.1 primary address is
+  // an execution-transport sentinel, so resolve its identity from the
+  // catalogued interface in the system's actual zone instead.
+  const ip = cfg('target_ip') ?? cfg('ip.primary') ?? resolveMachineIdentityAddress(machine, zone);
+  if (!ip) return [];
+
   const infraType = infrastructure?.type ?? 'machine';
   const vmidStr = cfg('vmid');
   const vmid = vmidStr ? Number.parseInt(vmidStr, 10) : null;
@@ -352,15 +383,9 @@ export function backfillModuleSystems(db: DbClient): string[] {
     const hostname = cfg('hostname');
     if (!hostname) continue;
 
-    // IP: resolved target_ip, else ip.primary, else the assigned machine's own
-    // IP (machine-pool deploy where neither was written). Mirrors
-    // recordDeployedSystemForModule. CIDR is stripped by upsertDeployedSystem.
-    let ip = cfg('target_ip') ?? cfg('ip.primary');
     const machine = infra.machineId
       ? db.select().from(machines).where(eq(machines.id, infra.machineId)).get()
       : undefined;
-    if (!ip) ip = machine?.ipAddress;
-    if (!ip) continue;
 
     const decl = declared[0];
     // Same precedence as recordDeployedSystemForModule: the machine a deploy
@@ -373,6 +398,13 @@ export function backfillModuleSystems(db: DbClient): string[] {
     // backfill (a re-deploy will record it properly).
     if (!zone) continue;
 
+    // Mirrors recordDeployedSystemForModule. For celilo's local machine,
+    // 127.0.0.1 is only a local-transport sentinel; the zone-matching
+    // catalogued interface is the deployed system's address.
+    const ip =
+      cfg('target_ip') ?? cfg('ip.primary') ?? resolveMachineIdentityAddress(machine, zone);
+    if (!ip) continue;
+
     const vmidStr = cfg('vmid');
     const vmid = vmidStr ? Number.parseInt(vmidStr, 10) : null;
 

package/src/services/dns-provider-backfill.ts

@@ -12,6 +12,36 @@
  * All DNS mechanics (which zones, register vs delete) live in the module's
  * hook; this file only decides WHICH hosts and invokes the hook. That is the
  * D5 division of labour: celilo owns host inventory, the module owns DNS.
+ *
+ * WHY THIS SURVIVED THE GENERIC PROVIDER-ARRIVAL BACKFILL
+ * (openspec/changes/capability-owned-tables task 1.7). `provider-arrival.ts`
+ * replaced `public-web-republish.ts` and covers `firewall` for the first time.
+ * It does NOT cover either function here, and deleting this file over it would
+ * silently take internal DNS with it. Two independent reasons, both checked
+ * against the manifests rather than assumed:
+ *
+ *   1. `backfillProviderDns` replays celilo's HOST INVENTORY — one record per
+ *      deployed system, across every module. Those systems are hosts, not
+ *      consumers. There is no `requires`/`optional` edge to fan out along
+ *      because none of them asked `dns_internal` for anything. It is not a
+ *      capability registration set, so a mechanism defined over consumer
+ *      declarations cannot reach it.
+ *
+ *   2. `backfillWebRouteDns` covers the FQDNs modules publish through
+ *      `public_web`. Those modules declare `public_web` — `authentik`,
+ *      `forgejo`, `celilo-registry`, `celilo-apt-repo`, `celilo-website`,
+ *      `npm-cache-node` and the two hello fixtures. NOT ONE declares
+ *      `dns_internal`. The only module that does is `caddy-internal`, and it
+ *      declares it `optional`. So a `dns_internal` provider arriving has no
+ *      edge to the modules whose records it needs to learn.
+ *
+ * Point 2 is still second-implementation debt, and it is already ledgered:
+ * `module-business-baseline.ts` carries this file under S3 (#945). Retiring it
+ * needs a `public_web` fan-out triggered by a `dns_internal` arrival — a
+ * CROSS-capability edge the pull-shaped design does not define — because
+ * `register_route` (packages/capabilities/src/public-web.ts) already writes
+ * exactly the record `backfillWebRouteDns` reconstructs by hand, with the same
+ * value and the same `zoneRoutableValue`.
  */
 
 import type { DnsInternalCapability, HookLogger } from '@celilo/capabilities';

package/src/services/fleet-checks.test.ts

@@ -650,6 +650,32 @@ describe('findBrokenCapabilityDerivations (shared predicate)', () => {
     expect(problems).toHaveLength(0);
   });
 
+  it('accepts a broken optional derivation when the manifest declares a fallback', () => {
+    const withFallback = baseManifest({
+      id: 'secondary-dns',
+      variables: {
+        owns: [
+          {
+            name: 'managed_domains',
+            type: 'array',
+            required: false,
+            default: [],
+            source: 'capability',
+            derive_from: '$capability:dns_internal.dns.managed_domains',
+          },
+        ],
+        imports: [],
+      },
+    });
+
+    expect(findBrokenCapabilityDerivations('secondary-dns', withFallback, {})).toEqual([]);
+    expect(
+      findBrokenCapabilityDerivations('secondary-dns', withFallback, {
+        dns_internal: { dns: { managed_domains: '$self:managed_domains' } },
+      }),
+    ).toEqual([]);
+  });
+
   it('walks dotted paths', () => {
     const m = baseManifest({
       id: 'consumer',

package/src/services/fleet-checks.ts

@@ -502,6 +502,8 @@ interface CapabilityRef {
   variable: string;
   capability: string;
   path: string;
+  /** A manifest default makes an absent/broken optional derivation safe. */
+  hasFallback: boolean;
 }
 
 export type CapabilityDerivationReason = 'no-provider' | 'empty-value' | 'unresolved-ref';
@@ -531,7 +533,12 @@ function parseCapabilityRefs(manifest: ModuleManifest): CapabilityRef[] {
     const re = /\$\{?capability:([\w-]+)\.([\w.]+)/g;
     let m: RegExpExecArray | null = re.exec(v.derive_from);
     while (m !== null) {
-      refs.push({ variable: v.name, capability: m[1], path: m[2] });
+      refs.push({
+        variable: v.name,
+        capability: m[1],
+        path: m[2],
+        hasFallback: Object.prototype.hasOwnProperty.call(v, 'default'),
+      });
       m = re.exec(v.derive_from);
     }
   }
@@ -578,15 +585,18 @@ export function findBrokenCapabilityDerivations(
     };
     const data = capabilities[ref.capability];
     if (!data) {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'no-provider' });
       continue;
     }
     const value = getNested(data, ref.path);
     if (value === undefined || value === null || value === '') {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'empty-value' });
       continue;
     }
     if (typeof value === 'string' && UNRESOLVED_REF.test(value)) {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'unresolved-ref', value });
     }
   }

package/src/services/module-deploy.ts

@@ -40,8 +40,12 @@ import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
 import { ensureRequiredNetworks } from './network-ensure';
+import {
+  type ProviderBackfillResult,
+  loadProviderBackfillPlan,
+  runProviderBackfill,
+} from './provider-arrival';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
-import { republishStaticWebConsumers } from './public-web-republish';
 import { LOCAL_MACHINE_IP, deleteTemporarySshKey, writeTemporarySshKey } from './ssh-key-manager';
 import { buildTerraformEnvForService } from './terraform-env';
 
@@ -224,6 +228,81 @@ async function invokeHookWithEnsureRetry(
   }
 }
 
+/**
+ * A provider just arrived — re-run the consumers that were already here.
+ *
+ * The generic mirror of `consumer-cleanup.ts`. It replaces
+ * `republishStaticWebConsumers`, which did this for `public_web` alone, and it
+ * covers `firewall` for the first time — three modules provide `firewall` and
+ * none of their consumers inherited anything on deploy (celilo#1011).
+ *
+ * Never fatal to the provider's own deploy. The provider deployed fine; what
+ * can fail is one consumer's re-registration, and the honest report is that
+ * consumer named with the command that retries it. Silence would report a clean
+ * deploy over a fleet where some consumers never re-registered.
+ */
+// ponytail: fans out on EVERY deploy of a module that provides anything, so a
+// caddy redeploy re-runs all eight `public_web` consumers' `on_install` even
+// though caddy was already their provider and they have nothing to gain. The
+// hooks are idempotent by contract and a provider deploy is already heavyweight,
+// so this is accepted rather than guessed at — every predicate for "this
+// consumer has something to gain" that does not need new state is wrong in some
+// case (a redeploy of the incumbent provider looks identical to a first
+// deploy). If it measurably hurts, the upgrade is to record which provider a
+// consumer last bound to and re-run only where that changed.
+async function backfillArrivedProvider(moduleId: string, db: DbClient): Promise<void> {
+  const plan = loadProviderBackfillPlan(moduleId, db);
+  if (plan.length === 0) return;
+
+  const attempts = plan.filter((target) => !target.skip);
+  const gauge = new FuelGauge(`Re-registering ${attempts.length} consumer(s) with ${moduleId}`, {
+    skipAnimation: !process.stdout.isTTY,
+  });
+  gauge.start();
+  let result: ProviderBackfillResult;
+  try {
+    const logger = createGaugeLogger(gauge, moduleId, 'provider_arrival');
+    result = await runProviderBackfill(moduleId, plan, db, logger);
+    gauge.stop(result.failures.length === 0);
+  } catch (error) {
+    gauge.stop(false);
+    const msg = error instanceof Error ? error.message : String(error);
+    log.warn(`Consumer re-registration against '${moduleId}' could not run: ${msg}`);
+    return;
+  }
+
+  if (result.rerun.length > 0) {
+    log.success(
+      `Re-registered ${result.rerun.length} consumer(s) with '${moduleId}': ${result.rerun.join(', ')}`,
+    );
+  }
+  // Re-surfaced outside the gauge: the gauge's own preview scrolls away, and a
+  // consumer that never re-registered is exactly what an operator must still be
+  // able to read once the deploy has finished.
+  //
+  // INFO, not warn, and the level is load-bearing. `unpause --cascade`
+  // redeploys in topological order and clears `pausedAt` only AFTER each
+  // redeploy succeeds (`module-pause.ts#executeUnpause`), so the provider's own
+  // redeploy runs while every consumer in the cascade is still PAUSED. Warning
+  // there would put one alarm per consumer immediately before the cascade
+  // redeploys each of them successfully — an alarm about the normal path.
+  //
+  // Nothing is at risk either way: a paused module cannot return to service
+  // without a redeploy, and that redeploy re-resolves its capabilities. That is
+  // the same guarantee `remove-guard.ts` relies on to treat a paused module as
+  // not a dependent. So this is worth saying and not worth warning about.
+  for (const paused of result.skipped.filter((s) => s.reason === 'paused')) {
+    log.info(
+      `'${paused.consumerId}' is paused, so it was not re-registered with '${moduleId}'. It rebinds when it is unpaused and redeployed.`,
+    );
+  }
+  for (const failure of result.failures) {
+    log.warn(
+      `'${failure.consumerId}' failed to re-register with '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.consumerId}\` to retry.`,
+    );
+  }
+}
+
 /**
  * Orchestrate module deployment workflow
  * Orchestrator function - coordinates deployment phases
@@ -836,26 +915,10 @@ async function deployModuleImpl(
         log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
       }
 
-      // A public_web provider deploy rebuilds its host and takes /srv/www with
-      // it, while consumers' route rows survive — so the fresh provider would
-      // render site blocks pointing at directories that no longer exist. Re-run
-      // each static consumer's on_install to refill them. Idempotent; failures
-      // are logged, never fatal to the provider's own deploy.
-      const republish = await republishStaticWebConsumers({
-        providerModuleId: moduleId,
-        manifest,
-        db,
-      });
-      if (republish.republished.length > 0) {
-        log.success(
-          `Re-published ${republish.republished.length} static site(s) to '${moduleId}': ${republish.republished.join(', ')}`,
-        );
-      }
-      for (const failure of republish.failures) {
-        log.warn(
-          `Failed to re-publish '${failure.moduleId}' to '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.moduleId}\` to restore its content.`,
-        );
-      }
+      // Whatever this module provides, the consumers that were already here
+      // never got to ask it for anything. Re-run them so they register through
+      // the path that worked the first time (design D7).
+      await backfillArrivedProvider(moduleId, db);
 
       // Mirror the infrastructure-path success message at the end of
       // a successful deploy. Without this, config-only deploys end
@@ -1514,26 +1577,10 @@ async function deployModuleImpl(
         log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
       }
 
-      // A public_web provider deploy rebuilds its host and takes /srv/www with
-      // it, while consumers' route rows survive — so the fresh provider would
-      // render site blocks pointing at directories that no longer exist. Re-run
-      // each static consumer's on_install to refill them. Idempotent; failures
-      // are logged, never fatal to the provider's own deploy.
-      const republish = await republishStaticWebConsumers({
-        providerModuleId: moduleId,
-        manifest,
-        db,
-      });
-      if (republish.republished.length > 0) {
-        log.success(
-          `Re-published ${republish.republished.length} static site(s) to '${moduleId}': ${republish.republished.join(', ')}`,
-        );
-      }
-      for (const failure of republish.failures) {
-        log.warn(
-          `Failed to re-publish '${failure.moduleId}' to '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.moduleId}\` to restore its content.`,
-        );
-      }
+      // Whatever this module provides, the consumers that were already here
+      // never got to ask it for anything. Re-run them so they register through
+      // the path that worked the first time (design D7).
+      await backfillArrivedProvider(moduleId, db);
 
       log.success(`Module '${moduleId}' deployed successfully`);
       return {