@celilo/cli 1.6.0 → 1.8.0

package/src/services/deployed-systems.ts

@@ -236,6 +236,35 @@ function asZone(value: string | null | undefined): NetworkZone | null {
     : null;
 }
 
+/** True for an IPv4 loopback address, including CIDR-form input. */
+function isLoopbackIpv4(value: string): boolean {
+  return stripCidr(value).startsWith('127.');
+}
+
+/**
+ * The machine pool uses 127.0.0.1 as a transport sentinel for the machine
+ * running celilo itself: it selects Ansible's local connection rather than
+ * SSH. That value is not the machine's network identity and must never leak
+ * into module_systems (and, downstream, internal DNS).
+ *
+ * For an ordinary machine, its primary address remains canonical. For the
+ * local sentinel, use the catalogued non-loopback interface in the system's
+ * resolved zone. Deliberately do not fall back to an arbitrary interface: a
+ * control plane can also have an upstream/Wi-Fi address, and publishing that
+ * would be a quieter version of the same identity bug.
+ */
+export function resolveMachineIdentityAddress(
+  machine: typeof machines.$inferSelect | undefined,
+  zone: NetworkZone,
+): string | undefined {
+  if (!machine) return undefined;
+  if (!isLoopbackIpv4(machine.ipAddress)) return machine.ipAddress;
+
+  return machine.interfaces.find(
+    (iface) => asZone(iface.zone) === zone && !isLoopbackIpv4(iface.ipAddress),
+  )?.ipAddress;
+}
+
 /**
  * Resolve a module's deployed system(s) from the deploy state and persist them
  * to `module_systems`. Called during deploy after infrastructure variables are
@@ -265,14 +294,9 @@ export async function recordDeployedSystemForModule(
   // No hostname → not yet addressable. A modeled state, not an error.
   if (!hostname) return [];
 
-  // IP: the module's resolved target_ip, else ip.primary, else the assigned
-  // machine's own IP (machine-pool deploys where neither was written).
-  let ip = cfg('target_ip') ?? cfg('ip.primary');
   const machine = infrastructure?.machineId
     ? db.select().from(machines).where(eq(machines.id, infrastructure.machineId)).get()
     : undefined;
-  if (!ip) ip = machine?.ipAddress;
-  if (!ip) return [];
 
   // Single-system transition: take the first declared system's name + zone.
   const decl = declared[0];
@@ -289,6 +313,13 @@ export async function recordDeployedSystemForModule(
     );
   }
 
+  // IP: the module's resolved target_ip, else ip.primary, else the assigned
+  // machine's network identity. A local machine's 127.0.0.1 primary address is
+  // an execution-transport sentinel, so resolve its identity from the
+  // catalogued interface in the system's actual zone instead.
+  const ip = cfg('target_ip') ?? cfg('ip.primary') ?? resolveMachineIdentityAddress(machine, zone);
+  if (!ip) return [];
+
   const infraType = infrastructure?.type ?? 'machine';
   const vmidStr = cfg('vmid');
   const vmid = vmidStr ? Number.parseInt(vmidStr, 10) : null;
@@ -352,15 +383,9 @@ export function backfillModuleSystems(db: DbClient): string[] {
     const hostname = cfg('hostname');
     if (!hostname) continue;
 
-    // IP: resolved target_ip, else ip.primary, else the assigned machine's own
-    // IP (machine-pool deploy where neither was written). Mirrors
-    // recordDeployedSystemForModule. CIDR is stripped by upsertDeployedSystem.
-    let ip = cfg('target_ip') ?? cfg('ip.primary');
     const machine = infra.machineId
       ? db.select().from(machines).where(eq(machines.id, infra.machineId)).get()
       : undefined;
-    if (!ip) ip = machine?.ipAddress;
-    if (!ip) continue;
 
     const decl = declared[0];
     // Same precedence as recordDeployedSystemForModule: the machine a deploy
@@ -373,6 +398,13 @@ export function backfillModuleSystems(db: DbClient): string[] {
     // backfill (a re-deploy will record it properly).
     if (!zone) continue;
 
+    // Mirrors recordDeployedSystemForModule. For celilo's local machine,
+    // 127.0.0.1 is only a local-transport sentinel; the zone-matching
+    // catalogued interface is the deployed system's address.
+    const ip =
+      cfg('target_ip') ?? cfg('ip.primary') ?? resolveMachineIdentityAddress(machine, zone);
+    if (!ip) continue;
+
     const vmidStr = cfg('vmid');
     const vmid = vmidStr ? Number.parseInt(vmidStr, 10) : null;
 

package/src/services/dns-internal-records.test.ts

@@ -4,8 +4,9 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { DnsRecordRequest } from '@celilo/capabilities';
 import type { ViewOverride } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { modules } from '../db/schema';
+import { dnsInternalRecords, modules } from '../db/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
 import {
   listDnsInternalRecords,
@@ -197,3 +198,73 @@ describe('dns-internal-records ledger', () => {
     });
   });
 });
+
+/**
+ * celilo#1010 — swapping the internal DNS provider must not delete the ledger.
+ *
+ * `dns_internal_records` used to cascade on `provider_module_id` as well as on
+ * the consumer, so removing `technitium` to install `knot-unbound-internal` took
+ * the fleet's entire internal DNS ledger with it, `zone_routable_ip` view
+ * overrides included — the durable desired state the resolver's split-horizon
+ * config is reconciled from. `web_routes` cascades on its consumer only, and the
+ * two tables' docblocks claimed to be siblings, so the divergence read as intent
+ * and was not.
+ *
+ * The claim on a capability-owned table is the CONSUMER
+ * (openspec/changes/capability-owned-tables D3/D8). Migration 0027 makes the
+ * table agree with the declaration, which cannot express anything else.
+ */
+describe('provider cascade (celilo#1010)', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'dns-cascade-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    for (const id of ['technitium', 'knot-unbound-internal', 'caddy']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          sourcePath: `/tmp/${id}`,
+          manifestData: {},
+        })
+        .run();
+    }
+    db.insert(dnsInternalRecords)
+      .values({
+        providerModuleId: 'technitium',
+        consumerModuleId: 'caddy',
+        host: 'auth.example.org',
+        ip: '192.168.0.253',
+        zoneRoutableIp: '10.0.10.14',
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('keeps the ledger when the PROVIDER is removed, view overrides included', () => {
+    db.delete(modules).where(eq(modules.id, 'technitium')).run();
+
+    const left = db.select().from(dnsInternalRecords).all();
+    expect(left).toHaveLength(1);
+    // The override is the part whose loss is silent: the resolver keeps
+    // answering, just with the wrong address for in-zone clients.
+    expect(left[0]?.zoneRoutableIp).toBe('10.0.10.14');
+  });
+
+  it('still dies with its CONSUMER, which is the rule that did not change', () => {
+    db.delete(modules).where(eq(modules.id, 'caddy')).run();
+
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
+  });
+});

package/src/services/dns-provider-backfill.ts

@@ -12,6 +12,36 @@
  * All DNS mechanics (which zones, register vs delete) live in the module's
  * hook; this file only decides WHICH hosts and invokes the hook. That is the
  * D5 division of labour: celilo owns host inventory, the module owns DNS.
+ *
+ * WHY THIS SURVIVED THE GENERIC PROVIDER-ARRIVAL BACKFILL
+ * (openspec/changes/capability-owned-tables task 1.7). `provider-arrival.ts`
+ * replaced `public-web-republish.ts` and covers `firewall` for the first time.
+ * It does NOT cover either function here, and deleting this file over it would
+ * silently take internal DNS with it. Two independent reasons, both checked
+ * against the manifests rather than assumed:
+ *
+ *   1. `backfillProviderDns` replays celilo's HOST INVENTORY — one record per
+ *      deployed system, across every module. Those systems are hosts, not
+ *      consumers. There is no `requires`/`optional` edge to fan out along
+ *      because none of them asked `dns_internal` for anything. It is not a
+ *      capability registration set, so a mechanism defined over consumer
+ *      declarations cannot reach it.
+ *
+ *   2. `backfillWebRouteDns` covers the FQDNs modules publish through
+ *      `public_web`. Those modules declare `public_web` — `authentik`,
+ *      `forgejo`, `celilo-registry`, `celilo-apt-repo`, `celilo-website`,
+ *      `npm-cache-node` and the two hello fixtures. NOT ONE declares
+ *      `dns_internal`. The only module that does is `caddy-internal`, and it
+ *      declares it `optional`. So a `dns_internal` provider arriving has no
+ *      edge to the modules whose records it needs to learn.
+ *
+ * Point 2 is still second-implementation debt, and it is already ledgered:
+ * `module-business-baseline.ts` carries this file under S3 (#945). Retiring it
+ * needs a `public_web` fan-out triggered by a `dns_internal` arrival — a
+ * CROSS-capability edge the pull-shaped design does not define — because
+ * `register_route` (packages/capabilities/src/public-web.ts) already writes
+ * exactly the record `backfillWebRouteDns` reconstructs by hand, with the same
+ * value and the same `zoneRoutableValue`.
  */
 
 import type { DnsInternalCapability, HookLogger } from '@celilo/capabilities';

package/src/services/fleet-checks.test.ts

@@ -650,6 +650,32 @@ describe('findBrokenCapabilityDerivations (shared predicate)', () => {
     expect(problems).toHaveLength(0);
   });
 
+  it('accepts a broken optional derivation when the manifest declares a fallback', () => {
+    const withFallback = baseManifest({
+      id: 'secondary-dns',
+      variables: {
+        owns: [
+          {
+            name: 'managed_domains',
+            type: 'array',
+            required: false,
+            default: [],
+            source: 'capability',
+            derive_from: '$capability:dns_internal.dns.managed_domains',
+          },
+        ],
+        imports: [],
+      },
+    });
+
+    expect(findBrokenCapabilityDerivations('secondary-dns', withFallback, {})).toEqual([]);
+    expect(
+      findBrokenCapabilityDerivations('secondary-dns', withFallback, {
+        dns_internal: { dns: { managed_domains: '$self:managed_domains' } },
+      }),
+    ).toEqual([]);
+  });
+
   it('walks dotted paths', () => {
     const m = baseManifest({
       id: 'consumer',

package/src/services/fleet-checks.ts

@@ -502,6 +502,8 @@ interface CapabilityRef {
   variable: string;
   capability: string;
   path: string;
+  /** A manifest default makes an absent/broken optional derivation safe. */
+  hasFallback: boolean;
 }
 
 export type CapabilityDerivationReason = 'no-provider' | 'empty-value' | 'unresolved-ref';
@@ -531,7 +533,12 @@ function parseCapabilityRefs(manifest: ModuleManifest): CapabilityRef[] {
     const re = /\$\{?capability:([\w-]+)\.([\w.]+)/g;
     let m: RegExpExecArray | null = re.exec(v.derive_from);
     while (m !== null) {
-      refs.push({ variable: v.name, capability: m[1], path: m[2] });
+      refs.push({
+        variable: v.name,
+        capability: m[1],
+        path: m[2],
+        hasFallback: Object.prototype.hasOwnProperty.call(v, 'default'),
+      });
       m = re.exec(v.derive_from);
     }
   }
@@ -578,15 +585,18 @@ export function findBrokenCapabilityDerivations(
     };
     const data = capabilities[ref.capability];
     if (!data) {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'no-provider' });
       continue;
     }
     const value = getNested(data, ref.path);
     if (value === undefined || value === null || value === '') {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'empty-value' });
       continue;
     }
     if (typeof value === 'string' && UNRESOLVED_REF.test(value)) {
+      if (ref.hasFallback) continue;
       problems.push({ ...base, reason: 'unresolved-ref', value });
     }
   }

package/src/services/module-deploy.ts

@@ -40,8 +40,12 @@ import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
 import { ensureRequiredNetworks } from './network-ensure';
+import {
+  type ProviderBackfillResult,
+  loadProviderBackfillPlan,
+  runProviderBackfill,
+} from './provider-arrival';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
-import { republishStaticWebConsumers } from './public-web-republish';
 import { LOCAL_MACHINE_IP, deleteTemporarySshKey, writeTemporarySshKey } from './ssh-key-manager';
 import { buildTerraformEnvForService } from './terraform-env';
 
@@ -224,6 +228,81 @@ async function invokeHookWithEnsureRetry(
   }
 }
 
+/**
+ * A provider just arrived — re-run the consumers that were already here.
+ *
+ * The generic mirror of `consumer-cleanup.ts`. It replaces
+ * `republishStaticWebConsumers`, which did this for `public_web` alone, and it
+ * covers `firewall` for the first time — three modules provide `firewall` and
+ * none of their consumers inherited anything on deploy (celilo#1011).
+ *
+ * Never fatal to the provider's own deploy. The provider deployed fine; what
+ * can fail is one consumer's re-registration, and the honest report is that
+ * consumer named with the command that retries it. Silence would report a clean
+ * deploy over a fleet where some consumers never re-registered.
+ */
+// ponytail: fans out on EVERY deploy of a module that provides anything, so a
+// caddy redeploy re-runs all eight `public_web` consumers' `on_install` even
+// though caddy was already their provider and they have nothing to gain. The
+// hooks are idempotent by contract and a provider deploy is already heavyweight,
+// so this is accepted rather than guessed at — every predicate for "this
+// consumer has something to gain" that does not need new state is wrong in some
+// case (a redeploy of the incumbent provider looks identical to a first
+// deploy). If it measurably hurts, the upgrade is to record which provider a
+// consumer last bound to and re-run only where that changed.
+async function backfillArrivedProvider(moduleId: string, db: DbClient): Promise<void> {
+  const plan = loadProviderBackfillPlan(moduleId, db);
+  if (plan.length === 0) return;
+
+  const attempts = plan.filter((target) => !target.skip);
+  const gauge = new FuelGauge(`Re-registering ${attempts.length} consumer(s) with ${moduleId}`, {
+    skipAnimation: !process.stdout.isTTY,
+  });
+  gauge.start();
+  let result: ProviderBackfillResult;
+  try {
+    const logger = createGaugeLogger(gauge, moduleId, 'provider_arrival');
+    result = await runProviderBackfill(moduleId, plan, db, logger);
+    gauge.stop(result.failures.length === 0);
+  } catch (error) {
+    gauge.stop(false);
+    const msg = error instanceof Error ? error.message : String(error);
+    log.warn(`Consumer re-registration against '${moduleId}' could not run: ${msg}`);
+    return;
+  }
+
+  if (result.rerun.length > 0) {
+    log.success(
+      `Re-registered ${result.rerun.length} consumer(s) with '${moduleId}': ${result.rerun.join(', ')}`,
+    );
+  }
+  // Re-surfaced outside the gauge: the gauge's own preview scrolls away, and a
+  // consumer that never re-registered is exactly what an operator must still be
+  // able to read once the deploy has finished.
+  //
+  // INFO, not warn, and the level is load-bearing. `unpause --cascade`
+  // redeploys in topological order and clears `pausedAt` only AFTER each
+  // redeploy succeeds (`module-pause.ts#executeUnpause`), so the provider's own
+  // redeploy runs while every consumer in the cascade is still PAUSED. Warning
+  // there would put one alarm per consumer immediately before the cascade
+  // redeploys each of them successfully — an alarm about the normal path.
+  //
+  // Nothing is at risk either way: a paused module cannot return to service
+  // without a redeploy, and that redeploy re-resolves its capabilities. That is
+  // the same guarantee `remove-guard.ts` relies on to treat a paused module as
+  // not a dependent. So this is worth saying and not worth warning about.
+  for (const paused of result.skipped.filter((s) => s.reason === 'paused')) {
+    log.info(
+      `'${paused.consumerId}' is paused, so it was not re-registered with '${moduleId}'. It rebinds when it is unpaused and redeployed.`,
+    );
+  }
+  for (const failure of result.failures) {
+    log.warn(
+      `'${failure.consumerId}' failed to re-register with '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.consumerId}\` to retry.`,
+    );
+  }
+}
+
 /**
  * Orchestrate module deployment workflow
  * Orchestrator function - coordinates deployment phases
@@ -836,26 +915,10 @@ async function deployModuleImpl(
         log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
       }
 
-      // A public_web provider deploy rebuilds its host and takes /srv/www with
-      // it, while consumers' route rows survive — so the fresh provider would
-      // render site blocks pointing at directories that no longer exist. Re-run
-      // each static consumer's on_install to refill them. Idempotent; failures
-      // are logged, never fatal to the provider's own deploy.
-      const republish = await republishStaticWebConsumers({
-        providerModuleId: moduleId,
-        manifest,
-        db,
-      });
-      if (republish.republished.length > 0) {
-        log.success(
-          `Re-published ${republish.republished.length} static site(s) to '${moduleId}': ${republish.republished.join(', ')}`,
-        );
-      }
-      for (const failure of republish.failures) {
-        log.warn(
-          `Failed to re-publish '${failure.moduleId}' to '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.moduleId}\` to restore its content.`,
-        );
-      }
+      // Whatever this module provides, the consumers that were already here
+      // never got to ask it for anything. Re-run them so they register through
+      // the path that worked the first time (design D7).
+      await backfillArrivedProvider(moduleId, db);
 
       // Mirror the infrastructure-path success message at the end of
       // a successful deploy. Without this, config-only deploys end
@@ -1514,26 +1577,10 @@ async function deployModuleImpl(
         log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
       }
 
-      // A public_web provider deploy rebuilds its host and takes /srv/www with
-      // it, while consumers' route rows survive — so the fresh provider would
-      // render site blocks pointing at directories that no longer exist. Re-run
-      // each static consumer's on_install to refill them. Idempotent; failures
-      // are logged, never fatal to the provider's own deploy.
-      const republish = await republishStaticWebConsumers({
-        providerModuleId: moduleId,
-        manifest,
-        db,
-      });
-      if (republish.republished.length > 0) {
-        log.success(
-          `Re-published ${republish.republished.length} static site(s) to '${moduleId}': ${republish.republished.join(', ')}`,
-        );
-      }
-      for (const failure of republish.failures) {
-        log.warn(
-          `Failed to re-publish '${failure.moduleId}' to '${moduleId}': ${failure.error}. Run \`celilo module deploy ${failure.moduleId}\` to restore its content.`,
-        );
-      }
+      // Whatever this module provides, the consumers that were already here
+      // never got to ask it for anything. Re-run them so they register through
+      // the path that worked the first time (design D7).
+      await backfillArrivedProvider(moduleId, db);
 
       log.success(`Module '${moduleId}' deployed successfully`);
       return {

package/src/services/module-validator/capability-versions.test.ts

@@ -5,6 +5,7 @@
  */
 
 import { describe, expect, test } from 'bun:test';
+import { CAPABILITY_CONTRACT_VERSIONS } from '@celilo/capabilities';
 import { validateCapabilityVersions } from './capability-versions';
 
 describe('validateCapabilityVersions', () => {
@@ -31,7 +32,11 @@ describe('validateCapabilityVersions', () => {
     expect(errors).toHaveLength(1);
     expect(errors[0]).toContain('provides[public_web]');
     expect(errors[0]).toContain('1.0.0');
-    expect(errors[0]).toContain('3.1.0');
+    // Read from the registry, not pinned to today's number. The assertion is
+    // "the message names the RUNTIME version so the author knows what to move
+    // to"; a literal here turns a behaviour test into a version tracker that
+    // goes red on every legitimate bump, which is what happened at 3.1.0 → 3.2.0.
+    expect(errors[0]).toContain(CAPABILITY_CONTRACT_VERSIONS.public_web);
   });
 
   test('error when provides[X].version is newer than runtime', () => {

package/src/services/port-forwards.test.ts

@@ -7,7 +7,8 @@ import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { portForwards } from '../db/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
-import { buildPortForwardStore, deletePortForwardsForModule } from './port-forwards';
+import { deleteClaimedRows } from './capability-table-rows';
+import { buildPortForwardStore } from './port-forwards';
 
 const FW = '192.168.0.254';
 const CADDY = { internalIp: '10.0.20.5', protocol: 'TCP' as const, description: 'caddy' };
@@ -113,7 +114,10 @@ describe('port-forward store', () => {
     other.replace(FW, { ...CADDY, description: 'also 443' }, [443]);
     expect(store.list(FW)).toHaveLength(2);
 
-    deletePortForwardsForModule(db, 'caddy');
+    // Through the GENERIC declaration-driven path, which replaced
+    // `deletePortForwardsForModule`. The property is unchanged; what changed is
+    // that core no longer names this capability's table to clear it.
+    deleteClaimedRows(db, 'caddy');
 
     const left = store.list(FW);
     expect(left).toHaveLength(1);

package/src/services/port-forwards.ts

@@ -99,14 +99,3 @@ export function buildPortForwardStore(db: DbClient, registeredBy: string): PortF
     },
   };
 }
-
-/**
- * Drop every forward a departing module owns, across every firewall.
- *
- * `registered_by` is plain text rather than a FK, so unlike `web_routes` these
- * rows do NOT die with the `modules` row — core deletes them explicitly, AFTER
- * the provider has converged without them (D4).
- */
-export function deletePortForwardsForModule(db: DbClient, moduleId: string): void {
-  db.delete(portForwards).where(eq(portForwards.registeredBy, moduleId)).run();
-}