@celilo/cli 0.27.0 → 1.1.0

package/src/services/trusted-sources.test.ts

@@ -40,27 +40,63 @@ describe('trusted-source store', () => {
 
   it('stamps registeredBy from the binding, not the caller', () => {
     const store = buildTrustedSourceStore(db, 'wireguard');
-    store.add(FW, { subnet: VPN, description: 'admin VPN clients' });
+    store.replace(FW, { subnets: [VPN], description: 'admin VPN clients' });
 
     expect(store.list(FW)).toEqual([
       { subnet: VPN, description: 'admin VPN clients', registeredBy: 'wireguard' },
     ]);
   });
 
-  it('upserts by subnet — registering twice yields one row', () => {
+  it('replace is idempotent — declaring the same set twice yields one row', () => {
     const store = buildTrustedSourceStore(db, 'wireguard');
-    store.add(FW, { subnet: VPN, description: 'first' });
-    store.add(FW, { subnet: VPN, description: 'second' });
+    store.replace(FW, { subnets: [VPN], description: 'first' });
+    store.replace(FW, { subnets: [VPN], description: 'second' });
 
     expect(store.list(FW)).toHaveLength(1);
     expect(store.list(FW)[0].description).toBe('second');
   });
 
-  it('removes by subnet and leaves other firewalls alone', () => {
-    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
-    buildTrustedSourceStore(db, 'other').add('10.0.0.1', { subnet: VPN, description: 'elsewhere' });
+  // D5b: the set is DECLARED, so changing an admin VPN's client subnet revokes
+  // the old one's reach. It used to keep reaching every zone forever.
+  it('a subnet left out of a later declaration loses its reach', () => {
+    const store = buildTrustedSourceStore(db, 'wireguard');
+    store.replace(FW, { subnets: [VPN, '10.9.9.0/24'], description: 'vpn' });
+    store.replace(FW, { subnets: [VPN], description: 'vpn' });
 
-    buildTrustedSourceStore(db, 'wireguard').remove(FW, VPN);
+    expect(store.list(FW).map((s) => s.subnet)).toEqual([VPN]);
+  });
+
+  it('an empty declaration withdraws the consumer’s whole set', () => {
+    const store = buildTrustedSourceStore(db, 'wireguard');
+    store.replace(FW, { subnets: [VPN], description: 'vpn' });
+    store.replace(FW, { subnets: [], description: 'vpn' });
+
+    expect(store.list(FW)).toEqual([]);
+  });
+
+  // D5a, the refcount case: the owner is in the unique index, so two modules
+  // trusting the same subnet are two rows and one withdrawing does not revoke
+  // the other's reach.
+  it('two consumers trusting the same subnet are two rows; one withdrawing leaves the other', () => {
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [VPN], description: 'vpn' });
+    buildTrustedSourceStore(db, 'other').replace(FW, { subnets: [VPN], description: 'also vpn' });
+    expect(listTrustedSourcesFor(db, FW)).toHaveLength(2);
+
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [], description: 'vpn' });
+
+    expect(listTrustedSourcesFor(db, FW)).toEqual([
+      { subnet: VPN, description: 'also vpn', registeredBy: 'other' },
+    ]);
+  });
+
+  it('withdrawing on one firewall leaves other firewalls alone', () => {
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [VPN], description: 'vpn' });
+    buildTrustedSourceStore(db, 'other').replace('10.0.0.1', {
+      subnets: [VPN],
+      description: 'elsewhere',
+    });
+
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [], description: 'vpn' });
 
     expect(buildTrustedSourceStore(db, 'wireguard').list(FW)).toEqual([]);
     expect(buildTrustedSourceStore(db, 'other').list('10.0.0.1')).toHaveLength(1);
@@ -69,9 +105,9 @@ describe('trusted-source store', () => {
   it('reads one firewall’s registrations without a module binding', () => {
     // Trust registered against one firewall is not trust granted by another —
     // the read is scoped, and the reader needs no identity to look.
-    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
-    buildTrustedSourceStore(db, 'other').add('10.0.0.1', {
-      subnet: '172.16.9.0/24',
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [VPN], description: 'vpn' });
+    buildTrustedSourceStore(db, 'other').replace('10.0.0.1', {
+      subnets: ['172.16.9.0/24'],
       description: 'elsewhere',
     });
 
@@ -80,7 +116,7 @@ describe('trusted-source store', () => {
   });
 
   it('reports every registration across firewalls, with who registered it', () => {
-    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, { subnets: [VPN], description: 'vpn' });
 
     expect(listAllTrustedSources(db)).toEqual([
       { firewallIp: FW, subnet: VPN, description: 'vpn', registeredBy: 'wireguard' },
@@ -104,7 +140,10 @@ describe('the render input excludes registrations; the reporting view includes t
     process.env.CELILO_DB_PATH = dbPath;
     db = await setupTestDatabase(dbPath);
     db.insert(systemConfig).values({ key: 'network.internal.subnet', value: CONTROL_PLANE }).run();
-    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'admin VPN' });
+    buildTrustedSourceStore(db, 'wireguard').replace(FW, {
+      subnets: [VPN],
+      description: 'admin VPN',
+    });
   });
   afterEach(() => {
     db.$client.close();

package/src/services/trusted-sources.ts

@@ -38,31 +38,41 @@ export function buildTrustedSourceStore(db: DbClient, registeredBy: string): Tru
         }));
     },
 
-    add(firewallIp: string, source: RegisterTrustedSourceRequest): void {
-      // Idempotent upsert on (firewall, subnet) — matches buildPortForwardStore.
+    replace(firewallIp: string, source: RegisterTrustedSourceRequest): void {
+      // The consumer's COMPLETE set for this firewall (D5b), scoped to its own
+      // rows: a subnet it trusted before and omits now loses its reach, and a
+      // subnet another module also trusts keeps it. Changing an admin VPN's
+      // client subnet used to leave the old one reaching every zone forever.
       db.delete(trustedSources)
         .where(
-          and(eq(trustedSources.firewallIp, firewallIp), eq(trustedSources.subnet, source.subnet)),
+          and(
+            eq(trustedSources.firewallIp, firewallIp),
+            eq(trustedSources.registeredBy, registeredBy),
+          ),
         )
         .run();
-      db.insert(trustedSources)
-        .values({
-          firewallIp,
-          subnet: source.subnet,
-          description: source.description,
-          registeredBy,
-        })
-        .run();
-    },
 
-    remove(firewallIp: string, subnet: string): void {
-      db.delete(trustedSources)
-        .where(and(eq(trustedSources.firewallIp, firewallIp), eq(trustedSources.subnet, subnet)))
+      if (source.subnets.length === 0) return;
+
+      db.insert(trustedSources)
+        .values(
+          source.subnets.map((subnet) => ({
+            firewallIp,
+            subnet,
+            description: source.description,
+            registeredBy,
+          })),
+        )
         .run();
     },
   };
 }
 
+/** Drop every trusted source a departing module owns, across every firewall. */
+export function deleteTrustedSourcesForModule(db: DbClient, moduleId: string): void {
+  db.delete(trustedSources).where(eq(trustedSources.registeredBy, moduleId)).run();
+}
+
 /** Where a trusted subnet came from — reach into every tier must be attributable. */
 export type TrustedSubnetOrigin = 'derived-control-plane' | 'registered' | 'operator-override';
 

package/src/templates/generator.ts

@@ -654,38 +654,58 @@ celilo module import modules/${moduleId}
  * @returns Generation result
  */
 /** Narrower than `GenerateResult`: this step produces no files, only an outcome. */
-export type DnsIngressResult = { success: true } | { success: false; error: string };
+export type IngressIpResult = { success: true } | { success: false; error: string };
 
 /**
- * Allocate-and-reserve the module's dedicated DNS-ingress IP, ONCE (ISS-0156).
+ * Allocate-and-reserve a module's dedicated `internal`-subnet ingress IPs, ONCE
+ * (ISS-0156, celilo#879).
  *
- * A `dns_internal` provider deploys into a PROTECTED zone (dmz) so it can see
- * protected-zone query sources for split-horizon views. `internal` devices have
- * no route into the 10-net, so they reach the resolver through a firewall DNAT
- * on a dedicated `internal`-subnet address. A module opts in by declaring a
- * `dns_ingress_ip` infrastructure variable.
+ * A service can need to live in a PROTECTED zone — the `dns_internal` resolver
+ * sits in `dmz` so it can see protected-zone query sources for split-horizon,
+ * and `caddy-internal` sits there so systems in the segmented zones can reach
+ * it. `internal` devices have no route into the 10-net, so they reach such a
+ * service through a firewall DNAT on a dedicated `internal`-subnet address.
+ * That DNAT is internal-side only: it is NOT a public port-forward, and the two
+ * are routinely confused.
+ *
+ * A module opts in by declaring an infrastructure variable whose name ends in
+ * `ingress_ip` — `dns_ingress_ip` for the resolver's `:53`, `ingress_ip` for a
+ * private web ingress's `:80/:443`. The hook then passes the stored value to
+ * `firewall.exposeService({ ingressIp })`.
  *
  * **Idempotence is the whole point, and it is load-bearing.** `module generate`
  * runs repeatedly over a module's life. Re-allocating here on the second run
- * would move the address internal clients use to reach DNS, every time — while
- * every command still reports success. That is why the stored value is reused
- * rather than re-derived, and why this is a named function instead of a branch
- * buried in `generateTemplates`: an invariant nothing can call is an invariant
- * nothing can test, and this one had no test at all.
+ * would move the address internal clients use to reach the service, every time
+ * — while every command still reports success. That is why the stored value is
+ * reused rather than re-derived, and why this is a named function instead of a
+ * branch buried in `generateTemplates`: an invariant nothing can call is an
+ * invariant nothing can test, and this one had no test at all.
  */
-export async function ensureDnsIngressIp(
+export async function ensureIngressIps(
   moduleId: string,
   manifest: ModuleManifest,
   db: DbClient,
-): Promise<DnsIngressResult> {
-  const wantsDnsIngress = manifest.variables?.owns?.some(
-    (v) => v.name === 'dns_ingress_ip' && v.source === 'infrastructure',
+): Promise<IngressIpResult> {
+  const wanted = (manifest.variables?.owns ?? []).filter(
+    (v) => v.name.endsWith('ingress_ip') && v.source === 'infrastructure',
   );
-  if (!wantsDnsIngress) return { success: true };
+  if (wanted.length === 0) return { success: true };
+
+  for (const variable of wanted) {
+    const result = await allocateIngressIp(moduleId, variable.name, db);
+    if (!result.success) return result;
+  }
+  return { success: true };
+}
 
-  const existing = getModuleConfigValue(moduleId, 'dns_ingress_ip', db)?.value;
+async function allocateIngressIp(
+  moduleId: string,
+  variableName: string,
+  db: DbClient,
+): Promise<IngressIpResult> {
+  const existing = getModuleConfigValue(moduleId, variableName, db)?.value;
   if (typeof existing === 'string' && existing.length > 0) {
-    log.success(`Using existing DNS-ingress IP ${existing} for ${moduleId}`);
+    log.success(`Using existing ingress IP ${existing} (${variableName}) for ${moduleId}`);
     return { success: true };
   }
 
@@ -695,9 +715,7 @@ export async function ensureDnsIngressIp(
   if (!subnetRow?.value) {
     return {
       success: false,
-      error:
-        'network.internal.subnet is not configured — required to allocate the ' +
-        'dns_internal DNS-ingress IP (ISS-0156). Ensure the internal network is set up first.',
+      error: `network.internal.subnet is not configured — required to allocate the ${variableName} ingress IP (ISS-0156). Ensure the internal network is set up first.`,
     };
   }
 
@@ -705,14 +723,14 @@ export async function ensureDnsIngressIp(
   const { stripCIDR } = await import('../ipam/subnet-parser');
   try {
     const ip = stripCIDR(await allocateIPFromSubnet(subnetRow.value, 'internal', db));
-    await reserveIP(ip, 'internal', `dns-ingress:${moduleId}`, null, db);
-    upsertModuleConfig(db, moduleId, 'dns_ingress_ip', ip);
-    log.success(`Allocated DNS-ingress IP ${ip} (internal subnet) for ${moduleId}`);
+    await reserveIP(ip, 'internal', `ingress:${moduleId}:${variableName}`, null, db);
+    upsertModuleConfig(db, moduleId, variableName, ip);
+    log.success(`Allocated ingress IP ${ip} (internal subnet, ${variableName}) for ${moduleId}`);
     return { success: true };
   } catch (error) {
     return {
       success: false,
-      error: `DNS-ingress IP allocation failed: ${error instanceof Error ? error.message : String(error)}`,
+      error: `Ingress IP allocation failed for ${variableName}: ${error instanceof Error ? error.message : String(error)}`,
     };
   }
 }
@@ -879,8 +897,8 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
     }
   }
 
-  const dnsIngress = await ensureDnsIngressIp(moduleId, manifest, db);
-  if (!dnsIngress.success) return dnsIngress;
+  const ingress = await ensureIngressIps(moduleId, manifest, db);
+  if (!ingress.success) return ingress;
 
   // Infrastructure Properties Resolution (Proxmox provider config)
   // For Proxmox services, extract provider config and store as temporary values

package/src/templates/{dns-ingress-ip.test.ts → ingress-ip.test.ts}

@@ -1,11 +1,11 @@
 /**
- * The DNS-ingress allocate-and-reserve guard (ISS-0156).
+ * The ingress-IP allocate-and-reserve guard (ISS-0156, celilo#879).
  *
  * This invariant had NO test. Losing it is not a crash: `module generate`
  * re-allocates a different address on every run, silently moving the address
- * internal clients use to reach DNS, while every command still reports success.
- * `module generate` runs repeatedly over a module's life, so "on the second
- * run" is the normal case, not an edge one.
+ * internal clients use to reach the service, while every command still reports
+ * success. `module generate` runs repeatedly over a module's life, so "on the
+ * second run" is the normal case, not an edge one.
  */
 
 import { beforeEach, describe, expect, test } from 'bun:test';
@@ -13,7 +13,7 @@ import type { DbClient } from '../db/client';
 import type { ModuleManifest } from '../manifest/schema';
 import { getModuleConfigValue } from '../services/module-config';
 import { setupTestDatabase } from '../test-utils/database';
-import { ensureDnsIngressIp } from './generator';
+import { ensureIngressIps } from './generator';
 
 let db: DbClient;
 
@@ -27,12 +27,17 @@ const operatorSupplied = {
   variables: { owns: [{ name: 'dns_ingress_ip', source: 'user_input' }] },
 } as unknown as ModuleManifest;
 
+/** How `caddy-internal` opts in — a web ingress rather than a DNS one. */
+const wantsWebIngress = {
+  variables: { owns: [{ name: 'ingress_ip', source: 'infrastructure' }] },
+} as unknown as ModuleManifest;
+
 /**
  * Typed as `string | undefined` rather than `unknown`: every assertion here is
  * about an address, and an untyped read pushes a cast onto each one.
  */
-const storedIp = (moduleId: string): string | undefined => {
-  const value = getModuleConfigValue(moduleId, 'dns_ingress_ip', db)?.value;
+const storedIp = (moduleId: string, variable = 'dns_ingress_ip'): string | undefined => {
+  const value = getModuleConfigValue(moduleId, variable, db)?.value;
   return typeof value === 'string' ? value : undefined;
 };
 
@@ -43,9 +48,9 @@ beforeEach(async () => {
     .run('network.internal.subnet', '10.226.1.0/24');
 });
 
-describe('ensureDnsIngressIp', () => {
+describe('ensureIngressIps', () => {
   test('allocates an address from the internal subnet on first generate', async () => {
-    const result = await ensureDnsIngressIp('technitium', wantsIngress, db);
+    const result = await ensureIngressIps('technitium', wantsIngress, db);
 
     expect(result.success).toBe(true);
     expect(storedIp('technitium')).toMatch(/^10\.226\.1\.\d+$/);
@@ -54,21 +59,21 @@ describe('ensureDnsIngressIp', () => {
   test('REUSES the same address on a second generate', async () => {
     // The guard itself. Re-allocating here moves the resolver's DNAT ingress
     // every time the module is regenerated, and nothing reports a problem.
-    await ensureDnsIngressIp('technitium', wantsIngress, db);
+    await ensureIngressIps('technitium', wantsIngress, db);
     const first = storedIp('technitium');
 
-    await ensureDnsIngressIp('technitium', wantsIngress, db);
+    await ensureIngressIps('technitium', wantsIngress, db);
     const second = storedIp('technitium');
 
     expect(second).toBe(first);
   });
 
   test('stays stable across many generates, not just two', async () => {
-    await ensureDnsIngressIp('technitium', wantsIngress, db);
+    await ensureIngressIps('technitium', wantsIngress, db);
     const first = storedIp('technitium');
 
     for (let i = 0; i < 5; i++) {
-      await ensureDnsIngressIp('technitium', wantsIngress, db);
+      await ensureIngressIps('technitium', wantsIngress, db);
     }
 
     expect(storedIp('technitium')).toBe(first);
@@ -77,7 +82,7 @@ describe('ensureDnsIngressIp', () => {
   test('RESERVES the address, so it is never handed out to something else', async () => {
     // Allocation without reservation is the same bug one step later: a
     // container gets the resolver's ingress address and DNS goes dark.
-    await ensureDnsIngressIp('technitium', wantsIngress, db);
+    await ensureIngressIps('technitium', wantsIngress, db);
     const ip = storedIp('technitium');
 
     const reserved = db.$client
@@ -85,20 +90,20 @@ describe('ensureDnsIngressIp', () => {
       .get(ip ?? '') as { ip_start: string; reason: string } | undefined;
 
     expect(reserved?.ip_start).toBe(ip);
-    // The reason names the owner, so an operator reading the table can tell
-    // what an otherwise anonymous held address is for.
-    expect(reserved?.reason).toBe('dns-ingress:technitium');
+    // The reason names the owner AND the variable, so an operator reading the
+    // table can tell what an otherwise anonymous held address is for.
+    expect(reserved?.reason).toBe('ingress:technitium:dns_ingress_ip');
   });
 
   test('two modules get two different addresses', async () => {
-    await ensureDnsIngressIp('technitium', wantsIngress, db);
-    await ensureDnsIngressIp('knot-unbound-internal', wantsIngress, db);
+    await ensureIngressIps('technitium', wantsIngress, db);
+    await ensureIngressIps('knot-unbound-internal', wantsIngress, db);
 
     expect(storedIp('knot-unbound-internal')).not.toBe(storedIp('technitium'));
   });
 
   test('does nothing for a module that never asked for one', async () => {
-    const result = await ensureDnsIngressIp('caddy', {} as ModuleManifest, db);
+    const result = await ensureIngressIps('caddy', {} as ModuleManifest, db);
 
     expect(result.success).toBe(true);
     expect(storedIp('caddy')).toBeUndefined();
@@ -107,7 +112,7 @@ describe('ensureDnsIngressIp', () => {
   test('only `source: infrastructure` opts in', async () => {
     // A same-named variable the operator supplies is theirs to set; allocating
     // over it would overwrite an operator's deliberate choice.
-    await ensureDnsIngressIp('technitium', operatorSupplied, db);
+    await ensureIngressIps('technitium', operatorSupplied, db);
 
     expect(storedIp('technitium')).toBeUndefined();
   });
@@ -115,9 +120,20 @@ describe('ensureDnsIngressIp', () => {
   test('fails with an actionable message when the internal subnet is unset', async () => {
     db.$client.prepare('DELETE FROM system_config WHERE key = ?').run('network.internal.subnet');
 
-    const result = await ensureDnsIngressIp('technitium', wantsIngress, db);
+    const result = await ensureIngressIps('technitium', wantsIngress, db);
 
     expect(result.success).toBe(false);
     expect(result.success === false && result.error).toContain('network.internal.subnet');
   });
+
+  // celilo#879. The opt-in used to be the literal name `dns_ingress_ip`, so a
+  // dmz-resident WEB ingress had no way to ask for the same treatment — which
+  // is how `caddy-internal` came to be pinned into the `internal` zone with a
+  // manifest comment claiming a dmz ingress could not be reached from a LAN.
+  test('a `ingress_ip` variable opts in the same way, for a non-DNS ingress', async () => {
+    const result = await ensureIngressIps('caddy-internal', wantsWebIngress, db);
+
+    expect(result.success).toBe(true);
+    expect(storedIp('caddy-internal', 'ingress_ip')).toMatch(/^10\.226\.1\.\d+$/);
+  });
 });

package/src/test-utils/cli-context.ts

@@ -351,9 +351,22 @@ export class CLIContext {
       new Promise<CommandResponse>((resolve, reject) => {
         this.pendingResponses.set(id, { resolve, reject });
       }),
-      // Timeout promise
+      // Timeout promise. Name the command and the actual elapsed time, not
+      // just the budget — a bare "timed out after 30000ms" reads as a hang
+      // in the command under test, indistinguishable from a real deploy
+      // defect. `Date.now() - startTime` at fire time is normally ~= timeout,
+      // but under CI load the event loop can be too busy to run this
+      // callback promptly, so a MUCH larger elapsed-than-budget is itself a
+      // load signal, not a fluke to explain away (celilo#804).
       new Promise<CommandResponse>((_, reject) =>
-        setTimeout(() => reject(new Error(`Command timed out after ${timeout}ms`)), timeout),
+        setTimeout(() => {
+          const elapsed = Date.now() - startTime;
+          reject(
+            new Error(
+              `Command #${id} "${command}" timed out after ${timeout}ms (elapsed ${elapsed}ms)`,
+            ),
+          );
+        }, timeout),
       ),
     ]);