@celilo/e2e 0.10.3 → 0.11.0

package/src/address-plan.test.ts

@@ -0,0 +1,95 @@
+/**
+ * The recurrence gate for #539.
+ *
+ * The sim used to number its zones exactly like a real celilo fleet, so a
+ * leaked stack on celilo's own forgejo-builder (which lives at 10.0.10.12 in
+ * the REAL dmz) put a second interface on the builder's own /24 and blackholed
+ * every containerized CI job's route to the forge. These tests make the two
+ * properties that prevent it non-negotiable:
+ *
+ *   1. every simulated private zone lives inside SIM_PRIVATE_SUPERNET, and
+ *   2. the retired fleet-colliding prefixes appear NOWHERE in the suite.
+ *
+ * (2) is the one that matters in practice: the plan itself is one table, but
+ * shell scripts, Knot zones and dnsmasq configs can't import TypeScript and so
+ * hard-code the values. This is what catches "I changed it in one place".
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { execSync } from 'node:child_process';
+import { SIM_PRIVATE_SUPERNET, ZONE_GATEWAYS, ZONE_SUBNETS, type Zone } from './types';
+import { inCidr } from './zone-classifier';
+
+const REPO_ROOT = execSync('git rev-parse --show-toplevel', {
+  cwd: import.meta.dir,
+  encoding: 'utf-8',
+}).trim();
+
+/**
+ * Prefixes the sim must never use again: the fleet's zone /24s and the
+ * operator's LAN. Prefix match — `10.0.10.` catches an address, a CIDR and a
+ * prose mention alike, which is what we want.
+ */
+const RETIRED_PREFIXES = ['10.0.10.', '10.0.20.', '10.0.30.', '10.0.120.', '192.168.'];
+
+/**
+ * Match the plain form AND the regex-escaped form (`192\.168\.0\.10`). Two
+ * escaped literals inside `toThrow(/…/)` survived the #539 sweep precisely
+ * because a naive search for the plain text does not see them.
+ */
+function toPattern(prefix: string): string {
+  return prefix.replace(/\./g, '\\\\?\\.');
+}
+
+/** Everything that can put an address on a simulated network or assert one. */
+const SCANNED_PATHS = ['packages/e2e', 'e2e/tests', 'modules/*/e2e'];
+
+/**
+ * CHANGELOG is history — it describes what the addresses WERE. This test file
+ * necessarily names the retired prefixes to ban them.
+ */
+const EXCLUDED = [/^packages\/e2e\/CHANGELOG\.md$/, /address-plan\.test\.ts$/];
+
+function grepRetired(): string[] {
+  const pattern = RETIRED_PREFIXES.map(toPattern).join('|');
+  let out = '';
+  try {
+    out = execSync(`git grep -nE '${pattern}' -- ${SCANNED_PATHS.join(' ')}`, {
+      cwd: REPO_ROOT,
+      encoding: 'utf-8',
+      maxBuffer: 8 * 1024 * 1024,
+    });
+  } catch {
+    // git grep exits 1 when there are no matches — that is the passing case.
+    return [];
+  }
+  return out
+    .split('\n')
+    .filter(Boolean)
+    .filter((line) => !EXCLUDED.some((re) => re.test(line.split(':')[0])));
+}
+
+describe('simulated address plan (#539)', () => {
+  test('every zone lives inside the sim supernet', () => {
+    for (const zone of Object.keys(ZONE_SUBNETS) as Zone[]) {
+      const base = ZONE_SUBNETS[zone].split('/')[0];
+      expect(inCidr(base, SIM_PRIVATE_SUPERNET)).toBe(true);
+      expect(inCidr(ZONE_GATEWAYS[zone], SIM_PRIVATE_SUPERNET)).toBe(true);
+      expect(inCidr(ZONE_GATEWAYS[zone], ZONE_SUBNETS[zone])).toBe(true);
+    }
+  });
+
+  test('zones do not overlap each other', () => {
+    const subnets = Object.values(ZONE_SUBNETS);
+    expect(new Set(subnets).size).toBe(subnets.length);
+  });
+
+  test('the fleet-colliding prefixes appear nowhere in the suite', () => {
+    const hits = grepRetired();
+    expect(
+      hits,
+      `The e2e suite must not use a real fleet's addresses (#539). Offending lines:\n${hits.join('\n')}\n\n` +
+        `Use zoneIp()/ZONE_SUBNETS from packages/e2e/src/types.ts, or an address inside ${SIM_PRIVATE_SUPERNET}.`,
+    ).toEqual([]);
+  });
+});

package/src/cli/scaffold.ts

@@ -60,8 +60,8 @@ describe('${name}', () => {
     }
 
     net = await network({ projectName: PROJECT_NAME })
-      .dmz({ 'machine-1': '10.0.10.10' })
-      // .app({ 'app-1': '10.0.20.10' })
+      .dmz({ 'machine-1': '10.226.10.10' })
+      // .app({ 'app-1': '10.226.20.10' })
       .start();
   }, 120_000);
 

package/src/container-manager.ts

@@ -93,14 +93,23 @@ const activeProjects: Set<string> = new Set();
 export function projectTeardownCommands(project: string): {
   listContainers: string;
   listNetworks: string;
+  listVolumes: string;
 } {
   return {
     listContainers: `docker ps -aq --filter name=${project}`,
     listNetworks: `docker network ls --format {{.Name}} --filter name=${project}`,
+    // Volumes were missing here, and the compose path that would have removed
+    // them (`down --volumes`) is the same best-effort call that silently does
+    // nothing without a resolvable compose file. So every run leaked its
+    // volumes: 58 `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder,
+    // the oldest months old. Nothing breaks from a leaked volume the way a
+    // leaked network breaks routing — it just accumulates unbounded, and the
+    // pile is the evidence that teardown has not been running.
+    listVolumes: `docker volume ls -q --filter name=${project}`,
   };
 }
 
-/** Force-remove a project's containers and networks by name. Never throws. */
+/** Force-remove a project's containers, networks and volumes by name. Never throws. */
 function forceRemoveProject(project: string): void {
   const cmds = projectTeardownCommands(project);
   try {
@@ -123,6 +132,19 @@ function forceRemoveProject(project: string): void {
       } catch {}
     }
   } catch {}
+  // Volumes last: a volume still attached to a container cannot be removed, so
+  // this must follow the container sweep above.
+  try {
+    const vols = execSync(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
+      .toString()
+      .split('\n')
+      .filter(Boolean);
+    for (const vol of vols) {
+      try {
+        execSync(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch {}
+    }
+  } catch {}
 }
 
 function cleanupOnExit() {
@@ -164,18 +186,20 @@ process.on('SIGINT', () => {
 // FAILURE path, where the test never reaches `handle.stop()`, teardown was
 // never running and the per-test networks leaked.
 //
-// That leak is not confined to the run that caused it. The sim's networks use
-// the same subnets as a real celilo fleet (dmz 10.0.10.0/24, app 10.0.20.0/24,
-// …), so on a host that lives in those subnets — celilo's own forgejo-builder
-// does — a leaked bridge leaves a DUPLICATE route for a production prefix.
-// Traffic from any later container to the real host at that address is routed
+// That leak used not to be confined to the run that caused it. The sim's
+// networks were numbered exactly like a real celilo fleet's zones, so on a host
+// that lives in those subnets — celilo's own forgejo-builder does — a leaked
+// bridge left a DUPLICATE route for a production prefix, and
+// traffic from any later container to the real host at that address was routed
 // into the dead simulated network and blackholed. That took out three
 // consecutive release runs, each dying in `git fetch` after a ~132s connect
 // timeout, long after the e2e run that stranded the network had ended.
 //
-// The start-of-run prefix sweep stays as the crash-recovery backstop (it is
-// the only thing that can clean up after SIGKILL); this makes the common case
-// clean up after itself instead of poisoning the next job. See #508.
+// The sim now lives in SIM_PRIVATE_SUPERNET (#539), so a leak can no longer
+// collide with a fleet address — but a leak is still a leak. The start-of-run
+// prefix sweep stays as the crash-recovery backstop (it is the only thing that
+// can clean up after SIGKILL); this makes the common case clean up after itself
+// instead of leaving containers and volumes behind. See #508.
 process.on('exit', () => {
   cleanupOnExit();
 });
@@ -271,9 +295,7 @@ export async function scrubDnsZones(): Promise<void> {
   }
   if (served !== null && !served.split(/\s+/).includes(expected)) {
     throw new Error(
-      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} ` +
-        `(website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". ` +
-        `The DNS reset did not take effect — shared DNS would bleed across tests.`,
+      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} (website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". The DNS reset did not take effect — shared DNS would bleed across tests.`,
     );
   }
 }
@@ -330,9 +352,7 @@ function assertCliVersion(projectName: string, composeDir: string): void {
   const result = dockerExec(projectName, composeDir, 'management', 'celilo --version', 8_000);
   if (result.exitCode !== 0) {
     throw new Error(
-      `Could not read celilo version from the management image ` +
-        `(harness requires >=${MIN_CLI_VERSION}). ` +
-        `\`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
+      `Could not read celilo version from the management image (harness requires >=${MIN_CLI_VERSION}). \`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
     );
   }
   const problem = checkCliVersion(result.stdout);
@@ -564,11 +584,7 @@ function buildNetworkHandle(
       );
       // Spawn detached. nohup + & + disown cleanly survives the
       // dockerExec session ending.
-      const spawnCmd =
-        `nohup celilo events respond --values ${valuesPath} ` +
-        '--idle-timeout 1h --max-duration 2h ' +
-        '--emittedBy cele2e-responder ' +
-        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
+      const spawnCmd = `nohup celilo events respond --values ${valuesPath} --idle-timeout 1h --max-duration 2h --emittedBy cele2e-responder > /tmp/cele2e-responder.log 2>&1 < /dev/null & disown`;
       const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
       if (spawnResult.exitCode !== 0) {
         throw new Error(
@@ -772,9 +788,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     if (heavy.length > 0) {
       const names = heavy.map((c) => `   - ${c}`).join('\n');
       throw new Error(
-        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\n` +
-          'Live and e2e environments are mutually exclusive.\n' +
-          'Stop live containers first: cele2e down --all\n',
+        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\nLive and e2e environments are mutually exclusive.\nStop live containers first: cele2e down --all\n`,
       );
     }
   } catch (e) {
@@ -803,25 +817,33 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       }
     } catch {}
 
-    // Remove stale per-test networks
+    // Remove stale per-test networks AND their volumes, via the same
+    // by-name teardown the exit handler uses. This used to lean on
+    // `docker compose -p <project> down --volumes` for the volume half, which
+    // has no `-f` and no reliable cwd here either — compose exits "no
+    // configuration file provided" and the catch swallows it, so volumes were
+    // never removed on this path either. Reusing forceRemoveProject keeps the
+    // crash-recovery sweep and the exit handler from drifting apart.
     try {
       const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
         .split('\n')
         .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
-      if (networks.length > 0) {
-        const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
-        for (const project of projects) {
-          try {
-            run(`docker compose -p ${project} down --volumes --remove-orphans`, {
-              timeout: 30_000,
-            });
-          } catch {}
-        }
-        for (const net of networks) {
-          try {
-            run(`docker network rm ${net}`, { timeout: 5_000 });
-          } catch {}
-        }
+      const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
+      for (const project of projects) {
+        forceRemoveProject(project);
+      }
+    } catch {}
+    // Volumes can outlive every container and network of their project — that
+    // is exactly the 58-volume pile — so sweep by prefix too, not only for
+    // projects that still have a network to be discovered by.
+    try {
+      const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
+        .split('\n')
+        .filter(Boolean);
+      for (const vol of vols) {
+        try {
+          run(`docker volume rm ${vol}`, { timeout: 5_000 });
+        } catch {}
       }
     } catch {}
   } catch {}
@@ -941,6 +963,21 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       // in ~2-5s).
       60_000,
       `${machine.name} target-setup`,
+      // Self-diagnosing via waitFor's onTimeout (e2e-confidence #255). This
+      // timeout has twice been read as a simulator or deployment bug when the
+      // real cause was inside the unit — target-setup fetching from a slow
+      // third party (#560). The unit's own journal says which line it is on.
+      () => {
+        const dump = (label: string, cmd: string): string => {
+          const r = dockerExec(projectName, composeDir, machine.name, cmd);
+          return `--- ${label} ---\n${(r.stdout || r.stderr || '(no output)').trim()}`;
+        };
+        return [
+          `=== target-setup diagnostics (${machine.name}) ===`,
+          dump('systemctl status', 'systemctl status target-setup --no-pager 2>&1 | head -20'),
+          dump('journal', 'journalctl -u target-setup --no-pager 2>&1 | tail -30'),
+        ].join('\n');
+      },
     );
   }
 
@@ -1023,8 +1060,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
           // The control plane needs its subnet declared if celilo-mgr lives
           // there OR if a machine does — infrastructure selection for a module
           // declaring `zone: secure-mgmt` reads it either way (#436).
-          config.managementZone === 'secure-mgmt' ||
-          (config.secureMgmtMachines ?? []).length > 0
+          config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
             ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
         }dns.primary=100.100.0.1 \
@@ -1084,8 +1120,7 @@ export function reconnectNetwork(projectName: string): NetworkHandle {
     }
   } catch (err) {
     throw new Error(
-      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
-        `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
+      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\nFix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
     );
   }
 

package/src/docker-compose-generator.ts

@@ -3,8 +3,14 @@ import { join } from 'node:path';
 import { stringify } from 'yaml';
 import { normalizeObservers, observerEnv, observerPlacement } from './observer';
 import { SIMULATOR_IPS } from './simulator-ips';
-import type { MachineSpec, NetworkConfig, TopologyPreset } from './types';
-import { ZONE_GATEWAYS } from './types';
+import type { MachineSpec, NetworkConfig, TopologyPreset, Zone } from './types';
+import {
+  ZONE_GATEWAYS,
+  ZONE_SUBNETS,
+  greenwaveRouterIp,
+  internalResolverIp,
+  zoneIp,
+} from './types';
 
 /**
  * Volumes for the registry container. Picks between source-bootstrap and
@@ -52,6 +58,15 @@ function networkDef(subnet: string, gateway: string) {
   };
 }
 
+/**
+ * A zone's bridge, derived entirely from the address plan in `types.ts`. The
+ * bridge itself takes `.250` (Docker's own gateway address on the network — NOT
+ * the routing gateway, which is the firewall at `ZONE_GATEWAYS[zone]`).
+ */
+function zoneNetworkDef(zone: Zone) {
+  return networkDef(ZONE_SUBNETS[zone], zoneIp(zone, 250));
+}
+
 /**
  * Image tag prefix for pre-built e2e images. When images with these
  * tags exist locally, `docker compose build` skips rebuilding them.
@@ -117,19 +132,29 @@ export const SHARED_NETWORKS = {
  * where celilo-mgr itself lives, mirroring the production topology the suite
  * previously could not express.
  */
-const SECURE_MGMT_SUBNET = '10.0.120.0/24';
-const SECURE_MGMT_GATEWAY = '10.0.120.1';
-const SECURE_MGMT_MANAGEMENT_IP = '10.0.120.100';
+const SECURE_MGMT_GATEWAY = ZONE_GATEWAYS['secure-mgmt'];
+const SECURE_MGMT_MANAGEMENT_IP = zoneIp('secure-mgmt', 100);
+
+/**
+ * The zones fw-main routes between WITHOUT NAT, so the dmz-resident resolver
+ * sees each client's real zone (source-based split-horizon, ISS-0156).
+ * `internal` is deliberately absent: a LAN device's query must arrive SNAT'd to
+ * a zone-gateway address so it matches the passthrough view and gets the natIp
+ * answer, not the protected one.
+ */
+const PROTECTED_SUBNETS = (['dmz', 'app', 'secure', 'secure-mgmt'] as const)
+  .map((zone) => ZONE_SUBNETS[zone])
+  .join(' ');
 
 function defaultTopologyServices(): Record<string, unknown> {
   return {
     'fw-main': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.firewall' },
       networks: {
-        internal: { ipv4_address: '192.168.0.254' },
-        dmz: { ipv4_address: '10.0.10.1' },
-        app: { ipv4_address: '10.0.20.1' },
-        secure: { ipv4_address: '10.0.30.1' },
+        internal: { ipv4_address: ZONE_GATEWAYS.internal },
+        dmz: { ipv4_address: ZONE_GATEWAYS.dmz },
+        app: { ipv4_address: ZONE_GATEWAYS.app },
+        secure: { ipv4_address: ZONE_GATEWAYS.secure },
       },
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
       // A firewall may also terminate a VPN (the admin tunnel the `wireguard`
@@ -138,11 +163,12 @@ function defaultTopologyServices(): Record<string, unknown> {
       devices: ['/dev/net/tun'],
       sysctls: ROUTER_SYSCTLS,
       volumes: ['ssh-keys:/ssh-keys:ro'],
+      environment: { PROTECTED_SUBNETS },
     }),
     'fw-isp': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.greenwave-sim' },
       networks: {
-        internal: { ipv4_address: '192.168.0.1' },
+        internal: { ipv4_address: greenwaveRouterIp() },
         'isp-external': { ipv4_address: '100.100.0.100' },
       },
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
@@ -166,10 +192,10 @@ function directInternetTopologyServices(): Record<string, unknown> {
     'fw-main': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.firewall' },
       networks: {
-        internal: { ipv4_address: '192.168.0.254' },
-        dmz: { ipv4_address: '10.0.10.1' },
-        app: { ipv4_address: '10.0.20.1' },
-        secure: { ipv4_address: '10.0.30.1' },
+        internal: { ipv4_address: ZONE_GATEWAYS.internal },
+        dmz: { ipv4_address: ZONE_GATEWAYS.dmz },
+        app: { ipv4_address: ZONE_GATEWAYS.app },
+        secure: { ipv4_address: ZONE_GATEWAYS.secure },
         'isp-external': { ipv4_address: '100.100.0.100' },
       },
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
@@ -179,6 +205,7 @@ function directInternetTopologyServices(): Record<string, unknown> {
       devices: ['/dev/net/tun'],
       sysctls: ROUTER_SYSCTLS,
       volumes: ['ssh-keys:/ssh-keys:ro'],
+      environment: { PROTECTED_SUBNETS },
     }),
     // No fw-isp — fw-main handles external routing directly
   };
@@ -379,15 +406,13 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   // land whether or not the management box shares the network (#436).
   const needsSecureMgmt = mgmtOnOwnNetwork || (config.secureMgmtMachines ?? []).length > 0;
   const networks: Record<string, unknown> = {
-    internal: networkDef('192.168.0.0/24', '192.168.0.250'),
-    dmz: networkDef('10.0.10.0/24', '10.0.10.250'),
-    app: networkDef('10.0.20.0/24', '10.0.20.250'),
-    secure: networkDef('10.0.30.0/24', '10.0.30.250'),
+    internal: zoneNetworkDef('internal'),
+    dmz: zoneNetworkDef('dmz'),
+    app: zoneNetworkDef('app'),
+    secure: zoneNetworkDef('secure'),
     // Only when celilo-mgr lives off the internal LAN — keeps the default
     // topology's generated compose byte-identical for every existing test.
-    ...(needsSecureMgmt
-      ? { 'secure-mgmt': networkDef(SECURE_MGMT_SUBNET, '10.0.120.250') }
-      : {}),
+    ...(needsSecureMgmt ? { 'secure-mgmt': zoneNetworkDef('secure-mgmt') } : {}),
     'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
     // internet-external is owned by shared infra; real-internet is per-test
     // (it just bridges fw-ext ↔ comcast-resolver, no shared service uses it).
@@ -411,7 +436,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
 
   // --- Management (always present) ---
   const managementDefaultGw =
-    config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
+    config.topology === 'direct-internet' ? ZONE_GATEWAYS.internal : greenwaveRouterIp();
 
   // Pick the management image based on variant. Both branches use
   // `image:` (not `build:`) so docker compose uses the existing tag —
@@ -476,24 +501,27 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
 
   services.management = mgmtOnOwnNetwork
     ? emitManagement(config.managementVariant, SECURE_MGMT_MANAGEMENT_IP, 'secure-mgmt')
-    : emitManagement(config.managementVariant, '192.168.0.100');
+    : emitManagement(config.managementVariant, zoneIp('internal', 100));
 
   if (config.secondaryManagementVariant) {
-    services['celilo-mgr-2'] = emitManagement(config.secondaryManagementVariant, '192.168.0.101');
+    services['celilo-mgr-2'] = emitManagement(
+      config.secondaryManagementVariant,
+      zoneIp('internal', 101),
+    );
   }
 
   // --- Internal split-horizon DNS resolver ---
   // Resolves the test domain to caddy's internal DMZ IP so management can
   // reach it via a direct route rather than through the external interface
   // (hairpin NAT would be needed otherwise). Management already has
-  // nameserver 192.168.0.10 as its primary resolver in /etc/resolv.conf.
+  // `internalResolverIp()` as its primary resolver in /etc/resolv.conf.
   const caddyIp =
     config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??
     config.dmzMachines[0]?.ip ??
-    '10.0.10.10';
+    zoneIp('dmz', 10);
   services['dns-int'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.internal-resolver' },
-    networks: { internal: { ipv4_address: '192.168.0.10' } },
+    networks: { internal: { ipv4_address: internalResolverIp() } },
     cap_add: ['NET_ADMIN'],
     environment: {
       CADDY_IP: caddyIp,
@@ -574,7 +602,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   if (config.dhcpClient) {
     services['dhcp-client'] = baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.dhcp-client' },
-      networks: { internal: { ipv4_address: '192.168.0.210' } },
+      networks: { internal: { ipv4_address: zoneIp('internal', 210) } },
       cap_add: ['NET_ADMIN'],
     });
   }
@@ -586,7 +614,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   if (config.signalCli) {
     services['signal-cli'] = baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.signal-cli' },
-      networks: { internal: { ipv4_address: '192.168.0.90' } },
+      networks: { internal: { ipv4_address: zoneIp('internal', 90) } },
       cap_add: ['NET_ADMIN'],
     });
   }
@@ -608,7 +636,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   if (config.signalSim) {
     services['signal-sim'] = baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.signal-sim' },
-      networks: { internal: { ipv4_address: '192.168.0.91' } },
+      networks: { internal: { ipv4_address: zoneIp('internal', 91) } },
       cap_add: ['NET_ADMIN'],
       environment: {
         SIGNAL_ACCOUNT: '+15551234567',
@@ -649,10 +677,10 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
 
 export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): string {
   const networks: Record<string, unknown> = {
-    internal: networkDef('192.168.0.0/24', '192.168.0.250'),
-    dmz: networkDef('10.0.10.0/24', '10.0.10.250'),
-    app: networkDef('10.0.20.0/24', '10.0.20.250'),
-    secure: networkDef('10.0.30.0/24', '10.0.30.250'),
+    internal: zoneNetworkDef('internal'),
+    dmz: zoneNetworkDef('dmz'),
+    app: zoneNetworkDef('app'),
+    secure: zoneNetworkDef('secure'),
     'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
     'internet-external': networkDef('100.64.0.0/24', '100.64.0.250'),
     'real-internet': {
@@ -668,11 +696,11 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   const services: Record<string, unknown> = {};
 
   const managementDefaultGw =
-    config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
+    config.topology === 'direct-internet' ? ZONE_GATEWAYS.internal : greenwaveRouterIp();
 
   services.management = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.management' },
-    networks: { internal: { ipv4_address: '192.168.0.100' } },
+    networks: { internal: { ipv4_address: zoneIp('internal', 100) } },
     cap_add: ['NET_ADMIN'],
     volumes: [
       ...(celiloRoot ? [`${celiloRoot}:/celilo`] : []),
@@ -692,10 +720,10 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   const legacyCaddyIp =
     config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??
     config.dmzMachines[0]?.ip ??
-    '10.0.10.10';
+    zoneIp('dmz', 10);
   services['dns-int'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.internal-resolver' },
-    networks: { internal: { ipv4_address: '192.168.0.10' } },
+    networks: { internal: { ipv4_address: internalResolverIp() } },
     cap_add: ['NET_ADMIN'],
     environment: {
       CADDY_IP: legacyCaddyIp,
@@ -831,7 +859,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   if (config.dhcpClient) {
     services['dhcp-client'] = baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.dhcp-client' },
-      networks: { internal: { ipv4_address: '192.168.0.210' } },
+      networks: { internal: { ipv4_address: zoneIp('internal', 210) } },
       cap_add: ['NET_ADMIN'],
     });
   }

package/src/exit-cleanup.test.ts

@@ -11,10 +11,10 @@
  * The leak's mechanism was a wiring bug, so the gate is on the wiring.
  */
 
+import { expect, test } from 'bun:test';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { expect, test } from 'bun:test';
 import { projectTeardownCommands } from './container-manager';
 
 const SRC = readFileSync(
@@ -34,7 +34,7 @@ test('cleanup honors --keep / --reuse so a failed run stays debuggable', () => {
   // an operator asked to keep, since --keep matters most after a failure.
   const body = SRC.slice(
     SRC.indexOf('function cleanupOnExit'),
-    SRC.indexOf('process.on(\'SIGTERM\''),
+    SRC.indexOf("process.on('SIGTERM'"),
   );
   expect(body).toContain('CELILO_E2E_KEEP');
   expect(body).toContain('CELILO_E2E_REUSE');
@@ -54,6 +54,20 @@ test('teardown does not depend on a compose file being findable', () => {
   }
 });
 
+// Containers and networks were removed; volumes were not. The only thing that
+// would have removed them was `docker compose down --volumes`, which is the
+// same best-effort call that silently does nothing without a resolvable compose
+// file — so in practice they were never removed at all. 58 leaked
+// `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder, the oldest
+// months old. A leaked volume breaks nothing on its own, which is why it went
+// unnoticed for months; the pile is the evidence teardown was not running.
+test('teardown removes volumes, not just containers and networks', () => {
+  const cmds = projectTeardownCommands('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).toContain('docker volume ls');
+  expect(cmds.listVolumes).toContain('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).not.toContain('docker compose');
+});
+
 test('teardown cannot match the shared project', () => {
   // Per-test projects carry a timestamp; `celilo-e2e-shared` does not, so a
   // name filter for one can never match the other. Shared infra must survive —

package/src/fixtures.ts

@@ -1,14 +1,14 @@
 import { NetworkBuilder, network } from './network-builder';
 
 /** Single caddy machine in the DMZ — basic web server deployment */
-export const CADDY_DEPLOYMENT = (): NetworkBuilder => network().dmz({ caddy: '10.0.10.10' });
+export const CADDY_DEPLOYMENT = (): NetworkBuilder => network().dmz({ caddy: '10.226.10.10' });
 
 /** Full stack: web server, identity provider, database */
 export const FULL_STACK = (): NetworkBuilder =>
   network()
-    .dmz({ caddy: '10.0.10.10' })
-    .app({ idp: '10.0.20.100' })
-    .secure({ db: '10.0.30.50' });
+    .dmz({ caddy: '10.226.10.10' })
+    .app({ idp: '10.226.20.100' })
+    .secure({ db: '10.226.30.50' });
 
 /** Minimal: no dynamic machines, just fixed infrastructure */
 export const INFRASTRUCTURE_ONLY = (): NetworkBuilder => network();
@@ -16,25 +16,25 @@ export const INFRASTRUCTURE_ONLY = (): NetworkBuilder => network();
 /** Caddy + Authentik: web server in DMZ, identity provider in app zone */
 export const AUTHENTIK_DEPLOYMENT = (): NetworkBuilder =>
   network()
-    .dmz({ caddy: '10.0.10.10' })
-    .app({ authentik: '10.0.20.30' });
+    .dmz({ caddy: '10.226.10.10' })
+    .app({ authentik: '10.226.20.30' });
 
 /** Multiple DMZ machines for testing multi-service deployments */
 export const MULTI_DMZ = (): NetworkBuilder =>
   network().dmz({
-    caddy: '10.0.10.10',
-    'web-app': '10.0.10.20',
+    caddy: '10.226.10.10',
+    'web-app': '10.226.10.20',
   });
 
 /** Technitium DNS + Caddy: internal DNS resolver + full web stack */
 export const TECHNITIUM_STACK = (): NetworkBuilder =>
   network()
-    .internal({ 'dns-int': '192.168.0.53' })
-    .dmz({ caddy: '10.0.10.10' });
+    .internal({ 'dns-int': '10.226.1.53' })
+    .dmz({ caddy: '10.226.10.10' });
 
 /** Technitium DNS with DHCP client: tests DHCP DNS propagation */
 export const TECHNITIUM_DHCP_STACK = (): NetworkBuilder =>
   network()
-    .internal({ 'dns-int': '192.168.0.53' })
-    .dmz({ caddy: '10.0.10.10' })
+    .internal({ 'dns-int': '10.226.1.53' })
+    .dmz({ caddy: '10.226.10.10' })
     .withDhcpClient();