@celilo/e2e 0.10.0 → 0.10.2

package/config/routing/fw-ext-routes.sh

@@ -13,7 +13,22 @@ ip route add 10.0.30.0/24 via 100.100.0.100       # secure via fw-isp
 
 # IP forwarding is set via sysctls in docker-compose
 
-# NAT/forward between isp-external and internet-external
+# NAT/forward between isp-external and internet-external.
+#
+# The customer's public /24 is ROUTED, never re-NAT'd. An ISP does not NAT a
+# subscriber that already holds a public address — the whole point of
+# 100.100.0.100 is that it IS the customer's address on the internet. fw-ext
+# used to MASQUERADE it anyway, so every packet leaving the fleet reached the
+# public simulators bearing fw-ext's own 100.64.0.1.
+#
+# That is not cosmetic. Namecheap-style DDNS registers the SOURCE address of
+# the update when the caller omits `ip=` (which is now how celilo registers
+# public names — #464/#466), so the double NAT published 100.64.0.1 for every
+# public hostname: an address that DNATs to nothing, breaking ACME validation
+# and any inbound reach. It also silently clobbered seeded apex records.
+# Anything that reasons about "the address the internet sees us as" was
+# measuring the simulator's own artefact.
+iptables -t nat -A POSTROUTING -s 100.100.0.0/24 -j RETURN
 iptables -t nat -A POSTROUTING -j MASQUERADE
 iptables -P FORWARD ACCEPT
 

package/config/routing/management-routes.sh

@@ -32,6 +32,11 @@ ip route add default via "$GATEWAY"
 ip route add 10.0.10.0/24 via "$FW_MAIN"    # dmz via fw-main
 ip route add 10.0.20.0/24 via "$FW_MAIN"    # app via fw-main
 ip route add 10.0.30.0/24 via "$FW_MAIN"    # secure via fw-main
+# Control plane. Only reachable when a secure-mgmt network exists — that is,
+# when celilo-mgr lives there itself, or when a module declaring
+# `zone: secure-mgmt` needs a system to land on (#436). Tolerated silently
+# otherwise so the default topology is unchanged.
+ip route add 10.0.120.0/24 via "$FW_MAIN" 2>/dev/null || true
 
 # DNS — Technitium internal resolver (split-horizon) with public fallback.
 # Technitium at 192.168.0.10 is authoritative for iamtheinternet.org internally,

package/config/routing/minio-startup.sh

@@ -7,6 +7,15 @@
 # `mc mb --ignore-existing`.
 set -e
 
+# Route the customer's public prefix via the ISP edge (fw-ext), like every
+# other host on internet-external. Docker's default bridge gateway has no path
+# across networks, so without this a reply to the customer's WAN address is
+# dropped and `backup --storage` hangs. fw-ext is per-test, so this is
+# expected to fail while shared infra runs alone — hence the tolerance.
+# (`busybox ip` because the MinIO image ships no iproute2.)
+busybox ip route del default 2>/dev/null || true
+busybox ip route add default via 100.64.0.1 2>/dev/null || true
+
 # Serve the S3 API on :80 so the management box reaches it over the SAME
 # proven HTTP egress path as the other public sims (apt-repo, npm-registry):
 # fw-ext's Squid-bypass exception is scoped to dport 80/443 → 100.64.0.0/24, so

package/config/routing/public-sim-entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Entrypoint wrapper for a simulator on `internet-external` — a host out on
+# the simulated public internet.
+#
+# Such a host must reach the customer's public prefix (100.100.0.0/24) through
+# the ISP edge, fw-ext. Docker's default bridge gateway has no path across
+# networks, so without this route a reply to the customer's WAN address is
+# simply dropped and the connection hangs.
+#
+# This used to be invisible: fw-ext MASQUERADEd the customer to its own
+# 100.64.0.1, which is on-link here, so replies never needed routing at all.
+# Once fw-ext stopped re-NATing a subscriber that already holds a public
+# address (the ISP routes it; it does not NAT it), the omission surfaced —
+# every sim that had this route already kept working, and the four that did
+# not started timing out. celilo-website-sim, isitup, pebble, namecheap-dns
+# and the DNS hierarchy have carried it all along; this is the same two lines.
+#
+# fw-ext is per-test, so the route is unreachable while shared infra runs
+# alone. That is harmless — nothing dials the customer before a test brings
+# fw-ext up — hence the tolerant `|| true`.
+ip route del default 2>/dev/null || true
+ip route add default via 100.64.0.1 2>/dev/null || true
+
+exec "$@"

package/docker/Dockerfile.apt-repo-sim

@@ -49,8 +49,16 @@ RUN set -eux; \
       -o APT::FTPArchive::Release::Architectures="amd64 arm64" \
       release dists/stable > dists/stable/Release
 
+# Route the customer's public prefix via the ISP edge, like every other host
+# on internet-external. See config/routing/public-sim-entrypoint.sh.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends iproute2 \
+ && rm -rf /var/lib/apt/lists/*
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
 EXPOSE 80
 
 # python3's http.server is enough for apt (GET + range requests over static
 # files). No special MIME/headers needed.
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
 CMD ["python3", "-m", "http.server", "80", "--directory", "/repo"]

package/docker/Dockerfile.cpanel-host-sim

@@ -67,4 +67,9 @@ EXPOSE 7822 80 443
 # which is what we want from a fixture host. The runtime/lock dirs live under
 # /var/run, which the container wipes at start, so recreate them before apache
 # reads DefaultRuntimeDir.
+# Route the customer's public prefix via the ISP edge, like every other host
+# on internet-external. See config/routing/public-sim-entrypoint.sh.
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
 CMD ["/bin/sh", "-c", "mkdir -p /var/run/apache2 /var/lock/apache2 /run/sshd && /usr/sbin/sshd && . /etc/apache2/envvars && exec apache2 -D FOREGROUND"]

package/docker/Dockerfile.minio

@@ -19,8 +19,15 @@
 
 FROM minio/mc:latest AS mc
 
+# Static busybox purely for its `ip` applet: the MinIO image has no iproute2
+# and no package manager worth invoking, but it still needs the ISP route
+# every host on internet-external carries (see minio-startup.sh).
+FROM busybox:stable-musl AS busybox
+
 FROM minio/minio:latest
 
+COPY --from=busybox /bin/busybox /usr/local/bin/busybox
+
 # Bring the MinIO client in from the mc image so the entrypoint can create the
 # bucket once the server is up.
 COPY --from=mc /usr/bin/mc /usr/bin/mc

package/docker/Dockerfile.npm-registry-sim

@@ -22,5 +22,13 @@ ENV PACKUMENT_DIR=/var/lib/npm \
     PORT=80 \
     REGISTRY_URL=http://npm-registry.lab
 
+# Route the customer's public prefix via the ISP edge, like every other host
+# on internet-external. See config/routing/public-sim-entrypoint.sh.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends iproute2 \
+ && rm -rf /var/lib/apt/lists/*
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
 EXPOSE 80
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
 CMD ["bun", "run", "src/server.ts"]

package/docker/Dockerfile.registry

@@ -30,5 +30,13 @@ ENV DATA_DIR=/var/lib/celilo-registry \
 # somewhere to put files before the server runs.
 RUN mkdir -p /uploads
 
+# Route the customer's public prefix via the ISP edge, like every other host
+# on internet-external. See config/routing/public-sim-entrypoint.sh.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends iproute2 \
+ && rm -rf /var/lib/apt/lists/*
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
 EXPOSE 80
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
 CMD ["bun", "run", "src/index.ts"]

package/docker/Dockerfile.signal-release

@@ -33,4 +33,12 @@ RUN mkdir -p "/out/v${SIGNAL_CLI_VERSION}" \
 
 FROM nginx:alpine
 COPY --from=fetch /out /usr/share/nginx/html
+
+# Route the customer's public prefix via the ISP edge, like every other host on
+# internet-external. See config/routing/public-sim-entrypoint.sh. (Alpine's
+# busybox already provides `ip`, so nothing to install.)
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
 EXPOSE 80
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@celilo/cli-display": "^0.1.9",
-    "@celilo/event-bus": "^0.1.8",
+    "@celilo/event-bus": "^0.1.9",
     "yaml": "^2.8.0",
     "zod": "^3.24.1"
   },

package/registry-server/src/bootstrap.ts

@@ -129,11 +129,15 @@ export function bootstrapIndexEntry(entry: BootstrapEntry): IndexEntry {
  * `walkDir`'s output conforms to the canonical rule. `walkDir` is exported for
  * that test.
  */
-export function walkDir(dir: string, base = dir): string[] {
+export function walkDir(dir: string, base = dir, inNodeModules = false): string[] {
   const results: string[] = [];
   for (const entry of readdirSync(dir, { withFileTypes: true })) {
     if (entry.name.startsWith('.')) continue;
     if (entry.name.endsWith('.test.ts')) continue;
+    // The module's own tsconfig.json is dev-only (CI typechecks hooks with it);
+    // build.ts drops it too. A dependency's tsconfig.json inside node_modules is
+    // that package's published content and stays — build.ts keeps those as well.
+    if (entry.name === 'tsconfig.json' && !inNodeModules) continue;
 
     const fullPath = join(dir, entry.name);
 
@@ -152,14 +156,14 @@ export function walkDir(dir: string, base = dir): string[] {
         // Other node_modules: collect only @celilo/capabilities.
         const capDir = join(fullPath, '@celilo', 'capabilities');
         if (existsSync(capDir)) {
-          results.push(...walkDir(capDir, base));
+          results.push(...walkDir(capDir, base, true));
         }
       }
       continue;
     }
 
     if (entry.isDirectory()) {
-      results.push(...walkDir(fullPath, base));
+      results.push(...walkDir(fullPath, base, inNodeModules));
     } else {
       results.push(relative(base, fullPath));
     }

package/simulators/signal-cli/server.ts

@@ -9,7 +9,10 @@
  *     that checks `response.ok` believes a rejected send succeeded;
  *   - an unlinked/revoked account answers HTTP perfectly and can send nothing;
  *   - `receive` returns receipts and typing indicators alongside real
- *     messages, and those are not replies.
+ *     messages, and those are not replies;
+ *   - `receive` is REFUSED unless the daemon was started with
+ *     `--receive-mode=manual`, which is the real default and the reason the
+ *     ack path shipped broken.
  *
  * It pushes back rather than saying yes. Sending to a number that has not been
  * registered with the simulator fails the way Signal fails.
@@ -36,6 +39,23 @@
 const ACCOUNT = process.env.SIGNAL_ACCOUNT ?? '+15551234567';
 const PORT = Number(process.env.SIGNAL_RPC_PORT ?? 8080);
 
+/**
+ * Mirrors `signal-cli daemon --receive-mode=<mode>`, and DEFAULTS TO THE REAL
+ * DEFAULT (`on-start`) rather than to the convenient one.
+ *
+ * In on-start the daemon registers a strong receive handler, drains every
+ * inbound message into its own SSE stream, and REFUSES the `receive` JSON-RPC
+ * call outright. A simulator that answers `receive` regardless is a placebo
+ * for the whole ack path: that is precisely how the deployed module shipped
+ * with an unreadable transport while this suite stayed green. Only `manual`
+ * lets `receive` work — so a caller has to be started the way it must be
+ * deployed.
+ *
+ * Verified in signal-cli 0.14.6: DaemonCommand.java:78, :114;
+ * ManagerImpl.java:1494-1501, :1605; ReceiveCommand.java:125.
+ */
+const RECEIVE_MODE = process.env.SIGNAL_RECEIVE_MODE ?? 'on-start';
+
 /** Numbers the simulator will accept as recipients. Anything else is rejected. */
 const KNOWN_RECIPIENTS = new Set(
   (process.env.SIGNAL_KNOWN_RECIPIENTS ?? '')
@@ -126,6 +146,18 @@ function handleSend(id: unknown, params: Record<string, unknown>): Response {
  * will misbehave on them.
  */
 function handleReceive(id: unknown): Response {
+  // The refusal comes BEFORE the link check, as it does in the real daemon:
+  // the receive thread is started per account at daemon start, so an
+  // already-receiving manager rejects the call without ever looking at what
+  // is queued. Verbatim message from ReceiveCommand.java:125.
+  if (RECEIVE_MODE !== 'manual') {
+    return rpcError(
+      id,
+      -32001,
+      'Receive command cannot be used if messages are already being received.',
+    );
+  }
+
   if (!linked) {
     return rpcError(id, -32000, 'Account is not registered or the device link was removed');
   }
@@ -142,13 +174,35 @@ function handleReceive(id: unknown): Response {
 
   while (inboundQueue.length > 0) {
     const item = inboundQueue.shift() as QueuedInbound;
-    envelopes.push({
-      envelope: {
-        sourceNumber: item.from,
-        timestamp: item.timestamp,
-        dataMessage: { message: item.body },
-      },
-    });
+    // A reply from the linked account itself — the default single-operator
+    // setup, where the route points at the number the transport is a
+    // secondary device of — is a NOTE-TO-SELF, and the real daemon delivers
+    // it as a sync transcript, never as a dataMessage. Emitting it as a
+    // dataMessage would let a client that only reads dataMessage look
+    // correct here while dropping every reply in production (#460).
+    envelopes.push(
+      item.from === ACCOUNT
+        ? {
+            envelope: {
+              sourceNumber: ACCOUNT,
+              timestamp: nextTimestamp(),
+              syncMessage: {
+                sentMessage: {
+                  destinationNumber: ACCOUNT,
+                  timestamp: item.timestamp,
+                  message: item.body,
+                },
+              },
+            },
+          }
+        : {
+            envelope: {
+              sourceNumber: item.from,
+              timestamp: item.timestamp,
+              dataMessage: { message: item.body },
+            },
+          },
+    );
     console.log(`[signal-sim] <- ${item.from}: ${item.body}`);
   }
 
@@ -261,7 +315,9 @@ Bun.serve({
   },
 });
 
-console.log(`[signal-sim] signal-cli JSON-RPC simulator on :${PORT} as ${ACCOUNT}`);
+console.log(
+  `[signal-sim] signal-cli JSON-RPC simulator on :${PORT} as ${ACCOUNT} (receive-mode=${RECEIVE_MODE})`,
+);
 if (KNOWN_RECIPIENTS.size > 0) {
   console.log(`[signal-sim] known recipients: ${[...KNOWN_RECIPIENTS].join(', ')}`);
 }

package/src/container-manager.ts

@@ -4,7 +4,11 @@ import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
 import { startBrowser } from './browser';
 import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
-import { SHARED_PROJECT_NAME, generateTestComposeYaml } from './docker-compose-generator';
+import {
+  SHARED_PROJECT_NAME,
+  generateTestComposeYaml,
+  getAllMachines,
+} from './docker-compose-generator';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
 import { startSocksProxy } from './socks-proxy';
@@ -67,8 +71,16 @@ const activeProjects: Set<string> = new Set();
  *
  * NOTE: only tears down per-test projects. Shared infra is managed
  * separately (by the test runner or explicit stopSharedInfra call).
+ *
+ * MUST stay fully synchronous: it runs from an `exit` handler, where the
+ * event loop is already closed and any async work is silently dropped.
  */
 function cleanupOnExit() {
+  // `--keep` / `--reuse` mean the operator wants the stack to survive for
+  // debugging — and they want it MOST after a failure, which is exactly the
+  // path this handler covers. Tearing down here would delete the evidence.
+  if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') return;
+
   for (const project of activeProjects) {
     try {
       execSync(`docker compose -p ${project} down --volumes --remove-orphans`, {
@@ -77,6 +89,7 @@ function cleanupOnExit() {
       });
     } catch {}
   }
+  activeProjects.clear();
   try {
     execSync('docker network prune -f', { timeout: 10_000, stdio: 'pipe' });
   } catch {}
@@ -90,7 +103,25 @@ process.on('SIGINT', () => {
   cleanupOnExit();
   process.exit(1);
 });
-process.on('beforeExit', () => {
+// `exit`, NOT `beforeExit`: the runner always terminates via `process.exit()`
+// (runner.ts ends with `process.exit(failed > 0 ? 1 : 0)`), and `beforeExit`
+// does not fire for an explicit exit — nor after an uncaught throw. So on the
+// FAILURE path, where the test never reaches `handle.stop()`, teardown was
+// never running and the per-test networks leaked.
+//
+// That leak is not confined to the run that caused it. The sim's networks use
+// the same subnets as a real celilo fleet (dmz 10.0.10.0/24, app 10.0.20.0/24,
+// …), so on a host that lives in those subnets — celilo's own forgejo-builder
+// does — a leaked bridge leaves a DUPLICATE route for a production prefix.
+// Traffic from any later container to the real host at that address is routed
+// into the dead simulated network and blackholed. That took out three
+// consecutive release runs, each dying in `git fetch` after a ~132s connect
+// timeout, long after the e2e run that stranded the network had ended.
+//
+// The start-of-run prefix sweep stays as the crash-recovery backstop (it is
+// the only thing that can clean up after SIGKILL); this makes the common case
+// clean up after itself instead of poisoning the next job. See #508.
+process.on('exit', () => {
   cleanupOnExit();
 });
 
@@ -829,13 +860,11 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     );
   }
 
-  // Wait for all target machines to complete setup (systemd boot + routing)
-  const allMachines = [
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-    ...config.internalMachines,
-  ];
+  // Wait for all target machines to complete setup (systemd boot + routing).
+  // Via getAllMachines so a newly-supported zone cannot be silently skipped —
+  // this list was hand-maintained and omitted `secure-mgmt`, so the first
+  // machine placed there raced `machine add` and failed on SSH intermittently.
+  const allMachines = getAllMachines(config);
   if (allMachines.length > 0) {
     console.log('[progress:start] waiting for target machines | target machines ready');
   }
@@ -936,8 +965,12 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
         network.internal.subnet=${ZONE_SUBNETS.internal} \
         network.internal.gateway=${ZONE_GATEWAYS.internal} \
         ${
-          config.managementZone === 'secure-mgmt'
-            ? 'network.secure-mgmt.subnet=10.0.120.0/24 network.secure-mgmt.gateway=10.0.120.1 '
+          // The control plane needs its subnet declared if celilo-mgr lives
+          // there OR if a machine does — infrastructure selection for a module
+          // declaring `zone: secure-mgmt` reads it either way (#436).
+          config.managementZone === 'secure-mgmt' ||
+          (config.secureMgmtMachines ?? []).length > 0
+            ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
         }dns.primary=100.100.0.1 \
         dns.fallback=1.0.0.1,8.8.8.8`,

package/src/docker-compose-generator.ts

@@ -374,6 +374,10 @@ export function generateSharedInfraYaml(): string {
  */
 export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: string): string {
   const mgmtOnOwnNetwork = config.managementZone === 'secure-mgmt';
+  // The control-plane network exists if celilo-mgr lives there OR if any
+  // machine does — a module declaring `zone: secure-mgmt` needs somewhere to
+  // land whether or not the management box shares the network (#436).
+  const needsSecureMgmt = mgmtOnOwnNetwork || (config.secureMgmtMachines ?? []).length > 0;
   const networks: Record<string, unknown> = {
     internal: networkDef('192.168.0.0/24', '192.168.0.250'),
     dmz: networkDef('10.0.10.0/24', '10.0.10.250'),
@@ -381,7 +385,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
     secure: networkDef('10.0.30.0/24', '10.0.30.250'),
     // Only when celilo-mgr lives off the internal LAN — keeps the default
     // topology's generated compose byte-identical for every existing test.
-    ...(mgmtOnOwnNetwork
+    ...(needsSecureMgmt
       ? { 'secure-mgmt': networkDef(SECURE_MGMT_SUBNET, '10.0.120.250') }
       : {}),
     'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
@@ -506,7 +510,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   // so the management box is routed like any other segmented network rather than
   // being stranded. Applied as a post-step so the default topology's generated
   // compose is untouched.
-  if (mgmtOnOwnNetwork) {
+  if (needsSecureMgmt) {
     const fwMain = services['fw-main'] as { networks?: Record<string, unknown> } | undefined;
     if (fwMain?.networks) {
       fwMain.networks['secure-mgmt'] = { ipv4_address: SECURE_MGMT_GATEWAY };
@@ -535,12 +539,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   });
 
   // --- Dynamic test machines ---
-  const allMachines = [
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-    ...config.internalMachines,
-  ];
+  const allMachines = getAllMachines(config);
 
   for (const machine of allMachines) {
     const zoneNetwork = machine.zone;
@@ -614,6 +613,11 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
       environment: {
         SIGNAL_ACCOUNT: '+15551234567',
         SIGNAL_KNOWN_RECIPIENTS: '+15550001,+15550002',
+        // Mirrors the flag the module's systemd unit must pass. Without it the
+        // simulator refuses `receive`, exactly as the real daemon does — which
+        // is the point: a test may only read replies from a daemon started the
+        // way celilo has to deploy it.
+        SIGNAL_RECEIVE_MODE: 'manual',
       },
     });
   }
@@ -795,12 +799,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     volumes: getRegistryVolumes(),
   });
 
-  const allMachines = [
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-    ...config.internalMachines,
-  ];
+  const allMachines = getAllMachines(config);
 
   for (const machine of allMachines) {
     const zoneNetwork = machine.zone;
@@ -847,5 +846,6 @@ export function getAllMachines(config: NetworkConfig): MachineSpec[] {
     ...config.appMachines,
     ...config.secureMachines,
     ...config.internalMachines,
+    ...(config.secureMgmtMachines ?? []),
   ];
 }

package/src/exit-cleanup.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Recurrence gate for #508: a failed e2e run must not leak its Docker networks.
+ *
+ * The sim's networks use the same subnets as a real celilo fleet, so on a host
+ * that lives in those subnets (celilo's forgejo-builder does) a stranded bridge
+ * leaves a duplicate route for a production prefix and blackholes later
+ * containers' traffic to the real address. Three consecutive release runs died
+ * that way — in `git fetch`, ~132s connect timeout — well after the e2e run
+ * that stranded the network had finished.
+ *
+ * The leak's mechanism was a wiring bug, so the gate is on the wiring.
+ */
+
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { expect, test } from 'bun:test';
+
+const SRC = readFileSync(
+  join(dirname(fileURLToPath(import.meta.url)), 'container-manager.ts'),
+  'utf-8',
+);
+
+test("teardown is wired to 'exit', which is the event an explicit process.exit() fires", () => {
+  // `beforeExit` alone was the bug: the runner ends with
+  // `process.exit(failed > 0 ? 1 : 0)`, so on the failure path — where the test
+  // never reaches handle.stop() — nothing ever tore the network down.
+  expect(SRC).toContain("process.on('exit'");
+});
+
+test('cleanup honors --keep / --reuse so a failed run stays debuggable', () => {
+  // Without this guard, adding the `exit` hook would destroy exactly the stack
+  // an operator asked to keep, since --keep matters most after a failure.
+  const body = SRC.slice(
+    SRC.indexOf('function cleanupOnExit'),
+    SRC.indexOf('process.on(\'SIGTERM\''),
+  );
+  expect(body).toContain('CELILO_E2E_KEEP');
+  expect(body).toContain('CELILO_E2E_REUSE');
+});
+
+test("the premise holds: process.exit() fires 'exit' but NOT 'beforeExit'", async () => {
+  // The whole fix rests on this runtime behavior. If it ever changes, the
+  // reasoning above is void and this should fail loudly rather than silently
+  // leaving the leak unfixed.
+  const proc = Bun.spawn([
+    'bun',
+    '-e',
+    "process.on('beforeExit', () => console.log('BEFORE_EXIT')); process.on('exit', () => console.log('EXIT')); process.exit(0);",
+  ]);
+  const out = await new Response(proc.stdout).text();
+  await proc.exited;
+
+  expect(out).toContain('EXIT');
+  expect(out).not.toContain('BEFORE_EXIT');
+});

package/src/index.ts

@@ -48,6 +48,8 @@ export type {
 } from './types';
 export {
   CeliloCommandError,
+  externalWanIp,
+  externalWanSubnet,
   greenwaveRouterIp,
   internalNatIp,
   internalResolverIp,
@@ -84,6 +86,11 @@ export {
   TECHNITIUM_DHCP_STACK,
 } from './fixtures';
 
+// Simulator addresses on `internet-external` — canonical for code, so a test
+// asserting against the authoritative DNS server never hardcodes its IP.
+export { SIMULATOR_IPS, SIMULATOR_IP_ENTRIES } from './simulator-ips';
+export type { SimulatorIpName } from './simulator-ips';
+
 // Shared infrastructure management (for test runners)
 export { ensureSharedInfra, stopSharedInfra, isSharedInfraRunning } from './shared-infra';
 

package/src/network-builder.ts

@@ -38,6 +38,7 @@ export class NetworkBuilder {
     appMachines: [],
     secureMachines: [],
     internalMachines: [],
+    secureMgmtMachines: [],
     dhcpClient: false,
     signalCli: false,
     signalSim: false,
@@ -94,6 +95,19 @@ export class NetworkBuilder {
     return this.addMachines(this.config.internalMachines, 'internal', machines);
   }
 
+  /**
+   * Add machines on celilo's control plane (`secure-mgmt`).
+   *
+   * For a module whose manifest declares `zone: secure-mgmt` — the signal
+   * transport does, because it holds the operator's Signal identity and must
+   * depend on nothing it reports about. Without this the module deploys onto
+   * whatever zone a test happened to add a machine in, which is the placement
+   * defect #435 is about, or fails selection outright.
+   */
+  secureMgmt(machines: Record<string, string>): this {
+    return this.addMachines(this.config.secureMgmtMachines, 'secure-mgmt', machines);
+  }
+
   /** Add internal-zone machines that need the Docker-capable image (e.g., Technitium) */
   internalDocker(machines: Record<string, string>): this {
     return this.addMachines(this.config.internalMachines, 'internal', machines, { docker: true });

package/src/public-sim-routes.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Recurrence gate: every simulator on `internet-external` must route the
+ * customer's public prefix via the ISP edge.
+ *
+ * `internet-external` models the public internet. A host there replies to a
+ * customer's public address through its ISP — modelled by defaulting via
+ * fw-ext (100.64.0.1). Docker's bridge gateway has no path across networks, so
+ * a simulator without that route silently blackholes every reply.
+ *
+ * This was invisible for as long as fw-ext MASQUERADEd the customer to its own
+ * on-link 100.64.0.1: nothing needed routing, so five simulators shipped
+ * without it. Removing that second NAT (an ISP routes a subscriber's public
+ * prefix, it does not NAT it) turned the omission into a hang — first in
+ * `bun add -g` against the sim npm registry, then in the signal module's
+ * release download. Both cost a full build-and-run cycle to find.
+ *
+ * A static check because the honest runtime one is impossible here: fw-ext is
+ * per-test, so at shared-infra readiness the route's next hop does not exist
+ * yet. Reading the compose the generator actually emits (rather than a
+ * hand-kept list) means a NEW simulator is covered the day it is added.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { parse } from 'yaml';
+import { generateSharedInfraYaml, generateTestComposeYaml } from './docker-compose-generator';
+import type { NetworkConfig } from './types';
+
+/** fw-ext's address on internet-external — the ISP edge every public host exits by. */
+const ISP_EDGE = '100.64.0.1';
+
+/** The one container that IS the edge, so it cannot route through itself. */
+const EDGE_SERVICE = 'fw-ext';
+
+interface ComposeService {
+  build?: { dockerfile?: string };
+  networks?: Record<string, unknown>;
+}
+
+/** Every optional simulator switched on, so none escapes the sweep. */
+const EVERYTHING: NetworkConfig = {
+  topology: 'default',
+  dmzMachines: [],
+  appMachines: [],
+  secureMachines: [],
+  internalMachines: [],
+  secureMgmtMachines: [],
+  domain: 'iamtheinternet.org',
+  ddnsPassword: 'test123',
+  verifyRouting: false,
+  managementVolumes: [],
+  dhcpClient: true,
+  signalCli: true,
+  signalSim: true,
+  signalRelease: true,
+};
+
+function publicSimulators(yaml: string): Array<{ name: string; dockerfile: string }> {
+  const compose = parse(yaml) as { services?: Record<string, ComposeService> };
+  return Object.entries(compose.services ?? {})
+    .filter(([name, svc]) => name !== EDGE_SERVICE && svc.networks?.['internet-external'])
+    .map(([name, svc]) => ({ name, dockerfile: svc.build?.dockerfile ?? '' }));
+}
+
+/**
+ * True if the image establishes the route — either through the shared
+ * entrypoint or by naming the ISP edge itself (a few sims predate the shared
+ * script and do it inline in their own entrypoint).
+ */
+function establishesIspRoute(dockerfile: string): boolean {
+  const e2eDir = join(import.meta.dir, '..');
+  const text = readFileSync(join(e2eDir, dockerfile), 'utf-8');
+  if (text.includes('public-sim-entrypoint.sh') || text.includes(ISP_EDGE)) return true;
+
+  // Otherwise the route may live in a script the image copies in. Follow every
+  // COPY source that looks like a script and check those too.
+  for (const match of text.matchAll(/^COPY\s+(?:--from=\S+\s+)?(\S+\.sh)\s/gm)) {
+    try {
+      if (readFileSync(join(e2eDir, match[1]), 'utf-8').includes(ISP_EDGE)) return true;
+    } catch {
+      // A COPY source outside the build context is not a route carrier.
+    }
+  }
+
+  // Some sims ship their entrypoint in a simulators/<name>/ directory copied
+  // wholesale; check the obvious sibling.
+  for (const match of text.matchAll(/^COPY\s+(?:--from=\S+\s+)?(simulators\/\S+)\s/gm)) {
+    try {
+      if (readFileSync(join(e2eDir, match[1], 'entrypoint.sh'), 'utf-8').includes(ISP_EDGE)) {
+        return true;
+      }
+    } catch {
+      // Not an entrypoint-bearing directory.
+    }
+  }
+  return false;
+}
+
+describe('public simulators route back through the ISP edge', () => {
+  const simulators = [
+    ...publicSimulators(generateSharedInfraYaml()),
+    ...publicSimulators(generateTestComposeYaml(EVERYTHING)),
+  ];
+
+  test('the sweep actually found simulators', () => {
+    // Guard against the check silently passing because the filter broke.
+    expect(simulators.length).toBeGreaterThan(5);
+  });
+
+  for (const { name, dockerfile } of simulators) {
+    test(`${name} routes the customer prefix via ${ISP_EDGE}`, () => {
+      expect(dockerfile).not.toBe('');
+      expect(establishesIspRoute(dockerfile)).toBe(true);
+    });
+  }
+});

package/src/types.ts

@@ -1,4 +1,13 @@
-export type Zone = 'dmz' | 'app' | 'secure' | 'internal';
+/**
+ * A placement zone a machine can be brought up in.
+ *
+ * `secure-mgmt` is celilo's own control plane, not a data-plane tier. It was
+ * expressible for the management box alone until a module declared
+ * `zone: secure-mgmt` (#436, the signal transport) and infrastructure
+ * selection had nowhere to put it — so a module that named the zone correctly
+ * could not be deployed by any test.
+ */
+export type Zone = 'dmz' | 'app' | 'secure' | 'internal' | 'secure-mgmt';
 
 /**
  * Where a network assertion is made FROM — named by the box's network location
@@ -38,6 +47,11 @@ export interface NetworkConfig {
   appMachines: MachineSpec[];
   secureMachines: MachineSpec[];
   internalMachines: MachineSpec[];
+  /**
+   * Machines on celilo's control-plane network. Bringing one up implies the
+   * `secure-mgmt` network exists, whether or not celilo-mgr itself sits there.
+   */
+  secureMgmtMachines: MachineSpec[];
   domain: string;
   ddnsPassword: string;
   verifyRouting: boolean;
@@ -398,6 +412,7 @@ export const ZONE_GATEWAYS: Record<Zone, string> = {
   app: '10.0.20.1',
   secure: '10.0.30.1',
   internal: '192.168.0.254',
+  'secure-mgmt': '10.0.120.1',
 };
 
 export const ZONE_SUBNETS: Record<Zone, string> = {
@@ -405,6 +420,7 @@ export const ZONE_SUBNETS: Record<Zone, string> = {
   app: '10.0.20.0/24',
   secure: '10.0.30.0/24',
   internal: '192.168.0.0/24',
+  'secure-mgmt': '10.0.120.0/24',
 };
 
 /**
@@ -428,6 +444,26 @@ export function greenwaveRouterIp(): string {
   return '192.168.0.1';
 }
 
+/**
+ * The customer firewall's EXTERNAL (WAN) address — what the public internet
+ * sees, and the only address a public A record for a fronted service may hold
+ * (CLAUDE.md inviolable rule #1). Also the source address a DDNS update
+ * acquires on its way out, which is what makes source-IP registration correct
+ * by construction. Reference this instead of hardcoding it.
+ */
+export function externalWanIp(): string {
+  return '100.100.0.100';
+}
+
+/**
+ * The customer's public /24 on `isp-external`. The ISP ROUTES this prefix; it
+ * never re-NATs it, so a packet leaving the fleet still carries the firewall's
+ * WAN address when it reaches a public service.
+ */
+export function externalWanSubnet(): string {
+  return '100.100.0.0/24';
+}
+
 /** The internal split-horizon DNS resolver (dns-int) in the e2e topology. */
 export function internalResolverIp(): string {
   return '192.168.0.10';