@celilo/e2e 0.8.0 → 0.9.1

package/src/container-manager.ts

@@ -1,8 +1,9 @@
 import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
-import { copyFileSync, existsSync, writeFileSync } from 'node:fs';
+import { existsSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
 import { startBrowser } from './browser';
+import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
 import { SHARED_PROJECT_NAME, generateTestComposeYaml } from './docker-compose-generator';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
@@ -16,7 +17,7 @@ import type {
   ProxyOptions,
   SocksProxyHandle,
 } from './types';
-import { CeliloCommandError, internalNatIp, ZONE_GATEWAYS, ZONE_SUBNETS } from './types';
+import { CeliloCommandError, ZONE_GATEWAYS, ZONE_SUBNETS, internalNatIp } from './types';
 
 /** Package root — where docker/, config/, simulators/ live */
 const PACKAGE_ROOT = join(__dirname, '..');
@@ -126,45 +127,68 @@ const SHARED_CONTAINERS = new Set([
 ]);
 
 /**
- * Scrub DNS zone files to baseline templates before a test starts.
+ * Reset the SHARED authoritative DNS (namecheap-dns) to its pristine seed
+ * before a test starts, so a prior test's DDNS writes don't bleed forward.
  *
- * Copies slim template zones (containing only @, ns1, www) over the
- * host-side zone files (which are bind-mounted into namecheap-dns),
- * then reloads Knot to pick up the changes. This prevents DDNS-written
- * subdomains from one test leaking into the next.
+ * namecheap-dns is SHARED across the whole suite. Knot serves the WRITABLE
+ * `/config/<zone>.zone` copies, which the DDNS simulator mutates at runtime;
+ * `/seed/<zone>.zone` is the read-only pristine mount, copied to `/config` once
+ * by namecheap-startup.sh at container start. So the only correct reset is to
+ * re-copy `/seed -> /config` inside the container and reload Knot.
  *
- * Called automatically by startNetwork unless config.skipDnsScrub is
- * set (useful for --keep debugging).
+ * History: the previous implementation rewrote the HOST-side zone file (which
+ * is bind-mounted to `/seed`, not `/config`) and reloaded — Knot never re-read
+ * it, so the scrub was a silent no-op. iamtheinternet.org survived only because
+ * every test re-registers its own hosts; celilo.computer's apex is never
+ * re-registered, so once a celilo.computer-managed test clobbered it (apex ->
+ * the fw-ext source IP instead of the 100.64.0.58 website-sim) it stayed broken
+ * for the rest of the run — silently failing install-sh. (ISS: cele2e DNS bleed.)
+ *
+ * Called automatically by startNetwork unless config.skipDnsScrub is set
+ * (useful for --keep debugging).
+ *
+ * Exported for the effect-proving integration test (e2e-confidence #256).
  */
-async function scrubDnsZones(): Promise<void> {
-  const zones = ['iamtheinternet.org', 'example.net'];
-  const composeDir = PACKAGE_ROOT;
-
-  for (const zone of zones) {
-    const templatePath = join(PACKAGE_ROOT, 'config', 'dns', 'templates', `${zone}.zone`);
-    const liveZonePath = join(PACKAGE_ROOT, 'config', 'dns', `${zone}.zone`);
+export async function scrubDnsZones(): Promise<void> {
+  const exec = (cmd: string): string =>
+    run(
+      `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns sh -c ${JSON.stringify(
+        cmd,
+      )}`,
+      { cwd: PACKAGE_ROOT },
+    );
 
-    try {
-      // Copy template over the host-side zone file. The file is
-      // bind-mounted into the container, so overwriting the host file
-      // makes the new content available inside.
-      copyFileSync(templatePath, liveZonePath);
-    } catch (err) {
-      // Non-fatal: log and continue. The scrub is a hygiene measure;
-      // if it fails (template missing, permissions), tests can still run
-      // with stale DNS state.
-      console.warn(`[dns-scrub] Failed to copy ${zone} template: ${err}`);
-    }
+  try {
+    exec('cp -f /seed/*.zone /config/ && knotc zone-reload');
+  } catch (err) {
+    // Non-fatal hygiene measure: log and continue. A failed reset just means a
+    // test may see stale DNS — better than aborting the run.
+    console.warn(`[dns-scrub] Failed to reset zones to seed: ${err}`);
+    return;
   }
 
-  // Reload all zones in one shot so Knot re-reads from the bind-mount
+  // Verify the reset ACTUALLY took effect by querying the RUNNING authoritative
+  // server (not a file): the old scrub silently wrote /seed while Knot served
+  // /config, so a file check would have looked fine while the server served
+  // stale records. Assert celilo.computer's apex now resolves to the website-sim
+  // and throw loudly on a confirmed mismatch, so a broken reset can never again
+  // silently bleed DNS state across tests. (e2e-confidence #253.)
+  const expected = SIMULATOR_IPS.WEBSITE;
+  let served: string | null = null;
   try {
-    run(
-      `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns knotc zone-reload`,
-      { cwd: composeDir },
-    );
+    served = exec('kdig @127.0.0.1 celilo.computer A +short').trim();
   } catch (err) {
-    console.warn(`[dns-scrub] Failed to reload zones: ${err}`);
+    // Couldn't run the verification query (e.g. the query tool is unavailable).
+    // Warn rather than abort the suite over a missing diagnostic — we only fail
+    // on a CONFIRMED wrong answer below.
+    console.warn(`[dns-scrub] could not verify reset (DNS query failed): ${err}`);
+  }
+  if (served !== null && !served.split(/\s+/).includes(expected)) {
+    throw new Error(
+      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} ` +
+        `(website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". ` +
+        `The DNS reset did not take effect — shared DNS would bleed across tests.`,
+    );
   }
 }
 
@@ -187,15 +211,48 @@ function dockerExec(
     );
     return { stdout, stderr: '', exitCode: 0 };
   } catch (err: unknown) {
-    const e = err as { stdout?: string; stderr?: string; status?: number };
+    const e = err as {
+      stdout?: string;
+      stderr?: string;
+      status?: number;
+      code?: string;
+      signal?: string;
+      killed?: boolean;
+    };
+    // execSync's timeout surfaces as an opaque error (ETIMEDOUT / SIGTERM /
+    // Bun's "canceled") with empty stderr — the classic e2e time-sink where a
+    // hung command reads as a mystery. Replace it with what actually happened.
+    const timedOut = e.code === 'ETIMEDOUT' || e.signal === 'SIGTERM' || e.killed === true;
+    const stderr = timedOut
+      ? `timed out after ${Math.round(timeoutMs / 1000)}s running: ${cmd}`
+      : (e.stderr?.toString() ?? '');
     return {
       stdout: e.stdout?.toString() ?? '',
-      stderr: e.stderr?.toString() ?? '',
-      exitCode: e.status ?? 1,
+      stderr,
+      exitCode: e.status ?? (timedOut ? 124 : 1),
     };
   }
 }
 
+/**
+ * Fail fast if the management image's baked celilo CLI is older than the
+ * harness needs (ce-5qp). Runs before `system init` so a version mismatch
+ * surfaces in <10s with an actionable message instead of a 90s hang ending
+ * in "canceled".
+ */
+function assertCliVersion(projectName: string, composeDir: string): void {
+  const result = dockerExec(projectName, composeDir, 'management', 'celilo --version', 8_000);
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `Could not read celilo version from the management image ` +
+        `(harness requires >=${MIN_CLI_VERSION}). ` +
+        `\`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
+    );
+  }
+  const problem = checkCliVersion(result.stdout);
+  if (problem) throw new Error(problem);
+}
+
 function dockerExecAsync(
   projectName: string,
   composeDir: string,
@@ -255,6 +312,7 @@ async function waitFor(
   check: () => Promise<boolean>,
   timeoutMs: number,
   label: string,
+  onTimeout?: () => string | Promise<string>,
 ): Promise<void> {
   const start = Date.now();
   while (Date.now() - start < timeoutMs) {
@@ -265,7 +323,20 @@ async function waitFor(
     }
     await new Promise((r) => setTimeout(r, 2000));
   }
-  throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms`);
+  // Self-diagnosing timeout (e2e-confidence #255): a readiness wait must attach
+  // EVIDENCE, never speculate ("X likely stalled or Y is down"). When the caller
+  // supplies an onTimeout collector, capture the live state and append it so the
+  // failure pinpoints the layer on first occurrence — no hand-instrumentation
+  // after the fact.
+  let diagnostics = '';
+  if (onTimeout) {
+    try {
+      diagnostics = `\n${await onTimeout()}`;
+    } catch (err) {
+      diagnostics = `\n(onTimeout diagnostics failed: ${err instanceof Error ? err.message : String(err)})`;
+    }
+  }
+  throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms${diagnostics}`);
 }
 
 /**
@@ -328,369 +399,111 @@ async function streamingBuild(
   });
 }
 
-export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
-  // Pre-flight: fail if live containers are running that will compete for resources.
-  try {
-    const running = run('docker ps --format "{{.Names}}" 2>/dev/null').split('\n').filter(Boolean);
-    const heavyPatterns = ['authentik-', 'caddy-', 'build-your-own-internet-'];
-    const heavy = running.filter((name) => heavyPatterns.some((p) => name.startsWith(p)));
-    if (heavy.length > 0) {
-      const names = heavy.map((c) => `   - ${c}`).join('\n');
-      throw new Error(
-        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\n` +
-          'Live and e2e environments are mutually exclusive.\n' +
-          'Stop live containers first: cele2e down --all\n',
-      );
-    }
-  } catch (e) {
-    if (e instanceof Error && e.message.includes('Cannot start e2e tests')) throw e;
-  }
+/**
+ * Build the NetworkHandle shared by startNetwork (fresh network) and
+ * reconnectNetwork (--keep reuse). Both expose the identical method set;
+ * only the setup that precedes handle creation differs, so the handle
+ * methods live here once instead of being duplicated in both functions.
+ */
+function buildNetworkHandle(
+  projectName: string,
+  composeDir: string,
+  celiloRoot: string | undefined,
+): NetworkHandle {
+  // Track host-side resources spawned via the handle (SOCKS proxies,
+  // Playwright browsers). stop() tears these down before the docker
+  // network itself.
+  const activeProxies: SocksProxyHandle[] = [];
+  const activeBrowsers: BrowserHandle[] = [];
 
-  // Ensure shared infrastructure (DNS, Pebble, registry, etc.) is running.
-  // This is idempotent — if already up, it just verifies health.
-  await ensureSharedInfra();
+  const handle: NetworkHandle = {
+    projectName,
 
-  // Scrub DNS zone files to baseline before each test (H5 in E2E_TEST_UPDATES.md).
-  // Prevents DDNS-written records from one test leaking into the next.
-  // Tests can opt out with config.skipDnsScrub (e.g., for --reuse debugging).
-  if (!config.skipDnsScrub) {
-    await scrubDnsZones();
-  }
+    async celilo(
+      cmd: string,
+      optsOrTimeout?: number | { check?: boolean; timeoutMs?: number },
+    ): Promise<ExecResult> {
+      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
+      const timeoutMs =
+        typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);
 
-  // Clean up stale per-test containers (self-healing from prior crashes)
-  console.log('[progress:start] cleaning up stale test resources | cleanup complete');
-  try {
-    // Force-kill per-test containers (not shared infra)
-    try {
-      const containers = run('docker ps -aq --filter "name=celilo-e2e-1" 2>/dev/null');
-      if (containers.trim()) {
-        run(`docker rm -f ${containers.replace(/\n/g, ' ')}`, { timeout: 30_000 });
-      }
-    } catch {}
+      const result = await dockerExecAsync(
+        projectName,
+        composeDir,
+        'management',
+        `celilo ${cmd}`,
+        timeoutMs,
+      );
 
-    // Remove stale per-test networks
-    try {
-      const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
-        .split('\n')
-        .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
-      if (networks.length > 0) {
-        const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
-        for (const project of projects) {
-          try {
-            run(`docker compose -p ${project} down --volumes --remove-orphans`, {
-              timeout: 30_000,
-            });
-          } catch {}
-        }
-        for (const net of networks) {
-          try {
-            run(`docker network rm ${net}`, { timeout: 5_000 });
-          } catch {}
-        }
+      if (check && result.exitCode !== 0) {
+        throw new CeliloCommandError(cmd, result);
       }
-    } catch {}
-  } catch {}
-
-  const projectName = `celilo-e2e-${Date.now()}`;
-  activeProjects.add(projectName);
-
-  const composeDir = PACKAGE_ROOT;
-  const celiloRoot = config.celiloRoot ?? findCeliloRoot();
-
-  // Generate per-test compose (references shared infra networks as external)
-  const yaml = generateTestComposeYaml(config, celiloRoot);
-  writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
-
-  // Build and start per-test containers only — stream output so the runner
-  // can show which service is currently being built (can take minutes on cold cache).
-  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
+      return result;
+    },
 
-  console.log('[progress:start] starting containers | containers running');
-  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, {
-    cwd: composeDir,
-    timeout: 120_000,
-  });
+    exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
+      return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
+    },
 
-  // Wait for DNS convergence (comcast-resolver is per-test, needs to reach shared DNS)
-  console.log('[progress:start] waiting for DNS convergence | DNS converged');
-  await waitFor(
-    async () => {
-      const result = dockerExec(
+    async respondWith(values): Promise<void> {
+      // Write the values JSON inside the management container at a
+      // stable path, then start a detached `celilo events respond
+      // --values <path>` process. The responder polls the in-container
+      // bus and answers config/secret/ensure events as the deploy
+      // emits them. The responder dies with the container at network
+      // teardown, so no explicit cleanup hook is needed.
+      //
+      // Long timeouts: the responder's --idle-timeout has to outlast
+      // any quiet stretch between deploys (a long `module deploy`
+      // might not emit interview events for several minutes if it's
+      // building images). 1h idle / 2h max-duration covers any single
+      // test comfortably without leaking. The container goes away at
+      // network stop regardless.
+      const valuesJson = JSON.stringify(values).replace(/'/g, "'\\''");
+      const valuesPath = '/tmp/cele2e-responder-values.json';
+      const writeCmd = `printf '%s' '${valuesJson}' > ${valuesPath}`;
+      const writeResult = dockerExec(projectName, composeDir, 'management', writeCmd, 10_000);
+      if (writeResult.exitCode !== 0) {
+        throw new Error(
+          `respondWith: failed to write values file inside management container: ${writeResult.stderr}`,
+        );
+      }
+      // Kill any prior responder (idempotent: replaces prior values map).
+      dockerExec(
         projectName,
         composeDir,
-        'comcast-resolver',
-        'dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2',
+        'management',
+        'pkill -f "celilo events respond" 2>/dev/null || true',
+        5_000,
       );
-      return result.exitCode === 0 && result.stdout.trim().length > 0;
-    },
-    60_000,
-    'DNS convergence',
-  );
-
-  // Wait for internal resolver (split-horizon DNS for management container).
-  // Skipped when the test specifies its own machine named `dns-int` — that
-  // means the test is replacing the infra resolver with a module under test
-  // (e.g., knot-unbound-internal). The bare target machine has no DNS server
-  // until its module deploys, so this wait would always time out. Management
-  // can still resolve names via the fallback `nameserver 100.100.0.1` in
-  // its resolv.conf during deploy.
-  //
-  // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
-  // places the dns_internal provider in a PROTECTED zone (dmz) so it can see
-  // protected-zone query sources for split-horizon views. So check every zone's
-  // machines for a `dns-int`, not only internalMachines.
-  const dnsIntReplaced = [
-    ...config.internalMachines,
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-  ].some((m) => m.name === 'dns-int');
-  if (!dnsIntReplaced) {
-    console.log('[progress:start] waiting for internal resolver | internal resolver ready');
-    await waitFor(
-      async () => {
-        const result = dockerExec(
-          projectName,
-          composeDir,
-          'dns-int',
-          'dig @127.0.0.1 iamtheinternet.org A +short +timeout=2',
+      // Spawn detached. nohup + & + disown cleanly survives the
+      // dockerExec session ending.
+      const spawnCmd =
+        `nohup celilo events respond --values ${valuesPath} ` +
+        '--idle-timeout 1h --max-duration 2h ' +
+        '--emittedBy cele2e-responder ' +
+        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
+      const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
+      if (spawnResult.exitCode !== 0) {
+        throw new Error(
+          `respondWith: failed to spawn responder inside management container: ${spawnResult.stderr}`,
         );
-        return result.exitCode === 0 && result.stdout.trim().length > 0;
-      },
-      30_000,
-      'internal DNS resolver',
-    );
-  }
+      }
+      // Brief settle so the responder has registered its watches
+      // before any deploy fires events. Without this, a fast deploy
+      // could emit before the responder polls.
+      await new Promise((r) => setTimeout(r, 500));
+    },
 
-  // Optional routing verification
-  if (config.verifyRouting) {
-    console.log('[progress:start] verifying routing | routing verified');
-    await waitFor(
-      async () => {
-        const result = dockerExec(
-          projectName,
-          composeDir,
-          'management',
-          `ping -c1 -W2 ${SIMULATOR_IPS.ROOT_DNS}`,
-        );
-        return result.exitCode === 0;
-      },
-      30_000,
-      'routing verification',
-    );
-  }
-
-  // Wait for all target machines to complete setup (systemd boot + routing)
-  const allMachines = [
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-    ...config.internalMachines,
-  ];
-  if (allMachines.length > 0) {
-    console.log('[progress:start] waiting for target machines | target machines ready');
-  }
-  for (const machine of allMachines) {
-    await waitFor(
-      async () => {
-        const result = dockerExec(
-          projectName,
-          composeDir,
-          machine.name,
-          'systemctl is-active target-setup 2>/dev/null',
-        );
-        return result.stdout.trim() === 'active';
-      },
-      // App-zone machines use Dockerfile.target-machine-docker which bakes in
-      // dockerd; systemd boot + dockerd init takes ~25-35s, leaving a tight
-      // margin against a 30s timeout. 60s covers the observed worst case
-      // without slowing down dmz/internal-only tests (which still hit this
-      // in ~2-5s).
-      60_000,
-      `${machine.name} target-setup`,
-    );
-  }
-
-  // Wait for DHCP client lease
-  if (config.dhcpClient) {
-    console.log('[progress:start] waiting for DHCP client lease | DHCP lease acquired');
-    await waitFor(
-      async () => {
-        const result = dockerExec(
-          projectName,
-          composeDir,
-          'dhcp-client',
-          'test -f /var/lib/dhcp/dhclient.leases && grep -c lease /var/lib/dhcp/dhclient.leases',
-        );
-        return result.exitCode === 0 && Number.parseInt(result.stdout.trim()) > 0;
-      },
-      60_000,
-      'DHCP client lease',
-    );
-  }
-
-  // Wait for routing to Pebble (through shared infra networks)
-  if (allMachines.length > 0) {
-    await waitFor(
-      async () => {
-        const result = dockerExec(
-          projectName,
-          composeDir,
-          'management',
-          `ping -c1 -W2 ${SIMULATOR_IPS.PEBBLE}`,
-        );
-        return result.exitCode === 0;
-      },
-      30_000,
-      'routing to Pebble',
-    );
-  }
-
-  // Initialize celilo. Skipped for the vanilla management variant —
-  // there's no celilo binary baked into that image; the caller is
-  // responsible for installing it (typically by running install.sh)
-  // and then calling `celilo system init` themselves.
-  if (config.managementVariant !== 'vanilla') {
-    console.log('[progress:start] initializing celilo | celilo initialized');
-    // Honest harness (v2/NETWORK_CONFIG_TO_FIREWALL.md): seed ONLY the
-    // `internal` zone — the network the management box is genuinely on —
-    // plus DNS. dmz/app/secure are NOT pre-seeded; they come into being
-    // when a test deploys the firewall (net.deployFirewall()), exactly as
-    // in production. A test that places services in those zones must call
-    // deployFirewall first; internal-only tests need nothing more.
-    const initResult = dockerExec(
-      projectName,
-      composeDir,
-      'management',
-      `celilo system init --accept-defaults \
-        network.internal.subnet=${ZONE_SUBNETS.internal} \
-        network.internal.gateway=${ZONE_GATEWAYS.internal} \
-        dns.primary=100.100.0.1 \
-        dns.fallback=1.0.0.1,8.8.8.8`,
-    );
-    if (initResult.exitCode !== 0) {
-      throw new Error(`Celilo init failed: ${initResult.stderr}`);
-    }
-
-    // Start the event-bus dispatcher (ISS-0035 / ISS-0042), now that `system
-    // init` has created the bus DB. Deploys emit bus events —
-    // public_web.routes_changed (caddy route reconcile), system.created (DNS
-    // providers) — that only take effect when a dispatcher delivers them.
-    // Production runs this as a systemd unit; the e2e mgmt box has no systemd,
-    // so run it detached (nohup + disown, like the cele2e responder). One
-    // dispatcher per management container serves every deploy in the test, so
-    // individual tests don't need to start their own (delivery claims are
-    // atomic, so a test that still does won't double-deliver).
-    console.log('[progress:start] starting event dispatcher | dispatcher running');
-    const dispatcherResult = dockerExec(
-      projectName,
-      composeDir,
-      'management',
-      'nohup celilo events run --poll-ms 500 --concurrency 4 > /tmp/cele2e-dispatcher.log 2>&1 < /dev/null & disown',
-    );
-    if (dispatcherResult.exitCode !== 0) {
-      throw new Error(`Failed to start event dispatcher: ${dispatcherResult.stderr}`);
-    }
-  }
-
-  console.log('[progress:done] network ready');
-  // Emit project name so the runner can persist it if --keep is set
-  console.log(`[e2e:project] ${projectName}`);
-
-  // Track host-side resources spawned via the handle (SOCKS proxies,
-  // Playwright browsers). stop() tears these down before the docker
-  // network itself.
-  const activeProxies: SocksProxyHandle[] = [];
-  const activeBrowsers: BrowserHandle[] = [];
-
-  const handle: NetworkHandle = {
-    projectName,
-
-    async celilo(
-      cmd: string,
-      optsOrTimeout?: number | { check?: boolean; timeoutMs?: number },
-    ): Promise<ExecResult> {
-      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
-      const timeoutMs =
-        typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);
-
-      const result = await dockerExecAsync(
-        projectName,
-        composeDir,
-        'management',
-        `celilo ${cmd}`,
-        timeoutMs,
-      );
-
-      if (check && result.exitCode !== 0) {
-        throw new CeliloCommandError(cmd, result);
-      }
-      return result;
-    },
-
-    exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
-      return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
-    },
-
-    async respondWith(values): Promise<void> {
-      // Write the values JSON inside the management container at a
-      // stable path, then start a detached `celilo events respond
-      // --values <path>` process. The responder polls the in-container
-      // bus and answers config/secret/ensure events as the deploy
-      // emits them. The responder dies with the container at network
-      // teardown, so no explicit cleanup hook is needed.
-      //
-      // Long timeouts: the responder's --idle-timeout has to outlast
-      // any quiet stretch between deploys (a long `module deploy`
-      // might not emit interview events for several minutes if it's
-      // building images). 1h idle / 2h max-duration covers any single
-      // test comfortably without leaking. The container goes away at
-      // network stop regardless.
-      const valuesJson = JSON.stringify(values).replace(/'/g, "'\\''");
-      const valuesPath = '/tmp/cele2e-responder-values.json';
-      const writeCmd = `printf '%s' '${valuesJson}' > ${valuesPath}`;
-      const writeResult = dockerExec(projectName, composeDir, 'management', writeCmd, 10_000);
-      if (writeResult.exitCode !== 0) {
-        throw new Error(
-          `respondWith: failed to write values file inside management container: ${writeResult.stderr}`,
-        );
-      }
-      // Kill any prior responder (idempotent: replaces prior values map).
-      dockerExec(
-        projectName,
-        composeDir,
-        'management',
-        'pkill -f "celilo events respond" 2>/dev/null || true',
-        5_000,
-      );
-      // Spawn detached. nohup + & + disown cleanly survives the
-      // dockerExec session ending.
-      const spawnCmd =
-        `nohup celilo events respond --values ${valuesPath} ` +
-        '--idle-timeout 1h --max-duration 2h ' +
-        '--emittedBy cele2e-responder ' +
-        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
-      const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
-      if (spawnResult.exitCode !== 0) {
-        throw new Error(
-          `respondWith: failed to spawn responder inside management container: ${spawnResult.stderr}`,
-        );
-      }
-      // Brief settle so the responder has registered its watches
-      // before any deploy fires events. Without this, a fast deploy
-      // could emit before the responder polls.
-      await new Promise((r) => setTimeout(r, 500));
-    },
-
-    async deployFirewall(opts = {}): Promise<void> {
-      const zones = opts.zones ?? ['dmz', 'app', 'secure'];
-      const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
-      const natIp = opts.natIp ?? internalNatIp();
-      const providedNetworks = zones.map((zone) => ({
-        zone,
-        subnet: ZONE_SUBNETS[zone],
-        gateway: ZONE_GATEWAYS[zone],
-      }));
+    async deployFirewall(opts = {}): Promise<void> {
+      const zones = opts.zones ?? ['dmz', 'app', 'secure'];
+      const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
+      const natIp = opts.natIp ?? internalNatIp();
+      const providedNetworks = zones.map((zone) => ({
+        zone,
+        subnet: ZONE_SUBNETS[zone],
+        gateway: ZONE_GATEWAYS[zone],
+      }));
 
       // fw-main is a firewall container, not a target machine, so the
       // network readiness wait (target-setup) does NOT cover its sshd. Poll
@@ -738,12 +551,12 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     },
 
     async browser(options: BrowserOptions): Promise<BrowserHandle> {
-      const handle = await startBrowser(projectName, options);
-      activeBrowsers.push(handle);
+      const browserHandle = await startBrowser(projectName, options);
+      activeBrowsers.push(browserHandle);
       // browser owns its proxy lifecycle; track it here too so stop()
       // doesn't try to stop it twice (idempotent stop() handles this)
-      activeProxies.push(handle.proxy);
-      return handle;
+      activeProxies.push(browserHandle.proxy);
+      return browserHandle;
     },
 
     async dig(name: string): Promise<string> {
@@ -864,257 +677,329 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   return handle;
 }
 
-/**
- * Reconnect to an existing test network (previously kept alive with --keep).
- * Returns a NetworkHandle pointing to the existing Docker project. Skips
- * all the setup (image build, container start, celilo init) — assumes the
- * network is already running and healthy.
- *
- * If the project doesn't exist, throws with an actionable error.
- */
-export function reconnectNetwork(projectName: string): NetworkHandle {
-  const composeDir = PACKAGE_ROOT;
-
-  // Verify the project actually exists
+export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
+  // Pre-flight: fail if live containers are running that will compete for resources.
   try {
-    const result = run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} ps -q`, {
-      cwd: composeDir,
-      timeout: 10_000,
-    });
-    if (!result.trim()) {
-      throw new Error(`No containers found for project ${projectName}`);
+    const running = run('docker ps --format "{{.Names}}" 2>/dev/null').split('\n').filter(Boolean);
+    const heavyPatterns = ['authentik-', 'caddy-', 'build-your-own-internet-'];
+    const heavy = running.filter((name) => heavyPatterns.some((p) => name.startsWith(p)));
+    if (heavy.length > 0) {
+      const names = heavy.map((c) => `   - ${c}`).join('\n');
+      throw new Error(
+        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\n` +
+          'Live and e2e environments are mutually exclusive.\n' +
+          'Stop live containers first: cele2e down --all\n',
+      );
     }
-  } catch (err) {
-    throw new Error(
-      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
-        `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
-    );
+  } catch (e) {
+    if (e instanceof Error && e.message.includes('Cannot start e2e tests')) throw e;
   }
 
-  activeProjects.add(projectName);
-  console.log(`[progress:done] reconnected to existing network (${projectName})`);
-  console.log(`[e2e:project] ${projectName}`);
-
-  // Track host-side resources spawned via the handle (mirrors startNetwork).
-  const activeProxies: SocksProxyHandle[] = [];
-  const activeBrowsers: BrowserHandle[] = [];
-
-  const handle: NetworkHandle = {
-    projectName,
+  // Ensure shared infrastructure (DNS, Pebble, registry, etc.) is running.
+  // This is idempotent — if already up, it just verifies health.
+  await ensureSharedInfra();
 
-    async celilo(
-      cmd: string,
-      optsOrTimeout?: number | { check?: boolean; timeoutMs?: number },
-    ): Promise<ExecResult> {
-      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
-      const timeoutMs =
-        typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);
+  // Scrub DNS zone files to baseline before each test (H5 in E2E_TEST_UPDATES.md).
+  // Prevents DDNS-written records from one test leaking into the next.
+  // Tests can opt out with config.skipDnsScrub (e.g., for --reuse debugging).
+  if (!config.skipDnsScrub) {
+    await scrubDnsZones();
+  }
 
-      const result = await dockerExecAsync(
-        projectName,
-        composeDir,
-        'management',
-        `celilo ${cmd}`,
-        timeoutMs,
-      );
+  // Clean up stale per-test containers (self-healing from prior crashes)
+  console.log('[progress:start] cleaning up stale test resources | cleanup complete');
+  try {
+    // Force-kill per-test containers (not shared infra)
+    try {
+      const containers = run('docker ps -aq --filter "name=celilo-e2e-1" 2>/dev/null');
+      if (containers.trim()) {
+        run(`docker rm -f ${containers.replace(/\n/g, ' ')}`, { timeout: 30_000 });
+      }
+    } catch {}
 
-      if (check && result.exitCode !== 0) {
-        throw new CeliloCommandError(cmd, result);
+    // Remove stale per-test networks
+    try {
+      const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
+        .split('\n')
+        .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
+      if (networks.length > 0) {
+        const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
+        for (const project of projects) {
+          try {
+            run(`docker compose -p ${project} down --volumes --remove-orphans`, {
+              timeout: 30_000,
+            });
+          } catch {}
+        }
+        for (const net of networks) {
+          try {
+            run(`docker network rm ${net}`, { timeout: 5_000 });
+          } catch {}
+        }
       }
-      return result;
-    },
+    } catch {}
+  } catch {}
 
-    exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
-      return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
-    },
+  const projectName = `celilo-e2e-${Date.now()}`;
+  activeProjects.add(projectName);
 
-    async respondWith(values): Promise<void> {
-      // Write the values JSON inside the management container at a
-      // stable path, then start a detached `celilo events respond
-      // --values <path>` process. The responder polls the in-container
-      // bus and answers config/secret/ensure events as the deploy
-      // emits them. The responder dies with the container at network
-      // teardown, so no explicit cleanup hook is needed.
-      //
-      // Long timeouts: the responder's --idle-timeout has to outlast
-      // any quiet stretch between deploys (a long `module deploy`
-      // might not emit interview events for several minutes if it's
-      // building images). 1h idle / 2h max-duration covers any single
-      // test comfortably without leaking. The container goes away at
-      // network stop regardless.
-      const valuesJson = JSON.stringify(values).replace(/'/g, "'\\''");
-      const valuesPath = '/tmp/cele2e-responder-values.json';
-      const writeCmd = `printf '%s' '${valuesJson}' > ${valuesPath}`;
-      const writeResult = dockerExec(projectName, composeDir, 'management', writeCmd, 10_000);
-      if (writeResult.exitCode !== 0) {
-        throw new Error(
-          `respondWith: failed to write values file inside management container: ${writeResult.stderr}`,
-        );
-      }
-      // Kill any prior responder (idempotent: replaces prior values map).
-      dockerExec(
-        projectName,
-        composeDir,
-        'management',
-        'pkill -f "celilo events respond" 2>/dev/null || true',
-        5_000,
-      );
-      // Spawn detached. nohup + & + disown cleanly survives the
-      // dockerExec session ending.
-      const spawnCmd =
-        `nohup celilo events respond --values ${valuesPath} ` +
-        '--idle-timeout 1h --max-duration 2h ' +
-        '--emittedBy cele2e-responder ' +
-        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
-      const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
-      if (spawnResult.exitCode !== 0) {
-        throw new Error(
-          `respondWith: failed to spawn responder inside management container: ${spawnResult.stderr}`,
-        );
-      }
-      // Brief settle so the responder has registered its watches
-      // before any deploy fires events. Without this, a fast deploy
-      // could emit before the responder polls.
-      await new Promise((r) => setTimeout(r, 500));
-    },
+  const composeDir = PACKAGE_ROOT;
+  const celiloRoot = config.celiloRoot ?? findCeliloRoot();
 
-    async deployFirewall(opts = {}): Promise<void> {
-      const zones = opts.zones ?? ['dmz', 'app', 'secure'];
-      const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
-      const natIp = opts.natIp ?? internalNatIp();
-      const providedNetworks = zones.map((zone) => ({
-        zone,
-        subnet: ZONE_SUBNETS[zone],
-        gateway: ZONE_GATEWAYS[zone],
-      }));
+  // Generate per-test compose (references shared infra networks as external)
+  const yaml = generateTestComposeYaml(config, celiloRoot);
+  writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
-      await handle.celilo(
-        `machine add ${firewallIp} --ssh-user root --ssh-key-file /root/.ssh/id_ed25519 --zone internal`,
-      );
-      await handle.celilo('module import iptables');
-      await handle.celilo(`module config set iptables firewall_ip ${firewallIp}`);
-      await handle.celilo(`module config set iptables nat_ip ${natIp}`);
-      await handle.celilo(
-        `module config set iptables zones '${JSON.stringify(['internal', ...zones])}'`,
-      );
-      await handle.celilo(
-        `module config set iptables provided_networks '${JSON.stringify(providedNetworks)}'`,
+  // Build and start per-test containers only — stream output so the runner
+  // can show which service is currently being built (can take minutes on cold cache).
+  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
+
+  console.log('[progress:start] starting containers | containers running');
+  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, {
+    cwd: composeDir,
+    timeout: 120_000,
+  });
+
+  // Wait for DNS convergence (comcast-resolver is per-test, needs to reach shared DNS)
+  console.log('[progress:start] waiting for DNS convergence | DNS converged');
+  await waitFor(
+    async () => {
+      const result = dockerExec(
+        projectName,
+        composeDir,
+        'comcast-resolver',
+        'dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2',
       );
-      await handle.celilo('module deploy iptables', 180_000);
+      return result.exitCode === 0 && result.stdout.trim().length > 0;
     },
+    60_000,
+    'DNS convergence',
+  );
 
-    async socksProxy(options: ProxyOptions = {}): Promise<SocksProxyHandle> {
-      const proxy = await startSocksProxy(projectName, options);
-      activeProxies.push(proxy);
-      return proxy;
-    },
+  // Wait for internal resolver (split-horizon DNS for management container).
+  // Skipped when the test specifies its own machine named `dns-int` — that
+  // means the test is replacing the infra resolver with a module under test
+  // (e.g., knot-unbound-internal). The bare target machine has no DNS server
+  // until its module deploys, so this wait would always time out. Management
+  // can still resolve names via the fallback `nameserver 100.100.0.1` in
+  // its resolv.conf during deploy.
+  //
+  // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
+  // places the dns_internal provider in a PROTECTED zone (dmz) so it can see
+  // protected-zone query sources for split-horizon views. So check every zone's
+  // machines for a `dns-int`, not only internalMachines.
+  const dnsIntReplaced = [
+    ...config.internalMachines,
+    ...config.dmzMachines,
+    ...config.appMachines,
+    ...config.secureMachines,
+  ].some((m) => m.name === 'dns-int');
+  if (!dnsIntReplaced) {
+    console.log('[progress:start] waiting for internal resolver | internal resolver ready');
+    await waitFor(
+      async () => {
+        const result = dockerExec(
+          projectName,
+          composeDir,
+          'dns-int',
+          'dig @127.0.0.1 iamtheinternet.org A +short +timeout=2',
+        );
+        return result.exitCode === 0 && result.stdout.trim().length > 0;
+      },
+      30_000,
+      'internal DNS resolver',
+    );
+  }
 
-    async browser(options: BrowserOptions): Promise<BrowserHandle> {
-      const browserHandle = await startBrowser(projectName, options);
-      activeBrowsers.push(browserHandle);
-      activeProxies.push(browserHandle.proxy);
-      return browserHandle;
-    },
+  // Optional routing verification
+  if (config.verifyRouting) {
+    console.log('[progress:start] verifying routing | routing verified');
+    await waitFor(
+      async () => {
+        const result = dockerExec(
+          projectName,
+          composeDir,
+          'management',
+          `ping -c1 -W2 ${SIMULATOR_IPS.ROOT_DNS}`,
+        );
+        return result.exitCode === 0;
+      },
+      30_000,
+      'routing verification',
+    );
+  }
 
-    async dig(name: string): Promise<string> {
-      const result = dockerExec(projectName, composeDir, 'management', `dig +short ${name}`);
-      return result.stdout
-        .split('\n')
-        .filter((l) => !l.startsWith(';;'))
-        .join('\n')
-        .trim();
-    },
+  // Wait for all target machines to complete setup (systemd boot + routing)
+  const allMachines = [
+    ...config.dmzMachines,
+    ...config.appMachines,
+    ...config.secureMachines,
+    ...config.internalMachines,
+  ];
+  if (allMachines.length > 0) {
+    console.log('[progress:start] waiting for target machines | target machines ready');
+  }
+  for (const machine of allMachines) {
+    await waitFor(
+      async () => {
+        const result = dockerExec(
+          projectName,
+          composeDir,
+          machine.name,
+          'systemctl is-active target-setup 2>/dev/null',
+        );
+        return result.stdout.trim() === 'active';
+      },
+      // App-zone machines use Dockerfile.target-machine-docker which bakes in
+      // dockerd; systemd boot + dockerd init takes ~25-35s, leaving a tight
+      // margin against a 30s timeout. 60s covers the observed worst case
+      // without slowing down dmz/internal-only tests (which still hit this
+      // in ~2-5s).
+      60_000,
+      `${machine.name} target-setup`,
+    );
+  }
 
-    async debug(container = 'management'): Promise<void> {
-      console.log(`[debug:pause] ${projectName} ${container}`);
-      const signalFile = join(composeDir, `.debug-resume-${process.pid}`);
-      const start = Date.now();
-      const timeout = 86_400_000;
-      while (!existsSync(signalFile) && Date.now() - start < timeout) {
-        await new Promise((r) => setTimeout(r, 500));
-      }
-      try {
-        require('node:fs').unlinkSync(signalFile);
-      } catch {}
-      console.log('[debug:resumed]');
-    },
+  // Wait for DHCP client lease
+  if (config.dhcpClient) {
+    console.log('[progress:start] waiting for DHCP client lease | DHCP lease acquired');
+    // Self-diagnosing via waitFor's onTimeout (e2e-confidence #255): a DHCP-lease
+    // timeout is an intermittent in-suite flake (passes solo). On timeout, dump
+    // the dhcp-client's interface + dhclient transcript so the failure shows
+    // whether DISCOVER/OFFER/REQUEST/ACK completed (server silent) vs. a lease
+    // that landed but wasn't recorded.
+    await waitFor(
+      async () => {
+        const result = dockerExec(
+          projectName,
+          composeDir,
+          'dhcp-client',
+          'test -f /var/lib/dhcp/dhclient.leases && grep -c lease /var/lib/dhcp/dhclient.leases',
+        );
+        return result.exitCode === 0 && Number.parseInt(result.stdout.trim()) > 0;
+      },
+      60_000,
+      'DHCP client lease',
+      () => {
+        const dump = (label: string, cmd: string): string => {
+          const r = dockerExec(projectName, composeDir, 'dhcp-client', cmd);
+          return `--- ${label} ---\n${(r.stdout || r.stderr || '(no output)').trim()}`;
+        };
+        return [
+          '=== DHCP diagnostics (dhcp-client) ===',
+          dump('ip addr', 'ip -o addr show 2>&1'),
+          dump('dhclient.leases', 'cat /var/lib/dhcp/dhclient.leases 2>&1 | tail -25'),
+          dump(
+            'dhclient transcript',
+            "journalctl -u dhclient --no-pager 2>/dev/null | tail -25 || cat /var/log/dhclient.log 2>/dev/null | tail -25 || echo '(no dhclient log)'",
+          ),
+        ].join('\n');
+      },
+    );
+  }
 
-    waitFor,
+  // Wait for routing to Pebble (through shared infra networks)
+  if (allMachines.length > 0) {
+    await waitFor(
+      async () => {
+        const result = dockerExec(
+          projectName,
+          composeDir,
+          'management',
+          `ping -c1 -W2 ${SIMULATOR_IPS.PEBBLE}`,
+        );
+        return result.exitCode === 0;
+      },
+      30_000,
+      'routing to Pebble',
+    );
+  }
 
-    async configureAcme(): Promise<void> {
-      await handle.celilo(
-        'module config set caddy acme_ca https://acme-v02.api.letsencrypt.org/dir',
-      );
-    },
+  // Initialize celilo. Skipped for the vanilla management variant —
+  // there's no celilo binary baked into that image; the caller is
+  // responsible for installing it (typically by running install.sh)
+  // and then calling `celilo system init` themselves.
+  if (config.managementVariant !== 'vanilla') {
+    assertCliVersion(projectName, composeDir);
+    console.log('[progress:start] initializing celilo | celilo initialized');
+    // Honest harness (v2/NETWORK_CONFIG_TO_FIREWALL.md): seed ONLY the
+    // `internal` zone — the network the management box is genuinely on —
+    // plus DNS. dmz/app/secure are NOT pre-seeded; they come into being
+    // when a test deploys the firewall (net.deployFirewall()), exactly as
+    // in production. A test that places services in those zones must call
+    // deployFirewall first; internal-only tests need nothing more.
+    const initResult = dockerExec(
+      projectName,
+      composeDir,
+      'management',
+      `celilo system init --accept-defaults \
+        network.internal.subnet=${ZONE_SUBNETS.internal} \
+        network.internal.gateway=${ZONE_GATEWAYS.internal} \
+        dns.primary=100.100.0.1 \
+        dns.fallback=1.0.0.1,8.8.8.8`,
+    );
+    if (initResult.exitCode !== 0) {
+      throw new Error(`Celilo init failed: ${initResult.stderr}`);
+    }
 
-    async publishModule(localPath: string): Promise<void> {
-      const absPath = resolve(localPath);
-      const isNetapp = absPath.endsWith('.netapp');
-      const moduleId = isNetapp ? basename(absPath).slice(0, -7) : basename(absPath);
+    // Start the event-bus dispatcher (ISS-0035 / ISS-0042), now that `system
+    // init` has created the bus DB. Deploys emit bus events —
+    // public_web.routes_changed (caddy route reconcile), system.created (DNS
+    // providers) — that only take effect when a dispatcher delivers them.
+    // Production runs this as a systemd unit; the e2e mgmt box has no systemd,
+    // so run it detached (nohup + disown, like the cele2e responder). One
+    // dispatcher per management container serves every deploy in the test, so
+    // individual tests don't need to start their own (delivery claims are
+    // atomic, so a test that still does won't double-deliver).
+    console.log('[progress:start] starting event dispatcher | dispatcher running');
+    const dispatcherResult = dockerExec(
+      projectName,
+      composeDir,
+      'management',
+      'nohup celilo events run --poll-ms 500 --concurrency 4 > /tmp/cele2e-dispatcher.log 2>&1 < /dev/null & disown',
+    );
+    if (dispatcherResult.exitCode !== 0) {
+      throw new Error(`Failed to start event dispatcher: ${dispatcherResult.stderr}`);
+    }
+  }
 
-      let netappPath: string;
-      let cleanup = false;
+  console.log('[progress:done] network ready');
+  // Emit project name so the runner can persist it if --keep is set
+  console.log(`[e2e:project] ${projectName}`);
 
-      if (isNetapp) {
-        netappPath = absPath;
-      } else {
-        netappPath = join(tmpdir(), `${moduleId}-${Date.now()}.netapp`);
-        cleanup = true;
-        const resolvedCeliloRoot = findCeliloRoot();
-        const celiloCliPath = join(
-          resolvedCeliloRoot ?? PACKAGE_ROOT,
-          'apps/celilo/src/cli/index.ts',
-        );
-        try {
-          run(
-            `bun run ${JSON.stringify(celiloCliPath)} package ${JSON.stringify(absPath)} --output ${JSON.stringify(netappPath)}`,
-            { timeout: 60_000 },
-          );
-        } catch (err) {
-          throw new Error(`Failed to package module at ${absPath}: ${String(err)}`);
-        }
-      }
+  return buildNetworkHandle(projectName, composeDir, celiloRoot);
+}
 
-      try {
-        run(
-          `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} cp ${JSON.stringify(netappPath)} registry:/uploads/${moduleId}.netapp`,
-          { cwd: composeDir, timeout: 120_000 },
-        );
-      } finally {
-        if (cleanup)
-          try {
-            execSync(`rm -f ${JSON.stringify(netappPath)}`, { stdio: 'pipe' });
-          } catch {}
-      }
-    },
+/**
+ * Reconnect to an existing test network (previously kept alive with --keep).
+ * Returns a NetworkHandle pointing to the existing Docker project. Skips
+ * all the setup (image build, container start, celilo init) — assumes the
+ * network is already running and healthy.
+ *
+ * If the project doesn't exist, throws with an actionable error.
+ */
+export function reconnectNetwork(projectName: string): NetworkHandle {
+  const composeDir = PACKAGE_ROOT;
 
-    async stop(): Promise<void> {
-      for (const browser of activeBrowsers) {
-        try {
-          await browser.close();
-        } catch {}
-      }
-      for (const proxy of activeProxies) {
-        try {
-          await proxy.stop();
-        } catch {}
-      }
+  // Verify the project actually exists
+  try {
+    const result = run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} ps -q`, {
+      cwd: composeDir,
+      timeout: 10_000,
+    });
+    if (!result.trim()) {
+      throw new Error(`No containers found for project ${projectName}`);
+    }
+  } catch (err) {
+    throw new Error(
+      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
+        `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
+    );
+  }
 
-      // Respect --keep / --reuse: don't tear down
-      if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') {
-        console.log(`[progress:done] network kept alive (project: ${projectName})`);
-        activeProjects.delete(projectName);
-        return;
-      }
-      try {
-        run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`, {
-          cwd: composeDir,
-          timeout: 60_000,
-        });
-      } catch {}
-      activeProjects.delete(projectName);
-    },
-  };
+  activeProjects.add(projectName);
+  console.log(`[progress:done] reconnected to existing network (${projectName})`);
+  console.log(`[e2e:project] ${projectName}`);
 
-  return handle;
+  return buildNetworkHandle(projectName, composeDir, findCeliloRoot());
 }