@celilo/e2e 0.19.3 → 0.20.0

package/src/container-manager.ts

@@ -1,19 +1,29 @@
-import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
+import {
+  type ChildProcessWithoutNullStreams,
+  type ExecSyncOptions,
+  type SpawnOptionsWithoutStdio,
+  execSync,
+  spawn,
+} from 'node:child_process';
 import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
+import { parseIpv4, subnetContains } from '@celilo/capabilities';
 import { parse as parseYaml } from 'yaml';
 import { startBrowser } from './browser';
 import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
 import {
+  type FirewallLeg,
   SHARED_PROJECT_NAME,
   firewallZoneLegs,
+  generateSharedInfraYaml,
   generateTestComposeYaml,
   getAllMachines,
+  referencedImages,
   registryUploadsHostDir,
 } from './docker-compose-generator';
-import { explainBuildFailure } from './doctor';
 import { type ModuleHost, parseModuleHost } from './module-host';
+import { CONTAINER_PREFIX, GUEST_PROJECT_LABEL } from './proxmox-provisioner';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
 import { startSocksProxy } from './socks-proxy';
@@ -35,9 +45,114 @@ import {
   internalNatIp,
 } from './types';
 
+/**
+ * Every way this module reaches the docker CLI, injectable so the cleanup
+ * sweeps, the exec wrappers and the image accounting are unit-testable
+ * without a daemon (ce-h4no, lane A1 of openspec/changes/e2e-suite-recovery).
+ *
+ * Follows the DoctorProbe pattern in doctor.ts: an interface of typed
+ * operations with the real implementation as the default, so no caller
+ * changes. proxmox-provisioner.ts owns the narrower `DockerRunner` name
+ * (argv-only, sync, no streaming), so this one is named for what it wraps:
+ * the docker CLI itself. Tests substitute a fake via `withDockerCli`.
+ */
+export interface DockerCli {
+  /** Synchronous `docker ...` shell command. Throws on non-zero exit, like execSync. */
+  exec(command: string, opts?: ExecSyncOptions): string;
+  /**
+   * Streaming `docker ...` for long-running compose build and exec. Callers
+   * never pass stdio (pipe default), so the streams are non-null.
+   */
+  spawn(args: string[], opts?: SpawnOptionsWithoutStdio): ChildProcessWithoutNullStreams;
+}
+
+export const realDockerCli: DockerCli = {
+  exec(command, opts) {
+    // Encoding is pinned last so TS narrows the return to `string` even if
+    // a caller passes an opts object that could in principle override it.
+    return execSync(command, { ...opts, encoding: 'utf-8' });
+  },
+  spawn(args, opts) {
+    return spawn('docker', args, opts);
+  },
+};
+
+let dockerCli: DockerCli = realDockerCli;
+
+/**
+ * Run `fn` with `runner` as this module's docker access, restoring the
+ * previous runner afterwards — even when `fn` throws. A sync callback runs
+ * and restores synchronously; a promise callback keeps the override in force
+ * across its awaits and restores it when it settles. Concurrent async scopes
+ * would restore out of order, so tests await one scope at a time.
+ */
+export function withDockerCli<T>(runner: DockerCli, fn: () => T): T;
+export function withDockerCli<T>(runner: DockerCli, fn: () => Promise<T>): Promise<T>;
+export function withDockerCli<T>(runner: DockerCli, fn: () => T | Promise<T>): T | Promise<T> {
+  const previous = dockerCli;
+  dockerCli = runner;
+  try {
+    const result = fn();
+    if (result instanceof Promise) {
+      return result.finally(() => {
+        dockerCli = previous;
+      });
+    }
+    dockerCli = previous;
+    return result;
+  } catch (err) {
+    dockerCli = previous;
+    throw err;
+  }
+}
+
 /** Package root — where docker/, config/, simulators/ live */
 const PACKAGE_ROOT = join(__dirname, '..');
 
+/**
+ * The `.netapp` build-infra already staged for this module, if nothing in the
+ * source has changed since.
+ *
+ * `publishModule` used to repackage from source unconditionally, inside a
+ * running stack, under a 60s cap. Measured 2026-09-04: build-infra packaged
+ * `celilo-registry` (an 82MB netapp) in 11 seconds before any stack existed,
+ * then publishModule exceeded 60 seconds on identical work because the stack was
+ * up and eating the host. `registry-pipeline` failed on exactly that, and it is
+ * not one of the three flakes that suite is quarantined for (celilo#1258).
+ *
+ * `find -newer -quit` asks the filesystem instead of walking in JS and stops at
+ * the first newer file. A staleness check that cannot answer repackages: reusing
+ * on an inconclusive result is how a test silently runs against a stale module.
+ */
+function stagedNetappIfCurrent(moduleDir: string, moduleId: string): string | null {
+  const staged = join(PACKAGE_ROOT, 'netapps', `${moduleId}.netapp`);
+  if (!existsSync(staged)) return null;
+
+  // Only paths the package actually contains can make it stale. `e2e/` is
+  // excluded from a module package wholesale and `node_modules/.bin` is excluded
+  // from the hook runtime closure — see `classifyModulePath` and
+  // `includeNodeModulesPath` in apps/celilo/src/module/packaging/, which are the
+  // authority. They are restated rather than imported because packages/e2e has
+  // no import path into apps/celilo, and registry-server's bootstrap.ts already
+  // carries the same duplication for the same reason.
+  //
+  // Getting these wrong is safe in one direction only, and it is this one:
+  // counting a non-packaged path makes us repackage needlessly (slow), while
+  // MISSING a packaged path would reuse a stale netapp (wrong). Every exclusion
+  // here is a path the packager does not ship, so it cannot cause the latter.
+  // Without them nothing is ever reused: `bun install` touches
+  // `e2e/node_modules/.bin/*` and every module looks permanently dirty.
+  try {
+    const newer = execSync(
+      `find ${JSON.stringify(moduleDir)} -newer ${JSON.stringify(staged)} -not -path '*/e2e/*' -not -path '*/node_modules/.bin/*' -print -quit`,
+      { encoding: 'utf-8', timeout: 30_000 },
+    ).trim();
+    return newer === '' ? staged : null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Find the Celilo project root by walking up from the package directory.
  * Looks for apps/celilo/ as the marker.
@@ -104,11 +219,14 @@ const activeProjects: Set<string> = new Set();
  */
 export function projectTeardownCommands(project: string): {
   listContainers: string;
+  /** Sim-created LXC guests of this project — by label, since their names carry no timestamp. */
+  listGuests: string;
   listNetworks: string;
   listVolumes: string;
 } {
   return {
     listContainers: `docker ps -aq --filter name=${project}`,
+    listGuests: `docker ps -aq --filter label=${GUEST_PROJECT_LABEL}=${project}`,
     listNetworks: `docker network ls --format {{.Name}} --filter name=${project}`,
     // Volumes were missing here, and the compose path that would have removed
     // them (`down --volumes`) is the same best-effort call that silently does
@@ -121,42 +239,220 @@ export function projectTeardownCommands(project: string): {
   };
 }
 
-/** Force-remove a project's containers, networks and volumes by name. Never throws. */
-function forceRemoveProject(project: string): void {
+/**
+ * Command listing the containers attached to one of the project's networks.
+ *
+ * A Proxmox guest is named `celilo-e2e-lxc-<vmid>` and carries NO project name
+ * (it is created by `docker run`, not compose), so the project-scoped container
+ * sweep never sees it. A live guest keeps `docker network rm` failing forever,
+ * and the leaked network holds the sim's subnet — every later run then dies at
+ * `docker compose up` with `invalid pool request: Pool overlaps` (ce-ywix:
+ * three full regressions lost to one crashed suite's guest).
+ */
+export function networkContainersInspectCommand(network: string): string {
+  return `docker network inspect ${network} --format {{json .Containers}}`;
+}
+
+/**
+ * One best-effort cleanup step that failed and was carried past. A sweep step
+ * never aborts its sweep (the next step's resources are no less leaked for
+ * it), but it is also never silent: each failure is logged as a
+ * `[cleanup:failed]` line and counted in the sweep's return value, so the
+ * progress line cannot claim "cleanup complete" over a sweep that did not
+ * complete (ce-yuzi, lane A2 of openspec/changes/e2e-suite-recovery).
+ */
+export interface CleanupFailure {
+  /** The step that failed, named the way the sweep runs it, e.g. `docker rm -f ...`. */
+  step: string;
+  /** The error's own message, not a restatement of it. */
+  detail: string;
+}
+
+/** Log one best-effort step that failed where there is no sweep to count it. */
+function logCleanupFailure(step: string, error: unknown): void {
+  const detail = error instanceof Error ? error.message : String(error);
+  console.error(`[cleanup:failed] ${step} | ${detail}`);
+}
+
+/** Record a carried-past failure: log it AND count it, so the caller's
+ * progress line reflects reality. */
+function recordFailure(failures: CleanupFailure[], step: string, error: unknown): void {
+  failures.push({ step, detail: error instanceof Error ? error.message : String(error) });
+  logCleanupFailure(step, error);
+}
+
+/**
+ * The done-side of the stale-resource sweep's progress line. Honest by
+ * construction: "complete" appears only when the sweep returned zero
+ * failures.
+ */
+export function cleanupProgress(failures: CleanupFailure[]): string {
+  if (failures.length === 0) return 'cleanup complete';
+  return `cleanup incomplete: ${failures.length} step(s) failed (see [cleanup:failed] lines above)`;
+}
+
+/**
+ * Force-remove a project's containers, networks and volumes by name. Never
+ * throws. Returns one CleanupFailure per step that failed and was carried
+ * past, so callers can report what the sweep could not do.
+ */
+export function forceRemoveProject(project: string): CleanupFailure[] {
+  const failures: CleanupFailure[] = [];
   const cmds = projectTeardownCommands(project);
-  try {
-    const ids = execSync(cmds.listContainers, { timeout: 15_000, stdio: 'pipe' })
-      .toString()
-      .split('\n')
-      .filter(Boolean);
-    if (ids.length > 0) {
-      execSync(`docker rm -f ${ids.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
+  const listed = (cmd: string, step: string): string[] => {
+    try {
+      return run(cmd, { timeout: 15_000, stdio: 'pipe' }).split('\n').filter(Boolean);
+    } catch (err) {
+      // A listing that fails must not look like an empty result: an empty
+      // answer and a broken docker are indistinguishable to the sweep, and
+      // the sweep would skip what it cannot see.
+      recordFailure(failures, step, err);
+      return [];
+    }
+  };
+  // Guests are listed by LABEL, not only by name: a sim-created guest is named
+  // celilo-e2e-lxc-<vmid>, which the name filter cannot match, and a guest left
+  // running holds its zone network's endpoint — the network rm below then fails
+  // with "has active endpoints" and the subnet stays allocated, colliding with
+  // every later suite in the run (celilo#1247).
+  const ids = [
+    ...new Set([
+      ...listed(cmds.listContainers, `docker ps name=${project}`),
+      ...listed(cmds.listGuests, `docker ps label=${GUEST_PROJECT_LABEL}=${project}`),
+    ]),
+  ];
+  if (ids.length > 0) {
+    try {
+      run(`docker rm -f ${ids.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
+    } catch (err) {
+      recordFailure(failures, `docker rm -f ${project} containers`, err);
     }
-  } catch {}
+  }
   try {
-    const nets = execSync(cmds.listNetworks, { timeout: 15_000, stdio: 'pipe' })
-      .toString()
+    const nets = run(cmds.listNetworks, { timeout: 15_000, stdio: 'pipe' })
       .split('\n')
       .filter(Boolean);
     for (const net of nets) {
+      // Remove the e2e containers ON the network first, or the rm below fails
+      // while a guest is attached. Only ever touches containers carrying the
+      // provisioner's own CONTAINER_PREFIX containment contract, so anything
+      // else attached by hand is left alone and the network rm fails (the
+      // doctor's leaked-stacks check is what names that case).
+      try {
+        // Through the seam (run/dockerCli), not raw execSync, so the sweep is
+        // unit-testable and every failure lands in the same channel.
+        const attached = JSON.parse(
+          run(networkContainersInspectCommand(net), { timeout: 15_000, stdio: 'pipe' }) || '{}',
+        ) as Record<string, { Name?: string }>;
+        const ours = Object.values(attached)
+          .map((c) => c.Name ?? '')
+          .filter((n) => n.startsWith(CONTAINER_PREFIX));
+        if (ours.length > 0) {
+          run(`docker rm -f ${ours.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
+        }
+      } catch (err) {
+        recordFailure(failures, `attached-container sweep on ${net}`, err);
+      }
       try {
-        execSync(`docker network rm ${net}`, { timeout: 10_000, stdio: 'pipe' });
-      } catch {}
+        run(`docker network rm ${net}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch (err) {
+        // A network that fails to rm must not stop the volume sweep below.
+        recordFailure(failures, `docker network rm ${net}`, err);
+      }
     }
-  } catch {}
+  } catch (err) {
+    recordFailure(failures, `network sweep for ${project}`, err);
+  }
   // Volumes last: a volume still attached to a container cannot be removed, so
   // this must follow the container sweep above.
   try {
-    const vols = execSync(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
-      .toString()
+    const vols = run(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
+      .split('\n')
+      .filter(Boolean);
+    for (const vol of vols) {
+      try {
+        run(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch (err) {
+        recordFailure(failures, `docker volume rm ${vol}`, err);
+      }
+    }
+  } catch (err) {
+    recordFailure(failures, `volume sweep for ${project}`, err);
+  }
+  return failures;
+}
+
+/**
+ * The start-of-run stale-resource sweep: per-test containers, sim-created
+ * guests of ANY project (their names carry no timestamp, celilo#1247), whole
+ * per-test projects found via their networks, and stale volumes. Moved
+ * verbatim out of startNetwork so its reach and its failure behavior are
+ * unit-testable (ce-yuzi). Returns the failures it carried past; the caller
+ * turns that into the progress line.
+ */
+export function sweepStaleTestResources(): CleanupFailure[] {
+  const failures: CleanupFailure[] = [];
+  // Force-kill per-test containers (not shared infra)
+  try {
+    const containers = run('docker ps -aq --filter "name=celilo-e2e-1" 2>/dev/null');
+    if (containers.trim()) {
+      run(`docker rm -f ${containers.replace(/\n/g, ' ')}`, { timeout: 30_000 });
+    }
+  } catch (err) {
+    recordFailure(failures, 'stale container sweep', err);
+  }
+
+  // Sim-created LXC guests. The name filter above cannot match them (their
+  // names are celilo-e2e-lxc-<vmid>, with no timestamp), and a guest left
+  // running holds its zone network's endpoint — so the network sweep below
+  // cannot free that subnet either, and every later suite in the run dies
+  // creating it (celilo#1247). Any container carrying the guest label is
+  // ours by construction: only the provisioner sets it.
+  try {
+    const guests = run(`docker ps -aq --filter label=${GUEST_PROJECT_LABEL}`);
+    if (guests.trim()) {
+      run(`docker rm -f ${guests.replace(/\n/g, ' ')}`, { timeout: 30_000 });
+    }
+  } catch (err) {
+    recordFailure(failures, 'stale guest sweep', err);
+  }
+
+  // Remove stale per-test networks AND their volumes, via the same
+  // by-name teardown the exit handler uses. This used to lean on
+  // `docker compose -p <project> down --volumes` for the volume half, which
+  // has no `-f` and no reliable cwd here either — compose exits "no
+  // configuration file provided" and the catch swallows it, so volumes were
+  // never removed on this path either. Reusing forceRemoveProject keeps the
+  // crash-recovery sweep and the exit handler from drifting apart.
+  try {
+    const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
+      .split('\n')
+      .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
+    const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
+    for (const project of projects) {
+      failures.push(...forceRemoveProject(project));
+    }
+  } catch (err) {
+    recordFailure(failures, 'stale project sweep', err);
+  }
+  // Volumes can outlive every container and network of their project — that
+  // is exactly the 58-volume pile — so sweep by prefix too, not only for
+  // projects that still have a network to be discovered by.
+  try {
+    const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
       .split('\n')
       .filter(Boolean);
     for (const vol of vols) {
       try {
-        execSync(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
-      } catch {}
+        run(`docker volume rm ${vol}`, { timeout: 5_000 });
+      } catch (err) {
+        recordFailure(failures, `docker volume rm ${vol}`, err);
+      }
     }
-  } catch {}
+  } catch (err) {
+    recordFailure(failures, 'stale volume sweep', err);
+  }
+  return failures;
 }
 
 function cleanupOnExit() {
@@ -169,19 +465,27 @@ function cleanupOnExit() {
     // Compose first when it can work — it also drops volumes and orphans — but
     // it is best-effort, so never rely on it having done anything.
     try {
-      execSync(`docker compose -f ${COMPOSE_FILE} -p ${project} down --volumes --remove-orphans`, {
+      run(`docker compose -f ${COMPOSE_FILE} -p ${project} down --volumes --remove-orphans`, {
         cwd: PACKAGE_ROOT,
         timeout: 30_000,
         stdio: 'pipe',
       });
-    } catch {}
+    } catch (err) {
+      // Best effort: an exit handler has no reliable cwd, so compose can fail
+      // to find the file. forceRemoveProject below is the authoritative,
+      // compose-free teardown, but the compose attempt still gets logged so a
+      // systematic failure is visible instead of absorbed.
+      logCleanupFailure(`compose down ${project}`, err);
+    }
     // Authoritative: needs no compose file, no cwd, no working directory state.
     forceRemoveProject(project);
   }
   activeProjects.clear();
   try {
-    execSync('docker network prune -f', { timeout: 10_000, stdio: 'pipe' });
-  } catch {}
+    run('docker network prune -f', { timeout: 10_000, stdio: 'pipe' });
+  } catch (err) {
+    logCleanupFailure('docker network prune', err);
+  }
 }
 
 process.on('SIGTERM', () => {
@@ -219,12 +523,14 @@ process.on('exit', () => {
 function run(cmd: string, opts?: ExecSyncOptions): string {
   // Encoding is pinned last so TS narrows the return to `string` even if
   // a caller passes an opts object that could in principle override it.
-  return execSync(cmd, {
-    timeout: COMPOSE_TIMEOUT,
-    stdio: ['pipe', 'pipe', 'pipe'],
-    ...opts,
-    encoding: 'utf-8',
-  }).trim();
+  return dockerCli
+    .exec(cmd, {
+      timeout: COMPOSE_TIMEOUT,
+      stdio: ['pipe', 'pipe', 'pipe'],
+      ...opts,
+      encoding: 'utf-8',
+    })
+    .trim();
 }
 
 const COMPOSE_FILE = 'docker-compose.test.yml';
@@ -312,7 +618,8 @@ export async function scrubDnsZones(): Promise<void> {
   }
 }
 
-function dockerExec(
+/** Exported for the unit tests that drive it through a fake DockerCli. */
+export function dockerExec(
   projectName: string,
   composeDir: string,
   container: string,
@@ -349,11 +656,96 @@ function dockerExec(
     return {
       stdout: e.stdout?.toString() ?? '',
       stderr,
-      exitCode: e.status ?? (timedOut ? 124 : 1),
+      // The timeout verdict wins over any status the dying client left behind.
+      // `docker compose exec` traps the SIGTERM execSync sends on timeout and
+      // exits 130 by its own convention (ce-013r, celilo#1293), so the real
+      // error carries status: 130 — the client's death code, not the command's
+      // verdict. Adopting it read every harness timeout as a mystery exit 130.
+      exitCode: timedOut ? 124 : (e.status ?? 1),
     };
   }
 }
 
+/**
+ * Subnets the simulated internet actually hosts. A nameserver outside all of
+ * them blackholes: nothing in the sim owns the address, so every query to it
+ * eats its full timeout. ZONE_SUBNETS covers machines a test deploys (a
+ * dns_internal provider among them); the three external networks cover the
+ * sim's public edge — isp-external (comcast-resolver, the fleet's
+ * `dns.primary`), internet-external (the public simulators), and the
+ * real-internet transit network behind fw-ext.
+ */
+const SIM_ROUTABLE_SUBNETS: ReadonlyArray<{ cidr: string; label: string }> = [
+  ...Object.entries(ZONE_SUBNETS).map(([zone, cidr]) => ({ cidr, label: `${zone} zone` })),
+  { cidr: '203.0.113.0/24', label: 'isp-external' },
+  { cidr: '100.64.0.0/24', label: 'internet-external' },
+  { cidr: '172.30.0.0/24', label: 'real-internet' },
+];
+
+/**
+ * The nameserver problems in `celilo system config get` output, one string
+ * each. Pure so the gate is unit-testable without a daemon. `dns.fallback` is
+ * comma-separated; `dns.primary` a single IP — both split on either separator.
+ * A missing `dns.primary` is itself a problem: the birth nameserver list would
+ * be empty.
+ */
+export function fleetNameserverProblems(configOutput: string): string[] {
+  const entries = [...configOutput.matchAll(/^dns\.(primary|fallback) = (.+)$/gm)].map(
+    ([, key, value]) => ({ key: key as 'primary' | 'fallback', value: value.trim() }),
+  );
+  const problems: string[] = [];
+  if (!entries.some((e) => e.key === 'primary')) {
+    problems.push('dns.primary is not set — the birth nameserver list would be empty');
+  }
+  for (const { key, value } of entries) {
+    for (const ip of value.split(/[\s,]+/).filter(Boolean)) {
+      if (parseIpv4(ip) === null) {
+        problems.push(`dns.${key} entry "${ip}" is not an IPv4 address`);
+        continue;
+      }
+      const routable = SIM_ROUTABLE_SUBNETS.some((s) => subnetContains(s.cidr, ip));
+      if (!routable) {
+        problems.push(
+          `dns.${key} nameserver ${ip} is in no simulated subnet (${SIM_ROUTABLE_SUBNETS.map((s) => s.cidr).join(', ')}) — the sim cannot route to it, so every lookup to it blackholes`,
+        );
+      }
+    }
+  }
+  return problems;
+}
+
+/**
+ * Recurrence gate for the Gathering Facts hang (celilo#1290). A nameserver in
+ * fleet config that the sim cannot route blackholes exactly the lookups
+ * ansible's Gathering Facts performs, and the failure used to surface minutes
+ * later as an unrelated 600s command timeout. Read back what `system init`
+ * actually stored — core's own resolver discovery falls back to 1.1.1.1
+ * (dns-discovery.ts), which is unroutable here — and fail in seconds, naming
+ * the address.
+ */
+function assertFleetNameserversAreSimRoutable(projectName: string, composeDir: string): void {
+  const result = dockerExec(
+    projectName,
+    composeDir,
+    'management',
+    'celilo system config get dns.primary; celilo system config get dns.fallback',
+    30_000,
+  );
+  // A missing dns.fallback exits non-zero ("key not found"); that is fine and
+  // expected. Only treat the read as failed when even dns.primary is absent.
+  if (result.exitCode !== 0 && !result.stdout.includes('dns.primary = ')) {
+    throw new Error(
+      `Could not read fleet DNS config from the management container: ${result.stderr || result.stdout || '(no output)'}`,
+    );
+  }
+  const problems = fleetNameserverProblems(result.stdout);
+  if (problems.length > 0) {
+    throw new Error(
+      `Fleet DNS config names nameserver(s) the simulated internet cannot reach — the celilo#1290 failure shape (600s Gathering Facts hang):\n  ${problems.join('\n  ')}`,
+    );
+  }
+}
+
 /**
  * Fail fast if the management image's baked celilo CLI is older than the
  * harness needs (ce-5qp). Runs before `system init` so a version mismatch
@@ -378,22 +770,94 @@ function assertCliVersion(projectName: string, composeDir: string): void {
  * knows nothing about it, so `docker compose exec` cannot see it. Same wrapping
  * as `dockerExec` so both behave identically from a test's point of view.
  */
-function plainDockerExec(container: string, cmd: string, timeoutMs = 60_000): ExecResult {
+export function plainDockerExec(container: string, cmd: string, timeoutMs = 60_000): ExecResult {
   try {
     const stdout = run(`docker exec ${container} bash -c ${JSON.stringify(cmd)}`, {
       timeout: timeoutMs,
     });
     return { stdout, stderr: '', exitCode: 0 };
   } catch (err: unknown) {
-    const e = err as { stdout?: string; stderr?: string; status?: number };
+    const e = err as {
+      stdout?: string;
+      stderr?: string;
+      status?: number;
+      code?: string;
+      signal?: string;
+      killed?: boolean;
+    };
+    // Same timeout mapping as dockerExec: the harness's clock ended the exec,
+    // so it reports 124 with the actionable stderr, never the client's own
+    // exit status (ce-013r).
+    const timedOut = e.code === 'ETIMEDOUT' || e.signal === 'SIGTERM' || e.killed === true;
     return {
       stdout: e.stdout ?? '',
-      stderr: e.stderr ?? String(err),
-      exitCode: e.status ?? 1,
+      stderr: timedOut
+        ? `timed out after ${Math.round(timeoutMs / 1000)}s running: ${cmd}`
+        : (e.stderr ?? String(err)),
+      exitCode: timedOut ? 124 : (e.status ?? 1),
     };
   }
 }
 
+/**
+ * Finds the interface still carrying an address in `subnet` (the alien
+ * segment's subnet) inside container `service`, deletes it, and verifies the
+ * deletion by re-reading the interface table. Throws naming the surviving
+ * interface and container when the interface is still there — the detach path
+ * of attachAlienSegment, which re-reads and throws for the same reason:
+ * neither docker's exit code nor its own view can be believed for veth
+ * surgery, and a surviving alien interface is exactly what makes a later
+ * converge refuse somewhere far from the cause (celilo#1261).
+ *
+ * `exec` is injected so unit tests can drive this against a faked interface
+ * table with no docker at all.
+ */
+export function removeAlienInterface(opts: {
+  service: string;
+  subnet: string;
+  exec: (cmd: string) => ExecResult;
+  /** Captured failure of `docker network disconnect -f`, carried into the
+   * throw's report when the repair does not hold. It is evidence, not a
+   * verdict: disconnect legitimately fails on a container docker already
+   * considers unattached. */
+  disconnectError?: string;
+}): void {
+  const prefix = opts.subnet.replace(/\.\d+\/\d+$/, '').replace(/\./g, '\\.');
+  const orphanRegex = new RegExp(`(\\S+)\\s+inet\\s+${prefix}\\.\\d+`);
+  const orphanIn = (table: string) => orphanRegex.exec(table)?.[1];
+
+  const check = opts.exec('ip -o addr show scope global');
+  const orphan = orphanIn(check.stdout);
+  if (!orphan) return;
+
+  const del = opts.exec(`ip link del ${orphan}`);
+  const recheck = opts.exec('ip -o addr show scope global');
+  const survivor = orphanIn(recheck.stdout);
+  if (survivor) {
+    throw new Error(
+      [
+        `detachAlienSegment: '${opts.service}' still has interface ${survivor} on ${opts.subnet} after repair:`,
+        `  ip link del ${orphan} exited ${del.exitCode}, stderr: ${del.stderr || '(empty)'}`,
+        opts.disconnectError
+          ? `  docker network disconnect failed: ${opts.disconnectError}`
+          : undefined,
+        `  interface table after repair:\n${recheck.stdout}`,
+      ]
+        .filter((line) => line !== undefined)
+        .join('\n'),
+    );
+  }
+  // The delete reported failure but the interface is gone. docker's exit code
+  // is not to be believed for this operation (see attachAlienSegment), so this
+  // is surfaced rather than swallowed — and not fatal, since the actual goal
+  // state (no alien interface) holds.
+  if (del.exitCode !== 0) {
+    console.error(
+      `detachAlienSegment: 'ip link del ${orphan}' on '${opts.service}' exited ${del.exitCode} but the interface is gone. stderr: ${del.stderr || '(empty)'}`,
+    );
+  }
+}
+
 function dockerExecAsync(
   projectName: string,
   composeDir: string,
@@ -420,7 +884,7 @@ function dockerExecAsync(
       '-c',
       cmd,
     ];
-    const child = spawn('docker', args, { cwd: composeDir });
+    const child = dockerCli.spawn(args, { cwd: composeDir });
 
     let stdout = '';
     let stderr = '';
@@ -462,6 +926,7 @@ async function waitFor(
     } catch {
       // retry
     }
+    // e2e-sleep-ok: poll cadence inside waitFor; the loop re-checks the condition each iteration.
     await new Promise((r) => setTimeout(r, 2000));
   }
   // Self-diagnosing timeout (e2e-confidence #255): a readiness wait must attach
@@ -481,67 +946,28 @@ async function waitFor(
 }
 
 /**
- * Run `docker compose build` with per-service progress signals.
- * Uses --progress=plain so output is parseable on non-TTY.
- * Emits one [progress:start] per service being built so the runner
- * shows "building <service>" as each image is compiled.
+ * Which of these image tags docker does not have locally.
+ *
+ * One `docker images` listing rather than a per-image `inspect`, because this
+ * runs on the startup path of every test and forty process spawns there is its
+ * own cost. A docker failure returns "all of them", so the caller builds —
+ * being wrong in the direction of doing the work.
  */
-async function streamingBuild(
-  projectName: string,
-  composeDir: string,
-  composeFile: string,
-): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    const proc = spawn(
-      'docker',
-      ['compose', '-f', composeFile, '-p', projectName, 'build', '--progress=plain'],
-      { cwd: composeDir },
+/** Exported for the unit tests that drive it through a fake DockerCli. */
+export function missingImages(tags: string[]): string[] {
+  if (tags.length === 0) return [];
+  let present: Set<string>;
+  try {
+    present = new Set(
+      run('docker images --format "{{.Repository}}:{{.Tag}}"', { timeout: 15_000 })
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean),
     );
-
-    const seenServices = new Set<string>();
-    let stderr = '';
-
-    function handleLine(line: string) {
-      // --progress=plain lines look like: #5 [management 2/4] RUN apt-get ...
-      const svcMatch = line.match(/\[([a-z][a-z0-9-]*)\s+\d+\/\d+\]/);
-      if (svcMatch) {
-        const svc = svcMatch[1];
-        if (!seenServices.has(svc)) {
-          seenServices.add(svc);
-          // Each new service auto-finalizes the previous [progress:start] in the runner.
-          console.log(`[progress:start] building image: ${svc} | ${svc} image ready`);
-        }
-      }
-    }
-
-    let buf = '';
-    function onData(chunk: Buffer) {
-      buf += chunk.toString();
-      const lines = buf.split('\n');
-      buf = lines.pop() ?? '';
-      for (const line of lines) handleLine(line.trim());
-    }
-
-    proc.stdout.on('data', onData);
-    proc.stderr.on('data', (chunk: Buffer) => {
-      stderr += chunk.toString();
-      onData(chunk);
-    });
-
-    proc.on('close', (code) => {
-      if (buf.trim()) handleLine(buf.trim());
-      if (code === 0) {
-        console.log('[progress:done] images built');
-        resolve();
-      } else {
-        reject(
-          new Error(
-            `docker compose build failed (exit ${code}):\n${stderr.slice(-500)}${explainBuildFailure(stderr)}`,
-          ),
-        );
-      }
-    });
-  });
+  } catch {
+    return tags;
+  }
+  return tags.filter((tag) => !present.has(tag.includes(':') ? tag : `${tag}:latest`));
 }
 
 /**
@@ -739,10 +1165,40 @@ function buildNetworkHandle(
           `respondWith: failed to spawn responder inside management container: ${spawnResult.stderr}`,
         );
       }
-      // Brief settle so the responder has registered its watches
-      // before any deploy fires events. Without this, a fast deploy
-      // could emit before the responder polls.
-      await new Promise((r) => setTimeout(r, 500));
+      // The spawn returns as soon as the shell backgrounds the process; bun
+      // then has to boot inside the container and register the responder's
+      // bus watches. A fixed sleep both wastes time on an idle host and is
+      // not enough on a loaded one: dns-replication failed stage 1 when its
+      // deploy's first interview fired before the responder polled, and the
+      // missing-responder error read as a missing fixture value. The log
+      // line below is printed only after startProgrammaticResponder has
+      // registered every watch, so waiting for it is a readiness wait on the
+      // real prerequisite, not a sleep (same pattern as deployFirewall).
+      const responderLog = '/tmp/cele2e-responder.log';
+      await waitFor(
+        async () => {
+          const probe = dockerExec(
+            projectName,
+            composeDir,
+            'management',
+            `grep -q "programmatic responder running" ${responderLog}`,
+            5_000,
+          );
+          return probe.exitCode === 0;
+        },
+        60_000,
+        'the e2e responder to register its bus watches in the management container',
+        async () => {
+          const log = dockerExec(
+            projectName,
+            composeDir,
+            'management',
+            `cat ${responderLog} 2>/dev/null || echo '(no responder log)'`,
+            5_000,
+          );
+          return `responder log (${responderLog}):\n${log.stdout || log.stderr}`;
+        },
+      );
     },
 
     async deployFirewall(opts = {}): Promise<ExecResult> {
@@ -769,7 +1225,19 @@ function buildNetworkHandle(
       //
       // `external` is excluded: it is the residual, has no subnet, and is not a
       // zone modules are placed in.
-      const declaredZones = firewallZoneLegs(topology);
+      //
+      // `firewallZoneLegs` knows the topology's legs, but not the control-plane
+      // one: the generator adds fw-main's `secure-mgmt` leg as a post-step
+      // (when celilo-mgr lives off the LAN, when a machine declares the zone,
+      // or when the proxmox sim is on), so it never appears in a
+      // topology-derived list. A sim fleet's fw-main then held an address on a
+      // segment with no declared subnet, eth4 classified alien, and the
+      // iptables converge refused (ce-fvxd). The compose file is the wiring and
+      // the wiring decides — the same artifact `topologyFromComposeFile` reads.
+      const baseLegs = firewallZoneLegs(topology);
+      const declaredZones: FirewallLeg[] = fwMainHasSecureMgmtLeg(join(composeDir, COMPOSE_FILE))
+        ? [...baseLegs, 'secure-mgmt']
+        : baseLegs;
       const providedZones = declaredZones.filter((zone) => zone !== 'external');
       const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
       const natIp = opts.natIp ?? internalNatIp();
@@ -881,7 +1349,11 @@ function buildNetworkHandle(
         // would make a classification test pass for the wrong reason.
         try {
           run(`docker network connect --ip ${ip} ${netName} ${id}`, { timeout: 20_000 });
-        } catch {}
+        } catch {
+          // Ignorable: failure here is the COMMON outcome (the gateway-exists
+          // case documented above, hit by every attach), and a connect that
+          // genuinely failed cannot hide — the address check below throws.
+        }
 
         const check = dockerExec(projectName, composeDir, service, 'ip -o addr show scope global');
         if (!check.stdout.includes(ip)) {
@@ -902,33 +1374,44 @@ function buildNetworkHandle(
           cwd: composeDir,
           timeout: 20_000,
         });
-        if (id) {
-          try {
-            run(`docker network disconnect -f ${netName} ${id}`, { timeout: 20_000 });
-          } catch {}
+        if (!id) continue;
+        // `-f` makes disconnect report failure on a container docker considers
+        // unattached, which after a half-completed attach is exactly the state
+        // being cleaned up here. So the failure is captured for the repair step
+        // to carry in its report, not obeyed — whether the interface actually
+        // left is decided by the interface table, not by docker's exit code.
+        let disconnectError: string | undefined;
+        try {
+          run(`docker network disconnect -f ${netName} ${id}`, { timeout: 20_000 });
+        } catch (err: unknown) {
+          disconnectError = err instanceof Error ? err.message : String(err);
         }
+        removeAlienInterface({
+          service,
+          subnet: alienSegmentSubnet,
+          disconnectError,
+          exec: (cmd) => dockerExec(projectName, composeDir, service, cmd),
+        });
       }
       try {
         run(`docker network rm ${netName}`, { timeout: 20_000 });
-      } catch {}
-
-      // VERIFY, for the same reason attach does. A lingering address on a dead
-      // wire is still an interface celilo cannot attribute, so a test that goes
-      // on to expect a clean converge would fail somewhere far from the cause.
-      for (const service of attachedToAlienSegment) {
-        const check = dockerExec(projectName, composeDir, service, 'ip -o addr show scope global');
-        const prefix = alienSegmentSubnet.replace(/\.\d+\/\d+$/, '').replace(/\./g, '\\.');
-        const orphan = new RegExp(`(\\S+)\\s+inet\\s+${prefix}\\.\\d+`).exec(check.stdout);
-        if (orphan) {
-          dockerExec(projectName, composeDir, service, `ip link del ${orphan[1]}`);
-        }
+      } catch (err: unknown) {
+        console.error(
+          `detachAlienSegment: docker network rm ${netName} failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
       }
       attachedToAlienSegment.clear();
     },
 
     async deployGreenwave(): Promise<void> {
       const routerIp = greenwaveRouterIp();
-      await handle.celilo(`machine add ${routerIp} --ssh-user root --earmark greenwave`);
+      // No `machine add` for the router, and do not restore one: greenwave is an
+      // appliance module (schema.ts `apiOnly`, ISP router) that talks to it over
+      // HTTPS only (scripts/router-api.ts), deploys onto no system
+      // (manifest `requires: capabilities: []`, no `system:`), and never enters
+      // infrastructure selection. Adding the router as a root machine only ran
+      // the fleet aspects against a simulated ISP router over SSH.
+      // @psbanka - 2026-09: removed in ce-5b1v; see celilo#1263.
       await handle.celilo('module import greenwave');
       await handle.celilo(`module config set greenwave router_ip ${routerIp}`);
       await handle.celilo('module secret set greenwave router_username admin');
@@ -973,13 +1456,18 @@ function buildNetworkHandle(
       const start = Date.now();
       const timeout = 86_400_000; // 24 hour max debug session
       while (!existsSync(signalFile) && Date.now() - start < timeout) {
+        // e2e-sleep-ok: waits on a human exiting the debug shell; the signal file is re-checked above.
         await new Promise((r) => setTimeout(r, 500));
       }
 
-      // Clean up
+      // Clean up. Ignorable: the file only ends THIS pause's wait loop, its
+      // name is pid-unique so a leftover can never satisfy another session's
+      // wait, and the next debug pause writes its own.
       try {
         require('node:fs').unlinkSync(signalFile);
-      } catch {}
+      } catch {
+        // deliberately ignored, see above
+      }
 
       console.log('[debug:resumed]');
     },
@@ -997,17 +1485,29 @@ function buildNetworkHandle(
       if (isNetapp) {
         netappPath = absPath;
       } else {
-        // Package the module source directory using the celilo CLI
-        netappPath = join(tmpdir(), `${moduleId}-${Date.now()}.netapp`);
-        cleanup = true;
-        const celiloCliPath = join(celiloRoot ?? PACKAGE_ROOT, 'apps/celilo/src/cli/index.ts');
-        try {
-          run(
-            `bun run ${JSON.stringify(celiloCliPath)} package ${JSON.stringify(absPath)} --output ${JSON.stringify(netappPath)}`,
-            { timeout: 60_000 },
-          );
-        } catch (err) {
-          throw new Error(`Failed to package module at ${absPath}: ${String(err)}`);
+        const staged = stagedNetappIfCurrent(absPath, moduleId);
+        if (staged) {
+          // build-infra already produced this, before the stack was competing
+          // for the host. Repackaging it here is the work that times out.
+          netappPath = staged;
+        } else {
+          netappPath = join(tmpdir(), `${moduleId}-${Date.now()}.netapp`);
+          cleanup = true;
+          const celiloCliPath = join(celiloRoot ?? PACKAGE_ROOT, 'apps/celilo/src/cli/index.ts');
+          const startedAt = Date.now();
+          try {
+            // 180s, not 60s: this runs while the stack is up, and the same work
+            // that takes 11s on an idle host went past 60s under a live stack.
+            run(
+              `bun run ${JSON.stringify(celiloCliPath)} package ${JSON.stringify(absPath)} --output ${JSON.stringify(netappPath)}`,
+              { timeout: 180_000 },
+            );
+          } catch (err) {
+            const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);
+            throw new Error(
+              `Failed to package module ${moduleId} at ${absPath} after ${seconds}s (no current staged netapp, so it was rebuilt inside a live stack): ${String(err)}`,
+            );
+          }
         }
       }
 
@@ -1029,7 +1529,9 @@ function buildNetworkHandle(
         if (cleanup)
           try {
             execSync(`rm -f ${JSON.stringify(netappPath)}`, { stdio: 'pipe' });
-          } catch {}
+          } catch (err) {
+            logCleanupFailure(`rm staged netapp ${netappPath}`, err);
+          }
       }
     },
 
@@ -1046,12 +1548,18 @@ function buildNetworkHandle(
       for (const browser of activeBrowsers) {
         try {
           await browser.close();
-        } catch {}
+        } catch (err) {
+          // A browser that refuses to close must not stop the network teardown
+          // below — the leaked resource is the network, not the browser.
+          logCleanupFailure('browser close', err);
+        }
       }
       for (const proxy of activeProxies) {
         try {
           await proxy.stop();
-        } catch {}
+        } catch (err) {
+          logCleanupFailure('proxy stop', err);
+        }
       }
 
       // Respect --keep / --reuse: don't tear down the network
@@ -1061,13 +1569,30 @@ function buildNetworkHandle(
         return;
       }
       console.log('[progress:start] stopping network | network stopped');
+      // Kill this project's sim-created guests BEFORE compose down. A guest
+      // holds its zone network's endpoint and the ssh-keys volume, so compose
+      // down with the guest alive fails both removals, the error is swallowed
+      // below, and the subnet stays allocated for every later suite
+      // (celilo#1247).
+      try {
+        const guests = run(projectTeardownCommands(projectName).listGuests);
+        if (guests.trim()) {
+          run(`docker rm -f ${guests.replace(/\n/g, ' ')}`, { timeout: 30_000 });
+        }
+      } catch (err) {
+        // Compose down below still runs; without the guest gone it fails both
+        // removals (celilo#1247), so the failure is worth naming, not absorbing.
+        logCleanupFailure(`guest sweep ${projectName}`, err);
+      }
       try {
         run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`, {
           cwd: composeDir,
           timeout: 60_000,
         });
-      } catch {
-        // Best effort cleanup
+      } catch (err) {
+        // Best effort cleanup: stop() must never fail the caller over teardown.
+        // The failure is logged so a systematically broken compose is visible.
+        logCleanupFailure(`compose down ${projectName}`, err);
       }
       activeProjects.delete(projectName);
     },
@@ -1103,47 +1628,15 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     await scrubDnsZones();
   }
 
-  // Clean up stale per-test containers (self-healing from prior crashes)
-  console.log('[progress:start] cleaning up stale test resources | cleanup complete');
-  try {
-    // Force-kill per-test containers (not shared infra)
-    try {
-      const containers = run('docker ps -aq --filter "name=celilo-e2e-1" 2>/dev/null');
-      if (containers.trim()) {
-        run(`docker rm -f ${containers.replace(/\n/g, ' ')}`, { timeout: 30_000 });
-      }
-    } catch {}
-
-    // Remove stale per-test networks AND their volumes, via the same
-    // by-name teardown the exit handler uses. This used to lean on
-    // `docker compose -p <project> down --volumes` for the volume half, which
-    // has no `-f` and no reliable cwd here either — compose exits "no
-    // configuration file provided" and the catch swallows it, so volumes were
-    // never removed on this path either. Reusing forceRemoveProject keeps the
-    // crash-recovery sweep and the exit handler from drifting apart.
-    try {
-      const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
-        .split('\n')
-        .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
-      const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
-      for (const project of projects) {
-        forceRemoveProject(project);
-      }
-    } catch {}
-    // Volumes can outlive every container and network of their project — that
-    // is exactly the 58-volume pile — so sweep by prefix too, not only for
-    // projects that still have a network to be discovered by.
-    try {
-      const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
-        .split('\n')
-        .filter(Boolean);
-      for (const vol of vols) {
-        try {
-          run(`docker volume rm ${vol}`, { timeout: 5_000 });
-        } catch {}
-      }
-    } catch {}
-  } catch {}
+  // Clean up stale per-test containers (self-healing from prior crashes).
+  // The sweep is extracted so it is testable; the progress line is honest by
+  // construction: "cleanup complete" only when the sweep reported zero
+  // failures (ce-yuzi).
+  console.log('[progress:start] cleaning up stale test resources | sweeping');
+  const cleanupFailures = sweepStaleTestResources();
+  console.log(
+    `[progress:start] cleaning up stale test resources | ${cleanupProgress(cleanupFailures)}`,
+  );
 
   const projectName = `celilo-e2e-${Date.now()}`;
   activeProjects.add(projectName);
@@ -1155,9 +1648,21 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   const yaml = generateTestComposeYaml(config, celiloRoot);
   writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
-  // Build and start per-test containers only — stream output so the runner
-  // can show which service is currently being built (can take minutes on cold cache).
-  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
+  // The compose carries `image:` only (never `build:`), so the run-time path
+  // cannot build and cannot resolve a FROM from docker.io — the fetch that
+  // made 30 outputs fail with `lookup auth.docker.io` (tasks.md 5b.2). A
+  // missing baked tag is a loud failure naming the remedy. Checked for BOTH
+  // compose files here, before shared infra comes up, so an unpopulated
+  // machine is named in seconds rather than discovered mid-`up`.
+  const missing = missingImages([
+    ...referencedImages(yaml),
+    ...referencedImages(generateSharedInfraYaml()),
+  ]);
+  if (missing.length > 0) {
+    throw new Error(
+      `${missing.length} baked image(s) missing from the local store, and suite time never builds: ${missing.join(', ')}.\nRemedy: cele2e build-infra (it owns the network phase — docker base images, apt, caddy, docker-ce). The suite run stays hermetic.`,
+    );
+  }
 
   console.log('[progress:start] starting containers | containers running');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, {
@@ -1346,6 +1851,15 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   if (config.managementVariant !== 'vanilla') {
     assertCliVersion(projectName, composeDir);
     console.log('[progress:start] initializing celilo | celilo initialized');
+    // dns.fallback is deliberately NOT set. The sim hosts exactly one resolver
+    // the fleet may name — comcast-resolver at 203.0.113.1 — so the
+    // pre-resolver birth list is just that address. The former value
+    // (1.0.0.1,8.8.8.8) named addresses that exist nowhere in the sim: every
+    // lookup to them ate its full timeout, and ansible's Gathering Facts does
+    // reverse lookups, which is the 600s hang in celilo#1290. The sim's
+    // second public resolver (SIMULATOR_IPS.PUBLIC_RESOLVER) stays OUT of
+    // fleet config on purpose: the public_dns check needs a resolver celilo
+    // does not itself use.
     // Honest harness (openspec/specs/progressive-zone-disclosure/spec.md): seed ONLY the
     // `internal` zone — the network the management box is genuinely on —
     // plus DNS. dmz/app/secure are NOT pre-seeded; they come into being
@@ -1366,12 +1880,12 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
           config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
             ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
-        }dns.primary=203.0.113.1 \
-        dns.fallback=1.0.0.1,8.8.8.8`,
+        }dns.primary=203.0.113.1`,
     );
     if (initResult.exitCode !== 0) {
       throw new Error(`Celilo init failed: ${initResult.stderr}`);
     }
+    assertFleetNameserversAreSimRoutable(projectName, composeDir);
 
     // Start the event-bus dispatcher (ISS-0035 / ISS-0042), now that `system
     // init` has created the bus DB. Deploys emit bus events —
@@ -1429,6 +1943,28 @@ function topologyFromComposeFile(composePath: string): TopologyPreset {
   }
 }
 
+/**
+ * Whether the generated compose wires fw-main to the secure-mgmt control-plane
+ * network. The generator adds that leg as a post-step after building the
+ * topology services, so it is invisible to `firewallZoneLegs(topology)` — which
+ * is how a wired-but-undeclared eth4 reached the iptables converge and got
+ * refused (ce-fvxd). Exported for the leg-coherence test.
+ */
+export function fwMainHasSecureMgmtLeg(composePath: string): boolean {
+  try {
+    const compose = parseYaml(readFileSync(composePath, 'utf-8')) as {
+      services?: Record<string, { networks?: Record<string, unknown> }>;
+    };
+    return Boolean(compose.services?.['fw-main']?.networks?.['secure-mgmt']);
+  } catch {
+    // An unreadable compose file means the stack was not built by this
+    // generator. Reporting "no leg" reproduces the under-declaration, but the
+    // converge refuses loudly on the unaccounted interface, so the failure is
+    // not silent — the same trade `topologyFromComposeFile` makes.
+    return false;
+  }
+}
+
 export function reconnectNetwork(projectName: string): NetworkHandle {
   const composeDir = PACKAGE_ROOT;