@celilo/e2e 0.20.0 → 0.20.2

package/bin/e2e-bake-management

@@ -33,10 +33,12 @@ import {
   stageWebsiteDist,
 } from '../src/stage-simulator-inputs';
 import {
+  CONSUMER_FINGERPRINT_PREFIX,
   PUBLISHED_FINGERPRINT_PREFIX,
   SOURCE_LABEL,
 } from '../src/source-fingerprint';
 import { readFileSync } from 'node:fs';
+import { ZONE_GATEWAYS, greenwaveRouterIp } from '../src/types';
 
 const PACKAGE_ROOT = join(import.meta.dir, '..');
 const COMPOSE_FILE = join(PACKAGE_ROOT, 'docker-compose.test.yml');
@@ -209,8 +211,25 @@ function simStampFromManifest(): string {
  * whether the CLI under test is the code in the working tree. See
  * `src/source-fingerprint.ts` for why that has to be measured, not remembered.
  */
+/**
+ * `celilo --version` prints "celilo <semver>", and `docker commit --change
+ * "LABEL name=value"` splits on whitespace, so stamping the raw output fails
+ * the commit with `Syntax error - can't find = in "2.2.0"`. Take the last
+ * field, so the label carries the version alone (celilo#1318).
+ */
+function labelSafeVersion(version: string): string {
+  return version.trim().split(/\s+/).pop() || 'unknown';
+}
+
 function sourceStamp(published: boolean, version: string): string {
-  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${version || 'unknown'}`;
+  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
+  // Consumer mode: restageSimulatorInputs is the only writer of
+  // pack-manifest.json and it is skipped without a checkout, so there is no
+  // tree fingerprint to read — and none to record, since the tree does not
+  // exist. Stamp the version the bake just installed and verified (celilo#1318).
+  if (!findMonorepoRoot(PACKAGE_ROOT)) {
+    return `${CONSUMER_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
+  }
   return simStampFromManifest();
 }
 
@@ -229,9 +248,16 @@ function sourceStamp(published: boolean, version: string): string {
 function restageSimulatorInputs(): void {
   const repoRoot = findMonorepoRoot(PACKAGE_ROOT);
   if (!repoRoot) {
-    throw new Error(
-      'The default bake installs the monorepo DEV cli via the sim registry — it needs a celilo checkout. For a published-CLI bake use --published.',
-    );
+    // Consumer mode (no checkout): there is no tree to repack, and nothing to
+    // correct. stageFromPublic already fetched THIS run's tarballs and site
+    // dist into the sim caches, and the sim images were built from them before
+    // the bake started. celilo#1299 is a monorepo-mode hazard — there the bake
+    // would otherwise reinstall the previous build's cached CLI — so the
+    // restage is redundant here, not impossible-but-required. Throwing broke
+    // npm-consumer-smoke, whose whole point is a bake with no monorepo source
+    // (celilo#1318).
+    console.log('  restage ................ skipped (consumer mode: inputs staged this run)');
+    return;
   }
   stageWebsiteDist(repoRoot, PACKAGE_ROOT);
   packNpmRegistryTarballs(repoRoot, PACKAGE_ROOT);
@@ -331,8 +357,23 @@ async function bakeViaSim(): Promise<void> {
       throw new Error('Could not resolve management container id');
     }
     const superseded = imageIdOnTag('celilo-e2e/management:latest');
+    // The committed container runs under the sim topology's compose env, and
+    // commit captures its Env. Since the default managementZone became
+    // secure-mgmt, that baked FW_MAIN_HOP=10.226.120.1 (and the control-plane
+    // default gateway) into :latest — every internal-topology suite then
+    // inherited a nexthop that is not on-link from the internal LAN and the
+    // mgmt box crash-looped in management-routes.sh (celilo#1351). Reset the
+    // per-topology routing vars to the internal-topology defaults at commit
+    // time, so the image is topology-neutral; every compose sets both anyway.
     run(
-      `docker commit --change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(false, version)}`)} ${containerId} celilo-e2e/management:latest`,
+      [
+        'docker commit',
+        `--change ${JSON.stringify(`ENV DEFAULT_GATEWAY=${greenwaveRouterIp()}`)}`,
+        `--change ${JSON.stringify(`ENV FW_MAIN_HOP=${ZONE_GATEWAYS.internal}`)}`,
+        `--change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(false, version)}`)}`,
+        containerId,
+        'celilo-e2e/management:latest',
+      ].join(' '),
     );
     console.log(`✔ ${Math.round((Date.now() - t5) / 1000)}s`);
     removeSupersededImage(superseded, 'celilo-e2e/management:latest');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.20.0",
+  "version": "0.20.2",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -20,7 +20,8 @@
   "scripts": {
     "test": "bun test --timeout 30000 ./tests/ ./src/ ./npm-registry-server/ ./simulators/",
     "test:completion": "bun test tests/completion",
-    "test:integration": "bun test tests-integration/"
+    "test:integration": "bun test tests-integration/",
+    "test:weekly": "bun test --timeout 30000 ./weekly/"
   },
   "files": [
     "src/",
@@ -37,7 +38,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@celilo/capabilities": "^4.2.0",
+    "@celilo/capabilities": "^4.3.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/event-bus": "^0.6.0",
     "@celilo/terraform-fake": "^0.3.1",

package/src/block-timing.ts

@@ -35,10 +35,31 @@ export type BlockTiming = Record<string, Record<string, number>>;
 /**
  * A failed block below this did no work, so its duration measures nothing.
  *
- * Nothing that stands up a container, deploys a module or waits on DNS returns
- * in 50ms. A `requireStage` skip returns in a fraction of one.
+ * Measured, not guessed. Over the 58 recorded runs in `e2e/results/`, every
+ * failed block falls in one of two clumps with nothing in between:
+ *
+ *   did no work (a `requireStage` throw)   0.00ms .. 43.6ms
+ *   did real work                          over 2s, up to 140s
+ *
+ * The gap is narrower than those two clumps suggest, because the fastest real
+ * work seen anywhere is not 2s. `CASCADE_STDOUT` in the test file pins a real
+ * failure at 123.54ms, and celilo#1281 measured real blocks from 145ms. So the
+ * floor has to sit between roughly 44ms and 123ms, and where it sits decides
+ * which of two mistakes this file makes.
+ *
+ * At 50 the clearance over the worst observed skip is 1.15x. That is thin
+ * enough that one slow skip enters the duration record as a healthy fast
+ * block, which is the mistake that matters: it pollutes the baseline every
+ * later run is compared against. At 100 the clearance is 2.3x, and the cost is
+ * that a REAL failure under 100ms would be dropped. That is the cheaper
+ * mistake. This file exists to find blocks running OUT of budget, and one
+ * using 0.03% of its cap is not a candidate.
+ *
+ * Nothing observed is reclassified by the move. No failed block in the recorded
+ * runs sits between 43.6ms and 2s, and no fixture sits between 39.19ms and
+ * 123.54ms, so both keep their current verdict.
  */
-const NOT_A_MEASUREMENT_MS = 50;
+const NOT_A_MEASUREMENT_MS = 100;
 
 export function parseBlockDurations(lines: string[], junitXml?: string): Record<string, number> {
   const failed = failedBlocks(lines);

package/src/cli/build.test.ts

@@ -1,10 +1,22 @@
 import { afterEach, beforeEach, expect, test } from 'bun:test';
-import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  utimesSync,
+  writeFileSync,
+} from 'node:fs';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { basename, join } from 'node:path';
 import { gzipSync } from 'node:zlib';
 import {
   assertGzipValid,
+  bakeManagement,
+  packageNetapp,
+  reportBakeChildFailure,
   stageNetappsFromRegistry,
   verifyNetapp,
   verifyStagedNetapps,
@@ -21,7 +33,7 @@ beforeEach(() => {
 afterEach(() => {
   globalThis.fetch = realFetch;
   rmSync(dir, { recursive: true, force: true });
-  process.env.CELILO_REGISTRY_URL = undefined;
+  delete process.env.CELILO_REGISTRY_URL;
 });
 
 test('fetches each module latest version to <name>.netapp via the download endpoint', async () => {
@@ -95,3 +107,130 @@ test('verifyStagedNetapps accepts a valid .netapp and refuses a truncated one',
   expect(() => verifyNetapp(join(dir, 'technitium.netapp'))).toThrow(/technitium\.netapp/s);
   expect(() => assertGzipValid(join(dir, 'technitium.netapp'))).toThrow(/unexpected end of file/s);
 });
+
+/**
+ * Stub process.exit and console.error around a call expected to terminate.
+ * Returns the exit codes collected and everything written to stderr.
+ */
+function captureTerminalExit(run: () => void): { exitCodes: unknown[]; stderr: string } {
+  const realExit = process.exit;
+  const realError = console.error;
+  const exitCodes: unknown[] = [];
+  const stderrLines: string[] = [];
+  (process as unknown as { exit: (code?: number) => never }).exit = ((code?: number) => {
+    exitCodes.push(code);
+    throw new Error(`process.exit(${code})`);
+  }) as typeof process.exit;
+  console.error = (...args: unknown[]) => {
+    stderrLines.push(args.map(String).join(' '));
+  };
+  try {
+    try {
+      run();
+    } catch (err) {
+      // The stubbed exit throws to unwind; anything else is a real failure.
+      if (!String(err).startsWith('Error: process.exit(')) throw err;
+    }
+  } finally {
+    process.exit = realExit;
+    console.error = realError;
+  }
+  return { exitCodes, stderr: stderrLines.join('\n') };
+}
+
+// celilo#1302: a lock-free live stack on the builder made the bake child's
+// startup cleanup refuse (exit 3), and the parent reported "Bake step failed"
+// with three install.sh causes for a step that never executed.
+/**
+ * Stub console.log/console.error around `run`, returning what each captured.
+ */
+function captureConsole(run: () => void): { stdout: string; stderr: string } {
+  const realLog = console.log;
+  const realError = console.error;
+  const out: string[] = [];
+  const err: string[] = [];
+  console.log = (...args: unknown[]) => {
+    out.push(args.map(String).join(' '));
+  };
+  console.error = (...args: unknown[]) => {
+    err.push(args.map(String).join(' '));
+  };
+  try {
+    run();
+  } finally {
+    console.log = realLog;
+    console.error = realError;
+  }
+  return { stdout: out.join('\n'), stderr: err.join('\n') };
+}
+
+// celilo#1258: build-infra is mandatory after cele2e down, and it used to
+// repackage all 37 modules from scratch every run (about 5 minutes to reach a
+// 38 second test) because nothing reused a current .netapp.
+test('a second build-infra with no source change skips repackaging', () => {
+  const moduleDir = mkdtempSync(join(tmpdir(), 'module-fixture-'));
+  const netappsDir = mkdtempSync(join(tmpdir(), 'netapps-fixture-'));
+  writeFileSync(join(moduleDir, 'main.sh'), 'echo hello');
+  // First run's output: staged after the source, so it is current.
+  const staged = join(netappsDir, `${basename(moduleDir)}.netapp`);
+  writeFileSync(staged, 'netapp-bytes');
+
+  const { stdout, stderr } = captureConsole(() => packageNetapp(moduleDir, netappsDir));
+  expect(stdout).toContain('current, skipped');
+  // The skip returns before any CLI lookup: no packaging attempt happened.
+  expect(stderr).not.toContain('celilo CLI not found');
+
+  // Recurrence gate: touching a shipped source file flips the answer, and the
+  // same call proceeds toward packaging. In a tmpdir there is no monorepo CLI,
+  // so getting past the skip surfaces as the CLI-not-found report; the point
+  // is that the skip did not fire.
+  const later = new Date(Date.now() + 60_000);
+  const touched = join(moduleDir, 'main.sh');
+  utimesSync(touched, later, later);
+  const { stdout: stdoutAfterTouch, stderr: stderrAfterTouch } = captureConsole(() =>
+    packageNetapp(moduleDir, netappsDir),
+  );
+  expect(stderrAfterTouch).toContain('celilo CLI not found');
+  expect(stdoutAfterTouch).not.toContain('current, skipped');
+
+  rmSync(moduleDir, { recursive: true, force: true });
+  rmSync(netappsDir, { recursive: true, force: true });
+});
+
+test('a refusal exit (3) from the bake child reports the bake did not run, not a bake failure', () => {
+  const { exitCodes, stderr } = captureTerminalExit(() => reportBakeChildFailure(3, 0));
+  expect(exitCodes).toEqual([3]);
+  expect(stderr).toContain('did not run');
+  expect(stderr).toContain('cele2e down');
+  // The wrong hint must not appear: these causes are all false when the bake
+  // never started.
+  expect(stderr).not.toContain('Likely causes');
+  expect(stderr).not.toContain('install.sh regressed');
+});
+
+test('a genuine bake failure (exit 1) keeps the bake-failure report and its likely causes', () => {
+  const { exitCodes, stderr } = captureTerminalExit(() => reportBakeChildFailure(1, 12));
+  expect(exitCodes).toEqual([1]);
+  expect(stderr).toContain('Bake step failed after 12s');
+  expect(stderr).toContain('Likely causes');
+  expect(stderr).not.toContain('did not run');
+});
+
+test('bakeManagement classifies a real child exit 3 end to end and exits 3', () => {
+  // A fake bake script that behaves like the real one does when the startup
+  // cleanup refuses: print the refusal to stderr, exit 3.
+  const pkgDir = mkdtempSync(join(tmpdir(), 'bake-child-'));
+  mkdirSync(join(pkgDir, 'bin'), { recursive: true });
+  writeFileSync(
+    join(pkgDir, 'bin', 'e2e-bake-management'),
+    'console.error("refusing to clean up"); process.exit(3);\n',
+  );
+  try {
+    const { exitCodes, stderr } = captureTerminalExit(() => bakeManagement(pkgDir, false));
+    expect(exitCodes).toEqual([3]);
+    expect(stderr).toContain('did not run');
+    expect(stderr).not.toContain('Likely causes');
+  } finally {
+    rmSync(pkgDir, { recursive: true, force: true });
+  }
+});

package/src/cli/build.ts

@@ -35,6 +35,7 @@ import { gunzipSync } from 'node:zlib';
 import { stageAptRepo } from '../../scripts/stage-apt-repo';
 import { stageLibsignal } from '../../scripts/stage-libsignal';
 import { explainBuildFailure } from '../doctor';
+import { isNetappCurrent } from '../netapp-staleness';
 import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../registry-bundle';
 import { findMonorepoRoot } from '../repo-root';
 import { packNpmRegistryTarballs, stageWebsiteDist } from '../stage-simulator-inputs';
@@ -433,7 +434,12 @@ export async function fetchSiteFile(
   writeFileSync(join(destDir, name), Buffer.from(await res.arrayBuffer()));
 }
 
-function packageNetapp(moduleDir: string, netappsDir: string): void {
+/**
+ * Package one module directory into `netappsDir` as `<name>.netapp`, skipping
+ * the work entirely when the staged netapp is already current over the source
+ * (celilo#1258). Exported for its recurrence test.
+ */
+export function packageNetapp(moduleDir: string, netappsDir: string): void {
   const absDir = resolve(moduleDir);
   if (!existsSync(absDir)) {
     console.error(`  ${red}skip${reset}  ${moduleDir}  ${dim}(not found)${reset}`);
@@ -443,6 +449,15 @@ function packageNetapp(moduleDir: string, netappsDir: string): void {
   const name = basename(absDir);
   const out = join(netappsDir, `${name}.netapp`);
 
+  // Skip a module whose staged .netapp is newer than every source file
+  // (celilo#1258): repackaging it again would reproduce the same bytes, and
+  // doing that for all 37 modules cost about 5 minutes on every build-infra
+  // run. A source edit flips the newest mtime and the module repackages.
+  if (isNetappCurrent(out, absDir)) {
+    console.log(`  ${String(name).padEnd(20)} ${dim}· current, skipped${reset}`);
+    return;
+  }
+
   process.stdout.write(`  ${String(name).padEnd(20)} `);
   const start = Date.now();
 
@@ -684,7 +699,7 @@ function tagVanillaManagement(): void {
  * build-infra so the operator can't accidentally proceed with a
  * non-functional default management image.
  */
-function bakeManagement(pkgDir: string, published: boolean): void {
+export function bakeManagement(pkgDir: string, published: boolean): void {
   const bakeScript = join(pkgDir, 'bin', 'e2e-bake-management');
   if (!existsSync(bakeScript)) {
     console.error(
@@ -705,16 +720,43 @@ function bakeManagement(pkgDir: string, published: boolean): void {
   const elapsed = Math.round((Date.now() - start) / 1000);
 
   if (result.status !== 0) {
-    console.error(`\n${red}Bake step failed after ${elapsed}s${reset}`);
-    console.error(
-      `${dim}Likely causes: install.sh regressed, npm-registry-sim doesn't have the expected @celilo/* tarballs, or celilo-website-sim isn't serving install.sh. Run \`cele2e run install-sh\` for a focused reproducer with cleaner output.${reset}`,
-    );
-    process.exit(1);
+    console.log(`${red}✗${reset}  ${dim}${elapsed}s${reset}`);
+    // status is null when the child died to a signal — that is a genuine
+    // failure, not a refusal, so it falls through to the exit-1 report.
+    reportBakeChildFailure(result.status ?? 1, elapsed);
   }
 
   console.log(`\n${green}Baked in ${elapsed}s${reset}\n`);
 }
 
+/**
+ * Report a non-zero bake-child exit and terminate. Exit 3 is the live-stack
+ * refusal convention (celilo#1297 guard; the run runner keys on the same code
+ * in its `refused` field): the bake child's startup cleanup found a live
+ * celilo-e2e-* stack or a foreign run lock and refused before the bake did
+ * any work. Reporting that as a bake failure named three causes — install.sh
+ * rot, registry tarballs, website sim — for a step that never executed
+ * (celilo#1302: 9 of ~111 smoke runs, every refusal diagnosed as install.sh
+ * rot). A refusal is not a bake failure, so it gets its own message and exits
+ * 3 so callers can tell the two apart.
+ */
+export function reportBakeChildFailure(status: number, elapsed: number): never {
+  if (status === 3) {
+    console.error(
+      `\n${red}Bake step did not run — the startup cleanup refused: a live e2e stack is in the way.${reset}`,
+    );
+    console.error(
+      `${dim}A refusal is not a bake failure; install.sh was never exercised. Clear the stack with \`cele2e down\` (or free the foreign run lock) and re-run \`cele2e build-infra\`.${reset}`,
+    );
+    process.exit(3);
+  }
+  console.error(`\n${red}Bake step failed after ${elapsed}s${reset}`);
+  console.error(
+    `${dim}Likely causes: install.sh regressed, npm-registry-sim doesn't have the expected @celilo/* tarballs, or celilo-website-sim isn't serving install.sh. Run \`cele2e run install-sh\` for a focused reproducer with cleaner output.${reset}`,
+  );
+  process.exit(1);
+}
+
 /**
  * Pre-seed heavy application images (authentik server, postgres, redis) into
  * the app-zone preload cache. Without this the `docker-image-cache` is empty on

package/src/container-manager.ts

@@ -22,7 +22,7 @@ import {
   referencedImages,
   registryUploadsHostDir,
 } from './docker-compose-generator';
-import { type ModuleHost, parseModuleHost } from './module-host';
+import { type ModuleHost, parseModuleHost, parseModuleWhere } from './module-host';
 import { CONTAINER_PREFIX, GUEST_PROJECT_LABEL } from './proxmox-provisioner';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
@@ -1110,6 +1110,23 @@ function buildNetworkHandle(
       return host;
     },
 
+    async targetIp(moduleId: string): Promise<string> {
+      const where = dockerExec(
+        projectName,
+        composeDir,
+        'management',
+        `celilo module where ${moduleId} --json`,
+      );
+      const addresses = parseModuleWhere(where.stdout);
+      if (addresses.length === 0) {
+        throw new Error(
+          `Could not resolve an address for '${moduleId}'. Its deploy did not record one in the inventory\n` +
+            `(or the CLI answered something unparsable). Raw:\n${where.stdout.slice(0, 400)}`,
+        );
+      }
+      return addresses[0];
+    },
+
     async execOnModuleHost(moduleId, cmd, timeoutMs = 60_000): Promise<ExecResult> {
       const host = await this.moduleHost(moduleId);
       return host.reach === 'plain'
@@ -1280,6 +1297,35 @@ function buildNetworkHandle(
         await handle.celilo(`system config set network.${zone}.gateway ${ZONE_GATEWAYS[zone]}`);
       }
 
+      // When fw-main carries the secure-mgmt leg, the management box sits on the
+      // control-plane network behind this firewall, and the firewall must TRUST
+      // that subnet or default-DROP drops the SSH `machine add` and every later
+      // hook/converge needs (celilo#1353: 19 suites died at machine add with a
+      // misleading key-mismatch message; the live stack showed the packet
+      // timing out in fw-main's FORWARD chain, policy DROP, trusted sources =
+      // internal only).
+      //
+      // The firewall DERIVES control-plane trust from where the celilo-mgmt
+      // module is deployed (apps/celilo/src/hooks/capability-loader.ts
+      // loadControlPlaneSubnet), falling back to the internal subnet. Suites
+      // that machine-add before any celilo-mgmt deploy have neither, so the
+      // declared control-plane subnet is trusted by nothing. The operator in
+      // this topology declares it explicitly — `firewall.trusted_subnets` is the
+      // product surface for exactly that (composeTrustedSubnets origin:
+      // operator-override) — and the harness models that operator rather than
+      // widening the product's fallback, which the approved design
+      // (openspec/changes/recognize-management-network, D2) deliberately kept
+      // as "previous behaviour + report".
+      //
+      // Idempotent with the derived path: composeTrustedSubnets dedupes by
+      // subnet, so a suite that later deploys celilo-mgmt on secure-mgmt renders
+      // the same ruleset.
+      if (fwMainHasSecureMgmtLeg(join(composeDir, COMPOSE_FILE))) {
+        await handle.celilo(
+          `system config set firewall.trusted_subnets ${ZONE_SUBNETS['secure-mgmt']}`,
+        );
+      }
+
       // fw-main is registered as an internal-zone machine; iptables deploys to it.
       await handle.celilo(
         `machine add ${firewallIp} --ssh-user root --ssh-key-file /root/.ssh/id_ed25519 --zone internal`,

package/src/docker-compose-generator.ts

@@ -625,8 +625,9 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
     dmz: zoneNetworkDef('dmz'),
     app: zoneNetworkDef('app'),
     secure: zoneNetworkDef('secure'),
-    // Only when celilo-mgr lives off the internal LAN — keeps the default
-    // topology's generated compose byte-identical for every existing test.
+    // Only when the control plane is in use (celilo-mgr, a secure-mgmt
+    // machine, or the proxmox simulator). With the default managementZone of
+    // `secure-mgmt` this is now present in every generated compose.
     ...(needsSecureMgmt ? { 'secure-mgmt': zoneNetworkDef('secure-mgmt') } : {}),
     'isp-external': networkDef('203.0.113.0/24', '203.0.113.250'),
     // internet-external is owned by shared infra; real-internet is per-test
@@ -747,8 +748,27 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
         // On its own control-plane network the only router in reach is fw-main's
         // leg there — both for the default route and for the segmented zones.
         DEFAULT_GATEWAY: zone === 'secure-mgmt' ? SECURE_MGMT_GATEWAY : managementDefaultGw,
-        ...(zone === 'secure-mgmt' ? { FW_MAIN_HOP: SECURE_MGMT_GATEWAY } : {}),
+        // BOTH branches always set the hop, never let the image's baked value
+        // through: the bake `docker commit`s the management container, so a
+        // bake from the secure-mgmt default baked FW_MAIN_HOP=10.226.120.1
+        // into :latest, and every internal-topology suite then inherited a
+        // nexthop that is not on-link from the internal LAN. The mgmt box
+        // exited 2 at the segmented-zone routes in management-routes.sh and
+        // crash-looped (celilo#1351). Compose env overrides image env, so an
+        // always-present FW_MAIN_HOP immunizes every topology against any
+        // baked value, current and future.
+        FW_MAIN_HOP: zone === 'secure-mgmt' ? SECURE_MGMT_GATEWAY : ZONE_GATEWAYS.internal,
         CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
+        // The hook jail is REQUIRED in the e2e, not `auto` (peba, 2026-09-08).
+        // ce-rez7 made an unset policy resolve to `off`, which silently turned
+        // hook-jail-trespass into a test of nothing: its stage 2 (no policy set)
+        // and its stage 3 control (explicit `off`) became the same experiment.
+        // celilo#1329. `required` beats `auto` here because a missing bubblewrap
+        // backend is then a hard failure instead of a silent drop to unjailed, so
+        // it cannot rot the same way twice. A per-command `CELILO_HOOK_JAIL=off`
+        // prefix still overrides this, which is how the deliberate unjailed
+        // controls in both jail suites keep working.
+        CELILO_HOOK_JAIL: 'required',
         // `cele2e run --source-cli` sets this, and the image's shim reads it to
         // run the mounted workspace instead of the CLI install.sh installed.
         // Off by default: the installed CLI is the artifact under test, and
@@ -979,6 +999,16 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     environment: {
       DEFAULT_GATEWAY: managementDefaultGw,
       CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
+      // The hook jail is REQUIRED in the e2e, not `auto` (peba, 2026-09-08).
+      // ce-rez7 made an unset policy resolve to `off`, which silently turned
+      // hook-jail-trespass into a test of nothing: its stage 2 (no policy set)
+      // and its stage 3 control (explicit `off`) became the same experiment.
+      // celilo#1329. `required` beats `auto` here because a missing bubblewrap
+      // backend is then a hard failure instead of a silent drop to unjailed, so
+      // it cannot rot the same way twice. A per-command `CELILO_HOOK_JAIL=off`
+      // prefix still overrides this, which is how the deliberate unjailed
+      // controls in both jail suites keep working.
+      CELILO_HOOK_JAIL: 'required',
     },
   });
 

package/src/extract-failure.ts

@@ -28,26 +28,48 @@ export interface StageTally {
  * as "9 failed" in a 10-stage suite — nine counts of a defect that does not
  * exist, and a summary that buries the one that does.
  *
- * The `Skipped:` marker is the semantic signal, so that is what we key on; the
- * sub-millisecond durations those stages show are a symptom, not the contract.
+ * The `error: Skipped:` line is the semantic signal, and bun prints it in BOTH
+ * orders relative to the `(fail)` marker: a thrown error's block (source
+ * excerpt, error line, stack) lands BEFORE its marker, while a top-level error
+ * print and bun's own timeout pointer land AFTER it. A one-directional window
+ * misattributes both ways — measured as ce-59f9: a window after a timeout
+ * marker catches the NEXT test's excerpt quoting the `Skipped:` template (and
+ * calls the timeout a skip), and a skip's reason pushed past the window calls
+ * the skip a failure. So each reason line is attributed to its NEAREST marker
+ * instead, ties to the earlier one (a thrown-error block sits closest to the
+ * marker that follows it), and a marker no reason line claims is a failure on
+ * its own merits.
  */
 export function tallyStages(lines: string[]): StageTally {
   const plain = lines.map(stripAnsi);
   const isFailure = (l: string): boolean => /^\(fail\)\s+\S/.test(l);
-  let failed = 0;
-  let skipped = 0;
+  const isSkippedReason = (l: string): boolean => /^\s*error:\s*Skipped:\s/.test(l);
+
+  const markers: number[] = [];
+  for (let i = 0; i < plain.length; i++) {
+    if (isFailure(plain[i])) markers.push(i);
+  }
+
+  const skippedMarkers = new Set<number>();
+  const MAX_REASON_DISTANCE = 8;
   for (let i = 0; i < plain.length; i++) {
-    if (!isFailure(plain[i])) continue;
-    // bun prints the reason on the following lines, indented under the failure.
-    // The window MUST stop at the next failure: cascade-skips come in runs, and
-    // a window that reads past the boundary attributes the next stage's
-    // "Skipped:" to this one — which misreads the single real defect at the top
-    // of a cascade as just another skip, losing the only line worth reading.
-    const reason: string[] = [];
-    for (let j = i + 1; j < plain.length && j <= i + 4 && !isFailure(plain[j]); j++) {
-      reason.push(plain[j]);
+    if (!isSkippedReason(plain[i])) continue;
+    let nearest = -1;
+    let nearestDist = Number.POSITIVE_INFINITY;
+    for (const m of markers) {
+      const dist = Math.abs(m - i);
+      if (dist <= MAX_REASON_DISTANCE && dist < nearestDist) {
+        nearest = m;
+        nearestDist = dist;
+      }
     }
-    if (/(?:^|\W)Skipped:\s/.test(reason.join('\n'))) skipped++;
+    if (nearest >= 0) skippedMarkers.add(nearest);
+  }
+
+  let failed = 0;
+  let skipped = 0;
+  for (const m of markers) {
+    if (skippedMarkers.has(m)) skipped++;
     else failed++;
   }
   return { failed, skipped };