@celilo/e2e 0.19.3 → 0.20.0

package/src/run-args.ts

@@ -0,0 +1,89 @@
+import { basename } from 'node:path';
+
+/**
+ * Argument parsing for the e2e run script (runner.ts). Pure and side-effect
+ * free, so the runner's argv contract is unit-testable without touching
+ * Docker, the run-lock, or anything else main() reaches at import time.
+ */
+
+/**
+ * Flags `e2e-run` understands. `--seed=<n>` is value-carrying and recognized
+ * by prefix, not listed here.
+ */
+export const KNOWN_RUN_FLAGS: readonly string[] = [
+  '--keep',
+  '--reuse',
+  '--live',
+  '--published',
+  '--source-cli',
+  '--verbose',
+  '--ci',
+  '--no-ci',
+  '--complete',
+  // Passed through by `cele2e run` (harmless/no-op there, see cli/index.ts).
+  '--ci-safe',
+  '--notify',
+  '--shuffle',
+  '--no-interactive',
+];
+
+export interface ParsedRunArgs {
+  /** Positional suite-name patterns (substring match against test file names). */
+  patterns: string[];
+  /** `--`-prefixed arguments the runner does not know. Must be empty before a run starts. */
+  unknownFlags: string[];
+  /** True when `--help` was requested. */
+  helpRequested: boolean;
+}
+
+/**
+ * Split e2e-run argv into patterns and flags. Reports unrecognized `--` args
+ * in `unknownFlags` instead of discarding them: the old filter stripped every
+ * `--`-prefixed argument it did not know, so `cele2e run --help` (and any
+ * typo'd flag) left zero patterns, which is the runner's encoding of "run
+ * everything" — the one command an unsure operator types took the single most
+ * expensive resource in the city.
+ */
+export function parseRunArgs(argv: readonly string[]): ParsedRunArgs {
+  const patterns: string[] = [];
+  const unknownFlags: string[] = [];
+  let helpRequested = false;
+  for (const arg of argv) {
+    if (!arg.startsWith('--')) {
+      patterns.push(arg);
+      continue;
+    }
+    if (arg === '--help') {
+      helpRequested = true;
+    } else if (!isKnownFlag(arg)) {
+      unknownFlags.push(arg);
+    }
+  }
+  return { patterns, unknownFlags, helpRequested };
+}
+
+/**
+ * Filter collected test files by the run's positional patterns. A pattern
+ * that exactly equals a test name selects only that file; otherwise the old
+ * substring semantics apply. Without the exact rule, `cele2e run
+ * aspect-fanout caddy-internal-private` also ran aspect-fanout-new-systems:
+ * the substring OR matched the prefix whenever the top-level dir was in the
+ * collected set.
+ */
+export function filterTestFilesByPatterns(
+  files: readonly string[],
+  patterns: readonly string[],
+): string[] {
+  if (patterns.length === 0) return [...files];
+  const exactPatterns = new Set(
+    patterns.filter((p) => files.some((f) => basename(f, '.test.ts') === p)),
+  );
+  return files.filter((f) => {
+    const name = basename(f, '.test.ts');
+    return patterns.some((p) => (exactPatterns.has(p) ? name === p : name.includes(p)));
+  });
+}
+
+function isKnownFlag(arg: string): boolean {
+  return KNOWN_RUN_FLAGS.includes(arg) || arg.startsWith('--seed=');
+}

package/src/runner.ts

@@ -22,6 +22,16 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { basename, dirname, join, resolve } from 'node:path';
 import { type DisplayMode, ProgressDisplay } from '@celilo/cli-display';
 import { parse as parseYaml } from 'yaml';
+import {
+  type BlockTiming,
+  type OverBudgetBlock,
+  declaredBlockCaps,
+  mergeBlockTiming,
+  overBudgetBlocks,
+  parseBlockDurations,
+  serializeBlockTiming,
+  tightBlocks,
+} from './block-timing';
 import {
   emitRunCompleted,
   emitRunFailed,
@@ -33,6 +43,7 @@ import { diagnose, formatReport } from './doctor';
 import { type StageTally, extractFailureMessage, stripAnsi, tallyStages } from './extract-failure';
 import { writeLastRun } from './last-run';
 import { parseLine } from './parse-line';
+import { KNOWN_RUN_FLAGS, filterTestFilesByPatterns, parseRunArgs } from './run-args';
 import { E2eBusyError, acquireRunLock, markKept } from './run-lock';
 import { SIMULATOR_IPS } from './simulator-ips';
 
@@ -68,6 +79,8 @@ const testsPath = moduleDir
 /** Set by `cele2e run --all` so a module-dirs run ALSO picks up the repo's top-level e2e/tests/. */
 const topLevelTestsPath = process.env.E2E_TOP_LEVEL_TESTS;
 const timingFile = join(testDir, '.e2e-timing.json');
+/** Per-test-block durations. The 300s cap is per BLOCK, and nothing else records that. */
+const blockTimingFile = join(testDir, '.e2e-block-timing.json');
 const persistentFile = join(testDir, '.e2e-persistent.json');
 
 // Parse flags out of argv
@@ -76,6 +89,14 @@ const flagKeep = rawArgs.includes('--keep');
 const flagReuse = rawArgs.includes('--reuse');
 const flagLive = rawArgs.includes('--live');
 const flagPublished = rawArgs.includes('--published');
+// --source-cli: run the celilo CLI from the mounted workspace instead of the one
+// baked into the management image. Off by default because the baked CLI is the
+// artifact under test, and running the source over the bind mount roughly
+// doubles every celilo command's start-up (measured 0.14s vs 0.31s on virtiofs,
+// 0.16s vs 0.56s on sshfs), which a 25-40 command test pays each time. The
+// image's shim reads this from the container environment; the compose generator
+// forwards it. Use it to iterate without a build-infra.
+const flagSourceCli = rawArgs.includes('--source-cli');
 const flagVerbose = rawArgs.includes('--verbose');
 // CI mode: no animated spinner, no [progress:*] markers — just one clean
 // ✔/✗ line per step (plus sub-events) written straight to the log. Forgejo's
@@ -88,7 +109,40 @@ const flagVerbose = rawArgs.includes('--verbose');
 const flagCi = rawArgs.includes('--ci');
 const flagNoCi = rawArgs.includes('--no-ci');
 const ci = flagCi || (!flagNoCi && !!process.env.CI);
-const patterns = rawArgs.filter((a) => !a.startsWith('--'));
+const RUN_USAGE = `
+Usage: cele2e run [pattern...] [flags]
+
+Runs e2e suites. Each pattern is a substring matched against test file
+names; no pattern (bare) runs every suite in the current scope.
+
+Flags:
+  --keep            Keep network running after tests
+  --reuse           Reuse existing network if running
+  --live            Use live (non-simulated) internet
+  --published       Use published .netapp packages
+  --source-cli      Run the celilo CLI from the mounted workspace
+  --verbose         Verbose output
+  --ci / --no-ci    Force plain CI log output on/off (auto-on when CI env var set)
+  --complete        Include quarantined (cele2e-ci-unsafe) tests
+  --notify          Desktop notification when the run finishes
+  --shuffle [--seed=<n>]  Randomize test order; replay an order with its seed
+
+Full CLI help: cele2e --help   (run --all is a cele2e-level alias, not a runner flag)
+`;
+
+const parsedArgs = parseRunArgs(rawArgs);
+if (parsedArgs.helpRequested) {
+  console.log(RUN_USAGE);
+  process.exit(0);
+}
+if (parsedArgs.unknownFlags.length > 0) {
+  console.error(`Unknown flag(s): ${parsedArgs.unknownFlags.join(' ')}`);
+  console.error(
+    `Known flags: ${[...KNOWN_RUN_FLAGS, '--seed=<n>'].join(' ')}   (--help for run usage)`,
+  );
+  process.exit(2);
+}
+const patterns = parsedArgs.patterns;
 // Quarantine: tests whose file contains the marker `cele2e-ci-unsafe` (a comment
 // naming the reason + ISS) are known-failing / builder-flaky and tracked
 // separately, so they must never turn a broad run red. They are SKIPPED by
@@ -161,6 +215,7 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
+const yellow = '\x1b[33m';
 const reset = '\x1b[0m';
 
 const interactive =
@@ -182,6 +237,18 @@ function saveTiming(history: TimingHistory): void {
   writeFileSync(timingFile, `${JSON.stringify(history, Object.keys(history).sort(), 2)}\n`);
 }
 
+function loadBlockTiming(): BlockTiming {
+  try {
+    return JSON.parse(readFileSync(blockTimingFile, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+function saveBlockTiming(timing: BlockTiming): void {
+  writeFileSync(blockTimingFile, serializeBlockTiming(timing));
+}
+
 function formatDuration(secs: number): string {
   if (secs >= 60) return `${Math.floor(secs / 60)}m${Math.abs(secs) % 60}s`;
   return `${secs}s`;
@@ -202,12 +269,22 @@ interface TestResult {
   rawTail?: string;
   hadDebugPause?: boolean;
   projectName?: string;
+  /** Per-`test()` durations in ms, so the per-block cap has a durable record. */
+  blocks?: Record<string, number>;
   /**
    * Real stage failures vs stages a failing earlier stage blocked. Without the
    * split, one bad fixture line in stage 1 of a 10-stage suite reports as "9
    * failed" — nine counts of a defect that does not exist.
    */
   stages?: StageTally;
+  /**
+   * The test child exited 3: the live-stack guard refused to clean up because
+   * another run's stack is in the way (celilo#1297). Bun itself never exits 3
+   * — it uses 1 for failures — so the code is unambiguous. The run stops here:
+   * every remaining suite would hit the same refusal, and continuing would
+   * only pile more noise onto one cause.
+   */
+  refused?: boolean;
 }
 
 async function runTest(
@@ -219,6 +296,7 @@ async function runTest(
 ): Promise<TestResult> {
   const start = Date.now();
   const logFile = join(logDir, 'output.log');
+  const junitFile = join(logDir, 'junit.xml');
 
   display.reset(start, expectedDuration);
   let hadDebugPause = false;
@@ -238,6 +316,7 @@ async function runTest(
     if (flagPublished) {
       testEnv.CELILO_E2E_INFRA_ROOT = ''; // empty = force published mode
     }
+    if (flagSourceCli) testEnv.CELILO_E2E_SOURCE_CLI = '1';
     if (flagReuse) {
       // Read the persistent project and pass it to the test
       try {
@@ -252,12 +331,22 @@ async function runTest(
       }
     }
 
-    // 1 hour timeout per test (debug sessions can be long)
-    const proc = spawn('bun', ['test', '--timeout', '3600000', file], {
-      cwd: PKG_DIR,
-      stdio: ['pipe', 'pipe', 'pipe'],
-      env: testEnv,
-    });
+    // 1 hour timeout per test (debug sessions can be long). Each test() block's
+    // own third argument overrides this — measured on bun 1.3.3, including when
+    // the body swallows its own errors — so this is a backstop, not the budget.
+    //
+    // The JUnit report is the ONLY source of a duration for a block that PASSED:
+    // bun's console names a block only when it fails. It does not change the
+    // console output, so the display and the failure parsers are untouched.
+    const proc = spawn(
+      'bun',
+      ['test', '--timeout', '3600000', '--reporter=junit', `--reporter-outfile=${junitFile}`, file],
+      {
+        cwd: PKG_DIR,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: testEnv,
+      },
+    );
 
     let stdout = '';
     let stderr = '';
@@ -363,6 +452,7 @@ async function runTest(
 
       const expectMatch = stdout.match(/(\d+) expect\(\) calls/);
       const expectCount = expectMatch ? Number.parseInt(expectMatch[1], 10) : undefined;
+      const blocks = parseBlockDurations(lines, readJunit(junitFile));
 
       if (code === 0) {
         if (expectCount === 0) {
@@ -370,6 +460,7 @@ async function runTest(
             name,
             status: 'suspicious',
             duration,
+            blocks,
             hadDebugPause,
             projectName: capturedProjectName,
             error: '0 assertions — test likely exited before reaching assertions',
@@ -379,6 +470,7 @@ async function runTest(
             name,
             status: 'pass',
             duration,
+            blocks,
             hadDebugPause,
             projectName: capturedProjectName,
           });
@@ -403,17 +495,99 @@ async function runTest(
           name,
           status: 'fail',
           duration,
+          blocks,
           hadDebugPause,
           projectName: capturedProjectName,
           error: extractFailureMessage(lines, code ?? 1),
           rawTail,
           stages: tallyStages(lines),
+          refused: code === 3,
         });
       }
     });
   });
 }
 
+/** bun writes this only if it got far enough to; a crash leaves nothing. */
+function readJunit(path: string): string | undefined {
+  try {
+    return readFileSync(path, 'utf-8');
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Print any block that finished with less than 20% of its own budget left.
+ *
+ * This runs HERE, in the runner, because it is the only place that sees real
+ * durations under real load. A gate in CI cannot see this failure mode at all:
+ * on a quiet runner every block fits, and the run is green (celilo#1268,
+ * acceptance 4).
+ *
+ * It reports rather than fails. A block at 90% has not broken anything yet, and
+ * failing a passing suite would stop the grind that produces the measurements.
+ * The line is the durable signal — it lands in `output.log`, so `classify.sh`
+ * can promote a PASS whose blocks are out of room, instead of a later timeout
+ * arriving with nothing to distinguish "the host starved this" from "this never
+ * had room".
+ */
+/**
+ * Fail a suite whose block overran the cap that block itself declares.
+ *
+ * Reads the JUnit report directly — `overBudgetBlocks` bypasses the skip filter
+ * for the reason stated there (celilo#1291: the filter deleted the one block
+ * that mattered). bun's per-test timeout is not the enforcer; this is. A block
+ * that overruns while the suite still PASSES is the dangerous case — an
+ * unbudgeted lease on the rig that reads as green — so a pass with an overrun
+ * becomes a fail naming the block, its duration and its declaration. A failed
+ * suite keeps its own failure; the overruns are printed either way so the
+ * output.log carries the durable line. Debug sessions are exempt: a
+ * `net.debug()` pause inside a block inflates its junit time without the suite
+ * having run at all.
+ */
+function enforceBlockCaps(result: TestResult, suite: string, logDir: string, file: string): void {
+  if (result.hadDebugPause) return;
+  const junitXml = readJunit(join(logDir, 'junit.xml'));
+  if (!junitXml) return;
+  let overruns: OverBudgetBlock[];
+  try {
+    overruns = overBudgetBlocks(junitXml, readFileSync(file, 'utf-8'));
+  } catch {
+    return;
+  }
+  if (overruns.length === 0) return;
+  for (const t of overruns) {
+    console.log(
+      `${red}[block:over-budget]${reset} ${suite} — "${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s declaration (${Math.round(t.fraction * 100)}%)`,
+    );
+  }
+  if (result.status === 'pass') {
+    result.status = 'fail';
+    result.error = overruns
+      .map(
+        (t) =>
+          `"${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s cap`,
+      )
+      .join('; ');
+  }
+}
+
+function reportTightBlocks(suite: string, file: string, blocks: Record<string, number>): void {
+  let caps: Record<string, number>;
+  try {
+    caps = declaredBlockCaps(readFileSync(file, 'utf-8'));
+  } catch {
+    return;
+  }
+  for (const t of tightBlocks(blocks, caps)) {
+    const pct = Math.round(t.fraction * 100);
+    console.log(
+      `${yellow}[block:tight]${reset} ${suite} — "${t.block}" used ${Math.round(t.ms / 1000)}s of its ${Math.round(t.capMs / 1000)}s budget (${pct}%)`,
+    );
+  }
+}
+
 /**
  * " — 1 stage failed, 8 skipped (blocked by an earlier stage)". Rendered only
  * when a cascade actually happened, so an ordinary single-stage failure stays
@@ -615,7 +789,7 @@ async function main() {
   }
 
   if (patterns.length > 0) {
-    testFiles = testFiles.filter((f) => patterns.some((p) => basename(f, '.test.ts').includes(p)));
+    testFiles = filterTestFilesByPatterns(testFiles, patterns);
   }
 
   // Drop quarantined tests (file contains `cele2e-ci-unsafe`) from broad runs so
@@ -669,6 +843,7 @@ async function main() {
   }
 
   const history = loadTiming();
+  let blockTiming = loadBlockTiming();
   const testNames = testFiles.map((f) => basename(f, '.test.ts'));
   const hasHistory = testNames.some((n) => history[n]);
   const suiteExpected = testNames.reduce((sum, n) => sum + (history[n] || 0), 0);
@@ -758,6 +933,7 @@ async function main() {
     console.log(`\n${red}Interrupted. Stopping shared infrastructure...${reset}`);
     stopSharedInfra();
     saveTiming(history);
+    saveBlockTiming(blockTiming);
     emitRunFailed({
       runId,
       durationMs: Date.now() - suiteStart,
@@ -802,6 +978,8 @@ async function main() {
       result.error = `cross-test pollution (#254): this test left shared DNS dirty — ${pollution}`;
     }
 
+    enforceBlockCaps(result, name, logDir, file);
+
     emitTestCompleted({
       runId,
       name,
@@ -817,6 +995,13 @@ async function main() {
     if (result.duration >= 10 && !result.hadDebugPause) {
       history[name] = result.duration;
     }
+    // Recorded even for a short or failed run: a block that BLEW its cap is
+    // exactly the measurement this file exists for, and a debug pause inflates
+    // the suite total without touching the blocks that ran before it.
+    if (result.blocks) {
+      blockTiming = mergeBlockTiming(blockTiming, name, result.blocks);
+      reportTightBlocks(name, file, result.blocks);
+    }
 
     // Write persistent network info if --keep was set and we captured a project
     if (flagKeep && result.projectName) {
@@ -864,6 +1049,24 @@ async function main() {
     }
     console.log();
 
+    // The live-stack guard refused (celilo#1297): another run's stack is in
+    // the way. Stop the whole run with exit 3 — the same code the lock
+    // refusal uses — instead of grinding every remaining suite into the same
+    // wall. Nothing of ours is running, so there is nothing to tear down.
+    if (result.refused) {
+      console.error(
+        `\n${red}✗ startup cleanup refused — a live e2e stack is in the way:${reset}\n`,
+      );
+      if (result.rawTail) console.error(`${dim}${result.rawTail}${reset}\n`);
+      console.error(`${dim}Clear the live stack with: cele2e down${reset}\n`);
+      emitRunFailed({
+        runId,
+        durationMs: Date.now() - suiteStart,
+        error: result.error ?? 'startup cleanup refused: live e2e stack in the way',
+      });
+      process.exit(3);
+    }
+
     writeFileSync(
       join(logDir, 'result.json'),
       JSON.stringify({ status: result.status, duration: result.duration, error: result.error }),
@@ -875,6 +1078,7 @@ async function main() {
   }
 
   saveTiming(history);
+  saveBlockTiming(blockTiming);
 
   console.log(`${dim}Stopping shared infrastructure...${reset}`);
   stopSharedInfra();
@@ -947,6 +1151,7 @@ async function main() {
   const adds: string[] = [];
   if (!content.includes('results/')) adds.push('results/');
   if (!content.includes('.e2e-timing.json')) adds.push('.e2e-timing.json');
+  if (!content.includes('.e2e-block-timing.json')) adds.push('.e2e-block-timing.json');
   if (adds.length) writeFileSync(gitignore, `${content.trimEnd()}\n${adds.join('\n')}\n`);
 
   emitRunCompleted({

package/src/shared-infra.ts

@@ -19,7 +19,15 @@ import {
   generateSharedInfraYaml,
   prepareRegistryDropZone,
 } from './docker-compose-generator';
+import {
+  type DockerReader,
+  LiveStackError,
+  findLiveE2eStack,
+  realDocker,
+  refuseOnLiveStack,
+} from './live-stack';
 import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
+import { type LockStatus, lockStatus } from './run-lock';
 
 const SHARED_COMPOSE_FILE = 'docker-compose.shared.yml';
 const COMPOSE_TIMEOUT = 120_000;
@@ -35,40 +43,73 @@ function run(cmd: string, opts?: { cwd?: string; timeout?: number }): string {
 
 /**
  * Remove ALL celilo-e2e-* Docker resources by name prefix (#212). Used at
- * start-of-run, where the run-lock guarantees no other session's stack is live.
- * Graceful compose-down of the shared project first (clean network detach),
- * then a force sweep that catches orphans from any prior invocation regardless
- * of compose-project name or the path that created them.
+ * start-of-run, where the live-stack guard (findLiveE2eStack, called by
+ * startupCleanup before this runs) has verified nothing live remains to
+ * remove. Graceful compose-down of the shared project first (clean network
+ * detach), then a force sweep that catches orphans from any prior invocation
+ * regardless of compose-project name or the path that created them.
+ *
+ * Docker access goes through the injected runner (ce-h4no seam) so tests can
+ * prove the guard fires BEFORE any removal command runs.
  */
-function nukeE2eResources(e2eDir: string): void {
+function nukeE2eResources(e2eDir: string, docker: DockerReader): void {
   try {
-    run(
-      `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`,
-      { cwd: e2eDir, timeout: 30_000 },
+    docker(
+      [
+        'compose',
+        '-f',
+        SHARED_COMPOSE_FILE,
+        '-p',
+        SHARED_PROJECT_NAME,
+        'down',
+        '--volumes',
+        '--remove-orphans',
+      ],
+      { cwd: e2eDir, timeoutMs: 30_000 },
     );
   } catch {}
   try {
-    const ids = run('docker ps -aq --filter name=celilo-e2e', { timeout: 10_000 });
-    if (ids.trim()) run(`docker rm -f ${ids.replace(/\n/g, ' ')}`, { timeout: 60_000 });
+    const ids = docker(['ps', '-aq', '--filter', 'name=celilo-e2e'], { timeoutMs: 10_000 });
+    if (ids.trim()) docker(['rm', '-f', ...ids.split('\n').filter(Boolean)], { timeoutMs: 60_000 });
   } catch {}
   try {
-    const nets = run('docker network ls --format "{{.Name}}"')
+    const nets = docker(['network', 'ls', '--format', '{{.Name}}'])
       .split('\n')
       .filter((n) => n.startsWith('celilo-e2e'));
     for (const net of nets) {
       try {
-        run(`docker network rm ${net}`, { timeout: 5_000 });
+        docker(['network', 'rm', net], { timeoutMs: 5_000 });
       } catch {}
     }
   } catch {}
   try {
-    run('docker network prune -f', { timeout: 10_000 });
+    docker(['network', 'prune', '-f'], { timeoutMs: 10_000 });
   } catch {}
   try {
-    run('docker volume prune -f', { timeout: 10_000 });
+    docker(['volume', 'prune', '-f'], { timeoutMs: 10_000 });
   } catch {}
 }
 
+/**
+ * The start-of-run cleanup: refuse if a live stack is in the way
+ * (celilo#1297), otherwise sweep every celilo-e2e-* resource. The refusal
+ * throws LiveStackError — ensureSharedInfra turns it into exit 3 at its two
+ * call sites — so tests can drive this with a fake runner and assert the
+ * sweep removes nothing behind a refusal.
+ */
+export function startupCleanup(
+  e2eDir: string,
+  docker: DockerReader = realDocker,
+  lock: () => LockStatus = lockStatus,
+): void {
+  const refusal = findLiveE2eStack(docker, lock);
+  if (refusal) throw new LiveStackError(refusal);
+  console.log(
+    '[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete',
+  );
+  nukeE2eResources(e2eDir, docker);
+}
+
 /**
  * Check if the shared infrastructure is already running and healthy.
  *
@@ -129,26 +170,38 @@ export async function ensureSharedInfra(): Promise<void> {
         return; // Shared infra healthy, nothing to do
       }
     } catch {
-      // DNS check failed — restart shared infra
+      // DNS check failed — restart shared infra. The restart tears down the
+      // running stack, so the live-stack guard fires first (celilo#1297): a
+      // stack that is up is never force-removed, even when its DNS is wedged.
+      // The operator clears it with `cele2e down` and re-runs. This retires
+      // the automatic DNS-restart self-heal by decision (peba, 2026-09-07):
+      // fail loudly naming the holder beats silently clobbering a stack that
+      // might be another session's — the #1297 incident rode exactly this
+      // branch.
       console.log(
         '[progress:start] shared infra DNS check failed, restarting | shared infra restarted',
       );
+      refuseOnLiveStack();
       await stopSharedInfra();
     }
   }
 
   const e2eDir = getE2eDir();
 
-  // Start-of-run cleanup. We hold the run-lock here (acquired in the runner /
-  // build path before any docker mutation), so NO other session's stack is
-  // live — it is safe to remove EVERY celilo-e2e-* resource by name prefix.
-  // This self-heals orphans left by a crashed / cross-path / cross-checkout
-  // prior run (e.g. a botched-mount container that wedges the next `up`),
-  // which the old compose-project-scoped `down` couldn't see (#212).
-  console.log(
-    '[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete',
-  );
-  nukeE2eResources(e2eDir);
+  // Start-of-run cleanup, behind the live-stack guard. The run-lock held here
+  // (acquired in the runner / build path before any docker mutation) is no
+  // longer treated as proof that no other session's stack is live — the #1297
+  // incident got past it. startupCleanup checks Docker and the lock file at
+  // the removal site and refuses with exit 3 when anything live remains.
+  try {
+    startupCleanup(e2eDir);
+  } catch (err) {
+    if (err instanceof LiveStackError) {
+      console.error(`\n${err.message}\n`);
+      process.exit(3);
+    }
+    throw err;
+  }
 
   // Refresh the bundled registry-server source so Dockerfile.registry's
   // COPY resolves whether we're in the monorepo (regenerated from the
@@ -184,13 +237,10 @@ export async function ensureSharedInfra(): Promise<void> {
     copyFileSync(tpl, live);
   }
 
-  // Build and start
-  console.log('[progress:start] building shared infrastructure images | shared images built');
-  run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} build`, {
-    cwd: e2eDir,
-    timeout: 300_000,
-  });
-
+  // Start. The generated compose carries `image:` only (never `build:`), so
+  // `up` cannot build and cannot resolve a FROM from docker.io; a missing
+  // baked tag fails right here, naming the image. startNetwork checks the
+  // full referenced set first and names the remedy (cele2e build-infra).
   console.log('[progress:start] starting shared infrastructure | shared infra started');
   run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`, {
     cwd: e2eDir,
@@ -232,6 +282,7 @@ export async function ensureSharedInfra(): Promise<void> {
       } catch {}
       lastRestart = Date.now();
     }
+    // e2e-sleep-ok: poll cadence; convergence is re-checked each iteration, diagnostics on timeout.
     await new Promise((r) => setTimeout(r, 2000));
   }
   // Diagnostics before failing, so the log shows WHY it didn't converge.

package/src/socks-proxy.ts

@@ -190,6 +190,7 @@ async function waitForTcpAccept(port: number, timeoutMs = 10_000): Promise<void>
       return false;
     });
     if (ok) return;
+    // e2e-sleep-ok: poll cadence; tryTcpConnect is re-checked each iteration.
     await new Promise((r) => setTimeout(r, 100));
   }
   throw new Error(
@@ -340,6 +341,7 @@ async function waitForContainerListening(name: string): Promise<void> {
           `Container logs:\n${logs || '(none)'}`,
       );
     }
+    // e2e-sleep-ok: poll cadence; the container inspect is re-checked each iteration.
     await new Promise((r) => setTimeout(r, 200));
   }
   throw new Error(