@celilo/e2e 0.7.17 → 0.9.0

package/src/docker-compose-generator.ts

@@ -18,9 +18,11 @@ import { ZONE_GATEWAYS } from './types';
  * - **npm-installed consumer** (lunacycle and friends) — `../../modules`
  *   would resolve to `<consumer>/node_modules/modules`, which doesn't
  *   exist. Docker fails the mount with "permission denied" trying to
- *   create that path. Instead we bind the pre-shipped `./netapps/` dir
- *   (always present in the npm tarball) to `/uploads`, and the registry's
- *   scanUploadsDir picks up the .netapp files.
+ *   create that path. Instead we bind the local `./netapps/` dir to
+ *   `/uploads`, and the registry's scanUploadsDir picks up the .netapp
+ *   files. That dir is populated at `cele2e build-infra` time by fetching
+ *   the standard-module netapps from the public registry (see
+ *   stageNetappsFromRegistry in cli/build.ts) — consumers vendor nothing.
  *
  * Detection: check whether the would-be monorepo modules dir is real.
  * `import.meta.dir` is `<pkgDir>/src/`, so `../../../modules` from there

package/src/network-builder.ts

@@ -2,7 +2,7 @@ import { execSync } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { startNetwork } from './container-manager';
-import type { NetworkConfig, NetworkHandle, TopologyPreset, Vantage } from './types';
+import type { MachineSpec, NetworkConfig, NetworkHandle, TopologyPreset, Vantage, Zone } from './types';
 
 /**
  * If a mounted path looks like a celilo module (has scripts/package.json),
@@ -51,53 +51,45 @@ export class NetworkBuilder {
     return this;
   }
 
-  dmz(machines: Record<string, string>): this {
+  /** Append `{name, ip, zone, ...extra}` for each entry to a zone's machine list. */
+  private addMachines(
+    target: MachineSpec[],
+    zone: Zone,
+    machines: Record<string, string>,
+    extra?: Pick<MachineSpec, 'docker' | 'dockerfile'>,
+  ): this {
     for (const [name, ip] of Object.entries(machines)) {
-      this.config.dmzMachines.push({ name, ip, zone: 'dmz' });
+      target.push({ name, ip, zone, ...extra });
     }
     return this;
   }
 
+  dmz(machines: Record<string, string>): this {
+    return this.addMachines(this.config.dmzMachines, 'dmz', machines);
+  }
+
   app(machines: Record<string, string>): this {
-    for (const [name, ip] of Object.entries(machines)) {
-      this.config.appMachines.push({ name, ip, zone: 'app' });
-    }
-    return this;
+    return this.addMachines(this.config.appMachines, 'app', machines);
   }
 
   secure(machines: Record<string, string>): this {
-    for (const [name, ip] of Object.entries(machines)) {
-      this.config.secureMachines.push({ name, ip, zone: 'secure' });
-    }
-    return this;
+    return this.addMachines(this.config.secureMachines, 'secure', machines);
   }
 
   internal(machines: Record<string, string>): this {
-    for (const [name, ip] of Object.entries(machines)) {
-      this.config.internalMachines.push({ name, ip, zone: 'internal' });
-    }
-    return this;
+    return this.addMachines(this.config.internalMachines, 'internal', machines);
   }
 
   /** Add internal-zone machines that need the Docker-capable image (e.g., Technitium) */
   internalDocker(machines: Record<string, string>): this {
-    for (const [name, ip] of Object.entries(machines)) {
-      this.config.internalMachines.push({ name, ip, zone: 'internal', docker: true });
-    }
-    return this;
+    return this.addMachines(this.config.internalMachines, 'internal', machines, { docker: true });
   }
 
   /** Add a Technitium DNS machine using the dedicated pre-installed image */
   technitium(machines: Record<string, string>): this {
-    for (const [name, ip] of Object.entries(machines)) {
-      this.config.internalMachines.push({
-        name,
-        ip,
-        zone: 'internal',
-        dockerfile: 'docker/Dockerfile.target-technitium',
-      });
-    }
-    return this;
+    return this.addMachines(this.config.internalMachines, 'internal', machines, {
+      dockerfile: 'docker/Dockerfile.target-technitium',
+    });
   }
 
   withDhcpClient(): this {

package/src/run-lock.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  acquireRunLock,
+  clearLock,
+  E2eBusyError,
+  type LockHolder,
+  lockStatus,
+  markKept,
+  readHolder,
+  releaseRunLock,
+} from './run-lock';
+
+// Isolate every test on its own lock file (machine-global path is overridable).
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'e2e-lock-'));
+  process.env.CELILO_E2E_LOCK_PATH = join(dir, 'run.lock');
+});
+
+afterEach(() => {
+  clearLock();
+  rmSync(dir, { recursive: true, force: true });
+  delete process.env.CELILO_E2E_LOCK_PATH;
+});
+
+function writeRawHolder(h: Partial<LockHolder>): void {
+  const full: LockHolder = {
+    pid: 123,
+    hostname: hostname(),
+    session: 'other',
+    test: 'foo',
+    runId: 'r',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+    ...h,
+  };
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, JSON.stringify(full));
+}
+
+test('acquire writes a holder for this process; release removes it', () => {
+  acquireRunLock({ test: 'mytest', runId: 'run-1' });
+  const h = readHolder();
+  expect(h?.pid).toBe(process.pid);
+  expect(h?.test).toBe('mytest');
+  expect(lockStatus().free).toBe(false);
+
+  releaseRunLock();
+  expect(readHolder()).toBeNull();
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a live holder makes a second acquire throw E2eBusyError', () => {
+  acquireRunLock({ test: 't1', runId: 'r1' });
+  // The on-disk holder is us (a live pid) → busy.
+  expect(() => acquireRunLock({ test: 't2', runId: 'r2' })).toThrow(E2eBusyError);
+  releaseRunLock();
+});
+
+test('a dead-pid holder on the same host is stale and reclaimed', () => {
+  // 2e9 is far above any real pid → process.kill(pid, 0) throws ESRCH → dead.
+  writeRawHolder({ pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(true); // stale → reported free
+
+  acquireRunLock({ test: 'reclaimer', runId: 'r' }); // reclaims the stale lock
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('--keep leaves a kept lock that survives release and blocks the next run', () => {
+  acquireRunLock({ test: 'kept-test', runId: 'r' });
+  markKept();
+  releaseRunLock();
+
+  const h = readHolder();
+  expect(h?.state).toBe('kept');
+  expect(lockStatus().free).toBe(false); // kept is never stale
+
+  // A plain run is refused...
+  expect(() => acquireRunLock({ test: 'next', runId: 'r2' })).toThrow(E2eBusyError);
+  // ...but a --reuse run (allowKept) takes it over.
+  acquireRunLock({ test: 'reuse', runId: 'r3', allowKept: true });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('clearLock frees a kept lock (the `cele2e release` path)', () => {
+  writeRawHolder({ state: 'kept', pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(false);
+  expect(clearLock()).toBe(true);
+  expect(existsSync(process.env.CELILO_E2E_LOCK_PATH as string)).toBe(false);
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a corrupt lock file is reclaimed, not fatal', () => {
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, 'not json{');
+  expect(readHolder()).toBeNull();
+  acquireRunLock({ test: 'survivor', runId: 'r' });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});

package/src/run-lock.ts

@@ -0,0 +1,252 @@
+/**
+ * cele2e run-lock — cross-platform mutual exclusion for the shared e2e infra.
+ *
+ * cele2e is a single shared resource: the shared infra is one global compose
+ * project (celilo-e2e-shared) and start-of-run cleanup tears down every
+ * celilo-e2e-* container regardless of who started it. So two actors at once
+ * (two Claude sessions, or a session + an operator at the terminal) clobber
+ * each other. This lock serializes runs across processes — no external tools,
+ * identical on macOS and Linux.
+ *
+ * Design (Forgejo #243):
+ *  - Acquire via fs.openSync(path, 'wx') — POSIX-atomic exclusive create,
+ *    same behavior on mac+linux. The lockfile lives at a MACHINE-GLOBAL path
+ *    (~/.cache/celilo-e2e/run.lock), NOT under any repo/worktree, because the
+ *    Docker infra is global regardless of which checkout started it.
+ *  - Fail fast on contention with a message naming the holder. Sessions poll
+ *    `cele2e status` and decide for themselves whether to wait (no --wait).
+ *  - Staleness: on the SAME host, PID-liveness is authoritative (process.kill
+ *    (pid, 0)); a dead holder PID → reclaim. Cross-host we can't check the PID,
+ *    so fall back to a heartbeat TTL (beatAt older than STALE_TTL_MS).
+ *  - A `kept` lock (left by `--keep` / `up`) is NEVER auto-reclaimed — it
+ *    guards a stack that outlives the process and is cleared only by
+ *    `cele2e release` / `cele2e down`. This enforces the honor-system rule.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, openSync, closeSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { homedir, hostname } from 'node:os';
+import { basename, dirname, join } from 'node:path';
+
+const STALE_TTL_MS = 90_000;
+const HEARTBEAT_MS = 30_000;
+
+/**
+ * Machine-global lock path — NOT under any repo/worktree, because the Docker
+ * infra is global regardless of which checkout started it. Overridable via
+ * CELILO_E2E_LOCK_PATH (used by tests; also a handy operator escape hatch).
+ */
+export function lockPath(): string {
+  return process.env.CELILO_E2E_LOCK_PATH || join(homedir(), '.cache', 'celilo-e2e', 'run.lock');
+}
+
+export interface LockHolder {
+  pid: number;
+  hostname: string;
+  session: string;
+  test: string;
+  runId: string;
+  startedAt: string;
+  beatAt: number;
+  state: 'running' | 'kept';
+}
+
+interface Held {
+  keepOnRelease: boolean;
+  heartbeat: ReturnType<typeof setInterval> | null;
+  released: boolean;
+}
+
+/** Set once this process owns the lock; null otherwise. */
+let held: Held | null = null;
+
+export class E2eBusyError extends Error {
+  constructor(public holder: LockHolder) {
+    super(formatBusy(holder));
+    this.name = 'E2eBusyError';
+  }
+}
+
+/** Human label for who's running, e.g. "e2e busy: <session> running <test>, started 3m ago (pid 1234)". */
+export function formatBusy(h: LockHolder): string {
+  const age = ageString(Date.parse(h.startedAt));
+  const verb = h.state === 'kept' ? 'holds a kept stack from' : 'running';
+  const what = h.state === 'kept' ? `(run \`cele2e release\` to free it)` : `${h.test}, started ${age} ago (pid ${h.pid})`;
+  return `e2e busy: ${h.session} ${verb} ${what}`;
+}
+
+function ageString(since: number): string {
+  const s = Math.max(0, Math.floor((Date.now() - since) / 1000));
+  if (s < 60) return `${s}s`;
+  if (s < 3600) return `${Math.floor(s / 60)}m`;
+  return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
+}
+
+/** Branch + worktree path so a holder maps back to a specific session/thread. */
+function deriveSession(): string {
+  if (process.env.CELILO_E2E_SESSION) return process.env.CELILO_E2E_SESSION;
+  const cwd = process.cwd();
+  const git = (args: string[]): string => {
+    try {
+      return execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+    } catch {
+      return '';
+    }
+  };
+  const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']);
+  const top = git(['rev-parse', '--show-toplevel']);
+  const base = top ? basename(top) : basename(cwd);
+  if (branch && top) return `${branch} (${top})`;
+  return branch || base;
+}
+
+function pidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // ESRCH = no such process (dead). EPERM = alive but not ours (still alive).
+    return (err as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+export function readHolder(): LockHolder | null {
+  try {
+    return JSON.parse(readFileSync(lockPath(), 'utf-8')) as LockHolder;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is the on-disk holder stale (safe to reclaim)? A `kept` lock is never stale —
+ * it deliberately outlives its process. On the same host the PID is the source
+ * of truth; cross-host we can only use the heartbeat TTL.
+ */
+function isStale(h: LockHolder): boolean {
+  if (h.state === 'kept') return false;
+  if (h.hostname === hostname()) return !pidAlive(h.pid);
+  return Date.now() - h.beatAt > STALE_TTL_MS;
+}
+
+function writeHolder(fd: number, h: LockHolder): void {
+  writeFileSync(fd, JSON.stringify(h, null, 2));
+}
+
+/**
+ * Acquire the run lock for this process. Throws E2eBusyError if another live
+ * (non-stale) holder owns it. Registers an exit handler so the lock is released
+ * on any process.exit() path (normal, exception, or a SIGINT handler that
+ * exits). A signal-killed process with no exit handler leaks the lock, but the
+ * next run reclaims it via PID-liveness — that's exactly what staleness is for.
+ */
+export function acquireRunLock(opts: { test: string; runId: string; allowKept?: boolean }): void {
+  mkdirSync(dirname(lockPath()), { recursive: true });
+
+  const holder: LockHolder = {
+    pid: process.pid,
+    hostname: hostname(),
+    session: deriveSession(),
+    test: opts.test,
+    runId: opts.runId,
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let fd: number;
+    try {
+      fd = openSync(lockPath(), 'wx');
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
+      const existing = readHolder();
+      // Reclaim when: corrupt/unreadable, the holder is stale, OR this is a
+      // `--reuse` run taking over a `kept` stack it's deliberately reusing.
+      if (!existing || isStale(existing) || (opts.allowKept && existing.state === 'kept')) {
+        try { unlinkSync(lockPath()); } catch {}
+        continue;
+      }
+      throw new E2eBusyError(existing);
+    }
+    writeHolder(fd, holder);
+    closeSync(fd);
+
+    const heartbeat = setInterval(() => {
+      // Only refresh while running; a kept lock's beatAt is frozen by design.
+      if (!held || held.released || held.keepOnRelease) return;
+      try {
+        const cur = readHolder();
+        if (cur && cur.pid === process.pid) {
+          writeFileSync(lockPath(), JSON.stringify({ ...cur, beatAt: Date.now() }, null, 2));
+        }
+      } catch {}
+    }, HEARTBEAT_MS);
+    heartbeat.unref();
+
+    held = { keepOnRelease: false, heartbeat, released: false };
+    process.on('exit', releaseRunLock);
+    return;
+  }
+  // Two reclaim attempts both lost the race → someone else won fair and square.
+  const existing = readHolder();
+  throw new E2eBusyError(existing ?? holderUnknown());
+}
+
+function holderUnknown(): LockHolder {
+  return {
+    pid: 0,
+    hostname: hostname(),
+    session: 'unknown',
+    test: '?',
+    runId: '?',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+}
+
+/**
+ * Mark the lock to survive process exit in a `kept` state — call when a stack
+ * (network/containers) will outlive this process (`--keep`, `up`). The next run
+ * then refuses to clobber it until `cele2e release` / `cele2e down`.
+ */
+export function markKept(): void {
+  if (held) held.keepOnRelease = true;
+}
+
+/** Release the lock. Idempotent. Honors markKept() (kept-state) vs delete. */
+export function releaseRunLock(): void {
+  if (!held || held.released) return;
+  held.released = true;
+  if (held.heartbeat) clearInterval(held.heartbeat);
+
+  if (held.keepOnRelease) {
+    const cur = readHolder();
+    if (cur && cur.pid === process.pid) {
+      try { writeFileSync(lockPath(), JSON.stringify({ ...cur, state: 'kept', beatAt: Date.now() }, null, 2)); } catch {}
+    }
+    return;
+  }
+  try {
+    const cur = readHolder();
+    if (!cur || cur.pid === process.pid) unlinkSync(lockPath());
+  } catch {}
+}
+
+/** Forcibly clear the lock regardless of holder — used by `cele2e release`/`down`. */
+export function clearLock(): boolean {
+  try {
+    unlinkSync(lockPath());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Current lock state for `cele2e status` (and any poller). */
+export function lockStatus(): { free: boolean; holder: LockHolder | null } {
+  const h = readHolder();
+  if (!h || isStale(h)) return { free: true, holder: null };
+  return { free: false, holder: h };
+}

package/src/runner.ts

@@ -30,6 +30,8 @@ import {
   emitTestStarted,
 } from './bus-events';
 import { parseLine } from './parse-line';
+import { acquireRunLock, E2eBusyError, markKept } from './run-lock';
+import { SIMULATOR_IPS } from './simulator-ips';
 
 // ─── Config ──────────────────────────────────────────────────────────
 
@@ -87,6 +89,52 @@ const patterns = rawArgs.filter((a) => !a.startsWith('--'));
 // run everything. Passed through from the CLI (not stripped) so the runner can
 // apply the exclusion.
 const flagCiSafe = rawArgs.includes('--ci-safe');
+// --notify: fire a desktop notification when the run finishes (handy for long
+// detached/background runs). Best-effort — never affects the exit code.
+const flagNotify = rawArgs.includes('--notify');
+// --shuffle [--seed=N]: randomize test order to surface order-dependent bugs
+// (e2e-confidence #258). The install-sh DNS-pollution bug only manifested in a
+// specific order — a shuffle lane flushes that class out. Seeded + logged so a
+// failing order is reproducible: replay with the printed --seed=N.
+const flagShuffle = rawArgs.includes('--shuffle');
+const seedFlag = rawArgs.find((a) => a.startsWith('--seed='));
+const seedArg = seedFlag ? Number.parseInt(seedFlag.split('=')[1] ?? '', 10) : undefined;
+
+/** Deterministic Fisher-Yates shuffle seeded by a 32-bit value (mulberry32), so
+ * a randomized test order is reproducible via --seed=<n>. */
+function seededShuffle<T>(arr: readonly T[], seed: number): T[] {
+  let s = seed >>> 0;
+  const rand = (): number => {
+    s = (s + 0x6d2b79f5) >>> 0;
+    let t = s;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+  const out = [...arr];
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(rand() * (i + 1));
+    [out[i], out[j]] = [out[j], out[i]];
+  }
+  return out;
+}
+
+function notify(title: string, message: string): void {
+  if (!flagNotify) return;
+  try {
+    if (process.platform === 'darwin') {
+      const r = spawnSync('terminal-notifier', ['-title', title, '-message', message, '-sound', 'Glass'], { stdio: 'ignore' });
+      if (r.error) {
+        spawnSync('osascript', ['-e', `display notification ${JSON.stringify(message)} with title ${JSON.stringify(title)} sound name "Glass"`], { stdio: 'ignore' });
+      }
+      spawnSync('say', [message], { stdio: 'ignore' });
+    } else if (process.platform === 'linux') {
+      spawnSync('notify-send', [title, message], { stdio: 'ignore' });
+    }
+  } catch {
+    // ponytail: notification is a nicety; its failure must never break a run.
+  }
+}
 
 // ─── Colors ──────────────────────────────────────────────────────────
 
@@ -94,7 +142,6 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
-const cyan = '\x1b[36m';
 const reset = '\x1b[0m';
 
 const interactive =
@@ -386,19 +433,36 @@ function dockerComposeShared(cmd: string): string {
   }
 }
 
-function isSharedInfraHealthy(): boolean {
-  const containers = dockerComposeShared('ps -q');
-  if (!containers) return false;
-  return dockerComposeShared(
-    'exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2',
-  ).length > 0;
-}
-
 function stopSharedInfra(): void {
   dockerComposeShared('down --volumes --remove-orphans');
   try { execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 }); } catch {}
 }
 
+/**
+ * Detect cross-test pollution of the SHARED authoritative DNS at the test
+ * boundary (e2e-confidence #254). A test that leaves a protected sim record
+ * clobbered — canonically celilo.computer's apex off the website-sim, the bug
+ * that silently broke install-sh ~15 tests downstream — has violated the shared
+ * contract. Checked AFTER each test so the polluter is named HERE, at the cause,
+ * instead of a later test failing mysteriously. On detection we re-scrub
+ * (cp /seed -> /config, mirroring scrubDnsZones) so the damage doesn't cascade.
+ *
+ * Returns a description of the violation (and heals) if polluted, else null.
+ * A query failure (infra down / tool missing) is not treated as pollution.
+ */
+function detectSharedDnsPollution(): string | null {
+  const served = dockerComposeShared(
+    `exec -T namecheap-dns sh -c ${JSON.stringify('kdig @127.0.0.1 celilo.computer A +short')}`,
+  ).trim();
+  if (!served) return null;
+  if (served.split(/\s+/).includes(SIMULATOR_IPS.WEBSITE)) return null;
+  // Polluted — heal so the next test starts from a clean baseline.
+  dockerComposeShared(
+    `exec -T namecheap-dns sh -c ${JSON.stringify('cp -f /seed/*.zone /config/ && knotc zone-reload')}`,
+  );
+  return `celilo.computer apex resolves to "${served}" (expected the website-sim ${SIMULATOR_IPS.WEBSITE})`;
+}
+
 // ─── Docker Check ────────────────────────────────────────────────────
 
 function assertDockerAvailable(): void {
@@ -417,6 +481,24 @@ function assertDockerAvailable(): void {
 async function main() {
   assertDockerAvailable();
 
+  // Serialize against other cele2e runs sharing the global Docker infra. Fail
+  // fast naming the holder; pollers use `cele2e status`. Acquire BEFORE any
+  // docker mutation so a competing run can't wipe our shared infra mid-setup.
+  const lockLabel = patterns.length ? patterns.join(',') : moduleDirs.length ? 'modules' : 'all';
+  try {
+    acquireRunLock({ test: lockLabel, runId: process.env.CELE2E_RUN_ID ?? ambientRunId, allowKept: flagReuse });
+    // A --reuse run keeps using a kept stack that stays up afterwards — hold the
+    // lock in kept state so it isn't freed out from under the reused network.
+    if (flagReuse) markKept();
+  } catch (err) {
+    if (err instanceof E2eBusyError) {
+      console.error(`\n${red}${err.message}${reset}`);
+      console.error(`${dim}Poll with: cele2e status   (machine-readable: cele2e status --json)${reset}\n`);
+      process.exit(3);
+    }
+    throw err;
+  }
+
   let testFiles: string[] = [];
 
   if (moduleDirs.length > 0) {
@@ -515,6 +597,17 @@ async function main() {
     process.exit(1);
   }
 
+  // e2e-confidence #258: optionally randomize order to surface order-dependent
+  // bugs (the install-sh DNS bleed only manifested in a specific order). Seeded
+  // + logged so a failing order replays exactly via --seed=<n>.
+  if (flagShuffle) {
+    const seed = (seedArg !== undefined && Number.isFinite(seedArg) ? seedArg : Date.now()) >>> 0;
+    testFiles = seededShuffle(testFiles, seed);
+    console.log(
+      `${dim}--shuffle: randomized test order (seed=${seed}) — replay this exact order with --seed=${seed}${reset}`,
+    );
+  }
+
   // --keep keeps a single test network alive after the run; only one network
   // can live in `.e2e-persistent.json` at a time, so multiple tests would
   // either fight for that slot or run with no functional --keep at all. Bail
@@ -557,9 +650,8 @@ async function main() {
   mkdirSync(resultsDir, { recursive: true });
 
   // runId scopes every event in this suite invocation. Read from env to
-  // let callers (CI, the Claude tool wrapper) inject a known id and
-  // subscribe to it before launching cele2e; otherwise generate one and
-  // print it so the operator can correlate.
+  // let callers (CI) inject a known id and subscribe to it before launching
+  // cele2e; otherwise generate one and print it so the operator can correlate.
   const runId = process.env.CELE2E_RUN_ID || crypto.randomUUID();
   if (!process.env.CELE2E_RUN_ID) {
     console.log(`${dim}runId: ${runId}${reset}`);
@@ -574,7 +666,7 @@ async function main() {
 
   // 'auto' resolves to 'render' when isTTY is true (operator at a
   // terminal sees animated spinners) and 'protocol' when isTTY is
-  // false (cele2e-run.sh wrapper / any redirected stdout gets ASCII
+  // false (a backgrounded run / any redirected stdout gets ASCII
   // `[progress:*]` markers — grep-friendly, no braille noise).
   // Prior to this change the mode was hardcoded 'render', so non-TTY
   // callers got `⣾ phase started` / `✔ phase done` lines per phase
@@ -635,6 +727,17 @@ async function main() {
 
     emitTestStarted({ runId, name, expectedDurationS: expected });
     const result = await runTest(file, name, expected, logDir, display);
+
+    // e2e-confidence #254: name a test that leaves shared infra polluted, at the
+    // boundary — before a later test fails mysteriously downstream. detect…()
+    // also heals (re-scrubs), so a passing-but-polluting test is failed here AND
+    // the next test still starts clean.
+    const pollution = detectSharedDnsPollution();
+    if (pollution && result.status === 'pass') {
+      result.status = 'fail';
+      result.error = `cross-test pollution (#254): this test left shared DNS dirty — ${pollution}`;
+    }
+
     emitTestCompleted({
       runId,
       name,
@@ -663,6 +766,9 @@ async function main() {
       console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
       console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
       console.log(`${dim}  Tear down: cele2e down${reset}`);
+      // The kept network outlives this process — hold the lock in a `kept`
+      // state so the next run refuses to clobber it (cleared by `cele2e down`).
+      markKept();
     }
 
     if (result.status === 'pass') {
@@ -745,6 +851,7 @@ async function main() {
     resultsDir,
   });
 
+  notify('cele2e', `${passed} passed, ${failed} failed — ${formatDuration(suiteDuration)}`);
   process.exit(failed > 0 ? 1 : 0);
 }
 
@@ -762,5 +869,6 @@ main().catch((err) => {
     error: err instanceof Error ? err.message : String(err),
   });
   console.error(err);
+  notify('cele2e', 'run failed');
   process.exit(1);
 });