@celilo/e2e 0.11.3 → 0.13.0

package/src/router-swap.ts

@@ -0,0 +1,121 @@
+/**
+ * Swap the physical router the `fw-isp` simulator impersonates, on a RUNNING
+ * network — the harness half of the ISP-replaced-the-box scenario.
+ *
+ * `.axonRouter()` on the network builder cannot express this: it sets
+ * `ROUTER_VENDOR_PREFIX` in the generated compose file, which is read once when
+ * the simulator process starts. That is the right tool for "this network has an
+ * Axon in it"; it cannot say "this network had a GreenWave and now has an Axon",
+ * which is the event the whole module-pause change exists to survive.
+ *
+ * The swap models what actually happens to a fleet when an ISP replaces the
+ * hardware, and each part is load-bearing for what the migration test proves:
+ *
+ *  - the TR-181 vendor prefix changes, so the OLD module's spelling is wrong;
+ *  - the credentials change, which is why the fleet wedges rather than merely
+ *    misbehaving — the old module cannot authenticate, so it can neither drive
+ *    the new box nor clean up after itself;
+ *  - every port forward is gone, because a new device arrives empty. That is
+ *    the assertion with teeth: the forwards have to be recreated by unpausing
+ *    the consumers that own them, and a simulator that kept its old table would
+ *    let a completely broken unpause pass.
+ */
+
+import { greenwaveRouterIp } from './types';
+import type { NetworkHandle } from './types';
+
+/** The devices the shared simulator can stand in for. */
+export const ROUTER_DEVICES = {
+  /** GreenWave C4000XG — the default, driven by `modules/greenwave`. */
+  greenwave: { vendorPrefix: 'X_GWS_', username: 'admin', password: 'admin' },
+  /** Axon Networks Q1000K — driven by `modules/axon`. */
+  axon: { vendorPrefix: 'X_AXON_', username: 'axonadmin', password: 'axonsecret' },
+} as const;
+
+export type RouterDevice = keyof typeof ROUTER_DEVICES;
+
+export interface RouterSwapResult {
+  /** Port forwards the outgoing device was holding, all of which are now gone. */
+  forwardsDiscarded: number;
+  device: { vendorPrefix: string; username: string };
+}
+
+/**
+ * Replace the impersonated device. Runs the request from INSIDE the simulator
+ * container, so the control endpoint never has to be reachable from anywhere
+ * else on the simulated internet.
+ */
+export async function swapRouterDevice(
+  net: NetworkHandle,
+  device: RouterDevice,
+): Promise<RouterSwapResult> {
+  const spec = ROUTER_DEVICES[device];
+  const result = await net.exec(
+    'fw-isp',
+    `curl -sk -X POST https://${greenwaveRouterIp()}/sim/swap-device ` +
+      `-d 'vendorPrefix=${spec.vendorPrefix}&username=${spec.username}&password=${spec.password}'`,
+  );
+  if (result.exitCode !== 0) {
+    throw new Error(`swapRouterDevice(${device}) failed: ${result.stderr || result.stdout}`);
+  }
+  return JSON.parse(result.stdout) as RouterSwapResult;
+}
+
+/**
+ * The port forwards the device is currently holding, read back OFF the device
+ * rather than out of celilo's own records.
+ *
+ * This exists because "verify the contract, not the verdict" is the whole point
+ * of the migration test: celilo reporting that it registered a forward is not
+ * evidence the router has one, and those are exactly the two things that came
+ * apart when the hardware changed underneath.
+ */
+export async function listRouterForwards(net: NetworkHandle, device: RouterDevice) {
+  const spec = ROUTER_DEVICES[device];
+  const login = await net.exec(
+    'fw-isp',
+    `curl -sk -X POST https://${greenwaveRouterIp()}/cgi/cgi_action ` +
+      `-d 'username=${spec.username}&password=${spec.password}' -c /tmp/swap-cookies`,
+  );
+  if (login.exitCode !== 0) {
+    throw new Error(`could not log in to the ${device} router: ${login.stderr || login.stdout}`);
+  }
+  const listed = await net.exec(
+    'fw-isp',
+    `curl -sk 'https://${greenwaveRouterIp()}/cgi/cgi_get?Object=Device.NAT.PortMapping.' -b /tmp/swap-cookies`,
+  );
+  return listed.stdout;
+}
+
+/**
+ * The external ports currently forwarded on the device, as a sorted list.
+ *
+ * Assert on THIS rather than on a consumer's container IP. A forward's
+ * `InternalClient` is the firewall's natIp DNAT ingress (e.g. `10.226.1.253`),
+ * not the consumer's zone-side address — the delegation chain hops through the
+ * firewall — so matching a module's own IP looks correct and never matches.
+ */
+export function forwardedPorts(listing: string): string[] {
+  const ports = [...listing.matchAll(/"ParamName":"ExternalPort","ParamValue":"(\d+)"/g)].map(
+    (m) => m[1],
+  );
+  return [...new Set(ports)].sort();
+}
+
+/** The DHCP pool's advertised DNS servers, read back off the device. */
+export async function routerDhcpDnsServers(
+  net: NetworkHandle,
+  device: RouterDevice,
+): Promise<string> {
+  const spec = ROUTER_DEVICES[device];
+  await net.exec(
+    'fw-isp',
+    `curl -sk -X POST https://${greenwaveRouterIp()}/cgi/cgi_action ` +
+      `-d 'username=${spec.username}&password=${spec.password}' -c /tmp/dhcp-cookies`,
+  );
+  const listed = await net.exec(
+    'fw-isp',
+    `curl -sk 'https://${greenwaveRouterIp()}/cgi/cgi_get?Object=Device.DHCPv4.Server.Pool.' -b /tmp/dhcp-cookies`,
+  );
+  return listed.stdout;
+}

package/src/run-lock.test.ts

@@ -5,8 +5,12 @@ import { join } from 'node:path';
 import {
   E2eBusyError,
   type LockHolder,
+  SUSPECT_HEARTBEAT_MS,
   acquireRunLock,
   clearLock,
+  formatBusy,
+  heartbeatAgeMs,
+  isSuspect,
   lockStatus,
   markKept,
   readHolder,
@@ -25,6 +29,7 @@ afterEach(() => {
   clearLock();
   rmSync(dir, { recursive: true, force: true });
   delete process.env.CELILO_E2E_LOCK_PATH;
+  delete process.env.CELILO_E2E_SESSION;
 });
 
 function writeRawHolder(h: Partial<LockHolder>): void {
@@ -71,7 +76,8 @@ test('a dead-pid holder on the same host is stale and reclaimed', () => {
   releaseRunLock();
 });
 
-test('--keep leaves a kept lock that survives release and blocks the next run', () => {
+test('--keep leaves a kept lock that survives release and blocks other sessions', () => {
+  process.env.CELILO_E2E_SESSION = 'keeper (/keeper/worktree)';
   acquireRunLock({ test: 'kept-test', runId: 'r' });
   markKept();
   releaseRunLock();
@@ -80,7 +86,8 @@ test('--keep leaves a kept lock that survives release and blocks the next run',
   expect(h?.state).toBe('kept');
   expect(lockStatus().free).toBe(false); // kept is never stale
 
-  // A plain run is refused...
+  // A plain run from ANOTHER session is refused...
+  process.env.CELILO_E2E_SESSION = 'other (/other/worktree)';
   expect(() => acquireRunLock({ test: 'next', runId: 'r2' })).toThrow(E2eBusyError);
   // ...but a --reuse run (allowKept) takes it over.
   acquireRunLock({ test: 'reuse', runId: 'r3', allowKept: true });
@@ -96,6 +103,70 @@ test('clearLock frees a kept lock (the `cele2e release` path)', () => {
   expect(lockStatus().free).toBe(true);
 });
 
+test('a kept lock left by THIS session is auto-released by the next run', () => {
+  // The friction case: `run --keep` then `run` from the same session. Refusing
+  // here protected nobody — the only stack at risk was the caller's own — and
+  // the refusal was routinely misread as a finished run, because the previous
+  // run's results dir is still sitting there looking like a clean pass.
+  process.env.CELILO_E2E_SESSION = 'my-branch (/my/worktree)';
+  acquireRunLock({ test: 'kept-test', runId: 'r' });
+  markKept();
+  releaseRunLock();
+  expect(readHolder()?.state).toBe('kept');
+
+  const outcome = acquireRunLock({ test: 'next', runId: 'r2' });
+  expect(outcome.autoReleasedOwnKept?.test).toBe('kept-test');
+  expect(readHolder()?.pid).toBe(process.pid);
+  expect(readHolder()?.state).toBe('running');
+  releaseRunLock();
+});
+
+test('a kept lock from a DIFFERENT session is still refused', () => {
+  process.env.CELILO_E2E_SESSION = 'my-branch (/my/worktree)';
+  // A live pid so staleness can't be what frees it — the point is the session.
+  writeRawHolder({ state: 'kept', session: 'someone-else (/their/worktree)', pid: process.pid });
+  expect(() => acquireRunLock({ test: 'mine', runId: 'r' })).toThrow(E2eBusyError);
+});
+
+test('a live pid with a silent heartbeat is SUSPECT but never auto-reclaimed', () => {
+  // build-infra wedged at step [16/27]: the process is alive (so PID-liveness
+  // says "healthy") while its heartbeat has not ticked for half an hour. This
+  // is the one hang the existing staleness check structurally cannot see.
+  writeRawHolder({ pid: process.pid, beatAt: Date.now() - 1_913_000 });
+  const h = readHolder() as LockHolder;
+
+  expect(isSuspect(h)).toBe(true);
+  expect(heartbeatAgeMs(h)).toBeGreaterThan(SUSPECT_HEARTBEAT_MS);
+  expect(formatBusy(h)).toContain('SUSPECT');
+
+  const status = lockStatus();
+  expect(status.suspect).toBe(true);
+  // Suspect surfaces the hang; it does NOT kill someone's build for them.
+  expect(status.free).toBe(false);
+  expect(() => acquireRunLock({ test: 'other', runId: 'r' })).toThrow(E2eBusyError);
+});
+
+test('a fresh heartbeat is not suspect', () => {
+  writeRawHolder({ pid: process.pid, beatAt: Date.now() - 5_000 });
+  const h = readHolder() as LockHolder;
+  expect(isSuspect(h)).toBe(false);
+  expect(lockStatus().suspect).toBe(false);
+});
+
+test('a healthy build-infra that blocks the event loop is NOT suspect', () => {
+  // spawnSync('docker', ['build', …]) blocks the event loop, so the heartbeat
+  // stops for the whole of each image build. A live holder was observed 67s
+  // stale while making normal progress; the threshold has to clear that or the
+  // flag fires on every large image and stops meaning anything.
+  writeRawHolder({ pid: process.pid, beatAt: Date.now() - 120_000 });
+  expect(isSuspect(readHolder() as LockHolder)).toBe(false);
+});
+
+test('a kept holder is never suspect — its beatAt is frozen on purpose', () => {
+  writeRawHolder({ state: 'kept', pid: process.pid, beatAt: Date.now() - 3_600_000 });
+  expect(isSuspect(readHolder() as LockHolder)).toBe(false);
+});
+
 test('a corrupt lock file is reclaimed, not fatal', () => {
   writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, 'not json{');
   expect(readHolder()).toBeNull();

package/src/run-lock.ts

@@ -18,9 +18,14 @@
  *  - Staleness: on the SAME host, PID-liveness is authoritative (process.kill
  *    (pid, 0)); a dead holder PID → reclaim. Cross-host we can't check the PID,
  *    so fall back to a heartbeat TTL (beatAt older than STALE_TTL_MS).
- *  - A `kept` lock (left by `--keep` / `up`) is NEVER auto-reclaimed — it
- *    guards a stack that outlives the process and is cleared only by
- *    `cele2e release` / `cele2e down`. This enforces the honor-system rule.
+ *  - A `kept` lock (left by `--keep` / `up`) is never reclaimed by ANOTHER
+ *    actor — it guards a stack that outlives the process. Its OWN session
+ *    reclaims it automatically (see isSameSession); everyone else clears it
+ *    deliberately via `cele2e release` / `cele2e down`.
+ *  - SUSPECT holders: a live PID whose heartbeat has gone quiet is the one
+ *    hang the PID check cannot see (a build wedged mid-step keeps its process
+ *    alive and sleeping). isSuspect() names it so `status`/`doctor` can flag it
+ *    instead of leaving 20 silent minutes to be found by hand.
  */
 
 import { execFileSync } from 'node:child_process';
@@ -31,6 +36,23 @@ import { basename, dirname, join } from 'node:path';
 const STALE_TTL_MS = 90_000;
 const HEARTBEAT_MS = 30_000;
 
+/**
+ * How quiet a RUNNING holder's heartbeat may go before it is suspect.
+ *
+ * Calibrated against what a HEALTHY holder actually does. The heartbeat is a
+ * setInterval, and build-infra shells out with spawnSync — which blocks the
+ * event loop for the entire duration of each `docker build`. So a perfectly
+ * healthy build stops beating for as long as its slowest image takes: a live
+ * holder was observed 67s stale while making normal progress, and the largest
+ * images here are ~2GB. Anything near the 30s tick interval would flag those.
+ *
+ * Ten minutes sits above any single legitimate image build (the per-image
+ * watchdog caps one at 15) and far below the ~20 and ~32 minute hangs that
+ * went unnoticed. Deliberately NOT a reclaim threshold: the PID is alive, and
+ * killing someone's wedged build out from under them is the operator's call.
+ */
+export const SUSPECT_HEARTBEAT_MS = 600_000;
+
 /**
  * Machine-global lock path — NOT under any repo/worktree, because the Docker
  * infra is global regardless of which checkout started it. Overridable via
@@ -75,14 +97,22 @@ export function formatBusy(h: LockHolder): string {
     h.state === 'kept'
       ? '(run `cele2e release` to free it)'
       : `${h.test}, started ${age} ago (pid ${h.pid})`;
-  return `e2e busy: ${h.session} ${verb} ${what}`;
+  const suspect = isSuspect(h)
+    ? ` — SUSPECT: no heartbeat for ${formatAge(heartbeatAgeMs(h))} (process alive but not progressing)`
+    : '';
+  return `e2e busy: ${h.session} ${verb} ${what}${suspect}`;
 }
 
 function ageString(since: number): string {
-  const s = Math.max(0, Math.floor((Date.now() - since) / 1000));
+  return formatAge(Date.now() - since);
+}
+
+/** Human duration, e.g. "45s" / "32m" / "2h05m". */
+export function formatAge(ms: number): string {
+  const s = Math.max(0, Math.floor(ms / 1000));
   if (s < 60) return `${s}s`;
   if (s < 3600) return `${Math.floor(s / 60)}m`;
-  return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
+  return `${Math.floor(s / 3600)}h${String(Math.floor((s % 3600) / 60)).padStart(2, '0')}m`;
 }
 
 /** Branch + worktree path so a holder maps back to a specific session/thread. */
@@ -107,6 +137,28 @@ function deriveSession(): string {
   return branch || base;
 }
 
+/**
+ * Is this holder the caller's own session — same host, same branch+worktree?
+ * This is what separates "my own kept stack is in my way" (friction, auto-clear)
+ * from "someone else is mid-run" (contention, refuse).
+ */
+export function isSameSession(h: LockHolder): boolean {
+  return h.hostname === hostname() && h.session === deriveSession();
+}
+
+/** Milliseconds since the holder last refreshed its heartbeat. */
+export function heartbeatAgeMs(h: LockHolder): number {
+  return Math.max(0, Date.now() - h.beatAt);
+}
+
+/**
+ * A live process whose heartbeat has gone quiet — the hang PID-liveness misses.
+ * `kept` holders are excluded: their beatAt is frozen on purpose at release.
+ */
+export function isSuspect(h: LockHolder): boolean {
+  return h.state === 'running' && heartbeatAgeMs(h) > SUSPECT_HEARTBEAT_MS;
+}
+
 function pidAlive(pid: number): boolean {
   try {
     process.kill(pid, 0);
@@ -140,15 +192,32 @@ function writeHolder(fd: number, h: LockHolder): void {
   writeFileSync(fd, JSON.stringify(h, null, 2));
 }
 
+/** What the acquire had to clear on the way in, so callers can say so out loud. */
+export interface AcquireOutcome {
+  /** A `kept` lock left by this same session that we auto-released, else null. */
+  autoReleasedOwnKept: LockHolder | null;
+}
+
 /**
  * Acquire the run lock for this process. Throws E2eBusyError if another live
  * (non-stale) holder owns it. Registers an exit handler so the lock is released
  * on any process.exit() path (normal, exception, or a SIGINT handler that
  * exits). A signal-killed process with no exit handler leaks the lock, but the
  * next run reclaims it via PID-liveness — that's exactly what staleness is for.
+ *
+ * A `kept` lock left by THIS session (same host, same branch+worktree) is
+ * auto-released and reported in the outcome. Refusing there protected nobody:
+ * the only stack at risk was the caller's own, and the refusal was routinely
+ * misread as a finished run — the previous run's results dir is still sitting
+ * there looking like a clean pass. Another session's kept lock still refuses.
  */
-export function acquireRunLock(opts: { test: string; runId: string; allowKept?: boolean }): void {
+export function acquireRunLock(opts: {
+  test: string;
+  runId: string;
+  allowKept?: boolean;
+}): AcquireOutcome {
   mkdirSync(dirname(lockPath()), { recursive: true });
+  let autoReleasedOwnKept: LockHolder | null = null;
 
   const holder: LockHolder = {
     pid: process.pid,
@@ -168,9 +237,17 @@ export function acquireRunLock(opts: { test: string; runId: string; allowKept?:
     } catch (err) {
       if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
       const existing = readHolder();
-      // Reclaim when: corrupt/unreadable, the holder is stale, OR this is a
-      // `--reuse` run taking over a `kept` stack it's deliberately reusing.
-      if (!existing || isStale(existing) || (opts.allowKept && existing.state === 'kept')) {
+      // Reclaim when: corrupt/unreadable, the holder is stale, this is a
+      // `--reuse` run taking over a `kept` stack it's deliberately reusing, OR
+      // the `kept` stack belongs to this very session (see the doc comment).
+      const ownKept = !!existing && existing.state === 'kept' && isSameSession(existing);
+      if (
+        !existing ||
+        isStale(existing) ||
+        (opts.allowKept && existing.state === 'kept') ||
+        ownKept
+      ) {
+        if (ownKept && existing) autoReleasedOwnKept = existing;
         try {
           unlinkSync(lockPath());
         } catch {}
@@ -195,7 +272,7 @@ export function acquireRunLock(opts: { test: string; runId: string; allowKept?:
 
     held = { keepOnRelease: false, heartbeat, released: false };
     process.on('exit', releaseRunLock);
-    return;
+    return { autoReleasedOwnKept };
   }
   // Two reclaim attempts both lost the race → someone else won fair and square.
   const existing = readHolder();
@@ -258,9 +335,28 @@ export function clearLock(): boolean {
   }
 }
 
+export interface LockStatus {
+  free: boolean;
+  holder: LockHolder | null;
+  /** Milliseconds since the holder's last heartbeat; null when free. */
+  heartbeatAgeMs: number | null;
+  /** Live PID, quiet heartbeat — the hang the PID check cannot see. */
+  suspect: boolean;
+  /** The holder is this session's own kept stack, which the next run reclaims. */
+  ownKept: boolean;
+}
+
 /** Current lock state for `cele2e status` (and any poller). */
-export function lockStatus(): { free: boolean; holder: LockHolder | null } {
+export function lockStatus(): LockStatus {
   const h = readHolder();
-  if (!h || isStale(h)) return { free: true, holder: null };
-  return { free: false, holder: h };
+  if (!h || isStale(h)) {
+    return { free: true, holder: null, heartbeatAgeMs: null, suspect: false, ownKept: false };
+  }
+  return {
+    free: false,
+    holder: h,
+    heartbeatAgeMs: heartbeatAgeMs(h),
+    suspect: isSuspect(h),
+    ownKept: h.state === 'kept' && isSameSession(h),
+  };
 }

package/src/runner.ts

@@ -29,7 +29,9 @@ import {
   emitTestCompleted,
   emitTestStarted,
 } from './bus-events';
-import { extractFailureMessage, stripAnsi } from './extract-failure';
+import { diagnose, formatReport } from './doctor';
+import { type StageTally, extractFailureMessage, stripAnsi, tallyStages } from './extract-failure';
+import { writeLastRun } from './last-run';
 import { parseLine } from './parse-line';
 import { E2eBusyError, acquireRunLock, markKept } from './run-lock';
 import { SIMULATOR_IPS } from './simulator-ips';
@@ -200,6 +202,12 @@ interface TestResult {
   rawTail?: string;
   hadDebugPause?: boolean;
   projectName?: string;
+  /**
+   * Real stage failures vs stages a failing earlier stage blocked. Without the
+   * split, one bad fixture line in stage 1 of a 10-stage suite reports as "9
+   * failed" — nine counts of a defect that does not exist.
+   */
+  stages?: StageTally;
 }
 
 async function runTest(
@@ -399,12 +407,24 @@ async function runTest(
           projectName: capturedProjectName,
           error: extractFailureMessage(lines, code ?? 1),
           rawTail,
+          stages: tallyStages(lines),
         });
       }
     });
   });
 }
 
+/**
+ * " — 1 stage failed, 8 skipped (blocked by an earlier stage)". Rendered only
+ * when a cascade actually happened, so an ordinary single-stage failure stays
+ * as terse as it was.
+ */
+function stageSuffix(stages: StageTally | undefined): string {
+  if (!stages || stages.skipped === 0) return '';
+  const failed = `${stages.failed} stage${stages.failed === 1 ? '' : 's'} failed`;
+  return ` ${dim}— ${failed}, ${stages.skipped} skipped (blocked by an earlier stage)${reset}`;
+}
+
 // ─── Shared Infrastructure ───────────────────────────────────────────
 
 function dockerComposeShared(cmd: string): string {
@@ -475,11 +495,20 @@ async function main() {
   // docker mutation so a competing run can't wipe our shared infra mid-setup.
   const lockLabel = patterns.length ? patterns.join(',') : moduleDirs.length ? 'modules' : 'all';
   try {
-    acquireRunLock({
+    const outcome = acquireRunLock({
       test: lockLabel,
       runId: process.env.CELE2E_RUN_ID ?? ambientRunId,
       allowKept: flagReuse,
     });
+    if (outcome.autoReleasedOwnKept) {
+      const own = outcome.autoReleasedOwnKept;
+      console.log(
+        `${dim}Auto-released your own kept stack (${own.test}, held since ${own.startedAt}) and continuing.${reset}`,
+      );
+      console.log(
+        `${dim}Use \`cele2e run --reuse\` instead if you meant to run against it.${reset}`,
+      );
+    }
     // A --reuse run keeps using a kept stack that stays up afterwards — hold the
     // lock in kept state so it isn't freed out from under the reused network.
     if (flagReuse) markKept();
@@ -494,6 +523,25 @@ async function main() {
     throw err;
   }
 
+  // Preflight the environment BEFORE touching Docker. Every check here failed
+  // silently once and surfaced later somewhere unrelated — a missing bake as an
+  // SSH error against a firewall IP, a pruned base image as a TLS timeout 16
+  // images into a build. Refusing now costs seconds; not refusing cost a full
+  // run plus a debugging cycle each time. Lock state is excluded: we already
+  // hold the lock, so checkRunLock would report ourselves as contention.
+  const health = diagnose({ pkgDir: PKG_DIR, skipLock: true });
+  if (!health.ok) {
+    console.error(
+      `\n${red}✗ cele2e preflight failed — not starting a run that cannot succeed.${reset}`,
+    );
+    for (const line of formatReport(health)) console.error(line);
+    console.error(`\n${dim}Full environment report: cele2e doctor${reset}\n`);
+    process.exit(1);
+  }
+  for (const c of health.checks) {
+    if (c.status === 'warn') console.log(`${dim}! ${c.name}: ${c.detail}${reset}`);
+  }
+
   let testFiles: string[] = [];
 
   if (moduleDirs.length > 0) {
@@ -656,6 +704,22 @@ async function main() {
   if (!process.env.CELE2E_RUN_ID) {
     console.log(`${dim}runId: ${runId}${reset}`);
   }
+  // Print the results dir HERE, not only at the end: a run that dies mid-way
+  // still wrote logs there, and `ls -t results | head -1` is not a safe way to
+  // find them — after a refused start it returns the PREVIOUS run's numbers,
+  // which look exactly like a clean pass. `cele2e last` reads the same record.
+  console.log(`${dim}results:  ${resultsDir}${reset}`);
+  writeLastRun({
+    runId,
+    resultsDir,
+    startedAt: new Date(suiteStart).toISOString(),
+    status: 'running',
+    total: testFiles.length,
+    passed: 0,
+    failed: 0,
+    skipped: 0,
+    durationMs: 0,
+  });
   emitRunStarted({
     runId,
     scenario: testFiles.length > 1 ? 'multi-test' : 'single-test',
@@ -784,7 +848,7 @@ async function main() {
       console.log(`  ${red}  └─ ${result.error}${reset}`);
       failed++;
     } else {
-      console.log(`  ${red}✗ failed (${result.duration}s)${reset}`);
+      console.log(`  ${red}✗ failed (${result.duration}s)${reset}${stageSuffix(result.stages)}`);
       if (result.error) {
         console.log(`  ${red}  └─ ${result.error}${reset}`);
       }
@@ -817,18 +881,28 @@ async function main() {
 
   const suiteDuration = Math.floor((Date.now() - suiteStart) / 1000);
 
+  // Stages a failing earlier stage blocked are reported separately from real
+  // failures. Counting them together turns one bad fixture line into "9 failed"
+  // and buries the single defect that actually exists.
+  const stageSkipped = results.reduce((sum, r) => sum + (r.stages?.skipped ?? 0), 0);
+
   console.log();
   console.log(`${bold}╔══════════════════════════════════════════════════════════════${reset}`);
   console.log(
     `${bold}║  Results: ${green}${passed} passed${reset}${bold}, ${red}${failed} failed${reset}${bold} — ${formatDuration(suiteDuration)} total${reset}`,
   );
+  if (stageSkipped > 0) {
+    console.log(
+      `${bold}║${reset}  ${dim}${stageSkipped} stage(s) skipped — blocked by an earlier stage, not separate defects${reset}`,
+    );
+  }
   console.log(`${bold}╠══════════════════════════════════════════════════════════════${reset}`);
 
   for (const r of results) {
     const icon = r.status === 'pass' ? '✓' : '✗';
     const color = r.status === 'pass' ? green : red;
     console.log(
-      `${bold}║${reset}  ${color}${icon}${reset} ${r.name.padEnd(28)} ${dim}${(`${r.duration}s`).padStart(6)}${reset}`,
+      `${bold}║${reset}  ${color}${icon}${reset} ${r.name.padEnd(28)} ${dim}${(`${r.duration}s`).padStart(6)}${reset}${stageSuffix(r.stages)}`,
     );
     if (r.error) {
       console.log(`${bold}║${reset}    ${red}└─ ${r.error}${reset}`);
@@ -838,16 +912,36 @@ async function main() {
   console.log(`${bold}╚══════════════════════════════════════════════════════════════${reset}`);
   console.log();
   console.log(`${dim}Full logs: ${resultsDir}${reset}`);
+  console.log(`${dim}Machine-readable: cele2e last --json${reset}`);
 
   writeFileSync(
     join(resultsDir, 'summary.json'),
     JSON.stringify(
-      { timestamp, total: testFiles.length, passed, failed, duration: suiteDuration },
+      {
+        timestamp,
+        total: testFiles.length,
+        passed,
+        failed,
+        stageSkipped,
+        duration: suiteDuration,
+      },
       null,
       2,
     ),
   );
 
+  writeLastRun({
+    runId,
+    resultsDir,
+    startedAt: new Date(suiteStart).toISOString(),
+    status: failed > 0 ? 'failed' : 'completed',
+    total: testFiles.length,
+    passed,
+    failed,
+    skipped: stageSkipped,
+    durationMs: suiteDuration * 1000,
+  });
+
   const gitignore = join(testDir, '.gitignore');
   const content = existsSync(gitignore) ? readFileSync(gitignore, 'utf-8') : '';
   const adds: string[] = [];

package/src/simulator-ips.ts

@@ -44,6 +44,27 @@ export const SIMULATOR_IPS = {
   CPANEL_HOST: '100.64.0.63',
   /** signal-cli release host — serves the tarball the signal module downloads at deploy time. */
   SIGNAL_RELEASE: '100.64.0.62',
+  /**
+   * OFF-FLEET recursive resolver — the rig's stand-in for 1.1.1.1, and a peer
+   * of comcast-resolver rather than a replacement for it.
+   *
+   * The distinction is the whole point of celilo's `public_dns` check: the
+   * fleet's own resolver (its ISP's, or its internal split-horizon one)
+   * answers with whatever is correct for a client INSIDE, which is not
+   * evidence about what the internet sees. celilo REFUSES to use a resolver it
+   * is itself configured to use, so verifying public reachability requires a
+   * second, independent public resolver — and until this existed the topology
+   * had exactly one.
+   */
+  PUBLIC_RESOLVER: '100.64.0.64',
+  /**
+   * IP echo service — the rig's stand-in for api.ipify.org. Reports the source
+   * address a request appears to come from, which for the customer fleet is
+   * the firewall's external address after SNAT. The `public_dns` check's
+   * expectation comes from here rather than from the registrar's own response,
+   * which is self-agreement (and, for a Namecheap `www` update, false).
+   */
+  IP_ECHO: '100.64.0.65',
   /** Pebble ACME server (replaces production Let's Encrypt). */
   PEBBLE: '100.64.0.100',
 } as const;

package/src/socks-proxy.ts

@@ -44,7 +44,7 @@ export interface SocksProxyHandle {
  * Why custom: an off-the-shelf SOCKS5 image (e.g. serjs/go-socks5-proxy)
  * starts fine on the test network but can't actually route to anything.
  * Docker's IPAM-default gateway points at an IP with no listener (e.g.
- * 100.100.0.250 on isp-external), and docker rewrites resolv.conf to
+ * 203.0.113.250 on isp-external), and docker rewrites resolv.conf to
  * its embedded resolver, which can't cleanly forward to the simulated
  * network's DNS. The custom image's entrypoint replaces both.
  */
@@ -61,7 +61,7 @@ interface VantageConfig {
 const VANTAGE_CONFIGS: Record<SocksProxyVantage, VantageConfig> = {
   'isp-external': {
     network: 'isp-external',
-    ip: '100.100.0.150',
+    ip: '203.0.113.150',
   },
   internal: {
     network: 'internal',