@celilo/e2e 0.5.1 → 0.6.1

package/src/runner.ts

@@ -20,12 +20,37 @@
 import { spawn, spawnSync, execSync } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { basename, join, resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { ProgressDisplay } from '@celilo/cli-display';
+import { parseLine } from './parse-line';
 
 // ─── Config ──────────────────────────────────────────────────────────
 
 const PKG_DIR = resolve(import.meta.dir, '..');
 const testDir = process.env.E2E_TEST_DIR || process.cwd();
-const testsPath = join(testDir, 'tests');
+/** Set by `cele2e run <module-path>` to run tests from that module's e2e/ dir */
+const moduleDir = process.env.E2E_MODULE_DIR;
+/** Set by `cele2e run` (no args) when invoked outside a module — runs every discovered module's tests. */
+const moduleDirs = process.env.E2E_MODULE_DIRS
+  ? process.env.E2E_MODULE_DIRS.split('\n').filter(Boolean)
+  : moduleDir
+    ? [moduleDir]
+    : [];
+
+function resolveModuleTestsDir(dir: string): string {
+  const manifestPath = join(dir, 'manifest.yml');
+  if (existsSync(manifestPath)) {
+    try {
+      const parsed = parseYaml(readFileSync(manifestPath, 'utf-8')) as { e2e?: { tests_dir?: string } };
+      if (parsed?.e2e?.tests_dir) return resolve(dir, parsed.e2e.tests_dir);
+    } catch {}
+  }
+  return join(dir, 'e2e');
+}
+
+const testsPath = moduleDir
+  ? resolveModuleTestsDir(moduleDir)
+  : (process.env.E2E_TESTS_PATH ?? join(testDir, 'tests'));
 const timingFile = join(testDir, '.e2e-timing.json');
 const persistentFile = join(testDir, '.e2e-persistent.json');
 
@@ -33,6 +58,9 @@ const persistentFile = join(testDir, '.e2e-persistent.json');
 const rawArgs = process.argv.slice(2);
 const flagKeep = rawArgs.includes('--keep');
 const flagReuse = rawArgs.includes('--reuse');
+const flagLive = rawArgs.includes('--live');
+const flagPublished = rawArgs.includes('--published');
+const flagVerbose = rawArgs.includes('--verbose');
 const patterns = rawArgs.filter((a) => !a.startsWith('--'));
 
 // ─── Colors ──────────────────────────────────────────────────────────
@@ -44,11 +72,7 @@ const red = '\x1b[31m';
 const cyan = '\x1b[36m';
 const reset = '\x1b[0m';
 
-// Terminal control for interactive mode
-const CLEAR_LINE = '\x1b[2K\r';
-const MOVE_UP = '\x1b[A';
-
-const interactive = process.stdout.isTTY && !process.argv.includes('--no-interactive');
+const interactive = process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose;
 
 // ─── Timing ──────────────────────────────────────────────────────────
 
@@ -71,240 +95,58 @@ function formatDuration(secs: number): string {
   return `${secs}s`;
 }
 
-/** Format countdown: positive = remaining, negative = over estimate */
-function formatCountdown(elapsedSecs: number, expectedSecs: number | undefined): string {
-  if (!expectedSecs) return `${dim}${String(elapsedSecs).padStart(4)}s${reset}`;
-  const remaining = expectedSecs - elapsedSecs;
-  if (remaining >= 0) {
-    return `${dim}${String(remaining).padStart(4)}s${reset}`;
-  }
-  return `${red}${String(remaining).padStart(4)}s${reset}`;
-}
-
-// ─── Progress Display ────────────────────────────────────────────────
-
-interface PendingStep {
-  doingMsg: string;
-  doneMsg: string;
-  startTime: number;
-}
-
-class ProgressDisplay {
-  private pending: PendingStep | null = null;
-  private testStart = 0;
-  private expectedDuration: number | undefined;
-  /** Number of sub-event lines printed below the current "… doing" line */
-  private subEventCount = 0;
-
-  reset(testStart: number, expectedDuration: number | undefined) {
-    this.pending = null;
-    this.testStart = testStart;
-    this.expectedDuration = expectedDuration;
-    this.subEventCount = 0;
-  }
-
-  private time(): string {
-    const elapsed = Math.floor((Date.now() - this.testStart) / 1000);
-    return formatCountdown(elapsed, this.expectedDuration);
-  }
-
-  /**
-   * In interactive mode: erase the "… doing" line and all sub-event
-   * lines below it, then print the "✔ done" line in their place.
-   * In non-interactive mode: just append the done line.
-   */
-  private finalizePending() {
-    if (!this.pending) return;
-    if (interactive) {
-      // Erase sub-event lines + the pending "…" line
-      const linesToErase = this.subEventCount + 1;
-      for (let i = 0; i < linesToErase; i++) {
-        process.stdout.write(`${MOVE_UP}${CLEAR_LINE}`);
-      }
-    }
-    process.stdout.write(`  ${this.time()}  ${green}✔${reset} ${this.pending.doneMsg}\n`);
-    this.pending = null;
-    this.subEventCount = 0;
-  }
-
-  private finalizeFailed(message: string) {
-    if (interactive && this.pending) {
-      const linesToErase = this.subEventCount + 1;
-      for (let i = 0; i < linesToErase; i++) {
-        process.stdout.write(`${MOVE_UP}${CLEAR_LINE}`);
-      }
-    }
-    process.stdout.write(`  ${this.time()}  ${red}✗${reset} ${message}\n`);
-    this.pending = null;
-    this.subEventCount = 0;
-  }
-
-  startStep(doingMsg: string, doneMsg: string) {
-    this.finalizePending();
-    this.pending = { doingMsg, doneMsg, startTime: Date.now() };
-    this.subEventCount = 0;
-    process.stdout.write(`  ${this.time()}  ${dim}…${reset} ${doingMsg}\n`);
-  }
-
-  doneStep(overrideMessage?: string) {
-    if (this.pending && overrideMessage) {
-      this.pending.doneMsg = overrideMessage;
-    }
-    this.finalizePending();
-  }
-
-  failStep(message: string) {
-    this.finalizeFailed(message);
-  }
-
-  instantEvent(message: string) {
-    this.finalizePending();
-    process.stdout.write(`  ${this.time()}  ${message}\n`);
-  }
-
-  /**
-   * Show a sub-event under the current "… doing" step.
-   * Interactive: replaces the "… doing" line with the sub-event,
-   * then reprints "… doing" below it — so the pending line is always last.
-   * Non-interactive: just appends indented.
-   */
-  subEvent(message: string) {
-    if (interactive && this.pending) {
-      // Replace the "… doing" line with this sub-event (permanent)
-      process.stdout.write(`${MOVE_UP}${CLEAR_LINE}`);
-      process.stdout.write(`  ${this.time()}    ${message}\n`);
-      // Reprint the "… doing" line as the new bottom line
-      process.stdout.write(`  ${this.time()}  ${dim}…${reset} ${this.pending.doingMsg}\n`);
-      this.subEventCount++;
-    } else {
-      process.stdout.write(`  ${this.time()}    ${message}\n`);
-      if (this.pending) this.subEventCount++;
-    }
-  }
-
-  /** Finalize pending on success */
-  flush() {
-    this.finalizePending();
-  }
+// ─── Error Extraction ────────────────────────────────────────────────
 
-  /** Abandon pending step on failure (erase the … line in interactive mode) */
-  abandon() {
-    if (!this.pending) return;
-    if (interactive) {
-      const linesToErase = this.subEventCount + 1;
-      for (let i = 0; i < linesToErase; i++) {
-        process.stdout.write(`${MOVE_UP}${CLEAR_LINE}`);
-      }
-    }
-    this.pending = null;
-    this.subEventCount = 0;
-  }
+// Strips ANSI escape sequences so pattern matching works on bun's coloured output
+const ANSI_RE = /\x1b\[[0-9;]*m/g;
+function stripAnsi(s: string): string {
+  return s.replace(ANSI_RE, '');
 }
 
-// ─── Line Parsing ────────────────────────────────────────────────────
-
-function parseLine(line: string, display: ProgressDisplay): void {
-  // Protocol: [progress:start] doing | done
-  const startMatch = line.match(/\[progress:start\] (.*?) \| (.*)/);
-  if (startMatch) {
-    display.startStep(startMatch[1], startMatch[2]);
-    return;
-  }
-
-  // Protocol: [progress:done] optional message
-  const doneMatch = line.match(/\[progress:done\](.*)/);
-  if (doneMatch) {
-    const msg = doneMatch[1].trim();
-    display.doneStep(msg || undefined);
-    return;
-  }
-
-  // Protocol: [progress:fail] message
-  const failMatch = line.match(/\[progress:fail\] (.*)/);
-  if (failMatch) {
-    display.failStep(failMatch[1]);
-    return;
-  }
-
-  // Legacy: [progress] message (treat as instant event)
-  const legacyMatch = line.match(/\[progress\] (.*)/);
-  if (legacyMatch) {
-    display.instantEvent(legacyMatch[1]);
-    return;
-  }
-
-  // Celilo CLI signals — shown as sub-events under the current step.
-  // These must NOT finalize the test's progress step.
-  const deployMatch = line.match(/Module '([^']+)' deployed successfully/);
-  if (deployMatch) {
-    display.subEvent(`${green}✔${reset} ${deployMatch[1]} deployed`);
-    return;
-  }
-
-  const importMatch = line.match(/Successfully imported module: (\S+)/);
-  if (importMatch) {
-    display.subEvent(`${green}✔${reset} ${importMatch[1]} imported`);
-    return;
-  }
-
-  if (line.includes('Deploying software')) {
-    display.subEvent('running Ansible');
-    return;
-  }
-  if (line.includes('Infrastructure selected')) {
-    display.subEvent('infrastructure selected');
-    return;
-  }
-  if (line.includes('Terraform skipped')) {
-    display.subEvent('terraform skipped (existing machine)');
-    return;
-  }
-  if (line.includes('terraform init') || line.includes('Terraform init')) {
-    display.subEvent('running terraform init');
-    return;
-  }
-  if (line.includes('terraform apply') || line.includes('Terraform apply')) {
-    display.subEvent('running terraform apply');
-    return;
-  }
-
-  const retryMatch = line.match(/RETRYING.*?(\d+) retries left/);
-  if (retryMatch) {
-    display.subEvent(`health check (${retryMatch[1]} retries left)`);
-    return;
+function extractFailureMessage(lines: string[], exitCode: number): string {
+  const plain = lines.map(stripAnsi);
+
+  // bun's own test failure: "✗ test name [Xms]" — extract the failing test name
+  const bunFailIdx = plain.findIndex((l) => /^\s*✗\s+\S/.test(l) && !l.includes('failed ('));
+  if (bunFailIdx >= 0) {
+    const testName = plain[bunFailIdx].replace(/^\s*✗\s+/, '').replace(/\s*\[\d+ms\].*$/, '').trim();
+    // Grab the error detail on the next few lines (bun indents it under the test)
+    const detail = plain
+      .slice(bunFailIdx + 1, bunFailIdx + 6)
+      .map((l) => l.trim())
+      .filter((l) => l.length > 0 && !l.startsWith('at ') && !l.startsWith('│') && l !== '└─')
+      .slice(0, 2)
+      .join(' — ');
+    return detail ? `${testName}: ${detail}` : testName;
   }
 
-  // Error lines
-  if (line.includes('Error:') && !line.includes('bun test') && !line.includes('ipv4: Address already')) {
-    const err = line.replace(/.*Error: /, '').slice(0, 70);
-    display.subEvent(`${red}✗${reset} ${err}`);
-  }
-}
-
-// ─── Error Extraction ────────────────────────────────────────────────
-
-function extractFailureMessage(lines: string[], exitCode: number): string {
-  const errorIdx = lines.findIndex(
-    (l) => l.startsWith('error:') && !l.includes('bun test'),
+  // "error: ..." lines (may be indented in bun output)
+  const errorIdx = plain.findIndex(
+    (l) => l.trimStart().startsWith('error:') && !l.includes('bun test'),
   );
-
   if (errorIdx >= 0) {
-    const context = lines
+    const context = plain
       .slice(errorIdx, errorIdx + 5)
       .map((l) => l.trim())
       .filter((l) => l.length > 0 && !l.startsWith('at '));
     return context.join('\n    ');
   }
 
-  const celiloError = lines.find(
+  const celiloError = plain.find(
     (l) => l.includes('CeliloCommandError') || (l.includes('celilo') && l.includes('failed')),
   );
   if (celiloError) return celiloError.slice(0, 120);
 
-  const timeout = lines.find((l) => l.includes('Timeout') || l.includes('timed out'));
+  const timeout = plain.find((l) => l.includes('Timeout') || l.includes('timed out'));
   if (timeout) return timeout.slice(0, 120);
 
-  return `exit code ${exitCode}`;
+  // Last resort: print the last few non-empty, non-progress lines from the output
+  const tail = plain
+    .filter((l) => l.length > 0 && !l.startsWith('[progress') && !l.startsWith('[e2e:') && !l.startsWith('[ansible:'))
+    .slice(-3)
+    .map((l) => l.trim())
+    .join(' | ');
+  return tail || `exit code ${exitCode}`;
 }
 
 // ─── Test Execution ──────────────────────────────────────────────────
@@ -314,6 +156,8 @@ interface TestResult {
   status: 'pass' | 'fail' | 'suspicious';
   duration: number;
   error?: string;
+  /** Last ~30 lines of raw test output, shown on failure so the user doesn't have to dig in logs */
+  rawTail?: string;
   hadDebugPause?: boolean;
   projectName?: string;
 }
@@ -333,9 +177,19 @@ async function runTest(
   let capturedProjectName: string | undefined;
 
   return new Promise<TestResult>((resolveTest) => {
-    // Env vars let the test detect --keep / --reuse modes
+    // Env vars let the test detect --keep / --reuse / --live / --published modes
     const testEnv: NodeJS.ProcessEnv = { ...process.env };
     if (flagKeep) testEnv.CELILO_E2E_KEEP = '1';
+    if (flagLive) {
+      // Resolve infra root relative to the test directory (sibling repo convention)
+      const inferredRoot = resolve(testDir, '../../infra');
+      testEnv.CELILO_E2E_INFRA_ROOT = existsSync(join(inferredRoot, 'apps', 'celilo'))
+        ? inferredRoot
+        : resolve(PKG_DIR, '..', '..'); // fallback: @celilo/e2e is inside the infra monorepo
+    }
+    if (flagPublished) {
+      testEnv.CELILO_E2E_INFRA_ROOT = ''; // empty = force published mode
+    }
     if (flagReuse) {
       // Read the persistent project and pass it to the test
       try {
@@ -364,9 +218,9 @@ async function runTest(
 
       const signalFile = join(PKG_DIR, `.debug-resume-${proc.pid}`);
       const border = '═'.repeat(56);
-      process.stdout.write(`\n${bold}\x1b[33m╔${border}╗${reset}\n`);
-      process.stdout.write(`${bold}\x1b[33m║  DEBUG: test paused                                    ║${reset}\n`);
-      process.stdout.write(`${bold}\x1b[33m╠${border}╣${reset}\n`);
+      process.stdout.write(`\n${bold}\x1b[33m╔${border}${reset}\n`);
+      process.stdout.write(`${bold}\x1b[33m║  DEBUG: test paused${reset}\n`);
+      process.stdout.write(`${bold}\x1b[33m╠${border}${reset}\n`);
       process.stdout.write(`${bold}\x1b[33m║${reset}  Project:   ${bold}${project}${reset}\n`);
       process.stdout.write(`${bold}\x1b[33m║${reset}  Container: ${bold}${container}${reset}\n`);
       process.stdout.write(`${bold}\x1b[33m║${reset}\n`);
@@ -377,7 +231,7 @@ async function runTest(
 
       if (interactive) {
         process.stdout.write(`${bold}\x1b[33m║${reset}  ${dim}Or get a shell: type 'exit' to resume the test.${reset}\n`);
-        process.stdout.write(`${bold}\x1b[33m╚${border}╝${reset}\n\n`);
+        process.stdout.write(`${bold}\x1b[33m╚${border}${reset}\n\n`);
 
         spawnSync('docker', [
           'compose', '-f', 'docker-compose.test.yml', '-p', project,
@@ -388,7 +242,7 @@ async function runTest(
         });
       } else {
         process.stdout.write(`${bold}\x1b[33m║${reset}  ${dim}Resume: touch ${signalFile}${reset}\n`);
-        process.stdout.write(`${bold}\x1b[33m╚${border}╝${reset}\n\n`);
+        process.stdout.write(`${bold}\x1b[33m╚${border}${reset}\n\n`);
 
         // Non-interactive: just wait for the signal file (no shell)
         // The caller creates the file when they're done investigating.
@@ -423,6 +277,11 @@ async function runTest(
           continue;
         }
 
+        // Detect bun test failure so pending progress steps show ✗ instead of ✔
+        if (/^\s*\(fail\)\s/.test(trimmed) || /^\s*✗\s+.+\[/.test(trimmed)) {
+          display.markTestFailing();
+        }
+
         parseLine(trimmed, display);
       }
     }
@@ -438,7 +297,7 @@ async function runTest(
       if (code === 0) {
         display.flush();
       } else {
-        display.abandon();
+        display.abandonFailed();
       }
 
       writeFileSync(logFile, stdout + '\n---STDERR---\n' + stderr);
@@ -460,6 +319,20 @@ async function runTest(
           resolveTest({ name, status: 'pass', duration, hadDebugPause, projectName: capturedProjectName });
         }
       } else {
+        // Collect a raw tail: strip ANSI, drop internal protocol lines, keep last 30
+        const rawTail = (stdout + stderr)
+          .split('\n')
+          .map(stripAnsi)
+          .map((l) => l.trimEnd())
+          .filter((l) =>
+            l.length > 0 &&
+            !l.startsWith('[progress') &&
+            !l.startsWith('[e2e:') &&
+            !l.startsWith('[ansible:') &&
+            !l.startsWith('[debug:'),
+          )
+          .slice(-30)
+          .join('\n');
         resolveTest({
           name,
           status: 'fail',
@@ -467,6 +340,7 @@ async function runTest(
           hadDebugPause,
           projectName: capturedProjectName,
           error: extractFailureMessage(lines, code ?? 1),
+          rawTail,
         });
       }
     });
@@ -499,18 +373,63 @@ function stopSharedInfra(): void {
   try { execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 }); } catch {}
 }
 
-// ─── Main ────────────────────────────────────────────────────────────
+// ─── Docker Check ────────────────────────────────────────────────────
 
-async function main() {
-  if (!existsSync(testsPath)) {
-    console.error(`${red}No tests/ directory found at ${testsPath}${reset}`);
+function assertDockerAvailable(): void {
+  try {
+    execSync('docker info', { stdio: 'pipe', timeout: 5_000 });
+  } catch {
+    console.error(`\n${red}✗ Docker is not running.${reset}`);
+    console.error(`  Start it with: ${bold}colima start${reset}`);
+    console.error(`  Then retry:    ${bold}cele2e run${reset}\n`);
     process.exit(1);
   }
+}
+
+// ─── Main ────────────────────────────────────────────────────────────
+
+async function main() {
+  assertDockerAvailable();
 
-  let testFiles = readdirSync(testsPath)
-    .filter((f) => f.endsWith('.test.ts'))
-    .sort()
-    .map((f) => resolve(testsPath, f));
+  let testFiles: string[] = [];
+
+  if (moduleDirs.length > 0) {
+    // One or more modules — collect tests from each module's e2e dir.
+    for (const dir of moduleDirs) {
+      if (!existsSync(join(dir, 'manifest.yml'))) {
+        console.error(`${red}Not a celilo module: no manifest.yml found at ${dir}${reset}`);
+        process.exit(1);
+      }
+      const tp = resolveModuleTestsDir(dir);
+      if (!existsSync(tp)) {
+        console.error(`${red}No tests directory found at ${tp}${reset}`);
+        console.error(`${red}Set e2e.tests_dir in ${join(dir, 'manifest.yml')} to point to your test files.${reset}`);
+        process.exit(1);
+      }
+      // Auto-install e2e/ dependencies if package.json exists but node_modules is missing
+      const e2ePkg = join(tp, 'package.json');
+      const e2eNodeModules = join(tp, 'node_modules');
+      if (existsSync(e2ePkg) && !existsSync(e2eNodeModules)) {
+        process.stdout.write(`${dim}Installing e2e dependencies for ${basename(dir)}...${reset}\n`);
+        execSync('bun install', { cwd: tp, stdio: 'inherit' });
+        process.stdout.write('\n');
+      }
+      const files = readdirSync(tp)
+        .filter((f) => f.endsWith('.test.ts'))
+        .sort()
+        .map((f) => resolve(tp, f));
+      testFiles.push(...files);
+    }
+  } else {
+    if (!existsSync(testsPath)) {
+      console.error(`${red}No tests/ directory found at ${testsPath}${reset}`);
+      process.exit(1);
+    }
+    testFiles = readdirSync(testsPath)
+      .filter((f) => f.endsWith('.test.ts'))
+      .sort()
+      .map((f) => resolve(testsPath, f));
+  }
 
   if (patterns.length > 0) {
     testFiles = testFiles.filter((f) => patterns.some((p) => basename(f, '.test.ts').includes(p)));
@@ -521,20 +440,40 @@ async function main() {
     process.exit(1);
   }
 
+  // --keep keeps a single test network alive after the run; only one network
+  // can live in `.e2e-persistent.json` at a time, so multiple tests would
+  // either fight for that slot or run with no functional --keep at all. Bail
+  // early with a clear message instead of silently doing the wrong thing.
+  if (flagKeep && testFiles.length > 1) {
+    console.error(
+      `${red}--keep matched ${testFiles.length} tests; --keep only works with a single test.${reset}`,
+    );
+    console.error(
+      `${dim}Narrow the pattern (e.g. \`cele2e run ${basename(testFiles[0], '.test.ts')} --keep\`) or drop --keep.${reset}`,
+    );
+    process.exit(1);
+  }
+
   const history = loadTiming();
   const testNames = testFiles.map((f) => basename(f, '.test.ts'));
   const hasHistory = testNames.some((n) => history[n]);
   const suiteExpected = testNames.reduce((sum, n) => sum + (history[n] || 0), 0);
 
+  const moduleLabel =
+    moduleDirs.length === 1
+      ? ` — ${basename(moduleDirs[0])}`
+      : moduleDirs.length > 1
+        ? ` — ${moduleDirs.length} modules`
+        : '';
   console.log();
-  console.log(`${bold}╔══════════════════════════════════════════════════════════════╗${reset}`);
-  console.log(`${bold}║  Celilo E2E Tests                                          ║${reset}`);
+  console.log(`${bold}╔══════════════════════════════════════════════════════════════${reset}`);
+  console.log(`${bold}║  Celilo E2E Tests${moduleLabel}${reset}`);
   if (hasHistory) {
-    console.log(`${bold}║  ${testFiles.length} test(s) — expected ~${formatDuration(suiteExpected).padEnd(30)}║${reset}`);
+    console.log(`${bold}║  ${testFiles.length} test(s) — expected ~${formatDuration(suiteExpected)}${reset}`);
   } else {
-    console.log(`${bold}║  ${testFiles.length} test(s) — first run, no timing history             ║${reset}`);
+    console.log(`${bold}║  ${testFiles.length} test(s) — first run, no timing history${reset}`);
   }
-  console.log(`${bold}╚══════════════════════════════════════════════════════════════╝${reset}`);
+  console.log(`${bold}╚══════════════════════════════════════════════════════════════${reset}`);
   console.log();
 
   const suiteStart = Date.now();
@@ -542,7 +481,10 @@ async function main() {
   const resultsDir = join(testDir, 'results', timestamp);
   mkdirSync(resultsDir, { recursive: true });
 
-  const display = new ProgressDisplay();
+  const display = new ProgressDisplay({
+    mode: 'render',
+    out: { write: process.stdout.write.bind(process.stdout), isTTY: interactive },
+  });
   let passed = 0;
   let failed = 0;
   const results: TestResult[] = [];
@@ -600,7 +542,7 @@ async function main() {
       );
       console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
       console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
-      console.log(`${dim}  Tear down: e2e-down${reset}`);
+      console.log(`${dim}  Tear down: cele2e down${reset}`);
     }
 
     if (result.status === 'pass') {
@@ -615,6 +557,14 @@ async function main() {
       if (result.error) {
         console.log(`  ${red}  └─ ${result.error}${reset}`);
       }
+      if (result.rawTail) {
+        console.log();
+        console.log(`${dim}  ── test output ──────────────────────────────────────${reset}`);
+        for (const line of result.rawTail.split('\n')) {
+          console.log(`  ${dim}${line}${reset}`);
+        }
+        console.log(`${dim}  ─────────────────────────────────────────────────────${reset}`);
+      }
       failed++;
     }
     console.log();
@@ -635,11 +585,11 @@ async function main() {
   const suiteDuration = Math.floor((Date.now() - suiteStart) / 1000);
 
   console.log();
-  console.log(`${bold}╔══════════════════════════════════════════════════════════════╗${reset}`);
+  console.log(`${bold}╔══════════════════════════════════════════════════════════════${reset}`);
   console.log(
-    `${bold}║  Results: ${green}${passed} passed${reset}${bold}, ${red}${failed} failed${reset}${bold} — ${formatDuration(suiteDuration)} total${' '.repeat(20)}║${reset}`,
+    `${bold}║  Results: ${green}${passed} passed${reset}${bold}, ${red}${failed} failed${reset}${bold} — ${formatDuration(suiteDuration)} total${reset}`,
   );
-  console.log(`${bold}╠══════════════════════════════════════════════════════════════╣${reset}`);
+  console.log(`${bold}╠══════════════════════════════════════════════════════════════${reset}`);
 
   for (const r of results) {
     const icon = r.status === 'pass' ? '✓' : '✗';
@@ -650,7 +600,7 @@ async function main() {
     }
   }
 
-  console.log(`${bold}╚══════════════════════════════════════════════════════════════╝${reset}`);
+  console.log(`${bold}╚══════════════════════════════════════════════════════════════${reset}`);
   console.log();
   console.log(`${dim}Full logs: ${resultsDir}${reset}`);
 

package/src/simulator-ips.ts

@@ -0,0 +1,42 @@
+/**
+ * Single source of truth for the IPs of e2e simulator containers on
+ * the `internet-external` Docker network.
+ *
+ * Renumbering one of these used to mean tracking down the value in
+ * eight-ish places (compose files, generator, zone files, resolver
+ * configs, routing scripts). This module is the canonical record for
+ * code; non-code files (Knot zones, shell scripts) still hard-code the
+ * values, but `scripts/check-simulator-ips.sh` flags any drift in CI.
+ *
+ * To renumber a simulator:
+ *   1. Update the constant here.
+ *   2. Run `bash packages/e2e/scripts/check-simulator-ips.sh` — it
+ *      lists every non-code file that still has the old value.
+ *   3. Edit those files. Re-run the check until it passes.
+ *   4. `cele2e build-infra` to rebake the images.
+ */
+
+export const SIMULATOR_IPS = {
+  /** Authoritative root DNS (`.` zone). */
+  ROOT_DNS: '100.64.0.53',
+  /** Authoritative TLD DNS for `com.`, `org.`, `net.`. */
+  TLD_DNS: '100.64.0.54',
+  /** Authoritative DNS for the customer domains; also serves the DDNS update endpoint at port 8080. */
+  NAMECHEAP_DNS: '100.64.0.55',
+  /** Internal e2e module registry (replaces production celilo.computer in tests). */
+  REGISTRY: '100.64.0.56',
+  /** External-website prober simulator (mimics isitup.org). */
+  ISITUP: '100.64.0.57',
+  /** Pebble ACME server (replaces production Let's Encrypt). */
+  PEBBLE: '100.64.0.100',
+} as const;
+
+export type SimulatorIpName = keyof typeof SIMULATOR_IPS;
+
+/**
+ * Iterable list of `[name, ip]` pairs — useful for the sanity check
+ * script and any future consumer that wants to enumerate the set.
+ */
+export const SIMULATOR_IP_ENTRIES: ReadonlyArray<readonly [SimulatorIpName, string]> = (
+  Object.entries(SIMULATOR_IPS) as Array<[SimulatorIpName, string]>
+).map(([k, v]) => [k, v] as const);