@celilo/e2e 0.7.15 → 0.7.17

package/scripts/pack-celilo-packages.ts

@@ -13,9 +13,14 @@
  *   @celilo/capabilities → (none)
  *   @celilo/cli-display → (none)
  *
- * So 5 workspace packages need to land in the simulator. We rely on
- * `bun pm pack` to rewrite `workspace:^` deps to concrete versions, so
- * the published tarballs resolve cleanly without real npm.
+ * So 5 workspace packages need to land in the simulator. We rewrite each
+ * package's `workspace:^` deps to concrete `^<workspace-version>` ranges
+ * OURSELVES before `bun pm pack` — we do NOT rely on bun's lock-based
+ * expansion, which goes stale after a cross-caret version bump and pins
+ * dependents to an unresolvable old range (ISS-0103: a `0.3→0.4`
+ * capabilities bump left the lock at 0.3, so packed cli pinned
+ * `^0.3.0-alpha.0`, excluding the 0.4.x the sim served). package.json is
+ * restored after packing so `workspace:^` stays in source.
  *
  * Output directory is typically `packages/e2e/.npm-registry-cache/` —
  * staged into the npm-registry-sim Docker image at build time.
@@ -25,11 +30,73 @@
  */
 
 import { execSync } from 'node:child_process';
-import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 
 const REPO_ROOT = join(import.meta.dir, '..', '..', '..');
 
+interface PackageJson {
+  name?: string;
+  version?: string;
+  dependencies?: Record<string, string>;
+  devDependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  optionalDependencies?: Record<string, string>;
+}
+
+export interface WorkspaceRewrite {
+  depName: string;
+  oldSpec: string;
+  newSpec: string;
+}
+
+const DEP_BUCKETS = [
+  'dependencies',
+  'devDependencies',
+  'peerDependencies',
+  'optionalDependencies',
+] as const;
+
+/**
+ * Rewrite `workspace:` deps in `pkg` to concrete ranges using the
+ * name→version map, mutating `pkg`. Mirrors the publish flow's
+ * `rewriteWorkspaceDeps`:
+ *   workspace:^      → ^<version>
+ *   workspace:~      → ~<version>
+ *   workspace:*  / workspace:  → *           (any version)
+ *   workspace:X.Y.Z  → X.Y.Z                 (explicit, passes through)
+ *
+ * We do this ourselves rather than trust `bun pm pack`'s lock-based
+ * expansion, which goes stale after a cross-caret version bump (ISS-0103).
+ * Only deps present in `versions` (our own @celilo/* set) are touched.
+ * Returns the rewrites applied.
+ */
+export function rewriteWorkspacePins(
+  pkg: PackageJson,
+  versions: Map<string, string>,
+): WorkspaceRewrite[] {
+  const rewrites: WorkspaceRewrite[] = [];
+  for (const bucket of DEP_BUCKETS) {
+    const deps = pkg[bucket];
+    if (!deps) continue;
+    for (const [name, spec] of Object.entries(deps)) {
+      if (!spec.startsWith('workspace:')) continue;
+      const version = versions.get(name);
+      if (!version) continue;
+      const range = spec.slice('workspace:'.length);
+      let newSpec: string;
+      if (range === '*' || range === '') newSpec = '*';
+      else if (range === '^') newSpec = `^${version}`;
+      else if (range === '~') newSpec = `~${version}`;
+      else newSpec = range;
+      if (newSpec === spec) continue;
+      deps[name] = newSpec;
+      rewrites.push({ depName: name, oldSpec: spec, newSpec });
+    }
+  }
+  return rewrites;
+}
+
 /**
  * Workspace packages that install.sh's transitive @celilo/* closure depends
  * on. Hardcoded by name → relative path so a typo in package.json or a future
@@ -63,27 +130,46 @@ function parseArgs(): CliArgs {
   return { destination };
 }
 
-function pack(pkgPath: string, destination: string): string {
+function pack(pkgPath: string, destination: string, versions: Map<string, string>): string {
   const fullPath = join(REPO_ROOT, pkgPath);
-  if (!existsSync(join(fullPath, 'package.json'))) {
+  const pkgJsonPath = join(fullPath, 'package.json');
+  if (!existsSync(pkgJsonPath)) {
     throw new Error(`No package.json in ${fullPath}`);
   }
-  // Capture filenames before/after so we can identify the tarball this
-  // pack call produced even when the destination already contains tarballs
-  // from earlier packages in the loop.
-  const before = new Set(readdirSync(destination).filter((f) => f.endsWith('.tgz')));
-  execSync(`bun pm pack --destination=${destination}`, {
-    cwd: fullPath,
-    stdio: 'inherit',
-  });
-  const after = readdirSync(destination).filter((f) => f.endsWith('.tgz'));
-  const produced = after.filter((f) => !before.has(f));
-  if (produced.length !== 1) {
-    throw new Error(
-      `Expected exactly one new tarball from packing ${pkgPath}, got ${produced.length}: ${produced.join(', ')}`,
-    );
+
+  // Rewrite workspace:^ deps to concrete ranges ourselves (see header /
+  // ISS-0103), then restore the original so workspace:^ stays in source.
+  const originalPkgJson = readFileSync(pkgJsonPath, 'utf-8');
+  const parsed = JSON.parse(originalPkgJson) as PackageJson;
+  const rewrites = rewriteWorkspacePins(parsed, versions);
+  if (rewrites.length > 0) {
+    const trailingNl = originalPkgJson.endsWith('\n') ? '\n' : '';
+    writeFileSync(pkgJsonPath, JSON.stringify(parsed, null, 2) + trailingNl);
+    for (const r of rewrites) {
+      console.log(`  rewrote ${r.depName}: ${r.oldSpec} → ${r.newSpec}`);
+    }
+  }
+
+  try {
+    // Capture filenames before/after so we can identify the tarball this
+    // pack call produced even when the destination already contains tarballs
+    // from earlier packages in the loop.
+    const before = new Set(readdirSync(destination).filter((f) => f.endsWith('.tgz')));
+    execSync(`bun pm pack --destination=${destination}`, {
+      cwd: fullPath,
+      stdio: 'inherit',
+    });
+    const after = readdirSync(destination).filter((f) => f.endsWith('.tgz'));
+    const produced = after.filter((f) => !before.has(f));
+    if (produced.length !== 1) {
+      throw new Error(
+        `Expected exactly one new tarball from packing ${pkgPath}, got ${produced.length}: ${produced.join(', ')}`,
+      );
+    }
+    return produced[0];
+  } finally {
+    writeFileSync(pkgJsonPath, originalPkgJson);
   }
-  return produced[0];
 }
 
 function workspaceVersion(pkgPath: string): string {
@@ -104,11 +190,17 @@ function main(): void {
   console.log(`Packing @celilo/* packages → ${destination}`);
   console.log('');
 
+  // The name→version map used to rewrite each package's workspace:^ deps to
+  // concrete ranges that resolve against the co-packed versions (ISS-0103).
+  const versions = new Map<string, string>(
+    PACKAGES_TO_PACK.map(({ name, path }) => [name, workspaceVersion(path)]),
+  );
+
   const results: Array<{ name: string; version: string; tarball: string }> = [];
   for (const { name, path } of PACKAGES_TO_PACK) {
     const version = workspaceVersion(path);
     console.log(`▸ ${name}@${version}  (${path})`);
-    const tarball = pack(path, destination);
+    const tarball = pack(path, destination, versions);
     results.push({ name, version, tarball });
     console.log(`  → ${tarball}`);
     console.log('');

package/scripts/stage-apt-repo.ts

@@ -65,11 +65,24 @@ export function stageAptRepo(repoRoot: string, pkgDir: string): boolean {
   process.stdout.write(`  ${'apt-repo (build debs)'.padEnd(28)} `);
   const t0 = Date.now();
 
+  // The deb must install the EXACT version the npm-registry-sim serves locally
+  // (= apps/celilo/package.json). Pin CELILO_VERSION so the postinst runs
+  // `bun add @celilo/cli@<version>` rather than `@celilo/cli@alpha` (build-deb.sh's
+  // default): the sim has no local `alpha` dist-tag, so an `@alpha` install leaks
+  // to the real npm uplink and pulls a stale published alpha (ISS-0153). Pinning
+  // makes the deb version, the installed npm version, and the test expectation a
+  // single source of truth.
+  const version = currentCeliloVersion(repoRoot);
+
   // Reuse the canonical build-deb scripts so versioning/contents match a
   // real release exactly. build:deb builds both arches; build:deb:bootstrap
   // builds the arch-all meta-package.
   for (const script of ['build:deb', 'build:deb:bootstrap']) {
-    const result = spawnSync('bun', ['run', script], { cwd: repoRoot, stdio: 'pipe' });
+    const result = spawnSync('bun', ['run', script], {
+      cwd: repoRoot,
+      stdio: 'pipe',
+      env: { ...process.env, CELILO_VERSION: version },
+    });
     if (result.status !== 0) {
       console.log(`${red}✗${reset}`);
       console.error(result.stderr?.toString());
@@ -81,7 +94,6 @@ export function stageAptRepo(repoRoot: string, pkgDir: string): boolean {
   // Stage every freshly-built .deb into the pool. dist/ may also hold older
   // versions; copy only the current ones so the repo doesn't advertise
   // stale celilo versions the npm-registry-sim no longer serves.
-  const version = currentCeliloVersion(repoRoot);
   const distDir = join(repoRoot, 'dist');
 
   const debs = readdirSync(distDir).filter((f) => f.endsWith('.deb') && f.includes(version));

package/src/cli/build.ts

@@ -48,6 +48,7 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
+const yellow = '\x1b[33m';
 const reset = '\x1b[0m';
 
 interface BuildOptions {
@@ -320,6 +321,30 @@ function bakeManagement(pkgDir: string): void {
   console.log(`\n${green}Baked in ${elapsed}s${reset}\n`);
 }
 
+/**
+ * Pre-seed heavy application images (authentik server, postgres, redis) into
+ * the app-zone preload cache. Without this the `docker-image-cache` is empty on
+ * a fresh checkout / the builder, so every app-module deploy pulls GB through
+ * the simulated internet — slow enough to time the authentik deploy out
+ * (exit 124) and take down forgejo-deploy + full-stack-pipeline (ISS-0154).
+ * Best-effort: `e2e-cache-images` is per-image resilient and the preload
+ * service skips a missing tarball, so a transient registry hiccup degrades to a
+ * slow pull rather than failing build-infra.
+ */
+function cacheAppImages(pkgDir: string): void {
+  const cacheScript = join(pkgDir, 'bin', 'e2e-cache-images');
+  if (!existsSync(cacheScript)) return;
+
+  console.log(`${bold}Pre-seeding app-zone image cache...${reset}\n`);
+  const result = spawnSync('bash', [cacheScript], { stdio: 'inherit' });
+  if (result.status !== 0) {
+    console.error(
+      `${yellow}⚠ image cache step failed — app-zone deploys may pull over the sim${reset}`,
+    );
+  }
+  console.log('');
+}
+
 function saveImages(pkgDir: string): void {
   const cacheDir = join(pkgDir, '.docker-cache');
   mkdirSync(cacheDir, { recursive: true });
@@ -406,6 +431,10 @@ export function runBuild(options: BuildOptions): void {
   // install.sh, rather than a parallel `bun add -g` pinning.
   bakeManagement(pkgDir);
 
+  // Pre-seed heavy app images so app-zone deploys load them from the cache
+  // instead of pulling GB through the sim (ISS-0154).
+  cacheAppImages(pkgDir);
+
   if (save) {
     saveImages(pkgDir);
   }

package/src/cli/command-registry.ts

@@ -26,6 +26,8 @@ export const COMMANDS: CommandDef[] = [
     description: 'Run e2e tests — pass a module path or pattern, or --all / --all-modules',
     flags: [
       { name: '--all', description: 'Run the full regression: every module suite + top-level e2e/tests/' },
+      { name: '--complete', description: 'Alias of --all: run every test, including ci-unsafe (quarantined) ones' },
+      { name: '--ci-safe', description: 'Full regression EXCEPT tests marked cele2e-ci-unsafe (used by the nightly)' },
       { name: '--all-modules', description: 'Run e2e tests for all modules with e2e/ directories' },
       { name: '--keep', description: 'Keep network running after tests' },
       { name: '--reuse', description: 'Reuse existing network if running' },
@@ -99,7 +101,7 @@ export const COMMANDS: CommandDef[] = [
 
 export const UP_PRESETS = ['--caddy', '--full-stack', '--infrastructure'];
 
-export const RUN_FLAGS = ['--all', '--all-modules', '--keep', '--reuse', '--live', '--published'];
+export const RUN_FLAGS = ['--all', '--complete', '--ci-safe', '--all-modules', '--keep', '--reuse', '--live', '--published'];
 
 export const DOWN_FLAGS = ['--keep', '--all'];
 

package/src/cli/completion.ts

@@ -119,6 +119,8 @@ _cele2e() {
             '--reuse[Reuse existing network if running]' \\
             '--live[Use live (non-simulated) internet]' \\
             '--published[Use published .netapp packages]' \\
+            '--ci[Plain log output: one line per step, no spinner]' \\
+            '--no-ci[Force spinner even when a CI env var is set]' \\
             '*::test name or module path:_cele2e_run_args'
           ;;
         list)

package/src/cli/index.ts

@@ -139,6 +139,8 @@ Options for \`run\`:
   --reuse           Reuse existing network if running
   --live            Use live (non-simulated) internet
   --published       Use published .netapp packages
+  --ci              Plain log output: one ✔/✗ line per step, no spinner
+                    (auto-on when a CI env var is set; --no-ci forces off)
 
 Options for \`up\`:
   --caddy           Start with caddy machine in DMZ
@@ -187,8 +189,17 @@ switch (command) {
     // repo's top-level e2e/tests/ in one invocation; `--all-modules` runs just
     // the module suites. Both resolve the repo root via resolveSuiteRoot() so
     // they work through the `./cele2e` wrapper's cd into packages/e2e.
-    if (args.includes('--all') || args.includes('--all-modules')) {
-      const wantTopLevel = args.includes('--all');
+    if (
+      args.includes('--all') ||
+      args.includes('--all-modules') ||
+      args.includes('--complete') ||
+      args.includes('--ci-safe')
+    ) {
+      // --complete and --ci-safe both run the full discovery (every module +
+      // the repo's top-level e2e/tests/). --ci-safe additionally has the runner
+      // skip tests marked `cele2e-ci-unsafe`. --all-modules stays modules-only.
+      const wantTopLevel =
+        args.includes('--all') || args.includes('--complete') || args.includes('--ci-safe');
       const suiteRoot = resolveSuiteRoot();
       const allModules = findAllModules(suiteRoot);
       const env: Record<string, string> = {
@@ -206,7 +217,9 @@ switch (command) {
       }
       runScript(
         'e2e-run',
-        args.filter((a) => a !== '--all' && a !== '--all-modules'),
+        // Strip the discovery aliases; KEEP --ci-safe so the runner applies the
+        // `cele2e-ci-unsafe` quarantine exclusion.
+        args.filter((a) => a !== '--all' && a !== '--all-modules' && a !== '--complete'),
         env,
       );
     }

package/src/docker-compose-generator.ts

@@ -396,6 +396,17 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
         variant === 'vanilla' ? 'celilo-e2e/management:vanilla' : 'celilo-e2e/management:latest',
       networks: { internal: { ipv4_address: ip } },
       cap_add: ['NET_ADMIN'],
+      // ISS-0157: `module import`'s `bun install` hard-links packages from the
+      // bun cache into node_modules, which on overlayfs forces a copy-up + fsync
+      // per file. On a slow/contended builder disk that storm blows past the
+      // 120s import timeout (intermittently). Put the celilo data dir (the
+      // node_modules install target) on tmpfs (RAM): bun then copies cache→tmpfs
+      // (cross-fs, no overlay copy-up) at RAM speed, immune to host disk. NOT
+      // /root/.bun itself — that holds the baked `celilo` CLI (`bun add -g`).
+      // Tmpfs the bun *cache* subdir too: the copy-up + fsync storm is the
+      // overlay upper layer under /root/.bun/install/cache. RAM-backing it (cold
+      // → re-fetch over the fast sim/net) keeps the CLI intact. Ephemeral e2e.
+      tmpfs: ['/root/.local/share/celilo', '/root/.bun/install/cache'],
       volumes: [
         ...(celiloRoot ? [`${celiloRoot}:/celilo`] : []),
         'ssh-keys:/ssh-keys',

package/src/runner.ts

@@ -19,9 +19,9 @@
 
 import { spawn, spawnSync, execSync } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { basename, join, resolve } from 'node:path';
+import { basename, dirname, join, resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
-import { ProgressDisplay } from '@celilo/cli-display';
+import { type DisplayMode, ProgressDisplay } from '@celilo/cli-display';
 import {
   emitRunCompleted,
   emitRunFailed,
@@ -70,7 +70,23 @@ const flagReuse = rawArgs.includes('--reuse');
 const flagLive = rawArgs.includes('--live');
 const flagPublished = rawArgs.includes('--published');
 const flagVerbose = rawArgs.includes('--verbose');
+// CI mode: no animated spinner, no [progress:*] markers — just one clean
+// ✔/✗ line per step (plus sub-events) written straight to the log. Forgejo's
+// runner allocates a PTY, so process.stdout.isTTY is truthy there and the
+// interactive footer (redrawn via `\r` on every spinner tick) gets linearized
+// into one log line per tick — dozens of identical braille lines per step.
+// `--ci` (or any standard `CI` env var, e.g. Forgejo/GitHub Actions) forces
+// the non-interactive render path that sidesteps that entirely. `--no-ci`
+// opts back out when a CI env var is set but an operator wants the spinner.
+const flagCi = rawArgs.includes('--ci');
+const flagNoCi = rawArgs.includes('--no-ci');
+const ci = flagCi || (!flagNoCi && !!process.env.CI);
 const patterns = rawArgs.filter((a) => !a.startsWith('--'));
+// --ci-safe: skip tests marked `cele2e-ci-unsafe` in their file (quarantined —
+// known-failing or builder-flaky, each tracked by an ISS). `--complete`/`--all`
+// run everything. Passed through from the CLI (not stripped) so the runner can
+// apply the exclusion.
+const flagCiSafe = rawArgs.includes('--ci-safe');
 
 // ─── Colors ──────────────────────────────────────────────────────────
 
@@ -81,7 +97,8 @@ const red = '\x1b[31m';
 const cyan = '\x1b[36m';
 const reset = '\x1b[0m';
 
-const interactive = process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose;
+const interactive =
+  process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose && !ci;
 
 // ─── Timing ──────────────────────────────────────────────────────────
 
@@ -454,12 +471,45 @@ async function main() {
       .filter((f) => f.endsWith('.test.ts'))
       .sort()
       .map((f) => resolve(testsPath, f));
+    // Docker-backed integration tests live in a sibling `tests-integration/`
+    // so they stay OUT of the hermetic `bun test tests/` gate. cele2e is the
+    // heavy runner (Docker + build-infra present), so it covers BOTH tiers —
+    // this is what keeps the documented `cele2e run ansible-output` working
+    // after the test moved out of tests/.
+    const integrationPath = join(dirname(testsPath), 'tests-integration');
+    if (existsSync(integrationPath)) {
+      testFiles.push(
+        ...readdirSync(integrationPath)
+          .filter((f) => f.endsWith('.test.ts'))
+          .sort()
+          .map((f) => resolve(integrationPath, f)),
+      );
+    }
   }
 
   if (patterns.length > 0) {
     testFiles = testFiles.filter((f) => patterns.some((p) => basename(f, '.test.ts').includes(p)));
   }
 
+  // --ci-safe: drop quarantined tests — those whose file contains the marker
+  // `cele2e-ci-unsafe` (a comment naming the reason + ISS). Keeps the nightly
+  // green on the CI-safe subset while the quarantined tests are tracked + fixed
+  // separately. `--complete`/`--all` run everything (no exclusion).
+  if (flagCiSafe) {
+    const before = testFiles.length;
+    testFiles = testFiles.filter((f) => {
+      try {
+        return !readFileSync(f, 'utf-8').includes('cele2e-ci-unsafe');
+      } catch {
+        return true;
+      }
+    });
+    const skipped = before - testFiles.length;
+    if (skipped > 0) {
+      console.log(`${dim}--ci-safe: skipped ${skipped} quarantined (cele2e-ci-unsafe) test(s)${reset}`);
+    }
+  }
+
   if (testFiles.length === 0) {
     console.error(`${red}No test files matched${reset}`);
     process.exit(1);
@@ -524,14 +574,21 @@ async function main() {
 
   // 'auto' resolves to 'render' when isTTY is true (operator at a
   // terminal sees animated spinners) and 'protocol' when isTTY is
-  // false (CI / cele2e-run.sh wrapper / any redirected stdout gets
-  // ASCII `[progress:*]` markers — grep-friendly, no braille noise).
+  // false (cele2e-run.sh wrapper / any redirected stdout gets ASCII
+  // `[progress:*]` markers — grep-friendly, no braille noise).
   // Prior to this change the mode was hardcoded 'render', so non-TTY
   // callers got `⣾ phase started` / `✔ phase done` lines per phase
   // (e.g. ~29 braille codepoints in a typical test log) — cosmetic
   // but actively unhelpful for log scanning and tooling.
+  //
+  // CI mode forces 'render' with isTTY:false (non-interactive render):
+  // one ✔/✗ line per step, sub-events emitted inline, NO spinner and
+  // NO `[progress:*]` markers. We can't rely on 'auto' here because
+  // Forgejo's PTY makes isTTY truthy — which would pick the animated
+  // footer and spam the log with one braille line per spinner tick.
+  const displayMode: DisplayMode = ci ? 'render' : 'auto';
   const display = new ProgressDisplay({
-    mode: 'auto',
+    mode: displayMode,
     out: { write: process.stdout.write.bind(process.stdout), isTTY: interactive },
   });
   let passed = 0;

package/src/shared-infra.ts

@@ -12,7 +12,7 @@
 
 import { execSync } from 'node:child_process';
 import { join } from 'node:path';
-import { writeFileSync } from 'node:fs';
+import { copyFileSync, existsSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { generateSharedInfraYaml, SHARED_PROJECT_NAME, SHARED_NETWORKS } from './docker-compose-generator';
 import { ensureRegistryServerBundle } from './registry-bundle';
 
@@ -146,6 +146,23 @@ export async function ensureSharedInfra(): Promise<void> {
   const yaml = generateSharedInfraYaml();
   writeFileSync(join(e2eDir, SHARED_COMPOSE_FILE), yaml);
 
+  // Seed the live DNS zone files from their templates BEFORE the compose
+  // mounts them. config/dns/{iamtheinternet.org,example.net}.zone are
+  // gitignored runtime state (scrubDnsZones rewrites them per-test); on a
+  // fresh checkout they don't exist yet, and a Docker bind-mount of a missing
+  // host path silently creates an empty DIRECTORY — so Knot serves no zone and
+  // DNS convergence times out. Seeding here makes the bind-mount targets real
+  // files, so shared infra bootstraps from a clean checkout (e.g. CI builder).
+  for (const zone of ['iamtheinternet.org', 'example.net']) {
+    const tpl = join(e2eDir, 'config', 'dns', 'templates', `${zone}.zone`);
+    const live = join(e2eDir, 'config', 'dns', `${zone}.zone`);
+    // Remove an empty dir left by a prior bind-mount on a missing path.
+    if (existsSync(live) && statSync(live).isDirectory()) {
+      rmSync(live, { recursive: true, force: true });
+    }
+    copyFileSync(tpl, live);
+  }
+
   // Build and start
   console.log('[progress:start] building shared infrastructure images | shared images built');
   run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} build`,
@@ -155,11 +172,16 @@ export async function ensureSharedInfra(): Promise<void> {
   run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`,
     { cwd: e2eDir, timeout: 120_000 });
 
-  // Wait for DNS convergence
+  // Wait for DNS convergence — robustly. The old single 60s window with no
+  // recovery was an intermittent build-infra failure on the builder (#319:
+  // namecheap-dns hadn't served its zone within 60s, and the bare throw gave
+  // no clue why). Now: a longer window, RESTART the DNS container if it's
+  // stuck (re-triggers the zone load), and DUMP diagnostics before giving up.
   console.log('[progress:start] waiting for DNS convergence | DNS converged');
+  const dnsTimeout = 180_000;
   const start = Date.now();
-  const timeout = 60_000;
-  while (Date.now() - start < timeout) {
+  let lastRestart = start;
+  while (Date.now() - start < dnsTimeout) {
     try {
       const result = run(
         `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2`,
@@ -173,9 +195,34 @@ export async function ensureSharedInfra(): Promise<void> {
     } catch {
       // retry
     }
+    // Recovery: if ~45s have passed since the last (re)start without
+    // converging, the DNS container likely didn't load its zone — restart
+    // it to re-trigger the load instead of waiting out the whole window.
+    if (Date.now() - lastRestart > 45_000) {
+      console.log('[progress:sub] DNS not converged yet — restarting namecheap-dns');
+      try {
+        run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} restart namecheap-dns`,
+          { cwd: e2eDir, timeout: 30_000 });
+      } catch {}
+      lastRestart = Date.now();
+    }
     await new Promise(r => setTimeout(r, 2000));
   }
-  throw new Error('Timeout waiting for shared infrastructure DNS convergence');
+  // Diagnostics before failing, so the log shows WHY it didn't converge.
+  let psOut = '';
+  try {
+    psOut = run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} ps`,
+      { cwd: e2eDir, timeout: 10_000 });
+  } catch {}
+  let dnsLog = '';
+  try {
+    dnsLog = run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} logs --tail 40 namecheap-dns`,
+      { cwd: e2eDir, timeout: 10_000 });
+  } catch {}
+  throw new Error(
+    `Timeout waiting for shared infrastructure DNS convergence (${dnsTimeout / 1000}s)\n` +
+      `--- compose ps ---\n${psOut}\n--- namecheap-dns logs (tail 40) ---\n${dnsLog}`,
+  );
 }
 
 /**