@celilo/e2e 0.7.16 → 0.8.0

package/bin/e2e-build

@@ -22,6 +22,7 @@ NETAPPS_DIR="$E2E_DIR/netapps"
 cd "$E2E_DIR"
 
 GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
 DIM='\033[2m'
 BOLD='\033[1m'
 NC='\033[0m'
@@ -160,6 +161,25 @@ BAKE_END=$(date +%s)
 echo -e "${GREEN}Baked in $((BAKE_END - BAKE_START))s${NC}"
 echo ""
 
+# ── Step 2d: Pre-seed heavy app images into the app-zone preload cache ─────────
+#
+# Heavy application images (authentik server, postgres, redis) are pulled ONCE
+# on the host here and saved to docker-image-cache/, which app-zone target
+# machines bind-mount and load at startup via docker-image-preload.service.
+# Without this the cache is empty on a fresh checkout / the builder, so every
+# app-module deploy pulls GB through the simulated internet — slow enough to
+# time the authentik deploy out (exit 124) and take down forgejo-deploy +
+# full-stack-pipeline (ISS-0154). Best-effort: e2e-cache-images is per-image
+# resilient and the preload service skips a missing tarball, so a transient
+# registry hiccup degrades to a slow pull rather than failing build-infra.
+echo -e "${BOLD}Pre-seeding app-zone image cache...${NC}"
+if bash "$E2E_DIR/bin/e2e-cache-images"; then
+  :
+else
+  echo -e "${YELLOW}⚠ image cache step failed — app-zone deploys may pull over the sim${NC}" >&2
+fi
+echo ""
+
 # ── Step 3: Optionally save images to tarball ─────────────────────────────────
 
 if [ "$1" = "--save" ]; then

package/bin/e2e-cache-images

@@ -23,6 +23,7 @@ mkdir -p "$CACHE_DIR"
 
 BOLD='\033[1m'
 GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
 DIM='\033[2m'
 NC='\033[0m'
 
@@ -44,12 +45,18 @@ for filename in "${!IMAGES[@]}"; do
   printf "  %-50s " "$image"
   START=$(date +%s)
 
-  docker pull "$image" > /dev/null 2>&1
-  docker save "$image" -o "$tarball"
-
-  END=$(date +%s)
-  SIZE=$(du -h "$tarball" | cut -f1)
-  echo -e "${GREEN}✔${NC} ${DIM}$((END - START))s  ${SIZE}${NC}"
+  # Per-image best-effort: a transient registry failure for one image must not
+  # abort caching the others (this runs inside build-infra now). A guarded
+  # command does not trip `set -e`. The preload service skips a missing tarball
+  # gracefully — that module just pulls over the sim at deploy time.
+  if docker pull "$image" > /dev/null 2>&1 && docker save "$image" -o "$tarball"; then
+    END=$(date +%s)
+    SIZE=$(du -h "$tarball" | cut -f1)
+    echo -e "${GREEN}✔${NC} ${DIM}$((END - START))s  ${SIZE}${NC}"
+  else
+    rm -f "$tarball"
+    echo -e "${YELLOW}✘ failed — will pull over the sim at deploy time${NC}"
+  fi
 done
 
 echo ""

package/config/dns/example.net.zone

@@ -0,0 +1,16 @@
+$ORIGIN example.net.
+$TTL 300
+
+@ IN SOA ns1.example.net. admin.example.net. (
+    2024020108 300 60 604800 300
+)
+
+; Public DNS — RFC 1918 addresses NEVER appear here. The whole point of
+; the e2e simulation is to model real internet connectivity, where
+; private IPs are unreachable from outside. All public-facing names
+; resolve to the firewall's external IP (100.100.0.100), which DNATs
+; inbound to the right internal host.
+@       IN NS   ns1.example.net.
+ns1     IN A    100.64.0.55
+@       IN A    100.100.0.100
+www     IN A    100.100.0.100

package/config/dns/iamtheinternet.org.zone

@@ -0,0 +1,16 @@
+$ORIGIN iamtheinternet.org.
+$TTL 300
+
+@ IN SOA ns1.iamtheinternet.org. admin.iamtheinternet.org. (
+    2024020247 300 60 604800 300
+)
+
+; Public DNS — RFC 1918 addresses NEVER appear here. The whole point of
+; the e2e simulation is to model real internet connectivity, where
+; private IPs are unreachable from outside. All public-facing names
+; resolve to the firewall's external IP (100.100.0.100), which DNATs
+; inbound to the right internal host.
+@       IN NS   ns1.iamtheinternet.org.
+ns1     IN A    100.64.0.55
+@       IN A    0.0.0.0
+www     IN A    100.100.0.100

package/config/routing/fw-main-routes.sh

@@ -8,22 +8,39 @@ chmod 700 /root/.ssh
 chmod 600 /root/.ssh/authorized_keys
 /usr/sbin/sshd
 
-# Routing — detect if we have an isp-external interface (direct-internet topology)
+# IP forwarding is set via sysctls in docker-compose
+iptables -P FORWARD ACCEPT
+
+# Routing + egress NAT — depends on topology (detected by the presence of an
+# isp-external interface).
 ip route del default 2>/dev/null || true
 if ip -o addr show | grep -q '100.100.0'; then
-  # Direct internet: route via fw-ext on isp-external
+  # Direct-internet: fw-main has its own external (WAN) interface.
   ip route add default via 100.100.0.101
+  # Egress NAT SCOPED to the external/WAN interface (the one with the public
+  # 100.100.0.x address) — NOT a blanket MASQUERADE (ISS-0156). Scoping to the WAN
+  # means inter-zone traffic (protected↔protected, protected↔internal) keeps its
+  # source, so the dmz-resident resolver sees each client's real zone for
+  # source-based DNS views. This boot-time rule is what target machines need for
+  # egress/DNS BEFORE any module deploys; celilo's iptables module applies the
+  # same WAN-scoped MASQUERADE on deploy (idempotent — the production code path).
+  EXT_IFACE=$(ip -o addr show | awk '/100\.100\.0\./{print $2; exit}')
+  iptables -t nat -A POSTROUTING -o "$EXT_IFACE" -j MASQUERADE
 else
-  # Default: route via fw-isp on internal
+  # Two-layer: no external interface on fw-main; it routes outbound to the
+  # upstream firewall (fw-isp) and MASQUERADEs toward it (egress leaves via the
+  # internal-facing interface, so there is no single WAN interface to scope to).
   ip route add default via 192.168.0.1
+  # Protected↔protected (dmz/app/secure ↔ each other, all in 10/8) is NOT NAT'd —
+  # a real firewall routes between its segmented zones without NAT, preserving the
+  # client source so the dmz-resident resolver can serve source-based split-horizon
+  # (ISS-0156). Only protected↔internal and egress get MASQUERADE'd below. Without
+  # this RETURN, app→dmz and internal→dmz would both be SNAT'd to the dmz gateway
+  # and become indistinguishable (confirmed via live probe).
+  iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -d 10.0.0.0/8 -j RETURN
+  iptables -t nat -A POSTROUTING -j MASQUERADE
 fi
 
-# IP forwarding is set via sysctls in docker-compose
-
-# NAT all outbound traffic from dmz/app/secure
-iptables -t nat -A POSTROUTING -j MASQUERADE
-iptables -P FORWARD ACCEPT
-
 # DNS
 echo "nameserver 100.100.0.1" > /etc/resolv.conf
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.7.16",
+  "version": "0.8.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -19,7 +19,8 @@
   },
   "scripts": {
     "test": "bun test tests/",
-    "test:completion": "bun test tests/completion"
+    "test:completion": "bun test tests/completion",
+    "test:integration": "bun test tests-integration/"
   },
   "files": [
     "src/",

package/scripts/stage-apt-repo.ts

@@ -65,11 +65,24 @@ export function stageAptRepo(repoRoot: string, pkgDir: string): boolean {
   process.stdout.write(`  ${'apt-repo (build debs)'.padEnd(28)} `);
   const t0 = Date.now();
 
+  // The deb must install the EXACT version the npm-registry-sim serves locally
+  // (= apps/celilo/package.json). Pin CELILO_VERSION so the postinst runs
+  // `bun add @celilo/cli@<version>` rather than `@celilo/cli@alpha` (build-deb.sh's
+  // default): the sim has no local `alpha` dist-tag, so an `@alpha` install leaks
+  // to the real npm uplink and pulls a stale published alpha (ISS-0153). Pinning
+  // makes the deb version, the installed npm version, and the test expectation a
+  // single source of truth.
+  const version = currentCeliloVersion(repoRoot);
+
   // Reuse the canonical build-deb scripts so versioning/contents match a
   // real release exactly. build:deb builds both arches; build:deb:bootstrap
   // builds the arch-all meta-package.
   for (const script of ['build:deb', 'build:deb:bootstrap']) {
-    const result = spawnSync('bun', ['run', script], { cwd: repoRoot, stdio: 'pipe' });
+    const result = spawnSync('bun', ['run', script], {
+      cwd: repoRoot,
+      stdio: 'pipe',
+      env: { ...process.env, CELILO_VERSION: version },
+    });
     if (result.status !== 0) {
       console.log(`${red}✗${reset}`);
       console.error(result.stderr?.toString());
@@ -81,7 +94,6 @@ export function stageAptRepo(repoRoot: string, pkgDir: string): boolean {
   // Stage every freshly-built .deb into the pool. dist/ may also hold older
   // versions; copy only the current ones so the repo doesn't advertise
   // stale celilo versions the npm-registry-sim no longer serves.
-  const version = currentCeliloVersion(repoRoot);
   const distDir = join(repoRoot, 'dist');
 
   const debs = readdirSync(distDir).filter((f) => f.endsWith('.deb') && f.includes(version));

package/src/cli/build.ts

@@ -48,6 +48,7 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
+const yellow = '\x1b[33m';
 const reset = '\x1b[0m';
 
 interface BuildOptions {
@@ -320,6 +321,30 @@ function bakeManagement(pkgDir: string): void {
   console.log(`\n${green}Baked in ${elapsed}s${reset}\n`);
 }
 
+/**
+ * Pre-seed heavy application images (authentik server, postgres, redis) into
+ * the app-zone preload cache. Without this the `docker-image-cache` is empty on
+ * a fresh checkout / the builder, so every app-module deploy pulls GB through
+ * the simulated internet — slow enough to time the authentik deploy out
+ * (exit 124) and take down forgejo-deploy + full-stack-pipeline (ISS-0154).
+ * Best-effort: `e2e-cache-images` is per-image resilient and the preload
+ * service skips a missing tarball, so a transient registry hiccup degrades to a
+ * slow pull rather than failing build-infra.
+ */
+function cacheAppImages(pkgDir: string): void {
+  const cacheScript = join(pkgDir, 'bin', 'e2e-cache-images');
+  if (!existsSync(cacheScript)) return;
+
+  console.log(`${bold}Pre-seeding app-zone image cache...${reset}\n`);
+  const result = spawnSync('bash', [cacheScript], { stdio: 'inherit' });
+  if (result.status !== 0) {
+    console.error(
+      `${yellow}⚠ image cache step failed — app-zone deploys may pull over the sim${reset}`,
+    );
+  }
+  console.log('');
+}
+
 function saveImages(pkgDir: string): void {
   const cacheDir = join(pkgDir, '.docker-cache');
   mkdirSync(cacheDir, { recursive: true });
@@ -406,6 +431,10 @@ export function runBuild(options: BuildOptions): void {
   // install.sh, rather than a parallel `bun add -g` pinning.
   bakeManagement(pkgDir);
 
+  // Pre-seed heavy app images so app-zone deploys load them from the cache
+  // instead of pulling GB through the sim (ISS-0154).
+  cacheAppImages(pkgDir);
+
   if (save) {
     saveImages(pkgDir);
   }

package/src/cli/command-registry.ts

@@ -26,11 +26,14 @@ export const COMMANDS: CommandDef[] = [
     description: 'Run e2e tests — pass a module path or pattern, or --all / --all-modules',
     flags: [
       { name: '--all', description: 'Run the full regression: every module suite + top-level e2e/tests/' },
+      { name: '--complete', description: 'Alias of --all: run every test, including ci-unsafe (quarantined) ones' },
+      { name: '--ci-safe', description: 'Full regression EXCEPT tests marked cele2e-ci-unsafe (used by the nightly)' },
       { name: '--all-modules', description: 'Run e2e tests for all modules with e2e/ directories' },
       { name: '--keep', description: 'Keep network running after tests' },
       { name: '--reuse', description: 'Reuse existing network if running' },
       { name: '--live', description: 'Use live (non-simulated) internet' },
       { name: '--published', description: 'Use published .netapp packages' },
+      { name: '--notify', description: 'Desktop notification when the run finishes' },
     ],
   },
   {
@@ -73,7 +76,12 @@ export const COMMANDS: CommandDef[] = [
   },
   {
     name: 'status',
-    description: 'Show network status and DNS health',
+    description: 'Show run-lock holder + network status (exit 3 if busy)',
+    flags: [{ name: '--json', description: 'Emit a machine-readable lock snapshot for polling' }],
+  },
+  {
+    name: 'release',
+    description: 'Free a run-lock left by --keep/up (does not tear down)',
   },
   {
     name: 'load',
@@ -99,7 +107,7 @@ export const COMMANDS: CommandDef[] = [
 
 export const UP_PRESETS = ['--caddy', '--full-stack', '--infrastructure'];
 
-export const RUN_FLAGS = ['--all', '--all-modules', '--keep', '--reuse', '--live', '--published'];
+export const RUN_FLAGS = ['--all', '--complete', '--ci-safe', '--all-modules', '--keep', '--reuse', '--live', '--published', '--notify'];
 
 export const DOWN_FLAGS = ['--keep', '--all'];
 

package/src/cli/completion.ts

@@ -119,6 +119,9 @@ _cele2e() {
             '--reuse[Reuse existing network if running]' \\
             '--live[Use live (non-simulated) internet]' \\
             '--published[Use published .netapp packages]' \\
+            '--notify[Desktop notification when the run finishes]' \\
+            '--ci[Plain log output: one line per step, no spinner]' \\
+            '--no-ci[Force spinner even when a CI env var is set]' \\
             '*::test name or module path:_cele2e_run_args'
           ;;
         list)
@@ -146,7 +149,10 @@ _cele2e() {
         completion)
           _arguments '1: :_cele2e_completion_shells'
           ;;
-        status|load|clear-timing|list)
+        status)
+          _arguments '--json[Emit a machine-readable lock snapshot for polling]'
+          ;;
+        release|load|clear-timing)
           ;;
       esac
       ;;
@@ -162,7 +168,8 @@ _cele2e_commands() {
     'build-infra:Rebuild @celilo/e2e Docker images and standard module .netapps'
     'clear-timing:Clear saved timing history for test ETAs'
     'shell:Shell into a running container (default: management)'
-    'status:Show network status and DNS health'
+    'status:Show run-lock holder + network status (exit 3 if busy)'
+    'release:Free a run-lock left by --keep/up (does not tear down)'
     'load:Load cached Docker images from tarball'
     'scaffold:Generate a new test file from template'
     'version:Show cele2e version'

package/src/cli/index.ts

@@ -23,6 +23,7 @@ import { runBuild } from './build';
 import { runScaffold } from './scaffold';
 import { generateBashCompletion, generateZshCompletion, getCompletions } from './completion';
 import { findAllModules, findModulesWithMatchingTest } from './module-discovery';
+import { acquireRunLock, clearLock, E2eBusyError, formatBusy, lockStatus, markKept } from '../run-lock';
 
 const PKG_DIR = resolve(import.meta.dir, '../..');
 const BIN_DIR = join(PKG_DIR, 'bin');
@@ -127,7 +128,8 @@ Commands:
   clear-timing      Clear saved timing history used for test ETAs
   version           Show cele2e version
   shell [container] Shell into a running container (default: management)
-  status            Show network status and DNS health
+  status            Show run-lock holder + network status (exit 3 if busy)
+  release           Free a run-lock left by --keep/up (does not tear down)
   load              Load cached Docker images from tarball
   scaffold <name>   Generate a new test file from template
   completion <shell> Generate shell completion script (zsh|bash)
@@ -139,6 +141,9 @@ Options for \`run\`:
   --reuse           Reuse existing network if running
   --live            Use live (non-simulated) internet
   --published       Use published .netapp packages
+  --notify          Desktop notification when the run finishes (best-effort)
+  --ci              Plain log output: one ✔/✗ line per step, no spinner
+                    (auto-on when a CI env var is set; --no-ci forces off)
 
 Options for \`up\`:
   --caddy           Start with caddy machine in DMZ
@@ -187,8 +192,17 @@ switch (command) {
     // repo's top-level e2e/tests/ in one invocation; `--all-modules` runs just
     // the module suites. Both resolve the repo root via resolveSuiteRoot() so
     // they work through the `./cele2e` wrapper's cd into packages/e2e.
-    if (args.includes('--all') || args.includes('--all-modules')) {
-      const wantTopLevel = args.includes('--all');
+    if (
+      args.includes('--all') ||
+      args.includes('--all-modules') ||
+      args.includes('--complete') ||
+      args.includes('--ci-safe')
+    ) {
+      // --complete and --ci-safe both run the full discovery (every module +
+      // the repo's top-level e2e/tests/). --ci-safe additionally has the runner
+      // skip tests marked `cele2e-ci-unsafe`. --all-modules stays modules-only.
+      const wantTopLevel =
+        args.includes('--all') || args.includes('--complete') || args.includes('--ci-safe');
       const suiteRoot = resolveSuiteRoot();
       const allModules = findAllModules(suiteRoot);
       const env: Record<string, string> = {
@@ -206,7 +220,9 @@ switch (command) {
       }
       runScript(
         'e2e-run',
-        args.filter((a) => a !== '--all' && a !== '--all-modules'),
+        // Strip the discovery aliases; KEEP --ci-safe so the runner applies the
+        // `cele2e-ci-unsafe` quarantine exclusion.
+        args.filter((a) => a !== '--all' && a !== '--all-modules' && a !== '--complete'),
         env,
       );
     }
@@ -284,16 +300,55 @@ switch (command) {
     process.exit(0);
   }
 
-  case 'up':
+  case 'up': {
+    // `up` leaves an interactive network running, so it holds the run-lock in a
+    // kept state (cleared by `cele2e down`/`release`) to stop a concurrent run
+    // from wiping it.
+    try {
+      acquireRunLock({ test: 'up', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID() });
+      markKept();
+    } catch (err) {
+      if (err instanceof E2eBusyError) {
+        console.error(err.message);
+        process.exit(3);
+      }
+      throw err;
+    }
     runScript('e2e-up', args);
+  }
 
   case 'down':
+    // Tearing the network down frees the (possibly kept) lock.
+    clearLock();
     runScript('e2e-down', args, { E2E_TEST_DIR: stateDir });
 
   case 'shell':
     runScript('e2e-shell', args);
 
+  case 'release': {
+    // Free a lock left by `--keep`/`up` (a `kept` stack) so the next run can
+    // proceed. Does not tear the stack down — use `cele2e down` for that.
+    const had = clearLock();
+    console.log(had ? 'Released the e2e run-lock.' : 'No e2e run-lock held.');
+    process.exit(0);
+  }
+
   case 'status': {
+    // Run-lock is the contention surface other sessions poll. Report it first;
+    // exit 0 = free, 3 = busy. `--json` emits a machine-readable snapshot.
+    const lock = lockStatus();
+    if (args.includes('--json')) {
+      console.log(JSON.stringify(lock));
+      process.exit(lock.free ? 0 : 3);
+    }
+    console.log('=== Run Lock ===');
+    if (lock.free) {
+      console.log('  free');
+    } else if (lock.holder) {
+      console.log(`  BUSY — ${formatBusy(lock.holder)}`);
+    }
+    console.log('');
+
     const persistentFile = join(stateDir, '.e2e-persistent.json');
     let projectName: string | undefined;
     if (existsSync(persistentFile)) {
@@ -319,6 +374,17 @@ switch (command) {
     const save = args.includes('--save');
     const skipModules = args.includes('--skip-modules');
     const moduleDirs = args.filter((a) => !a.startsWith('--'));
+    // build-infra mutates the shared Docker images — serialize against runs.
+    // Released via the lock's exit handler (runBuild may process.exit on failure).
+    try {
+      acquireRunLock({ test: 'build-infra', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID() });
+    } catch (err) {
+      if (err instanceof E2eBusyError) {
+        console.error(err.message);
+        process.exit(3);
+      }
+      throw err;
+    }
     runBuild({ pkgDir: PKG_DIR, moduleDirs, save, skipModules });
     break;
   }

package/src/container-manager.ts

@@ -434,7 +434,17 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   // until its module deploys, so this wait would always time out. Management
   // can still resolve names via the fallback `nameserver 100.100.0.1` in
   // its resolv.conf during deploy.
-  const dnsIntReplaced = config.internalMachines.some((m) => m.name === 'dns-int');
+  //
+  // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
+  // places the dns_internal provider in a PROTECTED zone (dmz) so it can see
+  // protected-zone query sources for split-horizon views. So check every zone's
+  // machines for a `dns-int`, not only internalMachines.
+  const dnsIntReplaced = [
+    ...config.internalMachines,
+    ...config.dmzMachines,
+    ...config.appMachines,
+    ...config.secureMachines,
+  ].some((m) => m.name === 'dns-int');
   if (!dnsIntReplaced) {
     console.log('[progress:start] waiting for internal resolver | internal resolver ready');
     await waitFor(
@@ -682,6 +692,27 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
         gateway: ZONE_GATEWAYS[zone],
       }));
 
+      // fw-main is a firewall container, not a target machine, so the
+      // network readiness wait (target-setup) does NOT cover its sshd. Poll
+      // until the firewall accepts SSH before `machine add` — otherwise a
+      // transient first-connect times out (spawnSync ETIMEDOUT) and aborts the
+      // whole deploy, cascading into unrelated "module not found" failures
+      // (#222). This is a readiness wait on the real prerequisite, not a sleep.
+      await waitFor(
+        async () => {
+          const probe = dockerExec(
+            projectName,
+            composeDir,
+            'management',
+            `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -i /root/.ssh/id_ed25519 root@${firewallIp} hostname`,
+            15_000,
+          );
+          return probe.exitCode === 0;
+        },
+        60_000,
+        `firewall ${firewallIp} sshd`,
+      );
+
       // fw-main is registered as an internal-zone machine; iptables
       // deploys to it and (Phase 2) writes the provided zones to system
       // config from its on_install hook.

package/src/docker-compose-generator.ts

@@ -396,6 +396,17 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
         variant === 'vanilla' ? 'celilo-e2e/management:vanilla' : 'celilo-e2e/management:latest',
       networks: { internal: { ipv4_address: ip } },
       cap_add: ['NET_ADMIN'],
+      // ISS-0157: `module import`'s `bun install` hard-links packages from the
+      // bun cache into node_modules, which on overlayfs forces a copy-up + fsync
+      // per file. On a slow/contended builder disk that storm blows past the
+      // 120s import timeout (intermittently). Put the celilo data dir (the
+      // node_modules install target) on tmpfs (RAM): bun then copies cache→tmpfs
+      // (cross-fs, no overlay copy-up) at RAM speed, immune to host disk. NOT
+      // /root/.bun itself — that holds the baked `celilo` CLI (`bun add -g`).
+      // Tmpfs the bun *cache* subdir too: the copy-up + fsync storm is the
+      // overlay upper layer under /root/.bun/install/cache. RAM-backing it (cold
+      // → re-fetch over the fast sim/net) keeps the CLI intact. Ephemeral e2e.
+      tmpfs: ['/root/.local/share/celilo', '/root/.bun/install/cache'],
       volumes: [
         ...(celiloRoot ? [`${celiloRoot}:/celilo`] : []),
         'ssh-keys:/ssh-keys',

package/src/run-lock.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  acquireRunLock,
+  clearLock,
+  E2eBusyError,
+  type LockHolder,
+  lockStatus,
+  markKept,
+  readHolder,
+  releaseRunLock,
+} from './run-lock';
+
+// Isolate every test on its own lock file (machine-global path is overridable).
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'e2e-lock-'));
+  process.env.CELILO_E2E_LOCK_PATH = join(dir, 'run.lock');
+});
+
+afterEach(() => {
+  clearLock();
+  rmSync(dir, { recursive: true, force: true });
+  delete process.env.CELILO_E2E_LOCK_PATH;
+});
+
+function writeRawHolder(h: Partial<LockHolder>): void {
+  const full: LockHolder = {
+    pid: 123,
+    hostname: hostname(),
+    session: 'other',
+    test: 'foo',
+    runId: 'r',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+    ...h,
+  };
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, JSON.stringify(full));
+}
+
+test('acquire writes a holder for this process; release removes it', () => {
+  acquireRunLock({ test: 'mytest', runId: 'run-1' });
+  const h = readHolder();
+  expect(h?.pid).toBe(process.pid);
+  expect(h?.test).toBe('mytest');
+  expect(lockStatus().free).toBe(false);
+
+  releaseRunLock();
+  expect(readHolder()).toBeNull();
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a live holder makes a second acquire throw E2eBusyError', () => {
+  acquireRunLock({ test: 't1', runId: 'r1' });
+  // The on-disk holder is us (a live pid) → busy.
+  expect(() => acquireRunLock({ test: 't2', runId: 'r2' })).toThrow(E2eBusyError);
+  releaseRunLock();
+});
+
+test('a dead-pid holder on the same host is stale and reclaimed', () => {
+  // 2e9 is far above any real pid → process.kill(pid, 0) throws ESRCH → dead.
+  writeRawHolder({ pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(true); // stale → reported free
+
+  acquireRunLock({ test: 'reclaimer', runId: 'r' }); // reclaims the stale lock
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('--keep leaves a kept lock that survives release and blocks the next run', () => {
+  acquireRunLock({ test: 'kept-test', runId: 'r' });
+  markKept();
+  releaseRunLock();
+
+  const h = readHolder();
+  expect(h?.state).toBe('kept');
+  expect(lockStatus().free).toBe(false); // kept is never stale
+
+  // A plain run is refused...
+  expect(() => acquireRunLock({ test: 'next', runId: 'r2' })).toThrow(E2eBusyError);
+  // ...but a --reuse run (allowKept) takes it over.
+  acquireRunLock({ test: 'reuse', runId: 'r3', allowKept: true });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('clearLock frees a kept lock (the `cele2e release` path)', () => {
+  writeRawHolder({ state: 'kept', pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(false);
+  expect(clearLock()).toBe(true);
+  expect(existsSync(process.env.CELILO_E2E_LOCK_PATH as string)).toBe(false);
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a corrupt lock file is reclaimed, not fatal', () => {
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, 'not json{');
+  expect(readHolder()).toBeNull();
+  acquireRunLock({ test: 'survivor', runId: 'r' });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});

package/src/run-lock.ts

@@ -0,0 +1,252 @@
+/**
+ * cele2e run-lock — cross-platform mutual exclusion for the shared e2e infra.
+ *
+ * cele2e is a single shared resource: the shared infra is one global compose
+ * project (celilo-e2e-shared) and start-of-run cleanup tears down every
+ * celilo-e2e-* container regardless of who started it. So two actors at once
+ * (two Claude sessions, or a session + an operator at the terminal) clobber
+ * each other. This lock serializes runs across processes — no external tools,
+ * identical on macOS and Linux.
+ *
+ * Design (Forgejo #243):
+ *  - Acquire via fs.openSync(path, 'wx') — POSIX-atomic exclusive create,
+ *    same behavior on mac+linux. The lockfile lives at a MACHINE-GLOBAL path
+ *    (~/.cache/celilo-e2e/run.lock), NOT under any repo/worktree, because the
+ *    Docker infra is global regardless of which checkout started it.
+ *  - Fail fast on contention with a message naming the holder. Sessions poll
+ *    `cele2e status` and decide for themselves whether to wait (no --wait).
+ *  - Staleness: on the SAME host, PID-liveness is authoritative (process.kill
+ *    (pid, 0)); a dead holder PID → reclaim. Cross-host we can't check the PID,
+ *    so fall back to a heartbeat TTL (beatAt older than STALE_TTL_MS).
+ *  - A `kept` lock (left by `--keep` / `up`) is NEVER auto-reclaimed — it
+ *    guards a stack that outlives the process and is cleared only by
+ *    `cele2e release` / `cele2e down`. This enforces the honor-system rule.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, openSync, closeSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { homedir, hostname } from 'node:os';
+import { basename, dirname, join } from 'node:path';
+
+const STALE_TTL_MS = 90_000;
+const HEARTBEAT_MS = 30_000;
+
+/**
+ * Machine-global lock path — NOT under any repo/worktree, because the Docker
+ * infra is global regardless of which checkout started it. Overridable via
+ * CELILO_E2E_LOCK_PATH (used by tests; also a handy operator escape hatch).
+ */
+export function lockPath(): string {
+  return process.env.CELILO_E2E_LOCK_PATH || join(homedir(), '.cache', 'celilo-e2e', 'run.lock');
+}
+
+export interface LockHolder {
+  pid: number;
+  hostname: string;
+  session: string;
+  test: string;
+  runId: string;
+  startedAt: string;
+  beatAt: number;
+  state: 'running' | 'kept';
+}
+
+interface Held {
+  keepOnRelease: boolean;
+  heartbeat: ReturnType<typeof setInterval> | null;
+  released: boolean;
+}
+
+/** Set once this process owns the lock; null otherwise. */
+let held: Held | null = null;
+
+export class E2eBusyError extends Error {
+  constructor(public holder: LockHolder) {
+    super(formatBusy(holder));
+    this.name = 'E2eBusyError';
+  }
+}
+
+/** Human label for who's running, e.g. "e2e busy: <session> running <test>, started 3m ago (pid 1234)". */
+export function formatBusy(h: LockHolder): string {
+  const age = ageString(Date.parse(h.startedAt));
+  const verb = h.state === 'kept' ? 'holds a kept stack from' : 'running';
+  const what = h.state === 'kept' ? `(run \`cele2e release\` to free it)` : `${h.test}, started ${age} ago (pid ${h.pid})`;
+  return `e2e busy: ${h.session} ${verb} ${what}`;
+}
+
+function ageString(since: number): string {
+  const s = Math.max(0, Math.floor((Date.now() - since) / 1000));
+  if (s < 60) return `${s}s`;
+  if (s < 3600) return `${Math.floor(s / 60)}m`;
+  return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
+}
+
+/** Branch + worktree path so a holder maps back to a specific session/thread. */
+function deriveSession(): string {
+  if (process.env.CELILO_E2E_SESSION) return process.env.CELILO_E2E_SESSION;
+  const cwd = process.cwd();
+  const git = (args: string[]): string => {
+    try {
+      return execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+    } catch {
+      return '';
+    }
+  };
+  const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']);
+  const top = git(['rev-parse', '--show-toplevel']);
+  const base = top ? basename(top) : basename(cwd);
+  if (branch && top) return `${branch} (${top})`;
+  return branch || base;
+}
+
+function pidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // ESRCH = no such process (dead). EPERM = alive but not ours (still alive).
+    return (err as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+export function readHolder(): LockHolder | null {
+  try {
+    return JSON.parse(readFileSync(lockPath(), 'utf-8')) as LockHolder;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is the on-disk holder stale (safe to reclaim)? A `kept` lock is never stale —
+ * it deliberately outlives its process. On the same host the PID is the source
+ * of truth; cross-host we can only use the heartbeat TTL.
+ */
+function isStale(h: LockHolder): boolean {
+  if (h.state === 'kept') return false;
+  if (h.hostname === hostname()) return !pidAlive(h.pid);
+  return Date.now() - h.beatAt > STALE_TTL_MS;
+}
+
+function writeHolder(fd: number, h: LockHolder): void {
+  writeFileSync(fd, JSON.stringify(h, null, 2));
+}
+
+/**
+ * Acquire the run lock for this process. Throws E2eBusyError if another live
+ * (non-stale) holder owns it. Registers an exit handler so the lock is released
+ * on any process.exit() path (normal, exception, or a SIGINT handler that
+ * exits). A signal-killed process with no exit handler leaks the lock, but the
+ * next run reclaims it via PID-liveness — that's exactly what staleness is for.
+ */
+export function acquireRunLock(opts: { test: string; runId: string; allowKept?: boolean }): void {
+  mkdirSync(dirname(lockPath()), { recursive: true });
+
+  const holder: LockHolder = {
+    pid: process.pid,
+    hostname: hostname(),
+    session: deriveSession(),
+    test: opts.test,
+    runId: opts.runId,
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let fd: number;
+    try {
+      fd = openSync(lockPath(), 'wx');
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
+      const existing = readHolder();
+      // Reclaim when: corrupt/unreadable, the holder is stale, OR this is a
+      // `--reuse` run taking over a `kept` stack it's deliberately reusing.
+      if (!existing || isStale(existing) || (opts.allowKept && existing.state === 'kept')) {
+        try { unlinkSync(lockPath()); } catch {}
+        continue;
+      }
+      throw new E2eBusyError(existing);
+    }
+    writeHolder(fd, holder);
+    closeSync(fd);
+
+    const heartbeat = setInterval(() => {
+      // Only refresh while running; a kept lock's beatAt is frozen by design.
+      if (!held || held.released || held.keepOnRelease) return;
+      try {
+        const cur = readHolder();
+        if (cur && cur.pid === process.pid) {
+          writeFileSync(lockPath(), JSON.stringify({ ...cur, beatAt: Date.now() }, null, 2));
+        }
+      } catch {}
+    }, HEARTBEAT_MS);
+    heartbeat.unref();
+
+    held = { keepOnRelease: false, heartbeat, released: false };
+    process.on('exit', releaseRunLock);
+    return;
+  }
+  // Two reclaim attempts both lost the race → someone else won fair and square.
+  const existing = readHolder();
+  throw new E2eBusyError(existing ?? holderUnknown());
+}
+
+function holderUnknown(): LockHolder {
+  return {
+    pid: 0,
+    hostname: hostname(),
+    session: 'unknown',
+    test: '?',
+    runId: '?',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+}
+
+/**
+ * Mark the lock to survive process exit in a `kept` state — call when a stack
+ * (network/containers) will outlive this process (`--keep`, `up`). The next run
+ * then refuses to clobber it until `cele2e release` / `cele2e down`.
+ */
+export function markKept(): void {
+  if (held) held.keepOnRelease = true;
+}
+
+/** Release the lock. Idempotent. Honors markKept() (kept-state) vs delete. */
+export function releaseRunLock(): void {
+  if (!held || held.released) return;
+  held.released = true;
+  if (held.heartbeat) clearInterval(held.heartbeat);
+
+  if (held.keepOnRelease) {
+    const cur = readHolder();
+    if (cur && cur.pid === process.pid) {
+      try { writeFileSync(lockPath(), JSON.stringify({ ...cur, state: 'kept', beatAt: Date.now() }, null, 2)); } catch {}
+    }
+    return;
+  }
+  try {
+    const cur = readHolder();
+    if (!cur || cur.pid === process.pid) unlinkSync(lockPath());
+  } catch {}
+}
+
+/** Forcibly clear the lock regardless of holder — used by `cele2e release`/`down`. */
+export function clearLock(): boolean {
+  try {
+    unlinkSync(lockPath());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Current lock state for `cele2e status` (and any poller). */
+export function lockStatus(): { free: boolean; holder: LockHolder | null } {
+  const h = readHolder();
+  if (!h || isStale(h)) return { free: true, holder: null };
+  return { free: false, holder: h };
+}

package/src/runner.ts

@@ -19,9 +19,9 @@
 
 import { spawn, spawnSync, execSync } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { basename, join, resolve } from 'node:path';
+import { basename, dirname, join, resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
-import { ProgressDisplay } from '@celilo/cli-display';
+import { type DisplayMode, ProgressDisplay } from '@celilo/cli-display';
 import {
   emitRunCompleted,
   emitRunFailed,
@@ -30,6 +30,7 @@ import {
   emitTestStarted,
 } from './bus-events';
 import { parseLine } from './parse-line';
+import { acquireRunLock, E2eBusyError, markKept } from './run-lock';
 
 // ─── Config ──────────────────────────────────────────────────────────
 
@@ -70,7 +71,43 @@ const flagReuse = rawArgs.includes('--reuse');
 const flagLive = rawArgs.includes('--live');
 const flagPublished = rawArgs.includes('--published');
 const flagVerbose = rawArgs.includes('--verbose');
+// CI mode: no animated spinner, no [progress:*] markers — just one clean
+// ✔/✗ line per step (plus sub-events) written straight to the log. Forgejo's
+// runner allocates a PTY, so process.stdout.isTTY is truthy there and the
+// interactive footer (redrawn via `\r` on every spinner tick) gets linearized
+// into one log line per tick — dozens of identical braille lines per step.
+// `--ci` (or any standard `CI` env var, e.g. Forgejo/GitHub Actions) forces
+// the non-interactive render path that sidesteps that entirely. `--no-ci`
+// opts back out when a CI env var is set but an operator wants the spinner.
+const flagCi = rawArgs.includes('--ci');
+const flagNoCi = rawArgs.includes('--no-ci');
+const ci = flagCi || (!flagNoCi && !!process.env.CI);
 const patterns = rawArgs.filter((a) => !a.startsWith('--'));
+// --ci-safe: skip tests marked `cele2e-ci-unsafe` in their file (quarantined —
+// known-failing or builder-flaky, each tracked by an ISS). `--complete`/`--all`
+// run everything. Passed through from the CLI (not stripped) so the runner can
+// apply the exclusion.
+const flagCiSafe = rawArgs.includes('--ci-safe');
+// --notify: fire a desktop notification when the run finishes (handy for long
+// detached/background runs). Best-effort — never affects the exit code.
+const flagNotify = rawArgs.includes('--notify');
+
+function notify(title: string, message: string): void {
+  if (!flagNotify) return;
+  try {
+    if (process.platform === 'darwin') {
+      const r = spawnSync('terminal-notifier', ['-title', title, '-message', message, '-sound', 'Glass'], { stdio: 'ignore' });
+      if (r.error) {
+        spawnSync('osascript', ['-e', `display notification ${JSON.stringify(message)} with title ${JSON.stringify(title)} sound name "Glass"`], { stdio: 'ignore' });
+      }
+      spawnSync('say', [message], { stdio: 'ignore' });
+    } else if (process.platform === 'linux') {
+      spawnSync('notify-send', [title, message], { stdio: 'ignore' });
+    }
+  } catch {
+    // ponytail: notification is a nicety; its failure must never break a run.
+  }
+}
 
 // ─── Colors ──────────────────────────────────────────────────────────
 
@@ -78,10 +115,10 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
-const cyan = '\x1b[36m';
 const reset = '\x1b[0m';
 
-const interactive = process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose;
+const interactive =
+  process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose && !ci;
 
 // ─── Timing ──────────────────────────────────────────────────────────
 
@@ -369,14 +406,6 @@ function dockerComposeShared(cmd: string): string {
   }
 }
 
-function isSharedInfraHealthy(): boolean {
-  const containers = dockerComposeShared('ps -q');
-  if (!containers) return false;
-  return dockerComposeShared(
-    'exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2',
-  ).length > 0;
-}
-
 function stopSharedInfra(): void {
   dockerComposeShared('down --volumes --remove-orphans');
   try { execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 }); } catch {}
@@ -400,6 +429,24 @@ function assertDockerAvailable(): void {
 async function main() {
   assertDockerAvailable();
 
+  // Serialize against other cele2e runs sharing the global Docker infra. Fail
+  // fast naming the holder; pollers use `cele2e status`. Acquire BEFORE any
+  // docker mutation so a competing run can't wipe our shared infra mid-setup.
+  const lockLabel = patterns.length ? patterns.join(',') : moduleDirs.length ? 'modules' : 'all';
+  try {
+    acquireRunLock({ test: lockLabel, runId: process.env.CELE2E_RUN_ID ?? ambientRunId, allowKept: flagReuse });
+    // A --reuse run keeps using a kept stack that stays up afterwards — hold the
+    // lock in kept state so it isn't freed out from under the reused network.
+    if (flagReuse) markKept();
+  } catch (err) {
+    if (err instanceof E2eBusyError) {
+      console.error(`\n${red}${err.message}${reset}`);
+      console.error(`${dim}Poll with: cele2e status   (machine-readable: cele2e status --json)${reset}\n`);
+      process.exit(3);
+    }
+    throw err;
+  }
+
   let testFiles: string[] = [];
 
   if (moduleDirs.length > 0) {
@@ -454,12 +501,45 @@ async function main() {
       .filter((f) => f.endsWith('.test.ts'))
       .sort()
       .map((f) => resolve(testsPath, f));
+    // Docker-backed integration tests live in a sibling `tests-integration/`
+    // so they stay OUT of the hermetic `bun test tests/` gate. cele2e is the
+    // heavy runner (Docker + build-infra present), so it covers BOTH tiers —
+    // this is what keeps the documented `cele2e run ansible-output` working
+    // after the test moved out of tests/.
+    const integrationPath = join(dirname(testsPath), 'tests-integration');
+    if (existsSync(integrationPath)) {
+      testFiles.push(
+        ...readdirSync(integrationPath)
+          .filter((f) => f.endsWith('.test.ts'))
+          .sort()
+          .map((f) => resolve(integrationPath, f)),
+      );
+    }
   }
 
   if (patterns.length > 0) {
     testFiles = testFiles.filter((f) => patterns.some((p) => basename(f, '.test.ts').includes(p)));
   }
 
+  // --ci-safe: drop quarantined tests — those whose file contains the marker
+  // `cele2e-ci-unsafe` (a comment naming the reason + ISS). Keeps the nightly
+  // green on the CI-safe subset while the quarantined tests are tracked + fixed
+  // separately. `--complete`/`--all` run everything (no exclusion).
+  if (flagCiSafe) {
+    const before = testFiles.length;
+    testFiles = testFiles.filter((f) => {
+      try {
+        return !readFileSync(f, 'utf-8').includes('cele2e-ci-unsafe');
+      } catch {
+        return true;
+      }
+    });
+    const skipped = before - testFiles.length;
+    if (skipped > 0) {
+      console.log(`${dim}--ci-safe: skipped ${skipped} quarantined (cele2e-ci-unsafe) test(s)${reset}`);
+    }
+  }
+
   if (testFiles.length === 0) {
     console.error(`${red}No test files matched${reset}`);
     process.exit(1);
@@ -507,9 +587,8 @@ async function main() {
   mkdirSync(resultsDir, { recursive: true });
 
   // runId scopes every event in this suite invocation. Read from env to
-  // let callers (CI, the Claude tool wrapper) inject a known id and
-  // subscribe to it before launching cele2e; otherwise generate one and
-  // print it so the operator can correlate.
+  // let callers (CI) inject a known id and subscribe to it before launching
+  // cele2e; otherwise generate one and print it so the operator can correlate.
   const runId = process.env.CELE2E_RUN_ID || crypto.randomUUID();
   if (!process.env.CELE2E_RUN_ID) {
     console.log(`${dim}runId: ${runId}${reset}`);
@@ -524,14 +603,21 @@ async function main() {
 
   // 'auto' resolves to 'render' when isTTY is true (operator at a
   // terminal sees animated spinners) and 'protocol' when isTTY is
-  // false (CI / cele2e-run.sh wrapper / any redirected stdout gets
-  // ASCII `[progress:*]` markers — grep-friendly, no braille noise).
+  // false (a backgrounded run / any redirected stdout gets ASCII
+  // `[progress:*]` markers — grep-friendly, no braille noise).
   // Prior to this change the mode was hardcoded 'render', so non-TTY
   // callers got `⣾ phase started` / `✔ phase done` lines per phase
   // (e.g. ~29 braille codepoints in a typical test log) — cosmetic
   // but actively unhelpful for log scanning and tooling.
+  //
+  // CI mode forces 'render' with isTTY:false (non-interactive render):
+  // one ✔/✗ line per step, sub-events emitted inline, NO spinner and
+  // NO `[progress:*]` markers. We can't rely on 'auto' here because
+  // Forgejo's PTY makes isTTY truthy — which would pick the animated
+  // footer and spam the log with one braille line per spinner tick.
+  const displayMode: DisplayMode = ci ? 'render' : 'auto';
   const display = new ProgressDisplay({
-    mode: 'auto',
+    mode: displayMode,
     out: { write: process.stdout.write.bind(process.stdout), isTTY: interactive },
   });
   let passed = 0;
@@ -606,6 +692,9 @@ async function main() {
       console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
       console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
       console.log(`${dim}  Tear down: cele2e down${reset}`);
+      // The kept network outlives this process — hold the lock in a `kept`
+      // state so the next run refuses to clobber it (cleared by `cele2e down`).
+      markKept();
     }
 
     if (result.status === 'pass') {
@@ -688,6 +777,7 @@ async function main() {
     resultsDir,
   });
 
+  notify('cele2e', `${passed} passed, ${failed} failed — ${formatDuration(suiteDuration)}`);
   process.exit(failed > 0 ? 1 : 0);
 }
 
@@ -705,5 +795,6 @@ main().catch((err) => {
     error: err instanceof Error ? err.message : String(err),
   });
   console.error(err);
+  notify('cele2e', 'run failed');
   process.exit(1);
 });

package/src/shared-infra.ts

@@ -30,6 +30,34 @@ function run(cmd: string, opts?: { cwd?: string; timeout?: number }): string {
 
 let sharedInfraRunning = false;
 
+/**
+ * Remove ALL celilo-e2e-* Docker resources by name prefix (#212). Used at
+ * start-of-run, where the run-lock guarantees no other session's stack is live.
+ * Graceful compose-down of the shared project first (clean network detach),
+ * then a force sweep that catches orphans from any prior invocation regardless
+ * of compose-project name or the path that created them.
+ */
+function nukeE2eResources(e2eDir: string): void {
+  try {
+    run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`,
+      { cwd: e2eDir, timeout: 30_000 });
+  } catch {}
+  try {
+    const ids = run('docker ps -aq --filter name=celilo-e2e', { timeout: 10_000 });
+    if (ids.trim()) run(`docker rm -f ${ids.replace(/\n/g, ' ')}`, { timeout: 60_000 });
+  } catch {}
+  try {
+    const nets = run('docker network ls --format "{{.Name}}"')
+      .split('\n')
+      .filter((n) => n.startsWith('celilo-e2e'));
+    for (const net of nets) {
+      try { run(`docker network rm ${net}`, { timeout: 5_000 }); } catch {}
+    }
+  } catch {}
+  try { run('docker network prune -f', { timeout: 10_000 }); } catch {}
+  try { run('docker volume prune -f', { timeout: 10_000 }); } catch {}
+}
+
 /**
  * Check if the shared infrastructure is already running and healthy.
  *
@@ -104,38 +132,14 @@ export async function ensureSharedInfra(): Promise<void> {
 
   const e2eDir = getE2eDir();
 
-  // Clean up any stale shared infra resources
+  // Start-of-run cleanup. We hold the run-lock here (acquired in the runner /
+  // build path before any docker mutation), so NO other session's stack is
+  // live — it is safe to remove EVERY celilo-e2e-* resource by name prefix.
+  // This self-heals orphans left by a crashed / cross-path / cross-checkout
+  // prior run (e.g. a botched-mount container that wedges the next `up`),
+  // which the old compose-project-scoped `down` couldn't see (#212).
   console.log('[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete');
-  try {
-    run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`,
-      { cwd: e2eDir, timeout: 30_000 });
-  } catch {}
-
-  // Also clean up any per-test projects that reference our networks
-  try {
-    const networks = run('docker network ls --format "{{.Name}}"')
-      .split('\n')
-      .filter(n => n.startsWith('celilo-e2e-') && !n.startsWith(`${SHARED_PROJECT_NAME}_`));
-    if (networks.length > 0) {
-      const projects = [...new Set(networks.map(n => n.replace(/_[^_]+$/, '')))];
-      for (const project of projects) {
-        try { run(`docker compose -p ${project} down --volumes --remove-orphans`, { timeout: 30_000 }); } catch {}
-      }
-    }
-  } catch {}
-
-  try { run('docker network prune -f', { timeout: 10_000 }); } catch {}
-  try { run('docker volume prune -f', { timeout: 10_000 }); } catch {}
-
-  // Force-remove any leftover networks by name
-  try {
-    const staleNets = run('docker network ls --format "{{.Name}}"')
-      .split('\n')
-      .filter(n => n.startsWith('celilo-e2e'));
-    for (const net of staleNets) {
-      try { run(`docker network rm ${net}`, { timeout: 5_000 }); } catch {}
-    }
-  } catch {}
+  nukeE2eResources(e2eDir);
 
   // Refresh the bundled registry-server source so Dockerfile.registry's
   // COPY resolves whether we're in the monorepo (regenerated from the
@@ -172,11 +176,16 @@ export async function ensureSharedInfra(): Promise<void> {
   run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`,
     { cwd: e2eDir, timeout: 120_000 });
 
-  // Wait for DNS convergence
+  // Wait for DNS convergence — robustly. The old single 60s window with no
+  // recovery was an intermittent build-infra failure on the builder (#319:
+  // namecheap-dns hadn't served its zone within 60s, and the bare throw gave
+  // no clue why). Now: a longer window, RESTART the DNS container if it's
+  // stuck (re-triggers the zone load), and DUMP diagnostics before giving up.
   console.log('[progress:start] waiting for DNS convergence | DNS converged');
+  const dnsTimeout = 180_000;
   const start = Date.now();
-  const timeout = 60_000;
-  while (Date.now() - start < timeout) {
+  let lastRestart = start;
+  while (Date.now() - start < dnsTimeout) {
     try {
       const result = run(
         `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2`,
@@ -190,9 +199,34 @@ export async function ensureSharedInfra(): Promise<void> {
     } catch {
       // retry
     }
+    // Recovery: if ~45s have passed since the last (re)start without
+    // converging, the DNS container likely didn't load its zone — restart
+    // it to re-trigger the load instead of waiting out the whole window.
+    if (Date.now() - lastRestart > 45_000) {
+      console.log('[progress:sub] DNS not converged yet — restarting namecheap-dns');
+      try {
+        run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} restart namecheap-dns`,
+          { cwd: e2eDir, timeout: 30_000 });
+      } catch {}
+      lastRestart = Date.now();
+    }
     await new Promise(r => setTimeout(r, 2000));
   }
-  throw new Error('Timeout waiting for shared infrastructure DNS convergence');
+  // Diagnostics before failing, so the log shows WHY it didn't converge.
+  let psOut = '';
+  try {
+    psOut = run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} ps`,
+      { cwd: e2eDir, timeout: 10_000 });
+  } catch {}
+  let dnsLog = '';
+  try {
+    dnsLog = run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} logs --tail 40 namecheap-dns`,
+      { cwd: e2eDir, timeout: 10_000 });
+  } catch {}
+  throw new Error(
+    `Timeout waiting for shared infrastructure DNS convergence (${dnsTimeout / 1000}s)\n` +
+      `--- compose ps ---\n${psOut}\n--- namecheap-dns logs (tail 40) ---\n${dnsLog}`,
+  );
 }
 
 /**