@celilo/e2e 0.7.17 → 0.8.0

package/config/dns/example.net.zone

@@ -0,0 +1,16 @@
+$ORIGIN example.net.
+$TTL 300
+
+@ IN SOA ns1.example.net. admin.example.net. (
+    2024020108 300 60 604800 300
+)
+
+; Public DNS — RFC 1918 addresses NEVER appear here. The whole point of
+; the e2e simulation is to model real internet connectivity, where
+; private IPs are unreachable from outside. All public-facing names
+; resolve to the firewall's external IP (100.100.0.100), which DNATs
+; inbound to the right internal host.
+@       IN NS   ns1.example.net.
+ns1     IN A    100.64.0.55
+@       IN A    100.100.0.100
+www     IN A    100.100.0.100

package/config/routing/fw-main-routes.sh

@@ -8,22 +8,39 @@ chmod 700 /root/.ssh
 chmod 600 /root/.ssh/authorized_keys
 /usr/sbin/sshd
 
-# Routing — detect if we have an isp-external interface (direct-internet topology)
+# IP forwarding is set via sysctls in docker-compose
+iptables -P FORWARD ACCEPT
+
+# Routing + egress NAT — depends on topology (detected by the presence of an
+# isp-external interface).
 ip route del default 2>/dev/null || true
 if ip -o addr show | grep -q '100.100.0'; then
-  # Direct internet: route via fw-ext on isp-external
+  # Direct-internet: fw-main has its own external (WAN) interface.
   ip route add default via 100.100.0.101
+  # Egress NAT SCOPED to the external/WAN interface (the one with the public
+  # 100.100.0.x address) — NOT a blanket MASQUERADE (ISS-0156). Scoping to the WAN
+  # means inter-zone traffic (protected↔protected, protected↔internal) keeps its
+  # source, so the dmz-resident resolver sees each client's real zone for
+  # source-based DNS views. This boot-time rule is what target machines need for
+  # egress/DNS BEFORE any module deploys; celilo's iptables module applies the
+  # same WAN-scoped MASQUERADE on deploy (idempotent — the production code path).
+  EXT_IFACE=$(ip -o addr show | awk '/100\.100\.0\./{print $2; exit}')
+  iptables -t nat -A POSTROUTING -o "$EXT_IFACE" -j MASQUERADE
 else
-  # Default: route via fw-isp on internal
+  # Two-layer: no external interface on fw-main; it routes outbound to the
+  # upstream firewall (fw-isp) and MASQUERADEs toward it (egress leaves via the
+  # internal-facing interface, so there is no single WAN interface to scope to).
   ip route add default via 192.168.0.1
+  # Protected↔protected (dmz/app/secure ↔ each other, all in 10/8) is NOT NAT'd —
+  # a real firewall routes between its segmented zones without NAT, preserving the
+  # client source so the dmz-resident resolver can serve source-based split-horizon
+  # (ISS-0156). Only protected↔internal and egress get MASQUERADE'd below. Without
+  # this RETURN, app→dmz and internal→dmz would both be SNAT'd to the dmz gateway
+  # and become indistinguishable (confirmed via live probe).
+  iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -d 10.0.0.0/8 -j RETURN
+  iptables -t nat -A POSTROUTING -j MASQUERADE
 fi
 
-# IP forwarding is set via sysctls in docker-compose
-
-# NAT all outbound traffic from dmz/app/secure
-iptables -t nat -A POSTROUTING -j MASQUERADE
-iptables -P FORWARD ACCEPT
-
 # DNS
 echo "nameserver 100.100.0.1" > /etc/resolv.conf
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.7.17",
+  "version": "0.8.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -34,9 +34,7 @@
     "npm-registry-server/tsconfig.json",
     "npm-registry-server/src/",
     "scripts/",
-    "README.md",
-    "AGENTS.md",
-    "COVERAGE.md"
+    "README.md"
   ],
   "dependencies": {
     "@celilo/cli-display": "^0.1.9",

package/src/cli/command-registry.ts

@@ -33,6 +33,7 @@ export const COMMANDS: CommandDef[] = [
       { name: '--reuse', description: 'Reuse existing network if running' },
       { name: '--live', description: 'Use live (non-simulated) internet' },
       { name: '--published', description: 'Use published .netapp packages' },
+      { name: '--notify', description: 'Desktop notification when the run finishes' },
     ],
   },
   {
@@ -75,7 +76,12 @@ export const COMMANDS: CommandDef[] = [
   },
   {
     name: 'status',
-    description: 'Show network status and DNS health',
+    description: 'Show run-lock holder + network status (exit 3 if busy)',
+    flags: [{ name: '--json', description: 'Emit a machine-readable lock snapshot for polling' }],
+  },
+  {
+    name: 'release',
+    description: 'Free a run-lock left by --keep/up (does not tear down)',
   },
   {
     name: 'load',
@@ -101,7 +107,7 @@ export const COMMANDS: CommandDef[] = [
 
 export const UP_PRESETS = ['--caddy', '--full-stack', '--infrastructure'];
 
-export const RUN_FLAGS = ['--all', '--complete', '--ci-safe', '--all-modules', '--keep', '--reuse', '--live', '--published'];
+export const RUN_FLAGS = ['--all', '--complete', '--ci-safe', '--all-modules', '--keep', '--reuse', '--live', '--published', '--notify'];
 
 export const DOWN_FLAGS = ['--keep', '--all'];
 

package/src/cli/completion.ts

@@ -119,6 +119,7 @@ _cele2e() {
             '--reuse[Reuse existing network if running]' \\
             '--live[Use live (non-simulated) internet]' \\
             '--published[Use published .netapp packages]' \\
+            '--notify[Desktop notification when the run finishes]' \\
             '--ci[Plain log output: one line per step, no spinner]' \\
             '--no-ci[Force spinner even when a CI env var is set]' \\
             '*::test name or module path:_cele2e_run_args'
@@ -148,7 +149,10 @@ _cele2e() {
         completion)
           _arguments '1: :_cele2e_completion_shells'
           ;;
-        status|load|clear-timing|list)
+        status)
+          _arguments '--json[Emit a machine-readable lock snapshot for polling]'
+          ;;
+        release|load|clear-timing)
           ;;
       esac
       ;;
@@ -164,7 +168,8 @@ _cele2e_commands() {
     'build-infra:Rebuild @celilo/e2e Docker images and standard module .netapps'
     'clear-timing:Clear saved timing history for test ETAs'
     'shell:Shell into a running container (default: management)'
-    'status:Show network status and DNS health'
+    'status:Show run-lock holder + network status (exit 3 if busy)'
+    'release:Free a run-lock left by --keep/up (does not tear down)'
     'load:Load cached Docker images from tarball'
     'scaffold:Generate a new test file from template'
     'version:Show cele2e version'

package/src/cli/index.ts

@@ -23,6 +23,7 @@ import { runBuild } from './build';
 import { runScaffold } from './scaffold';
 import { generateBashCompletion, generateZshCompletion, getCompletions } from './completion';
 import { findAllModules, findModulesWithMatchingTest } from './module-discovery';
+import { acquireRunLock, clearLock, E2eBusyError, formatBusy, lockStatus, markKept } from '../run-lock';
 
 const PKG_DIR = resolve(import.meta.dir, '../..');
 const BIN_DIR = join(PKG_DIR, 'bin');
@@ -127,7 +128,8 @@ Commands:
   clear-timing      Clear saved timing history used for test ETAs
   version           Show cele2e version
   shell [container] Shell into a running container (default: management)
-  status            Show network status and DNS health
+  status            Show run-lock holder + network status (exit 3 if busy)
+  release           Free a run-lock left by --keep/up (does not tear down)
   load              Load cached Docker images from tarball
   scaffold <name>   Generate a new test file from template
   completion <shell> Generate shell completion script (zsh|bash)
@@ -139,6 +141,7 @@ Options for \`run\`:
   --reuse           Reuse existing network if running
   --live            Use live (non-simulated) internet
   --published       Use published .netapp packages
+  --notify          Desktop notification when the run finishes (best-effort)
   --ci              Plain log output: one ✔/✗ line per step, no spinner
                     (auto-on when a CI env var is set; --no-ci forces off)
 
@@ -297,16 +300,55 @@ switch (command) {
     process.exit(0);
   }
 
-  case 'up':
+  case 'up': {
+    // `up` leaves an interactive network running, so it holds the run-lock in a
+    // kept state (cleared by `cele2e down`/`release`) to stop a concurrent run
+    // from wiping it.
+    try {
+      acquireRunLock({ test: 'up', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID() });
+      markKept();
+    } catch (err) {
+      if (err instanceof E2eBusyError) {
+        console.error(err.message);
+        process.exit(3);
+      }
+      throw err;
+    }
     runScript('e2e-up', args);
+  }
 
   case 'down':
+    // Tearing the network down frees the (possibly kept) lock.
+    clearLock();
     runScript('e2e-down', args, { E2E_TEST_DIR: stateDir });
 
   case 'shell':
     runScript('e2e-shell', args);
 
+  case 'release': {
+    // Free a lock left by `--keep`/`up` (a `kept` stack) so the next run can
+    // proceed. Does not tear the stack down — use `cele2e down` for that.
+    const had = clearLock();
+    console.log(had ? 'Released the e2e run-lock.' : 'No e2e run-lock held.');
+    process.exit(0);
+  }
+
   case 'status': {
+    // Run-lock is the contention surface other sessions poll. Report it first;
+    // exit 0 = free, 3 = busy. `--json` emits a machine-readable snapshot.
+    const lock = lockStatus();
+    if (args.includes('--json')) {
+      console.log(JSON.stringify(lock));
+      process.exit(lock.free ? 0 : 3);
+    }
+    console.log('=== Run Lock ===');
+    if (lock.free) {
+      console.log('  free');
+    } else if (lock.holder) {
+      console.log(`  BUSY — ${formatBusy(lock.holder)}`);
+    }
+    console.log('');
+
     const persistentFile = join(stateDir, '.e2e-persistent.json');
     let projectName: string | undefined;
     if (existsSync(persistentFile)) {
@@ -332,6 +374,17 @@ switch (command) {
     const save = args.includes('--save');
     const skipModules = args.includes('--skip-modules');
     const moduleDirs = args.filter((a) => !a.startsWith('--'));
+    // build-infra mutates the shared Docker images — serialize against runs.
+    // Released via the lock's exit handler (runBuild may process.exit on failure).
+    try {
+      acquireRunLock({ test: 'build-infra', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID() });
+    } catch (err) {
+      if (err instanceof E2eBusyError) {
+        console.error(err.message);
+        process.exit(3);
+      }
+      throw err;
+    }
     runBuild({ pkgDir: PKG_DIR, moduleDirs, save, skipModules });
     break;
   }

package/src/container-manager.ts

@@ -434,7 +434,17 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   // until its module deploys, so this wait would always time out. Management
   // can still resolve names via the fallback `nameserver 100.100.0.1` in
   // its resolv.conf during deploy.
-  const dnsIntReplaced = config.internalMachines.some((m) => m.name === 'dns-int');
+  //
+  // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
+  // places the dns_internal provider in a PROTECTED zone (dmz) so it can see
+  // protected-zone query sources for split-horizon views. So check every zone's
+  // machines for a `dns-int`, not only internalMachines.
+  const dnsIntReplaced = [
+    ...config.internalMachines,
+    ...config.dmzMachines,
+    ...config.appMachines,
+    ...config.secureMachines,
+  ].some((m) => m.name === 'dns-int');
   if (!dnsIntReplaced) {
     console.log('[progress:start] waiting for internal resolver | internal resolver ready');
     await waitFor(
@@ -682,6 +692,27 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
         gateway: ZONE_GATEWAYS[zone],
       }));
 
+      // fw-main is a firewall container, not a target machine, so the
+      // network readiness wait (target-setup) does NOT cover its sshd. Poll
+      // until the firewall accepts SSH before `machine add` — otherwise a
+      // transient first-connect times out (spawnSync ETIMEDOUT) and aborts the
+      // whole deploy, cascading into unrelated "module not found" failures
+      // (#222). This is a readiness wait on the real prerequisite, not a sleep.
+      await waitFor(
+        async () => {
+          const probe = dockerExec(
+            projectName,
+            composeDir,
+            'management',
+            `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -i /root/.ssh/id_ed25519 root@${firewallIp} hostname`,
+            15_000,
+          );
+          return probe.exitCode === 0;
+        },
+        60_000,
+        `firewall ${firewallIp} sshd`,
+      );
+
       // fw-main is registered as an internal-zone machine; iptables
       // deploys to it and (Phase 2) writes the provided zones to system
       // config from its on_install hook.

package/src/run-lock.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  acquireRunLock,
+  clearLock,
+  E2eBusyError,
+  type LockHolder,
+  lockStatus,
+  markKept,
+  readHolder,
+  releaseRunLock,
+} from './run-lock';
+
+// Isolate every test on its own lock file (machine-global path is overridable).
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'e2e-lock-'));
+  process.env.CELILO_E2E_LOCK_PATH = join(dir, 'run.lock');
+});
+
+afterEach(() => {
+  clearLock();
+  rmSync(dir, { recursive: true, force: true });
+  delete process.env.CELILO_E2E_LOCK_PATH;
+});
+
+function writeRawHolder(h: Partial<LockHolder>): void {
+  const full: LockHolder = {
+    pid: 123,
+    hostname: hostname(),
+    session: 'other',
+    test: 'foo',
+    runId: 'r',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+    ...h,
+  };
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, JSON.stringify(full));
+}
+
+test('acquire writes a holder for this process; release removes it', () => {
+  acquireRunLock({ test: 'mytest', runId: 'run-1' });
+  const h = readHolder();
+  expect(h?.pid).toBe(process.pid);
+  expect(h?.test).toBe('mytest');
+  expect(lockStatus().free).toBe(false);
+
+  releaseRunLock();
+  expect(readHolder()).toBeNull();
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a live holder makes a second acquire throw E2eBusyError', () => {
+  acquireRunLock({ test: 't1', runId: 'r1' });
+  // The on-disk holder is us (a live pid) → busy.
+  expect(() => acquireRunLock({ test: 't2', runId: 'r2' })).toThrow(E2eBusyError);
+  releaseRunLock();
+});
+
+test('a dead-pid holder on the same host is stale and reclaimed', () => {
+  // 2e9 is far above any real pid → process.kill(pid, 0) throws ESRCH → dead.
+  writeRawHolder({ pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(true); // stale → reported free
+
+  acquireRunLock({ test: 'reclaimer', runId: 'r' }); // reclaims the stale lock
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('--keep leaves a kept lock that survives release and blocks the next run', () => {
+  acquireRunLock({ test: 'kept-test', runId: 'r' });
+  markKept();
+  releaseRunLock();
+
+  const h = readHolder();
+  expect(h?.state).toBe('kept');
+  expect(lockStatus().free).toBe(false); // kept is never stale
+
+  // A plain run is refused...
+  expect(() => acquireRunLock({ test: 'next', runId: 'r2' })).toThrow(E2eBusyError);
+  // ...but a --reuse run (allowKept) takes it over.
+  acquireRunLock({ test: 'reuse', runId: 'r3', allowKept: true });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});
+
+test('clearLock frees a kept lock (the `cele2e release` path)', () => {
+  writeRawHolder({ state: 'kept', pid: 2_000_000_000 });
+  expect(lockStatus().free).toBe(false);
+  expect(clearLock()).toBe(true);
+  expect(existsSync(process.env.CELILO_E2E_LOCK_PATH as string)).toBe(false);
+  expect(lockStatus().free).toBe(true);
+});
+
+test('a corrupt lock file is reclaimed, not fatal', () => {
+  writeFileSync(process.env.CELILO_E2E_LOCK_PATH as string, 'not json{');
+  expect(readHolder()).toBeNull();
+  acquireRunLock({ test: 'survivor', runId: 'r' });
+  expect(readHolder()?.pid).toBe(process.pid);
+  releaseRunLock();
+});

package/src/run-lock.ts

@@ -0,0 +1,252 @@
+/**
+ * cele2e run-lock — cross-platform mutual exclusion for the shared e2e infra.
+ *
+ * cele2e is a single shared resource: the shared infra is one global compose
+ * project (celilo-e2e-shared) and start-of-run cleanup tears down every
+ * celilo-e2e-* container regardless of who started it. So two actors at once
+ * (two Claude sessions, or a session + an operator at the terminal) clobber
+ * each other. This lock serializes runs across processes — no external tools,
+ * identical on macOS and Linux.
+ *
+ * Design (Forgejo #243):
+ *  - Acquire via fs.openSync(path, 'wx') — POSIX-atomic exclusive create,
+ *    same behavior on mac+linux. The lockfile lives at a MACHINE-GLOBAL path
+ *    (~/.cache/celilo-e2e/run.lock), NOT under any repo/worktree, because the
+ *    Docker infra is global regardless of which checkout started it.
+ *  - Fail fast on contention with a message naming the holder. Sessions poll
+ *    `cele2e status` and decide for themselves whether to wait (no --wait).
+ *  - Staleness: on the SAME host, PID-liveness is authoritative (process.kill
+ *    (pid, 0)); a dead holder PID → reclaim. Cross-host we can't check the PID,
+ *    so fall back to a heartbeat TTL (beatAt older than STALE_TTL_MS).
+ *  - A `kept` lock (left by `--keep` / `up`) is NEVER auto-reclaimed — it
+ *    guards a stack that outlives the process and is cleared only by
+ *    `cele2e release` / `cele2e down`. This enforces the honor-system rule.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, openSync, closeSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { homedir, hostname } from 'node:os';
+import { basename, dirname, join } from 'node:path';
+
+const STALE_TTL_MS = 90_000;
+const HEARTBEAT_MS = 30_000;
+
+/**
+ * Machine-global lock path — NOT under any repo/worktree, because the Docker
+ * infra is global regardless of which checkout started it. Overridable via
+ * CELILO_E2E_LOCK_PATH (used by tests; also a handy operator escape hatch).
+ */
+export function lockPath(): string {
+  return process.env.CELILO_E2E_LOCK_PATH || join(homedir(), '.cache', 'celilo-e2e', 'run.lock');
+}
+
+export interface LockHolder {
+  pid: number;
+  hostname: string;
+  session: string;
+  test: string;
+  runId: string;
+  startedAt: string;
+  beatAt: number;
+  state: 'running' | 'kept';
+}
+
+interface Held {
+  keepOnRelease: boolean;
+  heartbeat: ReturnType<typeof setInterval> | null;
+  released: boolean;
+}
+
+/** Set once this process owns the lock; null otherwise. */
+let held: Held | null = null;
+
+export class E2eBusyError extends Error {
+  constructor(public holder: LockHolder) {
+    super(formatBusy(holder));
+    this.name = 'E2eBusyError';
+  }
+}
+
+/** Human label for who's running, e.g. "e2e busy: <session> running <test>, started 3m ago (pid 1234)". */
+export function formatBusy(h: LockHolder): string {
+  const age = ageString(Date.parse(h.startedAt));
+  const verb = h.state === 'kept' ? 'holds a kept stack from' : 'running';
+  const what = h.state === 'kept' ? `(run \`cele2e release\` to free it)` : `${h.test}, started ${age} ago (pid ${h.pid})`;
+  return `e2e busy: ${h.session} ${verb} ${what}`;
+}
+
+function ageString(since: number): string {
+  const s = Math.max(0, Math.floor((Date.now() - since) / 1000));
+  if (s < 60) return `${s}s`;
+  if (s < 3600) return `${Math.floor(s / 60)}m`;
+  return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
+}
+
+/** Branch + worktree path so a holder maps back to a specific session/thread. */
+function deriveSession(): string {
+  if (process.env.CELILO_E2E_SESSION) return process.env.CELILO_E2E_SESSION;
+  const cwd = process.cwd();
+  const git = (args: string[]): string => {
+    try {
+      return execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+    } catch {
+      return '';
+    }
+  };
+  const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']);
+  const top = git(['rev-parse', '--show-toplevel']);
+  const base = top ? basename(top) : basename(cwd);
+  if (branch && top) return `${branch} (${top})`;
+  return branch || base;
+}
+
+function pidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // ESRCH = no such process (dead). EPERM = alive but not ours (still alive).
+    return (err as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+export function readHolder(): LockHolder | null {
+  try {
+    return JSON.parse(readFileSync(lockPath(), 'utf-8')) as LockHolder;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is the on-disk holder stale (safe to reclaim)? A `kept` lock is never stale —
+ * it deliberately outlives its process. On the same host the PID is the source
+ * of truth; cross-host we can only use the heartbeat TTL.
+ */
+function isStale(h: LockHolder): boolean {
+  if (h.state === 'kept') return false;
+  if (h.hostname === hostname()) return !pidAlive(h.pid);
+  return Date.now() - h.beatAt > STALE_TTL_MS;
+}
+
+function writeHolder(fd: number, h: LockHolder): void {
+  writeFileSync(fd, JSON.stringify(h, null, 2));
+}
+
+/**
+ * Acquire the run lock for this process. Throws E2eBusyError if another live
+ * (non-stale) holder owns it. Registers an exit handler so the lock is released
+ * on any process.exit() path (normal, exception, or a SIGINT handler that
+ * exits). A signal-killed process with no exit handler leaks the lock, but the
+ * next run reclaims it via PID-liveness — that's exactly what staleness is for.
+ */
+export function acquireRunLock(opts: { test: string; runId: string; allowKept?: boolean }): void {
+  mkdirSync(dirname(lockPath()), { recursive: true });
+
+  const holder: LockHolder = {
+    pid: process.pid,
+    hostname: hostname(),
+    session: deriveSession(),
+    test: opts.test,
+    runId: opts.runId,
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let fd: number;
+    try {
+      fd = openSync(lockPath(), 'wx');
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
+      const existing = readHolder();
+      // Reclaim when: corrupt/unreadable, the holder is stale, OR this is a
+      // `--reuse` run taking over a `kept` stack it's deliberately reusing.
+      if (!existing || isStale(existing) || (opts.allowKept && existing.state === 'kept')) {
+        try { unlinkSync(lockPath()); } catch {}
+        continue;
+      }
+      throw new E2eBusyError(existing);
+    }
+    writeHolder(fd, holder);
+    closeSync(fd);
+
+    const heartbeat = setInterval(() => {
+      // Only refresh while running; a kept lock's beatAt is frozen by design.
+      if (!held || held.released || held.keepOnRelease) return;
+      try {
+        const cur = readHolder();
+        if (cur && cur.pid === process.pid) {
+          writeFileSync(lockPath(), JSON.stringify({ ...cur, beatAt: Date.now() }, null, 2));
+        }
+      } catch {}
+    }, HEARTBEAT_MS);
+    heartbeat.unref();
+
+    held = { keepOnRelease: false, heartbeat, released: false };
+    process.on('exit', releaseRunLock);
+    return;
+  }
+  // Two reclaim attempts both lost the race → someone else won fair and square.
+  const existing = readHolder();
+  throw new E2eBusyError(existing ?? holderUnknown());
+}
+
+function holderUnknown(): LockHolder {
+  return {
+    pid: 0,
+    hostname: hostname(),
+    session: 'unknown',
+    test: '?',
+    runId: '?',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+  };
+}
+
+/**
+ * Mark the lock to survive process exit in a `kept` state — call when a stack
+ * (network/containers) will outlive this process (`--keep`, `up`). The next run
+ * then refuses to clobber it until `cele2e release` / `cele2e down`.
+ */
+export function markKept(): void {
+  if (held) held.keepOnRelease = true;
+}
+
+/** Release the lock. Idempotent. Honors markKept() (kept-state) vs delete. */
+export function releaseRunLock(): void {
+  if (!held || held.released) return;
+  held.released = true;
+  if (held.heartbeat) clearInterval(held.heartbeat);
+
+  if (held.keepOnRelease) {
+    const cur = readHolder();
+    if (cur && cur.pid === process.pid) {
+      try { writeFileSync(lockPath(), JSON.stringify({ ...cur, state: 'kept', beatAt: Date.now() }, null, 2)); } catch {}
+    }
+    return;
+  }
+  try {
+    const cur = readHolder();
+    if (!cur || cur.pid === process.pid) unlinkSync(lockPath());
+  } catch {}
+}
+
+/** Forcibly clear the lock regardless of holder — used by `cele2e release`/`down`. */
+export function clearLock(): boolean {
+  try {
+    unlinkSync(lockPath());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Current lock state for `cele2e status` (and any poller). */
+export function lockStatus(): { free: boolean; holder: LockHolder | null } {
+  const h = readHolder();
+  if (!h || isStale(h)) return { free: true, holder: null };
+  return { free: false, holder: h };
+}

package/src/runner.ts

@@ -30,6 +30,7 @@ import {
   emitTestStarted,
 } from './bus-events';
 import { parseLine } from './parse-line';
+import { acquireRunLock, E2eBusyError, markKept } from './run-lock';
 
 // ─── Config ──────────────────────────────────────────────────────────
 
@@ -87,6 +88,26 @@ const patterns = rawArgs.filter((a) => !a.startsWith('--'));
 // run everything. Passed through from the CLI (not stripped) so the runner can
 // apply the exclusion.
 const flagCiSafe = rawArgs.includes('--ci-safe');
+// --notify: fire a desktop notification when the run finishes (handy for long
+// detached/background runs). Best-effort — never affects the exit code.
+const flagNotify = rawArgs.includes('--notify');
+
+function notify(title: string, message: string): void {
+  if (!flagNotify) return;
+  try {
+    if (process.platform === 'darwin') {
+      const r = spawnSync('terminal-notifier', ['-title', title, '-message', message, '-sound', 'Glass'], { stdio: 'ignore' });
+      if (r.error) {
+        spawnSync('osascript', ['-e', `display notification ${JSON.stringify(message)} with title ${JSON.stringify(title)} sound name "Glass"`], { stdio: 'ignore' });
+      }
+      spawnSync('say', [message], { stdio: 'ignore' });
+    } else if (process.platform === 'linux') {
+      spawnSync('notify-send', [title, message], { stdio: 'ignore' });
+    }
+  } catch {
+    // ponytail: notification is a nicety; its failure must never break a run.
+  }
+}
 
 // ─── Colors ──────────────────────────────────────────────────────────
 
@@ -94,7 +115,6 @@ const bold = '\x1b[1m';
 const dim = '\x1b[2m';
 const green = '\x1b[32m';
 const red = '\x1b[31m';
-const cyan = '\x1b[36m';
 const reset = '\x1b[0m';
 
 const interactive =
@@ -386,14 +406,6 @@ function dockerComposeShared(cmd: string): string {
   }
 }
 
-function isSharedInfraHealthy(): boolean {
-  const containers = dockerComposeShared('ps -q');
-  if (!containers) return false;
-  return dockerComposeShared(
-    'exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2',
-  ).length > 0;
-}
-
 function stopSharedInfra(): void {
   dockerComposeShared('down --volumes --remove-orphans');
   try { execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 }); } catch {}
@@ -417,6 +429,24 @@ function assertDockerAvailable(): void {
 async function main() {
   assertDockerAvailable();
 
+  // Serialize against other cele2e runs sharing the global Docker infra. Fail
+  // fast naming the holder; pollers use `cele2e status`. Acquire BEFORE any
+  // docker mutation so a competing run can't wipe our shared infra mid-setup.
+  const lockLabel = patterns.length ? patterns.join(',') : moduleDirs.length ? 'modules' : 'all';
+  try {
+    acquireRunLock({ test: lockLabel, runId: process.env.CELE2E_RUN_ID ?? ambientRunId, allowKept: flagReuse });
+    // A --reuse run keeps using a kept stack that stays up afterwards — hold the
+    // lock in kept state so it isn't freed out from under the reused network.
+    if (flagReuse) markKept();
+  } catch (err) {
+    if (err instanceof E2eBusyError) {
+      console.error(`\n${red}${err.message}${reset}`);
+      console.error(`${dim}Poll with: cele2e status   (machine-readable: cele2e status --json)${reset}\n`);
+      process.exit(3);
+    }
+    throw err;
+  }
+
   let testFiles: string[] = [];
 
   if (moduleDirs.length > 0) {
@@ -557,9 +587,8 @@ async function main() {
   mkdirSync(resultsDir, { recursive: true });
 
   // runId scopes every event in this suite invocation. Read from env to
-  // let callers (CI, the Claude tool wrapper) inject a known id and
-  // subscribe to it before launching cele2e; otherwise generate one and
-  // print it so the operator can correlate.
+  // let callers (CI) inject a known id and subscribe to it before launching
+  // cele2e; otherwise generate one and print it so the operator can correlate.
   const runId = process.env.CELE2E_RUN_ID || crypto.randomUUID();
   if (!process.env.CELE2E_RUN_ID) {
     console.log(`${dim}runId: ${runId}${reset}`);
@@ -574,7 +603,7 @@ async function main() {
 
   // 'auto' resolves to 'render' when isTTY is true (operator at a
   // terminal sees animated spinners) and 'protocol' when isTTY is
-  // false (cele2e-run.sh wrapper / any redirected stdout gets ASCII
+  // false (a backgrounded run / any redirected stdout gets ASCII
   // `[progress:*]` markers — grep-friendly, no braille noise).
   // Prior to this change the mode was hardcoded 'render', so non-TTY
   // callers got `⣾ phase started` / `✔ phase done` lines per phase
@@ -663,6 +692,9 @@ async function main() {
       console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
       console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
       console.log(`${dim}  Tear down: cele2e down${reset}`);
+      // The kept network outlives this process — hold the lock in a `kept`
+      // state so the next run refuses to clobber it (cleared by `cele2e down`).
+      markKept();
     }
 
     if (result.status === 'pass') {
@@ -745,6 +777,7 @@ async function main() {
     resultsDir,
   });
 
+  notify('cele2e', `${passed} passed, ${failed} failed — ${formatDuration(suiteDuration)}`);
   process.exit(failed > 0 ? 1 : 0);
 }
 
@@ -762,5 +795,6 @@ main().catch((err) => {
     error: err instanceof Error ? err.message : String(err),
   });
   console.error(err);
+  notify('cele2e', 'run failed');
   process.exit(1);
 });

package/src/shared-infra.ts

@@ -30,6 +30,34 @@ function run(cmd: string, opts?: { cwd?: string; timeout?: number }): string {
 
 let sharedInfraRunning = false;
 
+/**
+ * Remove ALL celilo-e2e-* Docker resources by name prefix (#212). Used at
+ * start-of-run, where the run-lock guarantees no other session's stack is live.
+ * Graceful compose-down of the shared project first (clean network detach),
+ * then a force sweep that catches orphans from any prior invocation regardless
+ * of compose-project name or the path that created them.
+ */
+function nukeE2eResources(e2eDir: string): void {
+  try {
+    run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`,
+      { cwd: e2eDir, timeout: 30_000 });
+  } catch {}
+  try {
+    const ids = run('docker ps -aq --filter name=celilo-e2e', { timeout: 10_000 });
+    if (ids.trim()) run(`docker rm -f ${ids.replace(/\n/g, ' ')}`, { timeout: 60_000 });
+  } catch {}
+  try {
+    const nets = run('docker network ls --format "{{.Name}}"')
+      .split('\n')
+      .filter((n) => n.startsWith('celilo-e2e'));
+    for (const net of nets) {
+      try { run(`docker network rm ${net}`, { timeout: 5_000 }); } catch {}
+    }
+  } catch {}
+  try { run('docker network prune -f', { timeout: 10_000 }); } catch {}
+  try { run('docker volume prune -f', { timeout: 10_000 }); } catch {}
+}
+
 /**
  * Check if the shared infrastructure is already running and healthy.
  *
@@ -104,38 +132,14 @@ export async function ensureSharedInfra(): Promise<void> {
 
   const e2eDir = getE2eDir();
 
-  // Clean up any stale shared infra resources
+  // Start-of-run cleanup. We hold the run-lock here (acquired in the runner /
+  // build path before any docker mutation), so NO other session's stack is
+  // live — it is safe to remove EVERY celilo-e2e-* resource by name prefix.
+  // This self-heals orphans left by a crashed / cross-path / cross-checkout
+  // prior run (e.g. a botched-mount container that wedges the next `up`),
+  // which the old compose-project-scoped `down` couldn't see (#212).
   console.log('[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete');
-  try {
-    run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} down --volumes --remove-orphans`,
-      { cwd: e2eDir, timeout: 30_000 });
-  } catch {}
-
-  // Also clean up any per-test projects that reference our networks
-  try {
-    const networks = run('docker network ls --format "{{.Name}}"')
-      .split('\n')
-      .filter(n => n.startsWith('celilo-e2e-') && !n.startsWith(`${SHARED_PROJECT_NAME}_`));
-    if (networks.length > 0) {
-      const projects = [...new Set(networks.map(n => n.replace(/_[^_]+$/, '')))];
-      for (const project of projects) {
-        try { run(`docker compose -p ${project} down --volumes --remove-orphans`, { timeout: 30_000 }); } catch {}
-      }
-    }
-  } catch {}
-
-  try { run('docker network prune -f', { timeout: 10_000 }); } catch {}
-  try { run('docker volume prune -f', { timeout: 10_000 }); } catch {}
-
-  // Force-remove any leftover networks by name
-  try {
-    const staleNets = run('docker network ls --format "{{.Name}}"')
-      .split('\n')
-      .filter(n => n.startsWith('celilo-e2e'));
-    for (const net of staleNets) {
-      try { run(`docker network rm ${net}`, { timeout: 5_000 }); } catch {}
-    }
-  } catch {}
+  nukeE2eResources(e2eDir);
 
   // Refresh the bundled registry-server source so Dockerfile.registry's
   // COPY resolves whether we're in the monorepo (regenerated from the

package/AGENTS.md

@@ -1,117 +0,0 @@
-# AGENTS.md — `@celilo/e2e`
-
-`@celilo/e2e` is a **simulated internet in Docker** for end-to-end testing of
-[celilo](https://celilo.computer)-deployed apps. It stands up a DNS hierarchy
-(root → TLD → registrar → recursive resolver), an ACME server (Pebble, standing
-in for Let's Encrypt), an ISP router + customer firewall with real iptables NAT,
-zoned LAN networks, and systemd target machines — then runs celilo *inside* it as
-if it were deploying to a real home lab. This file orients an AI agent to write
-and run those tests. **Read the in-package docs below before searching the web** —
-they are the source of truth.
-
-This file ships **inside the npm package**: the `./`-prefixed docs in the doc map
-are installed alongside it (grep them offline). Repo-relative entries point at the
-source repository, which the tarball does not carry.
-
-## Mental model (learn these five things)
-
-- **The sim is a faithful internet, not a convenience harness.** Public DNS only
-  ever answers with the customer firewall's *external* IP; the firewall DNATs
-  inbound to the right internal host. ACME challenges validate over the public
-  path. A test that "passes" by short-circuiting any of this is testing a placebo
-  — see **Inviolable rules** below.
-- **`network()` builder** — declare zones and the machines in them, `.start()` the
-  topology, drive celilo with `.celilo(...)`, assert, then `.stop()`. This is the
-  whole library surface (`import { network } from '@celilo/e2e'`).
-- **Zones** — `internal` (home LAN), `dmz`, `app`, `secure` are the customer-side
-  networks; `isp-external` / `internet-external` model the public internet. Public
-  IPs live only on the public networks; the firewall containers are the only place
-  the two address spaces meet (via NAT/DNAT).
-- **Two consumption modes** — as a **library** (`network()` in a `bun test` file)
-  or via the **`cele2e` CLI** (`cele2e run <test>`, `build-infra`, `list`). The
-  shared infra (`celilo-e2e-shared`) is a *global* Docker compose project,
-  independent of which checkout started it.
-- **`build-infra` gate** — anything baked into a Docker image (`docker/`,
-  `config/`, `simulators/`, installed package tarballs) needs `cele2e build-infra`
-  before it takes effect. Pure `src/*.ts` changes do not (bun runs TS directly).
-
-## Writing a test (the happy path)
-
-```ts
-import { test, expect } from 'bun:test';
-import { network } from '@celilo/e2e';
-
-test('caddy serves TLS in the dmz', async () => {
-  const net = await network()
-    .dmz({ caddy: '10.0.10.10' })   // machine name → zone-side IP
-    .start();
-  try {
-    await net.celilo('module import caddy');
-    await net.celilo('module deploy caddy --no-interactive');
-    // assert against the deployed service via the public path, not the container IP
-  } finally {
-    await net.stop();
-  }
-});
-```
-
-- Prefer the **fixtures** in `./src/fixtures.ts` (`CADDY_DEPLOYMENT`,
-  `FULL_STACK`, `TECHNITIUM_STACK`, …) over hand-rolling topology.
-- Drive **everything through the celilo CLI** or the module's documented surface —
-  never `docker exec` to hand-edit a deployed container's config (that's the bug
-  you're trying to catch).
-- Poll on the **observable condition** the deploy is meant to satisfy
-  (`VantageProbe`, DNS resolution, an HTTP header) — never `setTimeout` to mask a
-  race.
-
-## CLI you'll use
-
-```
-cele2e list                     # enumerate every test (module suites + top-level)
-cele2e build-infra              # rebuild Docker images (required after image-baked changes)
-cele2e run <test> [--keep]      # run one test; --keep leaves the network up for debugging
-cele2e run --all                # full regression (every suite); --all-modules = modules only
-```
-
-For Claude-driven / detached runs, fire via the repo wrapper
-`infra/scripts/cele2e-run.sh <test> [--keep|--build]` (fire-and-forget RUNID/LOG
-contract) rather than calling `cele2e run` from a long-lived watcher.
-
-## Inviolable rules (breaking any turns the suite into a placebo)
-
-1. **No RFC 1918 leaks past the public boundary.** Public DNS never serves an A
-   record pointing at `10/8`, `172.16/12`, or `192.168/16` — only the firewall's
-   external IP.
-2. **Don't collapse the topology to fix a timing/connectivity bug.** Fix the
-   *deployment ordering* (DNS record / DNAT rule / firewall opening in place
-   before the dependent step), not the simulation.
-3. **Public IPs only on the public networks.** Firewall containers bridge public
-   and private — the only place those spaces meet, and only via NAT/DNAT.
-4. **ACME challenges go through the public path** to the firewall's external IP.
-5. **DDNS updates modify *public* DNS** with the firewall's external IP, never an
-   internal target IP.
-
-Cheats (`/etc/hosts` overrides, `iptables -F`, `--insecure` in module code,
-hardcoded internal IPs, `|| true` on a celilo command, sleeping past a race) are
-fine for **bisecting** a failure but are never a **final solution** — confirm the
-hypothesis, then ship the real fix. Full list + rationale in `./README.md` and the
-repo `CLAUDE.md`.
-
-## Doc map (read in this order)
-
-Shipped **inside this package** (offline, grep-friendly):
-
-- `./README.md` — network architecture, machines, routing, DNS, simulators,
-  manual usage. Start here.
-- `./COVERAGE.md` — the array/object/map coverage audit (where N>1 paths are or
-  aren't exercised). Consult before adding a multi-element feature test.
-- `./src/index.ts` — the public library exports (`network`, types, probes).
-- `./src/fixtures.ts` — ready-made topology presets to build tests on.
-- `./src/types.ts` — `NetworkHandle`, `Zone`, `MachineSpec`, `Vantage`, and the
-  zone subnet/gateway constants.
-
-In the **source repo** (not carried in the tarball):
-
-- `infra/docs/RUNNING_CELE2E_TESTS.md` — the full operator guide; read before
-  running the suite for real.
-- `https://celilo.computer/docs` — hosted docs (LAN; the repo docs lead).

package/COVERAGE.md

@@ -1,60 +0,0 @@
-# E2E Test Coverage Audit
-
-This document tracks how well the e2e suite exercises capability and module
-fields whose type is `array`, `object`, or `map` — anywhere the framework's
-public API admits "more than one of X."
-
-## Why this exists
-
-A test that only ever uses N=1 will pass even when the multi-element path is
-broken. The bug fixed by `dns_registrar` 4.0.0 (see
-`apps/celilo/designs/DNS_REGISTRAR_MULTI_DOMAIN.md`) escaped the existing e2e
-suite for exactly this reason: the website-deploy-simple test put caddy, namecheap,
-and the website all on a single domain, and the multi-domain code path was
-never exercised.
-
-## Audit checklist
-
-For every capability and every module manifest, identify each field whose
-type is `array`, `object`, or `map`. For each one, ask:
-
-1. Does at least one e2e test exercise this field with **N > 1** distinct
-   values?
-2. Are those values **meaningfully different** (different domains, different
-   zones, different ports — not just "two of the same kind of thing")?
-3. Does the test assert behavior that would be **wrong if only the first
-   element worked**?
-
-Anywhere all three answers aren't "yes," that's a gap to either close (write
-the test) or document why it's intentionally deferred.
-
-## Coverage table
-
-| Field | N>1? | Meaningfully different? | Asserts non-first works? | Status |
-|---|---|---|---|---|
-| `namecheap.domains` | yes (`website-deploy-cross-domain`) | yes (different TLDs: `iamtheinternet.org` + `example.net`) | yes (verifies example.net DNS+TLS while iamtheinternet.org is canonical) | covered |
-| `namecheap.ddns_passwords` (per-domain map) | yes (same test) | yes (distinct passwords per domain) | yes (uses example.net's own password to update its A record) | covered |
-| `caddy.hostnames` | yes (`website-deploy-simple`) | partial (same domain in `website-deploy-simple.test.ts`); yes (cross-TLD in cross-domain test) | yes (cross-domain test serves a hostname that isn't `hostnames[0]`) | covered |
-| `iptables.exposed_services` | unclear — needs audit | — | — | **GAP** |
-| `public_web` registrations across modules | partial (`website-deploy-simple.test.ts` registers auth + website on the same domain) | needs cross-domain variant | partial | **GAP** |
-| `<every secret: object field>` | typically only one key | usually no | — | **AUDIT EACH** |
-
-## Closing the gaps
-
-- **`iptables.exposed_services`**: write an e2e that exposes two services
-  on the same machine via different ports and verifies both are reachable
-  from outside.
-- **`public_web` cross-domain**: extend `website-deploy-simple.test.ts` (or a new
-  test) to register routes on two different domains and verify each is
-  served by the right backend.
-- **secret object fields**: as modules using these grow (e.g. third-party
-  API integrations with multiple keys), add per-key tests.
-
-## New-module checklist
-
-When authoring a module, list every field whose type permits multiple
-values. For each, write at least one e2e or integration test that uses
-**N ≥ 2** with meaningfully different values. Add a row to the table
-above (or document why you've deferred it).
-
-See `design/MODULE_DEVELOPMENT_GUIDE.md` for the full new-module checklist.