@celilo/e2e 0.19.3 → 0.20.1

package/src/live-stack.test.ts

@@ -0,0 +1,300 @@
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  type DockerReader,
+  LiveStackError,
+  type ProcessProbe,
+  SHARED_ORPHAN_MIN_AGE_MS,
+  findLiveE2eStack,
+  looksLikeE2eRunCommand,
+} from './live-stack';
+import type { LockStatus } from './run-lock';
+import { startupCleanup } from './shared-infra';
+
+/**
+ * The live-stack guard for the startup cleanup (celilo#1297, ce-h04y).
+ *
+ * The #1297 incident: a second cele2e invocation got past the host-global run
+ * lock and nukeE2eResources force-removed another run's mid-flight stack. The
+ * guard checks Docker and the lock file at the removal site. These tests prove
+ * the wiring through the injected runner: a live stack refuses with NOTHING
+ * removed, a dead one sweeps. Delete the guard call and the first test goes
+ * red — the removal commands run behind what should have been a refusal.
+ */
+
+const DIR = dirname(fileURLToPath(import.meta.url));
+const SHARED_INFRA_SRC = readFileSync(join(DIR, 'shared-infra.ts'), 'utf-8');
+
+/** Recording fake: answers `docker ps` from `psOutput`, records every command. */
+function fakeDocker(psOutput: string): { docker: DockerReader; commands: string[] } {
+  const commands: string[] = [];
+  return {
+    commands,
+    docker: (args) => {
+      commands.push(args.join(' '));
+      if (args[0] === 'ps') return psOutput;
+      return '';
+    },
+  };
+}
+
+const noLock: () => LockStatus = () => ({
+  free: true,
+  holder: null,
+  heartbeatAgeMs: null,
+  suspect: false,
+  ownKept: false,
+});
+
+function foreignLock(overrides: Partial<LockStatus> = {}): () => LockStatus {
+  return () => ({
+    free: false,
+    holder: {
+      pid: 1,
+      hostname: hostname(),
+      session: 'polecat/ce-9999 (some other worktree)',
+      test: 'crew-alerting',
+      runId: 'run-abc',
+      startedAt: new Date(Date.now() - 5 * 60_000).toISOString(),
+      beatAt: Date.now() - 45_000,
+      state: 'running',
+    },
+    heartbeatAgeMs: 45_000,
+    suspect: false,
+    ownKept: false,
+    ...overrides,
+  });
+}
+
+describe('findLiveE2eStack', () => {
+  test('refuses when a celilo-e2e-* container is running, naming it', () => {
+    const { docker } = fakeDocker(
+      'celilo-e2e-shared_namecheap-dns\trunning\ncelilo-e2e-1788769175864_fw-main\texited\n',
+    );
+    const refusal = findLiveE2eStack(docker, noLock);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns');
+    expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main');
+    // The remedy must be a command that EXISTS on the host printing the
+    // message (celilo#1314): a bare `cele2e down` does not — forgejo job
+    // workspaces are ephemeral, so the binary only exists inside a checkout.
+    expect(refusal?.reason).toContain('docker rm -f');
+    expect(refusal?.reason).not.toContain('cele2e down');
+    // The refusal must be greppable as an environment problem, not a check
+    // failure (celilo#1314 direction 3).
+    expect(refusal?.reason).toContain('[infra-refusal]');
+    expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_namecheap-dns']);
+  });
+
+  test('proceeds when every container is exited', () => {
+    const { docker } = fakeDocker(
+      'celilo-e2e-1788769175864_fw-main\texited\ncelilo-e2e-shared_registry\tdead\n',
+    );
+    expect(findLiveE2eStack(docker, noLock)).toBeNull();
+  });
+
+  test('treats paused and restarting containers as live — they hold real state', () => {
+    for (const state of ['paused', 'restarting', 'running']) {
+      const { docker } = fakeDocker(`celilo-e2e-shared_registry\t${state}\n`);
+      const refusal = findLiveE2eStack(docker, noLock);
+      expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_registry']);
+    }
+  });
+
+  test('refuses on a foreign lock with a fresh heartbeat, naming session, test and heartbeat age', () => {
+    const { docker } = fakeDocker('');
+    const refusal = findLiveE2eStack(docker, foreignLock());
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('polecat/ce-9999');
+    expect(refusal?.reason).toContain('crew-alerting');
+    expect(refusal?.reason).toContain('heartbeat 45s old');
+    expect(refusal?.holder?.runId).toBe('run-abc');
+  });
+
+  test('exempts our own lock — the caller holds it for its whole run', () => {
+    const { docker } = fakeDocker('');
+    const own: () => LockStatus = () => ({
+      free: false,
+      holder: {
+        pid: process.pid,
+        hostname: hostname(),
+        session: 'whatever (this worktree)',
+        test: 'caddy-internal-private',
+        runId: 'run-own',
+        startedAt: new Date().toISOString(),
+        beatAt: Date.now(),
+        state: 'running',
+      },
+      heartbeatAgeMs: 0,
+      suspect: false,
+      ownKept: false,
+    });
+    expect(findLiveE2eStack(docker, own)).toBeNull();
+  });
+
+  test('the container check has no own-lock exemption — a stolen lock says "ours" while another run owns the containers', () => {
+    // The #1297 suspected path: the second invocation ends up holding the lock
+    // while the first run's containers are still up. Only Docker can tell that
+    // truth, so running containers refuse even when the lock is ours.
+    const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
+    const refusal = findLiveE2eStack(docker, foreignLock({ holder: null }));
+    expect(refusal?.runningContainers.length).toBe(1);
+  });
+});
+
+describe('looksLikeE2eRunCommand', () => {
+  test('matches the cele2e CLI by any argv mention', () => {
+    expect(looksLikeE2eRunCommand('cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('./node_modules/.bin/cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/bin/cele2e.ts run --all')).toBe(true);
+  });
+
+  test('matches a direct bun test of this package, which has no cele2e in argv', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/e2e/tests/dns-replication.test.ts')).toBe(
+      true,
+    );
+    expect(looksLikeE2eRunCommand('bun test e2e/tests/smoke.test.ts')).toBe(true);
+  });
+
+  test('does not match unrelated bun test runs, editors, or scripts', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/mcp-server/src/tools.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('vi packages/e2e/src/live-stack.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/scripts/pack-celilo-packages.ts')).toBe(
+      false,
+    );
+    expect(looksLikeE2eRunCommand('bun build apps/celilo/src/index.ts')).toBe(false);
+  });
+});
+
+describe('shared-only orphan reap (celilo#1314)', () => {
+  // Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC".
+  const dockerAge = (msAgo: number): string => {
+    const d = new Date(Date.now() - msAgo);
+    return `${d.toISOString().slice(0, 10)} ${d.toISOString().slice(11, 19)} +0000 UTC`;
+  };
+  const noProcesses: ProcessProbe = () => [];
+
+  test('REAPS the exact shape measured on the builder: shared-only stack, 11h old, no per-test project, no cele2e process', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-shared_registry-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    // No containers at all is NOT the test: the guard must see the live shared
+    // containers and STILL clear the way, because every piece of run evidence
+    // is absent.
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+    startupCleanup('/tmp/e2e', docker, noLock, noProcesses);
+    // The reap is a real teardown, not a refusal: the shared compose down ran.
+    expect(commands.some((c) => c.includes(' down '))).toBe(true);
+  });
+
+  test('must NOT reap a shared stack with a live per-test project beside it (the celilo#1297 shape)', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-1788769175864_fw-main\trunning\t${dockerAge(5 * 60_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.runningContainers).toContain('celilo-e2e-shared_namecheap-dns-1');
+    expect(refusal?.runningContainers).toContain('celilo-e2e-1788769175864_fw-main');
+    expect(() => startupCleanup('/tmp/e2e', docker, noLock, noProcesses)).toThrow(LiveStackError);
+    expect(commands.some((c) => c.includes(' down '))).toBe(false);
+  });
+
+  test('must NOT reap while any other cele2e process is alive — a run may be between suites', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const oneRunner: ProcessProbe = () => [424242];
+    const refusal = findLiveE2eStack(docker, noLock, oneRunner);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('424242');
+  });
+
+  test(`must NOT reap a stack younger than ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m — the evidence is not old enough to be conclusive`, () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(5 * 60_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('5m');
+  });
+
+  test('must NOT reap when the age cannot be read — inconclusive evidence refuses, it never reaps', () => {
+    // Two fields, no CreatedAt: the pre-#1314 fake shape, and anything docker
+    // might print that the parser does not recognize.
+    const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('age');
+  });
+
+  test('a foreign fresh lock still refuses a shared-only orphan — the reap never overrides the lock', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, foreignLock(), noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('polecat/ce-9999');
+  });
+
+  test('exited containers of the shared project alone still proceed (unchanged)', () => {
+    const ps = `celilo-e2e-shared_registry-1\texited\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+  });
+});
+
+describe('startupCleanup', () => {
+  test('refuses behind a live stack and removes NOTHING (celilo#1297)', () => {
+    const { docker, commands } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
+    expect(() => startupCleanup('/tmp/e2e', docker, noLock)).toThrow(LiveStackError);
+    // Only the ps read happened. No compose down, no rm, no prune.
+    expect(commands).toEqual([expect.stringContaining('ps -a --filter name=celilo-e2e')]);
+    expect(commands.some((c) => c.includes(' down '))).toBe(false);
+    expect(commands.some((c) => c.includes('rm -f'))).toBe(false);
+    expect(commands.some((c) => c.includes('prune'))).toBe(false);
+  });
+
+  test('sweeps when the environment is provably dead, compose down before the force sweep', () => {
+    const { docker, commands } = fakeDocker('celilo-e2e-1788769175864_fw-main\texited\n');
+    startupCleanup('/tmp/e2e', docker, noLock);
+    const down = commands.findIndex((c) => c.includes('compose -f docker-compose.shared.yml'));
+    const rm = commands.findIndex((c) => c.includes('rm -f'));
+    const netPrune = commands.findIndex((c) => c.includes('network prune'));
+    const volPrune = commands.findIndex((c) => c.includes('volume prune'));
+    expect(down).toBeGreaterThanOrEqual(0);
+    expect(rm).toBeGreaterThan(down);
+    expect(netPrune).toBeGreaterThan(rm);
+    expect(volPrune).toBeGreaterThan(netPrune);
+    // The sweep keys off the celilo-e2e prefix; it must never name the shared
+    // project in a force removal (the graceful compose down above owns that).
+    for (const command of commands.filter((c) => c.includes('rm -f'))) {
+      expect(command).not.toContain('celilo-e2e-shared');
+    }
+  });
+});
+
+// The guard is only real if it is WIRED — a perfect guard nobody calls is the
+// same failure as no guard. These source assertions pin the two removal sites
+// in ensureSharedInfra; the behavioral tests above pin the guard itself.
+describe('wiring in shared-infra.ts', () => {
+  test('the start-of-run nuke runs through startupCleanup, which guards first', () => {
+    expect(SHARED_INFRA_SRC).toContain('startupCleanup(e2eDir)');
+    const cleanupBody = SHARED_INFRA_SRC.slice(
+      SHARED_INFRA_SRC.indexOf('export function startupCleanup'),
+      SHARED_INFRA_SRC.indexOf('nukeE2eResources(e2eDir, docker)'),
+    );
+    expect(cleanupBody).toContain('findLiveE2eStack(docker, lock, processes)');
+  });
+
+  test('the DNS-restart branch guards before tearing down the running stack', () => {
+    const branchBody = SHARED_INFRA_SRC.slice(
+      SHARED_INFRA_SRC.indexOf('DNS check failed'),
+      SHARED_INFRA_SRC.indexOf('await stopSharedInfra()'),
+    );
+    expect(branchBody).toContain('refuseOnLiveStack()');
+  });
+});

package/src/live-stack.ts

@@ -0,0 +1,355 @@
+/**
+ * Live-stack guard for the e2e startup cleanup (celilo#1297, ce-h04y).
+ *
+ * nukeE2eResources tears down EVERY celilo-e2e-* container by name prefix, on
+ * the assumption that holding the run-lock proves no other session's stack is
+ * live. The #1297 incident broke that assumption: a second invocation got past
+ * the lock anyway and force-removed another run's mid-flight stack, nine
+ * containers, the suite dead at 12.95s with exit 137. Which lock path let it
+ * through is not reconstructable, and does not need to be — the guard defends
+ * every path by checking ground truth (Docker and the lock file) at the
+ * removal site instead of trusting the lock's own verdict.
+ *
+ * Decided (peba, 2026-09-07, option a): before the startup cleanup removes
+ * anything, it refuses when any celilo-e2e-* container is still live or the
+ * run-lock heartbeat is fresh and foreign. It removes only what is provably
+ * dead. There is no --force: an operator who wants a live stack gone uses
+ * `cele2e down`.
+ *
+ * Deliberately NOT guarded: the runner's end-of-run stopSharedInfra teardown.
+ * The owner tearing down its own live stack is the normal exit path, and the
+ * guard's container check would refuse it by definition.
+ *
+ * The shared-stack orphan reap (celilo#1314). The #1297 guard is keyed on
+ * "any live celilo-e2e-* container", and the shared stack is designed never to
+ * be touched by name-protecting cleanup — so one abnormal exit wedges the
+ * host permanently: the orphaned shared stack refuses every later cleanup, it
+ * is the one thing the guard structurally cannot resolve. Measured on the
+ * builder 2026-09-08: 13 celilo-e2e-shared-* containers, 11h old, no per-test
+ * project, no cele2e process, and every npm-consumer-smoke run red on a check
+ * that never executed. So a stack whose ONLY live containers belong to the
+ * shared project gets an evidence test instead of a blanket refusal: no
+ * foreign lock, no live run process, and an age past a threshold means the
+ * stack is garbage and cleanup may act. Any single piece of run evidence
+ * present, or any age that cannot be read, refuses — inconclusive evidence
+ * never reaps, because a false reap kills a real run while a false refusal
+ * costs the operator one docker command.
+ *
+ * The refusal itself names a command that exists on the host printing it (a
+ * bare `cele2e down` does not: forgejo job workspaces are ephemeral, so the
+ * binary only exists inside a checkout), and every refusal reason carries the
+ * `[infra-refusal]` marker with its exit code, so a log search distinguishes
+ * an environment problem from a check failure (celilo#1314 directions 2+3).
+ */
+
+import type { ExecFileSyncOptions } from 'node:child_process';
+import { execFileSync } from 'node:child_process';
+import { SHARED_PROJECT_NAME } from './docker-compose-generator';
+import {
+  type LockHolder,
+  type LockStatus,
+  formatAge,
+  formatBusy,
+  heartbeatAgeMs,
+  isSameSession,
+  lockStatus,
+} from './run-lock';
+
+/**
+ * Docker access, injected so the guard and the sweep it protects are
+ * unit-testable without a daemon (the seam lane from ce-h4no). Unlike
+ * proxmox-provisioner's DockerRunner this carries per-call timeouts and cwd,
+ * which the compose sweep needs.
+ */
+export type DockerReader = (args: string[], opts?: { timeoutMs?: number; cwd?: string }) => string;
+
+export const realDocker: DockerReader = (args, opts) =>
+  execFileSync('docker', args, {
+    encoding: 'utf-8',
+    timeout: opts?.timeoutMs ?? 60_000,
+    cwd: opts?.cwd,
+    stdio: ['pipe', 'pipe', 'pipe'],
+  } satisfies ExecFileSyncOptions).trim();
+
+/** What the guard found, and the human-readable refusal naming it. */
+export interface LiveStackRefusal {
+  reason: string;
+  /** Container names still live, when the refusal is about containers. */
+  runningContainers: string[];
+  /** The foreign lock holder, when the refusal is about the lock. */
+  holder: LockHolder | null;
+}
+
+export class LiveStackError extends Error {
+  constructor(readonly refusal: LiveStackRefusal) {
+    super(refusal.reason);
+    this.name = 'LiveStackError';
+  }
+}
+
+/**
+ * How old the youngest live shared-stack container must be before a
+ * shared-only stack with no other run evidence is treated as an orphan.
+ * One hour sits far above any window in which a run process could have died
+ * without its containers dying too, and far below the 11h the builder sat
+ * wedged (celilo#1314). A stack younger than this refuses even with no other
+ * evidence: the reap is the dangerous direction, so it waits for certainty.
+ */
+export const SHARED_ORPHAN_MIN_AGE_MS = 60 * 60_000;
+
+/**
+ * Live pids of processes that may own an e2e run. Injectable so the reap's
+ * evidence test is unit-testable without a process table.
+ */
+export type ProcessProbe = () => number[];
+
+/**
+ * Does this process command line look like an e2e run? Pure so the probe's
+ * reach is testable. Matches the two ways a run actually exists:
+ *   - the cele2e CLI and anything whose argv names it (`cele2e run|up|down|...`)
+ *   - a direct `bun test` of this package's suites, which manages the shared
+ *     stack through ensureSharedInfra but has no cele2e in argv
+ * The bun-test arm requires bun AND e2e AND test in the command, so an
+ * unrelated `bun test` elsewhere only false-matches when its path names e2e —
+ * and a false match refuses (cheap), where a false miss reaps a live run
+ * (expensive). An editor with an e2e test file open does not match: it does
+ * not start with a bun invocation.
+ */
+export function looksLikeE2eRunCommand(command: string): boolean {
+  if (command.includes('cele2e')) return true;
+  return (
+    /(^|[\\/])bun(\.exe)?\s/.test(command) && command.includes('e2e') && command.includes('test')
+  );
+}
+
+/** pids of this process's ancestors, self included, bounded at 64 levels. */
+function familyOfSelf(): Set<number> {
+  const family = new Set<number>([process.pid]);
+  let pid: number | undefined = process.ppid;
+  for (let i = 0; pid !== undefined && pid > 1 && i < 64; i++) {
+    family.add(pid);
+    try {
+      const out = execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+      const ppid = Number.parseInt(out, 10);
+      pid = Number.isFinite(ppid) ? ppid : undefined;
+    } catch {
+      pid = undefined;
+    }
+  }
+  return family;
+}
+
+/**
+ * The real probe: one `ps` listing, filtered by looksLikeE2eRunCommand, with
+ * this process and its ancestry removed — the cleanup runs INSIDE the run it
+ * would otherwise see as evidence, and the run's own launcher shell carries
+ * the same strings in its argv.
+ */
+export const realProcessProbe: ProcessProbe = (): number[] => {
+  let listing: string;
+  try {
+    listing = execFileSync('ps', ['-axo', 'pid=,command='], {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    // A process table that cannot be read is missing evidence. The caller
+    // treats any probe failure conservatively; here that means reporting no
+    // matches, which the shared-only path then backstops with the age
+    // threshold and the unreadable-age refusal.
+    return [];
+  }
+  const family = familyOfSelf();
+  const pids: number[] = [];
+  for (const line of listing.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const sep = trimmed.indexOf(' ');
+    if (sep <= 0) continue;
+    const pid = Number.parseInt(trimmed.slice(0, sep), 10);
+    if (!Number.isFinite(pid) || family.has(pid)) continue;
+    if (looksLikeE2eRunCommand(trimmed.slice(sep + 1))) pids.push(pid);
+  }
+  return pids;
+};
+
+/**
+ * Every refusal goes through here so the printed reason is uniformly
+ * greppable as an infrastructure refusal (celilo#1314 direction 3) rather
+ * than reading as a check failure.
+ */
+function makeRefusal(
+  detail: string,
+  runningContainers: string[],
+  holder: LockHolder | null,
+): LiveStackRefusal {
+  const reason = `[infra-refusal] refusing to clean up: ${detail}\n(this is an environment problem, not a test failure — the run exits 3)`;
+  return { reason, runningContainers, holder };
+}
+
+/**
+ * The remedy printed with a live-stack refusal. A raw docker removal, because
+ * `cele2e down` does not exist on the hosts that print this message: forgejo
+ * job workspaces are ephemeral, so the binary only exists inside a checkout
+ * (celilo#1314 direction 2). Removing the containers is what unblocks the
+ * guard; the next cleanup sweeps whatever name-prefix resources survive.
+ */
+export const CLEAR_STACK_COMMAND = 'docker rm -f $(docker ps -aq --filter name=celilo-e2e)';
+
+/** Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC". */
+function parseDockerCreatedAt(raw: string | undefined): Date | null {
+  if (raw === undefined) return null;
+  const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.(\d+))? ([+-])(\d{2})(\d{2})/.exec(
+    raw.trim(),
+  );
+  if (!m) return null;
+  const [, date, time, frac, sign, offH, offM] = m;
+  const utcMs = Date.parse(`${date}T${time}Z`);
+  if (Number.isNaN(utcMs)) return null;
+  const offsetMs = (sign === '-' ? -1 : 1) * (Number(offH) * 60 + Number(offM)) * 60_000;
+  const fracMs = frac ? Number(frac.padEnd(3, '0').slice(0, 3)) : 0;
+  return new Date(utcMs - offsetMs + fracMs);
+}
+
+/**
+ * Containers of the shared compose project. Compose names them
+ * `<project>_<service>_<n>` (or with `-` separators on newer compose), so the
+ * character right after the project name is the tell. Everything else live —
+ * per-test projects AND sim-created guests, whose names carry no project —
+ * is run evidence.
+ */
+function isSharedStackContainer(name: string): boolean {
+  if (!name.startsWith(SHARED_PROJECT_NAME)) return false;
+  const sep = name[SHARED_PROJECT_NAME.length];
+  return sep === '_' || sep === '-';
+}
+
+/**
+ * Container states that mean the container cannot be holding a live stack.
+ * Everything else (running, paused, restarting) is live: paused containers
+ * keep their memory, restarting ones own their networks, and `docker rm -f`
+ * on either kills real work.
+ */
+const DEAD_STATES = new Set(['exited', 'created', 'dead']);
+
+function isOwnHolder(h: LockHolder): boolean {
+  return h.pid === process.pid || isSameSession(h);
+}
+
+/**
+ * Inspect the machine for a live e2e stack. Returns a refusal when the caller
+ * must not remove anything, null when the environment is provably dead.
+ *
+ * The lock check exempts our own process (the caller holds the lock for its
+ * whole run; without the exemption every run would refuse its own cleanup).
+ * The container check has NO exemption: a stolen lock (the #1297 suspected
+ * path) makes the lock file say "ours" while another run's containers are
+ * still up, so only Docker itself can tell that truth.
+ */
+export function findLiveE2eStack(
+  docker: DockerReader = realDocker,
+  lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
+): LiveStackRefusal | null {
+  const status = lock();
+  if (!status.free && status.holder && !isOwnHolder(status.holder)) {
+    const h = status.holder;
+    const heartbeat =
+      h.state === 'running' ? `, heartbeat ${formatAge(heartbeatAgeMs(h))} old` : '';
+    return makeRefusal(
+      `the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
+      [],
+      h,
+    );
+  }
+
+  const out = docker([
+    'ps',
+    '-a',
+    '--filter',
+    'name=celilo-e2e',
+    '--format',
+    '{{.Names}}\t{{.State}}\t{{.CreatedAt}}',
+  ]);
+  const live = out
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [name, state, createdAt] = line.split('\t');
+      return { name: name ?? '', state: state ?? '', createdAt: parseDockerCreatedAt(createdAt) };
+    })
+    .filter((c) => c.name !== '' && !DEAD_STATES.has(c.state));
+  if (live.length === 0) return null;
+
+  // A per-test container (or a guest, which carries no project name at all)
+  // means a run owns this host. Refuse regardless of the shared stack — this
+  // is the #1297 protection, unchanged.
+  const runOwned = live.filter((c) => !isSharedStackContainer(c.name));
+  if (runOwned.length > 0) {
+    return makeRefusal(
+      `${live.length} live celilo-e2e-* container(s):\n${live.map((c) => `  - ${c.name}`).join('\n')}\nA live stack owns these. If the run is gone and the stack is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}\n(there may be no cele2e binary outside a checkout — celilo#1314)`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  // celilo#1314: ONLY shared-stack containers are live. Refusing here
+  // unconditionally is what wedged the builder — an orphaned shared stack is
+  // the one state the guard could never resolve. Apply the orphan evidence
+  // test instead. Any inconclusive answer refuses.
+  const runners = processes();
+  if (runners.length > 0) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but a run process is still alive (pid ${runners.join(', ')}) — it may be between suites and about to use the stack. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  const youngest = Math.min(
+    ...live.map((c) => {
+      if (c.createdAt === null) return Number.NaN;
+      return c.createdAt.getTime();
+    }),
+  );
+  if (!Number.isFinite(youngest)) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.map((c) => c.name).join(', ')}), but its container age cannot be read from docker — the orphan evidence is inconclusive. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+  const ageMs = Date.now() - youngest;
+  if (ageMs < SHARED_ORPHAN_MIN_AGE_MS) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but its youngest container is only ${formatAge(ageMs)} old — under the ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m orphan threshold, so it may belong to a run the probe cannot see. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  // Every piece of run evidence is absent, and the stack is old enough that
+  // no live run can be hiding from the probe. Provably dead: reap it.
+  return null;
+}
+
+/**
+ * The startup-cleanup boundary: refuse with exit 3 when a live stack is in
+ * the way, return when the caller may remove. Exiting here (rather than
+ * throwing) is what makes the refusal survive every caller shape: the run
+ * path's bun test child, `cele2e up`, and the build paths all surface a
+ * process exit code without each one needing its own handling.
+ */
+export function refuseOnLiveStack(
+  docker: DockerReader = realDocker,
+  lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
+): void {
+  const refusal = findLiveE2eStack(docker, lock, processes);
+  if (!refusal) return;
+  console.error(`\n${refusal.reason}\n`);
+  process.exit(3);
+}