@celilo/e2e 0.20.0 → 0.20.2

package/src/live-stack.test.ts

@@ -3,7 +3,17 @@ import { readFileSync } from 'node:fs';
 import { hostname } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { type DockerReader, LiveStackError, findLiveE2eStack } from './live-stack';
+import {
+  type DockerReader,
+  LiveStackError,
+  type ProcessProbe,
+  SHARED_ORPHAN_MIN_AGE_MS,
+  UNIT_ONLY_ENV_ENTRY,
+  environMarksUnitOnly,
+  findLiveE2eStack,
+  isUnitOnlyProcess,
+  looksLikeE2eRunCommand,
+} from './live-stack';
 import type { LockStatus } from './run-lock';
 import { startupCleanup } from './shared-infra';
 
@@ -67,11 +77,24 @@ describe('findLiveE2eStack', () => {
     const { docker } = fakeDocker(
       'celilo-e2e-shared_namecheap-dns\trunning\ncelilo-e2e-1788769175864_fw-main\texited\n',
     );
-    const refusal = findLiveE2eStack(docker, noLock);
+    // The default probe reads the REAL host process table. Under another
+    // session's live e2e run (celilo#1332: the bun 1.4.2 grind) the
+    // shared-only + live-runner branch fires instead of the unreadable-age
+    // branch pinned here, and the refusal names pid counts instead of the
+    // container. A test pinning a message shape injects its inputs: an empty
+    // probe makes the pinned shape deterministic.
+    const refusal = findLiveE2eStack(docker, noLock, () => []);
     expect(refusal).not.toBeNull();
     expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns');
     expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main');
-    expect(refusal?.reason).toContain('cele2e down');
+    // The remedy must be a command that EXISTS on the host printing the
+    // message (celilo#1314): a bare `cele2e down` does not — forgejo job
+    // workspaces are ephemeral, so the binary only exists inside a checkout.
+    expect(refusal?.reason).toContain('docker rm -f');
+    expect(refusal?.reason).not.toContain('cele2e down');
+    // The refusal must be greppable as an environment problem, not a check
+    // failure (celilo#1314 direction 3).
+    expect(refusal?.reason).toContain('[infra-refusal]');
     expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_namecheap-dns']);
   });
 
@@ -131,6 +154,150 @@ describe('findLiveE2eStack', () => {
   });
 });
 
+describe('unit-only exclusion (celilo#1320)', () => {
+  const environOf =
+    (byPid: Record<number, string | null>) =>
+    (pid: number): string | null =>
+      byPid[pid] ?? null;
+
+  test('an environ carrying the marker is unit-only', () => {
+    expect(
+      environMarksUnitOnly(['PATH=/usr/bin', UNIT_ONLY_ENV_ENTRY, 'HOME=/root'].join('\0')),
+    ).toBe(true);
+  });
+
+  test('the marker must be a whole entry, not a substring', () => {
+    expect(environMarksUnitOnly(['PATH=/x', 'SOMETHING_CELILO_UNIT_ONLY=12'].join('\0'))).toBe(
+      false,
+    );
+    expect(environMarksUnitOnly('CELILO_UNIT_ONLY=12')).toBe(false);
+  });
+
+  test('an unreadable environ is NOT unit-only (inconclusive never reaps)', () => {
+    expect(environMarksUnitOnly(null)).toBe(false);
+  });
+
+  test('isUnitOnlyProcess reads the environ of the matched pid', () => {
+    const environ = environOf({
+      101: ['PATH=/x', UNIT_ONLY_ENV_ENTRY].join('\0'),
+      102: 'PATH=/x',
+      103: null,
+    });
+    expect(isUnitOnlyProcess(101, environ)).toBe(true);
+    expect(isUnitOnlyProcess(102, environ)).toBe(false);
+    expect(isUnitOnlyProcess(103, environ)).toBe(false);
+  });
+
+  test('the marker matches what test:unit actually exports', () => {
+    const rootPackageJson = JSON.parse(
+      readFileSync(join(DIR, '../../../package.json'), 'utf-8'),
+    ) as { scripts: Record<string, string> };
+    expect(rootPackageJson.scripts['test:unit']).toContain('CELILO_UNIT_ONLY=1');
+  });
+});
+
+describe('looksLikeE2eRunCommand', () => {
+  test('matches the cele2e CLI by any argv mention', () => {
+    expect(looksLikeE2eRunCommand('cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('./node_modules/.bin/cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/bin/cele2e.ts run --all')).toBe(true);
+  });
+
+  test('matches a direct bun test of this package, which has no cele2e in argv', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/e2e/tests/dns-replication.test.ts')).toBe(
+      true,
+    );
+    expect(looksLikeE2eRunCommand('bun test e2e/tests/smoke.test.ts')).toBe(true);
+  });
+
+  test('does not match unrelated bun test runs, editors, or scripts', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/mcp-server/src/tools.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('vi packages/e2e/src/live-stack.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/scripts/pack-celilo-packages.ts')).toBe(
+      false,
+    );
+    expect(looksLikeE2eRunCommand('bun build apps/celilo/src/index.ts')).toBe(false);
+  });
+});
+
+describe('shared-only orphan reap (celilo#1314)', () => {
+  // Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC".
+  const dockerAge = (msAgo: number): string => {
+    const d = new Date(Date.now() - msAgo);
+    return `${d.toISOString().slice(0, 10)} ${d.toISOString().slice(11, 19)} +0000 UTC`;
+  };
+  const noProcesses: ProcessProbe = () => [];
+
+  test('REAPS the exact shape measured on the builder: shared-only stack, 11h old, no per-test project, no cele2e process', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-shared_registry-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    // No containers at all is NOT the test: the guard must see the live shared
+    // containers and STILL clear the way, because every piece of run evidence
+    // is absent.
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+    startupCleanup('/tmp/e2e', docker, noLock, noProcesses);
+    // The reap is a real teardown, not a refusal: the shared compose down ran.
+    expect(commands.some((c) => c.includes(' down '))).toBe(true);
+  });
+
+  test('must NOT reap a shared stack with a live per-test project beside it (the celilo#1297 shape)', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-1788769175864_fw-main\trunning\t${dockerAge(5 * 60_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.runningContainers).toContain('celilo-e2e-shared_namecheap-dns-1');
+    expect(refusal?.runningContainers).toContain('celilo-e2e-1788769175864_fw-main');
+    expect(() => startupCleanup('/tmp/e2e', docker, noLock, noProcesses)).toThrow(LiveStackError);
+    expect(commands.some((c) => c.includes(' down '))).toBe(false);
+  });
+
+  test('must NOT reap while any other cele2e process is alive — a run may be between suites', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const oneRunner: ProcessProbe = () => [424242];
+    const refusal = findLiveE2eStack(docker, noLock, oneRunner);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('424242');
+  });
+
+  test(`must NOT reap a stack younger than ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m — the evidence is not old enough to be conclusive`, () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(5 * 60_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('5m');
+  });
+
+  test('must NOT reap when the age cannot be read — inconclusive evidence refuses, it never reaps', () => {
+    // Two fields, no CreatedAt: the pre-#1314 fake shape, and anything docker
+    // might print that the parser does not recognize.
+    const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('age');
+  });
+
+  test('a foreign fresh lock still refuses a shared-only orphan — the reap never overrides the lock', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, foreignLock(), noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('polecat/ce-9999');
+  });
+
+  test('exited containers of the shared project alone still proceed (unchanged)', () => {
+    const ps = `celilo-e2e-shared_registry-1\texited\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+  });
+});
+
 describe('startupCleanup', () => {
   test('refuses behind a live stack and removes NOTHING (celilo#1297)', () => {
     const { docker, commands } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
@@ -171,7 +338,7 @@ describe('wiring in shared-infra.ts', () => {
       SHARED_INFRA_SRC.indexOf('export function startupCleanup'),
       SHARED_INFRA_SRC.indexOf('nukeE2eResources(e2eDir, docker)'),
     );
-    expect(cleanupBody).toContain('findLiveE2eStack(docker, lock)');
+    expect(cleanupBody).toContain('findLiveE2eStack(docker, lock, processes)');
   });
 
   test('the DNS-restart branch guards before tearing down the running stack', () => {

package/src/live-stack.ts

@@ -19,10 +19,43 @@
  * Deliberately NOT guarded: the runner's end-of-run stopSharedInfra teardown.
  * The owner tearing down its own live stack is the normal exit path, and the
  * guard's container check would refuse it by definition.
+ *
+ * The shared-stack orphan reap (celilo#1314). The #1297 guard is keyed on
+ * "any live celilo-e2e-* container", and the shared stack is designed never to
+ * be touched by name-protecting cleanup — so one abnormal exit wedges the
+ * host permanently: the orphaned shared stack refuses every later cleanup, it
+ * is the one thing the guard structurally cannot resolve. Measured on the
+ * builder 2026-09-08: 13 celilo-e2e-shared-* containers, 11h old, no per-test
+ * project, no cele2e process, and every npm-consumer-smoke run red on a check
+ * that never executed. So a stack whose ONLY live containers belong to the
+ * shared project gets an evidence test instead of a blanket refusal: no
+ * foreign lock, no live run process, and an age past a threshold means the
+ * stack is garbage and cleanup may act. Any single piece of run evidence
+ * present, or any age that cannot be read, refuses — inconclusive evidence
+ * never reaps, because a false reap kills a real run while a false refusal
+ * costs the operator one docker command.
+ *
+ * The refusal itself names a command that exists on the host printing it (a
+ * bare `cele2e down` does not: forgejo job workspaces are ephemeral, so the
+ * binary only exists inside a checkout), and every refusal reason carries the
+ * `[infra-refusal]` marker with its exit code, so a log search distinguishes
+ * an environment problem from a check failure (celilo#1314 directions 2+3).
+ *
+ * The unit-only exclusion (celilo#1320). ci/validate runs `bun run test:unit`
+ * on the same persistent builder as npm-consumer-smoke, and every `bun test`
+ * child it spawns matched the probe's bun-test arm while managing no Docker
+ * at all — the smoke job's teardown then declined to clear a finished run's
+ * shared stack, and the next run inside the orphan hour refused. A process
+ * whose environ carries CELILO_UNIT_ONLY=1 (the root script exports it; every
+ * child inherits) is a unit-test run and is not run evidence. An unreadable
+ * environ keeps the match: the exclusion follows the same bias as the
+ * evidence test itself, inconclusive never reaps.
  */
 
 import type { ExecFileSyncOptions } from 'node:child_process';
 import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { SHARED_PROJECT_NAME } from './docker-compose-generator';
 import {
   type LockHolder,
   type LockStatus,
@@ -65,6 +98,193 @@ export class LiveStackError extends Error {
   }
 }
 
+/**
+ * How old the youngest live shared-stack container must be before a
+ * shared-only stack with no other run evidence is treated as an orphan.
+ * One hour sits far above any window in which a run process could have died
+ * without its containers dying too, and far below the 11h the builder sat
+ * wedged (celilo#1314). A stack younger than this refuses even with no other
+ * evidence: the reap is the dangerous direction, so it waits for certainty.
+ */
+export const SHARED_ORPHAN_MIN_AGE_MS = 60 * 60_000;
+
+/**
+ * Live pids of processes that may own an e2e run. Injectable so the reap's
+ * evidence test is unit-testable without a process table.
+ */
+export type ProcessProbe = () => number[];
+
+/**
+ * The env entry `bun run test:unit` exports (root package.json). A process
+ * carrying it is a unit-test run: the e2e unit suites exercise Docker only
+ * through injected readers, so it never manages the shared stack.
+ */
+export const UNIT_ONLY_ENV_ENTRY = 'CELILO_UNIT_ONLY=1';
+
+/**
+ * Raw environment of one pid (NUL-separated entries), or null when it cannot
+ * be read — /proc is Linux-only, and a foreign-uid environ is unreadable even
+ * where /proc exists.
+ */
+export type EnvironReader = (pid: number) => string | null;
+
+export const realEnvironReader: EnvironReader = (pid) => {
+  try {
+    return readFileSync(`/proc/${pid}/environ`, 'utf8');
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * Does a raw environ block mark its process as unit-only? Pure so the
+ * probe's exclusion is testable without a filesystem. A null (unreadable)
+ * environ is NOT unit-only: an unreadable answer is a missing answer, and the
+ * probe biases to refusing — a false refusal costs the operator one docker
+ * command, a false miss reaps a live run.
+ */
+export function environMarksUnitOnly(environ: string | null): boolean {
+  if (environ === null) return false;
+  return environ.split('\0').includes(UNIT_ONLY_ENV_ENTRY);
+}
+
+/** The exclusion realProcessProbe applies to its matches; injectable for tests. */
+export function isUnitOnlyProcess(
+  pid: number,
+  environ: EnvironReader = realEnvironReader,
+): boolean {
+  return environMarksUnitOnly(environ(pid));
+}
+
+/**
+ * Does this process command line look like an e2e run? Pure so the probe's
+ * reach is testable. Matches the two ways a run actually exists:
+ *   - the cele2e CLI and anything whose argv names it (`cele2e run|up|down|...`)
+ *   - a direct `bun test` of this package's suites, which manages the shared
+ *     stack through ensureSharedInfra but has no cele2e in argv
+ * The bun-test arm requires bun AND e2e AND test in the command, so an
+ * unrelated `bun test` elsewhere only false-matches when its path names e2e —
+ * and a false match refuses (cheap), where a false miss reaps a live run
+ * (expensive). An editor with an e2e test file open does not match: it does
+ * not start with a bun invocation.
+ */
+export function looksLikeE2eRunCommand(command: string): boolean {
+  if (command.includes('cele2e')) return true;
+  return (
+    /(^|[\\/])bun(\.exe)?\s/.test(command) && command.includes('e2e') && command.includes('test')
+  );
+}
+
+/** pids of this process's ancestors, self included, bounded at 64 levels. */
+function familyOfSelf(): Set<number> {
+  const family = new Set<number>([process.pid]);
+  let pid: number | undefined = process.ppid;
+  for (let i = 0; pid !== undefined && pid > 1 && i < 64; i++) {
+    family.add(pid);
+    try {
+      const out = execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+      const ppid = Number.parseInt(out, 10);
+      pid = Number.isFinite(ppid) ? ppid : undefined;
+    } catch {
+      pid = undefined;
+    }
+  }
+  return family;
+}
+
+/**
+ * The real probe: one `ps` listing, filtered by looksLikeE2eRunCommand, with
+ * this process and its ancestry removed — the cleanup runs INSIDE the run it
+ * would otherwise see as evidence, and the run's own launcher shell carries
+ * the same strings in its argv.
+ */
+export const realProcessProbe: ProcessProbe = (): number[] => {
+  let listing: string;
+  try {
+    listing = execFileSync('ps', ['-axo', 'pid=,command='], {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    // A process table that cannot be read is missing evidence. The caller
+    // treats any probe failure conservatively; here that means reporting no
+    // matches, which the shared-only path then backstops with the age
+    // threshold and the unreadable-age refusal.
+    return [];
+  }
+  const family = familyOfSelf();
+  const pids: number[] = [];
+  for (const line of listing.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const sep = trimmed.indexOf(' ');
+    if (sep <= 0) continue;
+    const pid = Number.parseInt(trimmed.slice(0, sep), 10);
+    if (!Number.isFinite(pid) || family.has(pid)) continue;
+    if (!looksLikeE2eRunCommand(trimmed.slice(sep + 1))) continue;
+    // celilo#1320: a unit-test run matches the command probe but manages no
+    // Docker. An unreadable environ keeps the match — inconclusive never
+    // reaps.
+    if (isUnitOnlyProcess(pid)) continue;
+    pids.push(pid);
+  }
+  return pids;
+};
+
+/**
+ * Every refusal goes through here so the printed reason is uniformly
+ * greppable as an infrastructure refusal (celilo#1314 direction 3) rather
+ * than reading as a check failure.
+ */
+function makeRefusal(
+  detail: string,
+  runningContainers: string[],
+  holder: LockHolder | null,
+): LiveStackRefusal {
+  const reason = `[infra-refusal] refusing to clean up: ${detail}\n(this is an environment problem, not a test failure — the run exits 3)`;
+  return { reason, runningContainers, holder };
+}
+
+/**
+ * The remedy printed with a live-stack refusal. A raw docker removal, because
+ * `cele2e down` does not exist on the hosts that print this message: forgejo
+ * job workspaces are ephemeral, so the binary only exists inside a checkout
+ * (celilo#1314 direction 2). Removing the containers is what unblocks the
+ * guard; the next cleanup sweeps whatever name-prefix resources survive.
+ */
+export const CLEAR_STACK_COMMAND = 'docker rm -f $(docker ps -aq --filter name=celilo-e2e)';
+
+/** Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC". */
+function parseDockerCreatedAt(raw: string | undefined): Date | null {
+  if (raw === undefined) return null;
+  const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.(\d+))? ([+-])(\d{2})(\d{2})/.exec(
+    raw.trim(),
+  );
+  if (!m) return null;
+  const [, date, time, frac, sign, offH, offM] = m;
+  const utcMs = Date.parse(`${date}T${time}Z`);
+  if (Number.isNaN(utcMs)) return null;
+  const offsetMs = (sign === '-' ? -1 : 1) * (Number(offH) * 60 + Number(offM)) * 60_000;
+  const fracMs = frac ? Number(frac.padEnd(3, '0').slice(0, 3)) : 0;
+  return new Date(utcMs - offsetMs + fracMs);
+}
+
+/**
+ * Containers of the shared compose project. Compose names them
+ * `<project>_<service>_<n>` (or with `-` separators on newer compose), so the
+ * character right after the project name is the tell. Everything else live —
+ * per-test projects AND sim-created guests, whose names carry no project —
+ * is run evidence.
+ */
+function isSharedStackContainer(name: string): boolean {
+  if (!name.startsWith(SHARED_PROJECT_NAME)) return false;
+  const sep = name[SHARED_PROJECT_NAME.length];
+  return sep === '_' || sep === '-';
+}
+
 /**
  * Container states that mean the container cannot be holding a live stack.
  * Everything else (running, paused, restarting) is live: paused containers
@@ -90,17 +310,18 @@ function isOwnHolder(h: LockHolder): boolean {
 export function findLiveE2eStack(
   docker: DockerReader = realDocker,
   lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
 ): LiveStackRefusal | null {
   const status = lock();
   if (!status.free && status.holder && !isOwnHolder(status.holder)) {
     const h = status.holder;
     const heartbeat =
       h.state === 'running' ? `, heartbeat ${formatAge(heartbeatAgeMs(h))} old` : '';
-    return {
-      reason: `refusing to clean up: the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
-      runningContainers: [],
-      holder: h,
-    };
+    return makeRefusal(
+      `the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
+      [],
+      h,
+    );
   }
 
   const out = docker([
@@ -109,21 +330,67 @@ export function findLiveE2eStack(
     '--filter',
     'name=celilo-e2e',
     '--format',
-    '{{.Names}}\t{{.State}}',
+    '{{.Names}}\t{{.State}}\t{{.CreatedAt}}',
   ]);
-  const running = out
+  const live = out
     .split('\n')
     .filter(Boolean)
-    .map((line) => line.split('\t'))
-    .filter((parts) => parts.length === 2 && !DEAD_STATES.has(parts[1] ?? ''))
-    .map((parts) => parts[0] ?? '');
-  if (running.length > 0) {
-    return {
-      reason: `refusing to clean up: ${running.length} live celilo-e2e-* container(s):\n${running.map((n) => `  - ${n}`).join('\n')}\nA live stack owns these. Clear it with: cele2e down`,
-      runningContainers: running,
-      holder: null,
-    };
+    .map((line) => {
+      const [name, state, createdAt] = line.split('\t');
+      return { name: name ?? '', state: state ?? '', createdAt: parseDockerCreatedAt(createdAt) };
+    })
+    .filter((c) => c.name !== '' && !DEAD_STATES.has(c.state));
+  if (live.length === 0) return null;
+
+  // A per-test container (or a guest, which carries no project name at all)
+  // means a run owns this host. Refuse regardless of the shared stack — this
+  // is the #1297 protection, unchanged.
+  const runOwned = live.filter((c) => !isSharedStackContainer(c.name));
+  if (runOwned.length > 0) {
+    return makeRefusal(
+      `${live.length} live celilo-e2e-* container(s):\n${live.map((c) => `  - ${c.name}`).join('\n')}\nA live stack owns these. If the run is gone and the stack is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}\n(there may be no cele2e binary outside a checkout — celilo#1314)`,
+      live.map((c) => c.name),
+      null,
+    );
   }
+
+  // celilo#1314: ONLY shared-stack containers are live. Refusing here
+  // unconditionally is what wedged the builder — an orphaned shared stack is
+  // the one state the guard could never resolve. Apply the orphan evidence
+  // test instead. Any inconclusive answer refuses.
+  const runners = processes();
+  if (runners.length > 0) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but a run process is still alive (pid ${runners.join(', ')}) — it may be between suites and about to use the stack. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  const youngest = Math.min(
+    ...live.map((c) => {
+      if (c.createdAt === null) return Number.NaN;
+      return c.createdAt.getTime();
+    }),
+  );
+  if (!Number.isFinite(youngest)) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.map((c) => c.name).join(', ')}), but its container age cannot be read from docker — the orphan evidence is inconclusive. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+  const ageMs = Date.now() - youngest;
+  if (ageMs < SHARED_ORPHAN_MIN_AGE_MS) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but its youngest container is only ${formatAge(ageMs)} old — under the ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m orphan threshold, so it may belong to a run the probe cannot see. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  // Every piece of run evidence is absent, and the stack is old enough that
+  // no live run can be hiding from the probe. Provably dead: reap it.
   return null;
 }
 
@@ -137,8 +404,9 @@ export function findLiveE2eStack(
 export function refuseOnLiveStack(
   docker: DockerReader = realDocker,
   lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
 ): void {
-  const refusal = findLiveE2eStack(docker, lock);
+  const refusal = findLiveE2eStack(docker, lock, processes);
   if (!refusal) return;
   console.error(`\n${refusal.reason}\n`);
   process.exit(3);

package/src/module-host.test.ts

@@ -8,7 +8,7 @@
  */
 
 import { describe, expect, test } from 'bun:test';
-import { parseModuleHost } from './module-host';
+import { parseModuleHost, parseModuleWhere } from './module-host';
 
 /** How the CLI really prints it — clack gutter and ANSI included. */
 const withGutter = (line: string) => `\x1b[1mModule: caddy\x1b[0m\n│  Placement:\n│  ${line}\n`;
@@ -69,3 +69,57 @@ describe('parseModuleHost', () => {
     expect(parseModuleHost(output)?.hostname).toBe('web');
   });
 });
+
+describe('parseModuleWhere', () => {
+  test('the inventory payload yields the recorded address', () => {
+    const payload = JSON.stringify({
+      module: 'caddy-internal',
+      systems: [
+        {
+          name: 'main',
+          hostname: 'caddy-internal',
+          ipv4_address: '10.226.30.14',
+          zone: 'dmz',
+          vmid: 2201,
+          infra_type: 'container_service',
+          placement: 'caddy-internal (vmid 2201) → pve1 (zone dmz)',
+          reachability: "firewall-segmented (dmz) — reach via the firewall's natIp DNAT",
+        },
+      ],
+    });
+
+    expect(parseModuleWhere(payload)).toEqual(['10.226.30.14']);
+  });
+
+  test('ANSI and whitespace around the payload do not break it', () => {
+    // The success path prints verbatim (celilo#698), but dev-mode shims have
+    // surprised us before, so the parse tolerates decoration.
+    const payload = `\x1b[1m${JSON.stringify({ systems: [{ ipv4_address: '10.226.30.15' }] })}\x1b[0m\n`;
+
+    expect(parseModuleWhere(payload)).toEqual(['10.226.30.15']);
+  });
+
+  test('a module with several systems yields them all, caller picks', () => {
+    const payload = JSON.stringify({
+      systems: [
+        { name: 'main', ipv4_address: '10.226.20.9' },
+        { name: 'replica', ipv4_address: '10.226.20.10' },
+      ],
+    });
+
+    expect(parseModuleWhere(payload)).toEqual(['10.226.20.9', '10.226.20.10']);
+  });
+
+  test('an API-only module yields an empty list, a real state', () => {
+    // namecheap has no host. Empty is not a parse failure — the caller decides
+    // whether "no recorded address" is expected for their module.
+    expect(parseModuleWhere(JSON.stringify({ systems: [] }))).toEqual([]);
+  });
+
+  test('a non-JSON answer yields an empty list, not a throw', () => {
+    // Crash text on stdout is a CLI failure and the harness method attaches
+    // the raw output to its error; the parser's contract is "addresses found",
+    // not "reasons why not".
+    expect(parseModuleWhere('Error: module not found: caddy\n')).toEqual([]);
+  });
+});

package/src/module-host.ts

@@ -70,3 +70,34 @@ export function parseModuleHost(statusOutput: string): ModuleHost | null {
 
   return null;
 }
+
+/**
+ * The address(es) a module's deploy recorded, from `celilo module where --json`.
+ *
+ * The old source was `grep target_ip` over the module's `generated/` tree
+ * (celilo#1334). D4 of control-plane-stops-building-modules makes that tree
+ * ephemeral — a successful deploy deletes it — so the inventory is where the
+ * address lives now: `module_systems` is written by the deploy itself and
+ * survives it. Every current module declares exactly one system, so callers
+ * take the first address; a module with none is an API-only module or an
+ * undeployed one, and that distinction belongs to the caller.
+ */
+export function parseModuleWhere(whereOutput: string): string[] {
+  // The success path prints the payload verbatim (celilo#698), but defensive
+  // ANSI stripping costs one line and dev-mode shims have surprised us before.
+  const stripped = whereOutput.replace(/\x1b\[[0-9;]*m/g, '').trim();
+
+  let parsed: { systems?: { ipv4_address?: string }[] };
+  try {
+    parsed = JSON.parse(stripped) as { systems?: { ipv4_address?: string }[] };
+  } catch {
+    // A non-JSON answer is a CLI failure (version drift, crash text) arriving
+    // on stdout. An empty list reads as "no systems", which is a real state a
+    // caller may legitimately handle — these are not the same problem.
+    return [];
+  }
+
+  return (parsed.systems ?? [])
+    .map((system) => system.ipv4_address ?? '')
+    .filter((address) => address !== '');
+}