@celilo/e2e 0.20.0 → 0.20.1

package/bin/e2e-bake-management

@@ -33,6 +33,7 @@ import {
   stageWebsiteDist,
 } from '../src/stage-simulator-inputs';
 import {
+  CONSUMER_FINGERPRINT_PREFIX,
   PUBLISHED_FINGERPRINT_PREFIX,
   SOURCE_LABEL,
 } from '../src/source-fingerprint';
@@ -209,8 +210,25 @@ function simStampFromManifest(): string {
  * whether the CLI under test is the code in the working tree. See
  * `src/source-fingerprint.ts` for why that has to be measured, not remembered.
  */
+/**
+ * `celilo --version` prints "celilo <semver>", and `docker commit --change
+ * "LABEL name=value"` splits on whitespace, so stamping the raw output fails
+ * the commit with `Syntax error - can't find = in "2.2.0"`. Take the last
+ * field, so the label carries the version alone (celilo#1318).
+ */
+function labelSafeVersion(version: string): string {
+  return version.trim().split(/\s+/).pop() || 'unknown';
+}
+
 function sourceStamp(published: boolean, version: string): string {
-  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${version || 'unknown'}`;
+  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
+  // Consumer mode: restageSimulatorInputs is the only writer of
+  // pack-manifest.json and it is skipped without a checkout, so there is no
+  // tree fingerprint to read — and none to record, since the tree does not
+  // exist. Stamp the version the bake just installed and verified (celilo#1318).
+  if (!findMonorepoRoot(PACKAGE_ROOT)) {
+    return `${CONSUMER_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
+  }
   return simStampFromManifest();
 }
 
@@ -229,9 +247,16 @@ function sourceStamp(published: boolean, version: string): string {
 function restageSimulatorInputs(): void {
   const repoRoot = findMonorepoRoot(PACKAGE_ROOT);
   if (!repoRoot) {
-    throw new Error(
-      'The default bake installs the monorepo DEV cli via the sim registry — it needs a celilo checkout. For a published-CLI bake use --published.',
-    );
+    // Consumer mode (no checkout): there is no tree to repack, and nothing to
+    // correct. stageFromPublic already fetched THIS run's tarballs and site
+    // dist into the sim caches, and the sim images were built from them before
+    // the bake started. celilo#1299 is a monorepo-mode hazard — there the bake
+    // would otherwise reinstall the previous build's cached CLI — so the
+    // restage is redundant here, not impossible-but-required. Throwing broke
+    // npm-consumer-smoke, whose whole point is a bake with no monorepo source
+    // (celilo#1318).
+    console.log('  restage ................ skipped (consumer mode: inputs staged this run)');
+    return;
   }
   stageWebsiteDist(repoRoot, PACKAGE_ROOT);
   packNpmRegistryTarballs(repoRoot, PACKAGE_ROOT);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.20.0",
+  "version": "0.20.1",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",

package/src/cli/build.test.ts

@@ -1,10 +1,20 @@
 import { afterEach, beforeEach, expect, test } from 'bun:test';
-import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { gzipSync } from 'node:zlib';
 import {
   assertGzipValid,
+  bakeManagement,
+  reportBakeChildFailure,
   stageNetappsFromRegistry,
   verifyNetapp,
   verifyStagedNetapps,
@@ -95,3 +105,74 @@ test('verifyStagedNetapps accepts a valid .netapp and refuses a truncated one',
   expect(() => verifyNetapp(join(dir, 'technitium.netapp'))).toThrow(/technitium\.netapp/s);
   expect(() => assertGzipValid(join(dir, 'technitium.netapp'))).toThrow(/unexpected end of file/s);
 });
+
+/**
+ * Stub process.exit and console.error around a call expected to terminate.
+ * Returns the exit codes collected and everything written to stderr.
+ */
+function captureTerminalExit(run: () => void): { exitCodes: unknown[]; stderr: string } {
+  const realExit = process.exit;
+  const realError = console.error;
+  const exitCodes: unknown[] = [];
+  const stderrLines: string[] = [];
+  (process as unknown as { exit: (code?: number) => never }).exit = ((code?: number) => {
+    exitCodes.push(code);
+    throw new Error(`process.exit(${code})`);
+  }) as typeof process.exit;
+  console.error = (...args: unknown[]) => {
+    stderrLines.push(args.map(String).join(' '));
+  };
+  try {
+    try {
+      run();
+    } catch (err) {
+      // The stubbed exit throws to unwind; anything else is a real failure.
+      if (!String(err).startsWith('Error: process.exit(')) throw err;
+    }
+  } finally {
+    process.exit = realExit;
+    console.error = realError;
+  }
+  return { exitCodes, stderr: stderrLines.join('\n') };
+}
+
+// celilo#1302: a lock-free live stack on the builder made the bake child's
+// startup cleanup refuse (exit 3), and the parent reported "Bake step failed"
+// with three install.sh causes for a step that never executed.
+test('a refusal exit (3) from the bake child reports the bake did not run, not a bake failure', () => {
+  const { exitCodes, stderr } = captureTerminalExit(() => reportBakeChildFailure(3, 0));
+  expect(exitCodes).toEqual([3]);
+  expect(stderr).toContain('did not run');
+  expect(stderr).toContain('cele2e down');
+  // The wrong hint must not appear: these causes are all false when the bake
+  // never started.
+  expect(stderr).not.toContain('Likely causes');
+  expect(stderr).not.toContain('install.sh regressed');
+});
+
+test('a genuine bake failure (exit 1) keeps the bake-failure report and its likely causes', () => {
+  const { exitCodes, stderr } = captureTerminalExit(() => reportBakeChildFailure(1, 12));
+  expect(exitCodes).toEqual([1]);
+  expect(stderr).toContain('Bake step failed after 12s');
+  expect(stderr).toContain('Likely causes');
+  expect(stderr).not.toContain('did not run');
+});
+
+test('bakeManagement classifies a real child exit 3 end to end and exits 3', () => {
+  // A fake bake script that behaves like the real one does when the startup
+  // cleanup refuses: print the refusal to stderr, exit 3.
+  const pkgDir = mkdtempSync(join(tmpdir(), 'bake-child-'));
+  mkdirSync(join(pkgDir, 'bin'), { recursive: true });
+  writeFileSync(
+    join(pkgDir, 'bin', 'e2e-bake-management'),
+    'console.error("refusing to clean up"); process.exit(3);\n',
+  );
+  try {
+    const { exitCodes, stderr } = captureTerminalExit(() => bakeManagement(pkgDir, false));
+    expect(exitCodes).toEqual([3]);
+    expect(stderr).toContain('did not run');
+    expect(stderr).not.toContain('Likely causes');
+  } finally {
+    rmSync(pkgDir, { recursive: true, force: true });
+  }
+});

package/src/cli/build.ts

@@ -684,7 +684,7 @@ function tagVanillaManagement(): void {
  * build-infra so the operator can't accidentally proceed with a
  * non-functional default management image.
  */
-function bakeManagement(pkgDir: string, published: boolean): void {
+export function bakeManagement(pkgDir: string, published: boolean): void {
   const bakeScript = join(pkgDir, 'bin', 'e2e-bake-management');
   if (!existsSync(bakeScript)) {
     console.error(
@@ -705,16 +705,43 @@ function bakeManagement(pkgDir: string, published: boolean): void {
   const elapsed = Math.round((Date.now() - start) / 1000);
 
   if (result.status !== 0) {
-    console.error(`\n${red}Bake step failed after ${elapsed}s${reset}`);
-    console.error(
-      `${dim}Likely causes: install.sh regressed, npm-registry-sim doesn't have the expected @celilo/* tarballs, or celilo-website-sim isn't serving install.sh. Run \`cele2e run install-sh\` for a focused reproducer with cleaner output.${reset}`,
-    );
-    process.exit(1);
+    console.log(`${red}✗${reset}  ${dim}${elapsed}s${reset}`);
+    // status is null when the child died to a signal — that is a genuine
+    // failure, not a refusal, so it falls through to the exit-1 report.
+    reportBakeChildFailure(result.status ?? 1, elapsed);
   }
 
   console.log(`\n${green}Baked in ${elapsed}s${reset}\n`);
 }
 
+/**
+ * Report a non-zero bake-child exit and terminate. Exit 3 is the live-stack
+ * refusal convention (celilo#1297 guard; the run runner keys on the same code
+ * in its `refused` field): the bake child's startup cleanup found a live
+ * celilo-e2e-* stack or a foreign run lock and refused before the bake did
+ * any work. Reporting that as a bake failure named three causes — install.sh
+ * rot, registry tarballs, website sim — for a step that never executed
+ * (celilo#1302: 9 of ~111 smoke runs, every refusal diagnosed as install.sh
+ * rot). A refusal is not a bake failure, so it gets its own message and exits
+ * 3 so callers can tell the two apart.
+ */
+export function reportBakeChildFailure(status: number, elapsed: number): never {
+  if (status === 3) {
+    console.error(
+      `\n${red}Bake step did not run — the startup cleanup refused: a live e2e stack is in the way.${reset}`,
+    );
+    console.error(
+      `${dim}A refusal is not a bake failure; install.sh was never exercised. Clear the stack with \`cele2e down\` (or free the foreign run lock) and re-run \`cele2e build-infra\`.${reset}`,
+    );
+    process.exit(3);
+  }
+  console.error(`\n${red}Bake step failed after ${elapsed}s${reset}`);
+  console.error(
+    `${dim}Likely causes: install.sh regressed, npm-registry-sim doesn't have the expected @celilo/* tarballs, or celilo-website-sim isn't serving install.sh. Run \`cele2e run install-sh\` for a focused reproducer with cleaner output.${reset}`,
+  );
+  process.exit(1);
+}
+
 /**
  * Pre-seed heavy application images (authentik server, postgres, redis) into
  * the app-zone preload cache. Without this the `docker-image-cache` is empty on

package/src/live-stack.test.ts

@@ -3,7 +3,14 @@ import { readFileSync } from 'node:fs';
 import { hostname } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { type DockerReader, LiveStackError, findLiveE2eStack } from './live-stack';
+import {
+  type DockerReader,
+  LiveStackError,
+  type ProcessProbe,
+  SHARED_ORPHAN_MIN_AGE_MS,
+  findLiveE2eStack,
+  looksLikeE2eRunCommand,
+} from './live-stack';
 import type { LockStatus } from './run-lock';
 import { startupCleanup } from './shared-infra';
 
@@ -71,7 +78,14 @@ describe('findLiveE2eStack', () => {
     expect(refusal).not.toBeNull();
     expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns');
     expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main');
-    expect(refusal?.reason).toContain('cele2e down');
+    // The remedy must be a command that EXISTS on the host printing the
+    // message (celilo#1314): a bare `cele2e down` does not — forgejo job
+    // workspaces are ephemeral, so the binary only exists inside a checkout.
+    expect(refusal?.reason).toContain('docker rm -f');
+    expect(refusal?.reason).not.toContain('cele2e down');
+    // The refusal must be greppable as an environment problem, not a check
+    // failure (celilo#1314 direction 3).
+    expect(refusal?.reason).toContain('[infra-refusal]');
     expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_namecheap-dns']);
   });
 
@@ -131,6 +145,108 @@ describe('findLiveE2eStack', () => {
   });
 });
 
+describe('looksLikeE2eRunCommand', () => {
+  test('matches the cele2e CLI by any argv mention', () => {
+    expect(looksLikeE2eRunCommand('cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('./node_modules/.bin/cele2e run smoke')).toBe(true);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/bin/cele2e.ts run --all')).toBe(true);
+  });
+
+  test('matches a direct bun test of this package, which has no cele2e in argv', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/e2e/tests/dns-replication.test.ts')).toBe(
+      true,
+    );
+    expect(looksLikeE2eRunCommand('bun test e2e/tests/smoke.test.ts')).toBe(true);
+  });
+
+  test('does not match unrelated bun test runs, editors, or scripts', () => {
+    expect(looksLikeE2eRunCommand('bun test packages/mcp-server/src/tools.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('vi packages/e2e/src/live-stack.test.ts')).toBe(false);
+    expect(looksLikeE2eRunCommand('bun run packages/e2e/scripts/pack-celilo-packages.ts')).toBe(
+      false,
+    );
+    expect(looksLikeE2eRunCommand('bun build apps/celilo/src/index.ts')).toBe(false);
+  });
+});
+
+describe('shared-only orphan reap (celilo#1314)', () => {
+  // Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC".
+  const dockerAge = (msAgo: number): string => {
+    const d = new Date(Date.now() - msAgo);
+    return `${d.toISOString().slice(0, 10)} ${d.toISOString().slice(11, 19)} +0000 UTC`;
+  };
+  const noProcesses: ProcessProbe = () => [];
+
+  test('REAPS the exact shape measured on the builder: shared-only stack, 11h old, no per-test project, no cele2e process', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-shared_registry-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    // No containers at all is NOT the test: the guard must see the live shared
+    // containers and STILL clear the way, because every piece of run evidence
+    // is absent.
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+    startupCleanup('/tmp/e2e', docker, noLock, noProcesses);
+    // The reap is a real teardown, not a refusal: the shared compose down ran.
+    expect(commands.some((c) => c.includes(' down '))).toBe(true);
+  });
+
+  test('must NOT reap a shared stack with a live per-test project beside it (the celilo#1297 shape)', () => {
+    const ps = [
+      `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`,
+      `celilo-e2e-1788769175864_fw-main\trunning\t${dockerAge(5 * 60_000)}`,
+    ].join('\n');
+    const { docker, commands } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.runningContainers).toContain('celilo-e2e-shared_namecheap-dns-1');
+    expect(refusal?.runningContainers).toContain('celilo-e2e-1788769175864_fw-main');
+    expect(() => startupCleanup('/tmp/e2e', docker, noLock, noProcesses)).toThrow(LiveStackError);
+    expect(commands.some((c) => c.includes(' down '))).toBe(false);
+  });
+
+  test('must NOT reap while any other cele2e process is alive — a run may be between suites', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const oneRunner: ProcessProbe = () => [424242];
+    const refusal = findLiveE2eStack(docker, noLock, oneRunner);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('424242');
+  });
+
+  test(`must NOT reap a stack younger than ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m — the evidence is not old enough to be conclusive`, () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(5 * 60_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('5m');
+  });
+
+  test('must NOT reap when the age cannot be read — inconclusive evidence refuses, it never reaps', () => {
+    // Two fields, no CreatedAt: the pre-#1314 fake shape, and anything docker
+    // might print that the parser does not recognize.
+    const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
+    const refusal = findLiveE2eStack(docker, noLock, noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('age');
+  });
+
+  test('a foreign fresh lock still refuses a shared-only orphan — the reap never overrides the lock', () => {
+    const ps = `celilo-e2e-shared_namecheap-dns-1\trunning\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    const refusal = findLiveE2eStack(docker, foreignLock(), noProcesses);
+    expect(refusal).not.toBeNull();
+    expect(refusal?.reason).toContain('polecat/ce-9999');
+  });
+
+  test('exited containers of the shared project alone still proceed (unchanged)', () => {
+    const ps = `celilo-e2e-shared_registry-1\texited\t${dockerAge(11 * 3_600_000)}`;
+    const { docker } = fakeDocker(ps);
+    expect(findLiveE2eStack(docker, noLock, noProcesses)).toBeNull();
+  });
+});
+
 describe('startupCleanup', () => {
   test('refuses behind a live stack and removes NOTHING (celilo#1297)', () => {
     const { docker, commands } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
@@ -171,7 +287,7 @@ describe('wiring in shared-infra.ts', () => {
       SHARED_INFRA_SRC.indexOf('export function startupCleanup'),
       SHARED_INFRA_SRC.indexOf('nukeE2eResources(e2eDir, docker)'),
     );
-    expect(cleanupBody).toContain('findLiveE2eStack(docker, lock)');
+    expect(cleanupBody).toContain('findLiveE2eStack(docker, lock, processes)');
   });
 
   test('the DNS-restart branch guards before tearing down the running stack', () => {

package/src/live-stack.ts

@@ -19,10 +19,32 @@
  * Deliberately NOT guarded: the runner's end-of-run stopSharedInfra teardown.
  * The owner tearing down its own live stack is the normal exit path, and the
  * guard's container check would refuse it by definition.
+ *
+ * The shared-stack orphan reap (celilo#1314). The #1297 guard is keyed on
+ * "any live celilo-e2e-* container", and the shared stack is designed never to
+ * be touched by name-protecting cleanup — so one abnormal exit wedges the
+ * host permanently: the orphaned shared stack refuses every later cleanup, it
+ * is the one thing the guard structurally cannot resolve. Measured on the
+ * builder 2026-09-08: 13 celilo-e2e-shared-* containers, 11h old, no per-test
+ * project, no cele2e process, and every npm-consumer-smoke run red on a check
+ * that never executed. So a stack whose ONLY live containers belong to the
+ * shared project gets an evidence test instead of a blanket refusal: no
+ * foreign lock, no live run process, and an age past a threshold means the
+ * stack is garbage and cleanup may act. Any single piece of run evidence
+ * present, or any age that cannot be read, refuses — inconclusive evidence
+ * never reaps, because a false reap kills a real run while a false refusal
+ * costs the operator one docker command.
+ *
+ * The refusal itself names a command that exists on the host printing it (a
+ * bare `cele2e down` does not: forgejo job workspaces are ephemeral, so the
+ * binary only exists inside a checkout), and every refusal reason carries the
+ * `[infra-refusal]` marker with its exit code, so a log search distinguishes
+ * an environment problem from a check failure (celilo#1314 directions 2+3).
  */
 
 import type { ExecFileSyncOptions } from 'node:child_process';
 import { execFileSync } from 'node:child_process';
+import { SHARED_PROJECT_NAME } from './docker-compose-generator';
 import {
   type LockHolder,
   type LockStatus,
@@ -65,6 +87,146 @@ export class LiveStackError extends Error {
   }
 }
 
+/**
+ * How old the youngest live shared-stack container must be before a
+ * shared-only stack with no other run evidence is treated as an orphan.
+ * One hour sits far above any window in which a run process could have died
+ * without its containers dying too, and far below the 11h the builder sat
+ * wedged (celilo#1314). A stack younger than this refuses even with no other
+ * evidence: the reap is the dangerous direction, so it waits for certainty.
+ */
+export const SHARED_ORPHAN_MIN_AGE_MS = 60 * 60_000;
+
+/**
+ * Live pids of processes that may own an e2e run. Injectable so the reap's
+ * evidence test is unit-testable without a process table.
+ */
+export type ProcessProbe = () => number[];
+
+/**
+ * Does this process command line look like an e2e run? Pure so the probe's
+ * reach is testable. Matches the two ways a run actually exists:
+ *   - the cele2e CLI and anything whose argv names it (`cele2e run|up|down|...`)
+ *   - a direct `bun test` of this package's suites, which manages the shared
+ *     stack through ensureSharedInfra but has no cele2e in argv
+ * The bun-test arm requires bun AND e2e AND test in the command, so an
+ * unrelated `bun test` elsewhere only false-matches when its path names e2e —
+ * and a false match refuses (cheap), where a false miss reaps a live run
+ * (expensive). An editor with an e2e test file open does not match: it does
+ * not start with a bun invocation.
+ */
+export function looksLikeE2eRunCommand(command: string): boolean {
+  if (command.includes('cele2e')) return true;
+  return (
+    /(^|[\\/])bun(\.exe)?\s/.test(command) && command.includes('e2e') && command.includes('test')
+  );
+}
+
+/** pids of this process's ancestors, self included, bounded at 64 levels. */
+function familyOfSelf(): Set<number> {
+  const family = new Set<number>([process.pid]);
+  let pid: number | undefined = process.ppid;
+  for (let i = 0; pid !== undefined && pid > 1 && i < 64; i++) {
+    family.add(pid);
+    try {
+      const out = execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+      const ppid = Number.parseInt(out, 10);
+      pid = Number.isFinite(ppid) ? ppid : undefined;
+    } catch {
+      pid = undefined;
+    }
+  }
+  return family;
+}
+
+/**
+ * The real probe: one `ps` listing, filtered by looksLikeE2eRunCommand, with
+ * this process and its ancestry removed — the cleanup runs INSIDE the run it
+ * would otherwise see as evidence, and the run's own launcher shell carries
+ * the same strings in its argv.
+ */
+export const realProcessProbe: ProcessProbe = (): number[] => {
+  let listing: string;
+  try {
+    listing = execFileSync('ps', ['-axo', 'pid=,command='], {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    // A process table that cannot be read is missing evidence. The caller
+    // treats any probe failure conservatively; here that means reporting no
+    // matches, which the shared-only path then backstops with the age
+    // threshold and the unreadable-age refusal.
+    return [];
+  }
+  const family = familyOfSelf();
+  const pids: number[] = [];
+  for (const line of listing.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const sep = trimmed.indexOf(' ');
+    if (sep <= 0) continue;
+    const pid = Number.parseInt(trimmed.slice(0, sep), 10);
+    if (!Number.isFinite(pid) || family.has(pid)) continue;
+    if (looksLikeE2eRunCommand(trimmed.slice(sep + 1))) pids.push(pid);
+  }
+  return pids;
+};
+
+/**
+ * Every refusal goes through here so the printed reason is uniformly
+ * greppable as an infrastructure refusal (celilo#1314 direction 3) rather
+ * than reading as a check failure.
+ */
+function makeRefusal(
+  detail: string,
+  runningContainers: string[],
+  holder: LockHolder | null,
+): LiveStackRefusal {
+  const reason = `[infra-refusal] refusing to clean up: ${detail}\n(this is an environment problem, not a test failure — the run exits 3)`;
+  return { reason, runningContainers, holder };
+}
+
+/**
+ * The remedy printed with a live-stack refusal. A raw docker removal, because
+ * `cele2e down` does not exist on the hosts that print this message: forgejo
+ * job workspaces are ephemeral, so the binary only exists inside a checkout
+ * (celilo#1314 direction 2). Removing the containers is what unblocks the
+ * guard; the next cleanup sweeps whatever name-prefix resources survive.
+ */
+export const CLEAR_STACK_COMMAND = 'docker rm -f $(docker ps -aq --filter name=celilo-e2e)';
+
+/** Docker's `{{.CreatedAt}}` format, e.g. "2026-09-07 18:49:14 +0000 UTC". */
+function parseDockerCreatedAt(raw: string | undefined): Date | null {
+  if (raw === undefined) return null;
+  const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.(\d+))? ([+-])(\d{2})(\d{2})/.exec(
+    raw.trim(),
+  );
+  if (!m) return null;
+  const [, date, time, frac, sign, offH, offM] = m;
+  const utcMs = Date.parse(`${date}T${time}Z`);
+  if (Number.isNaN(utcMs)) return null;
+  const offsetMs = (sign === '-' ? -1 : 1) * (Number(offH) * 60 + Number(offM)) * 60_000;
+  const fracMs = frac ? Number(frac.padEnd(3, '0').slice(0, 3)) : 0;
+  return new Date(utcMs - offsetMs + fracMs);
+}
+
+/**
+ * Containers of the shared compose project. Compose names them
+ * `<project>_<service>_<n>` (or with `-` separators on newer compose), so the
+ * character right after the project name is the tell. Everything else live —
+ * per-test projects AND sim-created guests, whose names carry no project —
+ * is run evidence.
+ */
+function isSharedStackContainer(name: string): boolean {
+  if (!name.startsWith(SHARED_PROJECT_NAME)) return false;
+  const sep = name[SHARED_PROJECT_NAME.length];
+  return sep === '_' || sep === '-';
+}
+
 /**
  * Container states that mean the container cannot be holding a live stack.
  * Everything else (running, paused, restarting) is live: paused containers
@@ -90,17 +252,18 @@ function isOwnHolder(h: LockHolder): boolean {
 export function findLiveE2eStack(
   docker: DockerReader = realDocker,
   lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
 ): LiveStackRefusal | null {
   const status = lock();
   if (!status.free && status.holder && !isOwnHolder(status.holder)) {
     const h = status.holder;
     const heartbeat =
       h.state === 'running' ? `, heartbeat ${formatAge(heartbeatAgeMs(h))} old` : '';
-    return {
-      reason: `refusing to clean up: the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
-      runningContainers: [],
-      holder: h,
-    };
+    return makeRefusal(
+      `the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
+      [],
+      h,
+    );
   }
 
   const out = docker([
@@ -109,21 +272,67 @@ export function findLiveE2eStack(
     '--filter',
     'name=celilo-e2e',
     '--format',
-    '{{.Names}}\t{{.State}}',
+    '{{.Names}}\t{{.State}}\t{{.CreatedAt}}',
   ]);
-  const running = out
+  const live = out
     .split('\n')
     .filter(Boolean)
-    .map((line) => line.split('\t'))
-    .filter((parts) => parts.length === 2 && !DEAD_STATES.has(parts[1] ?? ''))
-    .map((parts) => parts[0] ?? '');
-  if (running.length > 0) {
-    return {
-      reason: `refusing to clean up: ${running.length} live celilo-e2e-* container(s):\n${running.map((n) => `  - ${n}`).join('\n')}\nA live stack owns these. Clear it with: cele2e down`,
-      runningContainers: running,
-      holder: null,
-    };
+    .map((line) => {
+      const [name, state, createdAt] = line.split('\t');
+      return { name: name ?? '', state: state ?? '', createdAt: parseDockerCreatedAt(createdAt) };
+    })
+    .filter((c) => c.name !== '' && !DEAD_STATES.has(c.state));
+  if (live.length === 0) return null;
+
+  // A per-test container (or a guest, which carries no project name at all)
+  // means a run owns this host. Refuse regardless of the shared stack — this
+  // is the #1297 protection, unchanged.
+  const runOwned = live.filter((c) => !isSharedStackContainer(c.name));
+  if (runOwned.length > 0) {
+    return makeRefusal(
+      `${live.length} live celilo-e2e-* container(s):\n${live.map((c) => `  - ${c.name}`).join('\n')}\nA live stack owns these. If the run is gone and the stack is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}\n(there may be no cele2e binary outside a checkout — celilo#1314)`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  // celilo#1314: ONLY shared-stack containers are live. Refusing here
+  // unconditionally is what wedged the builder — an orphaned shared stack is
+  // the one state the guard could never resolve. Apply the orphan evidence
+  // test instead. Any inconclusive answer refuses.
+  const runners = processes();
+  if (runners.length > 0) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but a run process is still alive (pid ${runners.join(', ')}) — it may be between suites and about to use the stack. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  const youngest = Math.min(
+    ...live.map((c) => {
+      if (c.createdAt === null) return Number.NaN;
+      return c.createdAt.getTime();
+    }),
+  );
+  if (!Number.isFinite(youngest)) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.map((c) => c.name).join(', ')}), but its container age cannot be read from docker — the orphan evidence is inconclusive. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
   }
+  const ageMs = Date.now() - youngest;
+  if (ageMs < SHARED_ORPHAN_MIN_AGE_MS) {
+    return makeRefusal(
+      `a shared-only stack is live (${live.length} celilo-e2e-shared-* container(s)), but its youngest container is only ${formatAge(ageMs)} old — under the ${SHARED_ORPHAN_MIN_AGE_MS / 60_000}m orphan threshold, so it may belong to a run the probe cannot see. If it is truly abandoned, clear it with:\n  ${CLEAR_STACK_COMMAND}`,
+      live.map((c) => c.name),
+      null,
+    );
+  }
+
+  // Every piece of run evidence is absent, and the stack is old enough that
+  // no live run can be hiding from the probe. Provably dead: reap it.
   return null;
 }
 
@@ -137,8 +346,9 @@ export function findLiveE2eStack(
 export function refuseOnLiveStack(
   docker: DockerReader = realDocker,
   lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
 ): void {
-  const refusal = findLiveE2eStack(docker, lock);
+  const refusal = findLiveE2eStack(docker, lock, processes);
   if (!refusal) return;
   console.error(`\n${refusal.reason}\n`);
   process.exit(3);

package/src/shared-infra.ts

@@ -22,8 +22,10 @@ import {
 import {
   type DockerReader,
   LiveStackError,
+  type ProcessProbe,
   findLiveE2eStack,
   realDocker,
+  realProcessProbe,
   refuseOnLiveStack,
 } from './live-stack';
 import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
@@ -101,8 +103,9 @@ export function startupCleanup(
   e2eDir: string,
   docker: DockerReader = realDocker,
   lock: () => LockStatus = lockStatus,
+  processes: ProcessProbe = realProcessProbe,
 ): void {
-  const refusal = findLiveE2eStack(docker, lock);
+  const refusal = findLiveE2eStack(docker, lock, processes);
   if (refusal) throw new LiveStackError(refusal);
   console.log(
     '[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete',
@@ -173,7 +176,9 @@ export async function ensureSharedInfra(): Promise<void> {
       // DNS check failed — restart shared infra. The restart tears down the
       // running stack, so the live-stack guard fires first (celilo#1297): a
       // stack that is up is never force-removed, even when its DNS is wedged.
-      // The operator clears it with `cele2e down` and re-runs. This retires
+      // The operator clears it with the docker removal named in the refusal
+      // (a bare `cele2e down` does not exist on ephemeral CI hosts —
+      // celilo#1314) and re-runs. This retires
       // the automatic DNS-restart self-heal by decision (peba, 2026-09-07):
       // fail loudly naming the holder beats silently clobbering a stack that
       // might be another session's — the #1297 incident rode exactly this

package/src/source-fingerprint.ts

@@ -50,6 +50,17 @@ export const SOURCE_LABEL = 'computer.celilo.e2e.source';
  */
 export const PUBLISHED_FINGERPRINT_PREFIX = 'published:';
 
+/**
+ * Value stamped by a CONSUMER-mode bake: `cele2e build-infra` run from an
+ * installed `@celilo/e2e` with no monorepo checkout. The CLI comes from the sim
+ * registry's freshly staged tarballs, so it is neither real npm nor this tree,
+ * and no tree fingerprint exists to record. The version is the honest
+ * attribution. Deliberately NOT the published prefix: that one tells the
+ * freshness check there is nothing to be stale against, which would suppress a
+ * real warning if such an image were later inspected from a checkout.
+ */
+export const CONSUMER_FINGERPRINT_PREFIX = 'consumer:';
+
 /**
  * Repo-relative paths whose content reaches the baked CLI.
  *