@wjarka/cezarion 0.15.1 → 0.16.0

package/scripts/benchmark-stuck-destroys.ts

@@ -0,0 +1,143 @@
+/**
+ * Stuck-destroy benchmark (hearsay-tools/cezarion#879): what permanently stuck worker destroys
+ * cost an otherwise idle cezar. Run it unchanged before and after a change to the cleanup loops.
+ *
+ *   npm run bench:stuck-destroys -w @wjarka/cezarion                       # 20 workers, 10 minutes
+ *   node --import tsx packages/cezar/scripts/benchmark-stuck-destroys.ts --workers 0 --minutes 10
+ *
+ * Flags: --workers (20) --minutes (10) --out (a JSON file for the result row).
+ *
+ * Method:
+ * - One process builds what `serve` builds for a project: `RunStore` (keepLive), `RunManager` and
+ *   `DelegationService`, on a temp Git repo, with `CEZ_DRY_RUN=1` and production cadence (no
+ *   timer is overridden).
+ * - Each worker is settled the way a finished worker is: a complete execution checkpoint, status
+ *   `review`, a real owned worktree and a local scratch dir. A `sleep` child holds each of the
+ *   two, so neither can ever be removed: the destroy stays incomplete and the scratch is retained.
+ * - Every worker is destroyed once through the human route, as Clean up does. Only then does the
+ *   window start, so the measurement is the retry and reprobe loops alone.
+ * - CPU is this process (`process.cpuUsage`, every thread) plus the children it reaped during the
+ *   window (`/proc/self/stat` cutime/cstime: the git processes the loops spawn). The holders are
+ *   reaped only after the window. Event-loop delay is what an HTTP request would wait.
+ * - Compare N workers against `--workers 0`; the difference is what the stuck workers add.
+ */
+import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { cpus, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { parseArgs } from 'node:util';
+import type { RunStore as RunStoreInstance } from '../src/runs/store.ts';
+import type { RunManager as RunManagerInstance } from '../src/workflows/run.ts';
+
+process.env.CEZ_DRY_RUN = '1';
+process.env.CEZ_DELEGATION = '1';
+process.env.CEZ_AUTONAME = '0';
+// The repo's own sandbox for ~/.cezar, so a benchmark never touches the user's registry; one this
+// run creates is removed with the rest of its teardown.
+const ownHome = process.env.CEZ_HOME ? undefined : mkdtempSync(join(tmpdir(), 'cez-bench-home-'));
+if (ownHome) process.env.CEZ_HOME = ownHome;
+
+const { values } = parseArgs({ options: {
+  workers: { type: 'string', default: '20' },
+  minutes: { type: 'string', default: '10' },
+  out: { type: 'string' },
+} });
+const workers = Number(values.workers), minutes = Number(values.minutes);
+if (!Number.isInteger(workers) || workers < 0 || !(minutes > 0)) throw new Error('--workers must be a whole number >= 0 and --minutes > 0');
+
+const { RunStore } = await import('../src/runs/store.ts');
+const { RunManager } = await import('../src/workflows/run.ts');
+const { DelegationService } = await import('../src/delegation/service.ts');
+const { planOwnedWorkspace, ensureOwnedWorkspace } = await import('../src/delegation/workspace.ts');
+const { WorkspaceSemaphore } = await import('../src/workspace/semaphore.ts');
+const { agentTmpDir } = await import('../src/runs/agent-tmpdir.ts');
+
+/** Clock ticks of this process's reaped children (`cutime + cstime`), in ms; 0 off Linux. */
+function childrenCpuMs(): number {
+  if (process.platform !== 'linux') return 0;
+  const stat = readFileSync('/proc/self/stat', 'utf8');
+  const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+  // After the comm field, cutime and cstime are fields 16 and 17 of stat(5), i.e. index 13 and 14 here.
+  const ticks = Number(fields[13]) + Number(fields[14]);
+  const hz = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()) || 100;
+  return ticks * 1000 / hz;
+}
+
+const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim();
+const root = mkdtempSync(join(tmpdir(), 'cez-bench-stuck-'));
+git(root, 'init', '-q', '-b', 'main');
+git(root, 'config', 'gc.auto', '0');
+git(root, 'config', 'maintenance.auto', 'false');
+git(root, '-c', 'user.name=bench', '-c', 'user.email=bench@local', 'commit', '--allow-empty', '-qm', 'base');
+const sha = git(root, 'rev-parse', 'HEAD');
+const dataDir = join(root, '.ai/cezar');
+
+// Teardown runs however the run ends, so a failed setup, destroy or measurement leaves no
+// `sleep` holder and no temporary repository behind.
+const holders: ChildProcess[] = [];
+let store: RunStoreInstance | undefined, manager: RunManagerInstance | undefined, detach: (() => void) | undefined;
+try {
+  store = RunStore.open(dataDir, { keepLive: true });
+  manager = new RunManager(store, root, { semaphore: new WorkspaceSemaphore({ initial: { maxParallel: 1 } }) });
+  const service = new DelegationService();
+  detach = service.registerProject({ id: 'bench', root, store, manager });
+  const parent = store.createRun({ title: 'parent', task: 'parent', workflow: 'quick-task', steps: [] });
+  store.updateRun(parent.id, { status: 'waiting', delegation: { role: 'root', permissions: ['spawn'], receipts: [] } });
+
+  const hold = (cwd: string) => { holders.push(spawn('sleep', ['86400'], { cwd, stdio: 'ignore' })); };
+  const ids: string[] = [];
+  for (let i = 0; i < workers; i++) {
+    const id = randomUUID();
+    const workspace = await planOwnedWorkspace(root, id, sha);
+    store.createOwnedRun({ title: `worker ${i}`, task: 'stuck', workflow: 'quick-task', runner: 'claude', steps: [{ id: 'task', name: 'Task', kind: 'agent' }] },
+      parent.id, randomUUID(), { role: 'worker', permissions: [], parentRunId: parent.id, workspace }, 'a'.repeat(64));
+    const generation = store.commitWorkerExecutionStart(id);
+    const created = await ensureOwnedWorkspace(root, store.getRun(id)!);
+    store.updateRun(id, { status: 'review', worktreePath: created.path, branch: created.branch });
+    if (!store.commitWorkerExecutionComplete(id, generation)) throw new Error(`worker ${id} did not complete`);
+    const scratch = agentTmpDir(dataDir, id);
+    mkdirSync(scratch, { recursive: true });
+    hold(created.path); hold(scratch);
+    ids.push(id);
+  }
+  // What boot does after recovery: arm pending destroys and the scratch reprobes.
+  service.armDestroyRetries('bench');
+  for (const id of ids) {
+    const result = await service.destroyForHuman('bench', id).catch((error: Error) => ({ state: 'error', error: error.message }));
+    if (result.state === 'complete') throw new Error(`worker ${id} was not held: its destroy completed`);
+  }
+
+  const delay = monitorEventLoopDelay({ resolution: 10 });
+  delay.enable();
+  const cpu0 = process.cpuUsage(), children0 = childrenCpuMs(), wall0 = performance.now();
+  await sleep(minutes * 60_000);
+  const cpu = process.cpuUsage(cpu0), children = childrenCpuMs() - children0, wall = performance.now() - wall0;
+  delay.disable();
+
+  const opened = store;
+  const pending = ids.filter(id => { const run = opened.getRun(id); return run?.delegation?.role === 'worker' && run.delegation.destroy?.phase !== 'complete'; }).length;
+  const row = {
+    workers, minutes, pendingDestroys: pending,
+    userMs: Math.round(cpu.user / 1000), systemMs: Math.round(cpu.system / 1000), childrenMs: Math.round(children),
+    cpuPercent: Number(((cpu.user + cpu.system) / 1000 / wall * 100).toFixed(2)),
+    cpuWithChildrenPercent: Number((((cpu.user + cpu.system) / 1000 + children) / wall * 100).toFixed(2)),
+    loopDelayP99Ms: Number((delay.percentile(99) / 1e6).toFixed(1)), loopDelayMaxMs: Number((delay.max / 1e6).toFixed(1)),
+    cores: cpus().length, node: process.version,
+  };
+  console.log('| workers | minutes | pending | user ms | system ms | children ms | CPU % | CPU % incl. children | loop p99 ms | loop max ms |');
+  console.log('| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
+  console.log(`| ${row.workers} | ${row.minutes} | ${row.pendingDestroys} | ${row.userMs} | ${row.systemMs} | ${row.childrenMs} | ${row.cpuPercent} | ${row.cpuWithChildrenPercent} | ${row.loopDelayP99Ms} | ${row.loopDelayMaxMs} |`);
+  if (values.out) writeFileSync(values.out, `${JSON.stringify(row, null, 2)}\n`);
+
+} finally {
+  for (const child of holders) child.kill('SIGKILL');
+  detach?.();
+  manager?.dispose();
+  store?.close();
+  rmSync(root, { recursive: true, force: true });
+  if (ownHome) rmSync(ownHome, { recursive: true, force: true });
+}
+process.exit(0);

package/scripts/mock-autonomous.mjs

@@ -0,0 +1,11 @@
+// Shared responses only; each caller encodes its own native runner wire (#426).
+let cap = false;
+export function autonomousReply(prompt) {
+  const askCap = prompt.includes('mock:autonomous-ask-cap') || prompt.includes('mock:autonomous-readiness-idle');
+  if (prompt.includes('mock:autonomous-cap') || askCap) cap = true;
+  if (askCap) return 'CEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}';
+  if (prompt.startsWith('Continue working autonomously until the task is fully complete.')) {
+    return cap ? 'Still working.' : 'Autonomous work finished.\nCEZ:DONE';
+  }
+  return 'Still working.';
+}

package/scripts/mock-ci-tool.mjs

@@ -1,23 +1,30 @@
 // Offline harness fixtures share the real bundled MCP protocol, never a fake receipt.
 import { writeFileSync, existsSync } from 'node:fs';
 import { basename } from 'node:path';
-import { pathToFileURL } from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
 
 export async function probeCiTool(backend, wire, pr = process.env.CEZ_MOCK_CI_PR) {
   if (!pr) return;
+  return probeCezarTool(backend, wire, 'cezar_wait_for_ci', { pr });
+}
+
+// Calls one cezar tool the way the real harness would: through the wired MCP server (or Pi/OMP extension).
+export async function probeCezarTool(backend, wire, name, args) {
   let result;
-  if (backend === 'pi') {
+  if (backend === 'pi' || backend === 'omp') {
     const extensions = wire.flatMap((arg, index) => arg === '--extension' ? [wire[index + 1]] : []);
-    const path = extensions.find(path => basename(path) === 'pi-ci-wait.mjs');
+    const path = extensions.find(path => basename(path) === `${backend}-ci-wait.mjs`);
     if (!path) throw new Error('CI extension absent');
     // The real Pi loader compiles TS when running from source; installed script uses JS.
     if (existsSync(new URL('../src/ci-wait/mcp.ts', import.meta.url))) { const { register } = await import('tsx/esm/api'); register(); }
     const { default: extension } = await import(pathToFileURL(path));
     const tools = [];
     extension({ registerTool(tool) { tools.push(tool); } });
-    result = { names: tools.map(tool => tool.name), response: await tools[0].execute('ci-1', { pr }) };
+    const tool = tools.find(tool => tool.name === name);
+    if (!tool) throw new Error(`${name} not registered`);
+    result = { names: tools.map(tool => tool.name), response: await tool.execute('ci-1', args) };
   } else {
     let server; let env;
     if (backend === 'claude') {
@@ -39,7 +46,7 @@ export async function probeCiTool(backend, wire, pr = process.env.CEZ_MOCK_CI_PR
     try {
       await client.connect(new StdioClientTransport({ command: server.command, args: server.args, env, stderr: 'pipe' }));
       const list = await client.listTools();
-      result = { names: list.tools.map(tool => tool.name), response: await client.callTool({ name: 'cezar_wait_for_ci', arguments: { pr } }) };
+      result = { names: list.tools.map(tool => tool.name), response: await client.callTool({ name, arguments: args }) };
     } finally { await client.close(); }
   }
   if (process.env.CEZ_MOCK_CI_RESULT) writeFileSync(process.env.CEZ_MOCK_CI_RESULT, JSON.stringify(result));
@@ -51,3 +58,11 @@ export async function ciPrompt(backend, wire, text) {
   const result = await probeCiTool(backend, wire, pr);
   return JSON.stringify(result.response);
 }
+
+// `mock:preview-serve <port>` registers the e2e fixture app (`node preview-app.mjs <port>`) as a live preview.
+export async function previewPrompt(backend, wire, text) {
+  const port = Number(/mock:preview-serve\s+(\d+)/.exec(text)?.[1]);
+  const fixture = fileURLToPath(new URL('../../web/e2e/fixtures/preview-app.mjs', import.meta.url));
+  const result = await probeCezarTool(backend, wire, 'cezar_preview_serve', { command: `node "${fixture}" ${port}`, port, label: 'web' });
+  return JSON.stringify(result.response);
+}

package/scripts/mock-claude.mjs

@@ -14,6 +14,13 @@ function watchdogStall(prompt) {
   if (prompt.includes('held-pipe')) {
     watchdogSpawn(process.execPath, ['-e', 'setTimeout(() => {}, 5000)'], { stdio: ['ignore', process.stdout, process.stderr] });
   }
+  if (prompt.includes('leftover')) {
+    // hearsay-tools/cezarion#890: one child stays in the session's process group and ignores
+    // SIGTERM; one leaves the group with setsid. Both exit on their own as a cleanup backstop.
+    const leftover = (file) => `process.on('SIGTERM',()=>{}); require('fs').writeFileSync(${JSON.stringify(file)}, String(process.pid)); setTimeout(()=>{},20000)`;
+    watchdogSpawn(process.execPath, ['-e', leftover('leftover-group.pid')], { stdio: 'ignore' });
+    watchdogSpawn(process.execPath, ['-e', leftover('leftover-session.pid')], { stdio: 'ignore', detached: true }).unref();
+  }
   return true;
 }
 
@@ -27,6 +34,15 @@ import { createInterface } from 'node:readline';
 import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
 
 import * as parityGateFs from 'node:fs';
+async function waitForMockRelease(fallbackMs) {
+  const file = process.env.CEZ_MOCK_RELEASE_FILE;
+  if (!file) { if (fallbackMs) await sleep(fallbackMs); return; }
+  const deadline = Date.now() + 15_000;
+  while (!parityGateFs.existsSync(file)) {
+    if (Date.now() > deadline) throw new Error('CEZ_MOCK_RELEASE_FILE was not created');
+    await sleep(10);
+  }
+}
 
 // #401: let the test observe the actual monitoring park before releasing late wire frames.
 async function afterParityPark(prompt) {
@@ -118,7 +134,55 @@ function writeHandoffAndTodo() {
 }
 
 async function respond(userText, imageCount, uuid) {
+  if (userText.includes('mock:crash-stderr')) {
+    const { crashWithStderr } = await import('./mock-runner-crash.mjs');
+    if (crashWithStderr(userText, '{"type":"assistant","message":')) return;
+  }
   emitReplay(uuid, userText);
+  if (userText.includes('mock:autonomous') || userText.startsWith('Continue working autonomously until the task is fully complete.')) {
+    const { autonomousReply } = await import('./mock-autonomous.mjs');
+    const text = autonomousReply(userText);
+    emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text }] } });
+    emit({ type: 'result', subtype: 'success', result: text, user_message_uuids: [uuid].filter(Boolean), usage: { input_tokens: 10, output_tokens: 5 } });
+    return;
+  }
+  if (userText.includes('mock:silent-tail') || userText.includes('mock:tool-tail') || userText.includes('Your last turn ended without a message to the user.')) {
+    const silent = await import('./mock-silent-tail.mjs');
+    silent.noteSilentTailPrompt(userText);
+    if (silent.isFinalMessageNudge(userText)) {
+      const kind = silent.finalMessageNudgeKind();
+      if (kind === 'ack-only') return;
+      if (kind === 'late') await silent.sleep(silent.LATE_REPLY_MS);
+      if (kind === 'silent') {
+        emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'thinking', thinking: silent.SILENT_TAIL_REASONING }] } });
+        emit({ type: 'result', subtype: 'success', result: '', user_message_uuids: [uuid].filter(Boolean), usage: { input_tokens: 10, output_tokens: 5 } });
+      } else {
+        if (kind === 'slow-done') {
+          emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: silent.SLOW_DONE_PREFIX }] } });
+          await silent.sleep(silent.SLOW_DONE_TAIL_MS);
+        }
+        const text = kind === 'standing' ? silent.FINAL_MESSAGE_STANDING : silent.SILENT_TAIL_DONE;
+        emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text }] } });
+        emit({ type: 'result', subtype: 'success', result: text, user_message_uuids: [uuid].filter(Boolean), usage: { input_tokens: 10, output_tokens: 5 } });
+      }
+      return;
+    }
+    if (silent.isSilentTailScenario(userText)) {
+      emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: silent.SILENT_TAIL_OPENING }] } });
+      emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'tool_use', id: 'toolu_silent_gh', name: 'Bash', input: { command: 'gh issue create' } }] } });
+      emit({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_silent_gh', content: 'created' }] } });
+      emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'thinking', thinking: silent.SILENT_TAIL_REASONING }] } });
+      emit({ type: 'result', subtype: 'success', result: silent.SILENT_TAIL_OPENING, user_message_uuids: [uuid].filter(Boolean), usage: { input_tokens: 20, output_tokens: 10 } });
+      return;
+    }
+    if (silent.isToolTailScenario(userText)) {
+      emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: silent.TOOL_TAIL_OPENING }] } });
+      emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'tool_use', id: 'toolu_tool_tail', name: 'Bash', input: { command: 'git status --short' } }] } });
+      emit({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_tool_tail', content: ' M src/example.ts' }] } });
+      emit({ type: 'result', subtype: 'success', result: silent.TOOL_TAIL_OPENING, user_message_uuids: [uuid].filter(Boolean), usage: { input_tokens: 20, output_tokens: 10 } });
+      return;
+    }
+  }
   // `mock:steer-tool` → one tool call; lines written while it runs are consumed after
   // it (replayed with their uuid) and settled by the SAME result, like Claude 2.1.280 (#505).
   if (userText.includes('mock:steer-tool')) {
@@ -146,8 +210,15 @@ async function respond(userText, imageCount, uuid) {
     return;
   }
   // #401: result-only full text; mapResult and v1 use the same fallback.
+  if (userText.includes('mock:turn-messages:')) {
+    const { turnMessages } = await import('./mock-turn-messages.mjs');
+    const messages = turnMessages(userText);
+    for (const text of messages) emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text }] } });
+    emit({ type: 'result', subtype: 'success', result: messages.join('\n'), usage: { input_tokens: 10, output_tokens: 5 } });
+    return;
+  }
   if (!userText.includes('[cez-namer]') && userText.includes('mock:ask-snapshot')) {
-    emit({ type: 'result', subtype: 'success', result: 'Using the CEZ:ASK structured question format instead:\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}', usage: { input_tokens: 10, output_tokens: 5 } });
+    emit({ type: 'result', subtype: 'success', result: userText.includes('mock:ask-snapshot-bad') ? 'CEZ:ASK {not valid json' : 'Using the CEZ:ASK structured question format instead:\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}', usage: { input_tokens: 10, output_tokens: 5 } });
     return;
   }
   if (!userText.includes('[cez-namer]') && userText.includes('mock:ask-prose')) {
@@ -161,6 +232,13 @@ async function respond(userText, imageCount, uuid) {
     emit({ type: 'result', subtype: 'success', result: text, usage: { input_tokens: 20, output_tokens: 10 } });
     return;
   }
+  if (!userText.includes('[cez-namer]') && userText.includes('mock:preview-serve')) {
+    const { previewPrompt } = await import('./mock-ci-tool.mjs');
+    const text = await previewPrompt('claude', process.argv.slice(2), userText);
+    emit({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text }] } });
+    emit({ type: 'result', subtype: 'success', result: text, usage: { input_tokens: 20, output_tokens: 10 } });
+    return;
+  }
   turn += 1;
   await sleep(250);
   // `mock:done` anywhere in the message → the reply ends with the CEZ:DONE
@@ -271,6 +349,37 @@ async function respond(userText, imageCount, uuid) {
     emit({ type: 'result', subtype: 'success', result: 'busy complete' });
     return;
   }
+  if (userText.includes('mock:hold-done')) {
+    await waitForMockRelease(400);
+    const held = 'parity hold-done: content after the pause\n\nCEZ:DONE';
+    emit({
+      type: 'assistant',
+      message: { role: 'assistant', content: [{ type: 'text', text: held }], usage: { input_tokens: 20, output_tokens: 10 } },
+    });
+    emit({ type: 'result', subtype: 'success', result: held, usage: { input_tokens: 20, output_tokens: 10 }, total_cost_usd: 0.0001 });
+    return;
+  }
+  if (userText.includes('mock:hold-ask')) {
+    await waitForMockRelease(400);
+    const ask = 'Pick one.\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}';
+    emit({
+      type: 'assistant',
+      message: { role: 'assistant', content: [{ type: 'text', text: ask }], usage: { input_tokens: 20, output_tokens: 10 } },
+    });
+    emit({ type: 'result', subtype: 'success', result: ask, usage: { input_tokens: 20, output_tokens: 10 }, total_cost_usd: 0.0001 });
+    return;
+  }
+  if (userText.includes('mock:hold-gated')) {
+    await waitForMockRelease(250);
+    const gated = 'parity hold-gated: content after the pause';
+    emit({
+      type: 'assistant',
+      message: { role: 'assistant', content: [{ type: 'text', text: gated }], usage: { input_tokens: 20, output_tokens: 10 } },
+    });
+    await sleep(250);
+    emit({ type: 'result', subtype: 'success', result: gated, usage: { input_tokens: 20, output_tokens: 10 }, total_cost_usd: 0.0001 });
+    return;
+  }
   if (userText.includes('mock:hold')) {
     await sleep(250);
     const held = 'parity hold: content after the pause';

package/scripts/mock-codex-app-server.mjs

@@ -14,6 +14,13 @@ function watchdogStall(prompt) {
   if (prompt.includes('held-pipe')) {
     watchdogSpawn(process.execPath, ['-e', 'setTimeout(() => {}, 5000)'], { stdio: ['ignore', process.stdout, process.stderr] });
   }
+  if (prompt.includes('leftover')) {
+    // hearsay-tools/cezarion#890: one child stays in the session's process group and ignores
+    // SIGTERM; one leaves the group with setsid. Both exit on their own as a cleanup backstop.
+    const leftover = (file) => `process.on('SIGTERM',()=>{}); require('fs').writeFileSync(${JSON.stringify(file)}, String(process.pid)); setTimeout(()=>{},20000)`;
+    watchdogSpawn(process.execPath, ['-e', leftover('leftover-group.pid')], { stdio: 'ignore' });
+    watchdogSpawn(process.execPath, ['-e', leftover('leftover-session.pid')], { stdio: 'ignore', detached: true }).unref();
+  }
   return true;
 }
 
@@ -42,12 +49,15 @@ const write = (obj) => process.stdout.write(`${JSON.stringify(obj)}\n`);
 // #505: like the real app-server, any active main-thread turn accepts `turn/steer`.
 // Steers accepted by an ordinary scripted turn are read just before it completes:
 // a userMessage item (echoing clientUserMessageId as clientId) and an echo reply.
+let autonomousTurn = 0;
 let activeTurnId = null;
 let pendingSteers = [];
 let steerEchoSerial = 0;
 const emit = (obj) => {
   const ending = (obj.method === 'turn/completed' || obj.method === 'turn/failed') && (!obj.params?.threadId || obj.params.threadId === 'th_mock_1');
-  if (ending && obj.method === 'turn/completed' && pendingSteers.length) {
+  // #486 review: CEZ_MOCK_CODEX_NO_STEER_ECHO=1 admits the steer (RPC result) but never
+  // echoes its userMessage, so an accepted-but-unread follow-up can complete the turn.
+  if (ending && obj.method === 'turn/completed' && pendingSteers.length && process.env.CEZ_MOCK_CODEX_NO_STEER_ECHO !== '1') {
     const turnId = activeTurnId ?? 'turn_mock_1';
     for (const entry of pendingSteers.splice(0)) {
       const item = { type: 'userMessage', id: `item_user_generic_${++steerEchoSerial}`, clientId: entry.clientId, content: [{ type: 'text', text: entry.text }] };
@@ -71,6 +81,15 @@ let ciWire;
 let steerTurn = null;
 let steerSerial = 0;
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function waitForMockRelease(fallbackMs) {
+  const file = process.env.CEZ_MOCK_RELEASE_FILE;
+  if (!file) { if (fallbackMs) await sleep(fallbackMs); return; }
+  const deadline = Date.now() + 15_000;
+  while (!parityGateFs.existsSync(file)) {
+    if (Date.now() > deadline) throw new Error('CEZ_MOCK_RELEASE_FILE was not created');
+    await sleep(10);
+  }
+}
 const completeSteerTurn = (turnId) => {
   steerTurn = null;
   emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: turnId, status: 'completed' } } });
@@ -190,6 +209,12 @@ rl.on('line', async (line) => {
       emit({ id: msg.id, result: { thread: { id: msg.params?.threadId }, sandbox } });
     }
   } else if (msg.method === 'turn/start') {
+    const openingCrashText = msg.params?.input?.map?.((part) => part.text ?? '').join('\n') ?? '';
+    if (openingCrashText.includes('mock:crash-stderr-pre-ack')) {
+      const { crashWithStderr } = await import('./mock-runner-crash.mjs');
+      crashWithStderr(openingCrashText, '{"method":"turn/started","params":');
+      return;
+    }
     if (rejectedPermissionOverride) permissionWarning();
     activeTurnId = 'turn_mock_1';
     // owned-input-delivery.testkit.ts patches the exact `emit(...)` line below; keep it verbatim.
@@ -197,9 +222,17 @@ rl.on('line', async (line) => {
     emit({ id: msg.id, result: { turn: { id: 'turn_mock_1' } } });
     }
     const turnText = msg.params?.input?.map?.((part) => part.text ?? '').join('\n') ?? '';
+    let silent;
+    if (turnText.includes('mock:silent-tail') || turnText.includes('Your last turn ended without a message to the user.')) {
+      silent = await import('./mock-silent-tail.mjs');
+      silent.noteSilentTailPrompt(turnText);
+      if (silent.isAckOnlyNudge(turnText)) return;
+      if (silent.isLateNudge(turnText) || silent.isLateTurnStartNudge(turnText)) await silent.sleep(silent.LATE_REPLY_MS);
+    }
     if (!turnText.includes('mock:no-progress-ack-only')) {
     emit({ method: 'turn/started', params: { turn: { id: 'turn_mock_1', status: 'inProgress', items: [] } } });
     }
+    if (silent?.isLateTurnStartNudge(turnText)) await silent.sleep(silent.LATE_TURN_START_HOLD_MS);
     // The real app-server records the turn's own input as a userMessage item,
     // echoing clientUserMessageId as clientId (probe 0.155.1, #505).
     const opening = { type: 'userMessage', id: `item_user_open_${++steerEchoSerial}`, clientId: msg.params?.clientUserMessageId ?? null, content: [{ type: 'text', text: turnText }] };
@@ -207,6 +240,49 @@ rl.on('line', async (line) => {
       emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: opening } });
       emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: opening } });
     }
+    if (turnText.includes('mock:crash-stderr')) {
+      const { crashWithStderr } = await import('./mock-runner-crash.mjs');
+      await new Promise(resolve => setTimeout(resolve, 30));
+      if (crashWithStderr(turnText, '{"method":"item/commandExecution/outputDelta","params":')) return;
+    }
+    if (turnText.includes('mock:autonomous') || turnText.startsWith('Continue working autonomously until the task is fully complete.')) {
+      const { autonomousReply } = await import('./mock-autonomous.mjs');
+      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: `autonomous-${++autonomousTurn}`, text: autonomousReply(turnText) } } });
+      emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: 'turn_mock_1', status: 'completed' } } });
+      return;
+    }
+    if (turnText.includes('mock:silent-tail') || turnText.includes('mock:tool-tail') || turnText.includes('Your last turn ended without a message to the user.')) {
+      const silent = await import('./mock-silent-tail.mjs');
+      silent.noteSilentTailPrompt(turnText);
+      if (silent.isFinalMessageNudge(turnText)) {
+        const kind = silent.finalMessageNudgeKind();
+        if (kind === 'silent') {
+          emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'reasoning', id: 'silent-nudge-rsn', summary: silent.SILENT_TAIL_REASONING, content: silent.SILENT_TAIL_REASONING } } });
+        } else {
+          if (kind === 'slow-done') {
+            emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'silent-nudge-prefix', text: silent.SLOW_DONE_PREFIX } } });
+            await silent.sleep(silent.SLOW_DONE_TAIL_MS);
+          }
+          const text = kind === 'standing' ? silent.FINAL_MESSAGE_STANDING : silent.SILENT_TAIL_DONE;
+          emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'silent-nudge-done', text } } });
+        }
+        emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: 'turn_mock_1', status: 'completed' } } });
+        return;
+      }
+      if (silent.isSilentTailScenario(turnText)) {
+        emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'silent-open', text: silent.SILENT_TAIL_OPENING } } });
+        emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'commandExecution', id: 'silent-gh', command: ['bash', '-lc', 'gh issue create'], cwd: '/repo', status: 'completed', exitCode: 0 } } });
+        emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'reasoning', id: 'silent-rsn', summary: silent.SILENT_TAIL_REASONING, content: silent.SILENT_TAIL_REASONING } } });
+        emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: 'turn_mock_1', status: 'completed' } } });
+        return;
+      }
+      if (silent.isToolTailScenario(turnText)) {
+        emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'tool-tail-open', text: silent.TOOL_TAIL_OPENING } } });
+        emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'commandExecution', id: 'tool-tail-git', command: ['bash', '-lc', 'git status --short'], cwd: '/repo', status: 'completed', exitCode: 0 } } });
+        emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: 'turn_mock_1', status: 'completed' } } });
+        return;
+      }
+    }
     // Native thread attribution from collab-agent-tool-call.ndjson (#121/#401).
     if (turnText.includes('mock:subagent-after-park')) {
       emit({ method: 'item/started', params: { threadId: 'th_mock_1', item: { type: 'collabAgentToolCall', id: 'park-spawn', tool: 'spawnAgent', status: 'inProgress', receiverThreadIds: ['th_park_child'] } } });
@@ -219,8 +295,14 @@ rl.on('line', async (line) => {
       return;
     }
     // #401: agentMessage completion without deltas (same item envelope as baseline).
+    if (turnText.includes('mock:turn-messages:')) {
+      const { turnMessages } = await import('./mock-turn-messages.mjs');
+      for (const text of turnMessages(turnText)) emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: `item_multi_${++echoSerial}`, text } } });
+      emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
+      return;
+    }
     if (turnText.includes('mock:ask-snapshot')) {
-      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'ask-snapshot', text: 'Using the CEZ:ASK structured question format instead:\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}' } } });
+      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'ask-snapshot', text: turnText.includes('mock:ask-snapshot-bad') ? 'CEZ:ASK {not valid json' : 'Using the CEZ:ASK structured question format instead:\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}' } } });
       emit({ method: 'turn/completed', params: { threadId: 'th_mock_1', turn: { id: 'turn_mock_1', status: 'completed' } } });
       return;
     }
@@ -293,6 +375,26 @@ rl.on('line', async (line) => {
       } });
       return;
     }
+    if (turnText.includes('mock:hold-done')) {
+      await waitForMockRelease(400);
+      const full = 'parity hold-done: content after the pause\n\nCEZ:DONE';
+      emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_hd1', text: '' } } });
+      emit({ method: 'item/agentMessage/delta', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', itemId: 'item_hd1', delta: full } });
+      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_hd1', text: full } } });
+      emit({ method: 'thread/tokenUsage/updated', params: { threadId: 'th_mock_1', tokenUsage: { total: { totalTokens: 30, inputTokens: 20, outputTokens: 10 }, last: { totalTokens: 30, inputTokens: 20, outputTokens: 10 } } } });
+      emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
+      return;
+    }
+    if (turnText.includes('mock:hold-ask')) {
+      await waitForMockRelease(400);
+      const ask = 'Pick one.\n\nCEZ:ASK {"questions":[{"header":"Library","question":"Which test library?","options":[{"label":"Vitest"},{"label":"Node test"}]}]}';
+      emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_ha1', text: '' } } });
+      emit({ method: 'item/agentMessage/delta', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', itemId: 'item_ha1', delta: ask } });
+      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_ha1', text: ask } } });
+      emit({ method: 'thread/tokenUsage/updated', params: { threadId: 'th_mock_1', tokenUsage: { total: { totalTokens: 30, inputTokens: 20, outputTokens: 10 }, last: { totalTokens: 30, inputTokens: 20, outputTokens: 10 } } } });
+      emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
+      return;
+    }
     if (turnText.includes('mock:done')) {
       // A turn that DECLARES the task complete, so the run reaches cezar's
       // review gate instead of parking for the user. A markerless turn-end
@@ -315,6 +417,16 @@ rl.on('line', async (line) => {
       emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
       return;
     }
+    if (turnText.includes('mock:hold-gated')) {
+      await waitForMockRelease(250);
+      const gated = 'parity hold-gated: content after the pause';
+      emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_hg1', text: '' } } });
+      emit({ method: 'item/agentMessage/delta', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', itemId: 'item_hg1', delta: gated } });
+      emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_hg1', text: gated } } });
+      emit({ method: 'thread/tokenUsage/updated', params: { threadId: 'th_mock_1', tokenUsage: { total: { totalTokens: 30, inputTokens: 20, outputTokens: 10 }, last: { totalTokens: 30, inputTokens: 20, outputTokens: 10 } } } });
+      emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
+      return;
+    }
     if (turnText.includes('mock:hold')) {
       // The `turn/start` response and `turn/started` above are the ack. Holding
       // the content AND `turn/completed` behind it is what makes harness parity
@@ -413,6 +525,9 @@ rl.on('line', async (line) => {
     emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'commandExecution', id: 'item_c1', command: ['bash', '-lc', 'git status --short'], cwd: '/repo', status: 'inProgress' } } });
     emit({ method: 'item/commandExecution/outputDelta', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', itemId: 'item_c1', delta: ' M src/example.ts\n' } });
     emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'commandExecution', id: 'item_c1', command: ['bash', '-lc', 'git status --short'], cwd: '/repo', status: 'completed', exitCode: 0 } } });
+    emit({ method: 'item/started', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_m2', text: '' } } });
+    emit({ method: 'item/agentMessage/delta', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', itemId: 'item_m2', delta: 'Done with the first pass.' } });
+    emit({ method: 'item/completed', params: { threadId: 'th_mock_1', turnId: 'turn_mock_1', item: { type: 'agentMessage', id: 'item_m2', text: 'Done with the first pass.' } } });
     emit({ method: 'thread/tokenUsage/updated', params: { threadId: 'th_mock_1', tokenUsage: { total: { totalTokens: 1500, inputTokens: 1200, outputTokens: 300 }, last: { totalTokens: 1500, inputTokens: 1200, outputTokens: 300 } } } });
     emit({ method: 'turn/completed', params: { turn: { id: 'turn_mock_1', status: 'completed' } } });
   }

package/scripts/mock-codex-coalesce-steer.mjs

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+// #486 review: buffers each `turn/steer` RPC response and writes it in the SAME
+// stdout write as the following `turn/completed` (or `turn/failed`) notification,
+// preserving order. Reproduces Codex humanPromptsPending lagging a coalesced chunk.
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const child = spawn(process.execPath, [join(dirname(fileURLToPath(import.meta.url)), 'mock-codex-app-server.mjs'), ...process.argv.slice(2)], {
+  stdio: ['pipe', 'pipe', 'inherit'],
+  env: process.env,
+});
+const steerIds = new Set();
+let stdinBuf = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => {
+  child.stdin.write(chunk);
+  stdinBuf += chunk;
+  let idx;
+  while ((idx = stdinBuf.indexOf('\n')) >= 0) {
+    const line = stdinBuf.slice(0, idx);
+    stdinBuf = stdinBuf.slice(idx + 1);
+    try {
+      const msg = JSON.parse(line);
+      if (msg.method === 'turn/steer' && typeof msg.id === 'number') steerIds.add(msg.id);
+    } catch { /* ignore a partial or non-JSON line */ }
+  }
+});
+process.stdin.on('end', () => child.stdin.end());
+
+let outBuf = '';
+const held = [];
+child.stdout.setEncoding('utf8');
+child.stdout.on('data', chunk => {
+  outBuf += chunk;
+  let idx;
+  while ((idx = outBuf.indexOf('\n')) >= 0) {
+    const line = outBuf.slice(0, idx + 1);
+    outBuf = outBuf.slice(idx + 1);
+    let msg;
+    try { msg = JSON.parse(line); } catch { process.stdout.write(line); continue; }
+    if (typeof msg.id === 'number' && steerIds.has(msg.id) && (msg.result !== undefined || msg.error !== undefined)) {
+      steerIds.delete(msg.id);
+      held.push(line);
+      continue;
+    }
+    if ((msg.method === 'turn/completed' || msg.method === 'turn/failed') && held.length) {
+      process.stdout.write(held.splice(0).join('') + line);
+      continue;
+    }
+    process.stdout.write(line);
+  }
+});
+child.on('exit', (code, signal) => {
+  if (outBuf) process.stdout.write(outBuf);
+  if (held.length) process.stdout.write(held.join(''));
+  if (signal) process.kill(process.pid, signal);
+  else process.exit(code ?? 0);
+});