gestalt-mobile 0.35.0 → 0.35.2

package/dist/server/server/composition.contracts.js

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 import { once } from 'node:events';
+import { createHash } from 'node:crypto';
 import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises';
 import { DatabaseSync } from 'node:sqlite';
 import { tmpdir } from 'node:os';
@@ -12,12 +13,14 @@ import { describe, expect, it, onTestFinished, vi } from 'vitest';
 import WebSocket from 'ws';
 import { composeRelayApp } from './composition.js';
 import { AUTOPILOT_CONTINUATION_PROMPT } from './features/autopilot/application/policy.js';
+import { CodexJsonRpcError } from './platform/codex/json-rpc-client.js';
 import { SqliteAuthorizationStore } from './platform/auth/sqlite-authorization-store.js';
 import { authorizationSessionId, authorizedDeviceId, localOwnerId, webAuthnCredentialId, } from './features/auth/domain/identifiers.js';
 import { deviceNickname } from './features/auth/domain/device-nickname.js';
 import { workspaceId } from './platform/catalog/workspace-id.js';
 import { planStatusDirectoryPath, planStatusFilePath, } from './platform/plans/filesystem-plan-status-source.js';
 import { toOrgPlanAttentionToolResponse } from '../shared/contracts/org-plan-attention.js';
+import { toOrgPlanCheckpointToolResponse } from '../shared/contracts/org-plan-checkpoint.js';
 function fakeAppServer(calls) {
     return {
         rpc: {
@@ -42,7 +45,7 @@ function fakeAppServer(calls) {
 }
 function liveAppServer(handles) {
     return () => {
-        const handle = { calls: [], requests: [] };
+        const handle = { calls: [], requests: [], pendingResponses: new Map() };
         handles.push(handle);
         return {
             rpc: {
@@ -71,11 +74,45 @@ function liveAppServer(handles) {
                     return () => { };
                 },
                 onServerRequest: (listener) => {
-                    handle.request = listener;
+                    handle.request = async (request) => {
+                        try {
+                            const result = await listener(request);
+                            if (!handle.deferResponseSettlement)
+                                handle.responseSettled?.({ id: request.id, outcome: 'resultWritten' });
+                            else
+                                await new Promise((resolve, reject) => handle.pendingResponses.set(request.id, { resolve, reject }));
+                            return result;
+                        }
+                        catch (error) {
+                            if (!handle.deferResponseSettlement)
+                                handle.responseSettled?.({ id: request.id, outcome: 'errorWritten' });
+                            throw error;
+                        }
+                    };
                     return () => { };
                 },
+                onServerResponseSettled: (listener) => {
+                    handle.responseSettled = (settlement) => {
+                        listener(settlement);
+                        const pending = handle.pendingResponses.get(settlement.id);
+                        if (!pending)
+                            return;
+                        handle.pendingResponses.delete(settlement.id);
+                        if (settlement.outcome === 'resultWritten')
+                            pending.resolve();
+                        else
+                            pending.reject(new Error('CODEX_SERVER_REQUEST_WRITE_FAILED'));
+                    };
+                    return () => {
+                        handle.responseSettled = undefined;
+                    };
+                },
+            },
+            close: () => {
+                for (const pending of handle.pendingResponses.values())
+                    pending.reject(new Error('CODEX_SERVER_REQUEST_CANCELLED'));
+                handle.pendingResponses.clear();
             },
-            close: () => { },
             onExit: () => () => { },
         };
     };
@@ -158,6 +195,36 @@ async function createProductionAutopilotFixture(overrides = {}) {
     await installAutopilotPlan(app, sessionId, workspacePath, 'autopilot');
     return { app, dataDir, handles, root, sessionId, workspacePath };
 }
+async function holdCheckpointRequest(fixture, timers, requestId, expectHeld = true) {
+    timers.find((timer) => !timer.cancelled).callback();
+    await vi.waitFor(() => expect(fixture.handles.flatMap((candidate) => candidate.requests.filter((request) => request.method === 'turn/start'))).toHaveLength(1));
+    const handle = fixture.handles.find((candidate) => candidate.requests.some((request) => request.method === 'turn/start'));
+    await vi.waitFor(() => expect(handle.request).toBeDefined());
+    const session = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json();
+    const threadId = session.threadId;
+    const turnId = session.activeTurnId;
+    handle.notify({ method: 'turn/started', params: { threadId, turn: { id: turnId } } });
+    await writeFile(join(fixture.workspacePath, 'autopilot.org'), autopilotPlanText('DONE'));
+    await expect
+        .poll(async () => (await fixture.app.inject(`/api/sessions/${fixture.sessionId}/plan`)).json().steps[0]
+        .children[0].state)
+        .toBe('DONE');
+    handle.deferResponseSettlement = true;
+    const planIdentity = createHash('sha256')
+        .update(join(fixture.workspacePath, 'autopilot.org'))
+        .digest('hex');
+    const writersBefore = fixture.handles.length;
+    const startsBefore = fixture.handles
+        .flatMap((candidate) => candidate.requests)
+        .filter((request) => request.method === 'turn/start').length;
+    const response = handle.request(l2CheckpointCall(requestId, threadId, turnId, planIdentity));
+    // The caller can be synchronously rejected by a coordinator exception.
+    // Mark it observed immediately; callers still assert its exact outcome.
+    void response.catch(() => { });
+    if (expectHeld)
+        await vi.waitFor(() => expect(handle.pendingResponses.has(requestId)).toBe(true));
+    return { handle, response, startsBefore, threadId, turnId, writersBefore };
+}
 function attentionCall(id, reason = 'hardBlock') {
     return {
         id,
@@ -173,6 +240,29 @@ function attentionCall(id, reason = 'hardBlock') {
         },
     };
 }
+function l2CheckpointCall(id, threadId, turnId, planIdentity) {
+    return {
+        id,
+        method: 'item/tool/call',
+        params: {
+            threadId,
+            turnId,
+            tool: 'gestalt_org_plan_checkpoint',
+            arguments: {
+                version: 1,
+                kind: 'l2Completed',
+                planIdentity,
+                l1Id: 'parent',
+                l2Id: 'child',
+                position: 'L1.1',
+                status: 'DONE',
+                changes: 'Recorded the completed child.',
+                files: 'src/checkpoint.ts',
+                tests: 'Focused tests passed.',
+            },
+        },
+    };
+}
 function ownTemporaryPaths(...paths) {
     onTestFinished(async () => {
         await Promise.all(paths.map((path) => rm(path, { recursive: true, force: true })));
@@ -632,6 +722,585 @@ describe('production composition', () => {
             expect(accepted.handles.flatMap((handle) => handle.calls)).not.toContain('turn/start');
             await accepted.app.close();
         });
+        it('acknowledges a checkpoint before scheduling and drops an obsolete executor wait lease', async () => {
+            const timers = [];
+            let coordinator;
+            const fixture = await createProductionAutopilotFixture({
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false, fired: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+                onAutopilotCoordinator: (value) => {
+                    coordinator = value;
+                },
+            });
+            const first = timers.find((timer) => !timer.cancelled && !timer.fired);
+            expect(first).toBeDefined();
+            first.fired = true;
+            first.callback();
+            await vi.waitFor(() => expect(fixture.handles.flatMap((handle) => handle.requests.filter((request) => request.method === 'turn/start'))).toHaveLength(1));
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const active = database
+                .prepare('SELECT active_turn_id FROM relay_sessions WHERE id = ?')
+                .get(fixture.sessionId);
+            database.close();
+            expect(coordinator.registerProactiveWait(fixture.sessionId, {
+                id: 'pre-checkpoint-report',
+                leaseId: 'pre-checkpoint-lease',
+                wakeConditions: ['executorChanged'],
+                maxWaitMs: 60_000,
+            })).toBe(true);
+            await writeFile(join(fixture.workspacePath, 'autopilot.org'), autopilotPlanText('DONE'));
+            await expect
+                .poll(async () => (await fixture.app.inject(`/api/sessions/${fixture.sessionId}/plan`)).json().steps[0]
+                .children[0].state)
+                .toBe('DONE');
+            const handle = fixture.handles.find((candidate) => candidate.requests.some((request) => request.method === 'turn/start'));
+            const threadId = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json()
+                .threadId;
+            const schedulesBeforeCheckpoint = timers.length;
+            const planIdentity = createHash('sha256')
+                .update(join(fixture.workspacePath, 'autopilot.org'))
+                .digest('hex');
+            handle.deferResponseSettlement = true;
+            const checkpointResponse = handle.request(l2CheckpointCall(811, threadId, active.active_turn_id, planIdentity));
+            await vi.waitFor(() => expect(handle.pendingResponses.has(811)).toBe(true));
+            const beforeWriteDatabase = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const beforeWrite = beforeWriteDatabase
+                .prepare('SELECT resolved_at FROM pending_interactions WHERE session_id = ? AND request_id = ?')
+                .get(fixture.sessionId, '811');
+            beforeWriteDatabase.close();
+            expect(beforeWrite.resolved_at).toBeNull();
+            handle.responseSettled({ id: 811, outcome: 'resultWritten' });
+            await expect(checkpointResponse).resolves.toEqual(toOrgPlanCheckpointToolResponse());
+            const afterWriteDatabase = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const afterWrite = afterWriteDatabase
+                .prepare('SELECT resolved_at FROM pending_interactions WHERE session_id = ? AND request_id = ?')
+                .get(fixture.sessionId, '811');
+            afterWriteDatabase.close();
+            expect(afterWrite.resolved_at).toEqual(expect.any(String));
+            expect(timers).toHaveLength(schedulesBeforeCheckpoint);
+            expect(coordinator.snapshot(fixture.sessionId).health).not.toMatchObject({
+                phase: 'waitingForAgentEvent',
+            });
+            const startsBeforeFinal = fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start').length;
+            // A completion from another turn cannot release the durable checkpoint
+            // boundary, including after the short acknowledgement has returned.
+            handle.notify({
+                method: 'turn/completed',
+                params: { threadId, turn: { id: 'wrong-turn' } },
+            });
+            await Promise.resolve();
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(startsBeforeFinal);
+            // Destroy the old runtime and rebuild the coordinator over the same
+            // SQLite store. The persisted checkpoint remains inert until its owning
+            // root final is observed by the replacement runtime.
+            await fixture.app.close();
+            const rebuiltTimers = [];
+            const rebuiltHandles = [];
+            let rebuiltCoordinator;
+            const rebuilt = await composeAuthorizedApp({
+                root: fixture.root,
+                dataDir: fixture.dataDir,
+                relyingParty,
+                installedCodexVersion: 'codex-cli 0.144.3',
+                startAppServers: true,
+                launchAppServer: liveAppServer(rebuiltHandles),
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false, fired: false };
+                    rebuiltTimers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+                onAutopilotCoordinator: (value) => {
+                    rebuiltCoordinator = value;
+                },
+                profiles: {
+                    list: async () => [],
+                    require: async () => ({
+                        name: 'default',
+                        state: 'ok',
+                        status: 'ready',
+                    }),
+                },
+            });
+            await vi.waitFor(() => expect(rebuiltCoordinator).toBeDefined());
+            expect(rebuiltTimers).toHaveLength(0);
+            expect(rebuiltCoordinator.turnCompleted(fixture.sessionId)).toBe(true);
+            await vi.waitFor(() => expect(rebuiltTimers.length).toBe(1));
+            await rebuilt.close();
+        });
+        it('recycles a stuck checkpoint writer at the acknowledgement deadline and recovers once', async () => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                checkpointHandoffDeadlineMs: 100,
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            timers.find((timer) => !timer.cancelled).callback();
+            await vi.waitFor(() => expect(fixture.handles.flatMap((candidate) => candidate.requests.filter((request) => request.method === 'turn/start'))).toHaveLength(1));
+            await vi.waitFor(() => expect(fixture.handles.some((candidate) => candidate.request)).toBe(true));
+            const handle = fixture.handles.find((candidate) => candidate.requests.some((request) => request.method === 'turn/start'));
+            const threadId = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json()
+                .threadId;
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const active = database
+                .prepare('SELECT active_turn_id FROM relay_sessions WHERE id = ?')
+                .get(fixture.sessionId);
+            database.close();
+            await writeFile(join(fixture.workspacePath, 'autopilot.org'), autopilotPlanText('DONE'));
+            await expect
+                .poll(async () => (await fixture.app.inject(`/api/sessions/${fixture.sessionId}/plan`)).json().steps[0]
+                .children[0].state)
+                .toBe('DONE');
+            handle.notify({
+                method: 'turn/started',
+                params: { threadId, turn: { id: active.active_turn_id } },
+            });
+            handle.deferResponseSettlement = true;
+            const identity = createHash('sha256')
+                .update(join(fixture.workspacePath, 'autopilot.org'))
+                .digest('hex');
+            const checkpointResponse = handle.request(l2CheckpointCall(991, threadId, active.active_turn_id, identity));
+            const rejectedCheckpoint = expect(checkpointResponse).rejects.toThrow('CODEX_SERVER_REQUEST_CANCELLED');
+            await vi.waitFor(() => expect(handle.pendingResponses.has(991)).toBe(true));
+            const timersBeforeRecovery = timers.length;
+            await rejectedCheckpoint;
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1);
+            await vi.waitFor(() => expect(fixture.handles.length).toBeGreaterThan(1), { timeout: 1_000 });
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1);
+            await expect
+                .poll(() => {
+                const audit = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+                const count = audit
+                    .prepare("SELECT count(*) AS count FROM session_events WHERE type = 'org-plan.checkpoint-handoff-failed'")
+                    .get();
+                audit.close();
+                return count.count;
+            })
+                .toBe(1);
+            await expect
+                .poll(() => {
+                const recoveredState = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+                const lifecycle = recoveredState
+                    .prepare('SELECT lifecycle_json FROM autopilot_sessions WHERE session_id = ?')
+                    .get(fixture.sessionId);
+                recoveredState.close();
+                return JSON.parse(lifecycle.lifecycle_json).checkpoints;
+            })
+                .toMatchObject({ pendingTurnId: null, checkpointHandoffFailed: false });
+            await vi.waitFor(() => expect(timers.length).toBeGreaterThan(timersBeforeRecovery));
+            const recoveryTimer = timers.at(-1);
+            recoveryTimer.callback();
+            await vi.waitFor(() => expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(2));
+            await fixture.app.close();
+        });
+        it('leaves an unsupported checkpoint exception before persistence inert', async () => {
+            let coordinator;
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+                onAutopilotCoordinator: (value) => {
+                    coordinator = value;
+                },
+            });
+            const original = coordinator.checkpointAccepted.bind(coordinator);
+            vi.spyOn(coordinator, 'checkpointAccepted').mockImplementation(() => {
+                throw new Error('injected-before-checkpoint-persistence');
+            });
+            const { response, startsBefore, writersBefore } = await holdCheckpointRequest(fixture, timers, 995, false);
+            await expect(response).rejects.toThrow('CODEX_SERVER_REQUEST_UNSUPPORTED');
+            expect(coordinator.checkpointAccepted).not.toBe(original);
+            expect(fixture.handles).toHaveLength(writersBefore);
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(startsBefore);
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const events = database
+                .prepare("SELECT type FROM session_events WHERE type IN ('org-plan.step-checkpointed', 'org-plan.checkpoint-handoff-failed')")
+                .all();
+            const lifecycle = database
+                .prepare('SELECT lifecycle_json FROM autopilot_sessions WHERE session_id = ?')
+                .get(fixture.sessionId);
+            database.close();
+            expect(events).toEqual([]);
+            expect(JSON.parse(lifecycle.lifecycle_json).checkpoints).toBeUndefined();
+            expect((await fixture.app.inject(`/api/sessions/${fixture.sessionId}/history`)).json()
+                .interactions).not.toEqual(expect.arrayContaining([expect.objectContaining({ requestId: '995' })]));
+            await fixture.app.close();
+        });
+        it('turns a post-persistence checkpoint exception into one explicit failed handoff', async () => {
+            let coordinator;
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+                onAutopilotCoordinator: (value) => {
+                    coordinator = value;
+                },
+            });
+            const original = coordinator.checkpointAccepted.bind(coordinator);
+            vi.spyOn(coordinator, 'checkpointAccepted').mockImplementation((...input) => {
+                original(...input);
+                throw new Error('injected-after-checkpoint-persistence');
+            });
+            const { response, startsBefore, writersBefore } = await holdCheckpointRequest(fixture, timers, 996, false);
+            await expect(response).rejects.toThrow('CHECKPOINT_HANDOFF_FAILED');
+            await vi.waitFor(async () => {
+                const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+                const failures = database
+                    .prepare("SELECT count(*) AS count FROM session_events WHERE type = 'org-plan.checkpoint-handoff-failed'")
+                    .get();
+                database.close();
+                expect(failures.count).toBe(1);
+            });
+            const history = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}/history`)).json();
+            expect(history.interactions).toEqual(expect.arrayContaining([
+                expect.objectContaining({
+                    requestId: '996',
+                    kind: 'orgPlanCheckpoint',
+                    outcome: 'failed',
+                }),
+            ]));
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const lifecycle = database
+                .prepare('SELECT lifecycle_json FROM autopilot_sessions WHERE session_id = ?')
+                .get(fixture.sessionId);
+            database.close();
+            expect(JSON.parse(lifecycle.lifecycle_json).checkpoints).toMatchObject({
+                pendingTurnId: expect.any(String),
+                checkpointHandoffFailed: true,
+            });
+            expect(fixture.handles).toHaveLength(writersBefore);
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(startsBefore);
+            await fixture.app.close();
+        });
+        it('restores a pending checkpoint handoff once from the same database', async () => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            const { response } = await holdCheckpointRequest(fixture, timers, 997);
+            await fixture.app.close();
+            await expect(response).rejects.toThrow('CODEX_SERVER_REQUEST_CANCELLED');
+            const rebuiltHandles = [];
+            const rebuiltTimers = [];
+            let rebuiltCoordinator;
+            const rebuilt = await composeAuthorizedApp({
+                root: fixture.root,
+                dataDir: fixture.dataDir,
+                relyingParty,
+                installedCodexVersion: 'codex-cli 0.144.3',
+                startAppServers: true,
+                launchAppServer: liveAppServer(rebuiltHandles),
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    rebuiltTimers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+                onAutopilotCoordinator: (value) => {
+                    rebuiltCoordinator = value;
+                },
+                profiles: {
+                    list: async () => [],
+                    require: async () => ({
+                        name: 'default',
+                        state: 'ok',
+                        status: 'ready',
+                    }),
+                },
+            });
+            await rebuilt.listen({ port: 0 });
+            await vi.waitFor(() => expect(rebuiltCoordinator).toBeDefined());
+            await vi.waitFor(() => expect(rebuiltTimers.some((timer) => !timer.cancelled)).toBe(true));
+            rebuiltTimers.find((timer) => !timer.cancelled).callback();
+            await vi.waitFor(() => expect(rebuiltHandles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1));
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const failures = database
+                .prepare("SELECT count(*) AS count FROM session_events WHERE type = 'org-plan.checkpoint-handoff-failed'")
+                .get();
+            const interaction = database
+                .prepare('SELECT outcome FROM pending_interactions WHERE session_id = ? AND request_id = ?')
+                .get(fixture.sessionId, '997');
+            database.close();
+            expect(failures.count).toBe(1);
+            expect(interaction.outcome).toBe('failed');
+            expect(rebuiltCoordinator.snapshot(fixture.sessionId).health).toMatchObject({
+                phase: expect.not.stringMatching('checkpointRecovery'),
+            });
+            await rebuilt.close();
+        });
+        it.each([
+            'resultWritten-before-timeout',
+            'timeout-before-late-resultWritten',
+            'writeFailed',
+            'duplicate-settlement',
+        ])('settles checkpoint write/deadline races exactly once: %s', async (scenario) => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                checkpointHandoffDeadlineMs: 200,
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            const { handle, response, startsBefore, writersBefore } = await holdCheckpointRequest(fixture, timers, 992);
+            const starts = () => fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start').length;
+            const failures = () => {
+                const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+                const count = database
+                    .prepare("SELECT count(*) AS count FROM session_events WHERE type = 'org-plan.checkpoint-handoff-failed'")
+                    .get();
+                database.close();
+                return count.count;
+            };
+            if (scenario === 'resultWritten-before-timeout') {
+                handle.responseSettled({ id: 992, outcome: 'resultWritten' });
+                await expect(response).resolves.toEqual(toOrgPlanCheckpointToolResponse());
+                await new Promise((resolve) => setTimeout(resolve, 30));
+                expect(fixture.handles).toHaveLength(writersBefore);
+                expect(failures()).toBe(0);
+            }
+            else if (scenario === 'duplicate-settlement') {
+                handle.responseSettled({ id: 992, outcome: 'resultWritten' });
+                handle.responseSettled({ id: 992, outcome: 'resultWritten' });
+                await expect(response).resolves.toEqual(toOrgPlanCheckpointToolResponse());
+                await new Promise((resolve) => setTimeout(resolve, 30));
+                expect(fixture.handles).toHaveLength(writersBefore);
+                expect(failures()).toBe(0);
+            }
+            else {
+                if (scenario === 'writeFailed')
+                    handle.responseSettled({ id: 992, outcome: 'writeFailed' });
+                const lateSettlement = handle.responseSettled;
+                await expect(response).rejects.toThrow('CODEX_SERVER_REQUEST_CANCELLED');
+                await vi.waitFor(() => expect(fixture.handles.length).toBeGreaterThan(writersBefore), {
+                    timeout: 1_000,
+                });
+                if (scenario === 'timeout-before-late-resultWritten')
+                    lateSettlement({ id: 992, outcome: 'resultWritten' });
+                await new Promise((resolve) => setTimeout(resolve, 30));
+                expect(failures()).toBe(1);
+            }
+            expect(starts()).toBe(startsBefore);
+            await fixture.app.close();
+        });
+        it('does not apply checkpoint deadlines to quiz or approval requests', async () => {
+            const checkpointFailures = (dataDir) => {
+                const database = new DatabaseSync(join(dataDir, 'relay.sqlite'));
+                const count = database
+                    .prepare("SELECT count(*) AS count FROM session_events WHERE type = 'org-plan.checkpoint-handoff-failed'")
+                    .get();
+                database.close();
+                return count.count;
+            };
+            const fixture = await createProductionAutopilotFixture({ checkpointHandoffDeadlineMs: 1 });
+            const handle = fixture.handles.find((candidate) => candidate.request);
+            const started = await fixture.app.inject({
+                method: 'POST',
+                url: `/api/sessions/${fixture.sessionId}/turns`,
+                payload: { text: 'request user input' },
+            });
+            expect(started.statusCode).toBe(202);
+            await vi.waitFor(() => expect(handle.request).toBeDefined());
+            handle.notify({
+                method: 'turn/started',
+                params: {
+                    threadId: (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json()
+                        .threadId,
+                    turn: { id: started.json().activeTurnId },
+                },
+            });
+            handle.deferResponseSettlement = true;
+            const quizWriters = fixture.handles.length;
+            const quiz = handle.request({
+                id: 993,
+                method: 'item/tool/call',
+                params: {
+                    tool: 'gestalt_quiz',
+                    arguments: {
+                        questions: [
+                            {
+                                id: 'mode',
+                                header: 'Mode',
+                                question: 'Choose a mode.',
+                                choices: [
+                                    { label: 'Solo', description: 'Use one agent.' },
+                                    { label: 'Team', description: 'Use several agents.' },
+                                ],
+                                allowCustom: false,
+                            },
+                        ],
+                    },
+                },
+            });
+            await vi.waitFor(async () => expect((await fixture.app.inject(`/api/sessions/${fixture.sessionId}/history`)).json()
+                .interactions).toEqual(expect.arrayContaining([
+                expect.objectContaining({ requestId: '993', kind: 'quiz', resolvedAt: null }),
+            ])));
+            await new Promise((resolve) => setTimeout(resolve, 20));
+            expect(fixture.handles).toHaveLength(quizWriters);
+            expect(checkpointFailures(fixture.dataDir)).toBe(0);
+            expect(handle.pendingResponses).toHaveLength(0);
+            expect((await fixture.app.inject(`/api/sessions/${fixture.sessionId}/history`)).json()
+                .interactions).toEqual(expect.arrayContaining([expect.objectContaining({ requestId: '993', resolvedAt: null })]));
+            const quizSubmission = fixture.app.inject({
+                method: 'POST',
+                url: `/api/sessions/${fixture.sessionId}/interactions/993`,
+                payload: {
+                    success: true,
+                    contentItems: [{ type: 'inputText', text: '{"answers":{"mode":"Solo"}}' }],
+                },
+            });
+            await vi.waitFor(() => {
+                expect(handle.pendingResponses.has(993)).toBe(true);
+            });
+            handle.responseSettled({ id: 993, outcome: 'resultWritten' });
+            expect((await quizSubmission).statusCode).toBe(202);
+            await expect(quiz).resolves.toEqual(expect.anything());
+            expect(fixture.handles).toHaveLength(quizWriters);
+            await fixture.app.close();
+            const approvalFixture = await createProductionAutopilotFixture({
+                checkpointHandoffDeadlineMs: 1,
+            });
+            const approvalHandle = approvalFixture.handles.find((candidate) => candidate.request);
+            const approvalTurn = await approvalFixture.app.inject({
+                method: 'POST',
+                url: `/api/sessions/${approvalFixture.sessionId}/turns`,
+                payload: { text: 'approve' },
+            });
+            await vi.waitFor(() => expect(approvalHandle.request).toBeDefined());
+            approvalHandle.notify({
+                method: 'turn/started',
+                params: {
+                    threadId: (await approvalFixture.app.inject(`/api/sessions/${approvalFixture.sessionId}`)).json().threadId,
+                    turn: { id: approvalTurn.json().activeTurnId },
+                },
+            });
+            approvalHandle.deferResponseSettlement = true;
+            const approvalWriters = approvalFixture.handles.length;
+            const approval = approvalHandle.request({
+                id: 994,
+                method: 'item/commandExecution/requestApproval',
+                params: { command: 'git status' },
+            });
+            await vi.waitFor(async () => expect((await approvalFixture.app.inject(`/api/sessions/${approvalFixture.sessionId}/history`)).json().interactions).toEqual(expect.arrayContaining([
+                expect.objectContaining({
+                    requestId: '994',
+                    kind: 'commandApproval',
+                    resolvedAt: null,
+                }),
+            ])));
+            await new Promise((resolve) => setTimeout(resolve, 20));
+            expect(approvalFixture.handles).toHaveLength(approvalWriters);
+            expect(checkpointFailures(approvalFixture.dataDir)).toBe(0);
+            expect((await approvalFixture.app.inject(`/api/sessions/${approvalFixture.sessionId}/history`)).json().interactions).toEqual(expect.arrayContaining([
+                expect.objectContaining({ requestId: '994', kind: 'commandApproval', resolvedAt: null }),
+            ]));
+            const approvalSubmission = approvalFixture.app.inject({
+                method: 'POST',
+                url: `/api/sessions/${approvalFixture.sessionId}/interactions/994`,
+                payload: { decision: 'accept' },
+            });
+            await vi.waitFor(() => expect(approvalHandle.pendingResponses.has(994)).toBe(true));
+            approvalHandle.responseSettled({ id: 994, outcome: 'resultWritten' });
+            expect((await approvalSubmission).statusCode).toBe(202);
+            await expect(approval).resolves.toEqual(expect.anything());
+            expect(approvalTurn.statusCode).toBe(202);
+            await approvalFixture.app.close();
+        });
+        it('rejects a checkpoint delivered after its owning root final has closed', async () => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            const first = timers[0];
+            first.callback();
+            await vi.waitFor(() => expect(fixture.handles
+                .flatMap((handle) => handle.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1));
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const active = database
+                .prepare('SELECT active_turn_id FROM relay_sessions WHERE id = ?')
+                .get(fixture.sessionId);
+            database.close();
+            const handle = fixture.handles.find((candidate) => candidate.notify && candidate.request);
+            const threadId = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json()
+                .threadId;
+            handle.notify({
+                method: 'turn/completed',
+                params: { threadId, turn: { id: active.active_turn_id } },
+            });
+            await expect
+                .poll(async () => (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json().activeTurnId)
+                .toBeNull();
+            const timersAfterFinal = timers.length;
+            const startsAfterFinal = fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start').length;
+            const planIdentity = createHash('sha256')
+                .update(join(fixture.workspacePath, 'autopilot.org'))
+                .digest('hex');
+            await expect(handle.request(l2CheckpointCall(812, threadId, active.active_turn_id, planIdentity))).rejects.toThrow('CODEX_SERVER_REQUEST_UNSUPPORTED');
+            expect(timers).toHaveLength(timersAfterFinal);
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(startsAfterFinal);
+            await fixture.app.close();
+        });
         it('production attention request requires explicit re-enable and a complete plan transitions to completed', async () => {
             const fixture = await createProductionAutopilotFixture();
             const handle = fixture.handles.find((candidate) => candidate.request);
@@ -867,6 +1536,211 @@ describe('production composition', () => {
             expect(fixture.handles.flatMap((candidate) => candidate.requests.filter((call) => call.method === 'turn/start'))[0]?.params).toMatchObject({ threadId: 'child-1' });
             await fixture.app.close();
         });
+        it('production treats only a confirmed missing rollout as an executor rejection and starts one fenced replacement', async () => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotActivity: (sessionId) => {
+                    const observedAt = new Date().toISOString();
+                    return {
+                        sessionId,
+                        rootThreadId: 'thread-1',
+                        root: {
+                            state: 'idle',
+                            reason: 'turnCompleted',
+                            observedAt,
+                            lastActivityAt: observedAt,
+                        },
+                        subagents: [
+                            {
+                                id: 'child-1',
+                                threadId: 'child-1',
+                                taskPath: '/root/l1',
+                                canonicalTaskName: 'l1',
+                                canonicalPosition: 'L1',
+                                continuationGeneration: 1,
+                                outcome: 'partial',
+                                ownedProcesses: [],
+                                state: 'idle',
+                                reason: 'turnCompleted',
+                                observedAt,
+                                lastActivityAt: observedAt,
+                            },
+                        ],
+                        aggregateSubagents: 'idle',
+                        confidence: 'fresh',
+                    };
+                },
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false, fired: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            // Session setup may have used a retired writer; apply the failure seam to
+            // every live writer that can own the executor continuation.
+            for (const handle of fixture.handles)
+                handle.respond = (method, params) => {
+                    if (method === 'turn/start' && params.threadId === 'child-1')
+                        throw new CodexJsonRpcError(-32600, 'no rollout found for thread id child-1');
+                    return undefined;
+                };
+            const fireNext = async () => {
+                await vi.waitFor(() => expect(timers.some((candidate) => !candidate.cancelled && !candidate.fired)).toBe(true));
+                const timer = timers.find((candidate) => !candidate.cancelled && !candidate.fired);
+                expect(timer).toBeDefined();
+                timer.fired = true;
+                timer.callback();
+                await Promise.resolve();
+                await Promise.resolve();
+            };
+            expect((await fixture.app.inject({
+                method: 'PUT',
+                url: `/api/sessions/${fixture.sessionId}/autopilot`,
+                payload: { enabled: true },
+            })).statusCode).toBe(200);
+            // Three confirmed rejections consume the bounded executor retry budget;
+            // the fourth scheduled control is the root-owned physical g2 handoff.
+            await fireNext();
+            await fireNext();
+            await fireNext();
+            await fireNext();
+            await fireNext();
+            await fireNext();
+            await fireNext();
+            await vi.waitFor(() => expect(fixture.handles.flatMap((candidate) => candidate.requests.filter((request) => request.method === 'turn/start'))).toHaveLength(4));
+            const starts = fixture.handles.flatMap((candidate) => candidate.requests.filter((request) => request.method === 'turn/start'));
+            expect(starts.slice(0, 3).map((request) => request.params)).toEqual(Array.from({ length: 3 }, () => expect.objectContaining({ threadId: 'child-1' })));
+            // The root control may reacquire a new app-server writer after the
+            // rejected child continuation; it must never target the dead child.
+            expect((starts[3]?.params).threadId).not.toBe('child-1');
+            expect((starts[3]?.params).input?.[0]?.text).toContain('Launch task_name l1_g2 for canonical L1');
+            expect((starts[3]?.params).input?.[0]?.text).toContain('explicit model selected by the supervisor');
+            // Capacity recovery is allowed only for the active root control. It
+            // recycles that writer but must retain the one durable g2 handoff.
+            const rootHandle = fixture.handles.find((candidate) => candidate.requests.some((request) => request.method === 'turn/start' &&
+                request.params.threadId !== 'child-1'));
+            const active = (await fixture.app.inject(`/api/sessions/${fixture.sessionId}`)).json();
+            const writersBeforeRecovery = fixture.handles.length;
+            await expect(rootHandle.request({
+                id: 992,
+                method: 'item/tool/call',
+                params: {
+                    threadId: active.threadId,
+                    turnId: active.activeTurnId,
+                    tool: 'gestalt_agent_capacity_recovery',
+                    arguments: { version: 1, reason: 'agentThreadLimit' },
+                },
+            })).resolves.toMatchObject({ success: true });
+            await expect.poll(() => fixture.handles.length).toBeGreaterThan(writersBeforeRecovery);
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')
+                .map((request) => request.params.input?.[0]?.text)).not.toContain(expect.stringContaining('l1_g3'));
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const lifecycle = database
+                .prepare('SELECT lifecycle_json FROM autopilot_sessions WHERE session_id = ?')
+                .get(fixture.sessionId);
+            const replacements = database
+                .prepare("SELECT count(*) AS count FROM session_events WHERE session_id = ? AND type = 'autopilot.executor-replacement-scheduled'")
+                .get(fixture.sessionId);
+            database.close();
+            expect(JSON.parse(lifecycle.lifecycle_json)).toMatchObject({
+                executor: {
+                    replacement: { taskName: 'l1_g2', canonicalTaskName: 'l1', generation: 2 },
+                    commands: expect.arrayContaining([
+                        expect.objectContaining({ status: 'failed', threadId: 'child-1' }),
+                    ]),
+                },
+            });
+            expect(replacements.count).toBe(1);
+            await fixture.app.close();
+        });
+        it('production leaves a transport-lost executor command issued and never replays it as a replacement', async () => {
+            const timers = [];
+            const fixture = await createProductionAutopilotFixture({
+                autopilotActivity: (sessionId) => {
+                    const observedAt = new Date().toISOString();
+                    return {
+                        sessionId,
+                        rootThreadId: 'thread-1',
+                        root: {
+                            state: 'idle',
+                            reason: 'turnCompleted',
+                            observedAt,
+                            lastActivityAt: observedAt,
+                        },
+                        subagents: [
+                            {
+                                id: 'child-1',
+                                threadId: 'child-1',
+                                taskPath: '/root/l1',
+                                canonicalTaskName: 'l1',
+                                canonicalPosition: 'L1',
+                                continuationGeneration: 1,
+                                outcome: 'partial',
+                                ownedProcesses: [],
+                                state: 'idle',
+                                reason: 'turnCompleted',
+                                observedAt,
+                                lastActivityAt: observedAt,
+                            },
+                        ],
+                        aggregateSubagents: 'idle',
+                        confidence: 'fresh',
+                    };
+                },
+                autopilotSchedule: (callback) => {
+                    const timer = { callback, cancelled: false, fired: false };
+                    timers.push(timer);
+                    return () => {
+                        timer.cancelled = true;
+                    };
+                },
+            });
+            for (const handle of fixture.handles)
+                handle.respond = (method, params) => {
+                    if (method === 'turn/start' && params.threadId === 'child-1')
+                        throw new Error('transport dropped after request write');
+                    return undefined;
+                };
+            expect((await fixture.app.inject({
+                method: 'PUT',
+                url: `/api/sessions/${fixture.sessionId}/autopilot`,
+                payload: { enabled: true },
+            })).statusCode).toBe(200);
+            const timer = timers.find((candidate) => !candidate.cancelled && !candidate.fired);
+            timer.fired = true;
+            timer.callback();
+            await vi.waitFor(() => expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1));
+            // Drive the bounded reinspection once: the durable issued command blocks replay.
+            const refresh = timers.find((candidate) => !candidate.cancelled && !candidate.fired);
+            refresh.fired = true;
+            refresh.callback();
+            await Promise.resolve();
+            await Promise.resolve();
+            const database = new DatabaseSync(join(fixture.dataDir, 'relay.sqlite'));
+            const lifecycle = database
+                .prepare('SELECT lifecycle_json FROM autopilot_sessions WHERE session_id = ?')
+                .get(fixture.sessionId);
+            const replacements = database
+                .prepare("SELECT count(*) AS count FROM session_events WHERE session_id = ? AND type = 'autopilot.executor-replacement-scheduled'")
+                .get(fixture.sessionId);
+            database.close();
+            expect(JSON.parse(lifecycle.lifecycle_json)).toMatchObject({
+                executor: {
+                    commands: [expect.objectContaining({ status: 'issued', threadId: 'child-1' })],
+                },
+            });
+            expect(replacements.count).toBe(0);
+            expect(fixture.handles
+                .flatMap((candidate) => candidate.requests)
+                .filter((request) => request.method === 'turn/start')).toHaveLength(1);
+            await fixture.app.close();
+        });
         it('production blocked child wakes the idle supervisor for bounded recovery', async () => {
             const timers = [];
             const fixture = await createProductionAutopilotFixture({