gestalt-mobile 0.35.1 → 0.35.3

package/dist/server/server/composition.js

@@ -16,6 +16,7 @@ import { CodexModelCatalog } from './platform/codex/codex-model-catalog.js';
 import { createRecentThreadLister } from './platform/codex/recent-thread-lister.js';
 import { CodexSessionRuntime } from './platform/codex/session-runtime.js';
 import { normalizeCodexNotification } from './platform/codex/normalizer.js';
+import { isMissingCodexThreadRollout } from './platform/codex/json-rpc-client.js';
 import { migrate } from './platform/persistence/migrate.js';
 import { openRelayDatabase } from './platform/persistence/sqlite.js';
 import { SqliteAuthorizationStore } from './platform/auth/sqlite-authorization-store.js';
@@ -93,6 +94,8 @@ export async function composeRelayApp(options) {
     const idempotency = new SqliteIdempotencyStore(database);
     const autopilotStore = new SqliteAutopilotStore(database);
     const attentionResolutionOperations = new Map();
+    const checkpointHandoffTimers = new Map();
+    const checkpointHandoffRecoveries = new Set();
     const supervisedPlans = new SupervisedPlanRegistry();
     const planStatusSource = new FilesystemPlanStatusSource(join(dirname(databasePath), 'plans'));
     const planMeasurementHelperPath = options.planMeasurementHelperPath ?? process.env.GESTALT_MOBILE_ORG_PLAN_HELPER;
@@ -263,7 +266,17 @@ export async function composeRelayApp(options) {
                     .update(`${sessionId}:${threadId}:${generation}:${randomUUID()}`)
                     .digest('hex')
                     .slice(0, 16)}`;
-                const turnId = await runtime.startExecutorTurn(writer.session, threadId, `${AUTOPILOT_EXECUTOR_CONTINUATION_PROMPT}${context}`, clientId);
+                let turnId;
+                try {
+                    turnId = await runtime.startExecutorTurn(writer.session, threadId, `${AUTOPILOT_EXECUTOR_CONTINUATION_PROMPT}${context}`, clientId);
+                }
+                catch (error) {
+                    // A confirmed app-server missing-rollout response is a rejection;
+                    // transport loss remains ambiguous and must retain the issued fence.
+                    if (isMissingCodexThreadRollout(error))
+                        throw new Error('AUTOPILOT_EXECUTOR_REJECTED');
+                    throw error;
+                }
                 activity.observe({
                     sessionId,
                     occurredAt: new Date().toISOString(),
@@ -273,16 +286,21 @@ export async function composeRelayApp(options) {
                 });
             },
             refresh: (sessionId) => activity.refresh(sessionId),
-            transferProcess: (sessionId, threadId, processId) => {
+            interrupt: async (sessionId, threadId) => {
+                const session = sessions.find(sessionId);
+                return session && runtime ? runtime.interruptExecutor(session, threadId) : false;
+            },
+            transferProcess: (sessionId, threadId, processId, actionId) => {
+                void actionId;
                 activity.transferProcessOwnership(sessionId, threadId, processId, new Date().toISOString());
             },
-            consumeProcess: (sessionId, threadId, processId) => {
-                runtime?.consumeChildProcessResult(sessionId, threadId, processId);
+            consumeProcess: (sessionId, threadId, processId, _actionId) => {
+                runtime?.consumeChildProcessResult(sessionId, threadId, processId, _actionId);
             },
-            terminateProcess: async (sessionId, threadId, processId) => {
+            terminateProcess: async (sessionId, threadId, processId, actionId, processInstance) => {
                 const session = sessions.find(sessionId);
                 return session && runtime
-                    ? runtime.terminateChildProcess(session, threadId, processId)
+                    ? runtime.terminateChildProcess(session, threadId, processId, actionId, processInstance)
                     : false;
             },
         },
@@ -381,6 +399,91 @@ export async function composeRelayApp(options) {
                 publishInteractionResolved(sessionId, interaction.requestId, occurredAt, outcome);
         }
     };
+    const checkpointTimerKey = (sessionId, requestId) => `${sessionId}:${requestId}`;
+    const cancelCheckpointHandoffTimer = (sessionId, requestId) => {
+        const key = checkpointTimerKey(sessionId, requestId);
+        const timer = checkpointHandoffTimers.get(key);
+        if (!timer)
+            return;
+        clearTimeout(timer.handle);
+        checkpointHandoffTimers.delete(key);
+    };
+    /** Fail a held checkpoint exactly once and retain its durable root boundary. */
+    const failCheckpointHandoff = (sessionId, requestId, turnId) => {
+        cancelCheckpointHandoffTimer(sessionId, requestId);
+        const occurredAt = new Date().toISOString();
+        if (!interactions.resolve(sessionId, requestId, occurredAt, 'failed'))
+            return false;
+        autopilot.checkpointHandoffFailed(sessionId, turnId);
+        events.publish(journal.append(sessionId, 'org-plan.checkpoint-handoff-failed', { requestId, reason: 'checkpointHandoffFailed' }, occurredAt));
+        publishAttentionSettlement(sessionId, requestId, occurredAt, 'failed');
+        // Neither this timeout nor a failed transport may schedule continuation.
+        // If a successful response is already blocked in the writable stream, drop
+        // its local ownership so any late write cannot be mistaken for an ack.
+        const rejected = runtime?.rejectServerRequest(sessionId, requestId, new Error('CHECKPOINT_HANDOFF_FAILED'));
+        if (rejected === false) {
+            const recoveryKey = checkpointTimerKey(sessionId, requestId);
+            const ownedRuntime = runtime;
+            const session = sessions.find(sessionId);
+            if (ownedRuntime &&
+                session?.activeTurnId === turnId &&
+                !checkpointHandoffRecoveries.has(recoveryKey)) {
+                checkpointHandoffRecoveries.add(recoveryKey);
+                void (async () => {
+                    try {
+                        // Recycle only the session-local writer.  Its close cancels the
+                        // blocked output, and only successful reconstruction releases the
+                        // durable checkpoint through the runtime-recovery boundary.
+                        if (runtime !== ownedRuntime || sessions.find(sessionId)?.activeTurnId !== turnId)
+                            return;
+                        const rebuilt = await ownedRuntime.recycle(session, new Date().toISOString());
+                        if (runtime !== ownedRuntime)
+                            return;
+                        const recovered = turnId && rebuilt.activeTurnId === turnId
+                            ? RelaySession.rehydrate(rebuilt).completeTurn(turnId, new Date().toISOString())
+                                .snapshot
+                            : rebuilt;
+                        saveSession(recovered);
+                        activity.observe({
+                            sessionId,
+                            occurredAt: new Date().toISOString(),
+                            kind: 'turnCompleted',
+                        });
+                        autopilot.recoverCheckpointHandoff(sessionId);
+                    }
+                    catch {
+                        // The durable failure remains visible; a later normal recovery can retry.
+                    }
+                    finally {
+                        checkpointHandoffRecoveries.delete(recoveryKey);
+                    }
+                })();
+            }
+        }
+        return true;
+    };
+    const armCheckpointHandoffDeadline = (sessionId, requestId, turnId) => {
+        cancelCheckpointHandoffTimer(sessionId, requestId);
+        const key = checkpointTimerKey(sessionId, requestId);
+        const token = Symbol(key);
+        const handle = setTimeout(() => {
+            if (checkpointHandoffTimers.get(key)?.token !== token)
+                return;
+            checkpointHandoffTimers.delete(key);
+            failCheckpointHandoff(sessionId, requestId, turnId);
+        }, options.checkpointHandoffDeadlineMs ?? 5_000);
+        handle.unref?.();
+        checkpointHandoffTimers.set(key, { handle, token });
+    };
+    const failPendingCheckpointHandoffs = (sessionId, recoverBoundary) => {
+        for (const interaction of interactions.list(sessionId)) {
+            if (interaction.kind !== 'orgPlanCheckpoint')
+                continue;
+            failCheckpointHandoff(sessionId, interaction.requestId, interaction.turnId ?? null);
+        }
+        if (recoverBoundary)
+            autopilot.recoverCheckpointHandoff(sessionId);
+    };
     let closing = false;
     let runtime = null;
     const acceptPlanUpdate = (sessionId, update) => {
@@ -478,7 +581,7 @@ export async function composeRelayApp(options) {
                 const rootOwned = origin.kind === 'root' && origin.physicalTurnId === session.activeTurnId;
                 if (rootOwned)
                     autopilot.rejectProbe(sessionId);
-                return (runtime?.resolveServerRequest(sessionId, String(request.id), autopilotWaitLeaseToolResponse()) === true);
+                return (runtime?.resolveServerRequest(sessionId, String(request.id), autopilotWaitLeaseToolResponse(false)) === true);
             }
             if (!rawInteraction)
                 return false;
@@ -502,7 +605,7 @@ export async function composeRelayApp(options) {
                 if (!accepted) {
                     if (rootOwned)
                         autopilot.rejectProbe(sessionId);
-                    return (runtime?.resolveServerRequest(sessionId, rawInteraction.requestId, autopilotWaitLeaseToolResponse()) === true);
+                    return (runtime?.resolveServerRequest(sessionId, rawInteraction.requestId, autopilotWaitLeaseToolResponse(false)) === true);
                 }
                 return (runtime?.resolveServerRequest(sessionId, rawInteraction.requestId, autopilotWaitLeaseToolResponse()) === true);
             }
@@ -526,9 +629,42 @@ export async function composeRelayApp(options) {
                 });
                 if (!valid)
                     return false;
-                if (!autopilot.checkpointAccepted(sessionId, checkpoint, session.activeTurnId, new Date().toISOString()))
+                if (!session.activeTurnId)
+                    return false;
+                let accepted = false;
+                try {
+                    accepted = autopilot.checkpointAccepted(sessionId, checkpoint, session.activeTurnId, new Date().toISOString());
+                }
+                catch {
+                    if (!autopilot.checkpointHandoffFailed(sessionId, session.activeTurnId))
+                        return false;
+                    const interaction = {
+                        ...rawInteraction,
+                        turnId: session.activeTurnId,
+                        requestedAt: new Date().toISOString(),
+                    };
+                    interactions.add(sessionId, interaction);
+                    failCheckpointHandoff(sessionId, rawInteraction.requestId, session.activeTurnId);
+                    return true;
+                }
+                if (!accepted)
                     return false;
-                return (runtime?.resolveServerRequest(sessionId, rawInteraction.requestId, toOrgPlanCheckpointToolResponse()) === true);
+                const interaction = {
+                    ...rawInteraction,
+                    turnId: session.activeTurnId,
+                    requestedAt: new Date().toISOString(),
+                };
+                interactions.add(sessionId, interaction);
+                armCheckpointHandoffDeadline(sessionId, rawInteraction.requestId, session.activeTurnId);
+                try {
+                    runtime?.resolveServerRequest(sessionId, rawInteraction.requestId, toOrgPlanCheckpointToolResponse());
+                }
+                catch {
+                    // The accepted hold remains owned by the bounded deadline.
+                }
+                // `true` retains the checkpoint-specific held request. Its timer
+                // is cancelled solely by the JSON-RPC write-settlement callback.
+                return true;
             }
             const interaction = {
                 ...rawInteraction,
@@ -554,10 +690,24 @@ export async function composeRelayApp(options) {
             return true;
         }, (sessionId) => {
             activity.disconnected(sessionId, new Date().toISOString());
+            failPendingCheckpointHandoffs(sessionId, false);
             dismissPendingInteractions(sessionId, new Date().toISOString(), 'failed');
             recoverExitedSession(sessionId);
         }, resolveSkills, planStatusSource, acceptPlanUpdate, options.planMeasurementBaseUrl, 30_000, 64, root)
         : null;
+    runtime?.onServerResponseSettled((sessionId, requestId, outcome) => {
+        const interaction = interactions.find(sessionId, requestId);
+        if (interaction?.kind !== 'orgPlanCheckpoint')
+            return;
+        if (outcome !== 'resultWritten') {
+            failCheckpointHandoff(sessionId, requestId, interaction.turnId ?? null);
+            return;
+        }
+        cancelCheckpointHandoffTimer(sessionId, requestId);
+        const occurredAt = new Date().toISOString();
+        if (interactions.resolve(sessionId, requestId, occurredAt, 'answered'))
+            publishAttentionSettlement(sessionId, requestId, occurredAt, 'answered');
+    });
     if (runtime && planMeasurementHelperPath) {
         planMeasurementRefresh = new PlanMeasurementRefresh(async (sessionId) => {
             const session = sessions.find(sessionId);
@@ -1069,6 +1219,10 @@ export async function composeRelayApp(options) {
             }
             await runtime?.watchPlanStatus(session);
             autopilot.restore(session.id);
+            // A prior process cannot resolve its original JSON-RPC request.  Turn
+            // that bounded handoff into one durable failure and release the
+            // checkpoint only through the explicit runtime-recovery boundary.
+            failPendingCheckpointHandoffs(session.id, true);
         });
     };
     app.addHook('onListen', async () => {
@@ -1083,11 +1237,21 @@ export async function composeRelayApp(options) {
         for (const session of sessions.list()) {
             autopilot.dispose(session.id);
             activity.dispose(session.id);
+            for (const key of [...checkpointHandoffTimers.keys()]) {
+                if (key.startsWith(`${session.id}:`)) {
+                    const timer = checkpointHandoffTimers.get(key);
+                    if (timer)
+                        clearTimeout(timer.handle);
+                    checkpointHandoffTimers.delete(key);
+                }
+            }
             // Relay shutdown only releases this process's writer.  A typed attention
             // request remains a durable human-visible blocker for the next relay
             // instance; only an app-server-cleared request is a failed audit outcome.
             for (const interaction of interactions.list(session.id)) {
-                if (interaction.kind === 'orgPlanAttention')
+                // Checkpoints are recovered by the next relay: their original root
+                // boundary is durable and must not be replaced by a shutdown dismissal.
+                if (interaction.kind === 'orgPlanAttention' || interaction.kind === 'orgPlanCheckpoint')
                     continue;
                 const occurredAt = new Date().toISOString();
                 if (interactions.resolve(session.id, interaction.requestId, occurredAt, 'dismissed'))