mixdog 0.9.85 → 0.9.87

package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs

@@ -3,10 +3,10 @@
 // mutex-guarded turn loop (askSession) plus the abort-aware call wrapper
 // (_api_call_with_interrupt). Behavior-preserving move — every runtime
 // helper it used in manager.mjs is now imported from its split module.
-import { createHash } from 'crypto';
+import { createHash, randomUUID } from 'crypto';
 import { getProvider } from '../../providers/registry.mjs';
 import { normalizeCompactType, DEFAULT_COMPACT_TYPE } from '../compact.mjs';
-import { loadSession, saveSessionAsync, saveSessionAsyncDeferred, readSessionLifecycleFromDisk } from '../store.mjs';
+import { loadSession, saveSession, saveSessionAsync, saveSessionAsyncDeferred, readSessionLifecycleFromDisk } from '../store.mjs';
 import { createAbortController } from '../../../../shared/abort-controller.mjs';
 import { logLlmCall } from '../../../../shared/llm/usage-log.mjs';
 import { appendAgentTrace } from '../../agent-trace.mjs';
@@ -56,6 +56,12 @@ import {
     persistCompactedOutgoingAfterAskFailure,
 } from './message-sanitize.mjs';
 import { createTurnInterruptionTracker } from './turn-interruption.mjs';
+import {
+    clearTurnCheckpoint,
+    recoverTurnCheckpoint,
+    turnMessagesForCheckpoint,
+    writeTurnCheckpoint,
+} from './turn-checkpoint.mjs';
 import { _getAgentLoop } from './runtime-loaders.mjs';
 import { getAgentRuntimeSync } from './agent-runtime-singleton.mjs';
 import { recordProviderContextBaseline } from '../loop/compact-policy.mjs';
@@ -208,6 +214,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
     // Hoisted so the outer finally (which runs once after the whole turn loop)
     // can compare against the last turn's generation.
     let askGeneration = 0;
+    let _crashRecoveryChecked = false;
     try {
       // Turn loop (pendingMessages pattern): run the current prompt, then drain
       // any `agent type=send` messages that were queued while this turn was in
@@ -262,6 +269,17 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
         if (preSession.closed === true) {
             throw new SessionClosedError(sessionId, 'session already closed');
         }
+        if (!_crashRecoveryChecked) {
+            _crashRecoveryChecked = true;
+            const recovery = recoverTurnCheckpoint(preSession);
+            if (recovery.changed) {
+                saveSession(preSession, {
+                    sync: true,
+                    expectedGeneration: preSession.generation,
+                });
+                if (recovery.turnToken) clearTurnCheckpoint(sessionId, recovery.turnToken);
+            }
+        }
         // A prior crash/partial-save during compaction may have pinned
         // compaction.lastStage='compacting'. This ask is starting fresh, so
         // recover the stale transient stage before the loop's pre-send compact
@@ -306,11 +324,62 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
         let cancelledUserTurnContent = '';
         let _turnOutgoing = null;
         const _turnInterruption = createTurnInterruptionTracker();
+        const _turnCheckpointToken = randomUUID();
+        const _turnCheckpointStartedAt = Date.now();
+        const _TURN_CHECKPOINT_THROTTLE_MS = 150;
+        let _turnCheckpointTimer = null;
+        let _turnCheckpointStopped = false;
+        let _turnCheckpointLastAt = 0;
+        let _turnCheckpointWarned = false;
+        const _flushTurnCheckpoint = () => {
+            if (_turnCheckpointStopped || !_turnOutgoing || !cancelledUserTurnContent) return false;
+            try {
+                const written = writeTurnCheckpoint({
+                    sessionId,
+                    generation: askGeneration,
+                    turnToken: _turnCheckpointToken,
+                    startedAt: _turnCheckpointStartedAt,
+                    currentUserContent: cancelledUserTurnContent,
+                    turnMessages: turnMessagesForCheckpoint(_turnOutgoing, cancelledUserTurnContent),
+                    interruption: _turnInterruption.snapshot(),
+                });
+                if (written) _turnCheckpointLastAt = Date.now();
+                return written;
+            } catch (error) {
+                if (!_turnCheckpointWarned) {
+                    _turnCheckpointWarned = true;
+                    try { process.stderr.write(`[turn-checkpoint] write failed session=${sessionId}: ${error?.message || error}\n`); } catch {}
+                }
+                return false;
+            }
+        };
+        const _scheduleTurnCheckpoint = (immediate = false) => {
+            if (_turnCheckpointStopped || !_turnOutgoing) return;
+            if (immediate || Date.now() - _turnCheckpointLastAt >= _TURN_CHECKPOINT_THROTTLE_MS) {
+                if (_turnCheckpointTimer) clearTimeout(_turnCheckpointTimer);
+                _turnCheckpointTimer = null;
+                _flushTurnCheckpoint();
+                return;
+            }
+            if (_turnCheckpointTimer) return;
+            const waitMs = Math.max(1, _TURN_CHECKPOINT_THROTTLE_MS - (Date.now() - _turnCheckpointLastAt));
+            _turnCheckpointTimer = setTimeout(() => {
+                _turnCheckpointTimer = null;
+                _flushTurnCheckpoint();
+            }, waitMs);
+            _turnCheckpointTimer.unref?.();
+        };
+        const _stopTurnCheckpoint = () => {
+            _turnCheckpointStopped = true;
+            if (_turnCheckpointTimer) clearTimeout(_turnCheckpointTimer);
+            _turnCheckpointTimer = null;
+        };
         const _sessionStartMetaInjectedBeforeTurn = preSession.sessionStartMetaInjected === true;
         let _interruptionSnapshot = null;
         const _prepareCloseSnapshot = (abortReason) => {
             if (_interruptionSnapshot) return _interruptionSnapshot;
             if (!activeSession) return null;
+            _stopTurnCheckpoint();
             activeSession.liveTurnMessages = null;
             _turnInterruption.restoreTombstonedText();
             const finalized = _turnInterruption.finalize({
@@ -319,6 +388,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                 abortReason,
             });
             activeSession.messages = finalized.messages;
+            delete activeSession.activeTurnCheckpoint;
             if (!finalized.responsePreserved) {
                 if (finalized.userTurnPreserved) {
                     // A non-user detach keeps the provisional prompt but makes
@@ -455,6 +525,15 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
             // user turn is introduced.
             session.messages = filterModelVisibleSessionMessages(outgoing);
             session.liveTurnMessages = outgoing;
+            session.activeTurnCheckpoint = {
+                version: 1,
+                turnToken: _turnCheckpointToken,
+                startedAt: _turnCheckpointStartedAt,
+            };
+            // The sidecar lands synchronously before provider execution. Even
+            // if the async canonical-session preflight save has not reached its
+            // worker when the process is killed, recovery still has the prompt.
+            _scheduleTurnCheckpoint(true);
             saveSessionAsync(session, { expectedGeneration: askGeneration }).catch((err) => {
                 try { process.stderr.write(`[session] preflight user-turn save failed: ${err?.message || err}\n`); } catch {}
             });
@@ -488,6 +567,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
             const agentLoop = await _getAgentLoop();
             const _trackTextDelta = (chunk) => {
                 _turnInterruption.recordTextDelta(chunk);
+                _scheduleTurnCheckpoint();
                 if (typeof askOpts?.onTextDelta === 'function') askOpts.onTextDelta(chunk);
             };
             const _trackTextReset = async (detail) => {
@@ -504,23 +584,28 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                 }
                 if (!acknowledged) return false;
                 _turnInterruption.tombstoneText(detail?.chars);
+                _scheduleTurnCheckpoint(true);
                 return true;
             };
             const _trackReasoningDelta = (chunk) => {
                 _turnInterruption.recordReasoningDelta(chunk);
+                _scheduleTurnCheckpoint();
                 if (typeof askOpts?.onReasoningDelta === 'function') askOpts.onReasoningDelta(chunk);
             };
             const _trackAssistantText = (text) => {
                 _turnInterruption.recordAssistantText(text);
+                _scheduleTurnCheckpoint(true);
                 if (typeof askOpts?.onAssistantText === 'function') askOpts.onAssistantText(text);
             };
             const _trackedOnToolCall = async (iteration, calls) => {
                 _turnInterruption.recordToolCalls(calls);
+                _scheduleTurnCheckpoint(true);
                 if (typeof onToolCall === 'function') return await onToolCall(iteration, calls);
                 return undefined;
             };
             const _trackToolResult = (message) => {
                 _turnInterruption.recordToolResult(message);
+                _scheduleTurnCheckpoint(true);
                 if (typeof askOpts?.onToolResult === 'function') askOpts.onToolResult(message);
             };
             const priorToolApprovalHook = session.toolApprovalHook;
@@ -539,10 +624,22 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                     onReasoningDelta: _trackReasoningDelta,
                     onAssistantText: _trackAssistantText,
                     takeAssistantTranscriptMetadata: _takeAssistantTranscriptMetadata,
-                    onAssistantMessageCommitted: () => _turnInterruption.markAssistantMessageCommitted(),
-                    onAssistantToolCallObserved: (call, detail) => _turnInterruption.recordToolCalls([call], detail),
-                    onProviderSendStarted: () => _turnInterruption.markProviderSendStarted(),
-                    onToolPhaseStarted: () => _turnInterruption.markToolPhaseStarted(),
+                    onAssistantMessageCommitted: () => {
+                        _turnInterruption.markAssistantMessageCommitted();
+                        _scheduleTurnCheckpoint(true);
+                    },
+                    onAssistantToolCallObserved: (call, detail) => {
+                        _turnInterruption.recordToolCalls([call], detail);
+                        _scheduleTurnCheckpoint(true);
+                    },
+                    onProviderSendStarted: () => {
+                        _turnInterruption.markProviderSendStarted();
+                        _scheduleTurnCheckpoint(true);
+                    },
+                    onToolPhaseStarted: () => {
+                        _turnInterruption.markToolPhaseStarted();
+                        _scheduleTurnCheckpoint(true);
+                    },
                     onUsageDelta: (d) => {
                         persistIterationMetrics(d).catch(() => {});
                         // provider_send usage arrives before agentLoop appends
@@ -565,7 +662,10 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                     },
                     onToolResult: _trackToolResult,
                     onToolApproval: typeof askOpts?.onToolApproval === 'function' ? askOpts.onToolApproval : undefined,
-                    onCompactEvent: typeof askOpts?.onCompactEvent === 'function' ? askOpts.onCompactEvent : undefined,
+                    onCompactEvent: (event) => {
+                        _scheduleTurnCheckpoint(true);
+                        try { askOpts?.onCompactEvent?.(event); } catch {}
+                    },
                     // Claude Code parity: mid-chain queued prompt/notification
                     // drain is owned by agentLoop at provider-continuation
                     // boundaries (after a tool batch, before the next send).
@@ -586,7 +686,10 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                         // post-loop _pendingTail drain below.
                         return out;
                     },
-                    onSteerMessage: typeof askOpts?.onSteerMessage === 'function' ? askOpts.onSteerMessage : undefined,
+                    onSteerMessage: (text, detail) => {
+                        _scheduleTurnCheckpoint(true);
+                        try { askOpts?.onSteerMessage?.(text, detail); } catch {}
+                    },
                     notifyFn: typeof askOpts?.notifyFn === 'function' ? askOpts.notifyFn : undefined,
                     promptCacheKey: session.promptCacheKey || sessionId,
                     // Provider-scoped cache key (mixdog-codex, mixdog-claude…).
@@ -639,6 +742,8 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
             // Turn committed into session.messages; drop the live-turn alias so
             // contextStatus() reverts to the authoritative committed transcript.
             session.liveTurnMessages = null;
+            _stopTurnCheckpoint();
+            delete session.activeTurnCheckpoint;
             const _assistantTranscriptMeta = persistedAssistantTranscriptMetadata(_rawTranscriptMeta);
             if (result.content || result.reasoningContent) {
                 // Max-output recovery returns the complete concatenated text to
@@ -846,6 +951,10 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                 ? saveSessionAsyncDeferred
                 : saveSessionAsync;
             const terminalSave = saveTerminalSession(session, { expectedGeneration: askGeneration });
+            terminalSave.then(
+                () => clearTurnCheckpoint(sessionId, _turnCheckpointToken),
+                () => {},
+            );
             finalizePendingMessageDelivery(
                 session,
                 _turnPendingEntries,
@@ -867,10 +976,14 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
             markSessionDone(sessionId, { empty: isEmptyFinal });
             _result = terminalResultPreview;
         } catch (err) {
+            _stopTurnCheckpoint();
             // Cancellation/error paths bypass the commit point above; drop the
             // live-turn alias so contextStatus() stops estimating from the
             // stale in-flight array once the turn unwinds.
-            if (activeSession) activeSession.liveTurnMessages = null;
+            if (activeSession) {
+                activeSession.liveTurnMessages = null;
+                delete activeSession.activeTurnCheckpoint;
+            }
             // Restore before ANY finalization path. In particular, cancellation
             // can race the acknowledged non-streaming restart and surface as a
             // SessionClosedError; its interruption snapshot must include the
@@ -899,6 +1012,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                             } else {
                                 await durableSave;
                             }
+                            clearTurnCheckpoint(sessionId, _turnCheckpointToken);
                         } catch { /* cancellation cleanup is best-effort */ }
                         if (currentRuntime) currentRuntime.session = activeSession;
                     } else releasePendingMessages(sessionId, _turnPendingEntries);
@@ -918,6 +1032,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
             const preserveProviderPartial = restoredResetText
                 || err?.liveTextEmitted === true
                 || err?.unsafeToRetry === true;
+            let _errorStateDurable = false;
             if (preserveProviderPartial && activeSession && _turnInterruption.hasResponseStarted()) {
                 const finalized = _prepareCloseSnapshot('provider-error');
                 if (finalized.responsePreserved) recordPendingMessageDelivery(activeSession, _turnPendingEntries);
@@ -932,6 +1047,7 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                     } else {
                         await durableSave;
                     }
+                    _errorStateDurable = true;
                 } catch { /* provider-failure history persistence is best-effort */ }
                 const currentRuntime = _getRuntimeEntry(sessionId);
                 if (currentRuntime) currentRuntime.session = activeSession;
@@ -951,11 +1067,19 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
                             activeSession, _turnPendingEntries, durableSave,
                             () => saveSessionAsync(activeSession, { expectedGeneration: askGeneration }),
                         );
+                        _errorStateDurable = true;
                     } catch {}
                 } else {
                     releasePendingMessages(sessionId, _turnPendingEntries);
                 }
             }
+            if (!_errorStateDurable && activeSession) {
+                try {
+                    await saveSessionAsync(activeSession, { expectedGeneration: askGeneration });
+                    _errorStateDurable = true;
+                } catch { /* retain checkpoint when canonical persistence fails */ }
+            }
+            if (_errorStateDurable) clearTurnCheckpoint(sessionId, _turnCheckpointToken);
             markSessionError(sessionId, err && err.message ? err.message : String(err));
             throw err;
         }
@@ -1000,6 +1124,8 @@ export async function askSession(sessionId, prompt, context, onToolCall, cwdOver
         return _result;
       }
     } finally {
+        // A thrown setup/provider path must never leave a delayed checkpoint
+        // writer alive after the turn lock is released.
         // Clear the controller only if it's still ours (closeSession may have
         // swapped it). Leave the rest of the runtime entry intact so agent type=list
         // can still surface the final stage (done/error/cancelling).

package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs

@@ -569,18 +569,30 @@ export function hydratePendingMessages(sessionId, options = {}) {
 
 function clearPersistedPendingMessages(sessionId) {
     if (!isValidPendingSessionId(sessionId)) return;
-    try {
-        updateJsonAtomicSync(pendingMessagesPath(), (raw) => {
-            const next = normalizePendingStore(raw);
-            if (!Object.prototype.hasOwnProperty.call(next.sessions, sessionId)) return undefined;
-            delete next.sessions[sessionId];
-            if (next.sessionTouchedAt) delete next.sessionTouchedAt[sessionId];
-            next.updatedAt = Date.now();
-            return next;
-        }, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
-    } catch (err) {
-        try { process.stderr.write(`[session] pending-message clear failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
-    }
+    // ASYNC, chained behind this session's persist tail. The previous SYNC
+    // variant self-deadlocked: closeSession ran it on the event loop while an
+    // in-flight async spool op (persist/ack/hydrate) in the SAME process held
+    // the lock — the sync wait starved the holder's continuations, so every
+    // such race burned the full 8s lock timeout (observed as an 8s shard-child
+    // stall after each one-shot session close). In-memory queues are already
+    // dropped synchronously by _dropPendingMessageState; only the durable
+    // spool row lags behind, exactly like the ack path.
+    const preceding = _pendingPersistTails.get(sessionId) || Promise.resolve();
+    const operation = preceding.catch(() => {}).then(() => updateJsonAtomic(pendingMessagesPath(), (raw) => {
+        const next = normalizePendingStore(raw);
+        if (!Object.prototype.hasOwnProperty.call(next.sessions, sessionId)) return undefined;
+        delete next.sessions[sessionId];
+        if (next.sessionTouchedAt) delete next.sessionTouchedAt[sessionId];
+        next.updatedAt = Date.now();
+        return next;
+    }, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false }))
+        .catch((err) => {
+            try { process.stderr.write(`[session] pending-message clear failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
+        });
+    _pendingPersistTails.set(sessionId, operation);
+    operation.finally(() => {
+        if (_pendingPersistTails.get(sessionId) === operation) _pendingPersistTails.delete(sessionId);
+    }).catch(() => {});
 }
 
 function shouldEvictPendingSession(sessionId, ttlMs, entryTouchedAt, now = Date.now()) {
@@ -913,5 +925,12 @@ export function _dropPendingMessageState(id, { clearPersisted = true } = {}) {
 }
 
 setImmediate(() => {
+    // Spool hygiene belongs to the lead/daemon process. Worker-mode children
+    // (channels worker, agent shard) share the same spool file; a boot-time
+    // SYNC sweep in every child contends on the cross-process lock against
+    // the lead's writes (observed in the first live shard run: "atomic lock
+    // timeout after 8000ms: session-pending-messages.json.lock") for zero
+    // benefit — the lead already sweeps.
+    if (process.env.MIXDOG_WORKER_MODE === '1') return;
     try { sweepOrphanedPendingMessages(); } catch { /* ignore */ }
 });

package/src/runtime/agent/orchestrator/session/manager/session-close.mjs

@@ -11,6 +11,7 @@ import { SessionClosedError } from './session-errors.mjs';
 import { _dropPendingMessageState } from './pending-messages.mjs';
 import { _stopToolActivityHeartbeat, _getRuntimeEntry, _clearSessionRuntime } from './runtime-liveness.mjs';
 import { _closeBashSessionLazy } from './runtime-loaders.mjs';
+import { clearTurnCheckpoint } from './turn-checkpoint.mjs';
 
 /**
  * Close a session. Plants a `closed=true` tombstone on disk with a bumped
@@ -69,6 +70,9 @@ export function closeSession(id, reason = 'manual', opts = {}) {
     //    write could silently overwrite the session after the user resumes
     //    it back (BL: burned-session late-save clobber).
     const newGen = tombstone ? markSessionClosed(id, reason) : bumpSessionGeneration(id, reason);
+    // prepareCloseSnapshot was folded into the synchronous generation write
+    // above. Only now is it safe to remove the crash fallback.
+    if (typeof newGen === 'number') clearTurnCheckpoint(id);
     // 2. Mark runtime as closed so post-await validation in askSession fires.
     if (entry) {
         entry.closed = true;

package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs

@@ -34,6 +34,7 @@ import {
 import { getAgentRuntimeSync, warnAgentRuntimeResolveFailureOnce } from './agent-runtime-singleton.mjs';
 import { mintSessionId } from './session-id.mjs';
 import { providerCacheKey } from './provider-cache-key.mjs';
+import { clearTurnCheckpoint, recoverTurnCheckpoint } from './turn-checkpoint.mjs';
 
 function buildSessionProviderCacheOpts(providerName, sessionId, agent = null) {
     // Keep this in sync with createSession's provider-cache policy: only
@@ -657,6 +658,19 @@ export async function resumeSession(sessionId, preset, options = {}) {
         }
         return attached;
     }
+    const crashRecovery = recoverTurnCheckpoint(session);
+    if (crashRecovery.changed) {
+        // Resume must not expose a recovered frame that exists only in memory.
+        // Persist synchronously, then remove the sidecar; a second crash at
+        // either boundary therefore leaves at least one complete copy.
+        saveSession(session, {
+            sync: true,
+            expectedGeneration: session.generation,
+        });
+        if (crashRecovery.turnToken) {
+            clearTurnCheckpoint(sessionId, crashRecovery.turnToken);
+        }
+    }
     const oldTools = session.tools || [];
     const cached = _preparedResumes.get(sessionId);
     _preparedResumes.delete(sessionId);

package/src/runtime/agent/orchestrator/session/manager/turn-checkpoint.mjs

@@ -0,0 +1,170 @@
+import {
+    existsSync,
+    mkdirSync,
+    readFileSync,
+    unlinkSync,
+    writeFileSync,
+} from 'fs';
+import { randomBytes } from 'crypto';
+import { join } from 'path';
+import { getPluginData } from '../../config.mjs';
+import { renameWithRetrySync } from '../../../../shared/atomic-file.mjs';
+import { sanitizeContentForStoredHistory } from '../../providers/media-normalization.mjs';
+import { promptContentText } from './prompt-utils.mjs';
+import { finalizeTurnInterruptionSnapshot } from './turn-interruption.mjs';
+
+const TURN_CHECKPOINT_VERSION = 1;
+const SESSION_ID = /^[A-Za-z0-9_-]+$/;
+
+function checkpointDir() {
+    const dir = join(getPluginData(), 'turn-checkpoints');
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    return dir;
+}
+
+function turnCheckpointPath(sessionId) {
+    if (!SESSION_ID.test(String(sessionId || ''))) {
+        throw new Error(`[turn-checkpoint] invalid session id: ${JSON.stringify(sessionId)}`);
+    }
+    return join(checkpointDir(), `${sessionId}.json`);
+}
+
+function checkpointMessage(message) {
+    if (!message || typeof message !== 'object') return message;
+    const content = sanitizeContentForStoredHistory(message.content);
+    return content === message.content ? message : { ...message, content };
+}
+
+function matchingUserMessage(message, currentUserContent) {
+    return message?.role === 'user'
+        && (message.content === currentUserContent
+            || promptContentText(message.content) === promptContentText(currentUserContent));
+}
+
+export function turnMessagesForCheckpoint(messages, currentUserContent) {
+    const source = Array.isArray(messages) ? messages : [];
+    let start = -1;
+    for (let index = source.length - 1; index >= 0; index -= 1) {
+        if (matchingUserMessage(source[index], currentUserContent)) {
+            start = index;
+            break;
+        }
+    }
+    const turn = start >= 0 ? source.slice(start) : [];
+    return turn.map(checkpointMessage);
+}
+
+export function writeTurnCheckpoint(checkpoint) {
+    if (!checkpoint?.sessionId || !checkpoint?.turnToken) return false;
+    const target = turnCheckpointPath(checkpoint.sessionId);
+    const temp = `${target}.${randomBytes(6).toString('hex')}.tmp`;
+    const payload = {
+        ...checkpoint,
+        version: TURN_CHECKPOINT_VERSION,
+        currentUserContent: sanitizeContentForStoredHistory(checkpoint.currentUserContent),
+        turnMessages: (Array.isArray(checkpoint.turnMessages) ? checkpoint.turnMessages : [])
+            .map(checkpointMessage),
+        updatedAt: Date.now(),
+    };
+    try {
+        writeFileSync(temp, JSON.stringify(payload), 'utf8');
+        renameWithRetrySync(temp, target);
+        return true;
+    } catch (error) {
+        try { unlinkSync(temp); } catch {}
+        throw error;
+    }
+}
+
+export function readTurnCheckpoint(sessionId) {
+    const target = turnCheckpointPath(sessionId);
+    try {
+        const value = JSON.parse(readFileSync(target, 'utf8'));
+        if (value?.version !== TURN_CHECKPOINT_VERSION
+            || value?.sessionId !== sessionId
+            || !value?.turnToken
+            || !Array.isArray(value?.turnMessages)) {
+            return null;
+        }
+        return value;
+    } catch {
+        return null;
+    }
+}
+
+export function clearTurnCheckpoint(sessionId, turnToken = null) {
+    const target = turnCheckpointPath(sessionId);
+    if (!existsSync(target)) return true;
+    if (turnToken) {
+        const current = readTurnCheckpoint(sessionId);
+        // A token guard prevents an older turn's late terminal save from
+        // deleting the checkpoint already created by its queued follow-up.
+        if (current?.turnToken && current.turnToken !== turnToken) return false;
+    }
+    try {
+        unlinkSync(target);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+function findPersistedTurnStart(messages, currentUserContent) {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+        if (matchingUserMessage(messages[index], currentUserContent)) return index;
+    }
+    return -1;
+}
+
+/**
+ * Merge a force-killed turn checkpoint into a loaded session. The caller must
+ * durably save the mutated session before clearing the returned token.
+ */
+export function recoverTurnCheckpoint(session) {
+    if (!session?.id || session.closed === true) {
+        return { changed: false, recovered: false, turnToken: null };
+    }
+    const checkpoint = readTurnCheckpoint(session.id);
+    const marker = session.activeTurnCheckpoint;
+    if (!checkpoint) {
+        if (!marker) return { changed: false, recovered: false, turnToken: null };
+        delete session.activeTurnCheckpoint;
+        return { changed: true, recovered: false, turnToken: null };
+    }
+
+    const sessionGeneration = Number(session.generation) || 0;
+    const checkpointGeneration = Number(checkpoint.generation) || 0;
+    const markerToken = typeof marker?.turnToken === 'string' ? marker.turnToken : null;
+    const markerMatches = markerToken === checkpoint.turnToken;
+    const checkpointIsNewer = Number(checkpoint.updatedAt) > Number(session.updatedAt || 0);
+    if (checkpointGeneration !== sessionGeneration
+        || (markerToken && !markerMatches)
+        || (!markerToken && !checkpointIsNewer)) {
+        clearTurnCheckpoint(session.id, checkpoint.turnToken);
+        if (markerToken === checkpoint.turnToken) delete session.activeTurnCheckpoint;
+        return { changed: false, recovered: false, turnToken: null };
+    }
+
+    const finalized = finalizeTurnInterruptionSnapshot({
+        turnOutgoing: checkpoint.turnMessages,
+        currentUserContent: checkpoint.currentUserContent,
+        snapshot: checkpoint.interruption,
+        abortReason: 'process-crash',
+    });
+    const current = Array.isArray(session.messages) ? session.messages : [];
+    const start = findPersistedTurnStart(current, checkpoint.currentUserContent);
+    session.messages = [
+        ...(start >= 0 ? current.slice(0, start) : current),
+        ...finalized.messages,
+    ];
+    delete session.activeTurnCheckpoint;
+    delete session.providerState;
+    session.updatedAt = Date.now();
+    session.lastUsedAt = Date.now();
+    return {
+        changed: true,
+        recovered: true,
+        turnToken: checkpoint.turnToken,
+        responsePreserved: finalized.responsePreserved,
+    };
+}

package/src/runtime/agent/orchestrator/session/manager/turn-interruption.mjs

@@ -8,6 +8,8 @@ import { filterModelVisibleSessionMessages } from './message-sanitize.mjs';
 const INTERRUPT_MESSAGE = '[Request interrupted by user]';
 const INTERRUPT_MESSAGE_FOR_TOOL_USE = '[Request interrupted by user for tool use]';
 const STREAMING_INTERRUPTED_TOOL_RESULT = 'Interrupted by user';
+const PROCESS_RESTART_INTERRUPT_MESSAGE = '[Request interrupted by process restart]';
+const PROCESS_RESTART_TOOL_RESULT = 'Interrupted by process restart';
 const TOOL_USE_REJECT_RESULT = "The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.";
 
 // Abort reasons that represent an EXPLICIT user cancellation of this turn.
@@ -134,9 +136,11 @@ function finalizeInterruptedTurn({
         const executionStarted = phase === 'tools' || observed?.eagerStarted === true;
         messages.push({
             role: 'tool',
-            content: executionStarted
-                ? TOOL_USE_REJECT_RESULT
-                : STREAMING_INTERRUPTED_TOOL_RESULT,
+            content: abortReason === 'process-crash'
+                ? PROCESS_RESTART_TOOL_RESULT
+                : (executionStarted
+                    ? TOOL_USE_REJECT_RESULT
+                    : STREAMING_INTERRUPTED_TOOL_RESULT),
             toolCallId: callId,
             toolKind: 'error',
         });
@@ -149,9 +153,11 @@ function finalizeInterruptedTurn({
     if (abortReason !== 'interrupt' && abortReason !== 'provider-error') {
         pairedMessages.push({
             role: 'user',
-            content: phase === 'tools'
-                ? INTERRUPT_MESSAGE_FOR_TOOL_USE
-                : INTERRUPT_MESSAGE,
+            content: abortReason === 'process-crash'
+                ? PROCESS_RESTART_INTERRUPT_MESSAGE
+                : (phase === 'tools'
+                    ? INTERRUPT_MESSAGE_FOR_TOOL_USE
+                    : INTERRUPT_MESSAGE),
         });
     }
     return { messages: pairedMessages, responsePreserved: true, userTurnPreserved: true };
@@ -247,6 +253,19 @@ export function createTurnInterruptionTracker() {
             phase = 'tools';
             responseStarted = true;
         },
+        snapshot() {
+            return {
+                responseStarted,
+                // A reset acknowledged by the UI temporarily tombstones text.
+                // A process crash cannot complete that replacement, so retain
+                // the same visible bytes the normal error path restores.
+                partialAssistantContent: partialAssistantContent + tombstonedAssistantContent,
+                partialReasoningContent,
+                phase,
+                observedToolCalls: [...observedToolCalls.entries()],
+                observedToolResults: [...observedToolResults.entries()],
+            };
+        },
         finalize({ turnOutgoing, currentUserContent, abortReason = null }) {
             return finalizeInterruptedTurn({
                 turnOutgoing,
@@ -262,3 +281,23 @@ export function createTurnInterruptionTracker() {
         },
     };
 }
+
+export function finalizeTurnInterruptionSnapshot({
+    turnOutgoing,
+    currentUserContent,
+    snapshot,
+    abortReason = 'process-crash',
+}) {
+    const source = snapshot && typeof snapshot === 'object' ? snapshot : {};
+    return finalizeInterruptedTurn({
+        turnOutgoing,
+        currentUserContent,
+        responseStarted: source.responseStarted === true,
+        partialAssistantContent: String(source.partialAssistantContent || ''),
+        partialReasoningContent: String(source.partialReasoningContent || ''),
+        observedToolCalls: new Map(Array.isArray(source.observedToolCalls) ? source.observedToolCalls : []),
+        observedToolResults: new Map(Array.isArray(source.observedToolResults) ? source.observedToolResults : []),
+        phase: source.phase === 'tools' ? 'tools' : 'streaming',
+        abortReason,
+    });
+}