@sayknow-cli/coding-agent 0.7.6 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,6 +39,7 @@ import {
39
39
  resolveTelemetry,
40
40
  type StablePrefixSnapshot,
41
41
  ThinkingLevel,
42
+ TOOL_CALL_REENTRY_ERROR_CODE,
42
43
  } from "@sayknow-cli/agent-core";
43
44
  import { normalizeMessagesForProvider } from "@sayknow-cli/agent-core/agent-loop";
44
45
  import {
@@ -99,11 +100,20 @@ import type {
99
100
  import {
100
101
  classifyContextOverflow,
101
102
  clearAnthropicFastModeFallback,
103
+ createUpstreamAdmission,
102
104
  getSupportedEfforts,
103
105
  isContextOverflow,
106
+ isRecoveryAdmissionErrorMessage,
104
107
  isUsageLimitError,
105
108
  modelsAreEqual,
109
+ type OAuthRefreshOptions,
110
+ RecoveryAdmissionError,
111
+ RecoveryBudget,
112
+ type RecoveryRequestKind,
106
113
  resolveServiceTier,
114
+ SAME_MODEL_RECOVERY_BASE_DELAY_MS,
115
+ SAME_MODEL_RECOVERY_MAX_DELAY_MS,
116
+ SAME_MODEL_RECOVERY_MAX_REQUESTS,
107
117
  streamSimple,
108
118
  } from "@sayknow-cli/ai";
109
119
  import {
@@ -386,6 +396,23 @@ import { buildResponseLanguageReminder, type DetectedLanguage, detectResponseLan
386
396
 
387
397
  export { DefaultModelSelectionRecoveryError } from "./default-model-selection";
388
398
 
399
+ import {
400
+ AUTO_RESUME_FULL_PROMPT,
401
+ AUTO_RESUME_MARKER_CUSTOM_TYPE,
402
+ AUTO_RESUME_RESTRICTED_PROMPT,
403
+ type AutoResumeMarker,
404
+ type AutoResumeMode,
405
+ type AutoResumeTrigger,
406
+ hasAutoResumeMarkerSinceLastUser,
407
+ hasVisibleFailedPrefix,
408
+ INTERRUPTED_TOOL_RESULT_TEXT,
409
+ isRestartSafeTool,
410
+ isSynthesizedResumeResult,
411
+ isThinkingOnlyFailure,
412
+ RESTART_RESUME_MAX_REQUESTS,
413
+ RESTART_RESUME_WINDOW_MS,
414
+ resolveAutoResumePolicy,
415
+ } from "./auto-resume";
389
416
  import type {
390
417
  ClientBridge,
391
418
  ClientBridgePermissionOption,
@@ -401,16 +428,19 @@ import {
401
428
  import { pruneStaleFileMentions } from "./file-mention-pruning";
402
429
  import type { MemoryGuardRestoreResult } from "./memory-guard-checkpoint-participant";
403
430
  import {
431
+ AUTO_RESUME_CUSTOM_TYPE,
404
432
  type BashExecutionMessage,
405
433
  type CompactionSummaryMessage,
406
434
  type CustomMessage,
407
435
  convertToLlm,
408
436
  createPreAdmissionArtifactSpillPreview,
409
437
  type FileMentionMessage,
438
+ getVisibleAnswerChain,
410
439
  type PythonExecutionMessage,
411
440
  readPendingDisplayTag,
412
441
  SILENT_ABORT_MARKER,
413
442
  SKILL_PROMPT_MESSAGE_TYPE,
443
+ VISIBLE_CONTINUATION_CUSTOM_TYPE,
414
444
  } from "./messages";
415
445
  import { isLegacyProviderSafetyStopMessage } from "./provider-safety-stop";
416
446
  import { formatSessionDumpText } from "./session-dump-format";
@@ -868,6 +898,11 @@ type RetryErrorClassification =
868
898
 
869
899
  const BARE_DEFAULT_WATCHDOG_ERROR =
870
900
  /^(?:[A-Za-z][A-Za-z0-9-]*(?: [A-Za-z][A-Za-z0-9-]*){0,3} )stream (?:timed out while waiting for the first event|stalled while waiting for the next event)$/;
901
+ /** Server-side overload/unavailability wording. Bare numbers never count: "500 requests per minute" is a throttle. */
902
+ const BARE_DEFAULT_SERVER_OVERLOAD_ERROR = /overloaded|service.?unavailable|internal.?server.?error|\bapi_error\b/i;
903
+ /** Throttle or quota wording, in any phrasing a provider uses for it. */
904
+ const BARE_DEFAULT_THROTTLE_ERROR =
905
+ /rate.?limit|too many requests|quota|usage.?limit|requests? per (?:second|minute|hour|day)|\b(?:rpm|tpm|rpd)\b|throttl/i;
871
906
  const KIMI_CODE_FIRST_EVENT_TIMEOUT_MESSAGES = {
872
907
  "anthropic-messages": new Set([
873
908
  "Provider stream timed out while waiting for the first event",
@@ -899,6 +934,10 @@ const ALIBABA_TOKEN_PLAN_FIRST_EVENT_TIMEOUT_MESSAGES = {
899
934
  ]),
900
935
  } as const;
901
936
 
937
+ function isServerHttpStatus(status: number | undefined): boolean {
938
+ return status !== undefined && status >= 500 && status <= 599;
939
+ }
940
+
902
941
  function hasBareDefaultRetryDisqualifyingFacts(message: AssistantMessage): boolean {
903
942
  if (message.errorKind !== undefined || message.errorStatus !== undefined) return true;
904
943
  const facts = message.transportFailure;
@@ -920,6 +959,20 @@ function assistantMessageHasVisibleOrToolContent(message: AssistantMessage): boo
920
959
  });
921
960
  }
922
961
 
962
+ /** Serializers with a verified projection of a failed plain-text assistant prefix. */
963
+ const VISIBLE_CONTINUATION_APIS: ReadonlySet<string> = new Set(["openai-completions", "anthropic-messages"]);
964
+ const RECOVERY_CHECKPOINT_CUSTOM_TYPE = "recovery_checkpoint";
965
+ const VISIBLE_CONTINUATION_PROMPT =
966
+ "The connection was interrupted while you were answering. Your answer above is preserved exactly as the user saw it. Continue directly from where it stops: do not repeat any of it and do not redo completed work.";
967
+
968
+ /** Only plain text is a proved replay projection; signed/private blocks are never repaired. */
969
+ function isTextVisibleTail(message: AssistantMessage): boolean {
970
+ return (
971
+ message.content.every(block => block.type === "text") &&
972
+ message.content.some(block => block.type === "text" && block.text.trim().length > 0)
973
+ );
974
+ }
975
+
923
976
  function isLocalModelEndpoint(model: Model | undefined): boolean {
924
977
  if (!model) return false;
925
978
  if (model.provider === "ollama" || model.provider === "lm-studio" || model.provider === "llama.cpp") {
@@ -971,7 +1024,6 @@ export interface BtwConversationScope {
971
1024
  systemPrompt: string[];
972
1025
  messages: BtwRoleTextMessage[];
973
1026
  thinkingLevel: ThinkingLevel;
974
- hideThinkingSummary: boolean;
975
1027
  serviceTier: ServiceTier | undefined;
976
1028
  credentialSessionId: string;
977
1029
  providerAffinitySessionId: string;
@@ -1762,10 +1814,19 @@ export class AgentSession {
1762
1814
 
1763
1815
  /** Replay safety for the currently admitted top-level prompt/custom-message run. */
1764
1816
  #retryReplayEpoch = 0;
1817
+ /**
1818
+ * Epoch tainted by something outside the model transcript (extension hooks,
1819
+ * commands, user bash/python, abort). Visible model output and completed tool
1820
+ * pairs stay in the transcript a recovery keeps and are never re-executed, so
1821
+ * they do not taint the run; recovery stays same-model and transcript-only.
1822
+ */
1765
1823
  #retryReplayUnsafeEpoch: number | undefined;
1766
1824
 
1767
1825
  #resetRetryReplaySafety(): void {
1768
1826
  this.#retryReplayEpoch++;
1827
+ this.#uncertainUpstreamUnresolved = false;
1828
+ // A genuine new prompt supersedes an interrupted persisted step.
1829
+ this.#unresolvedRecoveryAtHydration = false;
1769
1830
  this.#retryReplayUnsafeEpoch = undefined;
1770
1831
  if (
1771
1832
  this.#extensionRunner?.hasHandlers("context") ||
@@ -1780,7 +1841,7 @@ export class AgentSession {
1780
1841
  if (this.#retryReplayEpoch > 0) this.#retryReplayUnsafeEpoch = this.#retryReplayEpoch;
1781
1842
  }
1782
1843
 
1783
- get #hasCleanRetryReplaySafety(): boolean {
1844
+ get #hasCleanContinuationSafety(): boolean {
1784
1845
  return this.#retryReplayEpoch > 0 && this.#retryReplayUnsafeEpoch !== this.#retryReplayEpoch;
1785
1846
  }
1786
1847
  #prePromptContextCheckPromise: Promise<void> | undefined = undefined;
@@ -1808,6 +1869,34 @@ export class AgentSession {
1808
1869
  #overflowMaintenanceAttempts = 0;
1809
1870
  #defaultFallbackExhaustedLastTurn = false;
1810
1871
  #transientStopTimestamp: number | undefined;
1872
+ /**
1873
+ * Shared same-model recovery envelope for the unfinished model step: every
1874
+ * concrete upstream request (initial, hidden resend, credential/token, session
1875
+ * retry) is admitted here. Created at the step's first request, cleared when the
1876
+ * step is accepted or a new user turn starts; never refunded.
1877
+ */
1878
+ #stepRecoveryBudget: RecoveryBudget | undefined;
1879
+ #visibleRecoveryReferenceIds: string[] = [];
1880
+ #visibleRecoveryContextIds: string[] = [];
1881
+ #stepRecoveryUncertainReasons: string[] = [];
1882
+ /**
1883
+ * Set when a step dispatched unobservable remote work. Ending the budget closes
1884
+ * admission but never launders that fact: manual retry stays refused until a new
1885
+ * user prompt starts a fresh replay epoch.
1886
+ */
1887
+ #uncertainUpstreamUnresolved = false;
1888
+ /** Durable record of in-step uncertainty; admission and resume decisions await it. */
1889
+ #pendingUncertainFlush: Promise<void> = Promise.resolve();
1890
+ #uncertainFlushFailed = false;
1891
+ /** The current run is an automatic resume restricted to restart-safe tools. */
1892
+ #activeAutoResume: { trigger: AutoResumeTrigger; mode: AutoResumeMode } | undefined;
1893
+ /** A restart resume owns the current envelope until its first step is accepted. */
1894
+ #restartResumeEnvelopePending = false;
1895
+ #restrictedToolMode = false;
1896
+ #stepRecoveryStepId: string | undefined;
1897
+ #stepRecoveryDeadlineAbort: AbortController | undefined;
1898
+ #cancelStepRecoveryDeadline: (() => void) | undefined;
1899
+ #unresolvedRecoveryAtHydration = false;
1811
1900
  /** Language of the latest genuine user prompt (or inherited from the parent agent). */
1812
1901
  #responseLanguage: DetectedLanguage | undefined;
1813
1902
  #fallbackInvocationId = 0;
@@ -2588,6 +2677,8 @@ export class AgentSession {
2588
2677
  // Per-tool TTSR reminders are folded into the matched tool's result via this hook.
2589
2678
  this.agent.afterToolCall = ctx => this.#ttsrAfterToolCall(ctx);
2590
2679
  this.agent.providerSessionState = this.#providerSessionState;
2680
+ this.agent.getUpstreamRequestLease = this.#getUpstreamRequestLease;
2681
+ this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
2591
2682
  this.#syncAgentSessionId();
2592
2683
  this.#removeEphemeralCustomMessages();
2593
2684
 
@@ -3458,38 +3549,9 @@ export class AgentSession {
3458
3549
 
3459
3550
  /** Internal handler for agent events - shared by subscribe and reconnect */
3460
3551
  #handleAgentEvent = async (event: AgentEvent): Promise<void> => {
3552
+ // Model output and completed tool pairs stay in the transcript a recovery
3553
+ // keeps; only an observing extension handler can make the run unsafe.
3461
3554
  if (this.#extensionRunner?.hasHandlers(event.type)) this.#markRetryReplayUnsafe();
3462
- if (
3463
- event.type === "tool_execution_start" ||
3464
- event.type === "tool_execution_update" ||
3465
- event.type === "tool_execution_end"
3466
- ) {
3467
- this.#markRetryReplayUnsafe();
3468
- } else if (event.type === "message_end") {
3469
- if (
3470
- event.message.role === "toolResult" ||
3471
- (event.message.role === "assistant" && assistantMessageHasVisibleOrToolContent(event.message))
3472
- ) {
3473
- this.#markRetryReplayUnsafe();
3474
- }
3475
- } else if (event.type === "message_update") {
3476
- const update = event.assistantMessageEvent;
3477
- if (
3478
- update.type === "toolcall_start" ||
3479
- update.type === "toolcall_delta" ||
3480
- update.type === "toolcall_end" ||
3481
- ((update.type === "text_delta" ||
3482
- update.type === "thinking_delta" ||
3483
- update.type === "reasoning_summary_delta") &&
3484
- update.delta.length > 0) ||
3485
- ((update.type === "text_end" ||
3486
- update.type === "thinking_end" ||
3487
- update.type === "reasoning_summary_end") &&
3488
- update.content.length > 0)
3489
- ) {
3490
- this.#markRetryReplayUnsafe();
3491
- }
3492
- }
3493
3555
  // Record a successful final yield before any asynchronous extension work so a
3494
3556
  // concurrently delivered agent_end cannot start post-turn maintenance first.
3495
3557
  if (event.type === "tool_execution_end" && event.toolName === "yield" && !event.isError) {
@@ -3531,6 +3593,11 @@ export class AgentSession {
3531
3593
  this.#displayDequeueAlreadyHandled.text === userMessageText,
3532
3594
  );
3533
3595
  if (userDisplayDequeueAlreadyHandled) this.#displayDequeueAlreadyHandled = undefined;
3596
+ // A user message reaching the model ends any restricted automatic resume: from here
3597
+ // on the user directs the turn. Queuing alone does not lift the restriction.
3598
+ if (event.type === "message_start" && event.message.role === "user" && this.#restrictedToolMode) {
3599
+ this.#endAutoResume("skipped");
3600
+ }
3534
3601
  if (event.type === "message_start" && event.message.role === "user" && !userDisplayDequeueAlreadyHandled) {
3535
3602
  const messageText = userMessageText;
3536
3603
  if (messageText) {
@@ -3871,17 +3938,32 @@ export class AgentSession {
3871
3938
  this.#skipPostTurnMaintenanceAssistantTimestamp = assistantMsg.timestamp;
3872
3939
  this.#suppressNextPostTurnMaintenanceAfterAutoHandoff = false;
3873
3940
  }
3941
+ // An accepted (non-error) model step closes its recovery envelope; the next
3942
+ // model step (e.g. after tool results) starts with a fresh budget.
3943
+ if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") {
3944
+ // A tool-call step stays in_flight on purpose: until the next step's own
3945
+ // checkpoint supersedes it, a crash during tool execution has unknown effects
3946
+ // and must restart fail closed. The final text step writes the completed marker.
3947
+ if (!assistantMsg.content.some(block => block.type === "toolCall")) {
3948
+ // The completed marker is appended synchronously. A flush lost to a crash
3949
+ // leaves the step unresolved, which restarts fail closed (no auto-resend).
3950
+ void this.#completeStepRecoveryCheckpoint(assistantMsg);
3951
+ }
3952
+ this.#endStepRecoveryBudget({ accepted: true });
3953
+ this.#restartResumeEnvelopePending = false;
3954
+ }
3874
3955
  if (
3875
3956
  assistantMsg.stopReason !== "error" &&
3876
3957
  assistantMsg.stopReason !== "aborted" &&
3877
3958
  this.#retryAttempt > 0
3878
3959
  ) {
3960
+ const recoveredAttempt = this.#retryAttempt;
3961
+ this.#retryAttempt = 0;
3879
3962
  await this.#emitSessionEvent({
3880
3963
  type: "auto_retry_end",
3881
3964
  success: true,
3882
- attempt: this.#retryAttempt,
3965
+ attempt: recoveredAttempt,
3883
3966
  });
3884
- this.#retryAttempt = 0;
3885
3967
  // Settle the retry gate here, colocated with the success event, rather
3886
3968
  // than relying on the generic #resolveRetry() at the end of the
3887
3969
  // agent_end branch. That tail resolver is bypassed by every early
@@ -3994,6 +4076,7 @@ export class AgentSession {
3994
4076
  const msg = this.#lastAssistantMessage ?? fallbackAssistant;
3995
4077
  this.#lastAssistantMessage = undefined;
3996
4078
  if (!msg) {
4079
+ if (this.#activeAutoResume) this.#endAutoResume("completed");
3997
4080
  this.#lastSuccessfulYieldToolCallId = undefined;
3998
4081
  this.#resolveRetry();
3999
4082
  return;
@@ -4010,12 +4093,14 @@ export class AgentSession {
4010
4093
  }
4011
4094
 
4012
4095
  if (this.#skipPostTurnMaintenanceAssistantTimestamp === msg.timestamp) {
4096
+ if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
4013
4097
  this.#skipPostTurnMaintenanceAssistantTimestamp = undefined;
4014
4098
  this.#lastSuccessfulYieldToolCallId = undefined;
4015
4099
  return;
4016
4100
  }
4017
4101
 
4018
4102
  if (this.#assistantEndedWithSuccessfulYield(msg)) {
4103
+ if (this.#activeAutoResume) this.#endAutoResume("completed");
4019
4104
  this.#lastSuccessfulYieldToolCallId = undefined;
4020
4105
  if (msg.stopReason !== "error" && msg.stopReason !== "aborted" && (await this.#checkGoalCompletion(msg))) {
4021
4106
  return;
@@ -4031,9 +4116,17 @@ export class AgentSession {
4031
4116
  const didRetry = await this.#handleRetryableError(msg, false, transportFailure);
4032
4117
  if (didRetry) return; // Retry was initiated, don't proceed to compaction
4033
4118
  }
4119
+ // The resumed run settled: its restricted tool view ends with it.
4120
+ if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
4121
+ // An interrupted step may resume once, on the same model, inside its budget.
4122
+ const resumeTrigger = this.#autoResumeTriggerFor(msg);
4123
+ const resumed = resumeTrigger !== undefined && (await this.#maybeAutoResume(resumeTrigger));
4124
+ // No automatic retry follows: the step's recovery envelope ends here, unless the
4125
+ // resume carries on inside it.
4126
+ if (!resumed) this.#endStepRecoveryBudget();
4034
4127
  if (this.#retryAttempt > 0) {
4035
- // A prior retry ended on a non-retryable (terminal) message: emit
4036
- // the terminal retry-end and reset so observers clear retry state.
4128
+ // The retry chain ended on a non-retryable (terminal) message, or handed over
4129
+ // to the resume: emit the terminal retry-end and reset so observers clear retry state.
4037
4130
  const attempt = this.#retryAttempt;
4038
4131
  this.#retryAttempt = 0;
4039
4132
  await this.#emitSessionEvent({
@@ -4044,6 +4137,7 @@ export class AgentSession {
4044
4137
  });
4045
4138
  }
4046
4139
  this.#resolveRetry();
4140
+ if (resumed) return;
4047
4141
 
4048
4142
  const compactionTask = this.#checkCompaction(msg);
4049
4143
  this.#trackPostPromptTask(compactionTask.then(() => undefined));
@@ -4161,6 +4255,8 @@ export class AgentSession {
4161
4255
  onSkip?: (reason: "generation_changed" | "aborted_signal" | "queue_drained" | "handoff_in_progress") => void;
4162
4256
  allowDuringCancelAndSubmit?: boolean;
4163
4257
  onError?: (error: unknown) => void;
4258
+ /** Run on the current model only: no managed chain, no fallback switch. */
4259
+ pinnedModel?: boolean;
4164
4260
  }): Promise<void> {
4165
4261
  const predecessorAgentEndHold = options?.suppressPredecessorAgentEnd
4166
4262
  ? this.#reserveDeferredAgentEndForContinuation()
@@ -4234,7 +4330,7 @@ export class AgentSession {
4234
4330
  this.agent.state.messages.at(-1)?.role === "assistant" && this.agent.hasQueuedMessages();
4235
4331
  try {
4236
4332
  await this.agent.continue({
4237
- ...this.#managedFallbackPromptOptions(),
4333
+ ...(options?.pinnedModel ? {} : this.#managedFallbackPromptOptions()),
4238
4334
  // Reset only after continue() has claimed the queued turn. Skipped or stale
4239
4335
  // continuations retain predecessor accounting, and resetAttemptBudget keeps
4240
4336
  // the sticky fallback cursor unchanged.
@@ -5317,6 +5413,16 @@ export class AgentSession {
5317
5413
  );
5318
5414
  }
5319
5415
 
5416
+ /**
5417
+ * Recovery safety follows the transcript, not the process: a session switch,
5418
+ * branch, fork, or handoff re-derives it from the persisted checkpoints and drops
5419
+ * in-memory uncertainty that belonged to the previous transcript.
5420
+ */
5421
+ #resyncRecoveryStateFromTranscript(): void {
5422
+ this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
5423
+ this.#uncertainUpstreamUnresolved = false;
5424
+ }
5425
+
5320
5426
  #rekeyHindsightMemoryForCurrentSessionId(): void {
5321
5427
  if (resolveMemoryBackend(this.settings).id !== "hindsight") return;
5322
5428
  const sid = this.agent.sessionId;
@@ -6714,12 +6820,58 @@ export class AgentSession {
6714
6820
  }
6715
6821
  }
6716
6822
 
6823
+ #hasUnresolvedPersistedRecovery(): boolean {
6824
+ const branch = this.sessionManager.getBranch();
6825
+ const checkpoint = branch.findLast(
6826
+ entry => entry.type === "custom" && entry.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE,
6827
+ );
6828
+ if (checkpoint?.type !== "custom") return false;
6829
+ if (!checkpoint.data || typeof checkpoint.data !== "object" || Array.isArray(checkpoint.data)) return true;
6830
+ const data = checkpoint.data as Record<string, unknown>;
6831
+ if (data.version !== 1 || data.state !== "completed") return true;
6832
+ if (!Array.isArray(data.visibleMessageIds) || data.visibleMessageIds.length === 0) return true;
6833
+ const branchIds = new Set(branch.map(entry => entry.id));
6834
+ return !data.visibleMessageIds.every(
6835
+ id => typeof id === "string" && branchIds.has(id) && this.sessionManager.getEntry(id)?.type === "message",
6836
+ );
6837
+ }
6838
+
6839
+ /**
6840
+ * Whether startup `--continue` should hand this session to the restart resume
6841
+ * policy: automatic resume is on and the persisted history ends in a step a process
6842
+ * exit interrupted. Every other tail stays idle until the user sends a message.
6843
+ */
6844
+ shouldResumeInterruptedStepOnStartup(): boolean {
6845
+ return this.settings.get("retry.autoResume") && this.#hasUnresolvedPersistedRecovery();
6846
+ }
6847
+
6717
6848
  /** Main startup calls this exactly once, after a strict open returned `kind: "opened"`. */
6718
6849
  async continuePersistedHistory(): Promise<void> {
6719
6850
  this.#assertNoHandoffTransition();
6720
6851
  this.#assertRecoveryHydrationPromoted();
6721
6852
  this.#removeEphemeralCustomMessages();
6722
6853
 
6854
+ // A process that died mid-step left an unresolved recovery checkpoint. Its old
6855
+ // budget, deadline and in-flight request outcome are unknown and never reused:
6856
+ // the step resumes at most once, in a small fresh envelope, without re-running
6857
+ // any call whose outcome was not observed. Otherwise the user resumes it.
6858
+ const tail = this.agent.state.messages.at(-1);
6859
+ let autoResumed = false;
6860
+ let resumeGeneration = this.#promptGeneration;
6861
+ if (
6862
+ this.#hasUnresolvedPersistedRecovery() ||
6863
+ (tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE)
6864
+ ) {
6865
+ const staleContinuation = tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE;
6866
+ autoResumed = !staleContinuation && (await this.#maybeAutoResume("restart"));
6867
+ resumeGeneration = this.#promptGeneration;
6868
+ if (!autoResumed) {
6869
+ throw new Error(
6870
+ "The previous answer was interrupted during automatic recovery. Its preserved output is shown above; send a message to continue.",
6871
+ );
6872
+ }
6873
+ }
6874
+
6723
6875
  if (!canContinuePersistedHistory(this.agent.state.messages)) {
6724
6876
  throw new Error("Cannot continue from persisted message history");
6725
6877
  }
@@ -6757,13 +6909,27 @@ export class AgentSession {
6757
6909
  // volatile-context/hindsight awaits above and this would otherwise start a
6758
6910
  // turn against the session being handed off.
6759
6911
  this.#assertNoHandoffTransition();
6912
+ // An abort or new prompt during the preparation above cancels the automatic
6913
+ // resume: it ended the restart envelope, and nothing may be sent outside it.
6914
+ if (autoResumed && (this.#promptGeneration !== resumeGeneration || !this.#stepRecoveryBudget)) {
6915
+ this.#endAutoResume("skipped");
6916
+ return;
6917
+ }
6760
6918
  await this.agent.continue({
6761
- ...this.#managedFallbackPromptOptions(),
6919
+ ...(autoResumed ? {} : this.#managedFallbackPromptOptions()),
6762
6920
  onRunAccepted: () => {
6763
6921
  if (hindsightRecall) hindsightState?.markRecallSnippetInjected(hindsightRecall);
6764
6922
  },
6765
6923
  });
6766
6924
  await this.#waitForPostPromptRecovery();
6925
+ } catch (error) {
6926
+ // A restart resume that failed before its run started releases its envelope
6927
+ // and restricted view; a later manual retry or prompt must not inherit them.
6928
+ if (autoResumed && this.#restartResumeEnvelopePending) {
6929
+ this.#endAutoResume("failed");
6930
+ this.#endStepRecoveryBudget();
6931
+ }
6932
+ throw error;
6767
6933
  } finally {
6768
6934
  this.#removeEphemeralCustomMessages();
6769
6935
  this.#endInFlight();
@@ -8073,6 +8239,17 @@ export class AgentSession {
8073
8239
  let hindsightRecall: string | undefined;
8074
8240
  try {
8075
8241
  this.#throwIfPromptPreflightCancelled(generation, preflightSignal);
8242
+ // A new user turn closes any earlier step's envelope (budget, deadline timer,
8243
+ // stepId, uncertainty) before the replay-safety reset clears its residue.
8244
+ if (message.role === "user") {
8245
+ // A real user turn ends any automatic resume before it is processed.
8246
+ if (this.#activeAutoResume) this.#endAutoResume("skipped");
8247
+ this.#endStepRecoveryBudget();
8248
+ // A failed uncertainty write closed admission for the previous step only. Wait
8249
+ // for that step's pending write so its outcome cannot leak into this turn.
8250
+ await this.#pendingUncertainFlush;
8251
+ this.#uncertainFlushFailed = false;
8252
+ }
8076
8253
  if (options?.resetRetryReplaySafety) this.#resetRetryReplaySafety();
8077
8254
  if (message.role === "user") {
8078
8255
  await this.#resetDefaultFallbackForNewTurn();
@@ -9256,6 +9433,8 @@ export class AgentSession {
9256
9433
  this.#silentAbortPending = options?.silent === true;
9257
9434
  this.#markRetryReplayUnsafe();
9258
9435
  this.abortRetry();
9436
+ // Cancellation admits no further upstream request for the interrupted step.
9437
+ this.#endStepRecoveryBudget();
9259
9438
  this.#promptGeneration++;
9260
9439
  this.#promptPreflightAbortController.abort();
9261
9440
  this.#promptPreflightAbortController = new AbortController();
@@ -9614,6 +9793,7 @@ export class AgentSession {
9614
9793
  }
9615
9794
  this.setTodoPhases([]);
9616
9795
  this.#syncAgentSessionId();
9796
+ this.#resyncRecoveryStateFromTranscript();
9617
9797
  this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
9618
9798
  this.#rekeyHindsightMemoryForCurrentSessionId();
9619
9799
  this.#resetHindsightConversationTrackingIfHindsight();
@@ -9687,6 +9867,7 @@ export class AgentSession {
9687
9867
  this.agent.reset();
9688
9868
  this.setTodoPhases([]);
9689
9869
  this.#syncAgentSessionId();
9870
+ this.#resyncRecoveryStateFromTranscript();
9690
9871
  this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
9691
9872
  this.#rekeyHindsightMemoryForCurrentSessionId();
9692
9873
  this.#resetHindsightConversationTrackingIfHindsight();
@@ -9779,6 +9960,7 @@ export class AgentSession {
9779
9960
  this.sessionManager.appendContextClearEntry({ sessionId });
9780
9961
  this.setTodoPhases([]);
9781
9962
  this.#syncAgentSessionId(sessionId);
9963
+ this.#resyncRecoveryStateFromTranscript();
9782
9964
  this.#steeringMessages = [];
9783
9965
  this.#followUpMessages = [];
9784
9966
  this.#pendingNextTurnMessages = [];
@@ -9848,6 +10030,7 @@ export class AgentSession {
9848
10030
  throw await discardPreparedNewSessionAfterFailure(this.sessionManager, prepared, error);
9849
10031
  }
9850
10032
  this.#syncAgentSessionId();
10033
+ this.#resyncRecoveryStateFromTranscript();
9851
10034
  this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
9852
10035
  this.#rekeyHindsightMemoryForCurrentSessionId();
9853
10036
 
@@ -11515,6 +11698,7 @@ export class AgentSession {
11515
11698
  committed = true;
11516
11699
  this.agent.reset();
11517
11700
  this.#syncAgentSessionId();
11701
+ this.#resyncRecoveryStateFromTranscript();
11518
11702
  this.#rekeyHindsightMemoryForCurrentSessionId();
11519
11703
  this.#steeringMessages = [];
11520
11704
  this.#followUpMessages = [];
@@ -11585,6 +11769,7 @@ export class AgentSession {
11585
11769
  // were never mutated before commit, so they survive intact.
11586
11770
  this.sessionManager.restoreState(rollbackSessionState);
11587
11771
  this.#syncAgentSessionId(rollbackSessionState.sessionId);
11772
+ this.#resyncRecoveryStateFromTranscript();
11588
11773
  this.#rekeyHindsightMemoryForCurrentSessionId();
11589
11774
  this.agent.replaceMessages(rollbackAgentMessages);
11590
11775
  this.agent.clearAllQueues();
@@ -13702,6 +13887,7 @@ export class AgentSession {
13702
13887
  }
13703
13888
 
13704
13889
  #isRetryableError(message: AssistantMessage): boolean {
13890
+ if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return false;
13705
13891
  if (this.#isTerminalProviderFirstEventTimeout(message)) return false;
13706
13892
  if (message.errorMessage?.startsWith("Model fallback chain exhausted;")) return false;
13707
13893
  // Already gave up on this transient failure; do not start a fresh budget.
@@ -13774,14 +13960,15 @@ export class AgentSession {
13774
13960
  /**
13775
13961
  * Whether a first-event timeout on the error's provider should fail closed —
13776
13962
  * i.e. retry a bounded number of times (capped at retry.maxRetries) and then
13777
- * surface, instead of joining the unbounded transient-retry class.
13963
+ * surface, instead of joining the transient class (bounded only by the shared
13964
+ * step budget of 7 requests / 15 minutes).
13778
13965
  *
13779
13966
  * Targets the ollama-chat API, which is exclusively ollama-cloud (local
13780
13967
  * Ollama uses the openai-responses API). That remote, queued backend can
13781
- * stall before its first token even for tiny prompts; an unbounded
13782
- * continuation retry re-issues the full request on every attempt and can
13783
- * silently spike upstream usage (#713). First-party providers keep their
13784
- * existing unbounded first-event-timeout retry behavior.
13968
+ * stall before its first token even for tiny prompts; each continuation retry
13969
+ * re-issues the full request and can spike upstream usage (#713). First-party
13970
+ * providers keep first-event timeouts in the transient class under the shared
13971
+ * step budget.
13785
13972
  */
13786
13973
  #shouldFailClosedOnFirstEventTimeout(message: AssistantMessage): boolean {
13787
13974
  // Prefer the active model's API (the model that produced the error);
@@ -13812,11 +13999,12 @@ export class AgentSession {
13812
13999
  /**
13813
14000
  * Ordered retry classification: typed safety stop (surface) -> legacy safety stop
13814
14001
  * (surface) -> overflow (compaction) -> terminal (surface) -> usage_limit
13815
- * (rotation) -> first_event_timeout (bounded retry) -> transient (unbounded retry) ->
14002
+ * (rotation) -> first_event_timeout (retry.maxRetries) -> transient (shared step budget) ->
13816
14003
  * unknown (bounded retry).
13817
14004
  */
13818
14005
  #classifyErrorForRetry(message: AssistantMessage): RetryErrorClassification {
13819
14006
  if (message.stopReason !== "error") return "none";
14007
+ if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return "terminal";
13820
14008
  if (message.errorKind === "provider_safety_stop") return "terminal";
13821
14009
  // A decode loop is deterministic for the submitted context: replaying the
13822
14010
  // identical conversation re-trips the guard and re-bills the full context.
@@ -13863,10 +14051,10 @@ export class AgentSession {
13863
14051
  if (this.#isTerminalProviderFirstEventTimeout(message)) {
13864
14052
  return "terminal";
13865
14053
  }
13866
- // A first-event timeout on ollama-cloud (the ollama-chat API) must not
13867
- // join the unbounded transient class: each continuation retry re-issues
13868
- // the full request to a remote, billable backend, so an unbounded loop
13869
- // can silently spike usage (#713). Bound it to retry.maxRetries instead.
14054
+ // A first-event timeout on ollama-cloud (the ollama-chat API) must not join
14055
+ // the transient class: each continuation retry re-issues the full request to
14056
+ // a remote, billable backend (#713). Bound it to retry.maxRetries instead of
14057
+ // the wider shared step budget.
13870
14058
  if (this.#isFirstEventTimeoutErrorMessage(err) && this.#shouldFailClosedOnFirstEventTimeout(message)) {
13871
14059
  return "first_event_timeout";
13872
14060
  }
@@ -14118,8 +14306,15 @@ export class AgentSession {
14118
14306
  }
14119
14307
 
14120
14308
  async #handleManagedAttemptOutcome(outcome: ManagedAttemptOutcome): Promise<ManagedAttemptDecision> {
14309
+ if (outcome.type === "visible_interrupted" || outcome.type === "unsafe_interrupted") {
14310
+ // The session does not supply managed recovery evidence, so a committed managed
14311
+ // failure is never replayed: preserve it and stop for an explicit user resume.
14312
+ this.#endStepRecoveryBudget();
14313
+ return { type: "pause", reason: outcome.evidence.tail };
14314
+ }
14121
14315
  if (outcome.type === "run_terminal") {
14122
14316
  this.#defaultFallbackChain().resetAttemptBudget();
14317
+ this.#endStepRecoveryBudget();
14123
14318
  return { type: "terminal", terminal: { stopReason: outcome.reason } };
14124
14319
  }
14125
14320
  if (outcome.type === "context_overflow_discarded") {
@@ -14205,7 +14400,7 @@ export class AgentSession {
14205
14400
  if (transport.class !== "other") return transport;
14206
14401
  // Managed fallback receives authoritative transport facts from the request
14207
14402
  // boundary. Once those facts classify as other, error prose must not upgrade
14208
- // the failure into an unbounded transient or quota retry.
14403
+ // the failure into a transient or quota retry.
14209
14404
  if (transportFailure) return { class: "unknown" };
14210
14405
  const classification = this.#classifyErrorForRetry(message);
14211
14406
  if (allowLegacyUsageLimit && classification === "usage_limit") {
@@ -14309,22 +14504,72 @@ export class AgentSession {
14309
14504
  async #markFailedManagedCredential(trigger: {
14310
14505
  class: FallbackTriggerClass;
14311
14506
  retryAfterMs?: number;
14507
+ }): Promise<boolean> {
14508
+ try {
14509
+ return await this.#markFailedManagedCredentialAdmitted(trigger);
14510
+ } catch (error) {
14511
+ // A refused admission ends maintenance locally; the next inference admission
14512
+ // reports the same budget verdict instead of this side path throwing.
14513
+ if (error instanceof RecoveryAdmissionError) return false;
14514
+ throw error;
14515
+ }
14516
+ }
14517
+
14518
+ async #markFailedManagedCredentialAdmitted(trigger: {
14519
+ class: FallbackTriggerClass;
14520
+ retryAfterMs?: number;
14312
14521
  }): Promise<boolean> {
14313
14522
  if (!this.model || (trigger.class !== "auth" && trigger.class !== "quota" && trigger.class !== "rate_limit")) {
14314
14523
  return false;
14315
14524
  }
14316
14525
  const authStorage = this.#modelRegistry.authStorage;
14526
+ const requestOptions = this.#stepCredentialRequestOptions();
14527
+ // Recovery maintenance outside a step budget would send unadmitted credential HTTP.
14528
+ if (!requestOptions.fetch) return false;
14317
14529
  if (trigger.class === "auth") {
14318
- const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
14530
+ const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
14319
14531
  if (!isAuthenticated(apiKey)) return false;
14320
- return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, { sessionId: this.sessionId });
14532
+ return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, {
14533
+ ...requestOptions,
14534
+ sessionId: this.sessionId,
14535
+ });
14321
14536
  }
14322
14537
  if (authStorage.hasRuntimeApiKey(this.model.provider)) return false;
14323
- const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
14538
+ const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
14324
14539
  const rotated = await authStorage.markUsageLimitReached(this.model.provider, this.sessionId, {
14325
14540
  retryAfterMs: trigger.retryAfterMs,
14541
+ ...requestOptions,
14326
14542
  });
14327
- return rotated && (await this.#modelRegistry.getApiKey(this.model, this.sessionId)) !== activeApiKey;
14543
+ return (
14544
+ rotated && (await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions)) !== activeApiKey
14545
+ );
14546
+ }
14547
+
14548
+ /**
14549
+ * Credential maintenance during recovery (rotation, restore, usage probes) spends
14550
+ * the same step budget as inference: every concrete token/usage HTTP is admitted
14551
+ * as kind "token", and opaque remote work marks the step unsafe.
14552
+ */
14553
+ #stepCredentialRequestOptions(): OAuthRefreshOptions {
14554
+ const budget = this.#stepRecoveryBudget;
14555
+ const deadlineAbort = this.#stepRecoveryDeadlineAbort;
14556
+ if (!budget || !deadlineAbort) return {};
14557
+ const validate = (): void => {
14558
+ if (this.#stepRecoveryBudget !== budget) throw new RecoveryAdmissionError("cancelled");
14559
+ budget.assertActive();
14560
+ };
14561
+ const admission = createUpstreamAdmission(
14562
+ kind => budget.reserve(kind),
14563
+ undefined,
14564
+ validate,
14565
+ () => this.#recordStepRecoveryFailure(budget),
14566
+ );
14567
+ return {
14568
+ signal: deadlineAbort.signal,
14569
+ fetch: admission.credentialFetch,
14570
+ onUpstreamRequest: admission.onUpstreamRequest,
14571
+ onUncertainUpstream: reason => this.#recordStepUncertainUpstream(budget, reason),
14572
+ };
14328
14573
  }
14329
14574
 
14330
14575
  /** Handle retryable errors with exponential backoff. */
@@ -14336,6 +14581,14 @@ export class AgentSession {
14336
14581
  const controller = this.#defaultFallbackChain();
14337
14582
  const managedFallback = controller.chain.entries.length > 1;
14338
14583
  const retrySettings = this.settings.getGroup("retry");
14584
+ // A rejected admission is the shared budget's own terminal answer, never a retry trigger.
14585
+ if (isRecoveryAdmissionErrorMessage(message.errorMessage)) {
14586
+ return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
14587
+ }
14588
+ // Broker/custom credential work with unobserved remote effects is never resent automatically.
14589
+ if (this.#stepRecoveryUncertainReasons.length > 0) {
14590
+ return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
14591
+ }
14339
14592
  const legacyRetryConfigured =
14340
14593
  this.settings.has("retry.enabled") ||
14341
14594
  this.settings.has("retry.maxRetries") ||
@@ -14345,14 +14598,37 @@ export class AgentSession {
14345
14598
  // user opt-out.
14346
14599
  if (!managedFallback && !retrySettings.enabled) return false;
14347
14600
  const classification = managedFallback ? undefined : this.#classifyErrorForRetry(message);
14348
- // Bare defaults admit only clean, side-effect-free canonical stream watchdog failures.
14601
+ // A failure after public output is never replayed by deleting that output.
14602
+ // A text-only tail on a verified serializer continues on the same model;
14603
+ // any other public tail (tool calls, images, tainted run) stops with the
14604
+ // output preserved for the user to resume explicitly. A failure that streamed
14605
+ // only reasoning holds no answer and its signed blocks cannot be replayed: it
14606
+ // is dropped and retried like a failure that showed nothing.
14607
+ const thinkingOnlyFailure = !managedOutcome && isThinkingOnlyFailure(message);
14608
+ const visibleFailure =
14609
+ !managedOutcome && !thinkingOnlyFailure && assistantMessageHasVisibleOrToolContent(message);
14610
+ const visibleContinuation = visibleFailure && this.#canContinueVisibleFailure(message);
14611
+ if (visibleFailure && !visibleContinuation) return false;
14612
+ // Bare defaults admit only clean canonical stream watchdog failures, plus a
14613
+ // transient server failure that streamed only reasoning (e.g. an SSE
14614
+ // overloaded_error after thinking_delta): the provider's own pre-content
14615
+ // retry cannot cover it, and it has no answer to preserve. Completed tool
14616
+ // pairs and preserved visible output stay in the transcript and are never
14617
+ // re-executed; only hooks/commands/user shell/abort make the run unsafe.
14349
14618
  if (!managedFallback && !legacyRetryConfigured) {
14350
- if (
14351
- hasBareDefaultRetryDisqualifyingFacts(message) ||
14352
- (classification !== "transient" && classification !== "first_event_timeout") ||
14353
- !BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "") ||
14354
- !this.#hasCleanRetryReplaySafety
14355
- ) {
14619
+ const bareWatchdog =
14620
+ !hasBareDefaultRetryDisqualifyingFacts(message) &&
14621
+ (classification === "transient" || classification === "first_event_timeout") &&
14622
+ BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "");
14623
+ const bareThinkingOnly =
14624
+ thinkingOnlyFailure &&
14625
+ classification === "transient" &&
14626
+ message.errorKind === undefined &&
14627
+ !BARE_DEFAULT_THROTTLE_ERROR.test(message.errorMessage ?? "") &&
14628
+ (BARE_DEFAULT_SERVER_OVERLOAD_ERROR.test(message.errorMessage ?? "") ||
14629
+ isServerHttpStatus(this.#extractExplicitHttpStatusFromErrorMessage(message.errorMessage ?? ""))) &&
14630
+ (!transportFailure || classifyFallbackTrigger(transportFailure).class === "server");
14631
+ if (!(bareWatchdog || bareThinkingOnly) || !this.#hasCleanContinuationSafety) {
14356
14632
  return false;
14357
14633
  }
14358
14634
  }
@@ -14362,8 +14638,29 @@ export class AgentSession {
14362
14638
  ? this.#managedFallbackExhaustionDecision(message, message.errorMessage || "Model fallback attempt failed")
14363
14639
  : false;
14364
14640
  }
14365
- const legacyUnbounded = classification === "transient";
14366
- const attemptsUsed = managedFallback ? controller.attemptsUsed || 1 : this.#retryAttempt + 1;
14641
+ const transientTrigger = trigger.class === "server" || trigger.class === "rate_limit";
14642
+ // Session retries and per-selector attempts are different units. Respect an
14643
+ // explicit additional-session-retry cap before credential rotation or replay.
14644
+ if (
14645
+ transientTrigger &&
14646
+ (!retrySettings.enabled ||
14647
+ this.#retryAttempt >= (this.settings.has("retry.maxRetries") ? retrySettings.maxRetries : 6))
14648
+ ) {
14649
+ this.#transientStopTimestamp = message.timestamp;
14650
+ return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
14651
+ }
14652
+ if (visibleContinuation && !transientTrigger) return false;
14653
+ // Default policy: transient same-model failures are bounded by the shared step
14654
+ // budget (7 requests / 15 minutes after the first failure) instead of an
14655
+ // unbounded loop or a per-entry cap. Explicit retry.maxRetries/fallback.maxAttempts
14656
+ // keep their own attempt units and also stay inside the shared budget.
14657
+ const sharedBudgetOwnsTransient = managedFallback
14658
+ ? !this.settings.has("fallback.maxAttempts")
14659
+ : classification === "transient" && !this.settings.has("retry.maxRetries");
14660
+ controller.sessionBudgetOwnsTransient = managedFallback && sharedBudgetOwnsTransient;
14661
+ const sharedTransientRetry = sharedBudgetOwnsTransient && transientTrigger;
14662
+ const attemptsUsed =
14663
+ managedFallback && !sharedTransientRetry ? controller.attemptsUsed || 1 : this.#retryAttempt + 1;
14367
14664
  const failedSelector = managedFallback ? controller.currentSelector() : undefined;
14368
14665
  const usageLimited = trigger.class === "quota" || trigger.class === "rate_limit";
14369
14666
  let credentialMarked = false;
@@ -14374,15 +14671,64 @@ export class AgentSession {
14374
14671
  // leave the model while another account can still serve it.
14375
14672
  credentialMarked = true;
14376
14673
  credentialRotated = await this.#markFailedManagedCredential(trigger);
14377
- }
14378
- let outcome = managedFallback
14379
- ? controller.onAttemptFailure(trigger.class, message.errorMessage || "Unknown error", { credentialRotated })
14380
- : legacyUnbounded || attemptsUsed <= retrySettings.maxRetries
14381
- ? "retry"
14382
- : "exhausted";
14674
+ if (this.#stepRecoveryUncertainReasons.length > 0) {
14675
+ return managedOutcome
14676
+ ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } }
14677
+ : false;
14678
+ }
14679
+ }
14680
+ // A resumed run stays on the interrupted model. On a managed chain only the shared
14681
+ // same-model transient path may follow; any other failure stops before the chain is
14682
+ // consulted, because consulting it may already move its cursor to the next model.
14683
+ const leavesResumedModel =
14684
+ this.#activeAutoResume !== undefined && managedFallback && !(sharedBudgetOwnsTransient && transientTrigger);
14685
+ let outcome = leavesResumedModel
14686
+ ? "stop"
14687
+ : managedFallback
14688
+ ? controller.onAttemptFailure(trigger.class, message.errorMessage || "Unknown error", { credentialRotated })
14689
+ : sharedBudgetOwnsTransient || attemptsUsed <= retrySettings.maxRetries
14690
+ ? "retry"
14691
+ : "exhausted";
14383
14692
  if (managedFallback && credentialRotated && outcome === "advance" && controller.restorePreviousEntryForRetry()) {
14384
14693
  outcome = "retry";
14385
14694
  }
14695
+ const stepBudget = this.#stepRecoveryBudget;
14696
+ if (outcome === "retry" && stepBudget && (transientTrigger || !managedFallback))
14697
+ this.#recordStepRecoveryFailure(stepBudget);
14698
+ // Typed transport Retry-After is an uncapped floor; a floor beyond the remaining
14699
+ // deadline stops locally rather than contacting upstream early. A hint parsed from
14700
+ // legacy error prose is advisory and stays capped at retry.maxDelayMs.
14701
+ const parsedRetryAfterMs = managedFallback
14702
+ ? undefined
14703
+ : this.#parseRetryAfterMsFromError(message.errorMessage || "Unknown error");
14704
+ const retryAfterMs =
14705
+ trigger.retryAfterMs ??
14706
+ (parsedRetryAfterMs === undefined ? undefined : Math.min(parsedRetryAfterMs, retrySettings.maxDelayMs));
14707
+ const backoffBaseMs =
14708
+ sharedTransientRetry && !this.settings.has("retry.baseDelayMs")
14709
+ ? SAME_MODEL_RECOVERY_BASE_DELAY_MS
14710
+ : retrySettings.baseDelayMs;
14711
+ const backoffMaxMs =
14712
+ sharedTransientRetry && !this.settings.has("retry.maxDelayMs")
14713
+ ? SAME_MODEL_RECOVERY_MAX_DELAY_MS
14714
+ : retrySettings.maxDelayMs;
14715
+ const delayMs =
14716
+ credentialRotated || outcome === "advance"
14717
+ ? 0
14718
+ : managedFallback || transientTrigger
14719
+ ? effectiveFallbackDelay(backoffBaseMs, backoffMaxMs, attemptsUsed, retryAfterMs)
14720
+ : retryAfterMs !== undefined
14721
+ ? Math.min(retryAfterMs, retrySettings.maxDelayMs)
14722
+ : cappedExponentialWithFullJitter(backoffBaseMs, backoffMaxMs, attemptsUsed);
14723
+ if (outcome === "retry" && stepBudget && !stepBudget.canAdmitAfter(delayMs)) {
14724
+ // The shared request/time budget cannot admit another same-model request
14725
+ // after the required wait: stop on the same model with the real provider
14726
+ // error instead of sending it. Never switch models for it.
14727
+ outcome = managedFallback && !transientTrigger ? "exhausted" : "stop";
14728
+ }
14729
+ if (outcome === "exhausted" && !managedFallback && sharedBudgetOwnsTransient) outcome = "stop";
14730
+ // retry.enabled=false is an explicit opt-out of automatic same-model transient recovery.
14731
+ if (outcome === "retry" && transientTrigger && !retrySettings.enabled) outcome = "stop";
14386
14732
  if (outcome === "stop") {
14387
14733
  // Transient failure past the same-model retry budget: surface the real
14388
14734
  // provider error and keep the current model; never switch models for it.
@@ -14413,16 +14759,6 @@ export class AgentSession {
14413
14759
 
14414
14760
  const generation = this.#promptGeneration;
14415
14761
  const errorMessage = message.errorMessage || "Unknown error";
14416
- const retryAfterMs =
14417
- trigger.retryAfterMs ?? (managedFallback ? undefined : this.#parseRetryAfterMsFromError(errorMessage));
14418
- const delayMs =
14419
- credentialRotated || outcome === "advance"
14420
- ? 0
14421
- : managedFallback
14422
- ? effectiveFallbackDelay(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed, retryAfterMs)
14423
- : retryAfterMs !== undefined
14424
- ? Math.min(retryAfterMs, retrySettings.maxDelayMs)
14425
- : cappedExponentialWithFullJitter(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed);
14426
14762
 
14427
14763
  // Only a spent quota window hides the model from later resolution; a burst
14428
14764
  // rate limit retries the same model and must not push later turns elsewhere.
@@ -14474,15 +14810,24 @@ export class AgentSession {
14474
14810
  await this.#emitSessionEvent({
14475
14811
  type: "auto_retry_start",
14476
14812
  attempt: this.#retryAttempt,
14477
- maxAttempts: managedFallback ? controller.maxAttempts : retrySettings.maxRetries,
14813
+ maxAttempts:
14814
+ sharedBudgetOwnsTransient && transientTrigger
14815
+ ? SAME_MODEL_RECOVERY_MAX_REQUESTS - 1
14816
+ : managedFallback
14817
+ ? controller.maxAttempts
14818
+ : retrySettings.maxRetries,
14478
14819
  delayMs,
14479
14820
  errorMessage,
14480
- unbounded: managedFallback ? false : legacyUnbounded,
14821
+ unbounded: false,
14481
14822
  });
14482
14823
 
14483
- const messages = this.agent.state.messages;
14484
- if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
14485
- this.agent.replaceMessages(messages.slice(0, -1));
14824
+ // A preserved public prefix is never deleted or re-emitted; it is continued
14825
+ // after the backoff instead (see below).
14826
+ if (!visibleContinuation) {
14827
+ const messages = this.agent.state.messages;
14828
+ if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
14829
+ this.agent.replaceMessages(messages.slice(0, -1));
14830
+ }
14486
14831
  }
14487
14832
 
14488
14833
  try {
@@ -14522,6 +14867,19 @@ export class AgentSession {
14522
14867
  if (this.#retryAbortController === retryAbortController) this.#retryAbortController = undefined;
14523
14868
  this.#retryNowRequested = false;
14524
14869
 
14870
+ if (visibleContinuation) {
14871
+ // Owner re-check before any durable write: a newer prompt supersedes it.
14872
+ if (this.#promptGeneration !== generation) {
14873
+ this.#failRetryRecovery("Retry continuation was superseded");
14874
+ return;
14875
+ }
14876
+ if (!(await this.#appendVisibleContinuation(message))) {
14877
+ this.#transientStopTimestamp = message.timestamp;
14878
+ this.#failRetryRecovery("Recovery checkpoint could not be saved");
14879
+ return;
14880
+ }
14881
+ }
14882
+
14525
14883
  if (managedOutcome) {
14526
14884
  try {
14527
14885
  await this.#checkEstimatedContextBeforePrompt();
@@ -14560,6 +14918,7 @@ export class AgentSession {
14560
14918
  delayMs: 1,
14561
14919
  generation,
14562
14920
  allowDuringCancelAndSubmit: true,
14921
+ shouldContinue: () => !visibleContinuation || this.#hasVisibleRecoveryReferences(),
14563
14922
  onError: () => this.#failRetryRecovery("Retry continuation failed to start"),
14564
14923
  onSkip: () => this.#failRetryRecovery("Retry continuation was superseded"),
14565
14924
  });
@@ -14580,6 +14939,580 @@ export class AgentSession {
14580
14939
  this.#resolveRetry();
14581
14940
  }
14582
14941
 
14942
+ /**
14943
+ * Single admission point for every concrete upstream request of the current
14944
+ * model step. Rejects (throws {@link RecoveryAdmissionError}) once the shared
14945
+ * same-model budget is spent or past its deadline, before anything is sent.
14946
+ */
14947
+ #getUpstreamRequestLease: NonNullable<AgentLoopConfig["getUpstreamRequestLease"]> = async (context, signal) => {
14948
+ if (
14949
+ !this.#stepRecoveryBudget &&
14950
+ this.#unresolvedRecoveryAtHydration &&
14951
+ context.messages.at(-1)?.role !== "user"
14952
+ ) {
14953
+ throw new RecoveryAdmissionError("cancelled");
14954
+ }
14955
+ // A restart resume's first step runs only inside its own small envelope: once that
14956
+ // envelope is gone before the step was accepted (abort, new prompt), no default
14957
+ // budget may take its place.
14958
+ if (!this.#stepRecoveryBudget && this.#restartResumeEnvelopePending) {
14959
+ throw new RecoveryAdmissionError("cancelled");
14960
+ }
14961
+ // The durable record of in-step uncertainty must land before anything else is sent.
14962
+ await this.#pendingUncertainFlush;
14963
+ if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
14964
+ // Unobservable remote work already happened in this step: no further upstream request
14965
+ // for it, whichever path (retry, rotation, fallback) asks next.
14966
+ if (this.#stepRecoveryBudget && this.#stepRecoveryUncertainReasons.length > 0) {
14967
+ throw new RecoveryAdmissionError("cancelled");
14968
+ }
14969
+ const generation = this.#promptGeneration;
14970
+ this.#stepRecoveryBudget ??= new RecoveryBudget({ maxSilentTimeouts: this.#maxSilentTimeouts() });
14971
+ this.#stepRecoveryDeadlineAbort ??= new AbortController();
14972
+ this.#stepRecoveryStepId ??= crypto.randomUUID();
14973
+ const budget = this.#stepRecoveryBudget;
14974
+ const deadlineAbort = this.#stepRecoveryDeadlineAbort;
14975
+ const stepId = this.#stepRecoveryStepId;
14976
+ const model = this.model;
14977
+ const validate = (): void => {
14978
+ if (
14979
+ signal?.aborted ||
14980
+ this.#isDisposed ||
14981
+ this.#promptGeneration !== generation ||
14982
+ this.#stepRecoveryBudget !== budget ||
14983
+ this.model !== model
14984
+ ) {
14985
+ throw new RecoveryAdmissionError("cancelled");
14986
+ }
14987
+ budget.assertActive();
14988
+ };
14989
+ validate();
14990
+ this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
14991
+ version: 1,
14992
+ state: "in_flight",
14993
+ logicalRunId: String(this.agent.currentManagedLogicalRunId ?? `${this.sessionId}:${generation}`),
14994
+ stepId,
14995
+ generation,
14996
+ attemptId: String(this.#retryAttempt),
14997
+ selector: model ? `${model.provider}/${model.id}` : undefined,
14998
+ api: model?.api,
14999
+ transport: model?.transport,
15000
+ uncertain: [...this.#stepRecoveryUncertainReasons],
15001
+ // Only this user turn's already-visible assistant output belongs to the step.
15002
+ visibleMessageIds: context.messages
15003
+ .slice(
15004
+ Math.max(
15005
+ 0,
15006
+ context.messages.findLastIndex(message => message.role === "user"),
15007
+ ),
15008
+ )
15009
+ .filter(message => message.role === "assistant")
15010
+ .map(getSessionMessageEntryId)
15011
+ .filter((id): id is string => id !== undefined),
15012
+ usedRequests: budget.snapshot().usedRequests,
15013
+ });
15014
+ await this.sessionManager.ensureOnDisk();
15015
+ await this.sessionManager.flush();
15016
+ validate();
15017
+ return {
15018
+ validate,
15019
+ signal: deadlineAbort.signal,
15020
+ // Records only; the identity check lives in #recordStepRecoveryFailure so the
15021
+ // original response/error is never replaced by an admission error.
15022
+ onRecoverableFailure: () => this.#recordStepRecoveryFailure(budget),
15023
+ onUncertainUpstream: (reason: string): void => this.#recordStepUncertainUpstream(budget, reason),
15024
+ onAttemptSettled: (message: AssistantMessage): void => this.#recordStepAttemptSettled(budget, message),
15025
+ onRequest: (kind: RecoveryRequestKind): void => {
15026
+ validate();
15027
+ if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
15028
+ // Unobservable remote work was reported inside this invocation: a resend could
15029
+ // duplicate it, so only the first concrete request of each kind remains admissible.
15030
+ if (kind === "resend" && this.#stepRecoveryUncertainReasons.length > 0) {
15031
+ throw new RecoveryAdmissionError("cancelled");
15032
+ }
15033
+ if (kind === "resend") this.#recordStepRecoveryFailure(budget);
15034
+ budget.reserve(kind);
15035
+ },
15036
+ };
15037
+ };
15038
+
15039
+ async #completeStepRecoveryCheckpoint(message: AssistantMessage): Promise<void> {
15040
+ if (!this.#stepRecoveryStepId || this.agent.state.messages.at(-1) !== message) return;
15041
+ this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
15042
+ version: 1,
15043
+ state: "completed",
15044
+ stepId: this.#stepRecoveryStepId,
15045
+ generation: this.#promptGeneration,
15046
+ visibleMessageIds: [getSessionMessageEntryId(message)].filter((id): id is string => id !== undefined),
15047
+ });
15048
+ try {
15049
+ await this.sessionManager.ensureOnDisk();
15050
+ await this.sessionManager.flush();
15051
+ } catch (error) {
15052
+ logger.warn("Recovery completion checkpoint failed", {
15053
+ error: error instanceof Error ? error.message : String(error),
15054
+ });
15055
+ }
15056
+ }
15057
+
15058
+ /**
15059
+ * Unobservable remote work cannot be proven idempotent: no automatic resend/continuation.
15060
+ * The reason is also persisted with the step so a restart never resumes it; the next
15061
+ * admission and every resume decision wait for that write.
15062
+ */
15063
+ #recordStepUncertainUpstream(budget: RecoveryBudget, reason: string): void {
15064
+ if (this.#stepRecoveryBudget !== budget) return;
15065
+ if (!this.#stepRecoveryUncertainReasons.includes(reason)) this.#stepRecoveryUncertainReasons.push(reason);
15066
+ this.#markRetryReplayUnsafe();
15067
+ const model = this.model;
15068
+ this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
15069
+ version: 1,
15070
+ state: "in_flight",
15071
+ stepId: this.#stepRecoveryStepId,
15072
+ generation: this.#promptGeneration,
15073
+ selector: model ? `${model.provider}/${model.id}` : undefined,
15074
+ api: model?.api,
15075
+ transport: model?.transport,
15076
+ uncertain: [...this.#stepRecoveryUncertainReasons],
15077
+ visibleMessageIds: [],
15078
+ usedRequests: budget.snapshot().usedRequests,
15079
+ });
15080
+ this.#pendingUncertainFlush = this.#pendingUncertainFlush
15081
+ .then(async () => {
15082
+ await this.sessionManager.ensureOnDisk();
15083
+ await this.sessionManager.flush();
15084
+ })
15085
+ .catch(error => {
15086
+ this.#uncertainFlushFailed = true;
15087
+ const reason = error instanceof Error ? error.message : String(error);
15088
+ logger.warn("Uncertain upstream checkpoint failed", { error: reason });
15089
+ this.emitNotice(
15090
+ "warning",
15091
+ `Stopped retrying this step: its recovery state could not be saved (${reason}).`,
15092
+ "retry",
15093
+ );
15094
+ });
15095
+ }
15096
+
15097
+ #maxSilentTimeouts(): number {
15098
+ return Math.min(7, Math.max(1, Math.trunc(this.settings.get("retry.maxSilentTimeouts")) || 3));
15099
+ }
15100
+
15101
+ /** Consecutive no-output timeouts close the step early, inside its request budget. */
15102
+ #recordStepAttemptSettled(budget: RecoveryBudget, message: AssistantMessage): void {
15103
+ if (this.#stepRecoveryBudget !== budget) return;
15104
+ const progressed = assistantMessageHasVisibleOrToolContent(message);
15105
+ const silentTimeout =
15106
+ message.stopReason === "error" && BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "");
15107
+ if (!budget.recordAttemptOutcome({ silentTimeout, progressed, selector: `${message.provider}/${message.model}` }))
15108
+ return;
15109
+ logger.info("silent_stall_tripped", { count: budget.snapshot().silentTimeouts });
15110
+ this.emitNotice(
15111
+ "warning",
15112
+ `Stopped after ${budget.snapshot().silentTimeouts} consecutive timeouts without any output.`,
15113
+ "retry",
15114
+ );
15115
+ }
15116
+
15117
+ #recordStepRecoveryFailure(budget: RecoveryBudget): void {
15118
+ if (this.#stepRecoveryBudget !== budget) return;
15119
+ budget.recordRecoverableFailure();
15120
+ if (this.#cancelStepRecoveryDeadline) return;
15121
+ const deadline = budget.snapshot().deadlineAtMs;
15122
+ if (deadline === undefined) return;
15123
+ const controller = this.#stepRecoveryDeadlineAbort;
15124
+ const timer = setTimeout(
15125
+ () => controller?.abort(new RecoveryAdmissionError("deadline")),
15126
+ Math.max(0, deadline - performance.now()),
15127
+ );
15128
+ this.#cancelStepRecoveryDeadline = () => clearTimeout(timer);
15129
+ }
15130
+
15131
+ /** Ends and permanently closes this lease; late callbacks never start a new budget. */
15132
+ #endStepRecoveryBudget(options?: { accepted?: boolean }): void {
15133
+ // Only an unfinished step leaves unobservable remote effects unresolved.
15134
+ if (!options?.accepted && this.#stepRecoveryUncertainReasons.length > 0) {
15135
+ this.#uncertainUpstreamUnresolved = true;
15136
+ }
15137
+ this.#stepRecoveryBudget?.cancel();
15138
+ this.#cancelStepRecoveryDeadline?.();
15139
+ this.#cancelStepRecoveryDeadline = undefined;
15140
+ this.#stepRecoveryDeadlineAbort = undefined;
15141
+ this.#stepRecoveryStepId = undefined;
15142
+ this.#visibleRecoveryReferenceIds = [];
15143
+ this.#visibleRecoveryContextIds = [];
15144
+ this.#stepRecoveryUncertainReasons = [];
15145
+ this.#stepRecoveryBudget = undefined;
15146
+ }
15147
+
15148
+ /**
15149
+ * Whether a failed step that already showed output can be continued on the same
15150
+ * model instead of replayed: the failure must be the live trailing assistant,
15151
+ * its public tail must be text (no tool calls/images), the serializer must have
15152
+ * a verified projection, and nothing outside the transcript tainted the run.
15153
+ */
15154
+ #canContinueVisibleFailure(message: AssistantMessage): boolean {
15155
+ if (message.stopReason !== "error") return false;
15156
+ if (!VISIBLE_CONTINUATION_APIS.has(message.api) || this.model?.transport === "pi-native") return false;
15157
+ if (!this.model || this.model.api !== message.api || this.model.provider !== message.provider) return false;
15158
+ if (this.model.id !== message.model) return false;
15159
+ if (!isTextVisibleTail(message)) return false;
15160
+ if (!this.#hasCleanContinuationSafety || this.#stepRecoveryUncertainReasons.length > 0) return false;
15161
+ return this.agent.state.messages.at(-1) === message && this.#getVisibleRecoveryReferences(message) !== undefined;
15162
+ }
15163
+
15164
+ /**
15165
+ * Persisted entry ids of the current user turn's failed text tails and completed
15166
+ * tool pairs; undefined when any is missing from the branch or a tail is not text.
15167
+ */
15168
+ #getVisibleRecoveryReferences(message: AssistantMessage):
15169
+ | {
15170
+ visibleMessageIds: string[];
15171
+ completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }>;
15172
+ }
15173
+ | undefined {
15174
+ const messages = this.agent.state.messages;
15175
+ const start = messages.findLastIndex(item => item.role === "user");
15176
+ const tail = messages.slice(Math.max(0, start));
15177
+ const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
15178
+ const visibleMessageIds: string[] = [];
15179
+ const completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }> = [];
15180
+ for (const item of tail) {
15181
+ if (item.role !== "assistant") continue;
15182
+ const entryId = getSessionMessageEntryId(item);
15183
+ if (!entryId || !branchIds.has(entryId)) return undefined;
15184
+ if (item.stopReason === "error" || item.stopReason === "aborted") {
15185
+ if (!isTextVisibleTail(item)) return undefined;
15186
+ visibleMessageIds.push(entryId);
15187
+ }
15188
+ for (const block of item.content) {
15189
+ if (block.type !== "toolCall") continue;
15190
+ const result = tail.find(candidate => candidate.role === "toolResult" && candidate.toolCallId === block.id);
15191
+ if (result?.role !== "toolResult" || result.isError) return undefined;
15192
+ const resultEntryId = getSessionMessageEntryId(result);
15193
+ if (!resultEntryId || !branchIds.has(resultEntryId)) return undefined;
15194
+ completedTools.push({ toolCallId: block.id, callEntryId: entryId, resultEntryId });
15195
+ }
15196
+ }
15197
+ const messageId = getSessionMessageEntryId(message);
15198
+ if (!messageId || !visibleMessageIds.includes(messageId)) return undefined;
15199
+ return { visibleMessageIds, completedTools };
15200
+ }
15201
+
15202
+ #hasVisibleRecoveryReferences(): boolean {
15203
+ if (this.#visibleRecoveryReferenceIds.length === 0) return false;
15204
+ const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
15205
+ if (!this.#visibleRecoveryReferenceIds.every(id => branchIds.has(id) && this.sessionManager.getEntry(id))) {
15206
+ return false;
15207
+ }
15208
+ // Compaction can keep the entries on the branch while replacing them in the model
15209
+ // context; the preserved prefix and completed tool pairs must still be what is sent.
15210
+ const liveIds = new Set(
15211
+ this.agent.state.messages.map(getSessionMessageEntryId).filter((id): id is string => id !== undefined),
15212
+ );
15213
+ return this.#visibleRecoveryContextIds.every(id => liveIds.has(id));
15214
+ }
15215
+
15216
+ /**
15217
+ * Durably checkpoints the preserved prefix and appends one continuation
15218
+ * instruction. Returns false (and sends nothing) when persistence fails.
15219
+ */
15220
+ async #appendVisibleContinuation(message: AssistantMessage): Promise<boolean> {
15221
+ if (this.agent.state.messages.at(-1) !== message) return false;
15222
+ const references = this.#getVisibleRecoveryReferences(message);
15223
+ if (!references) return false;
15224
+ const generation = this.#promptGeneration;
15225
+ const budget = this.#stepRecoveryBudget;
15226
+ const details = {
15227
+ version: 1,
15228
+ state: "recovering",
15229
+ runId: `${this.sessionId}:${generation}`,
15230
+ stepId: references.visibleMessageIds[0],
15231
+ attemptId: String(this.#retryAttempt),
15232
+ generation,
15233
+ selector: `${message.provider}/${message.model}`,
15234
+ ...references,
15235
+ uncertainTools: this.#stepRecoveryUncertainReasons.map(reason => ({ reason })),
15236
+ usedRequests: budget?.snapshot().usedRequests ?? 0,
15237
+ };
15238
+ const continuation: AgentMessage = {
15239
+ role: "custom",
15240
+ customType: VISIBLE_CONTINUATION_CUSTOM_TYPE,
15241
+ content: VISIBLE_CONTINUATION_PROMPT,
15242
+ display: false,
15243
+ details,
15244
+ attribution: "agent",
15245
+ timestamp: Date.now(),
15246
+ };
15247
+ try {
15248
+ const continuationMessageId = this.sessionManager.appendCustomMessageEntry(
15249
+ VISIBLE_CONTINUATION_CUSTOM_TYPE,
15250
+ VISIBLE_CONTINUATION_PROMPT,
15251
+ false,
15252
+ details,
15253
+ "agent",
15254
+ );
15255
+ const checkpointEntryId = this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
15256
+ ...details,
15257
+ continuationMessageId,
15258
+ });
15259
+ // New sessions defer their first write until an assistant exists. Explicitly
15260
+ // materialize before flushing; flush alone need not have a file to fsync.
15261
+ await this.sessionManager.ensureOnDisk();
15262
+ await this.sessionManager.flush();
15263
+ if (this.#isDisposed || this.#promptGeneration !== generation || this.#stepRecoveryBudget !== budget)
15264
+ return false;
15265
+ if (!budget?.canAdmit() || this.agent.state.messages.at(-1) !== message) return false;
15266
+ const contextIds = [
15267
+ ...references.visibleMessageIds,
15268
+ ...references.completedTools.flatMap(tool => [tool.callEntryId, tool.resultEntryId]),
15269
+ ];
15270
+ this.#visibleRecoveryReferenceIds = [checkpointEntryId, continuationMessageId, ...contextIds];
15271
+ this.#visibleRecoveryContextIds = contextIds;
15272
+ if (!this.#hasVisibleRecoveryReferences()) return false;
15273
+ this.agent.appendMessage(continuation);
15274
+ return true;
15275
+ } catch (error) {
15276
+ logger.warn("Visible continuation checkpoint failed", {
15277
+ error: error instanceof Error ? error.message : String(error),
15278
+ });
15279
+ return false;
15280
+ }
15281
+ }
15282
+
15283
+ /** Which in-process interruption, if any, the settled assistant represents. */
15284
+ #autoResumeTriggerFor(message: AssistantMessage): AutoResumeTrigger | undefined {
15285
+ // Managed chains settle here too (the session supplies no managed recovery evidence);
15286
+ // the resume itself is pinned to the interrupted model. Only a transient failure is
15287
+ // resumed: terminal provider answers and an explicit retry opt-out still stop.
15288
+ if (message.stopReason !== "error" || !this.settings.get("retry.enabled")) return undefined;
15289
+ if (!this.#isResumableFailure(message)) return undefined;
15290
+ // Work worth keeping exists: tools already ran or the user saw part of the answer.
15291
+ const turnStart = this.agent.state.messages.findLastIndex(item => item.role === "user");
15292
+ const followsTools = this.agent.state.messages.slice(turnStart + 1).some(item => item.role === "toolResult");
15293
+ return followsTools || hasVisibleFailedPrefix([message]) ? "interrupted" : undefined;
15294
+ }
15295
+
15296
+ /**
15297
+ * Error eligibility shared by in-process and restart resume. Only a transient
15298
+ * failure is resumed: terminal provider answers, rate limits and quota stop, since
15299
+ * the retry cap already decided to stop and another request would hit the same limit.
15300
+ */
15301
+ #isResumableFailure(message: AssistantMessage): boolean {
15302
+ const classification = this.#classifyErrorForRetry(message);
15303
+ if (classification !== "transient" && classification !== "first_event_timeout") return false;
15304
+ // Throttle copy can quote bare numbers ("500 requests per minute") that the
15305
+ // transient classifier reads as a status; it is a limit, never resumed.
15306
+ if (BARE_DEFAULT_THROTTLE_ERROR.test(message.errorMessage ?? "")) return false;
15307
+ const trigger = this.#fallbackTriggerFor(message, true, message.transportFailure);
15308
+ return trigger?.class !== "rate_limit" && trigger?.class !== "quota" && trigger?.class !== "auth";
15309
+ }
15310
+
15311
+ /** Last persisted recovery checkpoint of the branch, when it records an unfinished step. */
15312
+ #lastInFlightCheckpoint(): Record<string, unknown> | undefined {
15313
+ const entry = this.sessionManager
15314
+ .getBranch()
15315
+ .findLast(item => item.type === "custom" && item.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE);
15316
+ if (entry?.type !== "custom" || !entry.data || typeof entry.data !== "object" || Array.isArray(entry.data))
15317
+ return undefined;
15318
+ const data = entry.data as Record<string, unknown>;
15319
+ return data.state === "in_flight" || data.state === "recovering" ? data : undefined;
15320
+ }
15321
+
15322
+ /**
15323
+ * Single decision point for an automatic resume of an interrupted step. Runs at
15324
+ * most once per user turn, on the interrupted step's selector, and only after the
15325
+ * resume marker is durable. Returns true when a resume was scheduled.
15326
+ */
15327
+ async #maybeAutoResume(trigger: AutoResumeTrigger): Promise<boolean> {
15328
+ await this.#pendingUncertainFlush;
15329
+ const generation = this.#promptGeneration;
15330
+ const messages = this.agent.state.messages;
15331
+ const model = this.model;
15332
+ const checkpoint = trigger === "restart" ? this.#lastInFlightCheckpoint() : undefined;
15333
+ const failed = messages.findLast(message => message.role === "assistant") as AssistantMessage | undefined;
15334
+ // A restart re-checks the stored failure: a step that stopped on a rate limit,
15335
+ // quota, auth or terminal answer stays stopped across processes too.
15336
+ if (
15337
+ trigger === "restart" &&
15338
+ failed?.stopReason === "error" &&
15339
+ messages.lastIndexOf(failed) > messages.findLastIndex(message => message.role === "user") &&
15340
+ !this.#isResumableFailure(failed)
15341
+ ) {
15342
+ logger.debug("auto_resume_paused", { trigger, reason: "non_transient_failure" });
15343
+ return false;
15344
+ }
15345
+ const selector =
15346
+ trigger === "restart"
15347
+ ? typeof checkpoint?.selector === "string"
15348
+ ? checkpoint.selector
15349
+ : undefined
15350
+ : failed
15351
+ ? `${failed.provider}/${failed.model}`
15352
+ : undefined;
15353
+ const restartUncertain =
15354
+ trigger === "restart" &&
15355
+ (!checkpoint ||
15356
+ !Array.isArray(checkpoint.uncertain) ||
15357
+ checkpoint.uncertain.length > 0 ||
15358
+ typeof checkpoint.api !== "string" ||
15359
+ checkpoint.api === "cursor-agent" ||
15360
+ checkpoint.transport === "pi-native");
15361
+ const budget = this.#stepRecoveryBudget;
15362
+ const selectedTools = this.agent.state.tools;
15363
+ const decision = resolveAutoResumePolicy({
15364
+ enabled: this.settings.get("retry.autoResume") && !this.#isDisposed && !this.#handoffTransitionActive,
15365
+ trigger,
15366
+ messages,
15367
+ resumedThisTurn: hasAutoResumeMarkerSinceLastUser(this.sessionManager.getBranch()),
15368
+ uncertainUpstream:
15369
+ this.#uncertainFlushFailed ||
15370
+ this.#uncertainUpstreamUnresolved ||
15371
+ this.#stepRecoveryUncertainReasons.length > 0 ||
15372
+ restartUncertain,
15373
+ opaqueTransport: model?.api === "cursor-agent" || model?.transport === "pi-native",
15374
+ budgetAdmits: trigger === "restart" ? true : budget?.canAdmit() === true,
15375
+ userInputPending: this.agent.hasQueuedMessages() || this.#cancelAndSubmitInProgress,
15376
+ selectorAvailable: model !== undefined && selector === `${model.provider}/${model.id}`,
15377
+ prefixReplayable:
15378
+ failed !== undefined &&
15379
+ VISIBLE_CONTINUATION_APIS.has(failed.api) &&
15380
+ model?.api === failed.api &&
15381
+ this.#hasCleanContinuationSafety,
15382
+ selectedTools,
15383
+ });
15384
+ if (decision.type === "pause") {
15385
+ logger.debug("auto_resume_paused", { trigger, reason: decision.reason });
15386
+ if (decision.reason === "already_resumed") {
15387
+ this.emitNotice("info", "Automatic resume was already used for this request.", "retry");
15388
+ }
15389
+ return false;
15390
+ }
15391
+ if (!selector) return false;
15392
+ const marker: AutoResumeMarker = {
15393
+ version: 1,
15394
+ trigger,
15395
+ mode: decision.mode,
15396
+ selector,
15397
+ interruptedCallIds: decision.interruptedCalls.map(call => call.id),
15398
+ };
15399
+ const placeholders: ToolResultMessage[] = decision.interruptedCalls.map(call => ({
15400
+ role: "toolResult",
15401
+ toolCallId: call.id,
15402
+ toolName: call.name,
15403
+ content: [{ type: "text", text: INTERRUPTED_TOOL_RESULT_TEXT }],
15404
+ isError: true,
15405
+ timestamp: Date.now(),
15406
+ }));
15407
+ const prompt = decision.mode === "restricted" ? AUTO_RESUME_RESTRICTED_PROMPT : AUTO_RESUME_FULL_PROMPT;
15408
+ const tailBeforeFlush = messages.at(-1);
15409
+ try {
15410
+ this.sessionManager.appendCustomEntry(AUTO_RESUME_MARKER_CUSTOM_TYPE, marker);
15411
+ for (const placeholder of placeholders) this.sessionManager.appendMessage(placeholder);
15412
+ this.sessionManager.appendCustomMessageEntry(AUTO_RESUME_CUSTOM_TYPE, prompt, false, marker, "agent");
15413
+ await this.sessionManager.ensureOnDisk();
15414
+ await this.sessionManager.flush();
15415
+ } catch (error) {
15416
+ logger.warn("Automatic resume checkpoint failed", {
15417
+ error: error instanceof Error ? error.message : String(error),
15418
+ });
15419
+ return false;
15420
+ }
15421
+ if (
15422
+ this.#isDisposed ||
15423
+ this.#promptGeneration !== generation ||
15424
+ this.agent.state.messages !== messages ||
15425
+ messages.at(-1) !== tailBeforeFlush
15426
+ ) {
15427
+ // Fail-safe: nothing is sent. The durable marker still consumes this turn's resume.
15428
+ logger.warn("auto_resume_abandoned", { trigger, reason: "state changed while the marker was written" });
15429
+ return false;
15430
+ }
15431
+ // A failed attempt that showed nothing, or only private reasoning, is not part of the
15432
+ // answer; resending it only adds an empty assistant turn (the same rule manual retry applies).
15433
+ const tailMessage = this.agent.state.messages.at(-1);
15434
+ if (
15435
+ placeholders.length === 0 &&
15436
+ tailMessage?.role === "assistant" &&
15437
+ (tailMessage.stopReason === "error" || tailMessage.stopReason === "aborted") &&
15438
+ (isThinkingOnlyFailure(tailMessage) || !assistantMessageHasVisibleOrToolContent(tailMessage))
15439
+ ) {
15440
+ this.agent.replaceMessages(this.agent.state.messages.slice(0, -1));
15441
+ }
15442
+ // The durable placeholders replace in-memory pairing results that observed nothing.
15443
+ if (this.agent.state.messages.some(isSynthesizedResumeResult)) {
15444
+ this.agent.replaceMessages(this.agent.state.messages.filter(message => !isSynthesizedResumeResult(message)));
15445
+ }
15446
+ for (const placeholder of placeholders) this.agent.appendMessage(placeholder);
15447
+ this.agent.appendMessage({
15448
+ role: "custom",
15449
+ customType: AUTO_RESUME_CUSTOM_TYPE,
15450
+ content: prompt,
15451
+ display: false,
15452
+ details: marker,
15453
+ attribution: "agent",
15454
+ timestamp: Date.now(),
15455
+ });
15456
+ if (trigger === "restart") {
15457
+ this.#endStepRecoveryBudget();
15458
+ this.#stepRecoveryBudget = new RecoveryBudget({
15459
+ maxRequests: RESTART_RESUME_MAX_REQUESTS,
15460
+ windowMs: RESTART_RESUME_WINDOW_MS,
15461
+ maxSilentTimeouts: this.#maxSilentTimeouts(),
15462
+ });
15463
+ // The small envelope starts its clock now and arms its deadline: an unanswered
15464
+ // restart resume never waits longer, even mid-request.
15465
+ this.#stepRecoveryDeadlineAbort = new AbortController();
15466
+ this.#recordStepRecoveryFailure(this.#stepRecoveryBudget);
15467
+ this.#restartResumeEnvelopePending = true;
15468
+ this.#unresolvedRecoveryAtHydration = false;
15469
+ }
15470
+ if (decision.mode === "restricted") this.#setRestrictedToolMode(true);
15471
+ this.#activeAutoResume = { trigger, mode: decision.mode };
15472
+ this.emitNotice(
15473
+ "info",
15474
+ decision.mode === "restricted"
15475
+ ? "Resuming the interrupted step automatically (read-only tools)."
15476
+ : "Resuming the interrupted step automatically.",
15477
+ "retry",
15478
+ );
15479
+ logger.info("auto_resume_started", { trigger, mode: decision.mode, interruptedCalls: placeholders.length });
15480
+ if (trigger === "restart") return true;
15481
+ void this.#scheduleAgentContinue({
15482
+ delayMs: 1,
15483
+ generation,
15484
+ allowDuringCancelAndSubmit: false,
15485
+ pinnedModel: true,
15486
+ shouldContinue: () => !this.agent.hasQueuedMessages(),
15487
+ // A resume that never started closes the step it was carrying, unless a newer
15488
+ // turn already owns the envelope.
15489
+ onError: () => {
15490
+ this.#endAutoResume("failed");
15491
+ if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
15492
+ },
15493
+ onSkip: () => {
15494
+ this.#endAutoResume("skipped");
15495
+ if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
15496
+ },
15497
+ });
15498
+ return true;
15499
+ }
15500
+
15501
+ /** Restricted mode narrows only what the model sees and may execute; the selection is untouched. */
15502
+ #setRestrictedToolMode(enabled: boolean): void {
15503
+ if (this.#restrictedToolMode === enabled) return;
15504
+ this.#restrictedToolMode = enabled;
15505
+ this.agent.setToolFilter(enabled ? tools => tools.filter(tool => isRestartSafeTool(tool, tools)) : undefined);
15506
+ }
15507
+
15508
+ #endAutoResume(outcome: "completed" | "failed" | "skipped"): void {
15509
+ this.#restartResumeEnvelopePending = false;
15510
+ this.#setRestrictedToolMode(false);
15511
+ const active = this.#activeAutoResume;
15512
+ this.#activeAutoResume = undefined;
15513
+ if (active) logger.info("auto_resume_finished", { ...active, outcome });
15514
+ }
15515
+
14583
15516
  /**
14584
15517
  * Skip the current retry backoff and re-attempt immediately. Distinct from
14585
15518
  * abortRetry(), which cancels the retry and returns to idle. No-op when no
@@ -14672,20 +15605,21 @@ export class AgentSession {
14672
15605
  );
14673
15606
  }
14674
15607
 
14675
- #isUnresolvedToolUseAssistant(message: AssistantMessage): boolean {
14676
- return message.stopReason === "toolUse" && message.content.some(content => content.type === "toolCall");
14677
- }
14678
-
14679
15608
  /**
14680
15609
  * Manually retry the last failed assistant turn, or resume an interrupted tail
14681
15610
  * left by a non-graceful process exit after the user/custom/tool-result message
14682
15611
  * was persisted but before the agent emitted a terminal assistant response.
14683
- * Removes failed/aborted/unresolved tool-use assistant tails before
14684
- * re-attempting with a fresh retry budget.
14685
- * @returns true if retry/resume was initiated, false if no retryable tail exists or agent is busy
15612
+ * Removes only a failed/aborted assistant tail with no visible output or tool
15613
+ * calls, then re-attempts with a fresh retry budget.
15614
+ * @returns false when the agent is busy, no retryable tail exists, the tail shows
15615
+ * output or tool calls, a persisted recovery is unresolved, or the step dispatched
15616
+ * unobservable remote work; true when a retry/resume was scheduled.
14686
15617
  */
14687
15618
  async retry(): Promise<boolean> {
14688
15619
  if (this.isStreaming || this.isCompacting || this.isRetrying) return false;
15620
+ if (this.#unresolvedRecoveryAtHydration) return false;
15621
+ // Unobservable remote effects (broker/gateway) cannot be proven unsent.
15622
+ if (this.#uncertainUpstreamUnresolved || this.#stepRecoveryUncertainReasons.length > 0) return false;
14689
15623
  // A handoff transition owns the session; retrying would mutate the tail and
14690
15624
  // schedule a continuation against the session being handed off.
14691
15625
  if (this.isGeneratingHandoff || this.#handoffTransitionActive) return false;
@@ -14697,22 +15631,24 @@ export class AgentSession {
14697
15631
  if (lastMsg.role !== "assistant") {
14698
15632
  if (!this.#isInterruptedRetryTail(lastMsg)) return false;
14699
15633
  this.#retryAttempt = 0;
15634
+ this.#endStepRecoveryBudget();
14700
15635
  this.#scheduleAgentContinue({ delayMs: 1 });
14701
15636
  return true;
14702
15637
  }
14703
15638
 
14704
15639
  const assistantMsg = lastMsg as AssistantMessage;
14705
- const shouldDropAssistant =
14706
- assistantMsg.stopReason === "error" ||
14707
- assistantMsg.stopReason === "aborted" ||
14708
- this.#isUnresolvedToolUseAssistant(assistantMsg);
14709
- if (!shouldDropAssistant) return false;
15640
+ // Explicit retry does not establish the outcome of an interrupted operation
15641
+ // and is never permission to erase a prefix already shown to the user. A failure
15642
+ // that streamed only reasoning holds no answer and is dropped like an empty one.
15643
+ if (!isThinkingOnlyFailure(assistantMsg) && assistantMessageHasVisibleOrToolContent(assistantMsg)) return false;
15644
+ if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") return false;
14710
15645
 
14711
- // Remove the failed/aborted/incomplete assistant message before re-attempting.
15646
+ // Remove the empty failed/aborted assistant message before re-attempting.
14712
15647
  this.agent.replaceMessages(messages.slice(0, -1));
14713
15648
 
14714
- // Reset retry budget for a fresh attempt
15649
+ // Reset retry budget for a fresh attempt: a manual retry is a new invocation.
14715
15650
  this.#retryAttempt = 0;
15651
+ this.#endStepRecoveryBudget();
14716
15652
 
14717
15653
  // Re-attempt the turn
14718
15654
  this.#scheduleAgentContinue({ delayMs: 1 });
@@ -15325,7 +16261,6 @@ export class AgentSession {
15325
16261
  systemPrompt: [...this.systemPrompt, instruction],
15326
16262
  messages: this.#projectBtwVisibleText(this.buildDisplaySessionContext().messages),
15327
16263
  thinkingLevel: this.thinkingLevel ?? ThinkingLevel.Off,
15328
- hideThinkingSummary: this.agent.hideThinkingSummary ?? false,
15329
16264
  serviceTier: this.serviceTier,
15330
16265
  credentialSessionId: providerAffinitySessionId,
15331
16266
  providerAffinitySessionId,
@@ -15413,7 +16348,6 @@ export class AgentSession {
15413
16348
  this.sessionId,
15414
16349
  ),
15415
16350
  reasoning: toReasoningEffort(this.thinkingLevel),
15416
- hideThinkingSummary: this.agent.hideThinkingSummary,
15417
16351
  serviceTier: this.serviceTier,
15418
16352
  signal: args.signal,
15419
16353
  toolChoice: "none",
@@ -15486,7 +16420,6 @@ export class AgentSession {
15486
16420
  apiKey,
15487
16421
  sessionId: scope.sideSessionId,
15488
16422
  reasoning: toReasoningEffort(scope.thinkingLevel),
15489
- hideThinkingSummary: scope.hideThinkingSummary,
15490
16423
  serviceTier: scope.serviceTier,
15491
16424
  signal: requestSignal,
15492
16425
  toolChoice: "none",
@@ -15737,6 +16670,7 @@ export class AgentSession {
15737
16670
  // (#2797 / #2925).
15738
16671
  if (switchingToDifferentSession) await initializeLocalRoot(this.#localProtocolOptions());
15739
16672
  this.#syncAgentSessionId();
16673
+ this.#resyncRecoveryStateFromTranscript();
15740
16674
  this.#rekeyHindsightMemoryForCurrentSessionId();
15741
16675
 
15742
16676
  const sessionContext = this.buildDisplaySessionContext();
@@ -15847,6 +16781,7 @@ export class AgentSession {
15847
16781
  this.sessionManager.restoreState(previousSessionState);
15848
16782
  this.#defaultFallbackController = undefined;
15849
16783
  this.#syncAgentSessionId(previousSessionState.sessionId);
16784
+ this.#resyncRecoveryStateFromTranscript();
15850
16785
  this.#restoreWorkflowGateEmitter(suspendedWorkflowGateEmitter);
15851
16786
  this.#rekeyHindsightMemoryForCurrentSessionId();
15852
16787
  let restoreMcpError: unknown;
@@ -15964,6 +16899,7 @@ export class AgentSession {
15964
16899
 
15965
16900
  this.#syncTodoPhasesFromBranch();
15966
16901
  this.#syncAgentSessionId();
16902
+ this.#resyncRecoveryStateFromTranscript();
15967
16903
  this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
15968
16904
  this.#rekeyHindsightMemoryForCurrentSessionId();
15969
16905
  this.#resetHindsightConversationTrackingIfHindsight();
@@ -16712,15 +17648,22 @@ export class AgentSession {
16712
17648
  if (!lastAssistant) return undefined;
16713
17649
 
16714
17650
  let text = "";
16715
- for (const content of lastAssistant.content) {
16716
- if (content.type === "text") {
16717
- text += content.text;
17651
+ for (const assistant of this.getVisibleAnswerChain(lastAssistant)) {
17652
+ for (const content of assistant.content) {
17653
+ if (content.type === "text") {
17654
+ text += content.text;
17655
+ }
16718
17656
  }
16719
17657
  }
16720
17658
 
16721
17659
  return text.trim() || undefined;
16722
17660
  }
16723
17661
 
17662
+ /** The assistant messages that together form the visible answer ending at `assistant`. */
17663
+ getVisibleAnswerChain(assistant: AssistantMessage): AssistantMessage[] {
17664
+ return getVisibleAnswerChain(this.messages, assistant);
17665
+ }
17666
+
16724
17667
  hasCopyCandidateAssistantMessage(): boolean {
16725
17668
  return this.#getLastCopyCandidateAssistantMessage() !== undefined;
16726
17669
  }