@sayknow-cli/coding-agent 0.7.6 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/types/config/model-registry.d.ts +3 -4
- package/dist/types/config/settings-schema.d.ts +14 -0
- package/dist/types/extensibility/tool-proxy.d.ts +2 -0
- package/dist/types/session/agent-session.d.ts +7 -3
- package/dist/types/session/auto-resume.d.ts +83 -0
- package/dist/types/session/fallback-chain-controller.d.ts +7 -0
- package/dist/types/session/messages.d.ts +32 -0
- package/package.json +7 -7
- package/scripts/generate-sdk-operation-inventory.ts +1 -0
- package/src/config/model-registry.ts +5 -6
- package/src/config/settings-schema.ts +8 -0
- package/src/extensibility/tool-proxy.ts +11 -0
- package/src/internal-urls/docs-index.generated.ts +1 -1
- package/src/modes/print-mode.ts +13 -3
- package/src/sdk/bus/index.ts +3 -16
- package/src/sdk/protocol/operation-inventory.generated.json +11 -0
- package/src/sdk/session.ts +18 -5
- package/src/session/agent-session.ts +987 -95
- package/src/session/auto-resume.ts +215 -0
- package/src/session/fallback-chain-controller.ts +15 -0
- package/src/session/messages.ts +91 -0
- package/src/task/executor.ts +2 -1
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
resolveTelemetry,
|
|
40
40
|
type StablePrefixSnapshot,
|
|
41
41
|
ThinkingLevel,
|
|
42
|
+
TOOL_CALL_REENTRY_ERROR_CODE,
|
|
42
43
|
} from "@sayknow-cli/agent-core";
|
|
43
44
|
import { normalizeMessagesForProvider } from "@sayknow-cli/agent-core/agent-loop";
|
|
44
45
|
import {
|
|
@@ -99,11 +100,20 @@ import type {
|
|
|
99
100
|
import {
|
|
100
101
|
classifyContextOverflow,
|
|
101
102
|
clearAnthropicFastModeFallback,
|
|
103
|
+
createUpstreamAdmission,
|
|
102
104
|
getSupportedEfforts,
|
|
103
105
|
isContextOverflow,
|
|
106
|
+
isRecoveryAdmissionErrorMessage,
|
|
104
107
|
isUsageLimitError,
|
|
105
108
|
modelsAreEqual,
|
|
109
|
+
type OAuthRefreshOptions,
|
|
110
|
+
RecoveryAdmissionError,
|
|
111
|
+
RecoveryBudget,
|
|
112
|
+
type RecoveryRequestKind,
|
|
106
113
|
resolveServiceTier,
|
|
114
|
+
SAME_MODEL_RECOVERY_BASE_DELAY_MS,
|
|
115
|
+
SAME_MODEL_RECOVERY_MAX_DELAY_MS,
|
|
116
|
+
SAME_MODEL_RECOVERY_MAX_REQUESTS,
|
|
107
117
|
streamSimple,
|
|
108
118
|
} from "@sayknow-cli/ai";
|
|
109
119
|
import {
|
|
@@ -386,6 +396,22 @@ import { buildResponseLanguageReminder, type DetectedLanguage, detectResponseLan
|
|
|
386
396
|
|
|
387
397
|
export { DefaultModelSelectionRecoveryError } from "./default-model-selection";
|
|
388
398
|
|
|
399
|
+
import {
|
|
400
|
+
AUTO_RESUME_FULL_PROMPT,
|
|
401
|
+
AUTO_RESUME_MARKER_CUSTOM_TYPE,
|
|
402
|
+
AUTO_RESUME_RESTRICTED_PROMPT,
|
|
403
|
+
type AutoResumeMarker,
|
|
404
|
+
type AutoResumeMode,
|
|
405
|
+
type AutoResumeTrigger,
|
|
406
|
+
hasAutoResumeMarkerSinceLastUser,
|
|
407
|
+
hasVisibleFailedPrefix,
|
|
408
|
+
INTERRUPTED_TOOL_RESULT_TEXT,
|
|
409
|
+
isRestartSafeTool,
|
|
410
|
+
isSynthesizedResumeResult,
|
|
411
|
+
RESTART_RESUME_MAX_REQUESTS,
|
|
412
|
+
RESTART_RESUME_WINDOW_MS,
|
|
413
|
+
resolveAutoResumePolicy,
|
|
414
|
+
} from "./auto-resume";
|
|
389
415
|
import type {
|
|
390
416
|
ClientBridge,
|
|
391
417
|
ClientBridgePermissionOption,
|
|
@@ -401,16 +427,19 @@ import {
|
|
|
401
427
|
import { pruneStaleFileMentions } from "./file-mention-pruning";
|
|
402
428
|
import type { MemoryGuardRestoreResult } from "./memory-guard-checkpoint-participant";
|
|
403
429
|
import {
|
|
430
|
+
AUTO_RESUME_CUSTOM_TYPE,
|
|
404
431
|
type BashExecutionMessage,
|
|
405
432
|
type CompactionSummaryMessage,
|
|
406
433
|
type CustomMessage,
|
|
407
434
|
convertToLlm,
|
|
408
435
|
createPreAdmissionArtifactSpillPreview,
|
|
409
436
|
type FileMentionMessage,
|
|
437
|
+
getVisibleAnswerChain,
|
|
410
438
|
type PythonExecutionMessage,
|
|
411
439
|
readPendingDisplayTag,
|
|
412
440
|
SILENT_ABORT_MARKER,
|
|
413
441
|
SKILL_PROMPT_MESSAGE_TYPE,
|
|
442
|
+
VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
414
443
|
} from "./messages";
|
|
415
444
|
import { isLegacyProviderSafetyStopMessage } from "./provider-safety-stop";
|
|
416
445
|
import { formatSessionDumpText } from "./session-dump-format";
|
|
@@ -920,6 +949,20 @@ function assistantMessageHasVisibleOrToolContent(message: AssistantMessage): boo
|
|
|
920
949
|
});
|
|
921
950
|
}
|
|
922
951
|
|
|
952
|
+
/** Serializers with a verified projection of a failed plain-text assistant prefix. */
|
|
953
|
+
const VISIBLE_CONTINUATION_APIS: ReadonlySet<string> = new Set(["openai-completions", "anthropic-messages"]);
|
|
954
|
+
const RECOVERY_CHECKPOINT_CUSTOM_TYPE = "recovery_checkpoint";
|
|
955
|
+
const VISIBLE_CONTINUATION_PROMPT =
|
|
956
|
+
"The connection was interrupted while you were answering. Your answer above is preserved exactly as the user saw it. Continue directly from where it stops: do not repeat any of it and do not redo completed work.";
|
|
957
|
+
|
|
958
|
+
/** Only plain text is a proved replay projection; signed/private blocks are never repaired. */
|
|
959
|
+
function isTextVisibleTail(message: AssistantMessage): boolean {
|
|
960
|
+
return (
|
|
961
|
+
message.content.every(block => block.type === "text") &&
|
|
962
|
+
message.content.some(block => block.type === "text" && block.text.trim().length > 0)
|
|
963
|
+
);
|
|
964
|
+
}
|
|
965
|
+
|
|
923
966
|
function isLocalModelEndpoint(model: Model | undefined): boolean {
|
|
924
967
|
if (!model) return false;
|
|
925
968
|
if (model.provider === "ollama" || model.provider === "lm-studio" || model.provider === "llama.cpp") {
|
|
@@ -1762,10 +1805,19 @@ export class AgentSession {
|
|
|
1762
1805
|
|
|
1763
1806
|
/** Replay safety for the currently admitted top-level prompt/custom-message run. */
|
|
1764
1807
|
#retryReplayEpoch = 0;
|
|
1808
|
+
/**
|
|
1809
|
+
* Epoch tainted by something outside the model transcript (extension hooks,
|
|
1810
|
+
* commands, user bash/python, abort). Visible model output and completed tool
|
|
1811
|
+
* pairs stay in the transcript a recovery keeps and are never re-executed, so
|
|
1812
|
+
* they do not taint the run; recovery stays same-model and transcript-only.
|
|
1813
|
+
*/
|
|
1765
1814
|
#retryReplayUnsafeEpoch: number | undefined;
|
|
1766
1815
|
|
|
1767
1816
|
#resetRetryReplaySafety(): void {
|
|
1768
1817
|
this.#retryReplayEpoch++;
|
|
1818
|
+
this.#uncertainUpstreamUnresolved = false;
|
|
1819
|
+
// A genuine new prompt supersedes an interrupted persisted step.
|
|
1820
|
+
this.#unresolvedRecoveryAtHydration = false;
|
|
1769
1821
|
this.#retryReplayUnsafeEpoch = undefined;
|
|
1770
1822
|
if (
|
|
1771
1823
|
this.#extensionRunner?.hasHandlers("context") ||
|
|
@@ -1780,7 +1832,7 @@ export class AgentSession {
|
|
|
1780
1832
|
if (this.#retryReplayEpoch > 0) this.#retryReplayUnsafeEpoch = this.#retryReplayEpoch;
|
|
1781
1833
|
}
|
|
1782
1834
|
|
|
1783
|
-
get #
|
|
1835
|
+
get #hasCleanContinuationSafety(): boolean {
|
|
1784
1836
|
return this.#retryReplayEpoch > 0 && this.#retryReplayUnsafeEpoch !== this.#retryReplayEpoch;
|
|
1785
1837
|
}
|
|
1786
1838
|
#prePromptContextCheckPromise: Promise<void> | undefined = undefined;
|
|
@@ -1808,6 +1860,34 @@ export class AgentSession {
|
|
|
1808
1860
|
#overflowMaintenanceAttempts = 0;
|
|
1809
1861
|
#defaultFallbackExhaustedLastTurn = false;
|
|
1810
1862
|
#transientStopTimestamp: number | undefined;
|
|
1863
|
+
/**
|
|
1864
|
+
* Shared same-model recovery envelope for the unfinished model step: every
|
|
1865
|
+
* concrete upstream request (initial, hidden resend, credential/token, session
|
|
1866
|
+
* retry) is admitted here. Created at the step's first request, cleared when the
|
|
1867
|
+
* step is accepted or a new user turn starts; never refunded.
|
|
1868
|
+
*/
|
|
1869
|
+
#stepRecoveryBudget: RecoveryBudget | undefined;
|
|
1870
|
+
#visibleRecoveryReferenceIds: string[] = [];
|
|
1871
|
+
#visibleRecoveryContextIds: string[] = [];
|
|
1872
|
+
#stepRecoveryUncertainReasons: string[] = [];
|
|
1873
|
+
/**
|
|
1874
|
+
* Set when a step dispatched unobservable remote work. Ending the budget closes
|
|
1875
|
+
* admission but never launders that fact: manual retry stays refused until a new
|
|
1876
|
+
* user prompt starts a fresh replay epoch.
|
|
1877
|
+
*/
|
|
1878
|
+
#uncertainUpstreamUnresolved = false;
|
|
1879
|
+
/** Durable record of in-step uncertainty; admission and resume decisions await it. */
|
|
1880
|
+
#pendingUncertainFlush: Promise<void> = Promise.resolve();
|
|
1881
|
+
#uncertainFlushFailed = false;
|
|
1882
|
+
/** The current run is an automatic resume restricted to restart-safe tools. */
|
|
1883
|
+
#activeAutoResume: { trigger: AutoResumeTrigger; mode: AutoResumeMode } | undefined;
|
|
1884
|
+
/** A restart resume owns the current envelope until its first step is accepted. */
|
|
1885
|
+
#restartResumeEnvelopePending = false;
|
|
1886
|
+
#restrictedToolMode = false;
|
|
1887
|
+
#stepRecoveryStepId: string | undefined;
|
|
1888
|
+
#stepRecoveryDeadlineAbort: AbortController | undefined;
|
|
1889
|
+
#cancelStepRecoveryDeadline: (() => void) | undefined;
|
|
1890
|
+
#unresolvedRecoveryAtHydration = false;
|
|
1811
1891
|
/** Language of the latest genuine user prompt (or inherited from the parent agent). */
|
|
1812
1892
|
#responseLanguage: DetectedLanguage | undefined;
|
|
1813
1893
|
#fallbackInvocationId = 0;
|
|
@@ -2588,6 +2668,8 @@ export class AgentSession {
|
|
|
2588
2668
|
// Per-tool TTSR reminders are folded into the matched tool's result via this hook.
|
|
2589
2669
|
this.agent.afterToolCall = ctx => this.#ttsrAfterToolCall(ctx);
|
|
2590
2670
|
this.agent.providerSessionState = this.#providerSessionState;
|
|
2671
|
+
this.agent.getUpstreamRequestLease = this.#getUpstreamRequestLease;
|
|
2672
|
+
this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
|
|
2591
2673
|
this.#syncAgentSessionId();
|
|
2592
2674
|
this.#removeEphemeralCustomMessages();
|
|
2593
2675
|
|
|
@@ -3458,38 +3540,9 @@ export class AgentSession {
|
|
|
3458
3540
|
|
|
3459
3541
|
/** Internal handler for agent events - shared by subscribe and reconnect */
|
|
3460
3542
|
#handleAgentEvent = async (event: AgentEvent): Promise<void> => {
|
|
3543
|
+
// Model output and completed tool pairs stay in the transcript a recovery
|
|
3544
|
+
// keeps; only an observing extension handler can make the run unsafe.
|
|
3461
3545
|
if (this.#extensionRunner?.hasHandlers(event.type)) this.#markRetryReplayUnsafe();
|
|
3462
|
-
if (
|
|
3463
|
-
event.type === "tool_execution_start" ||
|
|
3464
|
-
event.type === "tool_execution_update" ||
|
|
3465
|
-
event.type === "tool_execution_end"
|
|
3466
|
-
) {
|
|
3467
|
-
this.#markRetryReplayUnsafe();
|
|
3468
|
-
} else if (event.type === "message_end") {
|
|
3469
|
-
if (
|
|
3470
|
-
event.message.role === "toolResult" ||
|
|
3471
|
-
(event.message.role === "assistant" && assistantMessageHasVisibleOrToolContent(event.message))
|
|
3472
|
-
) {
|
|
3473
|
-
this.#markRetryReplayUnsafe();
|
|
3474
|
-
}
|
|
3475
|
-
} else if (event.type === "message_update") {
|
|
3476
|
-
const update = event.assistantMessageEvent;
|
|
3477
|
-
if (
|
|
3478
|
-
update.type === "toolcall_start" ||
|
|
3479
|
-
update.type === "toolcall_delta" ||
|
|
3480
|
-
update.type === "toolcall_end" ||
|
|
3481
|
-
((update.type === "text_delta" ||
|
|
3482
|
-
update.type === "thinking_delta" ||
|
|
3483
|
-
update.type === "reasoning_summary_delta") &&
|
|
3484
|
-
update.delta.length > 0) ||
|
|
3485
|
-
((update.type === "text_end" ||
|
|
3486
|
-
update.type === "thinking_end" ||
|
|
3487
|
-
update.type === "reasoning_summary_end") &&
|
|
3488
|
-
update.content.length > 0)
|
|
3489
|
-
) {
|
|
3490
|
-
this.#markRetryReplayUnsafe();
|
|
3491
|
-
}
|
|
3492
|
-
}
|
|
3493
3546
|
// Record a successful final yield before any asynchronous extension work so a
|
|
3494
3547
|
// concurrently delivered agent_end cannot start post-turn maintenance first.
|
|
3495
3548
|
if (event.type === "tool_execution_end" && event.toolName === "yield" && !event.isError) {
|
|
@@ -3531,6 +3584,11 @@ export class AgentSession {
|
|
|
3531
3584
|
this.#displayDequeueAlreadyHandled.text === userMessageText,
|
|
3532
3585
|
);
|
|
3533
3586
|
if (userDisplayDequeueAlreadyHandled) this.#displayDequeueAlreadyHandled = undefined;
|
|
3587
|
+
// A user message reaching the model ends any restricted automatic resume: from here
|
|
3588
|
+
// on the user directs the turn. Queuing alone does not lift the restriction.
|
|
3589
|
+
if (event.type === "message_start" && event.message.role === "user" && this.#restrictedToolMode) {
|
|
3590
|
+
this.#endAutoResume("skipped");
|
|
3591
|
+
}
|
|
3534
3592
|
if (event.type === "message_start" && event.message.role === "user" && !userDisplayDequeueAlreadyHandled) {
|
|
3535
3593
|
const messageText = userMessageText;
|
|
3536
3594
|
if (messageText) {
|
|
@@ -3871,17 +3929,32 @@ export class AgentSession {
|
|
|
3871
3929
|
this.#skipPostTurnMaintenanceAssistantTimestamp = assistantMsg.timestamp;
|
|
3872
3930
|
this.#suppressNextPostTurnMaintenanceAfterAutoHandoff = false;
|
|
3873
3931
|
}
|
|
3932
|
+
// An accepted (non-error) model step closes its recovery envelope; the next
|
|
3933
|
+
// model step (e.g. after tool results) starts with a fresh budget.
|
|
3934
|
+
if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") {
|
|
3935
|
+
// A tool-call step stays in_flight on purpose: until the next step's own
|
|
3936
|
+
// checkpoint supersedes it, a crash during tool execution has unknown effects
|
|
3937
|
+
// and must restart fail closed. The final text step writes the completed marker.
|
|
3938
|
+
if (!assistantMsg.content.some(block => block.type === "toolCall")) {
|
|
3939
|
+
// The completed marker is appended synchronously. A flush lost to a crash
|
|
3940
|
+
// leaves the step unresolved, which restarts fail closed (no auto-resend).
|
|
3941
|
+
void this.#completeStepRecoveryCheckpoint(assistantMsg);
|
|
3942
|
+
}
|
|
3943
|
+
this.#endStepRecoveryBudget({ accepted: true });
|
|
3944
|
+
this.#restartResumeEnvelopePending = false;
|
|
3945
|
+
}
|
|
3874
3946
|
if (
|
|
3875
3947
|
assistantMsg.stopReason !== "error" &&
|
|
3876
3948
|
assistantMsg.stopReason !== "aborted" &&
|
|
3877
3949
|
this.#retryAttempt > 0
|
|
3878
3950
|
) {
|
|
3951
|
+
const recoveredAttempt = this.#retryAttempt;
|
|
3952
|
+
this.#retryAttempt = 0;
|
|
3879
3953
|
await this.#emitSessionEvent({
|
|
3880
3954
|
type: "auto_retry_end",
|
|
3881
3955
|
success: true,
|
|
3882
|
-
attempt:
|
|
3956
|
+
attempt: recoveredAttempt,
|
|
3883
3957
|
});
|
|
3884
|
-
this.#retryAttempt = 0;
|
|
3885
3958
|
// Settle the retry gate here, colocated with the success event, rather
|
|
3886
3959
|
// than relying on the generic #resolveRetry() at the end of the
|
|
3887
3960
|
// agent_end branch. That tail resolver is bypassed by every early
|
|
@@ -3994,6 +4067,7 @@ export class AgentSession {
|
|
|
3994
4067
|
const msg = this.#lastAssistantMessage ?? fallbackAssistant;
|
|
3995
4068
|
this.#lastAssistantMessage = undefined;
|
|
3996
4069
|
if (!msg) {
|
|
4070
|
+
if (this.#activeAutoResume) this.#endAutoResume("completed");
|
|
3997
4071
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
3998
4072
|
this.#resolveRetry();
|
|
3999
4073
|
return;
|
|
@@ -4010,12 +4084,14 @@ export class AgentSession {
|
|
|
4010
4084
|
}
|
|
4011
4085
|
|
|
4012
4086
|
if (this.#skipPostTurnMaintenanceAssistantTimestamp === msg.timestamp) {
|
|
4087
|
+
if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
|
|
4013
4088
|
this.#skipPostTurnMaintenanceAssistantTimestamp = undefined;
|
|
4014
4089
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
4015
4090
|
return;
|
|
4016
4091
|
}
|
|
4017
4092
|
|
|
4018
4093
|
if (this.#assistantEndedWithSuccessfulYield(msg)) {
|
|
4094
|
+
if (this.#activeAutoResume) this.#endAutoResume("completed");
|
|
4019
4095
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
4020
4096
|
if (msg.stopReason !== "error" && msg.stopReason !== "aborted" && (await this.#checkGoalCompletion(msg))) {
|
|
4021
4097
|
return;
|
|
@@ -4031,9 +4107,17 @@ export class AgentSession {
|
|
|
4031
4107
|
const didRetry = await this.#handleRetryableError(msg, false, transportFailure);
|
|
4032
4108
|
if (didRetry) return; // Retry was initiated, don't proceed to compaction
|
|
4033
4109
|
}
|
|
4110
|
+
// The resumed run settled: its restricted tool view ends with it.
|
|
4111
|
+
if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
|
|
4112
|
+
// An interrupted step may resume once, on the same model, inside its budget.
|
|
4113
|
+
const resumeTrigger = this.#autoResumeTriggerFor(msg);
|
|
4114
|
+
const resumed = resumeTrigger !== undefined && (await this.#maybeAutoResume(resumeTrigger));
|
|
4115
|
+
// No automatic retry follows: the step's recovery envelope ends here, unless the
|
|
4116
|
+
// resume carries on inside it.
|
|
4117
|
+
if (!resumed) this.#endStepRecoveryBudget();
|
|
4034
4118
|
if (this.#retryAttempt > 0) {
|
|
4035
|
-
//
|
|
4036
|
-
// the terminal retry-end and reset so observers clear retry state.
|
|
4119
|
+
// The retry chain ended on a non-retryable (terminal) message, or handed over
|
|
4120
|
+
// to the resume: emit the terminal retry-end and reset so observers clear retry state.
|
|
4037
4121
|
const attempt = this.#retryAttempt;
|
|
4038
4122
|
this.#retryAttempt = 0;
|
|
4039
4123
|
await this.#emitSessionEvent({
|
|
@@ -4044,6 +4128,7 @@ export class AgentSession {
|
|
|
4044
4128
|
});
|
|
4045
4129
|
}
|
|
4046
4130
|
this.#resolveRetry();
|
|
4131
|
+
if (resumed) return;
|
|
4047
4132
|
|
|
4048
4133
|
const compactionTask = this.#checkCompaction(msg);
|
|
4049
4134
|
this.#trackPostPromptTask(compactionTask.then(() => undefined));
|
|
@@ -4161,6 +4246,8 @@ export class AgentSession {
|
|
|
4161
4246
|
onSkip?: (reason: "generation_changed" | "aborted_signal" | "queue_drained" | "handoff_in_progress") => void;
|
|
4162
4247
|
allowDuringCancelAndSubmit?: boolean;
|
|
4163
4248
|
onError?: (error: unknown) => void;
|
|
4249
|
+
/** Run on the current model only: no managed chain, no fallback switch. */
|
|
4250
|
+
pinnedModel?: boolean;
|
|
4164
4251
|
}): Promise<void> {
|
|
4165
4252
|
const predecessorAgentEndHold = options?.suppressPredecessorAgentEnd
|
|
4166
4253
|
? this.#reserveDeferredAgentEndForContinuation()
|
|
@@ -4234,7 +4321,7 @@ export class AgentSession {
|
|
|
4234
4321
|
this.agent.state.messages.at(-1)?.role === "assistant" && this.agent.hasQueuedMessages();
|
|
4235
4322
|
try {
|
|
4236
4323
|
await this.agent.continue({
|
|
4237
|
-
...this.#managedFallbackPromptOptions(),
|
|
4324
|
+
...(options?.pinnedModel ? {} : this.#managedFallbackPromptOptions()),
|
|
4238
4325
|
// Reset only after continue() has claimed the queued turn. Skipped or stale
|
|
4239
4326
|
// continuations retain predecessor accounting, and resetAttemptBudget keeps
|
|
4240
4327
|
// the sticky fallback cursor unchanged.
|
|
@@ -5317,6 +5404,16 @@ export class AgentSession {
|
|
|
5317
5404
|
);
|
|
5318
5405
|
}
|
|
5319
5406
|
|
|
5407
|
+
/**
|
|
5408
|
+
* Recovery safety follows the transcript, not the process: a session switch,
|
|
5409
|
+
* branch, fork, or handoff re-derives it from the persisted checkpoints and drops
|
|
5410
|
+
* in-memory uncertainty that belonged to the previous transcript.
|
|
5411
|
+
*/
|
|
5412
|
+
#resyncRecoveryStateFromTranscript(): void {
|
|
5413
|
+
this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
|
|
5414
|
+
this.#uncertainUpstreamUnresolved = false;
|
|
5415
|
+
}
|
|
5416
|
+
|
|
5320
5417
|
#rekeyHindsightMemoryForCurrentSessionId(): void {
|
|
5321
5418
|
if (resolveMemoryBackend(this.settings).id !== "hindsight") return;
|
|
5322
5419
|
const sid = this.agent.sessionId;
|
|
@@ -6714,12 +6811,49 @@ export class AgentSession {
|
|
|
6714
6811
|
}
|
|
6715
6812
|
}
|
|
6716
6813
|
|
|
6814
|
+
#hasUnresolvedPersistedRecovery(): boolean {
|
|
6815
|
+
const branch = this.sessionManager.getBranch();
|
|
6816
|
+
const checkpoint = branch.findLast(
|
|
6817
|
+
entry => entry.type === "custom" && entry.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE,
|
|
6818
|
+
);
|
|
6819
|
+
if (checkpoint?.type !== "custom") return false;
|
|
6820
|
+
if (!checkpoint.data || typeof checkpoint.data !== "object" || Array.isArray(checkpoint.data)) return true;
|
|
6821
|
+
const data = checkpoint.data as Record<string, unknown>;
|
|
6822
|
+
if (data.version !== 1 || data.state !== "completed") return true;
|
|
6823
|
+
if (!Array.isArray(data.visibleMessageIds) || data.visibleMessageIds.length === 0) return true;
|
|
6824
|
+
const branchIds = new Set(branch.map(entry => entry.id));
|
|
6825
|
+
return !data.visibleMessageIds.every(
|
|
6826
|
+
id => typeof id === "string" && branchIds.has(id) && this.sessionManager.getEntry(id)?.type === "message",
|
|
6827
|
+
);
|
|
6828
|
+
}
|
|
6829
|
+
|
|
6717
6830
|
/** Main startup calls this exactly once, after a strict open returned `kind: "opened"`. */
|
|
6718
6831
|
async continuePersistedHistory(): Promise<void> {
|
|
6719
6832
|
this.#assertNoHandoffTransition();
|
|
6720
6833
|
this.#assertRecoveryHydrationPromoted();
|
|
6721
6834
|
this.#removeEphemeralCustomMessages();
|
|
6722
6835
|
|
|
6836
|
+
// A process that died mid-step left an unresolved recovery checkpoint. Its old
|
|
6837
|
+
// budget, deadline and in-flight request outcome are unknown and never reused:
|
|
6838
|
+
// the step resumes at most once, in a small fresh envelope, without re-running
|
|
6839
|
+
// any call whose outcome was not observed. Otherwise the user resumes it.
|
|
6840
|
+
const tail = this.agent.state.messages.at(-1);
|
|
6841
|
+
let autoResumed = false;
|
|
6842
|
+
let resumeGeneration = this.#promptGeneration;
|
|
6843
|
+
if (
|
|
6844
|
+
this.#hasUnresolvedPersistedRecovery() ||
|
|
6845
|
+
(tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE)
|
|
6846
|
+
) {
|
|
6847
|
+
const staleContinuation = tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE;
|
|
6848
|
+
autoResumed = !staleContinuation && (await this.#maybeAutoResume("restart"));
|
|
6849
|
+
resumeGeneration = this.#promptGeneration;
|
|
6850
|
+
if (!autoResumed) {
|
|
6851
|
+
throw new Error(
|
|
6852
|
+
"The previous answer was interrupted during automatic recovery. Its preserved output is shown above; send a message to continue.",
|
|
6853
|
+
);
|
|
6854
|
+
}
|
|
6855
|
+
}
|
|
6856
|
+
|
|
6723
6857
|
if (!canContinuePersistedHistory(this.agent.state.messages)) {
|
|
6724
6858
|
throw new Error("Cannot continue from persisted message history");
|
|
6725
6859
|
}
|
|
@@ -6757,13 +6891,27 @@ export class AgentSession {
|
|
|
6757
6891
|
// volatile-context/hindsight awaits above and this would otherwise start a
|
|
6758
6892
|
// turn against the session being handed off.
|
|
6759
6893
|
this.#assertNoHandoffTransition();
|
|
6894
|
+
// An abort or new prompt during the preparation above cancels the automatic
|
|
6895
|
+
// resume: it ended the restart envelope, and nothing may be sent outside it.
|
|
6896
|
+
if (autoResumed && (this.#promptGeneration !== resumeGeneration || !this.#stepRecoveryBudget)) {
|
|
6897
|
+
this.#endAutoResume("skipped");
|
|
6898
|
+
return;
|
|
6899
|
+
}
|
|
6760
6900
|
await this.agent.continue({
|
|
6761
|
-
...this.#managedFallbackPromptOptions(),
|
|
6901
|
+
...(autoResumed ? {} : this.#managedFallbackPromptOptions()),
|
|
6762
6902
|
onRunAccepted: () => {
|
|
6763
6903
|
if (hindsightRecall) hindsightState?.markRecallSnippetInjected(hindsightRecall);
|
|
6764
6904
|
},
|
|
6765
6905
|
});
|
|
6766
6906
|
await this.#waitForPostPromptRecovery();
|
|
6907
|
+
} catch (error) {
|
|
6908
|
+
// A restart resume that failed before its run started releases its envelope
|
|
6909
|
+
// and restricted view; a later manual retry or prompt must not inherit them.
|
|
6910
|
+
if (autoResumed && this.#restartResumeEnvelopePending) {
|
|
6911
|
+
this.#endAutoResume("failed");
|
|
6912
|
+
this.#endStepRecoveryBudget();
|
|
6913
|
+
}
|
|
6914
|
+
throw error;
|
|
6767
6915
|
} finally {
|
|
6768
6916
|
this.#removeEphemeralCustomMessages();
|
|
6769
6917
|
this.#endInFlight();
|
|
@@ -8073,6 +8221,17 @@ export class AgentSession {
|
|
|
8073
8221
|
let hindsightRecall: string | undefined;
|
|
8074
8222
|
try {
|
|
8075
8223
|
this.#throwIfPromptPreflightCancelled(generation, preflightSignal);
|
|
8224
|
+
// A new user turn closes any earlier step's envelope (budget, deadline timer,
|
|
8225
|
+
// stepId, uncertainty) before the replay-safety reset clears its residue.
|
|
8226
|
+
if (message.role === "user") {
|
|
8227
|
+
// A real user turn ends any automatic resume before it is processed.
|
|
8228
|
+
if (this.#activeAutoResume) this.#endAutoResume("skipped");
|
|
8229
|
+
this.#endStepRecoveryBudget();
|
|
8230
|
+
// A failed uncertainty write closed admission for the previous step only. Wait
|
|
8231
|
+
// for that step's pending write so its outcome cannot leak into this turn.
|
|
8232
|
+
await this.#pendingUncertainFlush;
|
|
8233
|
+
this.#uncertainFlushFailed = false;
|
|
8234
|
+
}
|
|
8076
8235
|
if (options?.resetRetryReplaySafety) this.#resetRetryReplaySafety();
|
|
8077
8236
|
if (message.role === "user") {
|
|
8078
8237
|
await this.#resetDefaultFallbackForNewTurn();
|
|
@@ -9256,6 +9415,8 @@ export class AgentSession {
|
|
|
9256
9415
|
this.#silentAbortPending = options?.silent === true;
|
|
9257
9416
|
this.#markRetryReplayUnsafe();
|
|
9258
9417
|
this.abortRetry();
|
|
9418
|
+
// Cancellation admits no further upstream request for the interrupted step.
|
|
9419
|
+
this.#endStepRecoveryBudget();
|
|
9259
9420
|
this.#promptGeneration++;
|
|
9260
9421
|
this.#promptPreflightAbortController.abort();
|
|
9261
9422
|
this.#promptPreflightAbortController = new AbortController();
|
|
@@ -9614,6 +9775,7 @@ export class AgentSession {
|
|
|
9614
9775
|
}
|
|
9615
9776
|
this.setTodoPhases([]);
|
|
9616
9777
|
this.#syncAgentSessionId();
|
|
9778
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9617
9779
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9618
9780
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9619
9781
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -9687,6 +9849,7 @@ export class AgentSession {
|
|
|
9687
9849
|
this.agent.reset();
|
|
9688
9850
|
this.setTodoPhases([]);
|
|
9689
9851
|
this.#syncAgentSessionId();
|
|
9852
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9690
9853
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9691
9854
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9692
9855
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -9779,6 +9942,7 @@ export class AgentSession {
|
|
|
9779
9942
|
this.sessionManager.appendContextClearEntry({ sessionId });
|
|
9780
9943
|
this.setTodoPhases([]);
|
|
9781
9944
|
this.#syncAgentSessionId(sessionId);
|
|
9945
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9782
9946
|
this.#steeringMessages = [];
|
|
9783
9947
|
this.#followUpMessages = [];
|
|
9784
9948
|
this.#pendingNextTurnMessages = [];
|
|
@@ -9848,6 +10012,7 @@ export class AgentSession {
|
|
|
9848
10012
|
throw await discardPreparedNewSessionAfterFailure(this.sessionManager, prepared, error);
|
|
9849
10013
|
}
|
|
9850
10014
|
this.#syncAgentSessionId();
|
|
10015
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9851
10016
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9852
10017
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9853
10018
|
|
|
@@ -11515,6 +11680,7 @@ export class AgentSession {
|
|
|
11515
11680
|
committed = true;
|
|
11516
11681
|
this.agent.reset();
|
|
11517
11682
|
this.#syncAgentSessionId();
|
|
11683
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
11518
11684
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
11519
11685
|
this.#steeringMessages = [];
|
|
11520
11686
|
this.#followUpMessages = [];
|
|
@@ -11585,6 +11751,7 @@ export class AgentSession {
|
|
|
11585
11751
|
// were never mutated before commit, so they survive intact.
|
|
11586
11752
|
this.sessionManager.restoreState(rollbackSessionState);
|
|
11587
11753
|
this.#syncAgentSessionId(rollbackSessionState.sessionId);
|
|
11754
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
11588
11755
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
11589
11756
|
this.agent.replaceMessages(rollbackAgentMessages);
|
|
11590
11757
|
this.agent.clearAllQueues();
|
|
@@ -13702,6 +13869,7 @@ export class AgentSession {
|
|
|
13702
13869
|
}
|
|
13703
13870
|
|
|
13704
13871
|
#isRetryableError(message: AssistantMessage): boolean {
|
|
13872
|
+
if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return false;
|
|
13705
13873
|
if (this.#isTerminalProviderFirstEventTimeout(message)) return false;
|
|
13706
13874
|
if (message.errorMessage?.startsWith("Model fallback chain exhausted;")) return false;
|
|
13707
13875
|
// Already gave up on this transient failure; do not start a fresh budget.
|
|
@@ -13774,14 +13942,15 @@ export class AgentSession {
|
|
|
13774
13942
|
/**
|
|
13775
13943
|
* Whether a first-event timeout on the error's provider should fail closed —
|
|
13776
13944
|
* i.e. retry a bounded number of times (capped at retry.maxRetries) and then
|
|
13777
|
-
* surface, instead of joining the
|
|
13945
|
+
* surface, instead of joining the transient class (bounded only by the shared
|
|
13946
|
+
* step budget of 7 requests / 15 minutes).
|
|
13778
13947
|
*
|
|
13779
13948
|
* Targets the ollama-chat API, which is exclusively ollama-cloud (local
|
|
13780
13949
|
* Ollama uses the openai-responses API). That remote, queued backend can
|
|
13781
|
-
* stall before its first token even for tiny prompts;
|
|
13782
|
-
*
|
|
13783
|
-
*
|
|
13784
|
-
*
|
|
13950
|
+
* stall before its first token even for tiny prompts; each continuation retry
|
|
13951
|
+
* re-issues the full request and can spike upstream usage (#713). First-party
|
|
13952
|
+
* providers keep first-event timeouts in the transient class under the shared
|
|
13953
|
+
* step budget.
|
|
13785
13954
|
*/
|
|
13786
13955
|
#shouldFailClosedOnFirstEventTimeout(message: AssistantMessage): boolean {
|
|
13787
13956
|
// Prefer the active model's API (the model that produced the error);
|
|
@@ -13812,11 +13981,12 @@ export class AgentSession {
|
|
|
13812
13981
|
/**
|
|
13813
13982
|
* Ordered retry classification: typed safety stop (surface) -> legacy safety stop
|
|
13814
13983
|
* (surface) -> overflow (compaction) -> terminal (surface) -> usage_limit
|
|
13815
|
-
* (rotation) -> first_event_timeout (
|
|
13984
|
+
* (rotation) -> first_event_timeout (retry.maxRetries) -> transient (shared step budget) ->
|
|
13816
13985
|
* unknown (bounded retry).
|
|
13817
13986
|
*/
|
|
13818
13987
|
#classifyErrorForRetry(message: AssistantMessage): RetryErrorClassification {
|
|
13819
13988
|
if (message.stopReason !== "error") return "none";
|
|
13989
|
+
if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return "terminal";
|
|
13820
13990
|
if (message.errorKind === "provider_safety_stop") return "terminal";
|
|
13821
13991
|
// A decode loop is deterministic for the submitted context: replaying the
|
|
13822
13992
|
// identical conversation re-trips the guard and re-bills the full context.
|
|
@@ -13863,10 +14033,10 @@ export class AgentSession {
|
|
|
13863
14033
|
if (this.#isTerminalProviderFirstEventTimeout(message)) {
|
|
13864
14034
|
return "terminal";
|
|
13865
14035
|
}
|
|
13866
|
-
// A first-event timeout on ollama-cloud (the ollama-chat API) must not
|
|
13867
|
-
//
|
|
13868
|
-
//
|
|
13869
|
-
//
|
|
14036
|
+
// A first-event timeout on ollama-cloud (the ollama-chat API) must not join
|
|
14037
|
+
// the transient class: each continuation retry re-issues the full request to
|
|
14038
|
+
// a remote, billable backend (#713). Bound it to retry.maxRetries instead of
|
|
14039
|
+
// the wider shared step budget.
|
|
13870
14040
|
if (this.#isFirstEventTimeoutErrorMessage(err) && this.#shouldFailClosedOnFirstEventTimeout(message)) {
|
|
13871
14041
|
return "first_event_timeout";
|
|
13872
14042
|
}
|
|
@@ -14118,8 +14288,15 @@ export class AgentSession {
|
|
|
14118
14288
|
}
|
|
14119
14289
|
|
|
14120
14290
|
async #handleManagedAttemptOutcome(outcome: ManagedAttemptOutcome): Promise<ManagedAttemptDecision> {
|
|
14291
|
+
if (outcome.type === "visible_interrupted" || outcome.type === "unsafe_interrupted") {
|
|
14292
|
+
// The session does not supply managed recovery evidence, so a committed managed
|
|
14293
|
+
// failure is never replayed: preserve it and stop for an explicit user resume.
|
|
14294
|
+
this.#endStepRecoveryBudget();
|
|
14295
|
+
return { type: "pause", reason: outcome.evidence.tail };
|
|
14296
|
+
}
|
|
14121
14297
|
if (outcome.type === "run_terminal") {
|
|
14122
14298
|
this.#defaultFallbackChain().resetAttemptBudget();
|
|
14299
|
+
this.#endStepRecoveryBudget();
|
|
14123
14300
|
return { type: "terminal", terminal: { stopReason: outcome.reason } };
|
|
14124
14301
|
}
|
|
14125
14302
|
if (outcome.type === "context_overflow_discarded") {
|
|
@@ -14205,7 +14382,7 @@ export class AgentSession {
|
|
|
14205
14382
|
if (transport.class !== "other") return transport;
|
|
14206
14383
|
// Managed fallback receives authoritative transport facts from the request
|
|
14207
14384
|
// boundary. Once those facts classify as other, error prose must not upgrade
|
|
14208
|
-
// the failure into
|
|
14385
|
+
// the failure into a transient or quota retry.
|
|
14209
14386
|
if (transportFailure) return { class: "unknown" };
|
|
14210
14387
|
const classification = this.#classifyErrorForRetry(message);
|
|
14211
14388
|
if (allowLegacyUsageLimit && classification === "usage_limit") {
|
|
@@ -14309,22 +14486,72 @@ export class AgentSession {
|
|
|
14309
14486
|
async #markFailedManagedCredential(trigger: {
|
|
14310
14487
|
class: FallbackTriggerClass;
|
|
14311
14488
|
retryAfterMs?: number;
|
|
14489
|
+
}): Promise<boolean> {
|
|
14490
|
+
try {
|
|
14491
|
+
return await this.#markFailedManagedCredentialAdmitted(trigger);
|
|
14492
|
+
} catch (error) {
|
|
14493
|
+
// A refused admission ends maintenance locally; the next inference admission
|
|
14494
|
+
// reports the same budget verdict instead of this side path throwing.
|
|
14495
|
+
if (error instanceof RecoveryAdmissionError) return false;
|
|
14496
|
+
throw error;
|
|
14497
|
+
}
|
|
14498
|
+
}
|
|
14499
|
+
|
|
14500
|
+
async #markFailedManagedCredentialAdmitted(trigger: {
|
|
14501
|
+
class: FallbackTriggerClass;
|
|
14502
|
+
retryAfterMs?: number;
|
|
14312
14503
|
}): Promise<boolean> {
|
|
14313
14504
|
if (!this.model || (trigger.class !== "auth" && trigger.class !== "quota" && trigger.class !== "rate_limit")) {
|
|
14314
14505
|
return false;
|
|
14315
14506
|
}
|
|
14316
14507
|
const authStorage = this.#modelRegistry.authStorage;
|
|
14508
|
+
const requestOptions = this.#stepCredentialRequestOptions();
|
|
14509
|
+
// Recovery maintenance outside a step budget would send unadmitted credential HTTP.
|
|
14510
|
+
if (!requestOptions.fetch) return false;
|
|
14317
14511
|
if (trigger.class === "auth") {
|
|
14318
|
-
const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
|
|
14512
|
+
const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
|
|
14319
14513
|
if (!isAuthenticated(apiKey)) return false;
|
|
14320
|
-
return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, {
|
|
14514
|
+
return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, {
|
|
14515
|
+
...requestOptions,
|
|
14516
|
+
sessionId: this.sessionId,
|
|
14517
|
+
});
|
|
14321
14518
|
}
|
|
14322
14519
|
if (authStorage.hasRuntimeApiKey(this.model.provider)) return false;
|
|
14323
|
-
const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
|
|
14520
|
+
const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
|
|
14324
14521
|
const rotated = await authStorage.markUsageLimitReached(this.model.provider, this.sessionId, {
|
|
14325
14522
|
retryAfterMs: trigger.retryAfterMs,
|
|
14523
|
+
...requestOptions,
|
|
14326
14524
|
});
|
|
14327
|
-
return
|
|
14525
|
+
return (
|
|
14526
|
+
rotated && (await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions)) !== activeApiKey
|
|
14527
|
+
);
|
|
14528
|
+
}
|
|
14529
|
+
|
|
14530
|
+
/**
|
|
14531
|
+
* Credential maintenance during recovery (rotation, restore, usage probes) spends
|
|
14532
|
+
* the same step budget as inference: every concrete token/usage HTTP is admitted
|
|
14533
|
+
* as kind "token", and opaque remote work marks the step unsafe.
|
|
14534
|
+
*/
|
|
14535
|
+
#stepCredentialRequestOptions(): OAuthRefreshOptions {
|
|
14536
|
+
const budget = this.#stepRecoveryBudget;
|
|
14537
|
+
const deadlineAbort = this.#stepRecoveryDeadlineAbort;
|
|
14538
|
+
if (!budget || !deadlineAbort) return {};
|
|
14539
|
+
const validate = (): void => {
|
|
14540
|
+
if (this.#stepRecoveryBudget !== budget) throw new RecoveryAdmissionError("cancelled");
|
|
14541
|
+
budget.assertActive();
|
|
14542
|
+
};
|
|
14543
|
+
const admission = createUpstreamAdmission(
|
|
14544
|
+
kind => budget.reserve(kind),
|
|
14545
|
+
undefined,
|
|
14546
|
+
validate,
|
|
14547
|
+
() => this.#recordStepRecoveryFailure(budget),
|
|
14548
|
+
);
|
|
14549
|
+
return {
|
|
14550
|
+
signal: deadlineAbort.signal,
|
|
14551
|
+
fetch: admission.credentialFetch,
|
|
14552
|
+
onUpstreamRequest: admission.onUpstreamRequest,
|
|
14553
|
+
onUncertainUpstream: reason => this.#recordStepUncertainUpstream(budget, reason),
|
|
14554
|
+
};
|
|
14328
14555
|
}
|
|
14329
14556
|
|
|
14330
14557
|
/** Handle retryable errors with exponential backoff. */
|
|
@@ -14336,6 +14563,14 @@ export class AgentSession {
|
|
|
14336
14563
|
const controller = this.#defaultFallbackChain();
|
|
14337
14564
|
const managedFallback = controller.chain.entries.length > 1;
|
|
14338
14565
|
const retrySettings = this.settings.getGroup("retry");
|
|
14566
|
+
// A rejected admission is the shared budget's own terminal answer, never a retry trigger.
|
|
14567
|
+
if (isRecoveryAdmissionErrorMessage(message.errorMessage)) {
|
|
14568
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14569
|
+
}
|
|
14570
|
+
// Broker/custom credential work with unobserved remote effects is never resent automatically.
|
|
14571
|
+
if (this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14572
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14573
|
+
}
|
|
14339
14574
|
const legacyRetryConfigured =
|
|
14340
14575
|
this.settings.has("retry.enabled") ||
|
|
14341
14576
|
this.settings.has("retry.maxRetries") ||
|
|
@@ -14345,13 +14580,22 @@ export class AgentSession {
|
|
|
14345
14580
|
// user opt-out.
|
|
14346
14581
|
if (!managedFallback && !retrySettings.enabled) return false;
|
|
14347
14582
|
const classification = managedFallback ? undefined : this.#classifyErrorForRetry(message);
|
|
14348
|
-
//
|
|
14583
|
+
// A failure after public output is never replayed by deleting that output.
|
|
14584
|
+
// A text-only tail on a verified serializer continues on the same model;
|
|
14585
|
+
// any other public tail (tool calls, images, tainted run) stops with the
|
|
14586
|
+
// output preserved for the user to resume explicitly.
|
|
14587
|
+
const visibleFailure = !managedOutcome && assistantMessageHasVisibleOrToolContent(message);
|
|
14588
|
+
const visibleContinuation = visibleFailure && this.#canContinueVisibleFailure(message);
|
|
14589
|
+
if (visibleFailure && !visibleContinuation) return false;
|
|
14590
|
+
// Bare defaults admit only clean canonical stream watchdog failures. Completed
|
|
14591
|
+
// tool pairs and preserved visible output stay in the transcript and are never
|
|
14592
|
+
// re-executed; only hooks/commands/user shell/abort make the run unsafe.
|
|
14349
14593
|
if (!managedFallback && !legacyRetryConfigured) {
|
|
14350
14594
|
if (
|
|
14351
14595
|
hasBareDefaultRetryDisqualifyingFacts(message) ||
|
|
14352
14596
|
(classification !== "transient" && classification !== "first_event_timeout") ||
|
|
14353
14597
|
!BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "") ||
|
|
14354
|
-
!this.#
|
|
14598
|
+
!this.#hasCleanContinuationSafety
|
|
14355
14599
|
) {
|
|
14356
14600
|
return false;
|
|
14357
14601
|
}
|
|
@@ -14362,8 +14606,29 @@ export class AgentSession {
|
|
|
14362
14606
|
? this.#managedFallbackExhaustionDecision(message, message.errorMessage || "Model fallback attempt failed")
|
|
14363
14607
|
: false;
|
|
14364
14608
|
}
|
|
14365
|
-
const
|
|
14366
|
-
|
|
14609
|
+
const transientTrigger = trigger.class === "server" || trigger.class === "rate_limit";
|
|
14610
|
+
// Session retries and per-selector attempts are different units. Respect an
|
|
14611
|
+
// explicit additional-session-retry cap before credential rotation or replay.
|
|
14612
|
+
if (
|
|
14613
|
+
transientTrigger &&
|
|
14614
|
+
(!retrySettings.enabled ||
|
|
14615
|
+
this.#retryAttempt >= (this.settings.has("retry.maxRetries") ? retrySettings.maxRetries : 6))
|
|
14616
|
+
) {
|
|
14617
|
+
this.#transientStopTimestamp = message.timestamp;
|
|
14618
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14619
|
+
}
|
|
14620
|
+
if (visibleContinuation && !transientTrigger) return false;
|
|
14621
|
+
// Default policy: transient same-model failures are bounded by the shared step
|
|
14622
|
+
// budget (7 requests / 15 minutes after the first failure) instead of an
|
|
14623
|
+
// unbounded loop or a per-entry cap. Explicit retry.maxRetries/fallback.maxAttempts
|
|
14624
|
+
// keep their own attempt units and also stay inside the shared budget.
|
|
14625
|
+
const sharedBudgetOwnsTransient = managedFallback
|
|
14626
|
+
? !this.settings.has("fallback.maxAttempts")
|
|
14627
|
+
: classification === "transient" && !this.settings.has("retry.maxRetries");
|
|
14628
|
+
controller.sessionBudgetOwnsTransient = managedFallback && sharedBudgetOwnsTransient;
|
|
14629
|
+
const sharedTransientRetry = sharedBudgetOwnsTransient && transientTrigger;
|
|
14630
|
+
const attemptsUsed =
|
|
14631
|
+
managedFallback && !sharedTransientRetry ? controller.attemptsUsed || 1 : this.#retryAttempt + 1;
|
|
14367
14632
|
const failedSelector = managedFallback ? controller.currentSelector() : undefined;
|
|
14368
14633
|
const usageLimited = trigger.class === "quota" || trigger.class === "rate_limit";
|
|
14369
14634
|
let credentialMarked = false;
|
|
@@ -14374,15 +14639,64 @@ export class AgentSession {
|
|
|
14374
14639
|
// leave the model while another account can still serve it.
|
|
14375
14640
|
credentialMarked = true;
|
|
14376
14641
|
credentialRotated = await this.#markFailedManagedCredential(trigger);
|
|
14377
|
-
|
|
14378
|
-
|
|
14379
|
-
|
|
14380
|
-
|
|
14381
|
-
|
|
14382
|
-
|
|
14642
|
+
if (this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14643
|
+
return managedOutcome
|
|
14644
|
+
? { type: "terminal", terminal: { stopReason: "error", messages: [message] } }
|
|
14645
|
+
: false;
|
|
14646
|
+
}
|
|
14647
|
+
}
|
|
14648
|
+
// A resumed run stays on the interrupted model. On a managed chain only the shared
|
|
14649
|
+
// same-model transient path may follow; any other failure stops before the chain is
|
|
14650
|
+
// consulted, because consulting it may already move its cursor to the next model.
|
|
14651
|
+
const leavesResumedModel =
|
|
14652
|
+
this.#activeAutoResume !== undefined && managedFallback && !(sharedBudgetOwnsTransient && transientTrigger);
|
|
14653
|
+
let outcome = leavesResumedModel
|
|
14654
|
+
? "stop"
|
|
14655
|
+
: managedFallback
|
|
14656
|
+
? controller.onAttemptFailure(trigger.class, message.errorMessage || "Unknown error", { credentialRotated })
|
|
14657
|
+
: sharedBudgetOwnsTransient || attemptsUsed <= retrySettings.maxRetries
|
|
14658
|
+
? "retry"
|
|
14659
|
+
: "exhausted";
|
|
14383
14660
|
if (managedFallback && credentialRotated && outcome === "advance" && controller.restorePreviousEntryForRetry()) {
|
|
14384
14661
|
outcome = "retry";
|
|
14385
14662
|
}
|
|
14663
|
+
const stepBudget = this.#stepRecoveryBudget;
|
|
14664
|
+
if (outcome === "retry" && stepBudget && (transientTrigger || !managedFallback))
|
|
14665
|
+
this.#recordStepRecoveryFailure(stepBudget);
|
|
14666
|
+
// Typed transport Retry-After is an uncapped floor; a floor beyond the remaining
|
|
14667
|
+
// deadline stops locally rather than contacting upstream early. A hint parsed from
|
|
14668
|
+
// legacy error prose is advisory and stays capped at retry.maxDelayMs.
|
|
14669
|
+
const parsedRetryAfterMs = managedFallback
|
|
14670
|
+
? undefined
|
|
14671
|
+
: this.#parseRetryAfterMsFromError(message.errorMessage || "Unknown error");
|
|
14672
|
+
const retryAfterMs =
|
|
14673
|
+
trigger.retryAfterMs ??
|
|
14674
|
+
(parsedRetryAfterMs === undefined ? undefined : Math.min(parsedRetryAfterMs, retrySettings.maxDelayMs));
|
|
14675
|
+
const backoffBaseMs =
|
|
14676
|
+
sharedTransientRetry && !this.settings.has("retry.baseDelayMs")
|
|
14677
|
+
? SAME_MODEL_RECOVERY_BASE_DELAY_MS
|
|
14678
|
+
: retrySettings.baseDelayMs;
|
|
14679
|
+
const backoffMaxMs =
|
|
14680
|
+
sharedTransientRetry && !this.settings.has("retry.maxDelayMs")
|
|
14681
|
+
? SAME_MODEL_RECOVERY_MAX_DELAY_MS
|
|
14682
|
+
: retrySettings.maxDelayMs;
|
|
14683
|
+
const delayMs =
|
|
14684
|
+
credentialRotated || outcome === "advance"
|
|
14685
|
+
? 0
|
|
14686
|
+
: managedFallback || transientTrigger
|
|
14687
|
+
? effectiveFallbackDelay(backoffBaseMs, backoffMaxMs, attemptsUsed, retryAfterMs)
|
|
14688
|
+
: retryAfterMs !== undefined
|
|
14689
|
+
? Math.min(retryAfterMs, retrySettings.maxDelayMs)
|
|
14690
|
+
: cappedExponentialWithFullJitter(backoffBaseMs, backoffMaxMs, attemptsUsed);
|
|
14691
|
+
if (outcome === "retry" && stepBudget && !stepBudget.canAdmitAfter(delayMs)) {
|
|
14692
|
+
// The shared request/time budget cannot admit another same-model request
|
|
14693
|
+
// after the required wait: stop on the same model with the real provider
|
|
14694
|
+
// error instead of sending it. Never switch models for it.
|
|
14695
|
+
outcome = managedFallback && !transientTrigger ? "exhausted" : "stop";
|
|
14696
|
+
}
|
|
14697
|
+
if (outcome === "exhausted" && !managedFallback && sharedBudgetOwnsTransient) outcome = "stop";
|
|
14698
|
+
// retry.enabled=false is an explicit opt-out of automatic same-model transient recovery.
|
|
14699
|
+
if (outcome === "retry" && transientTrigger && !retrySettings.enabled) outcome = "stop";
|
|
14386
14700
|
if (outcome === "stop") {
|
|
14387
14701
|
// Transient failure past the same-model retry budget: surface the real
|
|
14388
14702
|
// provider error and keep the current model; never switch models for it.
|
|
@@ -14413,16 +14727,6 @@ export class AgentSession {
|
|
|
14413
14727
|
|
|
14414
14728
|
const generation = this.#promptGeneration;
|
|
14415
14729
|
const errorMessage = message.errorMessage || "Unknown error";
|
|
14416
|
-
const retryAfterMs =
|
|
14417
|
-
trigger.retryAfterMs ?? (managedFallback ? undefined : this.#parseRetryAfterMsFromError(errorMessage));
|
|
14418
|
-
const delayMs =
|
|
14419
|
-
credentialRotated || outcome === "advance"
|
|
14420
|
-
? 0
|
|
14421
|
-
: managedFallback
|
|
14422
|
-
? effectiveFallbackDelay(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed, retryAfterMs)
|
|
14423
|
-
: retryAfterMs !== undefined
|
|
14424
|
-
? Math.min(retryAfterMs, retrySettings.maxDelayMs)
|
|
14425
|
-
: cappedExponentialWithFullJitter(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed);
|
|
14426
14730
|
|
|
14427
14731
|
// Only a spent quota window hides the model from later resolution; a burst
|
|
14428
14732
|
// rate limit retries the same model and must not push later turns elsewhere.
|
|
@@ -14474,15 +14778,24 @@ export class AgentSession {
|
|
|
14474
14778
|
await this.#emitSessionEvent({
|
|
14475
14779
|
type: "auto_retry_start",
|
|
14476
14780
|
attempt: this.#retryAttempt,
|
|
14477
|
-
maxAttempts:
|
|
14781
|
+
maxAttempts:
|
|
14782
|
+
sharedBudgetOwnsTransient && transientTrigger
|
|
14783
|
+
? SAME_MODEL_RECOVERY_MAX_REQUESTS - 1
|
|
14784
|
+
: managedFallback
|
|
14785
|
+
? controller.maxAttempts
|
|
14786
|
+
: retrySettings.maxRetries,
|
|
14478
14787
|
delayMs,
|
|
14479
14788
|
errorMessage,
|
|
14480
|
-
unbounded:
|
|
14789
|
+
unbounded: false,
|
|
14481
14790
|
});
|
|
14482
14791
|
|
|
14483
|
-
|
|
14484
|
-
|
|
14485
|
-
|
|
14792
|
+
// A preserved public prefix is never deleted or re-emitted; it is continued
|
|
14793
|
+
// after the backoff instead (see below).
|
|
14794
|
+
if (!visibleContinuation) {
|
|
14795
|
+
const messages = this.agent.state.messages;
|
|
14796
|
+
if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
|
|
14797
|
+
this.agent.replaceMessages(messages.slice(0, -1));
|
|
14798
|
+
}
|
|
14486
14799
|
}
|
|
14487
14800
|
|
|
14488
14801
|
try {
|
|
@@ -14522,6 +14835,19 @@ export class AgentSession {
|
|
|
14522
14835
|
if (this.#retryAbortController === retryAbortController) this.#retryAbortController = undefined;
|
|
14523
14836
|
this.#retryNowRequested = false;
|
|
14524
14837
|
|
|
14838
|
+
if (visibleContinuation) {
|
|
14839
|
+
// Owner re-check before any durable write: a newer prompt supersedes it.
|
|
14840
|
+
if (this.#promptGeneration !== generation) {
|
|
14841
|
+
this.#failRetryRecovery("Retry continuation was superseded");
|
|
14842
|
+
return;
|
|
14843
|
+
}
|
|
14844
|
+
if (!(await this.#appendVisibleContinuation(message))) {
|
|
14845
|
+
this.#transientStopTimestamp = message.timestamp;
|
|
14846
|
+
this.#failRetryRecovery("Recovery checkpoint could not be saved");
|
|
14847
|
+
return;
|
|
14848
|
+
}
|
|
14849
|
+
}
|
|
14850
|
+
|
|
14525
14851
|
if (managedOutcome) {
|
|
14526
14852
|
try {
|
|
14527
14853
|
await this.#checkEstimatedContextBeforePrompt();
|
|
@@ -14560,6 +14886,7 @@ export class AgentSession {
|
|
|
14560
14886
|
delayMs: 1,
|
|
14561
14887
|
generation,
|
|
14562
14888
|
allowDuringCancelAndSubmit: true,
|
|
14889
|
+
shouldContinue: () => !visibleContinuation || this.#hasVisibleRecoveryReferences(),
|
|
14563
14890
|
onError: () => this.#failRetryRecovery("Retry continuation failed to start"),
|
|
14564
14891
|
onSkip: () => this.#failRetryRecovery("Retry continuation was superseded"),
|
|
14565
14892
|
});
|
|
@@ -14580,6 +14907,559 @@ export class AgentSession {
|
|
|
14580
14907
|
this.#resolveRetry();
|
|
14581
14908
|
}
|
|
14582
14909
|
|
|
14910
|
+
/**
|
|
14911
|
+
* Single admission point for every concrete upstream request of the current
|
|
14912
|
+
* model step. Rejects (throws {@link RecoveryAdmissionError}) once the shared
|
|
14913
|
+
* same-model budget is spent or past its deadline, before anything is sent.
|
|
14914
|
+
*/
|
|
14915
|
+
#getUpstreamRequestLease: NonNullable<AgentLoopConfig["getUpstreamRequestLease"]> = async (context, signal) => {
|
|
14916
|
+
if (
|
|
14917
|
+
!this.#stepRecoveryBudget &&
|
|
14918
|
+
this.#unresolvedRecoveryAtHydration &&
|
|
14919
|
+
context.messages.at(-1)?.role !== "user"
|
|
14920
|
+
) {
|
|
14921
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14922
|
+
}
|
|
14923
|
+
// A restart resume's first step runs only inside its own small envelope: once that
|
|
14924
|
+
// envelope is gone before the step was accepted (abort, new prompt), no default
|
|
14925
|
+
// budget may take its place.
|
|
14926
|
+
if (!this.#stepRecoveryBudget && this.#restartResumeEnvelopePending) {
|
|
14927
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14928
|
+
}
|
|
14929
|
+
// The durable record of in-step uncertainty must land before anything else is sent.
|
|
14930
|
+
await this.#pendingUncertainFlush;
|
|
14931
|
+
if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
|
|
14932
|
+
// Unobservable remote work already happened in this step: no further upstream request
|
|
14933
|
+
// for it, whichever path (retry, rotation, fallback) asks next.
|
|
14934
|
+
if (this.#stepRecoveryBudget && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14935
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14936
|
+
}
|
|
14937
|
+
const generation = this.#promptGeneration;
|
|
14938
|
+
this.#stepRecoveryBudget ??= new RecoveryBudget({ maxSilentTimeouts: this.#maxSilentTimeouts() });
|
|
14939
|
+
this.#stepRecoveryDeadlineAbort ??= new AbortController();
|
|
14940
|
+
this.#stepRecoveryStepId ??= crypto.randomUUID();
|
|
14941
|
+
const budget = this.#stepRecoveryBudget;
|
|
14942
|
+
const deadlineAbort = this.#stepRecoveryDeadlineAbort;
|
|
14943
|
+
const stepId = this.#stepRecoveryStepId;
|
|
14944
|
+
const model = this.model;
|
|
14945
|
+
const validate = (): void => {
|
|
14946
|
+
if (
|
|
14947
|
+
signal?.aborted ||
|
|
14948
|
+
this.#isDisposed ||
|
|
14949
|
+
this.#promptGeneration !== generation ||
|
|
14950
|
+
this.#stepRecoveryBudget !== budget ||
|
|
14951
|
+
this.model !== model
|
|
14952
|
+
) {
|
|
14953
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14954
|
+
}
|
|
14955
|
+
budget.assertActive();
|
|
14956
|
+
};
|
|
14957
|
+
validate();
|
|
14958
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
14959
|
+
version: 1,
|
|
14960
|
+
state: "in_flight",
|
|
14961
|
+
logicalRunId: String(this.agent.currentManagedLogicalRunId ?? `${this.sessionId}:${generation}`),
|
|
14962
|
+
stepId,
|
|
14963
|
+
generation,
|
|
14964
|
+
attemptId: String(this.#retryAttempt),
|
|
14965
|
+
selector: model ? `${model.provider}/${model.id}` : undefined,
|
|
14966
|
+
api: model?.api,
|
|
14967
|
+
transport: model?.transport,
|
|
14968
|
+
uncertain: [...this.#stepRecoveryUncertainReasons],
|
|
14969
|
+
// Only this user turn's already-visible assistant output belongs to the step.
|
|
14970
|
+
visibleMessageIds: context.messages
|
|
14971
|
+
.slice(
|
|
14972
|
+
Math.max(
|
|
14973
|
+
0,
|
|
14974
|
+
context.messages.findLastIndex(message => message.role === "user"),
|
|
14975
|
+
),
|
|
14976
|
+
)
|
|
14977
|
+
.filter(message => message.role === "assistant")
|
|
14978
|
+
.map(getSessionMessageEntryId)
|
|
14979
|
+
.filter((id): id is string => id !== undefined),
|
|
14980
|
+
usedRequests: budget.snapshot().usedRequests,
|
|
14981
|
+
});
|
|
14982
|
+
await this.sessionManager.ensureOnDisk();
|
|
14983
|
+
await this.sessionManager.flush();
|
|
14984
|
+
validate();
|
|
14985
|
+
return {
|
|
14986
|
+
validate,
|
|
14987
|
+
signal: deadlineAbort.signal,
|
|
14988
|
+
// Records only; the identity check lives in #recordStepRecoveryFailure so the
|
|
14989
|
+
// original response/error is never replaced by an admission error.
|
|
14990
|
+
onRecoverableFailure: () => this.#recordStepRecoveryFailure(budget),
|
|
14991
|
+
onUncertainUpstream: (reason: string): void => this.#recordStepUncertainUpstream(budget, reason),
|
|
14992
|
+
onAttemptSettled: (message: AssistantMessage): void => this.#recordStepAttemptSettled(budget, message),
|
|
14993
|
+
onRequest: (kind: RecoveryRequestKind): void => {
|
|
14994
|
+
validate();
|
|
14995
|
+
if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
|
|
14996
|
+
// Unobservable remote work was reported inside this invocation: a resend could
|
|
14997
|
+
// duplicate it, so only the first concrete request of each kind remains admissible.
|
|
14998
|
+
if (kind === "resend" && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14999
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
15000
|
+
}
|
|
15001
|
+
if (kind === "resend") this.#recordStepRecoveryFailure(budget);
|
|
15002
|
+
budget.reserve(kind);
|
|
15003
|
+
},
|
|
15004
|
+
};
|
|
15005
|
+
};
|
|
15006
|
+
|
|
15007
|
+
async #completeStepRecoveryCheckpoint(message: AssistantMessage): Promise<void> {
|
|
15008
|
+
if (!this.#stepRecoveryStepId || this.agent.state.messages.at(-1) !== message) return;
|
|
15009
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15010
|
+
version: 1,
|
|
15011
|
+
state: "completed",
|
|
15012
|
+
stepId: this.#stepRecoveryStepId,
|
|
15013
|
+
generation: this.#promptGeneration,
|
|
15014
|
+
visibleMessageIds: [getSessionMessageEntryId(message)].filter((id): id is string => id !== undefined),
|
|
15015
|
+
});
|
|
15016
|
+
try {
|
|
15017
|
+
await this.sessionManager.ensureOnDisk();
|
|
15018
|
+
await this.sessionManager.flush();
|
|
15019
|
+
} catch (error) {
|
|
15020
|
+
logger.warn("Recovery completion checkpoint failed", {
|
|
15021
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15022
|
+
});
|
|
15023
|
+
}
|
|
15024
|
+
}
|
|
15025
|
+
|
|
15026
|
+
/**
|
|
15027
|
+
* Unobservable remote work cannot be proven idempotent: no automatic resend/continuation.
|
|
15028
|
+
* The reason is also persisted with the step so a restart never resumes it; the next
|
|
15029
|
+
* admission and every resume decision wait for that write.
|
|
15030
|
+
*/
|
|
15031
|
+
#recordStepUncertainUpstream(budget: RecoveryBudget, reason: string): void {
|
|
15032
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15033
|
+
if (!this.#stepRecoveryUncertainReasons.includes(reason)) this.#stepRecoveryUncertainReasons.push(reason);
|
|
15034
|
+
this.#markRetryReplayUnsafe();
|
|
15035
|
+
const model = this.model;
|
|
15036
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15037
|
+
version: 1,
|
|
15038
|
+
state: "in_flight",
|
|
15039
|
+
stepId: this.#stepRecoveryStepId,
|
|
15040
|
+
generation: this.#promptGeneration,
|
|
15041
|
+
selector: model ? `${model.provider}/${model.id}` : undefined,
|
|
15042
|
+
api: model?.api,
|
|
15043
|
+
transport: model?.transport,
|
|
15044
|
+
uncertain: [...this.#stepRecoveryUncertainReasons],
|
|
15045
|
+
visibleMessageIds: [],
|
|
15046
|
+
usedRequests: budget.snapshot().usedRequests,
|
|
15047
|
+
});
|
|
15048
|
+
this.#pendingUncertainFlush = this.#pendingUncertainFlush
|
|
15049
|
+
.then(async () => {
|
|
15050
|
+
await this.sessionManager.ensureOnDisk();
|
|
15051
|
+
await this.sessionManager.flush();
|
|
15052
|
+
})
|
|
15053
|
+
.catch(error => {
|
|
15054
|
+
this.#uncertainFlushFailed = true;
|
|
15055
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
15056
|
+
logger.warn("Uncertain upstream checkpoint failed", { error: reason });
|
|
15057
|
+
this.emitNotice(
|
|
15058
|
+
"warning",
|
|
15059
|
+
`Stopped retrying this step: its recovery state could not be saved (${reason}).`,
|
|
15060
|
+
"retry",
|
|
15061
|
+
);
|
|
15062
|
+
});
|
|
15063
|
+
}
|
|
15064
|
+
|
|
15065
|
+
#maxSilentTimeouts(): number {
|
|
15066
|
+
return Math.min(7, Math.max(1, Math.trunc(this.settings.get("retry.maxSilentTimeouts")) || 3));
|
|
15067
|
+
}
|
|
15068
|
+
|
|
15069
|
+
/** Consecutive no-output timeouts close the step early, inside its request budget. */
|
|
15070
|
+
#recordStepAttemptSettled(budget: RecoveryBudget, message: AssistantMessage): void {
|
|
15071
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15072
|
+
const progressed = assistantMessageHasVisibleOrToolContent(message);
|
|
15073
|
+
const silentTimeout =
|
|
15074
|
+
message.stopReason === "error" && BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "");
|
|
15075
|
+
if (!budget.recordAttemptOutcome({ silentTimeout, progressed, selector: `${message.provider}/${message.model}` }))
|
|
15076
|
+
return;
|
|
15077
|
+
logger.info("silent_stall_tripped", { count: budget.snapshot().silentTimeouts });
|
|
15078
|
+
this.emitNotice(
|
|
15079
|
+
"warning",
|
|
15080
|
+
`Stopped after ${budget.snapshot().silentTimeouts} consecutive timeouts without any output.`,
|
|
15081
|
+
"retry",
|
|
15082
|
+
);
|
|
15083
|
+
}
|
|
15084
|
+
|
|
15085
|
+
#recordStepRecoveryFailure(budget: RecoveryBudget): void {
|
|
15086
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15087
|
+
budget.recordRecoverableFailure();
|
|
15088
|
+
if (this.#cancelStepRecoveryDeadline) return;
|
|
15089
|
+
const deadline = budget.snapshot().deadlineAtMs;
|
|
15090
|
+
if (deadline === undefined) return;
|
|
15091
|
+
const controller = this.#stepRecoveryDeadlineAbort;
|
|
15092
|
+
const timer = setTimeout(
|
|
15093
|
+
() => controller?.abort(new RecoveryAdmissionError("deadline")),
|
|
15094
|
+
Math.max(0, deadline - performance.now()),
|
|
15095
|
+
);
|
|
15096
|
+
this.#cancelStepRecoveryDeadline = () => clearTimeout(timer);
|
|
15097
|
+
}
|
|
15098
|
+
|
|
15099
|
+
/** Ends and permanently closes this lease; late callbacks never start a new budget. */
|
|
15100
|
+
#endStepRecoveryBudget(options?: { accepted?: boolean }): void {
|
|
15101
|
+
// Only an unfinished step leaves unobservable remote effects unresolved.
|
|
15102
|
+
if (!options?.accepted && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
15103
|
+
this.#uncertainUpstreamUnresolved = true;
|
|
15104
|
+
}
|
|
15105
|
+
this.#stepRecoveryBudget?.cancel();
|
|
15106
|
+
this.#cancelStepRecoveryDeadline?.();
|
|
15107
|
+
this.#cancelStepRecoveryDeadline = undefined;
|
|
15108
|
+
this.#stepRecoveryDeadlineAbort = undefined;
|
|
15109
|
+
this.#stepRecoveryStepId = undefined;
|
|
15110
|
+
this.#visibleRecoveryReferenceIds = [];
|
|
15111
|
+
this.#visibleRecoveryContextIds = [];
|
|
15112
|
+
this.#stepRecoveryUncertainReasons = [];
|
|
15113
|
+
this.#stepRecoveryBudget = undefined;
|
|
15114
|
+
}
|
|
15115
|
+
|
|
15116
|
+
/**
|
|
15117
|
+
* Whether a failed step that already showed output can be continued on the same
|
|
15118
|
+
* model instead of replayed: the failure must be the live trailing assistant,
|
|
15119
|
+
* its public tail must be text (no tool calls/images), the serializer must have
|
|
15120
|
+
* a verified projection, and nothing outside the transcript tainted the run.
|
|
15121
|
+
*/
|
|
15122
|
+
#canContinueVisibleFailure(message: AssistantMessage): boolean {
|
|
15123
|
+
if (message.stopReason !== "error") return false;
|
|
15124
|
+
if (!VISIBLE_CONTINUATION_APIS.has(message.api) || this.model?.transport === "pi-native") return false;
|
|
15125
|
+
if (!this.model || this.model.api !== message.api || this.model.provider !== message.provider) return false;
|
|
15126
|
+
if (this.model.id !== message.model) return false;
|
|
15127
|
+
if (!isTextVisibleTail(message)) return false;
|
|
15128
|
+
if (!this.#hasCleanContinuationSafety || this.#stepRecoveryUncertainReasons.length > 0) return false;
|
|
15129
|
+
return this.agent.state.messages.at(-1) === message && this.#getVisibleRecoveryReferences(message) !== undefined;
|
|
15130
|
+
}
|
|
15131
|
+
|
|
15132
|
+
/**
|
|
15133
|
+
* Persisted entry ids of the current user turn's failed text tails and completed
|
|
15134
|
+
* tool pairs; undefined when any is missing from the branch or a tail is not text.
|
|
15135
|
+
*/
|
|
15136
|
+
#getVisibleRecoveryReferences(message: AssistantMessage):
|
|
15137
|
+
| {
|
|
15138
|
+
visibleMessageIds: string[];
|
|
15139
|
+
completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }>;
|
|
15140
|
+
}
|
|
15141
|
+
| undefined {
|
|
15142
|
+
const messages = this.agent.state.messages;
|
|
15143
|
+
const start = messages.findLastIndex(item => item.role === "user");
|
|
15144
|
+
const tail = messages.slice(Math.max(0, start));
|
|
15145
|
+
const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
|
|
15146
|
+
const visibleMessageIds: string[] = [];
|
|
15147
|
+
const completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }> = [];
|
|
15148
|
+
for (const item of tail) {
|
|
15149
|
+
if (item.role !== "assistant") continue;
|
|
15150
|
+
const entryId = getSessionMessageEntryId(item);
|
|
15151
|
+
if (!entryId || !branchIds.has(entryId)) return undefined;
|
|
15152
|
+
if (item.stopReason === "error" || item.stopReason === "aborted") {
|
|
15153
|
+
if (!isTextVisibleTail(item)) return undefined;
|
|
15154
|
+
visibleMessageIds.push(entryId);
|
|
15155
|
+
}
|
|
15156
|
+
for (const block of item.content) {
|
|
15157
|
+
if (block.type !== "toolCall") continue;
|
|
15158
|
+
const result = tail.find(candidate => candidate.role === "toolResult" && candidate.toolCallId === block.id);
|
|
15159
|
+
if (result?.role !== "toolResult" || result.isError) return undefined;
|
|
15160
|
+
const resultEntryId = getSessionMessageEntryId(result);
|
|
15161
|
+
if (!resultEntryId || !branchIds.has(resultEntryId)) return undefined;
|
|
15162
|
+
completedTools.push({ toolCallId: block.id, callEntryId: entryId, resultEntryId });
|
|
15163
|
+
}
|
|
15164
|
+
}
|
|
15165
|
+
const messageId = getSessionMessageEntryId(message);
|
|
15166
|
+
if (!messageId || !visibleMessageIds.includes(messageId)) return undefined;
|
|
15167
|
+
return { visibleMessageIds, completedTools };
|
|
15168
|
+
}
|
|
15169
|
+
|
|
15170
|
+
#hasVisibleRecoveryReferences(): boolean {
|
|
15171
|
+
if (this.#visibleRecoveryReferenceIds.length === 0) return false;
|
|
15172
|
+
const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
|
|
15173
|
+
if (!this.#visibleRecoveryReferenceIds.every(id => branchIds.has(id) && this.sessionManager.getEntry(id))) {
|
|
15174
|
+
return false;
|
|
15175
|
+
}
|
|
15176
|
+
// Compaction can keep the entries on the branch while replacing them in the model
|
|
15177
|
+
// context; the preserved prefix and completed tool pairs must still be what is sent.
|
|
15178
|
+
const liveIds = new Set(
|
|
15179
|
+
this.agent.state.messages.map(getSessionMessageEntryId).filter((id): id is string => id !== undefined),
|
|
15180
|
+
);
|
|
15181
|
+
return this.#visibleRecoveryContextIds.every(id => liveIds.has(id));
|
|
15182
|
+
}
|
|
15183
|
+
|
|
15184
|
+
/**
|
|
15185
|
+
* Durably checkpoints the preserved prefix and appends one continuation
|
|
15186
|
+
* instruction. Returns false (and sends nothing) when persistence fails.
|
|
15187
|
+
*/
|
|
15188
|
+
async #appendVisibleContinuation(message: AssistantMessage): Promise<boolean> {
|
|
15189
|
+
if (this.agent.state.messages.at(-1) !== message) return false;
|
|
15190
|
+
const references = this.#getVisibleRecoveryReferences(message);
|
|
15191
|
+
if (!references) return false;
|
|
15192
|
+
const generation = this.#promptGeneration;
|
|
15193
|
+
const budget = this.#stepRecoveryBudget;
|
|
15194
|
+
const details = {
|
|
15195
|
+
version: 1,
|
|
15196
|
+
state: "recovering",
|
|
15197
|
+
runId: `${this.sessionId}:${generation}`,
|
|
15198
|
+
stepId: references.visibleMessageIds[0],
|
|
15199
|
+
attemptId: String(this.#retryAttempt),
|
|
15200
|
+
generation,
|
|
15201
|
+
selector: `${message.provider}/${message.model}`,
|
|
15202
|
+
...references,
|
|
15203
|
+
uncertainTools: this.#stepRecoveryUncertainReasons.map(reason => ({ reason })),
|
|
15204
|
+
usedRequests: budget?.snapshot().usedRequests ?? 0,
|
|
15205
|
+
};
|
|
15206
|
+
const continuation: AgentMessage = {
|
|
15207
|
+
role: "custom",
|
|
15208
|
+
customType: VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
15209
|
+
content: VISIBLE_CONTINUATION_PROMPT,
|
|
15210
|
+
display: false,
|
|
15211
|
+
details,
|
|
15212
|
+
attribution: "agent",
|
|
15213
|
+
timestamp: Date.now(),
|
|
15214
|
+
};
|
|
15215
|
+
try {
|
|
15216
|
+
const continuationMessageId = this.sessionManager.appendCustomMessageEntry(
|
|
15217
|
+
VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
15218
|
+
VISIBLE_CONTINUATION_PROMPT,
|
|
15219
|
+
false,
|
|
15220
|
+
details,
|
|
15221
|
+
"agent",
|
|
15222
|
+
);
|
|
15223
|
+
const checkpointEntryId = this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15224
|
+
...details,
|
|
15225
|
+
continuationMessageId,
|
|
15226
|
+
});
|
|
15227
|
+
// New sessions defer their first write until an assistant exists. Explicitly
|
|
15228
|
+
// materialize before flushing; flush alone need not have a file to fsync.
|
|
15229
|
+
await this.sessionManager.ensureOnDisk();
|
|
15230
|
+
await this.sessionManager.flush();
|
|
15231
|
+
if (this.#isDisposed || this.#promptGeneration !== generation || this.#stepRecoveryBudget !== budget)
|
|
15232
|
+
return false;
|
|
15233
|
+
if (!budget?.canAdmit() || this.agent.state.messages.at(-1) !== message) return false;
|
|
15234
|
+
const contextIds = [
|
|
15235
|
+
...references.visibleMessageIds,
|
|
15236
|
+
...references.completedTools.flatMap(tool => [tool.callEntryId, tool.resultEntryId]),
|
|
15237
|
+
];
|
|
15238
|
+
this.#visibleRecoveryReferenceIds = [checkpointEntryId, continuationMessageId, ...contextIds];
|
|
15239
|
+
this.#visibleRecoveryContextIds = contextIds;
|
|
15240
|
+
if (!this.#hasVisibleRecoveryReferences()) return false;
|
|
15241
|
+
this.agent.appendMessage(continuation);
|
|
15242
|
+
return true;
|
|
15243
|
+
} catch (error) {
|
|
15244
|
+
logger.warn("Visible continuation checkpoint failed", {
|
|
15245
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15246
|
+
});
|
|
15247
|
+
return false;
|
|
15248
|
+
}
|
|
15249
|
+
}
|
|
15250
|
+
|
|
15251
|
+
/** Which in-process interruption, if any, the settled assistant represents. */
|
|
15252
|
+
#autoResumeTriggerFor(message: AssistantMessage): AutoResumeTrigger | undefined {
|
|
15253
|
+
// Managed chains settle here too (the session supplies no managed recovery evidence);
|
|
15254
|
+
// the resume itself is pinned to the interrupted model. Only a transient failure is
|
|
15255
|
+
// resumed: terminal provider answers and an explicit retry opt-out still stop.
|
|
15256
|
+
if (message.stopReason !== "error" || !this.settings.get("retry.enabled")) return undefined;
|
|
15257
|
+
const classification = this.#classifyErrorForRetry(message);
|
|
15258
|
+
if (classification !== "transient" && classification !== "first_event_timeout") return undefined;
|
|
15259
|
+
// A rate-limited or quota-limited provider is not resumed: its retry cap already
|
|
15260
|
+
// decided to stop, and another request would only hit the same limit.
|
|
15261
|
+
const trigger = this.#fallbackTriggerFor(message, true, message.transportFailure);
|
|
15262
|
+
if (trigger?.class === "rate_limit" || trigger?.class === "quota") return undefined;
|
|
15263
|
+
// Work worth keeping exists: tools already ran or the user saw part of the answer.
|
|
15264
|
+
const turnStart = this.agent.state.messages.findLastIndex(item => item.role === "user");
|
|
15265
|
+
const followsTools = this.agent.state.messages.slice(turnStart + 1).some(item => item.role === "toolResult");
|
|
15266
|
+
return followsTools || hasVisibleFailedPrefix([message]) ? "interrupted" : undefined;
|
|
15267
|
+
}
|
|
15268
|
+
|
|
15269
|
+
/** Last persisted recovery checkpoint of the branch, when it records an unfinished step. */
|
|
15270
|
+
#lastInFlightCheckpoint(): Record<string, unknown> | undefined {
|
|
15271
|
+
const entry = this.sessionManager
|
|
15272
|
+
.getBranch()
|
|
15273
|
+
.findLast(item => item.type === "custom" && item.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE);
|
|
15274
|
+
if (entry?.type !== "custom" || !entry.data || typeof entry.data !== "object" || Array.isArray(entry.data))
|
|
15275
|
+
return undefined;
|
|
15276
|
+
const data = entry.data as Record<string, unknown>;
|
|
15277
|
+
return data.state === "in_flight" || data.state === "recovering" ? data : undefined;
|
|
15278
|
+
}
|
|
15279
|
+
|
|
15280
|
+
/**
|
|
15281
|
+
* Single decision point for an automatic resume of an interrupted step. Runs at
|
|
15282
|
+
* most once per user turn, on the interrupted step's selector, and only after the
|
|
15283
|
+
* resume marker is durable. Returns true when a resume was scheduled.
|
|
15284
|
+
*/
|
|
15285
|
+
async #maybeAutoResume(trigger: AutoResumeTrigger): Promise<boolean> {
|
|
15286
|
+
await this.#pendingUncertainFlush;
|
|
15287
|
+
const generation = this.#promptGeneration;
|
|
15288
|
+
const messages = this.agent.state.messages;
|
|
15289
|
+
const model = this.model;
|
|
15290
|
+
const checkpoint = trigger === "restart" ? this.#lastInFlightCheckpoint() : undefined;
|
|
15291
|
+
const failed = messages.findLast(message => message.role === "assistant") as AssistantMessage | undefined;
|
|
15292
|
+
const selector =
|
|
15293
|
+
trigger === "restart"
|
|
15294
|
+
? typeof checkpoint?.selector === "string"
|
|
15295
|
+
? checkpoint.selector
|
|
15296
|
+
: undefined
|
|
15297
|
+
: failed
|
|
15298
|
+
? `${failed.provider}/${failed.model}`
|
|
15299
|
+
: undefined;
|
|
15300
|
+
const restartUncertain =
|
|
15301
|
+
trigger === "restart" &&
|
|
15302
|
+
(!checkpoint ||
|
|
15303
|
+
!Array.isArray(checkpoint.uncertain) ||
|
|
15304
|
+
checkpoint.uncertain.length > 0 ||
|
|
15305
|
+
typeof checkpoint.api !== "string" ||
|
|
15306
|
+
checkpoint.api === "cursor-agent" ||
|
|
15307
|
+
checkpoint.transport === "pi-native");
|
|
15308
|
+
const budget = this.#stepRecoveryBudget;
|
|
15309
|
+
const selectedTools = this.agent.state.tools;
|
|
15310
|
+
const decision = resolveAutoResumePolicy({
|
|
15311
|
+
enabled: this.settings.get("retry.autoResume") && !this.#isDisposed && !this.#handoffTransitionActive,
|
|
15312
|
+
trigger,
|
|
15313
|
+
messages,
|
|
15314
|
+
resumedThisTurn: hasAutoResumeMarkerSinceLastUser(this.sessionManager.getBranch()),
|
|
15315
|
+
uncertainUpstream:
|
|
15316
|
+
this.#uncertainFlushFailed ||
|
|
15317
|
+
this.#uncertainUpstreamUnresolved ||
|
|
15318
|
+
this.#stepRecoveryUncertainReasons.length > 0 ||
|
|
15319
|
+
restartUncertain,
|
|
15320
|
+
opaqueTransport: model?.api === "cursor-agent" || model?.transport === "pi-native",
|
|
15321
|
+
budgetAdmits: trigger === "restart" ? true : budget?.canAdmit() === true,
|
|
15322
|
+
userInputPending: this.agent.hasQueuedMessages() || this.#cancelAndSubmitInProgress,
|
|
15323
|
+
selectorAvailable: model !== undefined && selector === `${model.provider}/${model.id}`,
|
|
15324
|
+
prefixReplayable:
|
|
15325
|
+
failed !== undefined &&
|
|
15326
|
+
VISIBLE_CONTINUATION_APIS.has(failed.api) &&
|
|
15327
|
+
model?.api === failed.api &&
|
|
15328
|
+
this.#hasCleanContinuationSafety,
|
|
15329
|
+
selectedTools,
|
|
15330
|
+
});
|
|
15331
|
+
if (decision.type === "pause") {
|
|
15332
|
+
logger.debug("auto_resume_paused", { trigger, reason: decision.reason });
|
|
15333
|
+
if (decision.reason === "already_resumed") {
|
|
15334
|
+
this.emitNotice("info", "Automatic resume was already used for this request.", "retry");
|
|
15335
|
+
}
|
|
15336
|
+
return false;
|
|
15337
|
+
}
|
|
15338
|
+
if (!selector) return false;
|
|
15339
|
+
const marker: AutoResumeMarker = {
|
|
15340
|
+
version: 1,
|
|
15341
|
+
trigger,
|
|
15342
|
+
mode: decision.mode,
|
|
15343
|
+
selector,
|
|
15344
|
+
interruptedCallIds: decision.interruptedCalls.map(call => call.id),
|
|
15345
|
+
};
|
|
15346
|
+
const placeholders: ToolResultMessage[] = decision.interruptedCalls.map(call => ({
|
|
15347
|
+
role: "toolResult",
|
|
15348
|
+
toolCallId: call.id,
|
|
15349
|
+
toolName: call.name,
|
|
15350
|
+
content: [{ type: "text", text: INTERRUPTED_TOOL_RESULT_TEXT }],
|
|
15351
|
+
isError: true,
|
|
15352
|
+
timestamp: Date.now(),
|
|
15353
|
+
}));
|
|
15354
|
+
const prompt = decision.mode === "restricted" ? AUTO_RESUME_RESTRICTED_PROMPT : AUTO_RESUME_FULL_PROMPT;
|
|
15355
|
+
const tailBeforeFlush = messages.at(-1);
|
|
15356
|
+
try {
|
|
15357
|
+
this.sessionManager.appendCustomEntry(AUTO_RESUME_MARKER_CUSTOM_TYPE, marker);
|
|
15358
|
+
for (const placeholder of placeholders) this.sessionManager.appendMessage(placeholder);
|
|
15359
|
+
this.sessionManager.appendCustomMessageEntry(AUTO_RESUME_CUSTOM_TYPE, prompt, false, marker, "agent");
|
|
15360
|
+
await this.sessionManager.ensureOnDisk();
|
|
15361
|
+
await this.sessionManager.flush();
|
|
15362
|
+
} catch (error) {
|
|
15363
|
+
logger.warn("Automatic resume checkpoint failed", {
|
|
15364
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15365
|
+
});
|
|
15366
|
+
return false;
|
|
15367
|
+
}
|
|
15368
|
+
if (
|
|
15369
|
+
this.#isDisposed ||
|
|
15370
|
+
this.#promptGeneration !== generation ||
|
|
15371
|
+
this.agent.state.messages !== messages ||
|
|
15372
|
+
messages.at(-1) !== tailBeforeFlush
|
|
15373
|
+
) {
|
|
15374
|
+
// Fail-safe: nothing is sent. The durable marker still consumes this turn's resume.
|
|
15375
|
+
logger.warn("auto_resume_abandoned", { trigger, reason: "state changed while the marker was written" });
|
|
15376
|
+
return false;
|
|
15377
|
+
}
|
|
15378
|
+
// A failed attempt that showed nothing is not part of the answer; resending it only
|
|
15379
|
+
// adds an empty assistant turn (the same rule manual retry applies).
|
|
15380
|
+
const tailMessage = this.agent.state.messages.at(-1);
|
|
15381
|
+
if (
|
|
15382
|
+
placeholders.length === 0 &&
|
|
15383
|
+
tailMessage?.role === "assistant" &&
|
|
15384
|
+
(tailMessage.stopReason === "error" || tailMessage.stopReason === "aborted") &&
|
|
15385
|
+
!assistantMessageHasVisibleOrToolContent(tailMessage)
|
|
15386
|
+
) {
|
|
15387
|
+
this.agent.replaceMessages(this.agent.state.messages.slice(0, -1));
|
|
15388
|
+
}
|
|
15389
|
+
// The durable placeholders replace in-memory pairing results that observed nothing.
|
|
15390
|
+
if (this.agent.state.messages.some(isSynthesizedResumeResult)) {
|
|
15391
|
+
this.agent.replaceMessages(this.agent.state.messages.filter(message => !isSynthesizedResumeResult(message)));
|
|
15392
|
+
}
|
|
15393
|
+
for (const placeholder of placeholders) this.agent.appendMessage(placeholder);
|
|
15394
|
+
this.agent.appendMessage({
|
|
15395
|
+
role: "custom",
|
|
15396
|
+
customType: AUTO_RESUME_CUSTOM_TYPE,
|
|
15397
|
+
content: prompt,
|
|
15398
|
+
display: false,
|
|
15399
|
+
details: marker,
|
|
15400
|
+
attribution: "agent",
|
|
15401
|
+
timestamp: Date.now(),
|
|
15402
|
+
});
|
|
15403
|
+
if (trigger === "restart") {
|
|
15404
|
+
this.#endStepRecoveryBudget();
|
|
15405
|
+
this.#stepRecoveryBudget = new RecoveryBudget({
|
|
15406
|
+
maxRequests: RESTART_RESUME_MAX_REQUESTS,
|
|
15407
|
+
windowMs: RESTART_RESUME_WINDOW_MS,
|
|
15408
|
+
maxSilentTimeouts: this.#maxSilentTimeouts(),
|
|
15409
|
+
});
|
|
15410
|
+
// The small envelope starts its clock now and arms its deadline: an unanswered
|
|
15411
|
+
// restart resume never waits longer, even mid-request.
|
|
15412
|
+
this.#stepRecoveryDeadlineAbort = new AbortController();
|
|
15413
|
+
this.#recordStepRecoveryFailure(this.#stepRecoveryBudget);
|
|
15414
|
+
this.#restartResumeEnvelopePending = true;
|
|
15415
|
+
this.#unresolvedRecoveryAtHydration = false;
|
|
15416
|
+
}
|
|
15417
|
+
if (decision.mode === "restricted") this.#setRestrictedToolMode(true);
|
|
15418
|
+
this.#activeAutoResume = { trigger, mode: decision.mode };
|
|
15419
|
+
this.emitNotice(
|
|
15420
|
+
"info",
|
|
15421
|
+
decision.mode === "restricted"
|
|
15422
|
+
? "Resuming the interrupted step automatically (read-only tools)."
|
|
15423
|
+
: "Resuming the interrupted step automatically.",
|
|
15424
|
+
"retry",
|
|
15425
|
+
);
|
|
15426
|
+
logger.info("auto_resume_started", { trigger, mode: decision.mode, interruptedCalls: placeholders.length });
|
|
15427
|
+
if (trigger === "restart") return true;
|
|
15428
|
+
void this.#scheduleAgentContinue({
|
|
15429
|
+
delayMs: 1,
|
|
15430
|
+
generation,
|
|
15431
|
+
allowDuringCancelAndSubmit: false,
|
|
15432
|
+
pinnedModel: true,
|
|
15433
|
+
shouldContinue: () => !this.agent.hasQueuedMessages(),
|
|
15434
|
+
// A resume that never started closes the step it was carrying, unless a newer
|
|
15435
|
+
// turn already owns the envelope.
|
|
15436
|
+
onError: () => {
|
|
15437
|
+
this.#endAutoResume("failed");
|
|
15438
|
+
if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
|
|
15439
|
+
},
|
|
15440
|
+
onSkip: () => {
|
|
15441
|
+
this.#endAutoResume("skipped");
|
|
15442
|
+
if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
|
|
15443
|
+
},
|
|
15444
|
+
});
|
|
15445
|
+
return true;
|
|
15446
|
+
}
|
|
15447
|
+
|
|
15448
|
+
/** Restricted mode narrows only what the model sees and may execute; the selection is untouched. */
|
|
15449
|
+
#setRestrictedToolMode(enabled: boolean): void {
|
|
15450
|
+
if (this.#restrictedToolMode === enabled) return;
|
|
15451
|
+
this.#restrictedToolMode = enabled;
|
|
15452
|
+
this.agent.setToolFilter(enabled ? tools => tools.filter(tool => isRestartSafeTool(tool, tools)) : undefined);
|
|
15453
|
+
}
|
|
15454
|
+
|
|
15455
|
+
#endAutoResume(outcome: "completed" | "failed" | "skipped"): void {
|
|
15456
|
+
this.#restartResumeEnvelopePending = false;
|
|
15457
|
+
this.#setRestrictedToolMode(false);
|
|
15458
|
+
const active = this.#activeAutoResume;
|
|
15459
|
+
this.#activeAutoResume = undefined;
|
|
15460
|
+
if (active) logger.info("auto_resume_finished", { ...active, outcome });
|
|
15461
|
+
}
|
|
15462
|
+
|
|
14583
15463
|
/**
|
|
14584
15464
|
* Skip the current retry backoff and re-attempt immediately. Distinct from
|
|
14585
15465
|
* abortRetry(), which cancels the retry and returns to idle. No-op when no
|
|
@@ -14672,20 +15552,21 @@ export class AgentSession {
|
|
|
14672
15552
|
);
|
|
14673
15553
|
}
|
|
14674
15554
|
|
|
14675
|
-
#isUnresolvedToolUseAssistant(message: AssistantMessage): boolean {
|
|
14676
|
-
return message.stopReason === "toolUse" && message.content.some(content => content.type === "toolCall");
|
|
14677
|
-
}
|
|
14678
|
-
|
|
14679
15555
|
/**
|
|
14680
15556
|
* Manually retry the last failed assistant turn, or resume an interrupted tail
|
|
14681
15557
|
* left by a non-graceful process exit after the user/custom/tool-result message
|
|
14682
15558
|
* was persisted but before the agent emitted a terminal assistant response.
|
|
14683
|
-
* Removes failed/aborted
|
|
14684
|
-
* re-
|
|
14685
|
-
* @returns
|
|
15559
|
+
* Removes only a failed/aborted assistant tail with no visible output or tool
|
|
15560
|
+
* calls, then re-attempts with a fresh retry budget.
|
|
15561
|
+
* @returns false when the agent is busy, no retryable tail exists, the tail shows
|
|
15562
|
+
* output or tool calls, a persisted recovery is unresolved, or the step dispatched
|
|
15563
|
+
* unobservable remote work; true when a retry/resume was scheduled.
|
|
14686
15564
|
*/
|
|
14687
15565
|
async retry(): Promise<boolean> {
|
|
14688
15566
|
if (this.isStreaming || this.isCompacting || this.isRetrying) return false;
|
|
15567
|
+
if (this.#unresolvedRecoveryAtHydration) return false;
|
|
15568
|
+
// Unobservable remote effects (broker/gateway) cannot be proven unsent.
|
|
15569
|
+
if (this.#uncertainUpstreamUnresolved || this.#stepRecoveryUncertainReasons.length > 0) return false;
|
|
14689
15570
|
// A handoff transition owns the session; retrying would mutate the tail and
|
|
14690
15571
|
// schedule a continuation against the session being handed off.
|
|
14691
15572
|
if (this.isGeneratingHandoff || this.#handoffTransitionActive) return false;
|
|
@@ -14697,22 +15578,23 @@ export class AgentSession {
|
|
|
14697
15578
|
if (lastMsg.role !== "assistant") {
|
|
14698
15579
|
if (!this.#isInterruptedRetryTail(lastMsg)) return false;
|
|
14699
15580
|
this.#retryAttempt = 0;
|
|
15581
|
+
this.#endStepRecoveryBudget();
|
|
14700
15582
|
this.#scheduleAgentContinue({ delayMs: 1 });
|
|
14701
15583
|
return true;
|
|
14702
15584
|
}
|
|
14703
15585
|
|
|
14704
15586
|
const assistantMsg = lastMsg as AssistantMessage;
|
|
14705
|
-
|
|
14706
|
-
|
|
14707
|
-
|
|
14708
|
-
|
|
14709
|
-
if (!shouldDropAssistant) return false;
|
|
15587
|
+
// Explicit retry does not establish the outcome of an interrupted operation
|
|
15588
|
+
// and is never permission to erase a prefix already shown to the user.
|
|
15589
|
+
if (assistantMessageHasVisibleOrToolContent(assistantMsg)) return false;
|
|
15590
|
+
if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") return false;
|
|
14710
15591
|
|
|
14711
|
-
// Remove the failed/aborted
|
|
15592
|
+
// Remove the empty failed/aborted assistant message before re-attempting.
|
|
14712
15593
|
this.agent.replaceMessages(messages.slice(0, -1));
|
|
14713
15594
|
|
|
14714
|
-
// Reset retry budget for a fresh attempt
|
|
15595
|
+
// Reset retry budget for a fresh attempt: a manual retry is a new invocation.
|
|
14715
15596
|
this.#retryAttempt = 0;
|
|
15597
|
+
this.#endStepRecoveryBudget();
|
|
14716
15598
|
|
|
14717
15599
|
// Re-attempt the turn
|
|
14718
15600
|
this.#scheduleAgentContinue({ delayMs: 1 });
|
|
@@ -15737,6 +16619,7 @@ export class AgentSession {
|
|
|
15737
16619
|
// (#2797 / #2925).
|
|
15738
16620
|
if (switchingToDifferentSession) await initializeLocalRoot(this.#localProtocolOptions());
|
|
15739
16621
|
this.#syncAgentSessionId();
|
|
16622
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15740
16623
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15741
16624
|
|
|
15742
16625
|
const sessionContext = this.buildDisplaySessionContext();
|
|
@@ -15847,6 +16730,7 @@ export class AgentSession {
|
|
|
15847
16730
|
this.sessionManager.restoreState(previousSessionState);
|
|
15848
16731
|
this.#defaultFallbackController = undefined;
|
|
15849
16732
|
this.#syncAgentSessionId(previousSessionState.sessionId);
|
|
16733
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15850
16734
|
this.#restoreWorkflowGateEmitter(suspendedWorkflowGateEmitter);
|
|
15851
16735
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15852
16736
|
let restoreMcpError: unknown;
|
|
@@ -15964,6 +16848,7 @@ export class AgentSession {
|
|
|
15964
16848
|
|
|
15965
16849
|
this.#syncTodoPhasesFromBranch();
|
|
15966
16850
|
this.#syncAgentSessionId();
|
|
16851
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15967
16852
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
15968
16853
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15969
16854
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -16712,15 +17597,22 @@ export class AgentSession {
|
|
|
16712
17597
|
if (!lastAssistant) return undefined;
|
|
16713
17598
|
|
|
16714
17599
|
let text = "";
|
|
16715
|
-
for (const
|
|
16716
|
-
|
|
16717
|
-
|
|
17600
|
+
for (const assistant of this.getVisibleAnswerChain(lastAssistant)) {
|
|
17601
|
+
for (const content of assistant.content) {
|
|
17602
|
+
if (content.type === "text") {
|
|
17603
|
+
text += content.text;
|
|
17604
|
+
}
|
|
16718
17605
|
}
|
|
16719
17606
|
}
|
|
16720
17607
|
|
|
16721
17608
|
return text.trim() || undefined;
|
|
16722
17609
|
}
|
|
16723
17610
|
|
|
17611
|
+
/** The assistant messages that together form the visible answer ending at `assistant`. */
|
|
17612
|
+
getVisibleAnswerChain(assistant: AssistantMessage): AssistantMessage[] {
|
|
17613
|
+
return getVisibleAnswerChain(this.messages, assistant);
|
|
17614
|
+
}
|
|
17615
|
+
|
|
16724
17616
|
hasCopyCandidateAssistantMessage(): boolean {
|
|
16725
17617
|
return this.#getLastCopyCandidateAssistantMessage() !== undefined;
|
|
16726
17618
|
}
|