@sayknow-cli/coding-agent 0.7.6 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/types/config/model-registry.d.ts +3 -4
- package/dist/types/config/settings-schema.d.ts +14 -0
- package/dist/types/extensibility/tool-proxy.d.ts +2 -0
- package/dist/types/session/agent-session.d.ts +13 -4
- package/dist/types/session/auto-resume.d.ts +89 -0
- package/dist/types/session/fallback-chain-controller.d.ts +7 -0
- package/dist/types/session/messages.d.ts +32 -0
- package/package.json +7 -7
- package/scripts/generate-sdk-operation-inventory.ts +3 -0
- package/src/config/model-registry.ts +5 -6
- package/src/config/settings-schema.ts +8 -0
- package/src/extensibility/tool-proxy.ts +11 -0
- package/src/internal-urls/docs-index.generated.ts +1 -1
- package/src/main.ts +4 -1
- package/src/modes/print-mode.ts +13 -3
- package/src/sdk/bus/index.ts +3 -16
- package/src/sdk/protocol/operation-inventory.generated.json +22 -0
- package/src/sdk/session.ts +18 -5
- package/src/session/agent-session.ts +1047 -104
- package/src/session/auto-resume.ts +231 -0
- package/src/session/fallback-chain-controller.ts +15 -0
- package/src/session/messages.ts +91 -0
- package/src/task/executor.ts +2 -1
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
resolveTelemetry,
|
|
40
40
|
type StablePrefixSnapshot,
|
|
41
41
|
ThinkingLevel,
|
|
42
|
+
TOOL_CALL_REENTRY_ERROR_CODE,
|
|
42
43
|
} from "@sayknow-cli/agent-core";
|
|
43
44
|
import { normalizeMessagesForProvider } from "@sayknow-cli/agent-core/agent-loop";
|
|
44
45
|
import {
|
|
@@ -99,11 +100,20 @@ import type {
|
|
|
99
100
|
import {
|
|
100
101
|
classifyContextOverflow,
|
|
101
102
|
clearAnthropicFastModeFallback,
|
|
103
|
+
createUpstreamAdmission,
|
|
102
104
|
getSupportedEfforts,
|
|
103
105
|
isContextOverflow,
|
|
106
|
+
isRecoveryAdmissionErrorMessage,
|
|
104
107
|
isUsageLimitError,
|
|
105
108
|
modelsAreEqual,
|
|
109
|
+
type OAuthRefreshOptions,
|
|
110
|
+
RecoveryAdmissionError,
|
|
111
|
+
RecoveryBudget,
|
|
112
|
+
type RecoveryRequestKind,
|
|
106
113
|
resolveServiceTier,
|
|
114
|
+
SAME_MODEL_RECOVERY_BASE_DELAY_MS,
|
|
115
|
+
SAME_MODEL_RECOVERY_MAX_DELAY_MS,
|
|
116
|
+
SAME_MODEL_RECOVERY_MAX_REQUESTS,
|
|
107
117
|
streamSimple,
|
|
108
118
|
} from "@sayknow-cli/ai";
|
|
109
119
|
import {
|
|
@@ -386,6 +396,23 @@ import { buildResponseLanguageReminder, type DetectedLanguage, detectResponseLan
|
|
|
386
396
|
|
|
387
397
|
export { DefaultModelSelectionRecoveryError } from "./default-model-selection";
|
|
388
398
|
|
|
399
|
+
import {
|
|
400
|
+
AUTO_RESUME_FULL_PROMPT,
|
|
401
|
+
AUTO_RESUME_MARKER_CUSTOM_TYPE,
|
|
402
|
+
AUTO_RESUME_RESTRICTED_PROMPT,
|
|
403
|
+
type AutoResumeMarker,
|
|
404
|
+
type AutoResumeMode,
|
|
405
|
+
type AutoResumeTrigger,
|
|
406
|
+
hasAutoResumeMarkerSinceLastUser,
|
|
407
|
+
hasVisibleFailedPrefix,
|
|
408
|
+
INTERRUPTED_TOOL_RESULT_TEXT,
|
|
409
|
+
isRestartSafeTool,
|
|
410
|
+
isSynthesizedResumeResult,
|
|
411
|
+
isThinkingOnlyFailure,
|
|
412
|
+
RESTART_RESUME_MAX_REQUESTS,
|
|
413
|
+
RESTART_RESUME_WINDOW_MS,
|
|
414
|
+
resolveAutoResumePolicy,
|
|
415
|
+
} from "./auto-resume";
|
|
389
416
|
import type {
|
|
390
417
|
ClientBridge,
|
|
391
418
|
ClientBridgePermissionOption,
|
|
@@ -401,16 +428,19 @@ import {
|
|
|
401
428
|
import { pruneStaleFileMentions } from "./file-mention-pruning";
|
|
402
429
|
import type { MemoryGuardRestoreResult } from "./memory-guard-checkpoint-participant";
|
|
403
430
|
import {
|
|
431
|
+
AUTO_RESUME_CUSTOM_TYPE,
|
|
404
432
|
type BashExecutionMessage,
|
|
405
433
|
type CompactionSummaryMessage,
|
|
406
434
|
type CustomMessage,
|
|
407
435
|
convertToLlm,
|
|
408
436
|
createPreAdmissionArtifactSpillPreview,
|
|
409
437
|
type FileMentionMessage,
|
|
438
|
+
getVisibleAnswerChain,
|
|
410
439
|
type PythonExecutionMessage,
|
|
411
440
|
readPendingDisplayTag,
|
|
412
441
|
SILENT_ABORT_MARKER,
|
|
413
442
|
SKILL_PROMPT_MESSAGE_TYPE,
|
|
443
|
+
VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
414
444
|
} from "./messages";
|
|
415
445
|
import { isLegacyProviderSafetyStopMessage } from "./provider-safety-stop";
|
|
416
446
|
import { formatSessionDumpText } from "./session-dump-format";
|
|
@@ -868,6 +898,11 @@ type RetryErrorClassification =
|
|
|
868
898
|
|
|
869
899
|
const BARE_DEFAULT_WATCHDOG_ERROR =
|
|
870
900
|
/^(?:[A-Za-z][A-Za-z0-9-]*(?: [A-Za-z][A-Za-z0-9-]*){0,3} )stream (?:timed out while waiting for the first event|stalled while waiting for the next event)$/;
|
|
901
|
+
/** Server-side overload/unavailability wording. Bare numbers never count: "500 requests per minute" is a throttle. */
|
|
902
|
+
const BARE_DEFAULT_SERVER_OVERLOAD_ERROR = /overloaded|service.?unavailable|internal.?server.?error|\bapi_error\b/i;
|
|
903
|
+
/** Throttle or quota wording, in any phrasing a provider uses for it. */
|
|
904
|
+
const BARE_DEFAULT_THROTTLE_ERROR =
|
|
905
|
+
/rate.?limit|too many requests|quota|usage.?limit|requests? per (?:second|minute|hour|day)|\b(?:rpm|tpm|rpd)\b|throttl/i;
|
|
871
906
|
const KIMI_CODE_FIRST_EVENT_TIMEOUT_MESSAGES = {
|
|
872
907
|
"anthropic-messages": new Set([
|
|
873
908
|
"Provider stream timed out while waiting for the first event",
|
|
@@ -899,6 +934,10 @@ const ALIBABA_TOKEN_PLAN_FIRST_EVENT_TIMEOUT_MESSAGES = {
|
|
|
899
934
|
]),
|
|
900
935
|
} as const;
|
|
901
936
|
|
|
937
|
+
function isServerHttpStatus(status: number | undefined): boolean {
|
|
938
|
+
return status !== undefined && status >= 500 && status <= 599;
|
|
939
|
+
}
|
|
940
|
+
|
|
902
941
|
function hasBareDefaultRetryDisqualifyingFacts(message: AssistantMessage): boolean {
|
|
903
942
|
if (message.errorKind !== undefined || message.errorStatus !== undefined) return true;
|
|
904
943
|
const facts = message.transportFailure;
|
|
@@ -920,6 +959,20 @@ function assistantMessageHasVisibleOrToolContent(message: AssistantMessage): boo
|
|
|
920
959
|
});
|
|
921
960
|
}
|
|
922
961
|
|
|
962
|
+
/** Serializers with a verified projection of a failed plain-text assistant prefix. */
|
|
963
|
+
const VISIBLE_CONTINUATION_APIS: ReadonlySet<string> = new Set(["openai-completions", "anthropic-messages"]);
|
|
964
|
+
const RECOVERY_CHECKPOINT_CUSTOM_TYPE = "recovery_checkpoint";
|
|
965
|
+
const VISIBLE_CONTINUATION_PROMPT =
|
|
966
|
+
"The connection was interrupted while you were answering. Your answer above is preserved exactly as the user saw it. Continue directly from where it stops: do not repeat any of it and do not redo completed work.";
|
|
967
|
+
|
|
968
|
+
/** Only plain text is a proved replay projection; signed/private blocks are never repaired. */
|
|
969
|
+
function isTextVisibleTail(message: AssistantMessage): boolean {
|
|
970
|
+
return (
|
|
971
|
+
message.content.every(block => block.type === "text") &&
|
|
972
|
+
message.content.some(block => block.type === "text" && block.text.trim().length > 0)
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
|
|
923
976
|
function isLocalModelEndpoint(model: Model | undefined): boolean {
|
|
924
977
|
if (!model) return false;
|
|
925
978
|
if (model.provider === "ollama" || model.provider === "lm-studio" || model.provider === "llama.cpp") {
|
|
@@ -971,7 +1024,6 @@ export interface BtwConversationScope {
|
|
|
971
1024
|
systemPrompt: string[];
|
|
972
1025
|
messages: BtwRoleTextMessage[];
|
|
973
1026
|
thinkingLevel: ThinkingLevel;
|
|
974
|
-
hideThinkingSummary: boolean;
|
|
975
1027
|
serviceTier: ServiceTier | undefined;
|
|
976
1028
|
credentialSessionId: string;
|
|
977
1029
|
providerAffinitySessionId: string;
|
|
@@ -1762,10 +1814,19 @@ export class AgentSession {
|
|
|
1762
1814
|
|
|
1763
1815
|
/** Replay safety for the currently admitted top-level prompt/custom-message run. */
|
|
1764
1816
|
#retryReplayEpoch = 0;
|
|
1817
|
+
/**
|
|
1818
|
+
* Epoch tainted by something outside the model transcript (extension hooks,
|
|
1819
|
+
* commands, user bash/python, abort). Visible model output and completed tool
|
|
1820
|
+
* pairs stay in the transcript a recovery keeps and are never re-executed, so
|
|
1821
|
+
* they do not taint the run; recovery stays same-model and transcript-only.
|
|
1822
|
+
*/
|
|
1765
1823
|
#retryReplayUnsafeEpoch: number | undefined;
|
|
1766
1824
|
|
|
1767
1825
|
#resetRetryReplaySafety(): void {
|
|
1768
1826
|
this.#retryReplayEpoch++;
|
|
1827
|
+
this.#uncertainUpstreamUnresolved = false;
|
|
1828
|
+
// A genuine new prompt supersedes an interrupted persisted step.
|
|
1829
|
+
this.#unresolvedRecoveryAtHydration = false;
|
|
1769
1830
|
this.#retryReplayUnsafeEpoch = undefined;
|
|
1770
1831
|
if (
|
|
1771
1832
|
this.#extensionRunner?.hasHandlers("context") ||
|
|
@@ -1780,7 +1841,7 @@ export class AgentSession {
|
|
|
1780
1841
|
if (this.#retryReplayEpoch > 0) this.#retryReplayUnsafeEpoch = this.#retryReplayEpoch;
|
|
1781
1842
|
}
|
|
1782
1843
|
|
|
1783
|
-
get #
|
|
1844
|
+
get #hasCleanContinuationSafety(): boolean {
|
|
1784
1845
|
return this.#retryReplayEpoch > 0 && this.#retryReplayUnsafeEpoch !== this.#retryReplayEpoch;
|
|
1785
1846
|
}
|
|
1786
1847
|
#prePromptContextCheckPromise: Promise<void> | undefined = undefined;
|
|
@@ -1808,6 +1869,34 @@ export class AgentSession {
|
|
|
1808
1869
|
#overflowMaintenanceAttempts = 0;
|
|
1809
1870
|
#defaultFallbackExhaustedLastTurn = false;
|
|
1810
1871
|
#transientStopTimestamp: number | undefined;
|
|
1872
|
+
/**
|
|
1873
|
+
* Shared same-model recovery envelope for the unfinished model step: every
|
|
1874
|
+
* concrete upstream request (initial, hidden resend, credential/token, session
|
|
1875
|
+
* retry) is admitted here. Created at the step's first request, cleared when the
|
|
1876
|
+
* step is accepted or a new user turn starts; never refunded.
|
|
1877
|
+
*/
|
|
1878
|
+
#stepRecoveryBudget: RecoveryBudget | undefined;
|
|
1879
|
+
#visibleRecoveryReferenceIds: string[] = [];
|
|
1880
|
+
#visibleRecoveryContextIds: string[] = [];
|
|
1881
|
+
#stepRecoveryUncertainReasons: string[] = [];
|
|
1882
|
+
/**
|
|
1883
|
+
* Set when a step dispatched unobservable remote work. Ending the budget closes
|
|
1884
|
+
* admission but never launders that fact: manual retry stays refused until a new
|
|
1885
|
+
* user prompt starts a fresh replay epoch.
|
|
1886
|
+
*/
|
|
1887
|
+
#uncertainUpstreamUnresolved = false;
|
|
1888
|
+
/** Durable record of in-step uncertainty; admission and resume decisions await it. */
|
|
1889
|
+
#pendingUncertainFlush: Promise<void> = Promise.resolve();
|
|
1890
|
+
#uncertainFlushFailed = false;
|
|
1891
|
+
/** The current run is an automatic resume restricted to restart-safe tools. */
|
|
1892
|
+
#activeAutoResume: { trigger: AutoResumeTrigger; mode: AutoResumeMode } | undefined;
|
|
1893
|
+
/** A restart resume owns the current envelope until its first step is accepted. */
|
|
1894
|
+
#restartResumeEnvelopePending = false;
|
|
1895
|
+
#restrictedToolMode = false;
|
|
1896
|
+
#stepRecoveryStepId: string | undefined;
|
|
1897
|
+
#stepRecoveryDeadlineAbort: AbortController | undefined;
|
|
1898
|
+
#cancelStepRecoveryDeadline: (() => void) | undefined;
|
|
1899
|
+
#unresolvedRecoveryAtHydration = false;
|
|
1811
1900
|
/** Language of the latest genuine user prompt (or inherited from the parent agent). */
|
|
1812
1901
|
#responseLanguage: DetectedLanguage | undefined;
|
|
1813
1902
|
#fallbackInvocationId = 0;
|
|
@@ -2588,6 +2677,8 @@ export class AgentSession {
|
|
|
2588
2677
|
// Per-tool TTSR reminders are folded into the matched tool's result via this hook.
|
|
2589
2678
|
this.agent.afterToolCall = ctx => this.#ttsrAfterToolCall(ctx);
|
|
2590
2679
|
this.agent.providerSessionState = this.#providerSessionState;
|
|
2680
|
+
this.agent.getUpstreamRequestLease = this.#getUpstreamRequestLease;
|
|
2681
|
+
this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
|
|
2591
2682
|
this.#syncAgentSessionId();
|
|
2592
2683
|
this.#removeEphemeralCustomMessages();
|
|
2593
2684
|
|
|
@@ -3458,38 +3549,9 @@ export class AgentSession {
|
|
|
3458
3549
|
|
|
3459
3550
|
/** Internal handler for agent events - shared by subscribe and reconnect */
|
|
3460
3551
|
#handleAgentEvent = async (event: AgentEvent): Promise<void> => {
|
|
3552
|
+
// Model output and completed tool pairs stay in the transcript a recovery
|
|
3553
|
+
// keeps; only an observing extension handler can make the run unsafe.
|
|
3461
3554
|
if (this.#extensionRunner?.hasHandlers(event.type)) this.#markRetryReplayUnsafe();
|
|
3462
|
-
if (
|
|
3463
|
-
event.type === "tool_execution_start" ||
|
|
3464
|
-
event.type === "tool_execution_update" ||
|
|
3465
|
-
event.type === "tool_execution_end"
|
|
3466
|
-
) {
|
|
3467
|
-
this.#markRetryReplayUnsafe();
|
|
3468
|
-
} else if (event.type === "message_end") {
|
|
3469
|
-
if (
|
|
3470
|
-
event.message.role === "toolResult" ||
|
|
3471
|
-
(event.message.role === "assistant" && assistantMessageHasVisibleOrToolContent(event.message))
|
|
3472
|
-
) {
|
|
3473
|
-
this.#markRetryReplayUnsafe();
|
|
3474
|
-
}
|
|
3475
|
-
} else if (event.type === "message_update") {
|
|
3476
|
-
const update = event.assistantMessageEvent;
|
|
3477
|
-
if (
|
|
3478
|
-
update.type === "toolcall_start" ||
|
|
3479
|
-
update.type === "toolcall_delta" ||
|
|
3480
|
-
update.type === "toolcall_end" ||
|
|
3481
|
-
((update.type === "text_delta" ||
|
|
3482
|
-
update.type === "thinking_delta" ||
|
|
3483
|
-
update.type === "reasoning_summary_delta") &&
|
|
3484
|
-
update.delta.length > 0) ||
|
|
3485
|
-
((update.type === "text_end" ||
|
|
3486
|
-
update.type === "thinking_end" ||
|
|
3487
|
-
update.type === "reasoning_summary_end") &&
|
|
3488
|
-
update.content.length > 0)
|
|
3489
|
-
) {
|
|
3490
|
-
this.#markRetryReplayUnsafe();
|
|
3491
|
-
}
|
|
3492
|
-
}
|
|
3493
3555
|
// Record a successful final yield before any asynchronous extension work so a
|
|
3494
3556
|
// concurrently delivered agent_end cannot start post-turn maintenance first.
|
|
3495
3557
|
if (event.type === "tool_execution_end" && event.toolName === "yield" && !event.isError) {
|
|
@@ -3531,6 +3593,11 @@ export class AgentSession {
|
|
|
3531
3593
|
this.#displayDequeueAlreadyHandled.text === userMessageText,
|
|
3532
3594
|
);
|
|
3533
3595
|
if (userDisplayDequeueAlreadyHandled) this.#displayDequeueAlreadyHandled = undefined;
|
|
3596
|
+
// A user message reaching the model ends any restricted automatic resume: from here
|
|
3597
|
+
// on the user directs the turn. Queuing alone does not lift the restriction.
|
|
3598
|
+
if (event.type === "message_start" && event.message.role === "user" && this.#restrictedToolMode) {
|
|
3599
|
+
this.#endAutoResume("skipped");
|
|
3600
|
+
}
|
|
3534
3601
|
if (event.type === "message_start" && event.message.role === "user" && !userDisplayDequeueAlreadyHandled) {
|
|
3535
3602
|
const messageText = userMessageText;
|
|
3536
3603
|
if (messageText) {
|
|
@@ -3871,17 +3938,32 @@ export class AgentSession {
|
|
|
3871
3938
|
this.#skipPostTurnMaintenanceAssistantTimestamp = assistantMsg.timestamp;
|
|
3872
3939
|
this.#suppressNextPostTurnMaintenanceAfterAutoHandoff = false;
|
|
3873
3940
|
}
|
|
3941
|
+
// An accepted (non-error) model step closes its recovery envelope; the next
|
|
3942
|
+
// model step (e.g. after tool results) starts with a fresh budget.
|
|
3943
|
+
if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") {
|
|
3944
|
+
// A tool-call step stays in_flight on purpose: until the next step's own
|
|
3945
|
+
// checkpoint supersedes it, a crash during tool execution has unknown effects
|
|
3946
|
+
// and must restart fail closed. The final text step writes the completed marker.
|
|
3947
|
+
if (!assistantMsg.content.some(block => block.type === "toolCall")) {
|
|
3948
|
+
// The completed marker is appended synchronously. A flush lost to a crash
|
|
3949
|
+
// leaves the step unresolved, which restarts fail closed (no auto-resend).
|
|
3950
|
+
void this.#completeStepRecoveryCheckpoint(assistantMsg);
|
|
3951
|
+
}
|
|
3952
|
+
this.#endStepRecoveryBudget({ accepted: true });
|
|
3953
|
+
this.#restartResumeEnvelopePending = false;
|
|
3954
|
+
}
|
|
3874
3955
|
if (
|
|
3875
3956
|
assistantMsg.stopReason !== "error" &&
|
|
3876
3957
|
assistantMsg.stopReason !== "aborted" &&
|
|
3877
3958
|
this.#retryAttempt > 0
|
|
3878
3959
|
) {
|
|
3960
|
+
const recoveredAttempt = this.#retryAttempt;
|
|
3961
|
+
this.#retryAttempt = 0;
|
|
3879
3962
|
await this.#emitSessionEvent({
|
|
3880
3963
|
type: "auto_retry_end",
|
|
3881
3964
|
success: true,
|
|
3882
|
-
attempt:
|
|
3965
|
+
attempt: recoveredAttempt,
|
|
3883
3966
|
});
|
|
3884
|
-
this.#retryAttempt = 0;
|
|
3885
3967
|
// Settle the retry gate here, colocated with the success event, rather
|
|
3886
3968
|
// than relying on the generic #resolveRetry() at the end of the
|
|
3887
3969
|
// agent_end branch. That tail resolver is bypassed by every early
|
|
@@ -3994,6 +4076,7 @@ export class AgentSession {
|
|
|
3994
4076
|
const msg = this.#lastAssistantMessage ?? fallbackAssistant;
|
|
3995
4077
|
this.#lastAssistantMessage = undefined;
|
|
3996
4078
|
if (!msg) {
|
|
4079
|
+
if (this.#activeAutoResume) this.#endAutoResume("completed");
|
|
3997
4080
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
3998
4081
|
this.#resolveRetry();
|
|
3999
4082
|
return;
|
|
@@ -4010,12 +4093,14 @@ export class AgentSession {
|
|
|
4010
4093
|
}
|
|
4011
4094
|
|
|
4012
4095
|
if (this.#skipPostTurnMaintenanceAssistantTimestamp === msg.timestamp) {
|
|
4096
|
+
if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
|
|
4013
4097
|
this.#skipPostTurnMaintenanceAssistantTimestamp = undefined;
|
|
4014
4098
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
4015
4099
|
return;
|
|
4016
4100
|
}
|
|
4017
4101
|
|
|
4018
4102
|
if (this.#assistantEndedWithSuccessfulYield(msg)) {
|
|
4103
|
+
if (this.#activeAutoResume) this.#endAutoResume("completed");
|
|
4019
4104
|
this.#lastSuccessfulYieldToolCallId = undefined;
|
|
4020
4105
|
if (msg.stopReason !== "error" && msg.stopReason !== "aborted" && (await this.#checkGoalCompletion(msg))) {
|
|
4021
4106
|
return;
|
|
@@ -4031,9 +4116,17 @@ export class AgentSession {
|
|
|
4031
4116
|
const didRetry = await this.#handleRetryableError(msg, false, transportFailure);
|
|
4032
4117
|
if (didRetry) return; // Retry was initiated, don't proceed to compaction
|
|
4033
4118
|
}
|
|
4119
|
+
// The resumed run settled: its restricted tool view ends with it.
|
|
4120
|
+
if (this.#activeAutoResume) this.#endAutoResume(msg.stopReason === "error" ? "failed" : "completed");
|
|
4121
|
+
// An interrupted step may resume once, on the same model, inside its budget.
|
|
4122
|
+
const resumeTrigger = this.#autoResumeTriggerFor(msg);
|
|
4123
|
+
const resumed = resumeTrigger !== undefined && (await this.#maybeAutoResume(resumeTrigger));
|
|
4124
|
+
// No automatic retry follows: the step's recovery envelope ends here, unless the
|
|
4125
|
+
// resume carries on inside it.
|
|
4126
|
+
if (!resumed) this.#endStepRecoveryBudget();
|
|
4034
4127
|
if (this.#retryAttempt > 0) {
|
|
4035
|
-
//
|
|
4036
|
-
// the terminal retry-end and reset so observers clear retry state.
|
|
4128
|
+
// The retry chain ended on a non-retryable (terminal) message, or handed over
|
|
4129
|
+
// to the resume: emit the terminal retry-end and reset so observers clear retry state.
|
|
4037
4130
|
const attempt = this.#retryAttempt;
|
|
4038
4131
|
this.#retryAttempt = 0;
|
|
4039
4132
|
await this.#emitSessionEvent({
|
|
@@ -4044,6 +4137,7 @@ export class AgentSession {
|
|
|
4044
4137
|
});
|
|
4045
4138
|
}
|
|
4046
4139
|
this.#resolveRetry();
|
|
4140
|
+
if (resumed) return;
|
|
4047
4141
|
|
|
4048
4142
|
const compactionTask = this.#checkCompaction(msg);
|
|
4049
4143
|
this.#trackPostPromptTask(compactionTask.then(() => undefined));
|
|
@@ -4161,6 +4255,8 @@ export class AgentSession {
|
|
|
4161
4255
|
onSkip?: (reason: "generation_changed" | "aborted_signal" | "queue_drained" | "handoff_in_progress") => void;
|
|
4162
4256
|
allowDuringCancelAndSubmit?: boolean;
|
|
4163
4257
|
onError?: (error: unknown) => void;
|
|
4258
|
+
/** Run on the current model only: no managed chain, no fallback switch. */
|
|
4259
|
+
pinnedModel?: boolean;
|
|
4164
4260
|
}): Promise<void> {
|
|
4165
4261
|
const predecessorAgentEndHold = options?.suppressPredecessorAgentEnd
|
|
4166
4262
|
? this.#reserveDeferredAgentEndForContinuation()
|
|
@@ -4234,7 +4330,7 @@ export class AgentSession {
|
|
|
4234
4330
|
this.agent.state.messages.at(-1)?.role === "assistant" && this.agent.hasQueuedMessages();
|
|
4235
4331
|
try {
|
|
4236
4332
|
await this.agent.continue({
|
|
4237
|
-
...this.#managedFallbackPromptOptions(),
|
|
4333
|
+
...(options?.pinnedModel ? {} : this.#managedFallbackPromptOptions()),
|
|
4238
4334
|
// Reset only after continue() has claimed the queued turn. Skipped or stale
|
|
4239
4335
|
// continuations retain predecessor accounting, and resetAttemptBudget keeps
|
|
4240
4336
|
// the sticky fallback cursor unchanged.
|
|
@@ -5317,6 +5413,16 @@ export class AgentSession {
|
|
|
5317
5413
|
);
|
|
5318
5414
|
}
|
|
5319
5415
|
|
|
5416
|
+
/**
|
|
5417
|
+
* Recovery safety follows the transcript, not the process: a session switch,
|
|
5418
|
+
* branch, fork, or handoff re-derives it from the persisted checkpoints and drops
|
|
5419
|
+
* in-memory uncertainty that belonged to the previous transcript.
|
|
5420
|
+
*/
|
|
5421
|
+
#resyncRecoveryStateFromTranscript(): void {
|
|
5422
|
+
this.#unresolvedRecoveryAtHydration = this.#hasUnresolvedPersistedRecovery();
|
|
5423
|
+
this.#uncertainUpstreamUnresolved = false;
|
|
5424
|
+
}
|
|
5425
|
+
|
|
5320
5426
|
#rekeyHindsightMemoryForCurrentSessionId(): void {
|
|
5321
5427
|
if (resolveMemoryBackend(this.settings).id !== "hindsight") return;
|
|
5322
5428
|
const sid = this.agent.sessionId;
|
|
@@ -6714,12 +6820,58 @@ export class AgentSession {
|
|
|
6714
6820
|
}
|
|
6715
6821
|
}
|
|
6716
6822
|
|
|
6823
|
+
#hasUnresolvedPersistedRecovery(): boolean {
|
|
6824
|
+
const branch = this.sessionManager.getBranch();
|
|
6825
|
+
const checkpoint = branch.findLast(
|
|
6826
|
+
entry => entry.type === "custom" && entry.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE,
|
|
6827
|
+
);
|
|
6828
|
+
if (checkpoint?.type !== "custom") return false;
|
|
6829
|
+
if (!checkpoint.data || typeof checkpoint.data !== "object" || Array.isArray(checkpoint.data)) return true;
|
|
6830
|
+
const data = checkpoint.data as Record<string, unknown>;
|
|
6831
|
+
if (data.version !== 1 || data.state !== "completed") return true;
|
|
6832
|
+
if (!Array.isArray(data.visibleMessageIds) || data.visibleMessageIds.length === 0) return true;
|
|
6833
|
+
const branchIds = new Set(branch.map(entry => entry.id));
|
|
6834
|
+
return !data.visibleMessageIds.every(
|
|
6835
|
+
id => typeof id === "string" && branchIds.has(id) && this.sessionManager.getEntry(id)?.type === "message",
|
|
6836
|
+
);
|
|
6837
|
+
}
|
|
6838
|
+
|
|
6839
|
+
/**
|
|
6840
|
+
* Whether startup `--continue` should hand this session to the restart resume
|
|
6841
|
+
* policy: automatic resume is on and the persisted history ends in a step a process
|
|
6842
|
+
* exit interrupted. Every other tail stays idle until the user sends a message.
|
|
6843
|
+
*/
|
|
6844
|
+
shouldResumeInterruptedStepOnStartup(): boolean {
|
|
6845
|
+
return this.settings.get("retry.autoResume") && this.#hasUnresolvedPersistedRecovery();
|
|
6846
|
+
}
|
|
6847
|
+
|
|
6717
6848
|
/** Main startup calls this exactly once, after a strict open returned `kind: "opened"`. */
|
|
6718
6849
|
async continuePersistedHistory(): Promise<void> {
|
|
6719
6850
|
this.#assertNoHandoffTransition();
|
|
6720
6851
|
this.#assertRecoveryHydrationPromoted();
|
|
6721
6852
|
this.#removeEphemeralCustomMessages();
|
|
6722
6853
|
|
|
6854
|
+
// A process that died mid-step left an unresolved recovery checkpoint. Its old
|
|
6855
|
+
// budget, deadline and in-flight request outcome are unknown and never reused:
|
|
6856
|
+
// the step resumes at most once, in a small fresh envelope, without re-running
|
|
6857
|
+
// any call whose outcome was not observed. Otherwise the user resumes it.
|
|
6858
|
+
const tail = this.agent.state.messages.at(-1);
|
|
6859
|
+
let autoResumed = false;
|
|
6860
|
+
let resumeGeneration = this.#promptGeneration;
|
|
6861
|
+
if (
|
|
6862
|
+
this.#hasUnresolvedPersistedRecovery() ||
|
|
6863
|
+
(tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE)
|
|
6864
|
+
) {
|
|
6865
|
+
const staleContinuation = tail?.role === "custom" && tail.customType === VISIBLE_CONTINUATION_CUSTOM_TYPE;
|
|
6866
|
+
autoResumed = !staleContinuation && (await this.#maybeAutoResume("restart"));
|
|
6867
|
+
resumeGeneration = this.#promptGeneration;
|
|
6868
|
+
if (!autoResumed) {
|
|
6869
|
+
throw new Error(
|
|
6870
|
+
"The previous answer was interrupted during automatic recovery. Its preserved output is shown above; send a message to continue.",
|
|
6871
|
+
);
|
|
6872
|
+
}
|
|
6873
|
+
}
|
|
6874
|
+
|
|
6723
6875
|
if (!canContinuePersistedHistory(this.agent.state.messages)) {
|
|
6724
6876
|
throw new Error("Cannot continue from persisted message history");
|
|
6725
6877
|
}
|
|
@@ -6757,13 +6909,27 @@ export class AgentSession {
|
|
|
6757
6909
|
// volatile-context/hindsight awaits above and this would otherwise start a
|
|
6758
6910
|
// turn against the session being handed off.
|
|
6759
6911
|
this.#assertNoHandoffTransition();
|
|
6912
|
+
// An abort or new prompt during the preparation above cancels the automatic
|
|
6913
|
+
// resume: it ended the restart envelope, and nothing may be sent outside it.
|
|
6914
|
+
if (autoResumed && (this.#promptGeneration !== resumeGeneration || !this.#stepRecoveryBudget)) {
|
|
6915
|
+
this.#endAutoResume("skipped");
|
|
6916
|
+
return;
|
|
6917
|
+
}
|
|
6760
6918
|
await this.agent.continue({
|
|
6761
|
-
...this.#managedFallbackPromptOptions(),
|
|
6919
|
+
...(autoResumed ? {} : this.#managedFallbackPromptOptions()),
|
|
6762
6920
|
onRunAccepted: () => {
|
|
6763
6921
|
if (hindsightRecall) hindsightState?.markRecallSnippetInjected(hindsightRecall);
|
|
6764
6922
|
},
|
|
6765
6923
|
});
|
|
6766
6924
|
await this.#waitForPostPromptRecovery();
|
|
6925
|
+
} catch (error) {
|
|
6926
|
+
// A restart resume that failed before its run started releases its envelope
|
|
6927
|
+
// and restricted view; a later manual retry or prompt must not inherit them.
|
|
6928
|
+
if (autoResumed && this.#restartResumeEnvelopePending) {
|
|
6929
|
+
this.#endAutoResume("failed");
|
|
6930
|
+
this.#endStepRecoveryBudget();
|
|
6931
|
+
}
|
|
6932
|
+
throw error;
|
|
6767
6933
|
} finally {
|
|
6768
6934
|
this.#removeEphemeralCustomMessages();
|
|
6769
6935
|
this.#endInFlight();
|
|
@@ -8073,6 +8239,17 @@ export class AgentSession {
|
|
|
8073
8239
|
let hindsightRecall: string | undefined;
|
|
8074
8240
|
try {
|
|
8075
8241
|
this.#throwIfPromptPreflightCancelled(generation, preflightSignal);
|
|
8242
|
+
// A new user turn closes any earlier step's envelope (budget, deadline timer,
|
|
8243
|
+
// stepId, uncertainty) before the replay-safety reset clears its residue.
|
|
8244
|
+
if (message.role === "user") {
|
|
8245
|
+
// A real user turn ends any automatic resume before it is processed.
|
|
8246
|
+
if (this.#activeAutoResume) this.#endAutoResume("skipped");
|
|
8247
|
+
this.#endStepRecoveryBudget();
|
|
8248
|
+
// A failed uncertainty write closed admission for the previous step only. Wait
|
|
8249
|
+
// for that step's pending write so its outcome cannot leak into this turn.
|
|
8250
|
+
await this.#pendingUncertainFlush;
|
|
8251
|
+
this.#uncertainFlushFailed = false;
|
|
8252
|
+
}
|
|
8076
8253
|
if (options?.resetRetryReplaySafety) this.#resetRetryReplaySafety();
|
|
8077
8254
|
if (message.role === "user") {
|
|
8078
8255
|
await this.#resetDefaultFallbackForNewTurn();
|
|
@@ -9256,6 +9433,8 @@ export class AgentSession {
|
|
|
9256
9433
|
this.#silentAbortPending = options?.silent === true;
|
|
9257
9434
|
this.#markRetryReplayUnsafe();
|
|
9258
9435
|
this.abortRetry();
|
|
9436
|
+
// Cancellation admits no further upstream request for the interrupted step.
|
|
9437
|
+
this.#endStepRecoveryBudget();
|
|
9259
9438
|
this.#promptGeneration++;
|
|
9260
9439
|
this.#promptPreflightAbortController.abort();
|
|
9261
9440
|
this.#promptPreflightAbortController = new AbortController();
|
|
@@ -9614,6 +9793,7 @@ export class AgentSession {
|
|
|
9614
9793
|
}
|
|
9615
9794
|
this.setTodoPhases([]);
|
|
9616
9795
|
this.#syncAgentSessionId();
|
|
9796
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9617
9797
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9618
9798
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9619
9799
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -9687,6 +9867,7 @@ export class AgentSession {
|
|
|
9687
9867
|
this.agent.reset();
|
|
9688
9868
|
this.setTodoPhases([]);
|
|
9689
9869
|
this.#syncAgentSessionId();
|
|
9870
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9690
9871
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9691
9872
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9692
9873
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -9779,6 +9960,7 @@ export class AgentSession {
|
|
|
9779
9960
|
this.sessionManager.appendContextClearEntry({ sessionId });
|
|
9780
9961
|
this.setTodoPhases([]);
|
|
9781
9962
|
this.#syncAgentSessionId(sessionId);
|
|
9963
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9782
9964
|
this.#steeringMessages = [];
|
|
9783
9965
|
this.#followUpMessages = [];
|
|
9784
9966
|
this.#pendingNextTurnMessages = [];
|
|
@@ -9848,6 +10030,7 @@ export class AgentSession {
|
|
|
9848
10030
|
throw await discardPreparedNewSessionAfterFailure(this.sessionManager, prepared, error);
|
|
9849
10031
|
}
|
|
9850
10032
|
this.#syncAgentSessionId();
|
|
10033
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
9851
10034
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
9852
10035
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
9853
10036
|
|
|
@@ -11515,6 +11698,7 @@ export class AgentSession {
|
|
|
11515
11698
|
committed = true;
|
|
11516
11699
|
this.agent.reset();
|
|
11517
11700
|
this.#syncAgentSessionId();
|
|
11701
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
11518
11702
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
11519
11703
|
this.#steeringMessages = [];
|
|
11520
11704
|
this.#followUpMessages = [];
|
|
@@ -11585,6 +11769,7 @@ export class AgentSession {
|
|
|
11585
11769
|
// were never mutated before commit, so they survive intact.
|
|
11586
11770
|
this.sessionManager.restoreState(rollbackSessionState);
|
|
11587
11771
|
this.#syncAgentSessionId(rollbackSessionState.sessionId);
|
|
11772
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
11588
11773
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
11589
11774
|
this.agent.replaceMessages(rollbackAgentMessages);
|
|
11590
11775
|
this.agent.clearAllQueues();
|
|
@@ -13702,6 +13887,7 @@ export class AgentSession {
|
|
|
13702
13887
|
}
|
|
13703
13888
|
|
|
13704
13889
|
#isRetryableError(message: AssistantMessage): boolean {
|
|
13890
|
+
if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return false;
|
|
13705
13891
|
if (this.#isTerminalProviderFirstEventTimeout(message)) return false;
|
|
13706
13892
|
if (message.errorMessage?.startsWith("Model fallback chain exhausted;")) return false;
|
|
13707
13893
|
// Already gave up on this transient failure; do not start a fresh budget.
|
|
@@ -13774,14 +13960,15 @@ export class AgentSession {
|
|
|
13774
13960
|
/**
|
|
13775
13961
|
* Whether a first-event timeout on the error's provider should fail closed —
|
|
13776
13962
|
* i.e. retry a bounded number of times (capped at retry.maxRetries) and then
|
|
13777
|
-
* surface, instead of joining the
|
|
13963
|
+
* surface, instead of joining the transient class (bounded only by the shared
|
|
13964
|
+
* step budget of 7 requests / 15 minutes).
|
|
13778
13965
|
*
|
|
13779
13966
|
* Targets the ollama-chat API, which is exclusively ollama-cloud (local
|
|
13780
13967
|
* Ollama uses the openai-responses API). That remote, queued backend can
|
|
13781
|
-
* stall before its first token even for tiny prompts;
|
|
13782
|
-
*
|
|
13783
|
-
*
|
|
13784
|
-
*
|
|
13968
|
+
* stall before its first token even for tiny prompts; each continuation retry
|
|
13969
|
+
* re-issues the full request and can spike upstream usage (#713). First-party
|
|
13970
|
+
* providers keep first-event timeouts in the transient class under the shared
|
|
13971
|
+
* step budget.
|
|
13785
13972
|
*/
|
|
13786
13973
|
#shouldFailClosedOnFirstEventTimeout(message: AssistantMessage): boolean {
|
|
13787
13974
|
// Prefer the active model's API (the model that produced the error);
|
|
@@ -13812,11 +13999,12 @@ export class AgentSession {
|
|
|
13812
13999
|
/**
|
|
13813
14000
|
* Ordered retry classification: typed safety stop (surface) -> legacy safety stop
|
|
13814
14001
|
* (surface) -> overflow (compaction) -> terminal (surface) -> usage_limit
|
|
13815
|
-
* (rotation) -> first_event_timeout (
|
|
14002
|
+
* (rotation) -> first_event_timeout (retry.maxRetries) -> transient (shared step budget) ->
|
|
13816
14003
|
* unknown (bounded retry).
|
|
13817
14004
|
*/
|
|
13818
14005
|
#classifyErrorForRetry(message: AssistantMessage): RetryErrorClassification {
|
|
13819
14006
|
if (message.stopReason !== "error") return "none";
|
|
14007
|
+
if (message.errorCode === TOOL_CALL_REENTRY_ERROR_CODE) return "terminal";
|
|
13820
14008
|
if (message.errorKind === "provider_safety_stop") return "terminal";
|
|
13821
14009
|
// A decode loop is deterministic for the submitted context: replaying the
|
|
13822
14010
|
// identical conversation re-trips the guard and re-bills the full context.
|
|
@@ -13863,10 +14051,10 @@ export class AgentSession {
|
|
|
13863
14051
|
if (this.#isTerminalProviderFirstEventTimeout(message)) {
|
|
13864
14052
|
return "terminal";
|
|
13865
14053
|
}
|
|
13866
|
-
// A first-event timeout on ollama-cloud (the ollama-chat API) must not
|
|
13867
|
-
//
|
|
13868
|
-
//
|
|
13869
|
-
//
|
|
14054
|
+
// A first-event timeout on ollama-cloud (the ollama-chat API) must not join
|
|
14055
|
+
// the transient class: each continuation retry re-issues the full request to
|
|
14056
|
+
// a remote, billable backend (#713). Bound it to retry.maxRetries instead of
|
|
14057
|
+
// the wider shared step budget.
|
|
13870
14058
|
if (this.#isFirstEventTimeoutErrorMessage(err) && this.#shouldFailClosedOnFirstEventTimeout(message)) {
|
|
13871
14059
|
return "first_event_timeout";
|
|
13872
14060
|
}
|
|
@@ -14118,8 +14306,15 @@ export class AgentSession {
|
|
|
14118
14306
|
}
|
|
14119
14307
|
|
|
14120
14308
|
async #handleManagedAttemptOutcome(outcome: ManagedAttemptOutcome): Promise<ManagedAttemptDecision> {
|
|
14309
|
+
if (outcome.type === "visible_interrupted" || outcome.type === "unsafe_interrupted") {
|
|
14310
|
+
// The session does not supply managed recovery evidence, so a committed managed
|
|
14311
|
+
// failure is never replayed: preserve it and stop for an explicit user resume.
|
|
14312
|
+
this.#endStepRecoveryBudget();
|
|
14313
|
+
return { type: "pause", reason: outcome.evidence.tail };
|
|
14314
|
+
}
|
|
14121
14315
|
if (outcome.type === "run_terminal") {
|
|
14122
14316
|
this.#defaultFallbackChain().resetAttemptBudget();
|
|
14317
|
+
this.#endStepRecoveryBudget();
|
|
14123
14318
|
return { type: "terminal", terminal: { stopReason: outcome.reason } };
|
|
14124
14319
|
}
|
|
14125
14320
|
if (outcome.type === "context_overflow_discarded") {
|
|
@@ -14205,7 +14400,7 @@ export class AgentSession {
|
|
|
14205
14400
|
if (transport.class !== "other") return transport;
|
|
14206
14401
|
// Managed fallback receives authoritative transport facts from the request
|
|
14207
14402
|
// boundary. Once those facts classify as other, error prose must not upgrade
|
|
14208
|
-
// the failure into
|
|
14403
|
+
// the failure into a transient or quota retry.
|
|
14209
14404
|
if (transportFailure) return { class: "unknown" };
|
|
14210
14405
|
const classification = this.#classifyErrorForRetry(message);
|
|
14211
14406
|
if (allowLegacyUsageLimit && classification === "usage_limit") {
|
|
@@ -14309,22 +14504,72 @@ export class AgentSession {
|
|
|
14309
14504
|
async #markFailedManagedCredential(trigger: {
|
|
14310
14505
|
class: FallbackTriggerClass;
|
|
14311
14506
|
retryAfterMs?: number;
|
|
14507
|
+
}): Promise<boolean> {
|
|
14508
|
+
try {
|
|
14509
|
+
return await this.#markFailedManagedCredentialAdmitted(trigger);
|
|
14510
|
+
} catch (error) {
|
|
14511
|
+
// A refused admission ends maintenance locally; the next inference admission
|
|
14512
|
+
// reports the same budget verdict instead of this side path throwing.
|
|
14513
|
+
if (error instanceof RecoveryAdmissionError) return false;
|
|
14514
|
+
throw error;
|
|
14515
|
+
}
|
|
14516
|
+
}
|
|
14517
|
+
|
|
14518
|
+
async #markFailedManagedCredentialAdmitted(trigger: {
|
|
14519
|
+
class: FallbackTriggerClass;
|
|
14520
|
+
retryAfterMs?: number;
|
|
14312
14521
|
}): Promise<boolean> {
|
|
14313
14522
|
if (!this.model || (trigger.class !== "auth" && trigger.class !== "quota" && trigger.class !== "rate_limit")) {
|
|
14314
14523
|
return false;
|
|
14315
14524
|
}
|
|
14316
14525
|
const authStorage = this.#modelRegistry.authStorage;
|
|
14526
|
+
const requestOptions = this.#stepCredentialRequestOptions();
|
|
14527
|
+
// Recovery maintenance outside a step budget would send unadmitted credential HTTP.
|
|
14528
|
+
if (!requestOptions.fetch) return false;
|
|
14317
14529
|
if (trigger.class === "auth") {
|
|
14318
|
-
const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
|
|
14530
|
+
const apiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
|
|
14319
14531
|
if (!isAuthenticated(apiKey)) return false;
|
|
14320
|
-
return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, {
|
|
14532
|
+
return authStorage.invalidateCredentialMatching(this.model.provider, apiKey, {
|
|
14533
|
+
...requestOptions,
|
|
14534
|
+
sessionId: this.sessionId,
|
|
14535
|
+
});
|
|
14321
14536
|
}
|
|
14322
14537
|
if (authStorage.hasRuntimeApiKey(this.model.provider)) return false;
|
|
14323
|
-
const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId);
|
|
14538
|
+
const activeApiKey = await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions);
|
|
14324
14539
|
const rotated = await authStorage.markUsageLimitReached(this.model.provider, this.sessionId, {
|
|
14325
14540
|
retryAfterMs: trigger.retryAfterMs,
|
|
14541
|
+
...requestOptions,
|
|
14326
14542
|
});
|
|
14327
|
-
return
|
|
14543
|
+
return (
|
|
14544
|
+
rotated && (await this.#modelRegistry.getApiKey(this.model, this.sessionId, requestOptions)) !== activeApiKey
|
|
14545
|
+
);
|
|
14546
|
+
}
|
|
14547
|
+
|
|
14548
|
+
/**
|
|
14549
|
+
* Credential maintenance during recovery (rotation, restore, usage probes) spends
|
|
14550
|
+
* the same step budget as inference: every concrete token/usage HTTP is admitted
|
|
14551
|
+
* as kind "token", and opaque remote work marks the step unsafe.
|
|
14552
|
+
*/
|
|
14553
|
+
#stepCredentialRequestOptions(): OAuthRefreshOptions {
|
|
14554
|
+
const budget = this.#stepRecoveryBudget;
|
|
14555
|
+
const deadlineAbort = this.#stepRecoveryDeadlineAbort;
|
|
14556
|
+
if (!budget || !deadlineAbort) return {};
|
|
14557
|
+
const validate = (): void => {
|
|
14558
|
+
if (this.#stepRecoveryBudget !== budget) throw new RecoveryAdmissionError("cancelled");
|
|
14559
|
+
budget.assertActive();
|
|
14560
|
+
};
|
|
14561
|
+
const admission = createUpstreamAdmission(
|
|
14562
|
+
kind => budget.reserve(kind),
|
|
14563
|
+
undefined,
|
|
14564
|
+
validate,
|
|
14565
|
+
() => this.#recordStepRecoveryFailure(budget),
|
|
14566
|
+
);
|
|
14567
|
+
return {
|
|
14568
|
+
signal: deadlineAbort.signal,
|
|
14569
|
+
fetch: admission.credentialFetch,
|
|
14570
|
+
onUpstreamRequest: admission.onUpstreamRequest,
|
|
14571
|
+
onUncertainUpstream: reason => this.#recordStepUncertainUpstream(budget, reason),
|
|
14572
|
+
};
|
|
14328
14573
|
}
|
|
14329
14574
|
|
|
14330
14575
|
/** Handle retryable errors with exponential backoff. */
|
|
@@ -14336,6 +14581,14 @@ export class AgentSession {
|
|
|
14336
14581
|
const controller = this.#defaultFallbackChain();
|
|
14337
14582
|
const managedFallback = controller.chain.entries.length > 1;
|
|
14338
14583
|
const retrySettings = this.settings.getGroup("retry");
|
|
14584
|
+
// A rejected admission is the shared budget's own terminal answer, never a retry trigger.
|
|
14585
|
+
if (isRecoveryAdmissionErrorMessage(message.errorMessage)) {
|
|
14586
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14587
|
+
}
|
|
14588
|
+
// Broker/custom credential work with unobserved remote effects is never resent automatically.
|
|
14589
|
+
if (this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14590
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14591
|
+
}
|
|
14339
14592
|
const legacyRetryConfigured =
|
|
14340
14593
|
this.settings.has("retry.enabled") ||
|
|
14341
14594
|
this.settings.has("retry.maxRetries") ||
|
|
@@ -14345,14 +14598,37 @@ export class AgentSession {
|
|
|
14345
14598
|
// user opt-out.
|
|
14346
14599
|
if (!managedFallback && !retrySettings.enabled) return false;
|
|
14347
14600
|
const classification = managedFallback ? undefined : this.#classifyErrorForRetry(message);
|
|
14348
|
-
//
|
|
14601
|
+
// A failure after public output is never replayed by deleting that output.
|
|
14602
|
+
// A text-only tail on a verified serializer continues on the same model;
|
|
14603
|
+
// any other public tail (tool calls, images, tainted run) stops with the
|
|
14604
|
+
// output preserved for the user to resume explicitly. A failure that streamed
|
|
14605
|
+
// only reasoning holds no answer and its signed blocks cannot be replayed: it
|
|
14606
|
+
// is dropped and retried like a failure that showed nothing.
|
|
14607
|
+
const thinkingOnlyFailure = !managedOutcome && isThinkingOnlyFailure(message);
|
|
14608
|
+
const visibleFailure =
|
|
14609
|
+
!managedOutcome && !thinkingOnlyFailure && assistantMessageHasVisibleOrToolContent(message);
|
|
14610
|
+
const visibleContinuation = visibleFailure && this.#canContinueVisibleFailure(message);
|
|
14611
|
+
if (visibleFailure && !visibleContinuation) return false;
|
|
14612
|
+
// Bare defaults admit only clean canonical stream watchdog failures, plus a
|
|
14613
|
+
// transient server failure that streamed only reasoning (e.g. an SSE
|
|
14614
|
+
// overloaded_error after thinking_delta): the provider's own pre-content
|
|
14615
|
+
// retry cannot cover it, and it has no answer to preserve. Completed tool
|
|
14616
|
+
// pairs and preserved visible output stay in the transcript and are never
|
|
14617
|
+
// re-executed; only hooks/commands/user shell/abort make the run unsafe.
|
|
14349
14618
|
if (!managedFallback && !legacyRetryConfigured) {
|
|
14350
|
-
|
|
14351
|
-
hasBareDefaultRetryDisqualifyingFacts(message)
|
|
14352
|
-
(classification
|
|
14353
|
-
|
|
14354
|
-
|
|
14355
|
-
|
|
14619
|
+
const bareWatchdog =
|
|
14620
|
+
!hasBareDefaultRetryDisqualifyingFacts(message) &&
|
|
14621
|
+
(classification === "transient" || classification === "first_event_timeout") &&
|
|
14622
|
+
BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "");
|
|
14623
|
+
const bareThinkingOnly =
|
|
14624
|
+
thinkingOnlyFailure &&
|
|
14625
|
+
classification === "transient" &&
|
|
14626
|
+
message.errorKind === undefined &&
|
|
14627
|
+
!BARE_DEFAULT_THROTTLE_ERROR.test(message.errorMessage ?? "") &&
|
|
14628
|
+
(BARE_DEFAULT_SERVER_OVERLOAD_ERROR.test(message.errorMessage ?? "") ||
|
|
14629
|
+
isServerHttpStatus(this.#extractExplicitHttpStatusFromErrorMessage(message.errorMessage ?? ""))) &&
|
|
14630
|
+
(!transportFailure || classifyFallbackTrigger(transportFailure).class === "server");
|
|
14631
|
+
if (!(bareWatchdog || bareThinkingOnly) || !this.#hasCleanContinuationSafety) {
|
|
14356
14632
|
return false;
|
|
14357
14633
|
}
|
|
14358
14634
|
}
|
|
@@ -14362,8 +14638,29 @@ export class AgentSession {
|
|
|
14362
14638
|
? this.#managedFallbackExhaustionDecision(message, message.errorMessage || "Model fallback attempt failed")
|
|
14363
14639
|
: false;
|
|
14364
14640
|
}
|
|
14365
|
-
const
|
|
14366
|
-
|
|
14641
|
+
const transientTrigger = trigger.class === "server" || trigger.class === "rate_limit";
|
|
14642
|
+
// Session retries and per-selector attempts are different units. Respect an
|
|
14643
|
+
// explicit additional-session-retry cap before credential rotation or replay.
|
|
14644
|
+
if (
|
|
14645
|
+
transientTrigger &&
|
|
14646
|
+
(!retrySettings.enabled ||
|
|
14647
|
+
this.#retryAttempt >= (this.settings.has("retry.maxRetries") ? retrySettings.maxRetries : 6))
|
|
14648
|
+
) {
|
|
14649
|
+
this.#transientStopTimestamp = message.timestamp;
|
|
14650
|
+
return managedOutcome ? { type: "terminal", terminal: { stopReason: "error", messages: [message] } } : false;
|
|
14651
|
+
}
|
|
14652
|
+
if (visibleContinuation && !transientTrigger) return false;
|
|
14653
|
+
// Default policy: transient same-model failures are bounded by the shared step
|
|
14654
|
+
// budget (7 requests / 15 minutes after the first failure) instead of an
|
|
14655
|
+
// unbounded loop or a per-entry cap. Explicit retry.maxRetries/fallback.maxAttempts
|
|
14656
|
+
// keep their own attempt units and also stay inside the shared budget.
|
|
14657
|
+
const sharedBudgetOwnsTransient = managedFallback
|
|
14658
|
+
? !this.settings.has("fallback.maxAttempts")
|
|
14659
|
+
: classification === "transient" && !this.settings.has("retry.maxRetries");
|
|
14660
|
+
controller.sessionBudgetOwnsTransient = managedFallback && sharedBudgetOwnsTransient;
|
|
14661
|
+
const sharedTransientRetry = sharedBudgetOwnsTransient && transientTrigger;
|
|
14662
|
+
const attemptsUsed =
|
|
14663
|
+
managedFallback && !sharedTransientRetry ? controller.attemptsUsed || 1 : this.#retryAttempt + 1;
|
|
14367
14664
|
const failedSelector = managedFallback ? controller.currentSelector() : undefined;
|
|
14368
14665
|
const usageLimited = trigger.class === "quota" || trigger.class === "rate_limit";
|
|
14369
14666
|
let credentialMarked = false;
|
|
@@ -14374,15 +14671,64 @@ export class AgentSession {
|
|
|
14374
14671
|
// leave the model while another account can still serve it.
|
|
14375
14672
|
credentialMarked = true;
|
|
14376
14673
|
credentialRotated = await this.#markFailedManagedCredential(trigger);
|
|
14377
|
-
|
|
14378
|
-
|
|
14379
|
-
|
|
14380
|
-
|
|
14381
|
-
|
|
14382
|
-
|
|
14674
|
+
if (this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14675
|
+
return managedOutcome
|
|
14676
|
+
? { type: "terminal", terminal: { stopReason: "error", messages: [message] } }
|
|
14677
|
+
: false;
|
|
14678
|
+
}
|
|
14679
|
+
}
|
|
14680
|
+
// A resumed run stays on the interrupted model. On a managed chain only the shared
|
|
14681
|
+
// same-model transient path may follow; any other failure stops before the chain is
|
|
14682
|
+
// consulted, because consulting it may already move its cursor to the next model.
|
|
14683
|
+
const leavesResumedModel =
|
|
14684
|
+
this.#activeAutoResume !== undefined && managedFallback && !(sharedBudgetOwnsTransient && transientTrigger);
|
|
14685
|
+
let outcome = leavesResumedModel
|
|
14686
|
+
? "stop"
|
|
14687
|
+
: managedFallback
|
|
14688
|
+
? controller.onAttemptFailure(trigger.class, message.errorMessage || "Unknown error", { credentialRotated })
|
|
14689
|
+
: sharedBudgetOwnsTransient || attemptsUsed <= retrySettings.maxRetries
|
|
14690
|
+
? "retry"
|
|
14691
|
+
: "exhausted";
|
|
14383
14692
|
if (managedFallback && credentialRotated && outcome === "advance" && controller.restorePreviousEntryForRetry()) {
|
|
14384
14693
|
outcome = "retry";
|
|
14385
14694
|
}
|
|
14695
|
+
const stepBudget = this.#stepRecoveryBudget;
|
|
14696
|
+
if (outcome === "retry" && stepBudget && (transientTrigger || !managedFallback))
|
|
14697
|
+
this.#recordStepRecoveryFailure(stepBudget);
|
|
14698
|
+
// Typed transport Retry-After is an uncapped floor; a floor beyond the remaining
|
|
14699
|
+
// deadline stops locally rather than contacting upstream early. A hint parsed from
|
|
14700
|
+
// legacy error prose is advisory and stays capped at retry.maxDelayMs.
|
|
14701
|
+
const parsedRetryAfterMs = managedFallback
|
|
14702
|
+
? undefined
|
|
14703
|
+
: this.#parseRetryAfterMsFromError(message.errorMessage || "Unknown error");
|
|
14704
|
+
const retryAfterMs =
|
|
14705
|
+
trigger.retryAfterMs ??
|
|
14706
|
+
(parsedRetryAfterMs === undefined ? undefined : Math.min(parsedRetryAfterMs, retrySettings.maxDelayMs));
|
|
14707
|
+
const backoffBaseMs =
|
|
14708
|
+
sharedTransientRetry && !this.settings.has("retry.baseDelayMs")
|
|
14709
|
+
? SAME_MODEL_RECOVERY_BASE_DELAY_MS
|
|
14710
|
+
: retrySettings.baseDelayMs;
|
|
14711
|
+
const backoffMaxMs =
|
|
14712
|
+
sharedTransientRetry && !this.settings.has("retry.maxDelayMs")
|
|
14713
|
+
? SAME_MODEL_RECOVERY_MAX_DELAY_MS
|
|
14714
|
+
: retrySettings.maxDelayMs;
|
|
14715
|
+
const delayMs =
|
|
14716
|
+
credentialRotated || outcome === "advance"
|
|
14717
|
+
? 0
|
|
14718
|
+
: managedFallback || transientTrigger
|
|
14719
|
+
? effectiveFallbackDelay(backoffBaseMs, backoffMaxMs, attemptsUsed, retryAfterMs)
|
|
14720
|
+
: retryAfterMs !== undefined
|
|
14721
|
+
? Math.min(retryAfterMs, retrySettings.maxDelayMs)
|
|
14722
|
+
: cappedExponentialWithFullJitter(backoffBaseMs, backoffMaxMs, attemptsUsed);
|
|
14723
|
+
if (outcome === "retry" && stepBudget && !stepBudget.canAdmitAfter(delayMs)) {
|
|
14724
|
+
// The shared request/time budget cannot admit another same-model request
|
|
14725
|
+
// after the required wait: stop on the same model with the real provider
|
|
14726
|
+
// error instead of sending it. Never switch models for it.
|
|
14727
|
+
outcome = managedFallback && !transientTrigger ? "exhausted" : "stop";
|
|
14728
|
+
}
|
|
14729
|
+
if (outcome === "exhausted" && !managedFallback && sharedBudgetOwnsTransient) outcome = "stop";
|
|
14730
|
+
// retry.enabled=false is an explicit opt-out of automatic same-model transient recovery.
|
|
14731
|
+
if (outcome === "retry" && transientTrigger && !retrySettings.enabled) outcome = "stop";
|
|
14386
14732
|
if (outcome === "stop") {
|
|
14387
14733
|
// Transient failure past the same-model retry budget: surface the real
|
|
14388
14734
|
// provider error and keep the current model; never switch models for it.
|
|
@@ -14413,16 +14759,6 @@ export class AgentSession {
|
|
|
14413
14759
|
|
|
14414
14760
|
const generation = this.#promptGeneration;
|
|
14415
14761
|
const errorMessage = message.errorMessage || "Unknown error";
|
|
14416
|
-
const retryAfterMs =
|
|
14417
|
-
trigger.retryAfterMs ?? (managedFallback ? undefined : this.#parseRetryAfterMsFromError(errorMessage));
|
|
14418
|
-
const delayMs =
|
|
14419
|
-
credentialRotated || outcome === "advance"
|
|
14420
|
-
? 0
|
|
14421
|
-
: managedFallback
|
|
14422
|
-
? effectiveFallbackDelay(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed, retryAfterMs)
|
|
14423
|
-
: retryAfterMs !== undefined
|
|
14424
|
-
? Math.min(retryAfterMs, retrySettings.maxDelayMs)
|
|
14425
|
-
: cappedExponentialWithFullJitter(retrySettings.baseDelayMs, retrySettings.maxDelayMs, attemptsUsed);
|
|
14426
14762
|
|
|
14427
14763
|
// Only a spent quota window hides the model from later resolution; a burst
|
|
14428
14764
|
// rate limit retries the same model and must not push later turns elsewhere.
|
|
@@ -14474,15 +14810,24 @@ export class AgentSession {
|
|
|
14474
14810
|
await this.#emitSessionEvent({
|
|
14475
14811
|
type: "auto_retry_start",
|
|
14476
14812
|
attempt: this.#retryAttempt,
|
|
14477
|
-
maxAttempts:
|
|
14813
|
+
maxAttempts:
|
|
14814
|
+
sharedBudgetOwnsTransient && transientTrigger
|
|
14815
|
+
? SAME_MODEL_RECOVERY_MAX_REQUESTS - 1
|
|
14816
|
+
: managedFallback
|
|
14817
|
+
? controller.maxAttempts
|
|
14818
|
+
: retrySettings.maxRetries,
|
|
14478
14819
|
delayMs,
|
|
14479
14820
|
errorMessage,
|
|
14480
|
-
unbounded:
|
|
14821
|
+
unbounded: false,
|
|
14481
14822
|
});
|
|
14482
14823
|
|
|
14483
|
-
|
|
14484
|
-
|
|
14485
|
-
|
|
14824
|
+
// A preserved public prefix is never deleted or re-emitted; it is continued
|
|
14825
|
+
// after the backoff instead (see below).
|
|
14826
|
+
if (!visibleContinuation) {
|
|
14827
|
+
const messages = this.agent.state.messages;
|
|
14828
|
+
if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
|
|
14829
|
+
this.agent.replaceMessages(messages.slice(0, -1));
|
|
14830
|
+
}
|
|
14486
14831
|
}
|
|
14487
14832
|
|
|
14488
14833
|
try {
|
|
@@ -14522,6 +14867,19 @@ export class AgentSession {
|
|
|
14522
14867
|
if (this.#retryAbortController === retryAbortController) this.#retryAbortController = undefined;
|
|
14523
14868
|
this.#retryNowRequested = false;
|
|
14524
14869
|
|
|
14870
|
+
if (visibleContinuation) {
|
|
14871
|
+
// Owner re-check before any durable write: a newer prompt supersedes it.
|
|
14872
|
+
if (this.#promptGeneration !== generation) {
|
|
14873
|
+
this.#failRetryRecovery("Retry continuation was superseded");
|
|
14874
|
+
return;
|
|
14875
|
+
}
|
|
14876
|
+
if (!(await this.#appendVisibleContinuation(message))) {
|
|
14877
|
+
this.#transientStopTimestamp = message.timestamp;
|
|
14878
|
+
this.#failRetryRecovery("Recovery checkpoint could not be saved");
|
|
14879
|
+
return;
|
|
14880
|
+
}
|
|
14881
|
+
}
|
|
14882
|
+
|
|
14525
14883
|
if (managedOutcome) {
|
|
14526
14884
|
try {
|
|
14527
14885
|
await this.#checkEstimatedContextBeforePrompt();
|
|
@@ -14560,6 +14918,7 @@ export class AgentSession {
|
|
|
14560
14918
|
delayMs: 1,
|
|
14561
14919
|
generation,
|
|
14562
14920
|
allowDuringCancelAndSubmit: true,
|
|
14921
|
+
shouldContinue: () => !visibleContinuation || this.#hasVisibleRecoveryReferences(),
|
|
14563
14922
|
onError: () => this.#failRetryRecovery("Retry continuation failed to start"),
|
|
14564
14923
|
onSkip: () => this.#failRetryRecovery("Retry continuation was superseded"),
|
|
14565
14924
|
});
|
|
@@ -14580,6 +14939,580 @@ export class AgentSession {
|
|
|
14580
14939
|
this.#resolveRetry();
|
|
14581
14940
|
}
|
|
14582
14941
|
|
|
14942
|
+
/**
|
|
14943
|
+
* Single admission point for every concrete upstream request of the current
|
|
14944
|
+
* model step. Rejects (throws {@link RecoveryAdmissionError}) once the shared
|
|
14945
|
+
* same-model budget is spent or past its deadline, before anything is sent.
|
|
14946
|
+
*/
|
|
14947
|
+
#getUpstreamRequestLease: NonNullable<AgentLoopConfig["getUpstreamRequestLease"]> = async (context, signal) => {
|
|
14948
|
+
if (
|
|
14949
|
+
!this.#stepRecoveryBudget &&
|
|
14950
|
+
this.#unresolvedRecoveryAtHydration &&
|
|
14951
|
+
context.messages.at(-1)?.role !== "user"
|
|
14952
|
+
) {
|
|
14953
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14954
|
+
}
|
|
14955
|
+
// A restart resume's first step runs only inside its own small envelope: once that
|
|
14956
|
+
// envelope is gone before the step was accepted (abort, new prompt), no default
|
|
14957
|
+
// budget may take its place.
|
|
14958
|
+
if (!this.#stepRecoveryBudget && this.#restartResumeEnvelopePending) {
|
|
14959
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14960
|
+
}
|
|
14961
|
+
// The durable record of in-step uncertainty must land before anything else is sent.
|
|
14962
|
+
await this.#pendingUncertainFlush;
|
|
14963
|
+
if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
|
|
14964
|
+
// Unobservable remote work already happened in this step: no further upstream request
|
|
14965
|
+
// for it, whichever path (retry, rotation, fallback) asks next.
|
|
14966
|
+
if (this.#stepRecoveryBudget && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
14967
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14968
|
+
}
|
|
14969
|
+
const generation = this.#promptGeneration;
|
|
14970
|
+
this.#stepRecoveryBudget ??= new RecoveryBudget({ maxSilentTimeouts: this.#maxSilentTimeouts() });
|
|
14971
|
+
this.#stepRecoveryDeadlineAbort ??= new AbortController();
|
|
14972
|
+
this.#stepRecoveryStepId ??= crypto.randomUUID();
|
|
14973
|
+
const budget = this.#stepRecoveryBudget;
|
|
14974
|
+
const deadlineAbort = this.#stepRecoveryDeadlineAbort;
|
|
14975
|
+
const stepId = this.#stepRecoveryStepId;
|
|
14976
|
+
const model = this.model;
|
|
14977
|
+
const validate = (): void => {
|
|
14978
|
+
if (
|
|
14979
|
+
signal?.aborted ||
|
|
14980
|
+
this.#isDisposed ||
|
|
14981
|
+
this.#promptGeneration !== generation ||
|
|
14982
|
+
this.#stepRecoveryBudget !== budget ||
|
|
14983
|
+
this.model !== model
|
|
14984
|
+
) {
|
|
14985
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
14986
|
+
}
|
|
14987
|
+
budget.assertActive();
|
|
14988
|
+
};
|
|
14989
|
+
validate();
|
|
14990
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
14991
|
+
version: 1,
|
|
14992
|
+
state: "in_flight",
|
|
14993
|
+
logicalRunId: String(this.agent.currentManagedLogicalRunId ?? `${this.sessionId}:${generation}`),
|
|
14994
|
+
stepId,
|
|
14995
|
+
generation,
|
|
14996
|
+
attemptId: String(this.#retryAttempt),
|
|
14997
|
+
selector: model ? `${model.provider}/${model.id}` : undefined,
|
|
14998
|
+
api: model?.api,
|
|
14999
|
+
transport: model?.transport,
|
|
15000
|
+
uncertain: [...this.#stepRecoveryUncertainReasons],
|
|
15001
|
+
// Only this user turn's already-visible assistant output belongs to the step.
|
|
15002
|
+
visibleMessageIds: context.messages
|
|
15003
|
+
.slice(
|
|
15004
|
+
Math.max(
|
|
15005
|
+
0,
|
|
15006
|
+
context.messages.findLastIndex(message => message.role === "user"),
|
|
15007
|
+
),
|
|
15008
|
+
)
|
|
15009
|
+
.filter(message => message.role === "assistant")
|
|
15010
|
+
.map(getSessionMessageEntryId)
|
|
15011
|
+
.filter((id): id is string => id !== undefined),
|
|
15012
|
+
usedRequests: budget.snapshot().usedRequests,
|
|
15013
|
+
});
|
|
15014
|
+
await this.sessionManager.ensureOnDisk();
|
|
15015
|
+
await this.sessionManager.flush();
|
|
15016
|
+
validate();
|
|
15017
|
+
return {
|
|
15018
|
+
validate,
|
|
15019
|
+
signal: deadlineAbort.signal,
|
|
15020
|
+
// Records only; the identity check lives in #recordStepRecoveryFailure so the
|
|
15021
|
+
// original response/error is never replaced by an admission error.
|
|
15022
|
+
onRecoverableFailure: () => this.#recordStepRecoveryFailure(budget),
|
|
15023
|
+
onUncertainUpstream: (reason: string): void => this.#recordStepUncertainUpstream(budget, reason),
|
|
15024
|
+
onAttemptSettled: (message: AssistantMessage): void => this.#recordStepAttemptSettled(budget, message),
|
|
15025
|
+
onRequest: (kind: RecoveryRequestKind): void => {
|
|
15026
|
+
validate();
|
|
15027
|
+
if (this.#uncertainFlushFailed) throw new RecoveryAdmissionError("cancelled");
|
|
15028
|
+
// Unobservable remote work was reported inside this invocation: a resend could
|
|
15029
|
+
// duplicate it, so only the first concrete request of each kind remains admissible.
|
|
15030
|
+
if (kind === "resend" && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
15031
|
+
throw new RecoveryAdmissionError("cancelled");
|
|
15032
|
+
}
|
|
15033
|
+
if (kind === "resend") this.#recordStepRecoveryFailure(budget);
|
|
15034
|
+
budget.reserve(kind);
|
|
15035
|
+
},
|
|
15036
|
+
};
|
|
15037
|
+
};
|
|
15038
|
+
|
|
15039
|
+
async #completeStepRecoveryCheckpoint(message: AssistantMessage): Promise<void> {
|
|
15040
|
+
if (!this.#stepRecoveryStepId || this.agent.state.messages.at(-1) !== message) return;
|
|
15041
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15042
|
+
version: 1,
|
|
15043
|
+
state: "completed",
|
|
15044
|
+
stepId: this.#stepRecoveryStepId,
|
|
15045
|
+
generation: this.#promptGeneration,
|
|
15046
|
+
visibleMessageIds: [getSessionMessageEntryId(message)].filter((id): id is string => id !== undefined),
|
|
15047
|
+
});
|
|
15048
|
+
try {
|
|
15049
|
+
await this.sessionManager.ensureOnDisk();
|
|
15050
|
+
await this.sessionManager.flush();
|
|
15051
|
+
} catch (error) {
|
|
15052
|
+
logger.warn("Recovery completion checkpoint failed", {
|
|
15053
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15054
|
+
});
|
|
15055
|
+
}
|
|
15056
|
+
}
|
|
15057
|
+
|
|
15058
|
+
/**
|
|
15059
|
+
* Unobservable remote work cannot be proven idempotent: no automatic resend/continuation.
|
|
15060
|
+
* The reason is also persisted with the step so a restart never resumes it; the next
|
|
15061
|
+
* admission and every resume decision wait for that write.
|
|
15062
|
+
*/
|
|
15063
|
+
#recordStepUncertainUpstream(budget: RecoveryBudget, reason: string): void {
|
|
15064
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15065
|
+
if (!this.#stepRecoveryUncertainReasons.includes(reason)) this.#stepRecoveryUncertainReasons.push(reason);
|
|
15066
|
+
this.#markRetryReplayUnsafe();
|
|
15067
|
+
const model = this.model;
|
|
15068
|
+
this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15069
|
+
version: 1,
|
|
15070
|
+
state: "in_flight",
|
|
15071
|
+
stepId: this.#stepRecoveryStepId,
|
|
15072
|
+
generation: this.#promptGeneration,
|
|
15073
|
+
selector: model ? `${model.provider}/${model.id}` : undefined,
|
|
15074
|
+
api: model?.api,
|
|
15075
|
+
transport: model?.transport,
|
|
15076
|
+
uncertain: [...this.#stepRecoveryUncertainReasons],
|
|
15077
|
+
visibleMessageIds: [],
|
|
15078
|
+
usedRequests: budget.snapshot().usedRequests,
|
|
15079
|
+
});
|
|
15080
|
+
this.#pendingUncertainFlush = this.#pendingUncertainFlush
|
|
15081
|
+
.then(async () => {
|
|
15082
|
+
await this.sessionManager.ensureOnDisk();
|
|
15083
|
+
await this.sessionManager.flush();
|
|
15084
|
+
})
|
|
15085
|
+
.catch(error => {
|
|
15086
|
+
this.#uncertainFlushFailed = true;
|
|
15087
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
15088
|
+
logger.warn("Uncertain upstream checkpoint failed", { error: reason });
|
|
15089
|
+
this.emitNotice(
|
|
15090
|
+
"warning",
|
|
15091
|
+
`Stopped retrying this step: its recovery state could not be saved (${reason}).`,
|
|
15092
|
+
"retry",
|
|
15093
|
+
);
|
|
15094
|
+
});
|
|
15095
|
+
}
|
|
15096
|
+
|
|
15097
|
+
#maxSilentTimeouts(): number {
|
|
15098
|
+
return Math.min(7, Math.max(1, Math.trunc(this.settings.get("retry.maxSilentTimeouts")) || 3));
|
|
15099
|
+
}
|
|
15100
|
+
|
|
15101
|
+
/** Consecutive no-output timeouts close the step early, inside its request budget. */
|
|
15102
|
+
#recordStepAttemptSettled(budget: RecoveryBudget, message: AssistantMessage): void {
|
|
15103
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15104
|
+
const progressed = assistantMessageHasVisibleOrToolContent(message);
|
|
15105
|
+
const silentTimeout =
|
|
15106
|
+
message.stopReason === "error" && BARE_DEFAULT_WATCHDOG_ERROR.test(message.errorMessage ?? "");
|
|
15107
|
+
if (!budget.recordAttemptOutcome({ silentTimeout, progressed, selector: `${message.provider}/${message.model}` }))
|
|
15108
|
+
return;
|
|
15109
|
+
logger.info("silent_stall_tripped", { count: budget.snapshot().silentTimeouts });
|
|
15110
|
+
this.emitNotice(
|
|
15111
|
+
"warning",
|
|
15112
|
+
`Stopped after ${budget.snapshot().silentTimeouts} consecutive timeouts without any output.`,
|
|
15113
|
+
"retry",
|
|
15114
|
+
);
|
|
15115
|
+
}
|
|
15116
|
+
|
|
15117
|
+
#recordStepRecoveryFailure(budget: RecoveryBudget): void {
|
|
15118
|
+
if (this.#stepRecoveryBudget !== budget) return;
|
|
15119
|
+
budget.recordRecoverableFailure();
|
|
15120
|
+
if (this.#cancelStepRecoveryDeadline) return;
|
|
15121
|
+
const deadline = budget.snapshot().deadlineAtMs;
|
|
15122
|
+
if (deadline === undefined) return;
|
|
15123
|
+
const controller = this.#stepRecoveryDeadlineAbort;
|
|
15124
|
+
const timer = setTimeout(
|
|
15125
|
+
() => controller?.abort(new RecoveryAdmissionError("deadline")),
|
|
15126
|
+
Math.max(0, deadline - performance.now()),
|
|
15127
|
+
);
|
|
15128
|
+
this.#cancelStepRecoveryDeadline = () => clearTimeout(timer);
|
|
15129
|
+
}
|
|
15130
|
+
|
|
15131
|
+
/** Ends and permanently closes this lease; late callbacks never start a new budget. */
|
|
15132
|
+
#endStepRecoveryBudget(options?: { accepted?: boolean }): void {
|
|
15133
|
+
// Only an unfinished step leaves unobservable remote effects unresolved.
|
|
15134
|
+
if (!options?.accepted && this.#stepRecoveryUncertainReasons.length > 0) {
|
|
15135
|
+
this.#uncertainUpstreamUnresolved = true;
|
|
15136
|
+
}
|
|
15137
|
+
this.#stepRecoveryBudget?.cancel();
|
|
15138
|
+
this.#cancelStepRecoveryDeadline?.();
|
|
15139
|
+
this.#cancelStepRecoveryDeadline = undefined;
|
|
15140
|
+
this.#stepRecoveryDeadlineAbort = undefined;
|
|
15141
|
+
this.#stepRecoveryStepId = undefined;
|
|
15142
|
+
this.#visibleRecoveryReferenceIds = [];
|
|
15143
|
+
this.#visibleRecoveryContextIds = [];
|
|
15144
|
+
this.#stepRecoveryUncertainReasons = [];
|
|
15145
|
+
this.#stepRecoveryBudget = undefined;
|
|
15146
|
+
}
|
|
15147
|
+
|
|
15148
|
+
/**
|
|
15149
|
+
* Whether a failed step that already showed output can be continued on the same
|
|
15150
|
+
* model instead of replayed: the failure must be the live trailing assistant,
|
|
15151
|
+
* its public tail must be text (no tool calls/images), the serializer must have
|
|
15152
|
+
* a verified projection, and nothing outside the transcript tainted the run.
|
|
15153
|
+
*/
|
|
15154
|
+
#canContinueVisibleFailure(message: AssistantMessage): boolean {
|
|
15155
|
+
if (message.stopReason !== "error") return false;
|
|
15156
|
+
if (!VISIBLE_CONTINUATION_APIS.has(message.api) || this.model?.transport === "pi-native") return false;
|
|
15157
|
+
if (!this.model || this.model.api !== message.api || this.model.provider !== message.provider) return false;
|
|
15158
|
+
if (this.model.id !== message.model) return false;
|
|
15159
|
+
if (!isTextVisibleTail(message)) return false;
|
|
15160
|
+
if (!this.#hasCleanContinuationSafety || this.#stepRecoveryUncertainReasons.length > 0) return false;
|
|
15161
|
+
return this.agent.state.messages.at(-1) === message && this.#getVisibleRecoveryReferences(message) !== undefined;
|
|
15162
|
+
}
|
|
15163
|
+
|
|
15164
|
+
/**
|
|
15165
|
+
* Persisted entry ids of the current user turn's failed text tails and completed
|
|
15166
|
+
* tool pairs; undefined when any is missing from the branch or a tail is not text.
|
|
15167
|
+
*/
|
|
15168
|
+
#getVisibleRecoveryReferences(message: AssistantMessage):
|
|
15169
|
+
| {
|
|
15170
|
+
visibleMessageIds: string[];
|
|
15171
|
+
completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }>;
|
|
15172
|
+
}
|
|
15173
|
+
| undefined {
|
|
15174
|
+
const messages = this.agent.state.messages;
|
|
15175
|
+
const start = messages.findLastIndex(item => item.role === "user");
|
|
15176
|
+
const tail = messages.slice(Math.max(0, start));
|
|
15177
|
+
const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
|
|
15178
|
+
const visibleMessageIds: string[] = [];
|
|
15179
|
+
const completedTools: Array<{ toolCallId: string; callEntryId: string; resultEntryId: string }> = [];
|
|
15180
|
+
for (const item of tail) {
|
|
15181
|
+
if (item.role !== "assistant") continue;
|
|
15182
|
+
const entryId = getSessionMessageEntryId(item);
|
|
15183
|
+
if (!entryId || !branchIds.has(entryId)) return undefined;
|
|
15184
|
+
if (item.stopReason === "error" || item.stopReason === "aborted") {
|
|
15185
|
+
if (!isTextVisibleTail(item)) return undefined;
|
|
15186
|
+
visibleMessageIds.push(entryId);
|
|
15187
|
+
}
|
|
15188
|
+
for (const block of item.content) {
|
|
15189
|
+
if (block.type !== "toolCall") continue;
|
|
15190
|
+
const result = tail.find(candidate => candidate.role === "toolResult" && candidate.toolCallId === block.id);
|
|
15191
|
+
if (result?.role !== "toolResult" || result.isError) return undefined;
|
|
15192
|
+
const resultEntryId = getSessionMessageEntryId(result);
|
|
15193
|
+
if (!resultEntryId || !branchIds.has(resultEntryId)) return undefined;
|
|
15194
|
+
completedTools.push({ toolCallId: block.id, callEntryId: entryId, resultEntryId });
|
|
15195
|
+
}
|
|
15196
|
+
}
|
|
15197
|
+
const messageId = getSessionMessageEntryId(message);
|
|
15198
|
+
if (!messageId || !visibleMessageIds.includes(messageId)) return undefined;
|
|
15199
|
+
return { visibleMessageIds, completedTools };
|
|
15200
|
+
}
|
|
15201
|
+
|
|
15202
|
+
#hasVisibleRecoveryReferences(): boolean {
|
|
15203
|
+
if (this.#visibleRecoveryReferenceIds.length === 0) return false;
|
|
15204
|
+
const branchIds = new Set(this.sessionManager.getBranch().map(entry => entry.id));
|
|
15205
|
+
if (!this.#visibleRecoveryReferenceIds.every(id => branchIds.has(id) && this.sessionManager.getEntry(id))) {
|
|
15206
|
+
return false;
|
|
15207
|
+
}
|
|
15208
|
+
// Compaction can keep the entries on the branch while replacing them in the model
|
|
15209
|
+
// context; the preserved prefix and completed tool pairs must still be what is sent.
|
|
15210
|
+
const liveIds = new Set(
|
|
15211
|
+
this.agent.state.messages.map(getSessionMessageEntryId).filter((id): id is string => id !== undefined),
|
|
15212
|
+
);
|
|
15213
|
+
return this.#visibleRecoveryContextIds.every(id => liveIds.has(id));
|
|
15214
|
+
}
|
|
15215
|
+
|
|
15216
|
+
/**
|
|
15217
|
+
* Durably checkpoints the preserved prefix and appends one continuation
|
|
15218
|
+
* instruction. Returns false (and sends nothing) when persistence fails.
|
|
15219
|
+
*/
|
|
15220
|
+
async #appendVisibleContinuation(message: AssistantMessage): Promise<boolean> {
|
|
15221
|
+
if (this.agent.state.messages.at(-1) !== message) return false;
|
|
15222
|
+
const references = this.#getVisibleRecoveryReferences(message);
|
|
15223
|
+
if (!references) return false;
|
|
15224
|
+
const generation = this.#promptGeneration;
|
|
15225
|
+
const budget = this.#stepRecoveryBudget;
|
|
15226
|
+
const details = {
|
|
15227
|
+
version: 1,
|
|
15228
|
+
state: "recovering",
|
|
15229
|
+
runId: `${this.sessionId}:${generation}`,
|
|
15230
|
+
stepId: references.visibleMessageIds[0],
|
|
15231
|
+
attemptId: String(this.#retryAttempt),
|
|
15232
|
+
generation,
|
|
15233
|
+
selector: `${message.provider}/${message.model}`,
|
|
15234
|
+
...references,
|
|
15235
|
+
uncertainTools: this.#stepRecoveryUncertainReasons.map(reason => ({ reason })),
|
|
15236
|
+
usedRequests: budget?.snapshot().usedRequests ?? 0,
|
|
15237
|
+
};
|
|
15238
|
+
const continuation: AgentMessage = {
|
|
15239
|
+
role: "custom",
|
|
15240
|
+
customType: VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
15241
|
+
content: VISIBLE_CONTINUATION_PROMPT,
|
|
15242
|
+
display: false,
|
|
15243
|
+
details,
|
|
15244
|
+
attribution: "agent",
|
|
15245
|
+
timestamp: Date.now(),
|
|
15246
|
+
};
|
|
15247
|
+
try {
|
|
15248
|
+
const continuationMessageId = this.sessionManager.appendCustomMessageEntry(
|
|
15249
|
+
VISIBLE_CONTINUATION_CUSTOM_TYPE,
|
|
15250
|
+
VISIBLE_CONTINUATION_PROMPT,
|
|
15251
|
+
false,
|
|
15252
|
+
details,
|
|
15253
|
+
"agent",
|
|
15254
|
+
);
|
|
15255
|
+
const checkpointEntryId = this.sessionManager.appendCustomEntry(RECOVERY_CHECKPOINT_CUSTOM_TYPE, {
|
|
15256
|
+
...details,
|
|
15257
|
+
continuationMessageId,
|
|
15258
|
+
});
|
|
15259
|
+
// New sessions defer their first write until an assistant exists. Explicitly
|
|
15260
|
+
// materialize before flushing; flush alone need not have a file to fsync.
|
|
15261
|
+
await this.sessionManager.ensureOnDisk();
|
|
15262
|
+
await this.sessionManager.flush();
|
|
15263
|
+
if (this.#isDisposed || this.#promptGeneration !== generation || this.#stepRecoveryBudget !== budget)
|
|
15264
|
+
return false;
|
|
15265
|
+
if (!budget?.canAdmit() || this.agent.state.messages.at(-1) !== message) return false;
|
|
15266
|
+
const contextIds = [
|
|
15267
|
+
...references.visibleMessageIds,
|
|
15268
|
+
...references.completedTools.flatMap(tool => [tool.callEntryId, tool.resultEntryId]),
|
|
15269
|
+
];
|
|
15270
|
+
this.#visibleRecoveryReferenceIds = [checkpointEntryId, continuationMessageId, ...contextIds];
|
|
15271
|
+
this.#visibleRecoveryContextIds = contextIds;
|
|
15272
|
+
if (!this.#hasVisibleRecoveryReferences()) return false;
|
|
15273
|
+
this.agent.appendMessage(continuation);
|
|
15274
|
+
return true;
|
|
15275
|
+
} catch (error) {
|
|
15276
|
+
logger.warn("Visible continuation checkpoint failed", {
|
|
15277
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15278
|
+
});
|
|
15279
|
+
return false;
|
|
15280
|
+
}
|
|
15281
|
+
}
|
|
15282
|
+
|
|
15283
|
+
/** Which in-process interruption, if any, the settled assistant represents. */
|
|
15284
|
+
#autoResumeTriggerFor(message: AssistantMessage): AutoResumeTrigger | undefined {
|
|
15285
|
+
// Managed chains settle here too (the session supplies no managed recovery evidence);
|
|
15286
|
+
// the resume itself is pinned to the interrupted model. Only a transient failure is
|
|
15287
|
+
// resumed: terminal provider answers and an explicit retry opt-out still stop.
|
|
15288
|
+
if (message.stopReason !== "error" || !this.settings.get("retry.enabled")) return undefined;
|
|
15289
|
+
if (!this.#isResumableFailure(message)) return undefined;
|
|
15290
|
+
// Work worth keeping exists: tools already ran or the user saw part of the answer.
|
|
15291
|
+
const turnStart = this.agent.state.messages.findLastIndex(item => item.role === "user");
|
|
15292
|
+
const followsTools = this.agent.state.messages.slice(turnStart + 1).some(item => item.role === "toolResult");
|
|
15293
|
+
return followsTools || hasVisibleFailedPrefix([message]) ? "interrupted" : undefined;
|
|
15294
|
+
}
|
|
15295
|
+
|
|
15296
|
+
/**
|
|
15297
|
+
* Error eligibility shared by in-process and restart resume. Only a transient
|
|
15298
|
+
* failure is resumed: terminal provider answers, rate limits and quota stop, since
|
|
15299
|
+
* the retry cap already decided to stop and another request would hit the same limit.
|
|
15300
|
+
*/
|
|
15301
|
+
#isResumableFailure(message: AssistantMessage): boolean {
|
|
15302
|
+
const classification = this.#classifyErrorForRetry(message);
|
|
15303
|
+
if (classification !== "transient" && classification !== "first_event_timeout") return false;
|
|
15304
|
+
// Throttle copy can quote bare numbers ("500 requests per minute") that the
|
|
15305
|
+
// transient classifier reads as a status; it is a limit, never resumed.
|
|
15306
|
+
if (BARE_DEFAULT_THROTTLE_ERROR.test(message.errorMessage ?? "")) return false;
|
|
15307
|
+
const trigger = this.#fallbackTriggerFor(message, true, message.transportFailure);
|
|
15308
|
+
return trigger?.class !== "rate_limit" && trigger?.class !== "quota" && trigger?.class !== "auth";
|
|
15309
|
+
}
|
|
15310
|
+
|
|
15311
|
+
/** Last persisted recovery checkpoint of the branch, when it records an unfinished step. */
|
|
15312
|
+
#lastInFlightCheckpoint(): Record<string, unknown> | undefined {
|
|
15313
|
+
const entry = this.sessionManager
|
|
15314
|
+
.getBranch()
|
|
15315
|
+
.findLast(item => item.type === "custom" && item.customType === RECOVERY_CHECKPOINT_CUSTOM_TYPE);
|
|
15316
|
+
if (entry?.type !== "custom" || !entry.data || typeof entry.data !== "object" || Array.isArray(entry.data))
|
|
15317
|
+
return undefined;
|
|
15318
|
+
const data = entry.data as Record<string, unknown>;
|
|
15319
|
+
return data.state === "in_flight" || data.state === "recovering" ? data : undefined;
|
|
15320
|
+
}
|
|
15321
|
+
|
|
15322
|
+
/**
|
|
15323
|
+
* Single decision point for an automatic resume of an interrupted step. Runs at
|
|
15324
|
+
* most once per user turn, on the interrupted step's selector, and only after the
|
|
15325
|
+
* resume marker is durable. Returns true when a resume was scheduled.
|
|
15326
|
+
*/
|
|
15327
|
+
async #maybeAutoResume(trigger: AutoResumeTrigger): Promise<boolean> {
|
|
15328
|
+
await this.#pendingUncertainFlush;
|
|
15329
|
+
const generation = this.#promptGeneration;
|
|
15330
|
+
const messages = this.agent.state.messages;
|
|
15331
|
+
const model = this.model;
|
|
15332
|
+
const checkpoint = trigger === "restart" ? this.#lastInFlightCheckpoint() : undefined;
|
|
15333
|
+
const failed = messages.findLast(message => message.role === "assistant") as AssistantMessage | undefined;
|
|
15334
|
+
// A restart re-checks the stored failure: a step that stopped on a rate limit,
|
|
15335
|
+
// quota, auth or terminal answer stays stopped across processes too.
|
|
15336
|
+
if (
|
|
15337
|
+
trigger === "restart" &&
|
|
15338
|
+
failed?.stopReason === "error" &&
|
|
15339
|
+
messages.lastIndexOf(failed) > messages.findLastIndex(message => message.role === "user") &&
|
|
15340
|
+
!this.#isResumableFailure(failed)
|
|
15341
|
+
) {
|
|
15342
|
+
logger.debug("auto_resume_paused", { trigger, reason: "non_transient_failure" });
|
|
15343
|
+
return false;
|
|
15344
|
+
}
|
|
15345
|
+
const selector =
|
|
15346
|
+
trigger === "restart"
|
|
15347
|
+
? typeof checkpoint?.selector === "string"
|
|
15348
|
+
? checkpoint.selector
|
|
15349
|
+
: undefined
|
|
15350
|
+
: failed
|
|
15351
|
+
? `${failed.provider}/${failed.model}`
|
|
15352
|
+
: undefined;
|
|
15353
|
+
const restartUncertain =
|
|
15354
|
+
trigger === "restart" &&
|
|
15355
|
+
(!checkpoint ||
|
|
15356
|
+
!Array.isArray(checkpoint.uncertain) ||
|
|
15357
|
+
checkpoint.uncertain.length > 0 ||
|
|
15358
|
+
typeof checkpoint.api !== "string" ||
|
|
15359
|
+
checkpoint.api === "cursor-agent" ||
|
|
15360
|
+
checkpoint.transport === "pi-native");
|
|
15361
|
+
const budget = this.#stepRecoveryBudget;
|
|
15362
|
+
const selectedTools = this.agent.state.tools;
|
|
15363
|
+
const decision = resolveAutoResumePolicy({
|
|
15364
|
+
enabled: this.settings.get("retry.autoResume") && !this.#isDisposed && !this.#handoffTransitionActive,
|
|
15365
|
+
trigger,
|
|
15366
|
+
messages,
|
|
15367
|
+
resumedThisTurn: hasAutoResumeMarkerSinceLastUser(this.sessionManager.getBranch()),
|
|
15368
|
+
uncertainUpstream:
|
|
15369
|
+
this.#uncertainFlushFailed ||
|
|
15370
|
+
this.#uncertainUpstreamUnresolved ||
|
|
15371
|
+
this.#stepRecoveryUncertainReasons.length > 0 ||
|
|
15372
|
+
restartUncertain,
|
|
15373
|
+
opaqueTransport: model?.api === "cursor-agent" || model?.transport === "pi-native",
|
|
15374
|
+
budgetAdmits: trigger === "restart" ? true : budget?.canAdmit() === true,
|
|
15375
|
+
userInputPending: this.agent.hasQueuedMessages() || this.#cancelAndSubmitInProgress,
|
|
15376
|
+
selectorAvailable: model !== undefined && selector === `${model.provider}/${model.id}`,
|
|
15377
|
+
prefixReplayable:
|
|
15378
|
+
failed !== undefined &&
|
|
15379
|
+
VISIBLE_CONTINUATION_APIS.has(failed.api) &&
|
|
15380
|
+
model?.api === failed.api &&
|
|
15381
|
+
this.#hasCleanContinuationSafety,
|
|
15382
|
+
selectedTools,
|
|
15383
|
+
});
|
|
15384
|
+
if (decision.type === "pause") {
|
|
15385
|
+
logger.debug("auto_resume_paused", { trigger, reason: decision.reason });
|
|
15386
|
+
if (decision.reason === "already_resumed") {
|
|
15387
|
+
this.emitNotice("info", "Automatic resume was already used for this request.", "retry");
|
|
15388
|
+
}
|
|
15389
|
+
return false;
|
|
15390
|
+
}
|
|
15391
|
+
if (!selector) return false;
|
|
15392
|
+
const marker: AutoResumeMarker = {
|
|
15393
|
+
version: 1,
|
|
15394
|
+
trigger,
|
|
15395
|
+
mode: decision.mode,
|
|
15396
|
+
selector,
|
|
15397
|
+
interruptedCallIds: decision.interruptedCalls.map(call => call.id),
|
|
15398
|
+
};
|
|
15399
|
+
const placeholders: ToolResultMessage[] = decision.interruptedCalls.map(call => ({
|
|
15400
|
+
role: "toolResult",
|
|
15401
|
+
toolCallId: call.id,
|
|
15402
|
+
toolName: call.name,
|
|
15403
|
+
content: [{ type: "text", text: INTERRUPTED_TOOL_RESULT_TEXT }],
|
|
15404
|
+
isError: true,
|
|
15405
|
+
timestamp: Date.now(),
|
|
15406
|
+
}));
|
|
15407
|
+
const prompt = decision.mode === "restricted" ? AUTO_RESUME_RESTRICTED_PROMPT : AUTO_RESUME_FULL_PROMPT;
|
|
15408
|
+
const tailBeforeFlush = messages.at(-1);
|
|
15409
|
+
try {
|
|
15410
|
+
this.sessionManager.appendCustomEntry(AUTO_RESUME_MARKER_CUSTOM_TYPE, marker);
|
|
15411
|
+
for (const placeholder of placeholders) this.sessionManager.appendMessage(placeholder);
|
|
15412
|
+
this.sessionManager.appendCustomMessageEntry(AUTO_RESUME_CUSTOM_TYPE, prompt, false, marker, "agent");
|
|
15413
|
+
await this.sessionManager.ensureOnDisk();
|
|
15414
|
+
await this.sessionManager.flush();
|
|
15415
|
+
} catch (error) {
|
|
15416
|
+
logger.warn("Automatic resume checkpoint failed", {
|
|
15417
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15418
|
+
});
|
|
15419
|
+
return false;
|
|
15420
|
+
}
|
|
15421
|
+
if (
|
|
15422
|
+
this.#isDisposed ||
|
|
15423
|
+
this.#promptGeneration !== generation ||
|
|
15424
|
+
this.agent.state.messages !== messages ||
|
|
15425
|
+
messages.at(-1) !== tailBeforeFlush
|
|
15426
|
+
) {
|
|
15427
|
+
// Fail-safe: nothing is sent. The durable marker still consumes this turn's resume.
|
|
15428
|
+
logger.warn("auto_resume_abandoned", { trigger, reason: "state changed while the marker was written" });
|
|
15429
|
+
return false;
|
|
15430
|
+
}
|
|
15431
|
+
// A failed attempt that showed nothing, or only private reasoning, is not part of the
|
|
15432
|
+
// answer; resending it only adds an empty assistant turn (the same rule manual retry applies).
|
|
15433
|
+
const tailMessage = this.agent.state.messages.at(-1);
|
|
15434
|
+
if (
|
|
15435
|
+
placeholders.length === 0 &&
|
|
15436
|
+
tailMessage?.role === "assistant" &&
|
|
15437
|
+
(tailMessage.stopReason === "error" || tailMessage.stopReason === "aborted") &&
|
|
15438
|
+
(isThinkingOnlyFailure(tailMessage) || !assistantMessageHasVisibleOrToolContent(tailMessage))
|
|
15439
|
+
) {
|
|
15440
|
+
this.agent.replaceMessages(this.agent.state.messages.slice(0, -1));
|
|
15441
|
+
}
|
|
15442
|
+
// The durable placeholders replace in-memory pairing results that observed nothing.
|
|
15443
|
+
if (this.agent.state.messages.some(isSynthesizedResumeResult)) {
|
|
15444
|
+
this.agent.replaceMessages(this.agent.state.messages.filter(message => !isSynthesizedResumeResult(message)));
|
|
15445
|
+
}
|
|
15446
|
+
for (const placeholder of placeholders) this.agent.appendMessage(placeholder);
|
|
15447
|
+
this.agent.appendMessage({
|
|
15448
|
+
role: "custom",
|
|
15449
|
+
customType: AUTO_RESUME_CUSTOM_TYPE,
|
|
15450
|
+
content: prompt,
|
|
15451
|
+
display: false,
|
|
15452
|
+
details: marker,
|
|
15453
|
+
attribution: "agent",
|
|
15454
|
+
timestamp: Date.now(),
|
|
15455
|
+
});
|
|
15456
|
+
if (trigger === "restart") {
|
|
15457
|
+
this.#endStepRecoveryBudget();
|
|
15458
|
+
this.#stepRecoveryBudget = new RecoveryBudget({
|
|
15459
|
+
maxRequests: RESTART_RESUME_MAX_REQUESTS,
|
|
15460
|
+
windowMs: RESTART_RESUME_WINDOW_MS,
|
|
15461
|
+
maxSilentTimeouts: this.#maxSilentTimeouts(),
|
|
15462
|
+
});
|
|
15463
|
+
// The small envelope starts its clock now and arms its deadline: an unanswered
|
|
15464
|
+
// restart resume never waits longer, even mid-request.
|
|
15465
|
+
this.#stepRecoveryDeadlineAbort = new AbortController();
|
|
15466
|
+
this.#recordStepRecoveryFailure(this.#stepRecoveryBudget);
|
|
15467
|
+
this.#restartResumeEnvelopePending = true;
|
|
15468
|
+
this.#unresolvedRecoveryAtHydration = false;
|
|
15469
|
+
}
|
|
15470
|
+
if (decision.mode === "restricted") this.#setRestrictedToolMode(true);
|
|
15471
|
+
this.#activeAutoResume = { trigger, mode: decision.mode };
|
|
15472
|
+
this.emitNotice(
|
|
15473
|
+
"info",
|
|
15474
|
+
decision.mode === "restricted"
|
|
15475
|
+
? "Resuming the interrupted step automatically (read-only tools)."
|
|
15476
|
+
: "Resuming the interrupted step automatically.",
|
|
15477
|
+
"retry",
|
|
15478
|
+
);
|
|
15479
|
+
logger.info("auto_resume_started", { trigger, mode: decision.mode, interruptedCalls: placeholders.length });
|
|
15480
|
+
if (trigger === "restart") return true;
|
|
15481
|
+
void this.#scheduleAgentContinue({
|
|
15482
|
+
delayMs: 1,
|
|
15483
|
+
generation,
|
|
15484
|
+
allowDuringCancelAndSubmit: false,
|
|
15485
|
+
pinnedModel: true,
|
|
15486
|
+
shouldContinue: () => !this.agent.hasQueuedMessages(),
|
|
15487
|
+
// A resume that never started closes the step it was carrying, unless a newer
|
|
15488
|
+
// turn already owns the envelope.
|
|
15489
|
+
onError: () => {
|
|
15490
|
+
this.#endAutoResume("failed");
|
|
15491
|
+
if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
|
|
15492
|
+
},
|
|
15493
|
+
onSkip: () => {
|
|
15494
|
+
this.#endAutoResume("skipped");
|
|
15495
|
+
if (this.#promptGeneration === generation) this.#endStepRecoveryBudget();
|
|
15496
|
+
},
|
|
15497
|
+
});
|
|
15498
|
+
return true;
|
|
15499
|
+
}
|
|
15500
|
+
|
|
15501
|
+
/** Restricted mode narrows only what the model sees and may execute; the selection is untouched. */
|
|
15502
|
+
#setRestrictedToolMode(enabled: boolean): void {
|
|
15503
|
+
if (this.#restrictedToolMode === enabled) return;
|
|
15504
|
+
this.#restrictedToolMode = enabled;
|
|
15505
|
+
this.agent.setToolFilter(enabled ? tools => tools.filter(tool => isRestartSafeTool(tool, tools)) : undefined);
|
|
15506
|
+
}
|
|
15507
|
+
|
|
15508
|
+
#endAutoResume(outcome: "completed" | "failed" | "skipped"): void {
|
|
15509
|
+
this.#restartResumeEnvelopePending = false;
|
|
15510
|
+
this.#setRestrictedToolMode(false);
|
|
15511
|
+
const active = this.#activeAutoResume;
|
|
15512
|
+
this.#activeAutoResume = undefined;
|
|
15513
|
+
if (active) logger.info("auto_resume_finished", { ...active, outcome });
|
|
15514
|
+
}
|
|
15515
|
+
|
|
14583
15516
|
/**
|
|
14584
15517
|
* Skip the current retry backoff and re-attempt immediately. Distinct from
|
|
14585
15518
|
* abortRetry(), which cancels the retry and returns to idle. No-op when no
|
|
@@ -14672,20 +15605,21 @@ export class AgentSession {
|
|
|
14672
15605
|
);
|
|
14673
15606
|
}
|
|
14674
15607
|
|
|
14675
|
-
#isUnresolvedToolUseAssistant(message: AssistantMessage): boolean {
|
|
14676
|
-
return message.stopReason === "toolUse" && message.content.some(content => content.type === "toolCall");
|
|
14677
|
-
}
|
|
14678
|
-
|
|
14679
15608
|
/**
|
|
14680
15609
|
* Manually retry the last failed assistant turn, or resume an interrupted tail
|
|
14681
15610
|
* left by a non-graceful process exit after the user/custom/tool-result message
|
|
14682
15611
|
* was persisted but before the agent emitted a terminal assistant response.
|
|
14683
|
-
* Removes failed/aborted
|
|
14684
|
-
* re-
|
|
14685
|
-
* @returns
|
|
15612
|
+
* Removes only a failed/aborted assistant tail with no visible output or tool
|
|
15613
|
+
* calls, then re-attempts with a fresh retry budget.
|
|
15614
|
+
* @returns false when the agent is busy, no retryable tail exists, the tail shows
|
|
15615
|
+
* output or tool calls, a persisted recovery is unresolved, or the step dispatched
|
|
15616
|
+
* unobservable remote work; true when a retry/resume was scheduled.
|
|
14686
15617
|
*/
|
|
14687
15618
|
async retry(): Promise<boolean> {
|
|
14688
15619
|
if (this.isStreaming || this.isCompacting || this.isRetrying) return false;
|
|
15620
|
+
if (this.#unresolvedRecoveryAtHydration) return false;
|
|
15621
|
+
// Unobservable remote effects (broker/gateway) cannot be proven unsent.
|
|
15622
|
+
if (this.#uncertainUpstreamUnresolved || this.#stepRecoveryUncertainReasons.length > 0) return false;
|
|
14689
15623
|
// A handoff transition owns the session; retrying would mutate the tail and
|
|
14690
15624
|
// schedule a continuation against the session being handed off.
|
|
14691
15625
|
if (this.isGeneratingHandoff || this.#handoffTransitionActive) return false;
|
|
@@ -14697,22 +15631,24 @@ export class AgentSession {
|
|
|
14697
15631
|
if (lastMsg.role !== "assistant") {
|
|
14698
15632
|
if (!this.#isInterruptedRetryTail(lastMsg)) return false;
|
|
14699
15633
|
this.#retryAttempt = 0;
|
|
15634
|
+
this.#endStepRecoveryBudget();
|
|
14700
15635
|
this.#scheduleAgentContinue({ delayMs: 1 });
|
|
14701
15636
|
return true;
|
|
14702
15637
|
}
|
|
14703
15638
|
|
|
14704
15639
|
const assistantMsg = lastMsg as AssistantMessage;
|
|
14705
|
-
|
|
14706
|
-
|
|
14707
|
-
|
|
14708
|
-
|
|
14709
|
-
if (
|
|
15640
|
+
// Explicit retry does not establish the outcome of an interrupted operation
|
|
15641
|
+
// and is never permission to erase a prefix already shown to the user. A failure
|
|
15642
|
+
// that streamed only reasoning holds no answer and is dropped like an empty one.
|
|
15643
|
+
if (!isThinkingOnlyFailure(assistantMsg) && assistantMessageHasVisibleOrToolContent(assistantMsg)) return false;
|
|
15644
|
+
if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") return false;
|
|
14710
15645
|
|
|
14711
|
-
// Remove the failed/aborted
|
|
15646
|
+
// Remove the empty failed/aborted assistant message before re-attempting.
|
|
14712
15647
|
this.agent.replaceMessages(messages.slice(0, -1));
|
|
14713
15648
|
|
|
14714
|
-
// Reset retry budget for a fresh attempt
|
|
15649
|
+
// Reset retry budget for a fresh attempt: a manual retry is a new invocation.
|
|
14715
15650
|
this.#retryAttempt = 0;
|
|
15651
|
+
this.#endStepRecoveryBudget();
|
|
14716
15652
|
|
|
14717
15653
|
// Re-attempt the turn
|
|
14718
15654
|
this.#scheduleAgentContinue({ delayMs: 1 });
|
|
@@ -15325,7 +16261,6 @@ export class AgentSession {
|
|
|
15325
16261
|
systemPrompt: [...this.systemPrompt, instruction],
|
|
15326
16262
|
messages: this.#projectBtwVisibleText(this.buildDisplaySessionContext().messages),
|
|
15327
16263
|
thinkingLevel: this.thinkingLevel ?? ThinkingLevel.Off,
|
|
15328
|
-
hideThinkingSummary: this.agent.hideThinkingSummary ?? false,
|
|
15329
16264
|
serviceTier: this.serviceTier,
|
|
15330
16265
|
credentialSessionId: providerAffinitySessionId,
|
|
15331
16266
|
providerAffinitySessionId,
|
|
@@ -15413,7 +16348,6 @@ export class AgentSession {
|
|
|
15413
16348
|
this.sessionId,
|
|
15414
16349
|
),
|
|
15415
16350
|
reasoning: toReasoningEffort(this.thinkingLevel),
|
|
15416
|
-
hideThinkingSummary: this.agent.hideThinkingSummary,
|
|
15417
16351
|
serviceTier: this.serviceTier,
|
|
15418
16352
|
signal: args.signal,
|
|
15419
16353
|
toolChoice: "none",
|
|
@@ -15486,7 +16420,6 @@ export class AgentSession {
|
|
|
15486
16420
|
apiKey,
|
|
15487
16421
|
sessionId: scope.sideSessionId,
|
|
15488
16422
|
reasoning: toReasoningEffort(scope.thinkingLevel),
|
|
15489
|
-
hideThinkingSummary: scope.hideThinkingSummary,
|
|
15490
16423
|
serviceTier: scope.serviceTier,
|
|
15491
16424
|
signal: requestSignal,
|
|
15492
16425
|
toolChoice: "none",
|
|
@@ -15737,6 +16670,7 @@ export class AgentSession {
|
|
|
15737
16670
|
// (#2797 / #2925).
|
|
15738
16671
|
if (switchingToDifferentSession) await initializeLocalRoot(this.#localProtocolOptions());
|
|
15739
16672
|
this.#syncAgentSessionId();
|
|
16673
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15740
16674
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15741
16675
|
|
|
15742
16676
|
const sessionContext = this.buildDisplaySessionContext();
|
|
@@ -15847,6 +16781,7 @@ export class AgentSession {
|
|
|
15847
16781
|
this.sessionManager.restoreState(previousSessionState);
|
|
15848
16782
|
this.#defaultFallbackController = undefined;
|
|
15849
16783
|
this.#syncAgentSessionId(previousSessionState.sessionId);
|
|
16784
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15850
16785
|
this.#restoreWorkflowGateEmitter(suspendedWorkflowGateEmitter);
|
|
15851
16786
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15852
16787
|
let restoreMcpError: unknown;
|
|
@@ -15964,6 +16899,7 @@ export class AgentSession {
|
|
|
15964
16899
|
|
|
15965
16900
|
this.#syncTodoPhasesFromBranch();
|
|
15966
16901
|
this.#syncAgentSessionId();
|
|
16902
|
+
this.#resyncRecoveryStateFromTranscript();
|
|
15967
16903
|
this.#bindWorkflowGateEmitter(previousWorkflowGateSessionId);
|
|
15968
16904
|
this.#rekeyHindsightMemoryForCurrentSessionId();
|
|
15969
16905
|
this.#resetHindsightConversationTrackingIfHindsight();
|
|
@@ -16712,15 +17648,22 @@ export class AgentSession {
|
|
|
16712
17648
|
if (!lastAssistant) return undefined;
|
|
16713
17649
|
|
|
16714
17650
|
let text = "";
|
|
16715
|
-
for (const
|
|
16716
|
-
|
|
16717
|
-
|
|
17651
|
+
for (const assistant of this.getVisibleAnswerChain(lastAssistant)) {
|
|
17652
|
+
for (const content of assistant.content) {
|
|
17653
|
+
if (content.type === "text") {
|
|
17654
|
+
text += content.text;
|
|
17655
|
+
}
|
|
16718
17656
|
}
|
|
16719
17657
|
}
|
|
16720
17658
|
|
|
16721
17659
|
return text.trim() || undefined;
|
|
16722
17660
|
}
|
|
16723
17661
|
|
|
17662
|
+
/** The assistant messages that together form the visible answer ending at `assistant`. */
|
|
17663
|
+
getVisibleAnswerChain(assistant: AssistantMessage): AssistantMessage[] {
|
|
17664
|
+
return getVisibleAnswerChain(this.messages, assistant);
|
|
17665
|
+
}
|
|
17666
|
+
|
|
16724
17667
|
hasCopyCandidateAssistantMessage(): boolean {
|
|
16725
17668
|
return this.#getLastCopyCandidateAssistantMessage() !== undefined;
|
|
16726
17669
|
}
|