@oh-my-pi/pi-coding-agent 17.0.8 → 17.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -1
- package/README.md +1 -1
- package/dist/cli.js +5495 -5021
- package/dist/types/async/job-manager.d.ts +37 -1
- package/dist/types/cli/args.d.ts +2 -0
- package/dist/types/cli/bench-cli.d.ts +41 -0
- package/dist/types/commands/bench.d.ts +15 -3
- package/dist/types/commands/launch.d.ts +4 -0
- package/dist/types/config/__tests__/model-registry.test.d.ts +1 -0
- package/dist/types/config/model-registry.d.ts +30 -1
- package/dist/types/config/models-config-schema.d.ts +18 -6
- package/dist/types/config/models-config.d.ts +14 -5
- package/dist/types/config/provider-globals.d.ts +2 -2
- package/dist/types/config/service-tier.d.ts +5 -1
- package/dist/types/config/settings-schema.d.ts +350 -50
- package/dist/types/config/settings.d.ts +7 -0
- package/dist/types/dap/client.d.ts +20 -0
- package/dist/types/exec/bash-executor.d.ts +38 -0
- package/dist/types/exec/direnv.d.ts +33 -0
- package/dist/types/export/html/args.d.ts +15 -0
- package/dist/types/export/html/index.d.ts +11 -26
- package/dist/types/export/html/web-palette.d.ts +55 -116
- package/dist/types/extensibility/extensions/loader.d.ts +3 -0
- package/dist/types/extensibility/extensions/types.d.ts +17 -2
- package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +1 -0
- package/dist/types/extensibility/legacy-pi-tui-shim.d.ts +10 -0
- package/dist/types/extensibility/plugins/legacy-pi-compat.d.ts +14 -1
- package/dist/types/internal-urls/registry-helpers.d.ts +11 -0
- package/dist/types/live/controller.d.ts +40 -0
- package/dist/types/live/protocol.d.ts +85 -0
- package/dist/types/live/protocol.test.d.ts +1 -0
- package/dist/types/live/transport.d.ts +48 -0
- package/dist/types/live/visualizer.d.ts +33 -0
- package/dist/types/lsp/client.d.ts +18 -1
- package/dist/types/mcp/config-writer.test.d.ts +1 -0
- package/dist/types/mcp/manager.d.ts +6 -2
- package/dist/types/mcp/oauth-discovery.d.ts +2 -0
- package/dist/types/mcp/render.d.ts +2 -2
- package/dist/types/mcp/smithery-auth.d.ts +1 -1
- package/dist/types/mcp/smithery-auth.test.d.ts +1 -0
- package/dist/types/mcp/tool-bridge.d.ts +7 -3
- package/dist/types/mcp/types.d.ts +6 -0
- package/dist/types/memory-backend/tool-names.d.ts +2 -0
- package/dist/types/modes/components/login-dialog.d.ts +2 -2
- package/dist/types/modes/components/oauth-selector.d.ts +8 -0
- package/dist/types/modes/components/plan-review-overlay.d.ts +26 -0
- package/dist/types/modes/components/settings-defs.d.ts +8 -1
- package/dist/types/modes/components/status-line/component.jj-cache.test.d.ts +1 -0
- package/dist/types/modes/components/tool-execution.d.ts +9 -1
- package/dist/types/modes/controllers/command-controller.d.ts +1 -1
- package/dist/types/modes/controllers/event-controller.d.ts +6 -1
- package/dist/types/modes/controllers/live-command-controller.d.ts +14 -0
- package/dist/types/modes/controllers/mcp-command-controller.d.ts +3 -0
- package/dist/types/modes/interactive-mode.d.ts +5 -0
- package/dist/types/modes/rpc/rpc-client.d.ts +10 -1
- package/dist/types/modes/rpc/rpc-frame.d.ts +16 -0
- package/dist/types/modes/rpc/rpc-messages.d.ts +26 -0
- package/dist/types/modes/rpc/rpc-types.d.ts +40 -0
- package/dist/types/modes/setup-wizard/scenes/sign-in.d.ts +1 -1
- package/dist/types/modes/setup-wizard/scenes/types.d.ts +9 -1
- package/dist/types/modes/setup-wizard/scenes/web-search.d.ts +1 -1
- package/dist/types/modes/theme/theme.d.ts +1 -1
- package/dist/types/modes/types.d.ts +3 -1
- package/dist/types/modes/utils/context-usage.d.ts +12 -2
- package/dist/types/plan-mode/plan-files.d.ts +10 -0
- package/dist/types/registry/agent-lifecycle.d.ts +21 -9
- package/dist/types/registry/agent-registry.d.ts +11 -4
- package/dist/types/sdk.d.ts +15 -1
- package/dist/types/secrets/index.d.ts +15 -2
- package/dist/types/secrets/obfuscator.d.ts +113 -15
- package/dist/types/session/acp-permission-gate.d.ts +19 -0
- package/dist/types/session/agent-session-events.d.ts +84 -0
- package/dist/types/session/agent-session-types.d.ts +314 -0
- package/dist/types/session/agent-session.d.ts +120 -750
- package/dist/types/session/async-job-delivery.d.ts +28 -0
- package/dist/types/session/bash-runner.d.ts +75 -0
- package/dist/types/session/blob-store.d.ts +11 -1
- package/dist/types/session/blob-store.test.d.ts +1 -0
- package/dist/types/session/checkpoint-entries.d.ts +28 -0
- package/dist/types/session/eval-runner.d.ts +49 -0
- package/dist/types/session/irc-bridge.d.ts +46 -0
- package/dist/types/session/messages.d.ts +22 -0
- package/dist/types/session/model-controls.d.ts +180 -0
- package/dist/types/session/prewalk.d.ts +46 -0
- package/dist/types/session/queued-messages.d.ts +22 -0
- package/dist/types/session/redis-session-storage.d.ts +1 -0
- package/dist/types/session/retry-fallback-chains.d.ts +74 -0
- package/dist/types/session/role-models.d.ts +15 -0
- package/dist/types/session/session-advisors.d.ts +228 -0
- package/dist/types/session/session-entries.d.ts +8 -0
- package/dist/types/session/session-handoff.d.ts +70 -0
- package/dist/types/session/session-history-format.d.ts +6 -1
- package/dist/types/session/session-loader.d.ts +6 -4
- package/dist/types/session/session-maintenance.d.ts +246 -0
- package/dist/types/session/session-manager.d.ts +41 -0
- package/dist/types/session/session-memory.d.ts +56 -0
- package/dist/types/session/session-provider-boundary.d.ts +62 -0
- package/dist/types/session/session-stats.d.ts +48 -0
- package/dist/types/session/session-tools.d.ts +143 -0
- package/dist/types/session/session-workspace.d.ts +32 -0
- package/dist/types/session/stream-guards.d.ts +44 -0
- package/dist/types/session/todo-tracker.d.ts +56 -0
- package/dist/types/session/ttsr-coordinator.d.ts +51 -0
- package/dist/types/session/turn-recovery.d.ts +225 -0
- package/dist/types/system-prompt.d.ts +2 -0
- package/dist/types/task/executor.d.ts +4 -1
- package/dist/types/task/output-manager.d.ts +3 -1
- package/dist/types/task/types.d.ts +14 -0
- package/dist/types/tools/approval.d.ts +1 -0
- package/dist/types/tools/browser/launch.d.ts +5 -0
- package/dist/types/tools/hub/jobs.d.ts +1 -1
- package/dist/types/tools/image-gen.d.ts +4 -4
- package/dist/types/tools/image-providers.d.ts +38 -0
- package/dist/types/tools/index.d.ts +8 -0
- package/dist/types/tools/path-utils.d.ts +8 -0
- package/dist/types/tools/report-tool-issue.d.ts +24 -9
- package/dist/types/tools/todo.d.ts +10 -4
- package/dist/types/tools/xdev.d.ts +5 -1
- package/dist/types/tts/streaming-player.d.ts +14 -0
- package/dist/types/tts/vocalizer.d.ts +5 -0
- package/dist/types/utils/jj.d.ts +29 -0
- package/dist/types/utils/lang-from-path.d.ts +1 -0
- package/dist/types/utils/title-generator.d.ts +30 -0
- package/dist/types/vibe/runtime.d.ts +36 -8
- package/dist/types/web/search/index.d.ts +1 -1
- package/dist/types/web/search/provider.d.ts +15 -5
- package/dist/types/web/search/providers/firecrawl.d.ts +9 -0
- package/dist/types/web/search/types.d.ts +95 -1
- package/package.json +13 -12
- package/scripts/generate-share-viewer.ts +4 -6
- package/scripts/legacy-pi-virtual-module.ts +1 -1
- package/src/advisor/__tests__/advisor.test.ts +643 -0
- package/src/advisor/runtime.ts +144 -43
- package/src/async/job-manager.ts +90 -2
- package/src/cli/args.ts +2 -0
- package/src/cli/auth-gateway-cli.ts +18 -3
- package/src/cli/bench-cli.ts +395 -29
- package/src/cli/flag-tables.ts +3 -0
- package/src/cli/gc-cli.ts +2 -2
- package/src/cli/grievances-cli.ts +2 -2
- package/src/cli/usage-cli.ts +11 -2
- package/src/collab/host.ts +3 -2
- package/src/commands/bench.ts +18 -4
- package/src/commands/launch.ts +4 -0
- package/src/config/__tests__/model-registry.test.ts +147 -0
- package/src/config/model-registry.ts +50 -2
- package/src/config/model-resolver.ts +0 -1
- package/src/config/models-config-schema.ts +15 -4
- package/src/config/provider-globals.ts +9 -9
- package/src/config/service-tier.ts +26 -1
- package/src/config/settings-schema.ts +250 -54
- package/src/config/settings.ts +81 -2
- package/src/dap/client.ts +37 -4
- package/src/edit/index.ts +49 -12
- package/src/eval/py/executor.ts +103 -33
- package/src/exec/bash-executor.ts +89 -6
- package/src/exec/direnv.ts +145 -0
- package/src/export/html/args.ts +20 -0
- package/src/export/html/index.ts +45 -47
- package/src/export/html/template.css +19 -1
- package/src/export/html/template.html +6 -0
- package/src/export/html/template.js +21 -0
- package/src/export/html/tool-views.generated.js +31 -31
- package/src/export/html/web-palette.ts +116 -132
- package/src/export/share.ts +249 -49
- package/src/extensibility/extensions/loader.ts +29 -1
- package/src/extensibility/extensions/runner.ts +6 -0
- package/src/extensibility/extensions/types.ts +33 -2
- package/src/extensibility/legacy-pi-coding-agent-shim.ts +1 -0
- package/src/extensibility/legacy-pi-tui-shim.ts +10 -0
- package/src/extensibility/plugins/legacy-pi-compat.ts +870 -311
- package/src/hindsight/state.ts +64 -6
- package/src/internal-urls/local-protocol.ts +2 -1
- package/src/internal-urls/memory-protocol.ts +2 -2
- package/src/internal-urls/registry-helpers.ts +40 -0
- package/src/internal-urls/skill-protocol.ts +2 -2
- package/src/internal-urls/ssh-protocol.ts +2 -1
- package/src/internal-urls/vault-protocol.ts +2 -1
- package/src/live/audio-worklet.txt +59 -0
- package/src/live/browser-runtime.txt +221 -0
- package/src/live/controller.ts +464 -0
- package/src/live/prompts/agent-final-message.md +3 -0
- package/src/live/prompts/live-instructions.md +23 -0
- package/src/live/protocol.test.ts +140 -0
- package/src/live/protocol.ts +233 -0
- package/src/live/transport.ts +490 -0
- package/src/live/visualizer.ts +234 -0
- package/src/lsp/client.ts +52 -9
- package/src/lsp/clients/biome-client.ts +3 -4
- package/src/lsp/index.ts +44 -11
- package/src/main.ts +9 -1
- package/src/mcp/config-writer.test.ts +43 -0
- package/src/mcp/config-writer.ts +109 -82
- package/src/mcp/manager.ts +39 -8
- package/src/mcp/oauth-discovery.ts +10 -2
- package/src/mcp/render.ts +94 -35
- package/src/mcp/smithery-auth.test.ts +29 -0
- package/src/mcp/smithery-auth.ts +3 -2
- package/src/mcp/tool-bridge.ts +107 -11
- package/src/mcp/types.ts +7 -0
- package/src/memories/index.ts +40 -20
- package/src/memory-backend/tool-names.ts +2 -0
- package/src/mnemopi/backend.ts +24 -7
- package/src/modes/acp/acp-agent.ts +56 -10
- package/src/modes/acp/acp-event-mapper.ts +8 -1
- package/src/modes/components/agent-hub.ts +27 -10
- package/src/modes/components/login-dialog.ts +14 -4
- package/src/modes/components/oauth-selector.ts +24 -7
- package/src/modes/components/plan-review-overlay.ts +350 -35
- package/src/modes/components/settings-defs.ts +28 -3
- package/src/modes/components/settings-selector.ts +178 -6
- package/src/modes/components/status-line/component.jj-cache.test.ts +229 -0
- package/src/modes/components/status-line/component.ts +123 -2
- package/src/modes/components/tool-execution.test.ts +63 -2
- package/src/modes/components/tool-execution.ts +11 -2
- package/src/modes/controllers/command-controller.ts +70 -22
- package/src/modes/controllers/event-controller.ts +111 -16
- package/src/modes/controllers/extension-ui-controller.test.ts +1 -0
- package/src/modes/controllers/extension-ui-controller.ts +20 -4
- package/src/modes/controllers/live-command-controller.ts +178 -0
- package/src/modes/controllers/mcp-command-controller.ts +80 -41
- package/src/modes/controllers/omfg-controller.ts +44 -40
- package/src/modes/controllers/selector-controller.ts +18 -10
- package/src/modes/controllers/session-focus-controller.ts +6 -1
- package/src/modes/interactive-mode.ts +160 -38
- package/src/modes/rpc/rpc-client.ts +94 -3
- package/src/modes/rpc/rpc-frame.ts +164 -4
- package/src/modes/rpc/rpc-messages.ts +127 -0
- package/src/modes/rpc/rpc-mode.ts +79 -7
- package/src/modes/rpc/rpc-types.ts +34 -2
- package/src/modes/runtime-init.ts +2 -0
- package/src/modes/setup-wizard/scenes/model.ts +5 -7
- package/src/modes/setup-wizard/scenes/providers.ts +3 -2
- package/src/modes/setup-wizard/scenes/sign-in.ts +8 -2
- package/src/modes/setup-wizard/scenes/theme.ts +13 -3
- package/src/modes/setup-wizard/scenes/types.ts +9 -1
- package/src/modes/setup-wizard/scenes/web-search.ts +23 -10
- package/src/modes/setup-wizard/wizard-overlay.ts +1 -1
- package/src/modes/theme/theme.ts +1 -1
- package/src/modes/types.ts +3 -1
- package/src/modes/utils/context-usage.ts +23 -7
- package/src/modes/utils/ui-helpers.ts +10 -5
- package/src/plan-mode/plan-files.ts +40 -0
- package/src/prompts/bench/cache-prefix-chunk.md +1 -0
- package/src/prompts/bench/cache-prefix.md +3 -0
- package/src/prompts/bench/cache-suffix.md +1 -0
- package/src/prompts/goals/guided-goal-system.md +24 -3
- package/src/prompts/system/custom-system-prompt.md +1 -1
- package/src/prompts/system/project-prompt.md +8 -1
- package/src/prompts/system/subagent-async-pending.md +6 -0
- package/src/prompts/system/system-prompt.md +1 -1
- package/src/prompts/system/workflow-notice.md +48 -40
- package/src/prompts/system/xdev-mount-notice.md +4 -0
- package/src/prompts/tools/task.md +12 -2
- package/src/prompts/tools/todo.md +3 -1
- package/src/registry/agent-lifecycle.ts +116 -46
- package/src/registry/agent-registry.ts +35 -8
- package/src/registry/persisted-agents.ts +26 -2
- package/src/sdk.ts +255 -139
- package/src/secrets/index.ts +127 -4
- package/src/secrets/obfuscator.ts +2328 -105
- package/src/session/acp-permission-gate.ts +165 -0
- package/src/session/agent-session-error-log.test.ts +1 -1
- package/src/session/agent-session-events.ts +66 -0
- package/src/session/agent-session-types.ts +334 -0
- package/src/session/agent-session.ts +1910 -12213
- package/src/session/artifacts.ts +6 -5
- package/src/session/async-job-delivery.ts +74 -0
- package/src/session/bash-runner.ts +326 -0
- package/src/session/blob-store.test.ts +56 -0
- package/src/session/blob-store.ts +18 -2
- package/src/session/checkpoint-entries.ts +81 -0
- package/src/session/eval-runner.ts +212 -0
- package/src/session/indexed-session-storage.ts +9 -2
- package/src/session/irc-bridge.ts +203 -0
- package/src/session/messages.ts +263 -1
- package/src/session/model-controls.ts +714 -0
- package/src/session/prewalk.ts +252 -0
- package/src/session/queued-messages.ts +93 -0
- package/src/session/redis-session-storage.ts +51 -11
- package/src/session/retry-fallback-chains.ts +455 -0
- package/src/session/role-models.ts +85 -0
- package/src/session/session-advisors.ts +1679 -0
- package/src/session/session-context.test.ts +224 -1
- package/src/session/session-context.ts +47 -5
- package/src/session/session-entries.ts +8 -0
- package/src/session/session-handoff.ts +308 -0
- package/src/session/session-history-format.ts +20 -9
- package/src/session/session-loader.ts +10 -51
- package/src/session/session-maintenance.ts +2983 -0
- package/src/session/session-manager.ts +390 -30
- package/src/session/session-memory.ts +222 -0
- package/src/session/session-provider-boundary.ts +307 -0
- package/src/session/session-stats.ts +293 -0
- package/src/session/session-tools.ts +943 -0
- package/src/session/session-workspace.ts +53 -0
- package/src/session/stream-guards.ts +417 -0
- package/src/session/todo-tracker.ts +380 -0
- package/src/session/ttsr-coordinator.ts +496 -0
- package/src/session/turn-recovery.ts +1629 -0
- package/src/slash-commands/builtin-registry.ts +92 -11
- package/src/slash-commands/helpers/usage-report.ts +25 -4
- package/src/system-prompt.ts +41 -17
- package/src/task/executor.ts +288 -70
- package/src/task/index.ts +148 -111
- package/src/task/output-manager.ts +25 -3
- package/src/task/persisted-revive.ts +4 -3
- package/src/task/structured-subagent.ts +4 -1
- package/src/task/types.ts +16 -0
- package/src/tools/approval.ts +55 -10
- package/src/tools/bash-interactive.ts +26 -0
- package/src/tools/bash-skill-urls.ts +28 -2
- package/src/tools/bash.ts +339 -123
- package/src/tools/browser/launch.ts +9 -1
- package/src/tools/browser/tab-supervisor.ts +8 -5
- package/src/tools/browser.ts +94 -47
- package/src/tools/hub/index.ts +1 -1
- package/src/tools/hub/jobs.ts +67 -8
- package/src/tools/image-gen.ts +20 -28
- package/src/tools/image-providers.ts +50 -0
- package/src/tools/index.ts +12 -0
- package/src/tools/path-utils.ts +49 -2
- package/src/tools/read.ts +156 -88
- package/src/tools/renderers.ts +7 -1
- package/src/tools/report-tool-issue.ts +79 -28
- package/src/tools/todo.ts +101 -11
- package/src/tools/vibe.ts +1 -2
- package/src/tools/write.ts +51 -1
- package/src/tools/xdev.ts +67 -3
- package/src/tts/streaming-player.ts +62 -4
- package/src/tts/vocalizer.ts +18 -1
- package/src/utils/jj.ts +125 -4
- package/src/utils/lang-from-path.ts +7 -0
- package/src/utils/title-generator.ts +137 -1
- package/src/vibe/runtime.ts +852 -72
- package/src/web/search/index.ts +3 -6
- package/src/web/search/provider.ts +31 -16
- package/src/web/search/providers/firecrawl.ts +46 -13
- package/src/web/search/providers/xai.ts +7 -1
- package/src/web/search/types.ts +8 -1
|
@@ -0,0 +1,1629 @@
|
|
|
1
|
+
import { scheduler } from "node:timers/promises";
|
|
2
|
+
import {
|
|
3
|
+
type Agent,
|
|
4
|
+
AgentBusyError,
|
|
5
|
+
type AgentMessage,
|
|
6
|
+
isSyntheticToolResultMessage,
|
|
7
|
+
type ThinkingLevel,
|
|
8
|
+
} from "@oh-my-pi/pi-agent-core";
|
|
9
|
+
import type {
|
|
10
|
+
AssistantMessage,
|
|
11
|
+
AssistantRetryRecovery,
|
|
12
|
+
AssistantRetryRecoveryKind,
|
|
13
|
+
CodexCompactionContext,
|
|
14
|
+
Model,
|
|
15
|
+
TextContent,
|
|
16
|
+
ToolChoice,
|
|
17
|
+
} from "@oh-my-pi/pi-ai";
|
|
18
|
+
import { calculateRateLimitBackoffMs, parseRateLimitReason } from "@oh-my-pi/pi-ai";
|
|
19
|
+
import * as AIError from "@oh-my-pi/pi-ai/error";
|
|
20
|
+
import { kCursorExecResolved } from "@oh-my-pi/pi-ai/utils/block-symbols";
|
|
21
|
+
import { isFireworksFastModelId, toFireworksBaseModelId } from "@oh-my-pi/pi-catalog/fireworks-model-id";
|
|
22
|
+
import { extractRetryHint, logger, prompt } from "@oh-my-pi/pi-utils";
|
|
23
|
+
import type { ModelRegistry } from "../config/model-registry";
|
|
24
|
+
import { formatModelStringWithRouting, resolveModelOverride } from "../config/model-resolver";
|
|
25
|
+
import type { Settings } from "../config/settings";
|
|
26
|
+
import type { RecoveredRetryError } from "../extensibility/shared-events";
|
|
27
|
+
import emptyStopRetryTemplate from "../prompts/system/empty-stop-retry.md" with { type: "text" };
|
|
28
|
+
import thinkingLoopRedirectTemplate from "../prompts/system/thinking-loop-redirect.md" with { type: "text" };
|
|
29
|
+
import unexpectedStopRetryTemplate from "../prompts/system/unexpected-stop-retry.md" with { type: "text" };
|
|
30
|
+
import type { ConfiguredThinkingLevel } from "../thinking";
|
|
31
|
+
import type { AgentSessionEvent } from "./agent-session-events";
|
|
32
|
+
import type { InitialRetryFallbackState } from "./agent-session-types";
|
|
33
|
+
import { isEmptyErrorTurn } from "./messages";
|
|
34
|
+
import {
|
|
35
|
+
type ActiveRetryFallbackState,
|
|
36
|
+
calculateRetryBackoffDelayMs,
|
|
37
|
+
findRetryFallbackCandidates,
|
|
38
|
+
formatRetryFallbackSelector,
|
|
39
|
+
getRetryFallbackChains,
|
|
40
|
+
getRetryFallbackRevertPolicy,
|
|
41
|
+
parseRetryFallbackSelector,
|
|
42
|
+
type RetryFallbackChains,
|
|
43
|
+
type RetryFallbackResolutionContext,
|
|
44
|
+
type RetryFallbackRevertPolicy,
|
|
45
|
+
type RetryFallbackSelector,
|
|
46
|
+
resolveRetryFallbackChainKey,
|
|
47
|
+
validateRetryFallbackChains,
|
|
48
|
+
} from "./retry-fallback-chains";
|
|
49
|
+
import { getLatestCompactionEntry } from "./session-context";
|
|
50
|
+
import { EPHEMERAL_MODEL_CHANGE_ROLE, type SessionEntry } from "./session-entries";
|
|
51
|
+
import type { SessionManager } from "./session-manager";
|
|
52
|
+
import { sameMessageContent, sessionMessagePersistenceKey } from "./turn-persistence";
|
|
53
|
+
import { classifyUnexpectedStop, isUnexpectedStopCandidate } from "./unexpected-stop-classifier";
|
|
54
|
+
|
|
55
|
+
const THINKING_LOOP_REDIRECT_TYPE = "thinking-loop-redirect";
|
|
56
|
+
const UNEXPECTED_STOP_MAX_RETRIES = 3;
|
|
57
|
+
const UNEXPECTED_STOP_TIMEOUT_MS = 4000;
|
|
58
|
+
const EMPTY_STOP_MAX_RETRIES = 3;
|
|
59
|
+
const SIBLING_UNBLOCK_BUFFER_MS = 1_000;
|
|
60
|
+
const NON_WHITESPACE_RE = /\S/;
|
|
61
|
+
|
|
62
|
+
function hasNonWhitespace(value: string): boolean {
|
|
63
|
+
return NON_WHITESPACE_RE.test(value);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function syntheticToolResultTailStart(messages: readonly AgentMessage[]): number {
|
|
67
|
+
let index = messages.length;
|
|
68
|
+
while (index > 0 && isSyntheticToolResultMessage(messages[index - 1])) {
|
|
69
|
+
index--;
|
|
70
|
+
}
|
|
71
|
+
return index;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function retryableAssistantTurnEnd(messages: readonly AgentMessage[]): number | undefined {
|
|
75
|
+
const turnEnd = syntheticToolResultTailStart(messages);
|
|
76
|
+
const message = messages[turnEnd - 1];
|
|
77
|
+
if (message?.role !== "assistant") return undefined;
|
|
78
|
+
if (message.stopReason !== "error" && message.stopReason !== "aborted") return undefined;
|
|
79
|
+
return turnEnd;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Result shape shared with automatic maintenance recovery. */
|
|
83
|
+
export interface RecoveryCompactionResult {
|
|
84
|
+
deferredHandoff: boolean;
|
|
85
|
+
continuationScheduled: boolean;
|
|
86
|
+
automaticContinuationBlocked?: boolean;
|
|
87
|
+
historyRewritten?: boolean;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Capabilities borrowed from the owning AgentSession. */
|
|
91
|
+
export interface TurnRecoveryHost {
|
|
92
|
+
agent: Agent;
|
|
93
|
+
sessionManager: SessionManager;
|
|
94
|
+
settings: Settings;
|
|
95
|
+
modelRegistry: ModelRegistry;
|
|
96
|
+
configWarnings: string[];
|
|
97
|
+
model(): Model | undefined;
|
|
98
|
+
thinkingLevel(): ThinkingLevel | undefined;
|
|
99
|
+
configuredThinkingLevel(): ConfiguredThinkingLevel | undefined;
|
|
100
|
+
setThinkingLevel(level: ConfiguredThinkingLevel | undefined): void;
|
|
101
|
+
isDisposed(): boolean;
|
|
102
|
+
isStreaming(): boolean;
|
|
103
|
+
isCompacting(): boolean;
|
|
104
|
+
abortInProgress(): boolean;
|
|
105
|
+
streamingEditAbortTriggered(): boolean;
|
|
106
|
+
promptGeneration(): number;
|
|
107
|
+
sessionId(): string;
|
|
108
|
+
emitSessionEvent(event: AgentSessionEvent): Promise<void>;
|
|
109
|
+
scheduleAgentContinue(options: { delayMs?: number; generation?: number }): void;
|
|
110
|
+
waitForSessionMessagePersistence(message: AssistantMessage): Promise<void>;
|
|
111
|
+
appendSessionMessage(message: AssistantMessage): void;
|
|
112
|
+
sessionMessageAlreadyPersisted(message: AssistantMessage): boolean;
|
|
113
|
+
setModelWithProviderSessionReset(model: Model): void;
|
|
114
|
+
resetCurrentResponsesProviderSession(reason: string): void;
|
|
115
|
+
maybeAutoRedeemCodexReset(): Promise<boolean>;
|
|
116
|
+
runAutoCompaction(
|
|
117
|
+
reason: "overflow" | "threshold" | "idle" | "incomplete",
|
|
118
|
+
willRetry: boolean,
|
|
119
|
+
deferred?: boolean,
|
|
120
|
+
allowDefer?: boolean,
|
|
121
|
+
options?: {
|
|
122
|
+
autoContinue?: boolean;
|
|
123
|
+
triggerContextTokens?: number;
|
|
124
|
+
suppressContinuation?: boolean;
|
|
125
|
+
suppressHandoff?: boolean;
|
|
126
|
+
phase?: CodexCompactionContext["phase"];
|
|
127
|
+
terminalTextAnswer?: boolean;
|
|
128
|
+
},
|
|
129
|
+
): Promise<RecoveryCompactionResult>;
|
|
130
|
+
withBashBranchTransition<T>(operation: () => T): T;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Construction-time retry state restored from model selection. */
|
|
134
|
+
export interface TurnRecoveryOptions {
|
|
135
|
+
initialRetryFallback?: InitialRetryFallbackState;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
type PendingRecoveredRetryError = {
|
|
139
|
+
entryId: string;
|
|
140
|
+
persistenceKey: string;
|
|
141
|
+
recovery: AssistantRetryRecoveryKind;
|
|
142
|
+
attempt: number;
|
|
143
|
+
note: string;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/** Owns terminal-stop recovery, automatic retries, and fallback routing. */
|
|
147
|
+
export class TurnRecovery {
|
|
148
|
+
readonly #host: TurnRecoveryHost;
|
|
149
|
+
#retryAbortController: AbortController | undefined;
|
|
150
|
+
#retryAttempt = 0;
|
|
151
|
+
#retryPromise: Promise<void> | undefined;
|
|
152
|
+
#retryResolve: (() => void) | undefined;
|
|
153
|
+
#activeRetryFallback: ActiveRetryFallbackState | undefined;
|
|
154
|
+
#pendingRecoveredRetryErrors: PendingRecoveredRetryError[] = [];
|
|
155
|
+
#emptyStopRetryCount = 0;
|
|
156
|
+
#unexpectedStopRetryCount = 0;
|
|
157
|
+
#acceptTerminalEmptyStopForPrompt = false;
|
|
158
|
+
|
|
159
|
+
constructor(host: TurnRecoveryHost, options: TurnRecoveryOptions = {}) {
|
|
160
|
+
this.#host = host;
|
|
161
|
+
if (options.initialRetryFallback) {
|
|
162
|
+
this.#activeRetryFallback = {
|
|
163
|
+
...options.initialRetryFallback,
|
|
164
|
+
lastAppliedFallbackThinkingLevel: host.configuredThinkingLevel(),
|
|
165
|
+
pinned: options.initialRetryFallback.pinned ?? false,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
this.#validateRetryFallbackChains();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Current automatic retry attempt. */
|
|
172
|
+
get attempt(): number {
|
|
173
|
+
return this.#retryAttempt;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Promise settled when the active retry saga finishes. */
|
|
177
|
+
get retryPromise(): Promise<void> | undefined {
|
|
178
|
+
return this.#retryPromise;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Resolved selector while fallback routing owns the current model. */
|
|
182
|
+
get retryFallbackModel(): string | undefined {
|
|
183
|
+
const model = this.#host.model();
|
|
184
|
+
return this.#activeRetryFallback && model
|
|
185
|
+
? formatRetryFallbackSelector(model, this.#host.thinkingLevel())
|
|
186
|
+
: undefined;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Resets per-prompt recovery counters and terminal-stop acceptance. */
|
|
190
|
+
resetForNewPrompt(): void {
|
|
191
|
+
this.#emptyStopRetryCount = 0;
|
|
192
|
+
this.#unexpectedStopRetryCount = 0;
|
|
193
|
+
this.#acceptTerminalEmptyStopForPrompt = false;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Sets whether one terminal empty stop is accepted for the current prompt. */
|
|
197
|
+
setAcceptTerminalEmptyStop(accept: boolean): void {
|
|
198
|
+
this.#acceptTerminalEmptyStopForPrompt = accept;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Closes a successful retry saga and annotates recovered persisted errors. */
|
|
202
|
+
async onAssistantSettledSuccessfully(message: AssistantMessage): Promise<void> {
|
|
203
|
+
if (
|
|
204
|
+
message.stopReason === "error" ||
|
|
205
|
+
message.stopReason === "aborted" ||
|
|
206
|
+
this.#isEmptyAssistantStop(message) ||
|
|
207
|
+
this.#retryAttempt === 0
|
|
208
|
+
) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const model = this.#host.model();
|
|
212
|
+
if (this.#activeRetryFallback && model) {
|
|
213
|
+
await this.#host.emitSessionEvent({
|
|
214
|
+
type: "retry_fallback_succeeded",
|
|
215
|
+
model: formatRetryFallbackSelector(model, this.#host.thinkingLevel()),
|
|
216
|
+
role: this.#activeRetryFallback.role,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
const recoveredErrors = await this.#markPendingRecoveredRetryErrors(message);
|
|
220
|
+
await this.#host.emitSessionEvent({
|
|
221
|
+
type: "auto_retry_end",
|
|
222
|
+
success: true,
|
|
223
|
+
attempt: this.#retryAttempt,
|
|
224
|
+
recoveredErrors,
|
|
225
|
+
});
|
|
226
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
227
|
+
this.#retryAttempt = 0;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Closes a failed retry saga when no compaction continuation took ownership. */
|
|
231
|
+
async onErrorSettledWithoutRetry(message: AssistantMessage, compaction: RecoveryCompactionResult): Promise<void> {
|
|
232
|
+
if (message.stopReason !== "error" || this.#retryAttempt === 0 || compaction.continuationScheduled) return;
|
|
233
|
+
const attempt = this.#retryAttempt;
|
|
234
|
+
this.#retryAttempt = 0;
|
|
235
|
+
await this.#host.emitSessionEvent({
|
|
236
|
+
type: "auto_retry_end",
|
|
237
|
+
success: false,
|
|
238
|
+
attempt,
|
|
239
|
+
finalError: message.errorMessage,
|
|
240
|
+
});
|
|
241
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Persists an otherwise skipped terminal empty error turn. */
|
|
245
|
+
persistTerminalEmptyErrorTurn(message: AssistantMessage): Promise<void> {
|
|
246
|
+
return this.#persistTerminalEmptyErrorTurn(message);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Handles empty terminal assistant turns and schedules bounded recovery. */
|
|
250
|
+
handleEmptyAssistantStop(message: AssistantMessage): Promise<boolean> {
|
|
251
|
+
return this.#handleEmptyAssistantStop(message);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Classifies suspicious terminal stops and schedules bounded recovery. */
|
|
255
|
+
handleUnexpectedAssistantStop(message: AssistantMessage): Promise<boolean> {
|
|
256
|
+
return this.#handleUnexpectedAssistantStop(message);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Removes a persisted failed assistant turn after its persistence slot settles. */
|
|
260
|
+
dropPersistedAssistantTurn(message: AssistantMessage): Promise<void> {
|
|
261
|
+
return this.#dropPersistedAssistantTurn(message);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Runs recovery compaction and restores the failed turn when no rewrite occurs. */
|
|
265
|
+
runRecoveryCompactionWithRollback(
|
|
266
|
+
reason: "overflow" | "incomplete",
|
|
267
|
+
message: AssistantMessage,
|
|
268
|
+
allowDefer: boolean,
|
|
269
|
+
options: { autoContinue: boolean; triggerContextTokens?: number },
|
|
270
|
+
): Promise<RecoveryCompactionResult> {
|
|
271
|
+
return this.#runRecoveryCompactionWithRollback(reason, message, allowDefer, options);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Restores the configured primary after fallback cooldown expiry. */
|
|
275
|
+
maybeRestoreRetryFallbackPrimary(): Promise<void> {
|
|
276
|
+
return this.#maybeRestoreRetryFallbackPrimary();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Applies automatic retry, credential rotation, and model fallback policy. */
|
|
280
|
+
handleRetryableError(
|
|
281
|
+
message: AssistantMessage,
|
|
282
|
+
options?: {
|
|
283
|
+
allowModelFallback?: boolean;
|
|
284
|
+
fireworksFastFallback?: boolean;
|
|
285
|
+
hardErrorFallback?: boolean;
|
|
286
|
+
preserveFailedTurn?: boolean;
|
|
287
|
+
},
|
|
288
|
+
): Promise<boolean> {
|
|
289
|
+
return this.#handleRetryableError(message, options);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** Prompts after transient overlap with a prior agent run. */
|
|
293
|
+
promptAgentWithIdleRetry(messages: AgentMessage[], options?: { toolChoice?: ToolChoice }): Promise<void> {
|
|
294
|
+
return this.#promptAgentWithIdleRetry(messages, options);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Parses provider retry and rate-limit reset hints into a delay. */
|
|
298
|
+
parseRetryAfterMsFromError(errorMessage: string): number | undefined {
|
|
299
|
+
return this.#parseRetryAfterMsFromError(errorMessage);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Resolve the pending retry promise */
|
|
303
|
+
resolveRetry(): void {
|
|
304
|
+
if (this.#retryResolve) {
|
|
305
|
+
this.#retryResolve();
|
|
306
|
+
this.#retryResolve = undefined;
|
|
307
|
+
this.#retryPromise = undefined;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
#clearPendingRecoveredRetryErrors(): void {
|
|
312
|
+
this.#pendingRecoveredRetryErrors = [];
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Durably record a terminal empty error turn (`stopReason: "error"` with no
|
|
317
|
+
* substantive content) that `#persistSessionMessageIfMissing` skipped, so the
|
|
318
|
+
* session JSONL keeps a record of why the run stopped instead of ending at the
|
|
319
|
+
* last tool result. A no-op for non-empty/non-error turns and idempotent via
|
|
320
|
+
* the already-persisted guard; the turn is dropped from active context by the
|
|
321
|
+
* caller (or `isProviderRefusalMessage`/`isEmptyErrorTurn` filters) so it is
|
|
322
|
+
* never replayed on the wire. Used by the retry-lifecycle dead-ends and the
|
|
323
|
+
* non-retry terminal error tail.
|
|
324
|
+
*/
|
|
325
|
+
async #persistTerminalEmptyErrorTurn(message: AssistantMessage): Promise<void> {
|
|
326
|
+
await this.#host.waitForSessionMessagePersistence(message);
|
|
327
|
+
if (!isEmptyErrorTurn(message)) return;
|
|
328
|
+
if (this.#host.sessionMessageAlreadyPersisted(message)) return;
|
|
329
|
+
this.#host.appendSessionMessage(message);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
#retryRecoveryKind(
|
|
333
|
+
id: number,
|
|
334
|
+
switchedCredential: boolean,
|
|
335
|
+
switchedModel: boolean,
|
|
336
|
+
delayMs: number,
|
|
337
|
+
): AssistantRetryRecoveryKind {
|
|
338
|
+
if (switchedCredential) return "credential";
|
|
339
|
+
if (switchedModel) return "model";
|
|
340
|
+
if (AIError.is(id, AIError.Flag.UsageLimit) && delayMs > 0) return "wait";
|
|
341
|
+
return "plain";
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
#retryRecoveryNote(recovery: AssistantRetryRecoveryKind, rateLimited: boolean): string {
|
|
345
|
+
const parts: string[] = [];
|
|
346
|
+
if (rateLimited) {
|
|
347
|
+
parts.push("rate-limited");
|
|
348
|
+
} else if (recovery === "plain") {
|
|
349
|
+
parts.push("error");
|
|
350
|
+
}
|
|
351
|
+
if (recovery === "credential") {
|
|
352
|
+
parts.push("switched account");
|
|
353
|
+
} else if (recovery === "model") {
|
|
354
|
+
parts.push("switched model");
|
|
355
|
+
} else if (recovery === "wait") {
|
|
356
|
+
parts.push("waited");
|
|
357
|
+
}
|
|
358
|
+
parts.push("retried");
|
|
359
|
+
return parts.join("; ");
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
async #recordPendingRecoveredRetryError(
|
|
363
|
+
message: AssistantMessage,
|
|
364
|
+
id: number,
|
|
365
|
+
options: { switchedCredential: boolean; switchedModel: boolean; delayMs: number },
|
|
366
|
+
): Promise<void> {
|
|
367
|
+
await this.persistTerminalEmptyErrorTurn(message);
|
|
368
|
+
const persistenceKey = sessionMessagePersistenceKey(message);
|
|
369
|
+
if (!persistenceKey) return;
|
|
370
|
+
let branchEntry: SessionEntry | undefined;
|
|
371
|
+
for (const entry of this.#host.sessionManager.getBranch().slice().reverse()) {
|
|
372
|
+
if (entry.type !== "message" || entry.message.role !== "assistant") continue;
|
|
373
|
+
if (sessionMessagePersistenceKey(entry.message) !== persistenceKey) continue;
|
|
374
|
+
if (!sameMessageContent(entry.message, message) && !this.#isSameAssistantMessage(entry.message, message)) {
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
branchEntry = entry;
|
|
378
|
+
break;
|
|
379
|
+
}
|
|
380
|
+
if (!branchEntry) return;
|
|
381
|
+
if (this.#pendingRecoveredRetryErrors.some(error => error.entryId === branchEntry.id)) return;
|
|
382
|
+
const rateLimited = AIError.is(id, AIError.Flag.UsageLimit);
|
|
383
|
+
const recovery = this.#retryRecoveryKind(id, options.switchedCredential, options.switchedModel, options.delayMs);
|
|
384
|
+
const note = this.#retryRecoveryNote(recovery, rateLimited);
|
|
385
|
+
this.#pendingRecoveredRetryErrors.push({
|
|
386
|
+
entryId: branchEntry.id,
|
|
387
|
+
persistenceKey,
|
|
388
|
+
recovery,
|
|
389
|
+
attempt: this.#retryAttempt,
|
|
390
|
+
note,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
async #markPendingRecoveredRetryErrors(supersedingMessage: AssistantMessage): Promise<RecoveredRetryError[]> {
|
|
395
|
+
if (this.#pendingRecoveredRetryErrors.length === 0) return [];
|
|
396
|
+
const branch = this.#host.sessionManager.getBranch();
|
|
397
|
+
const branchById = new Map<string, SessionEntry>();
|
|
398
|
+
for (const entry of branch) {
|
|
399
|
+
branchById.set(entry.id, entry);
|
|
400
|
+
}
|
|
401
|
+
const recoveredAt = new Date().toISOString();
|
|
402
|
+
const supersededBy: AssistantRetryRecovery["supersededBy"] = {
|
|
403
|
+
timestamp: supersedingMessage.timestamp,
|
|
404
|
+
provider: supersedingMessage.provider,
|
|
405
|
+
model: supersedingMessage.model,
|
|
406
|
+
};
|
|
407
|
+
if (supersedingMessage.responseId) {
|
|
408
|
+
supersededBy.responseId = supersedingMessage.responseId;
|
|
409
|
+
}
|
|
410
|
+
const recoveredErrors: RecoveredRetryError[] = [];
|
|
411
|
+
for (const pending of this.#pendingRecoveredRetryErrors) {
|
|
412
|
+
let entry = branchById.get(pending.entryId);
|
|
413
|
+
if (entry?.type !== "message" || entry.message.role !== "assistant") {
|
|
414
|
+
entry = branch
|
|
415
|
+
.slice()
|
|
416
|
+
.reverse()
|
|
417
|
+
.find(
|
|
418
|
+
candidate =>
|
|
419
|
+
candidate.type === "message" &&
|
|
420
|
+
candidate.message.role === "assistant" &&
|
|
421
|
+
sessionMessagePersistenceKey(candidate.message) === pending.persistenceKey,
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
if (entry?.type !== "message" || entry.message.role !== "assistant") continue;
|
|
425
|
+
const retryRecovery: AssistantRetryRecovery = {
|
|
426
|
+
kind: "auto-retry",
|
|
427
|
+
status: "recovered",
|
|
428
|
+
attempt: pending.attempt,
|
|
429
|
+
recoveredAt,
|
|
430
|
+
recovery: pending.recovery,
|
|
431
|
+
note: pending.note,
|
|
432
|
+
supersededBy,
|
|
433
|
+
};
|
|
434
|
+
entry.message.retryRecovery = retryRecovery;
|
|
435
|
+
recoveredErrors.push({
|
|
436
|
+
entryId: entry.id,
|
|
437
|
+
persistenceKey: pending.persistenceKey,
|
|
438
|
+
note: retryRecovery.note,
|
|
439
|
+
retryRecovery,
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
if (recoveredErrors.length > 0) {
|
|
443
|
+
await this.#host.sessionManager.rewriteEntries();
|
|
444
|
+
}
|
|
445
|
+
return recoveredErrors;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
async #handleEmptyAssistantStop(assistantMessage: AssistantMessage): Promise<boolean> {
|
|
449
|
+
if (!this.#isEmptyAssistantStop(assistantMessage)) {
|
|
450
|
+
this.#emptyStopRetryCount = 0;
|
|
451
|
+
return false;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
if (this.#acceptTerminalEmptyStopForPrompt && assistantMessage.stopReason === "stop") {
|
|
455
|
+
this.#acceptTerminalEmptyStopForPrompt = false;
|
|
456
|
+
this.#discardAcceptedTerminalEmptyStop(assistantMessage);
|
|
457
|
+
this.#emptyStopRetryCount = 0;
|
|
458
|
+
return false;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
this.#emptyStopRetryCount++;
|
|
462
|
+
if (this.#emptyStopRetryCount > EMPTY_STOP_MAX_RETRIES) {
|
|
463
|
+
const attempts = this.#emptyStopRetryCount - 1;
|
|
464
|
+
const finalError =
|
|
465
|
+
"Assistant returned empty stop after retry cap; try switching models or `/shake images` to remove archived frames";
|
|
466
|
+
logger.warn(finalError, {
|
|
467
|
+
attempts,
|
|
468
|
+
model: assistantMessage.model,
|
|
469
|
+
provider: assistantMessage.provider,
|
|
470
|
+
});
|
|
471
|
+
await this.#host.emitSessionEvent({
|
|
472
|
+
type: "auto_retry_end",
|
|
473
|
+
success: false,
|
|
474
|
+
attempt: this.#retryAttempt > 0 ? this.#retryAttempt : attempts,
|
|
475
|
+
finalError,
|
|
476
|
+
});
|
|
477
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
478
|
+
this.#retryAttempt = 0;
|
|
479
|
+
this.resolveRetry();
|
|
480
|
+
// A zero-content turn carries no transcript value, while its provider usage
|
|
481
|
+
// can anchor the next prompt at the full failed-request size and re-trigger
|
|
482
|
+
// compaction at the same boundary. Remove every capped empty stop; toolUse
|
|
483
|
+
// orphans still need this for Anthropic message-history validity.
|
|
484
|
+
await this.dropPersistedAssistantTurn(assistantMessage);
|
|
485
|
+
return false;
|
|
486
|
+
}
|
|
487
|
+
this.discardAssistantTurn(assistantMessage);
|
|
488
|
+
this.#host.agent.appendMessage({
|
|
489
|
+
role: "developer",
|
|
490
|
+
content: [{ type: "text", text: this.#emptyStopRetryReminder() }],
|
|
491
|
+
attribution: "agent",
|
|
492
|
+
timestamp: Date.now(),
|
|
493
|
+
});
|
|
494
|
+
this.#host.scheduleAgentContinue({ generation: this.#host.promptGeneration() });
|
|
495
|
+
return true;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
#isEmptyAssistantStop(assistantMessage: AssistantMessage): boolean {
|
|
499
|
+
switch (assistantMessage.stopReason) {
|
|
500
|
+
case "stop":
|
|
501
|
+
// Unsigned thinking alone is not actionable, but a signature is
|
|
502
|
+
// provider-authenticated content and makes the stop terminal.
|
|
503
|
+
for (const content of assistantMessage.content) {
|
|
504
|
+
if (content.type === "toolCall") return false;
|
|
505
|
+
if (content.type === "text" && hasNonWhitespace(content.text)) return false;
|
|
506
|
+
if (content.type === "thinking" && hasNonWhitespace(content.thinkingSignature ?? "")) return false;
|
|
507
|
+
}
|
|
508
|
+
return true;
|
|
509
|
+
case "toolUse":
|
|
510
|
+
// An orphaned toolUse stop (no tool_use block) corrupts Anthropic history:
|
|
511
|
+
// a later tool_result has nothing to anchor to. Thinking alone cannot anchor
|
|
512
|
+
// a tool_result, so it does not rescue a toolUse stop here.
|
|
513
|
+
for (const content of assistantMessage.content) {
|
|
514
|
+
if (content.type === "toolCall") return false;
|
|
515
|
+
if (content.type === "text" && hasNonWhitespace(content.text)) return false;
|
|
516
|
+
}
|
|
517
|
+
return true;
|
|
518
|
+
default:
|
|
519
|
+
return false;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
#emptyStopRetryReminder(): string {
|
|
524
|
+
return prompt.render(emptyStopRetryTemplate, {
|
|
525
|
+
retryCount: this.#emptyStopRetryCount,
|
|
526
|
+
maxRetries: EMPTY_STOP_MAX_RETRIES,
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
async #handleUnexpectedAssistantStop(assistantMessage: AssistantMessage): Promise<boolean> {
|
|
530
|
+
if (!this.#host.settings.get("features.unexpectedStopDetection")) {
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
if (!isUnexpectedStopCandidate(assistantMessage)) {
|
|
534
|
+
this.#unexpectedStopRetryCount = 0;
|
|
535
|
+
return false;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const text = assistantMessage.content
|
|
539
|
+
.filter((content): content is TextContent => content.type === "text")
|
|
540
|
+
.map(content => content.text)
|
|
541
|
+
.join("\n");
|
|
542
|
+
if (!/\S/.test(text)) {
|
|
543
|
+
this.#unexpectedStopRetryCount = 0;
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
const controller = new AbortController();
|
|
548
|
+
const timeout = setTimeout(() => controller.abort(), UNEXPECTED_STOP_TIMEOUT_MS);
|
|
549
|
+
let classification: boolean | undefined;
|
|
550
|
+
try {
|
|
551
|
+
classification = await classifyUnexpectedStop(text, {
|
|
552
|
+
settings: this.#host.settings,
|
|
553
|
+
registry: this.#host.modelRegistry,
|
|
554
|
+
sessionId: this.#host.sessionId(),
|
|
555
|
+
metadataResolver: (provider: string) => this.#host.agent.metadataForProvider(provider),
|
|
556
|
+
signal: controller.signal,
|
|
557
|
+
});
|
|
558
|
+
} finally {
|
|
559
|
+
clearTimeout(timeout);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
if (classification !== true) {
|
|
563
|
+
this.#unexpectedStopRetryCount = 0;
|
|
564
|
+
return false;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
this.#unexpectedStopRetryCount++;
|
|
568
|
+
if (this.#unexpectedStopRetryCount > UNEXPECTED_STOP_MAX_RETRIES) {
|
|
569
|
+
logger.warn("Assistant returned unexpected stop after retry cap", {
|
|
570
|
+
attempts: this.#unexpectedStopRetryCount - 1,
|
|
571
|
+
model: assistantMessage.model,
|
|
572
|
+
provider: assistantMessage.provider,
|
|
573
|
+
});
|
|
574
|
+
this.#unexpectedStopRetryCount = 0;
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
this.#host.agent.appendMessage({
|
|
579
|
+
role: "developer",
|
|
580
|
+
content: [{ type: "text", text: this.#unexpectedStopRetryReminder() }],
|
|
581
|
+
attribution: "agent",
|
|
582
|
+
timestamp: Date.now(),
|
|
583
|
+
});
|
|
584
|
+
this.#host.scheduleAgentContinue({ generation: this.#host.promptGeneration() });
|
|
585
|
+
return true;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
#unexpectedStopRetryReminder(): string {
|
|
589
|
+
return prompt.render(unexpectedStopRetryTemplate, {
|
|
590
|
+
retryCount: this.#unexpectedStopRetryCount,
|
|
591
|
+
maxRetries: UNEXPECTED_STOP_MAX_RETRIES,
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
removeAssistantMessageFromActiveContext(
|
|
596
|
+
assistantMessage: AssistantMessage,
|
|
597
|
+
reason = "assistant-context-cleanup",
|
|
598
|
+
): void {
|
|
599
|
+
const messages = this.#host.agent.state.messages;
|
|
600
|
+
const lastMessage = messages[messages.length - 1];
|
|
601
|
+
const lastAssistant: AssistantMessage | undefined = lastMessage?.role === "assistant" ? lastMessage : undefined;
|
|
602
|
+
if (lastAssistant !== undefined && this.#isSameAssistantMessage(lastAssistant, assistantMessage)) {
|
|
603
|
+
this.#host.agent.replaceMessages(messages.slice(0, -1));
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
// A miss means the failed turn is still in active context (or was never
|
|
607
|
+
// there); log just enough to explain why the identity check failed.
|
|
608
|
+
logger.debug("agent active context assistant removal missed", {
|
|
609
|
+
reason,
|
|
610
|
+
lastRole: lastMessage?.role,
|
|
611
|
+
candidateTimestamp: assistantMessage.timestamp,
|
|
612
|
+
lastTimestamp: lastAssistant?.timestamp,
|
|
613
|
+
candidateStopReason: assistantMessage.stopReason,
|
|
614
|
+
lastStopReason: lastAssistant?.stopReason,
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Drop a recoverable assistant turn from the persisted session branch once a
|
|
620
|
+
* recovery path (context promotion or compaction) is committed. Waits for the
|
|
621
|
+
* in-flight `message_end` persistence slot first so the branch entry exists
|
|
622
|
+
* before we reparent past it. Active context removal is the caller's
|
|
623
|
+
* responsibility — recovery paths clear it eagerly so the retry never
|
|
624
|
+
* replays the failed turn, while no-recovery paths leave the persisted entry
|
|
625
|
+
* (and the user-visible transcript line) in place.
|
|
626
|
+
*/
|
|
627
|
+
async #dropPersistedAssistantTurn(assistantMessage: AssistantMessage): Promise<void> {
|
|
628
|
+
await this.#host.waitForSessionMessagePersistence(assistantMessage);
|
|
629
|
+
this.discardAssistantTurn(assistantMessage);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Drop the failed assistant turn from persisted history, run
|
|
634
|
+
* {@link #runAutoCompaction} for an `overflow` / `incomplete` recovery, and
|
|
635
|
+
* restore the assistant entry if compaction did not actually commit
|
|
636
|
+
* anything (no usable model/preparation, hook cancel, compaction error,
|
|
637
|
+
* or a no-progress automatic-continuation block before any summary was
|
|
638
|
+
* written).
|
|
639
|
+
*
|
|
640
|
+
* Compaction has to see a clean branch — otherwise its `prepareCompaction`
|
|
641
|
+
* pass would keep the failed turn in the kept region and the retry would
|
|
642
|
+
* replay it. But a return that was not paired with a fresh compaction
|
|
643
|
+
* summary or a successful history rewrite means no recovery is in progress,
|
|
644
|
+
* even if queued user input gets drained next. Restoring the failed turn
|
|
645
|
+
* before that continuation preserves the visible stop reason and rebuilds the
|
|
646
|
+
* active assistant tail that `Agent.continue()` needs to dequeue follow-ups.
|
|
647
|
+
*/
|
|
648
|
+
async #runRecoveryCompactionWithRollback(
|
|
649
|
+
reason: "overflow" | "incomplete",
|
|
650
|
+
assistantMessage: AssistantMessage,
|
|
651
|
+
allowDefer: boolean,
|
|
652
|
+
options: { autoContinue: boolean; triggerContextTokens?: number },
|
|
653
|
+
): Promise<RecoveryCompactionResult> {
|
|
654
|
+
const compactionEntryBefore = getLatestCompactionEntry(this.#host.sessionManager.getBranch());
|
|
655
|
+
await this.dropPersistedAssistantTurn(assistantMessage);
|
|
656
|
+
const result = await this.#host.runAutoCompaction(reason, true, false, allowDefer, {
|
|
657
|
+
autoContinue: options.autoContinue,
|
|
658
|
+
triggerContextTokens: options.triggerContextTokens,
|
|
659
|
+
phase: "mid_turn",
|
|
660
|
+
});
|
|
661
|
+
const compactionEntryAfter = getLatestCompactionEntry(this.#host.sessionManager.getBranch());
|
|
662
|
+
if (result.historyRewritten !== true && compactionEntryAfter === compactionEntryBefore) {
|
|
663
|
+
this.#restoreFailedAssistantTurn(assistantMessage);
|
|
664
|
+
}
|
|
665
|
+
return result;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
#restoreFailedAssistantTurn(assistantMessage: AssistantMessage): void {
|
|
669
|
+
if (!isEmptyErrorTurn(assistantMessage)) this.#host.sessionManager.appendMessage(assistantMessage);
|
|
670
|
+
const lastMessage = this.#host.agent.state.messages.at(-1);
|
|
671
|
+
if (
|
|
672
|
+
lastMessage?.role === "assistant" &&
|
|
673
|
+
this.#isSameAssistantMessage(lastMessage as AssistantMessage, assistantMessage)
|
|
674
|
+
) {
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
this.#host.agent.appendMessage(assistantMessage);
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
#discardAcceptedTerminalEmptyStop(assistantMessage: AssistantMessage): void {
|
|
681
|
+
const branch = this.#host.sessionManager.getBranch();
|
|
682
|
+
const branchEntry = branch
|
|
683
|
+
.slice()
|
|
684
|
+
.reverse()
|
|
685
|
+
.find(
|
|
686
|
+
entry =>
|
|
687
|
+
entry.type === "message" &&
|
|
688
|
+
entry.message.role === "assistant" &&
|
|
689
|
+
this.#isSameAssistantMessage(entry.message, assistantMessage),
|
|
690
|
+
);
|
|
691
|
+
const parentEntry =
|
|
692
|
+
branchEntry?.parentId === null || branchEntry?.parentId === undefined
|
|
693
|
+
? undefined
|
|
694
|
+
: branch.find(entry => entry.id === branchEntry.parentId);
|
|
695
|
+
const prunePrompt = parentEntry?.type === "custom_message";
|
|
696
|
+
|
|
697
|
+
this.removeAssistantMessageFromActiveContext(assistantMessage, "accepted-terminal-empty-stop");
|
|
698
|
+
if (prunePrompt && this.#host.agent.state.messages.at(-1)?.role === "custom") {
|
|
699
|
+
this.#host.agent.replaceMessages(this.#host.agent.state.messages.slice(0, -1));
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
if (!branchEntry) return;
|
|
703
|
+
const targetParentId = prunePrompt ? parentEntry.parentId : branchEntry.parentId;
|
|
704
|
+
this.#host.withBashBranchTransition(() => {
|
|
705
|
+
if (targetParentId === null) {
|
|
706
|
+
this.#host.sessionManager.resetLeaf();
|
|
707
|
+
} else {
|
|
708
|
+
this.#host.sessionManager.branch(targetParentId);
|
|
709
|
+
}
|
|
710
|
+
});
|
|
711
|
+
this.#host.sessionManager.appendCustomEntry("accepted-terminal-empty-stop");
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/**
|
|
715
|
+
* Drop an assistant turn from BOTH the live agent context and the persisted
|
|
716
|
+
* session branch (reparenting the leaf to the turn's parent), so a discarded
|
|
717
|
+
* turn does not resurface on reload. Used for empty/reasoning-only stops and
|
|
718
|
+
* the Gemini header-runaway interrupt, which must not replay a partial,
|
|
719
|
+
* loop-fueling thinking block.
|
|
720
|
+
*/
|
|
721
|
+
discardAssistantTurn(assistantMessage: AssistantMessage): void {
|
|
722
|
+
this.removeAssistantMessageFromActiveContext(assistantMessage);
|
|
723
|
+
|
|
724
|
+
const branchEntry = this.#host.sessionManager
|
|
725
|
+
.getBranch()
|
|
726
|
+
.slice()
|
|
727
|
+
.reverse()
|
|
728
|
+
.find(
|
|
729
|
+
entry =>
|
|
730
|
+
entry.type === "message" &&
|
|
731
|
+
entry.message.role === "assistant" &&
|
|
732
|
+
this.#isSameAssistantMessage(entry.message as AssistantMessage, assistantMessage),
|
|
733
|
+
);
|
|
734
|
+
if (!branchEntry) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
this.#host.withBashBranchTransition(() => {
|
|
738
|
+
if (branchEntry.parentId === null) {
|
|
739
|
+
this.#host.sessionManager.resetLeaf();
|
|
740
|
+
} else {
|
|
741
|
+
this.#host.sessionManager.branch(branchEntry.parentId);
|
|
742
|
+
}
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
#isSameAssistantMessage(left: AssistantMessage, right: AssistantMessage): boolean {
|
|
747
|
+
return (
|
|
748
|
+
left === right ||
|
|
749
|
+
(left.timestamp === right.timestamp &&
|
|
750
|
+
left.provider === right.provider &&
|
|
751
|
+
left.model === right.model &&
|
|
752
|
+
left.stopReason === right.stopReason)
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Classify retry decisions against the active session model. Test stream
|
|
758
|
+
* shims and provider adapters can emit generic assistant metadata, but retry
|
|
759
|
+
* policy belongs to the model that was actually requested for this turn.
|
|
760
|
+
*/
|
|
761
|
+
#classifyRetryMessage(message: AssistantMessage): number {
|
|
762
|
+
const activeModel = this.#host.model();
|
|
763
|
+
if (!activeModel || message.api === activeModel.api) {
|
|
764
|
+
return AIError.classifyMessage(message);
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const id = AIError.classifyMessage({
|
|
768
|
+
api: activeModel.api,
|
|
769
|
+
errorId: message.errorId,
|
|
770
|
+
errorMessage: message.errorMessage,
|
|
771
|
+
errorStatus: message.errorStatus,
|
|
772
|
+
});
|
|
773
|
+
message.errorId = id;
|
|
774
|
+
return id;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
#isGenericAbortSentinel(message: AssistantMessage): boolean {
|
|
778
|
+
return message.errorMessage === "Request was aborted" || message.errorMessage === "Request was aborted.";
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Retry an empty, reason-less provider abort: a turn with no content that
|
|
783
|
+
* carries the generic sentinel (bare `abort()`), whether the provider
|
|
784
|
+
* finalized it as `stopReason: "aborted"` or leaked it as `stopReason:
|
|
785
|
+
* "error"` (a stalled/dropped stream reported as an error rather than an
|
|
786
|
+
* abort — issue #5375). Only fires while the session is neither aborting nor
|
|
787
|
+
* tearing down. A user/lifecycle abort (`#abortInProgress`), a dispose-driven
|
|
788
|
+
* abort (`#isDisposed`), or a session-induced streaming-edit guard abort
|
|
789
|
+
* (`StreamingEditGuard.abortTriggered` — auto-generated-file guard or failed-patch
|
|
790
|
+
* preview) is deliberate and MUST settle the turn instead: routing it through
|
|
791
|
+
* retry would orphan `#retryPromise` on a continuation the guard skips
|
|
792
|
+
* (hanging the in-flight `prompt()`) or silently undo the guard's intended
|
|
793
|
+
* abort. Deliberate user interrupts (`UserInterrupt`) and silent aborts carry
|
|
794
|
+
* their own marker, not the generic sentinel, so they never match here.
|
|
795
|
+
*/
|
|
796
|
+
isRetryableReasonlessAbort(message: AssistantMessage): boolean {
|
|
797
|
+
if (
|
|
798
|
+
(message.stopReason !== "aborted" && message.stopReason !== "error") ||
|
|
799
|
+
message.content.length !== 0 ||
|
|
800
|
+
this.#host.abortInProgress() ||
|
|
801
|
+
this.#host.isDisposed() ||
|
|
802
|
+
this.#host.streamingEditAbortTriggered()
|
|
803
|
+
) {
|
|
804
|
+
return false;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
const id = this.#classifyRetryMessage(message);
|
|
808
|
+
if (message.stopReason === "aborted" && AIError.is(id, AIError.Flag.Abort)) return true;
|
|
809
|
+
if (!this.#isGenericAbortSentinel(message)) return false;
|
|
810
|
+
|
|
811
|
+
message.errorId = AIError.create(AIError.Flag.Abort);
|
|
812
|
+
return true;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Check if an error is retryable (transient errors or usage limits).
|
|
817
|
+
* Context overflow is NOT retryable (handled by compaction instead).
|
|
818
|
+
* Usage-limit errors are retryable because the retry handler performs credential switching.
|
|
819
|
+
*/
|
|
820
|
+
isRetryableError(message: AssistantMessage): boolean {
|
|
821
|
+
if (message.stopReason !== "error") return false;
|
|
822
|
+
|
|
823
|
+
const id = this.#classifyRetryMessage(message);
|
|
824
|
+
// Context overflow is handled by compaction, not retry
|
|
825
|
+
const contextWindow = this.#host.model()?.contextWindow ?? 0;
|
|
826
|
+
if (AIError.isContextOverflow(message, contextWindow)) return false;
|
|
827
|
+
|
|
828
|
+
if (this.isClassifierRefusal(message)) return true;
|
|
829
|
+
return AIError.retriable(id, { replayUnsafe: this.#hasReplayUnsafeToolOutput(message) });
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Resume a stalled turn after every emitted tool call has produced a result.
|
|
834
|
+
* Cursor calls must also carry the server-execution marker. The failed
|
|
835
|
+
* assistant/tool-result pair stays in context so completed side effects are
|
|
836
|
+
* continued from rather than replayed.
|
|
837
|
+
*/
|
|
838
|
+
canResumeResolvedStreamStall(message: AssistantMessage): boolean {
|
|
839
|
+
if (message.stopReason !== "error" || !message.errorMessage?.toLowerCase().includes("stream stall")) {
|
|
840
|
+
return false;
|
|
841
|
+
}
|
|
842
|
+
const id = this.#classifyRetryMessage(message);
|
|
843
|
+
if (!AIError.retriable(id)) return false;
|
|
844
|
+
|
|
845
|
+
const resolvedToolCallIds: string[] = [];
|
|
846
|
+
for (const block of message.content) {
|
|
847
|
+
if (block.type !== "toolCall") continue;
|
|
848
|
+
if (
|
|
849
|
+
message.provider === "cursor" &&
|
|
850
|
+
(!(kCursorExecResolved in block) || block[kCursorExecResolved] !== true)
|
|
851
|
+
) {
|
|
852
|
+
return false;
|
|
853
|
+
}
|
|
854
|
+
resolvedToolCallIds.push(block.id);
|
|
855
|
+
}
|
|
856
|
+
if (resolvedToolCallIds.length === 0) return false;
|
|
857
|
+
|
|
858
|
+
const messages = this.#host.agent.state.messages;
|
|
859
|
+
let assistantIndex = -1;
|
|
860
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
861
|
+
const candidate = messages[i];
|
|
862
|
+
if (candidate.role === "assistant" && this.#isSameAssistantMessage(candidate, message)) {
|
|
863
|
+
assistantIndex = i;
|
|
864
|
+
break;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
if (assistantIndex < 0) return false;
|
|
868
|
+
|
|
869
|
+
const unresolvedToolCallIds = new Set(resolvedToolCallIds);
|
|
870
|
+
for (let i = assistantIndex + 1; i < messages.length; i++) {
|
|
871
|
+
const candidate = messages[i];
|
|
872
|
+
if (candidate.role === "toolResult") unresolvedToolCallIds.delete(candidate.toolCallId);
|
|
873
|
+
}
|
|
874
|
+
return unresolvedToolCallIds.size === 0;
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* Retried turns remove the failed assistant message from active context.
|
|
878
|
+
* Text/thinking-only partials are safe to discard and replay. Retained
|
|
879
|
+
* tool calls are not: a completed tool call may already have emitted its
|
|
880
|
+
* tool result after this assistant message, so replaying can duplicate work.
|
|
881
|
+
*/
|
|
882
|
+
#hasReplayUnsafeToolOutput(message: AssistantMessage): boolean {
|
|
883
|
+
return message.content.some(block => block.type === "toolCall");
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/**
|
|
887
|
+
* OpenRouter can repeatedly close Gemini streams at the reasoning-to-payload
|
|
888
|
+
* transition. One retry covers a transient edge failure; the normal ten-retry
|
|
889
|
+
* budget would otherwise re-run the same expensive reasoning cycle unchanged.
|
|
890
|
+
*/
|
|
891
|
+
#isOpenRouterThinkingStreamClose(message: AssistantMessage): boolean {
|
|
892
|
+
return (
|
|
893
|
+
message.provider === "openrouter" &&
|
|
894
|
+
/server_error:\s*stream closed with reason:\s*error/i.test(message.errorMessage ?? "") &&
|
|
895
|
+
message.content.some(block => block.type === "thinking" && block.thinking.trim().length > 0)
|
|
896
|
+
);
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/** Checks whether a provider error represents a classifier refusal. */
|
|
900
|
+
isClassifierRefusal(message: AssistantMessage): boolean {
|
|
901
|
+
if (message.stopReason !== "error") return false;
|
|
902
|
+
const stopType = message.stopDetails?.type;
|
|
903
|
+
return stopType === "refusal" || stopType === "sensitive";
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
#getRetryFallbackResolutionContext(): RetryFallbackResolutionContext {
|
|
907
|
+
return {
|
|
908
|
+
chains: this.#getRetryFallbackChains(),
|
|
909
|
+
getModelRole: role => this.#host.settings.getModelRole(role),
|
|
910
|
+
modelLookup: this.#host.modelRegistry,
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
#getRetryFallbackChains(): RetryFallbackChains {
|
|
915
|
+
return getRetryFallbackChains(this.#host.settings);
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
#validateRetryFallbackChains(): void {
|
|
919
|
+
validateRetryFallbackChains(this.#host.settings, this.#host.modelRegistry, message =>
|
|
920
|
+
this.#host.configWarnings.push(message),
|
|
921
|
+
);
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
#getRetryFallbackRevertPolicy(): RetryFallbackRevertPolicy {
|
|
925
|
+
return getRetryFallbackRevertPolicy(this.#host.settings);
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/** Clears fallback ownership after an explicit model change. */
|
|
929
|
+
clearActiveRetryFallback(): void {
|
|
930
|
+
this.#activeRetryFallback = undefined;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/** Checks whether a fallback selector remains in cooldown. */
|
|
934
|
+
isRetryFallbackSelectorSuppressed(selector: RetryFallbackSelector): boolean {
|
|
935
|
+
return this.#host.modelRegistry.isSelectorSuppressed(selector.raw);
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/** Records the cooldown that should suppress a failing selector. */
|
|
939
|
+
noteRetryFallbackCooldown(currentSelector: string, retryAfterMs: number | undefined, errorMessage: string): void {
|
|
940
|
+
let cooldownMs = retryAfterMs;
|
|
941
|
+
if (!cooldownMs || cooldownMs <= 0) {
|
|
942
|
+
const reason = parseRateLimitReason(errorMessage);
|
|
943
|
+
cooldownMs = reason === "UNKNOWN" ? 5 * 60 * 1000 : calculateRateLimitBackoffMs(reason);
|
|
944
|
+
}
|
|
945
|
+
this.#host.modelRegistry.suppressSelector(currentSelector, Date.now() + cooldownMs);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Map the failing model selector to the chain key that owns it, by
|
|
950
|
+
* specificity: an exact model-selector key, then a `provider/*` wildcard,
|
|
951
|
+
* then a model role whose current assignment matches, then `default`.
|
|
952
|
+
* Model-oriented keys win over roles so a chain follows the model across
|
|
953
|
+
* role reassignments.
|
|
954
|
+
*/
|
|
955
|
+
resolveRetryFallbackRole(
|
|
956
|
+
currentSelector: string,
|
|
957
|
+
currentModel: Model | null | undefined = this.#host.model(),
|
|
958
|
+
): string | undefined {
|
|
959
|
+
return resolveRetryFallbackChainKey(this.#getRetryFallbackResolutionContext(), currentSelector, currentModel);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/** Finds fallback candidates that follow the active selector. */
|
|
963
|
+
findRetryFallbackCandidates(
|
|
964
|
+
role: string,
|
|
965
|
+
currentSelector: string,
|
|
966
|
+
currentModel: Model | null | undefined = this.#host.model(),
|
|
967
|
+
): RetryFallbackSelector[] {
|
|
968
|
+
return findRetryFallbackCandidates(
|
|
969
|
+
this.#getRetryFallbackResolutionContext(),
|
|
970
|
+
role,
|
|
971
|
+
currentSelector,
|
|
972
|
+
currentModel,
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
async applyRetryFallbackCandidate(
|
|
977
|
+
role: string,
|
|
978
|
+
selector: RetryFallbackSelector,
|
|
979
|
+
currentSelector: string,
|
|
980
|
+
options?: { pinFallback?: boolean; apiKey?: string; signal?: AbortSignal },
|
|
981
|
+
): Promise<void> {
|
|
982
|
+
const resolved = resolveModelOverride([selector.raw], this.#host.modelRegistry, this.#host.settings);
|
|
983
|
+
const candidate = resolved.model ?? this.#host.modelRegistry.find(selector.provider, selector.id);
|
|
984
|
+
if (!candidate) {
|
|
985
|
+
throw new Error(`Retry fallback model not found: ${selector.raw}`);
|
|
986
|
+
}
|
|
987
|
+
const apiKey =
|
|
988
|
+
options?.apiKey ?? (await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId(), options));
|
|
989
|
+
if (!apiKey) {
|
|
990
|
+
throw new Error(`No API key for retry fallback ${selector.raw}`);
|
|
991
|
+
}
|
|
992
|
+
if (options?.signal?.aborted) return;
|
|
993
|
+
|
|
994
|
+
// Capture the configured selector (auto-aware) so a fallback chain preserves
|
|
995
|
+
// `auto` instead of collapsing it to the level it resolved to this turn.
|
|
996
|
+
const currentThinkingLevel = this.#host.configuredThinkingLevel();
|
|
997
|
+
const nextThinkingLevel = selector.thinkingLevel ?? currentThinkingLevel;
|
|
998
|
+
const candidateSelector = formatModelStringWithRouting(candidate);
|
|
999
|
+
this.#host.setModelWithProviderSessionReset(candidate);
|
|
1000
|
+
this.#host.sessionManager.appendModelChange(candidateSelector, EPHEMERAL_MODEL_CHANGE_ROLE);
|
|
1001
|
+
this.#host.settings.getStorage()?.recordModelUsage(candidateSelector);
|
|
1002
|
+
this.#host.setThinkingLevel(nextThinkingLevel);
|
|
1003
|
+
if (!this.#activeRetryFallback) {
|
|
1004
|
+
this.#activeRetryFallback = {
|
|
1005
|
+
role,
|
|
1006
|
+
originalSelector: currentSelector,
|
|
1007
|
+
originalThinkingLevel: currentThinkingLevel,
|
|
1008
|
+
lastAppliedFallbackThinkingLevel: nextThinkingLevel,
|
|
1009
|
+
pinned: options?.pinFallback === true,
|
|
1010
|
+
};
|
|
1011
|
+
} else {
|
|
1012
|
+
this.#activeRetryFallback.lastAppliedFallbackThinkingLevel = nextThinkingLevel;
|
|
1013
|
+
this.#activeRetryFallback.pinned = this.#activeRetryFallback.pinned || options?.pinFallback === true;
|
|
1014
|
+
}
|
|
1015
|
+
await this.#host.emitSessionEvent({
|
|
1016
|
+
type: "retry_fallback_applied",
|
|
1017
|
+
from: currentSelector,
|
|
1018
|
+
to: selector.raw,
|
|
1019
|
+
role,
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
async #tryRetryModelFallback(currentSelector: string, options?: { pinFallback?: boolean }): Promise<boolean> {
|
|
1024
|
+
const role = this.#activeRetryFallback?.role ?? this.resolveRetryFallbackRole(currentSelector);
|
|
1025
|
+
if (!role) return false;
|
|
1026
|
+
|
|
1027
|
+
for (const selector of this.findRetryFallbackCandidates(role, currentSelector)) {
|
|
1028
|
+
if (this.isRetryFallbackSelectorSuppressed(selector)) continue;
|
|
1029
|
+
const resolved = resolveModelOverride([selector.raw], this.#host.modelRegistry, this.#host.settings);
|
|
1030
|
+
const candidate = resolved.model ?? this.#host.modelRegistry.find(selector.provider, selector.id);
|
|
1031
|
+
if (!candidate) continue;
|
|
1032
|
+
const apiKey = await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId());
|
|
1033
|
+
if (!apiKey) continue;
|
|
1034
|
+
await this.applyRetryFallbackCandidate(role, selector, currentSelector, options);
|
|
1035
|
+
return true;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
return false;
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/** The active model when it is a Fireworks Fast (`-fast`) variant, else undefined. */
|
|
1042
|
+
#activeFireworksFastModel(): Model | undefined {
|
|
1043
|
+
const model = this.#host.model();
|
|
1044
|
+
return model?.provider === "fireworks" && isFireworksFastModelId(model.id) ? model : undefined;
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* True when the current turn failed on a Fireworks Fast (`-fast`) model in a
|
|
1049
|
+
* way that should degrade to the reliable base (Standard) model. Fast is a
|
|
1050
|
+
* speed-optimized router with no SLA, so any *pre-content* failure — a
|
|
1051
|
+
* transient overload/5xx or a hard "router/model not found / unsupported" —
|
|
1052
|
+
* is worth retrying on the base id. Skips failures the base model shares:
|
|
1053
|
+
* context overflow (compaction's job), usage limits and auth errors (same
|
|
1054
|
+
* account/key), and turns that already emitted a tool call (replaying would
|
|
1055
|
+
* duplicate work). Requires the base model to exist in the registry.
|
|
1056
|
+
*/
|
|
1057
|
+
isFireworksFastFallbackEligible(message: AssistantMessage): boolean {
|
|
1058
|
+
const model = this.#activeFireworksFastModel();
|
|
1059
|
+
if (!model) return false;
|
|
1060
|
+
if (message.stopReason !== "error") return false;
|
|
1061
|
+
if (message.content.some(block => block.type === "toolCall")) return false;
|
|
1062
|
+
// A content refusal/sensitivity stop is the model's decision, not a route
|
|
1063
|
+
// failure — switching to the base model would just re-trigger it.
|
|
1064
|
+
if (this.isClassifierRefusal(message)) return false;
|
|
1065
|
+
const id = this.#classifyRetryMessage(message);
|
|
1066
|
+
if (AIError.isContextOverflow(message, model.contextWindow ?? 0)) return false;
|
|
1067
|
+
if (AIError.is(id, AIError.Flag.UsageLimit)) return false;
|
|
1068
|
+
if (AIError.is(id, AIError.Flag.AuthFailed)) return false;
|
|
1069
|
+
return this.#host.modelRegistry.find("fireworks", toFireworksBaseModelId(model.id)) !== undefined;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
/**
|
|
1073
|
+
* True when a turn failed with a hard (non-retryable) provider error but a
|
|
1074
|
+
* configured `retry.fallbackChains` entry covers the active model: the same
|
|
1075
|
+
* model is not worth retrying, yet a DIFFERENT model is a fresh chance, so
|
|
1076
|
+
* the chain is consulted before the error becomes final. Skips failures a
|
|
1077
|
+
* model switch cannot fix or must not replay: cancellations (abort-flavored
|
|
1078
|
+
* errors are not model faults), context overflow (compaction's job),
|
|
1079
|
+
* classifier refusals (chain consult is handled on the retryable path with
|
|
1080
|
+
* `pinFallback`), and turns that already emitted a tool call (replaying
|
|
1081
|
+
* could duplicate work).
|
|
1082
|
+
*/
|
|
1083
|
+
isHardErrorFallbackEligible(message: AssistantMessage): boolean {
|
|
1084
|
+
if (message.stopReason !== "error") return false;
|
|
1085
|
+
const model = this.#host.model();
|
|
1086
|
+
if (!model) return false;
|
|
1087
|
+
const retrySettings = this.#host.settings.getGroup("retry");
|
|
1088
|
+
if (!retrySettings.enabled || !retrySettings.modelFallback) return false;
|
|
1089
|
+
if (this.isClassifierRefusal(message)) return false;
|
|
1090
|
+
const id = this.#classifyRetryMessage(message);
|
|
1091
|
+
if (AIError.is(id, AIError.Flag.Abort) || AIError.is(id, AIError.Flag.UserInterrupt)) return false;
|
|
1092
|
+
if (AIError.isContextOverflow(message, model.contextWindow ?? 0)) return false;
|
|
1093
|
+
if (this.#hasReplayUnsafeToolOutput(message)) return false;
|
|
1094
|
+
const currentSelector = formatRetryFallbackSelector(model, this.#host.thinkingLevel());
|
|
1095
|
+
const role = this.#activeRetryFallback?.role ?? this.resolveRetryFallbackRole(currentSelector);
|
|
1096
|
+
if (!role) return false;
|
|
1097
|
+
return this.findRetryFallbackCandidates(role, currentSelector).length > 0;
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
/**
|
|
1101
|
+
* Switch the active model from a Fireworks Fast (`-fast`) variant to its base
|
|
1102
|
+
* (Standard) id and stick there for the rest of the session — the auto
|
|
1103
|
+
* fallback that makes Fast a safe default. Returns false when the current
|
|
1104
|
+
* model is not a fast variant, the base id is missing, or it has no key.
|
|
1105
|
+
*/
|
|
1106
|
+
async #tryFireworksFastFallback(currentSelector: string): Promise<boolean> {
|
|
1107
|
+
const model = this.#activeFireworksFastModel();
|
|
1108
|
+
if (!model) return false;
|
|
1109
|
+
const baseModel = this.#host.modelRegistry.find("fireworks", toFireworksBaseModelId(model.id));
|
|
1110
|
+
if (!baseModel) return false;
|
|
1111
|
+
const apiKey = await this.#host.modelRegistry.getApiKey(baseModel, this.#host.sessionId());
|
|
1112
|
+
if (!apiKey) return false;
|
|
1113
|
+
const baseSelector = formatModelStringWithRouting(baseModel);
|
|
1114
|
+
this.#host.setModelWithProviderSessionReset(baseModel);
|
|
1115
|
+
this.#host.sessionManager.appendModelChange(baseSelector, EPHEMERAL_MODEL_CHANGE_ROLE);
|
|
1116
|
+
this.#host.settings.getStorage()?.recordModelUsage(baseSelector);
|
|
1117
|
+
await this.#host.emitSessionEvent({
|
|
1118
|
+
type: "retry_fallback_applied",
|
|
1119
|
+
from: currentSelector,
|
|
1120
|
+
to: baseSelector,
|
|
1121
|
+
role: "fireworks-fast",
|
|
1122
|
+
});
|
|
1123
|
+
return true;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
async #maybeRestoreRetryFallbackPrimary(): Promise<void> {
|
|
1127
|
+
if (!this.#activeRetryFallback) return;
|
|
1128
|
+
if (this.#activeRetryFallback.pinned) return;
|
|
1129
|
+
if (this.#getRetryFallbackRevertPolicy() !== "cooldown-expiry") return;
|
|
1130
|
+
|
|
1131
|
+
const {
|
|
1132
|
+
originalSelector: originalSelectorRaw,
|
|
1133
|
+
originalThinkingLevel,
|
|
1134
|
+
lastAppliedFallbackThinkingLevel,
|
|
1135
|
+
} = this.#activeRetryFallback;
|
|
1136
|
+
const originalSelector = parseRetryFallbackSelector(originalSelectorRaw, this.#host.modelRegistry);
|
|
1137
|
+
if (!originalSelector) {
|
|
1138
|
+
this.clearActiveRetryFallback();
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
const currentModel = this.#host.model();
|
|
1143
|
+
if (!currentModel) return;
|
|
1144
|
+
const currentSelector = formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel());
|
|
1145
|
+
if (currentSelector === originalSelector.raw) {
|
|
1146
|
+
if (!this.isRetryFallbackSelectorSuppressed(originalSelector)) {
|
|
1147
|
+
this.clearActiveRetryFallback();
|
|
1148
|
+
}
|
|
1149
|
+
return;
|
|
1150
|
+
}
|
|
1151
|
+
if (this.isRetryFallbackSelectorSuppressed(originalSelector)) return;
|
|
1152
|
+
|
|
1153
|
+
const resolvedPrimary = resolveModelOverride(
|
|
1154
|
+
[originalSelector.raw],
|
|
1155
|
+
this.#host.modelRegistry,
|
|
1156
|
+
this.#host.settings,
|
|
1157
|
+
);
|
|
1158
|
+
const primaryModel =
|
|
1159
|
+
resolvedPrimary.model ?? this.#host.modelRegistry.find(originalSelector.provider, originalSelector.id);
|
|
1160
|
+
if (!primaryModel) return;
|
|
1161
|
+
const apiKey = await this.#host.modelRegistry.getApiKey(primaryModel, this.#host.sessionId());
|
|
1162
|
+
if (!apiKey) return;
|
|
1163
|
+
|
|
1164
|
+
const currentThinkingLevel = this.#host.configuredThinkingLevel();
|
|
1165
|
+
const thinkingToApply =
|
|
1166
|
+
currentThinkingLevel === lastAppliedFallbackThinkingLevel ? originalThinkingLevel : currentThinkingLevel;
|
|
1167
|
+
const primarySelector = formatModelStringWithRouting(primaryModel);
|
|
1168
|
+
this.#host.setModelWithProviderSessionReset(primaryModel);
|
|
1169
|
+
this.#host.sessionManager.appendModelChange(primarySelector, EPHEMERAL_MODEL_CHANGE_ROLE);
|
|
1170
|
+
this.#host.settings.getStorage()?.recordModelUsage(primarySelector);
|
|
1171
|
+
this.#host.setThinkingLevel(thinkingToApply);
|
|
1172
|
+
this.clearActiveRetryFallback();
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
#parseRetryAfterMsFromError(errorMessage: string): number | undefined {
|
|
1176
|
+
const now = Date.now();
|
|
1177
|
+
const retryAfterMsMatch = /retry-after-ms\s*[:=]\s*(\d+)/i.exec(errorMessage);
|
|
1178
|
+
if (retryAfterMsMatch) {
|
|
1179
|
+
return Math.max(0, Number(retryAfterMsMatch[1]));
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
const retryAfterMatch = /retry-after\s*[:=]\s*([^\s,;]+)/i.exec(errorMessage);
|
|
1183
|
+
if (retryAfterMatch) {
|
|
1184
|
+
const value = retryAfterMatch[1];
|
|
1185
|
+
const seconds = Number(value);
|
|
1186
|
+
if (!Number.isNaN(seconds)) {
|
|
1187
|
+
return Math.max(0, seconds * 1000);
|
|
1188
|
+
}
|
|
1189
|
+
const dateMs = Date.parse(value);
|
|
1190
|
+
if (!Number.isNaN(dateMs)) {
|
|
1191
|
+
return Math.max(0, dateMs - now);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
const retryHintMs = extractRetryHint(undefined, errorMessage);
|
|
1196
|
+
if (retryHintMs !== undefined) {
|
|
1197
|
+
return retryHintMs;
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
const resetMsMatch = /x-ratelimit-reset-ms\s*[:=]\s*(\d+)/i.exec(errorMessage);
|
|
1201
|
+
if (resetMsMatch) {
|
|
1202
|
+
const resetMs = Number(resetMsMatch[1]);
|
|
1203
|
+
if (!Number.isNaN(resetMs)) {
|
|
1204
|
+
if (resetMs > 1_000_000_000_000) {
|
|
1205
|
+
return Math.max(0, resetMs - now);
|
|
1206
|
+
}
|
|
1207
|
+
return Math.max(0, resetMs);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
const resetMatch = /x-ratelimit-reset\s*[:=]\s*(\d+)/i.exec(errorMessage);
|
|
1212
|
+
if (resetMatch) {
|
|
1213
|
+
const resetSeconds = Number(resetMatch[1]);
|
|
1214
|
+
if (!Number.isNaN(resetSeconds)) {
|
|
1215
|
+
if (resetSeconds > 1_000_000_000) {
|
|
1216
|
+
return Math.max(0, resetSeconds * 1000 - now);
|
|
1217
|
+
}
|
|
1218
|
+
return Math.max(0, resetSeconds * 1000);
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
// Smart Fallback if no exact headers found
|
|
1223
|
+
return undefined;
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
/**
|
|
1227
|
+
* Handle retryable errors with exponential backoff, credential rotation, and
|
|
1228
|
+
* model-fallback chains. Also entered for NON-retryable errors when a switch
|
|
1229
|
+
* is the recovery (`fireworksFastFallback`, `hardErrorFallback`): then a
|
|
1230
|
+
* successful model switch retries immediately, and a failed switch surfaces
|
|
1231
|
+
* the error without a same-model backoff retry.
|
|
1232
|
+
* @returns true if retry was initiated, false if max retries exceeded or disabled
|
|
1233
|
+
*/
|
|
1234
|
+
async #handleRetryableError(
|
|
1235
|
+
message: AssistantMessage,
|
|
1236
|
+
options?: {
|
|
1237
|
+
allowModelFallback?: boolean;
|
|
1238
|
+
fireworksFastFallback?: boolean;
|
|
1239
|
+
hardErrorFallback?: boolean;
|
|
1240
|
+
preserveFailedTurn?: boolean;
|
|
1241
|
+
},
|
|
1242
|
+
): Promise<boolean> {
|
|
1243
|
+
const retrySettings = this.#host.settings.getGroup("retry");
|
|
1244
|
+
// The Fireworks Fast→base degrade is an intrinsic model-selection safety net,
|
|
1245
|
+
// not a retry loop, so it runs even when the user disabled retries: it switches
|
|
1246
|
+
// the model once and lets the base turn proceed.
|
|
1247
|
+
if (!retrySettings.enabled && !options?.fireworksFastFallback) return false;
|
|
1248
|
+
const classifierRefusal = this.isClassifierRefusal(message);
|
|
1249
|
+
|
|
1250
|
+
const generation = this.#host.promptGeneration();
|
|
1251
|
+
this.#retryAttempt++;
|
|
1252
|
+
|
|
1253
|
+
// Create retry promise on first attempt so waitForRetry() can await it
|
|
1254
|
+
// Ensure only one promise exists (avoid orphaned promises from concurrent calls)
|
|
1255
|
+
if (!this.#retryPromise) {
|
|
1256
|
+
const { promise, resolve } = Promise.withResolvers<void>();
|
|
1257
|
+
this.#retryPromise = promise;
|
|
1258
|
+
this.#retryResolve = resolve;
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
// All attempts on the current model are spent. Don't fail yet: the
|
|
1262
|
+
// fallback chain below gets one last consult. Credential rotation can
|
|
1263
|
+
// consume the entire budget without the fallback branch ever running
|
|
1264
|
+
// (every rotation sets switchedCredential and skips it), so without
|
|
1265
|
+
// this last resort a provider-wide usage cap never fails over to the
|
|
1266
|
+
// configured chain.
|
|
1267
|
+
const maxRetries = this.#isOpenRouterThinkingStreamClose(message)
|
|
1268
|
+
? Math.min(retrySettings.maxRetries, 1)
|
|
1269
|
+
: retrySettings.maxRetries;
|
|
1270
|
+
const retryBudgetExhausted = this.#retryAttempt > maxRetries;
|
|
1271
|
+
|
|
1272
|
+
const errorMessage = message.errorMessage || "Unknown error";
|
|
1273
|
+
const id = this.#classifyRetryMessage(message);
|
|
1274
|
+
const staleOpenAIResponsesReplayError = AIError.is(id, AIError.Flag.StaleResponsesItem);
|
|
1275
|
+
const parsedRetryAfterMs = this.#parseRetryAfterMsFromError(errorMessage);
|
|
1276
|
+
let delayMs = staleOpenAIResponsesReplayError
|
|
1277
|
+
? 0
|
|
1278
|
+
: calculateRetryBackoffDelayMs(retrySettings.baseDelayMs, this.#retryAttempt);
|
|
1279
|
+
let switchedCredential = false;
|
|
1280
|
+
let switchedModel = false;
|
|
1281
|
+
// Set when a usage-limit error pinned the wait to credential
|
|
1282
|
+
// availability — suppresses the generic retry-after bump below.
|
|
1283
|
+
let usageLimitWaitMs: number | undefined;
|
|
1284
|
+
|
|
1285
|
+
if (staleOpenAIResponsesReplayError) {
|
|
1286
|
+
this.#host.resetCurrentResponsesProviderSession("stale replay error");
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
const activeModel = this.#host.model();
|
|
1290
|
+
if (
|
|
1291
|
+
!retryBudgetExhausted &&
|
|
1292
|
+
activeModel &&
|
|
1293
|
+
!staleOpenAIResponsesReplayError &&
|
|
1294
|
+
AIError.is(id, AIError.Flag.UsageLimit)
|
|
1295
|
+
) {
|
|
1296
|
+
const retryAfterMs = parsedRetryAfterMs ?? calculateRateLimitBackoffMs(parseRateLimitReason(errorMessage));
|
|
1297
|
+
const outcome = await this.#host.modelRegistry.authStorage.markUsageLimitReached(
|
|
1298
|
+
activeModel.provider,
|
|
1299
|
+
this.#host.sessionId(),
|
|
1300
|
+
{
|
|
1301
|
+
retryAfterMs,
|
|
1302
|
+
baseUrl: activeModel.baseUrl,
|
|
1303
|
+
modelId: activeModel.id,
|
|
1304
|
+
},
|
|
1305
|
+
);
|
|
1306
|
+
if (outcome.switched) {
|
|
1307
|
+
switchedCredential = true;
|
|
1308
|
+
delayMs = 0;
|
|
1309
|
+
} else if (await this.#host.maybeAutoRedeemCodexReset()) {
|
|
1310
|
+
// A live usage-limit 429 on the active Codex account, with a banked
|
|
1311
|
+
// reset and the opt-in setting on: spend the reset and retry
|
|
1312
|
+
// immediately instead of waiting out the window. Runs after the
|
|
1313
|
+
// free sibling-switch above and before model fallback below.
|
|
1314
|
+
switchedCredential = true;
|
|
1315
|
+
delayMs = 0;
|
|
1316
|
+
} else {
|
|
1317
|
+
// No sibling credential is usable right now. Wait for whichever
|
|
1318
|
+
// comes first: the provider's retry-after window for the current
|
|
1319
|
+
// account, or the earliest moment a temporarily blocked sibling
|
|
1320
|
+
// frees up (e.g. a 60s post-401 block or a 5-min usage-probe
|
|
1321
|
+
// block) — the next attempt's getApiKey re-ranks and picks it up.
|
|
1322
|
+
// Without this, one short-lived sibling block escalates a
|
|
1323
|
+
// recoverable situation into the provider's multi-hour wait and
|
|
1324
|
+
// trips the fail-fast cap below.
|
|
1325
|
+
usageLimitWaitMs = retryAfterMs;
|
|
1326
|
+
if (outcome.retryAtMs !== undefined) {
|
|
1327
|
+
const siblingWaitMs = Math.max(0, outcome.retryAtMs - Date.now()) + SIBLING_UNBLOCK_BUFFER_MS;
|
|
1328
|
+
if (siblingWaitMs < usageLimitWaitMs) {
|
|
1329
|
+
usageLimitWaitMs = siblingWaitMs;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
if (usageLimitWaitMs > delayMs) {
|
|
1333
|
+
delayMs = usageLimitWaitMs;
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
const allowModelFallback = options?.allowModelFallback !== false;
|
|
1339
|
+
const currentModel = this.#host.model();
|
|
1340
|
+
const currentSelector = currentModel
|
|
1341
|
+
? formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel())
|
|
1342
|
+
: undefined;
|
|
1343
|
+
if (!staleOpenAIResponsesReplayError && !switchedCredential && currentSelector) {
|
|
1344
|
+
// A refusal chain stops at the retry budget: the exhausted-attempt
|
|
1345
|
+
// last resort is for provider failures, not classifier decisions.
|
|
1346
|
+
if (allowModelFallback && retrySettings.modelFallback && !(retryBudgetExhausted && classifierRefusal)) {
|
|
1347
|
+
if (!classifierRefusal) {
|
|
1348
|
+
this.noteRetryFallbackCooldown(currentSelector, parsedRetryAfterMs, errorMessage);
|
|
1349
|
+
}
|
|
1350
|
+
switchedModel = await this.#tryRetryModelFallback(currentSelector, { pinFallback: classifierRefusal });
|
|
1351
|
+
}
|
|
1352
|
+
// Auto fallback from a Fireworks Fast variant to its base model. Independent
|
|
1353
|
+
// of the role-fallback setting: it's intrinsic to the Fast contract (speed
|
|
1354
|
+
// best-effort, degrade to Standard on failure) and triggers on hard router
|
|
1355
|
+
// errors the generic retry classifier would otherwise reject.
|
|
1356
|
+
if (!switchedModel && allowModelFallback && options?.fireworksFastFallback) {
|
|
1357
|
+
switchedModel = await this.#tryFireworksFastFallback(currentSelector);
|
|
1358
|
+
}
|
|
1359
|
+
if (switchedModel) {
|
|
1360
|
+
delayMs = 0;
|
|
1361
|
+
} else if (usageLimitWaitMs === undefined && parsedRetryAfterMs && parsedRetryAfterMs > delayMs) {
|
|
1362
|
+
delayMs = parsedRetryAfterMs;
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
if (retryBudgetExhausted) {
|
|
1366
|
+
if (!switchedModel) {
|
|
1367
|
+
await this.persistTerminalEmptyErrorTurn(message);
|
|
1368
|
+
// Max retries exceeded and no fallback model to switch to: emit
|
|
1369
|
+
// final failure and reset.
|
|
1370
|
+
await this.#host.emitSessionEvent({
|
|
1371
|
+
type: "auto_retry_end",
|
|
1372
|
+
success: false,
|
|
1373
|
+
attempt: this.#retryAttempt - 1,
|
|
1374
|
+
finalError: message.errorMessage,
|
|
1375
|
+
});
|
|
1376
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
1377
|
+
this.#retryAttempt = 0;
|
|
1378
|
+
this.resolveRetry(); // Resolve so waitForRetry() completes
|
|
1379
|
+
return false;
|
|
1380
|
+
}
|
|
1381
|
+
// The fallback model gets a fresh retry budget — leaving the spent
|
|
1382
|
+
// counter in place would exhaust it again on its first error.
|
|
1383
|
+
this.#retryAttempt = 1;
|
|
1384
|
+
}
|
|
1385
|
+
if (classifierRefusal && !switchedModel) {
|
|
1386
|
+
// A prior attempt in this saga already announced `auto_retry_start`
|
|
1387
|
+
// (retryAttempt was incremented for each call to this method, so > 1
|
|
1388
|
+
// means at least one earlier attempt started the loop) but this
|
|
1389
|
+
// attempt is not going to retry — the saga must close with its own
|
|
1390
|
+
// `auto_retry_end` so subscribers tracking retry-outstanding state
|
|
1391
|
+
// (e.g. suppressing a duplicate error toast) don't stay latched on
|
|
1392
|
+
// an announcement that never resolves.
|
|
1393
|
+
if (this.#retryAttempt > 1) {
|
|
1394
|
+
await this.persistTerminalEmptyErrorTurn(message);
|
|
1395
|
+
await this.#host.emitSessionEvent({
|
|
1396
|
+
type: "auto_retry_end",
|
|
1397
|
+
success: false,
|
|
1398
|
+
attempt: this.#retryAttempt - 1,
|
|
1399
|
+
finalError: errorMessage,
|
|
1400
|
+
});
|
|
1401
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
1402
|
+
}
|
|
1403
|
+
this.#retryAttempt = 0;
|
|
1404
|
+
this.resolveRetry();
|
|
1405
|
+
return false;
|
|
1406
|
+
}
|
|
1407
|
+
// A fallback switch was the whole reason we entered (Fast→base degrade or
|
|
1408
|
+
// a hard-error chain consult) but it could not happen (e.g. no candidate
|
|
1409
|
+
// has a credential). Don't fall through to backing-off and retrying the
|
|
1410
|
+
// failing model for an error the generic classifier wouldn't retry —
|
|
1411
|
+
// surface it instead.
|
|
1412
|
+
if (
|
|
1413
|
+
(options?.fireworksFastFallback || options?.hardErrorFallback) &&
|
|
1414
|
+
!switchedModel &&
|
|
1415
|
+
!this.isRetryableError(message)
|
|
1416
|
+
) {
|
|
1417
|
+
// Same auto_retry_end backstop as the classifier-refusal branch above.
|
|
1418
|
+
if (this.#retryAttempt > 1) {
|
|
1419
|
+
await this.persistTerminalEmptyErrorTurn(message);
|
|
1420
|
+
await this.#host.emitSessionEvent({
|
|
1421
|
+
type: "auto_retry_end",
|
|
1422
|
+
success: false,
|
|
1423
|
+
attempt: this.#retryAttempt - 1,
|
|
1424
|
+
finalError: errorMessage,
|
|
1425
|
+
});
|
|
1426
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
1427
|
+
}
|
|
1428
|
+
this.#retryAttempt = 0;
|
|
1429
|
+
this.resolveRetry();
|
|
1430
|
+
return false;
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
// Fail-fast cap: if the provider asks us to wait longer than
|
|
1434
|
+
// retry.maxDelayMs and we have no fallback credential or model to
|
|
1435
|
+
// switch to, surface the error instead of sleeping. Defends against
|
|
1436
|
+
// 3-hour Anthropic rate-limit windows that would otherwise leave a
|
|
1437
|
+
// subagent (or interactive session) silently hung. The original
|
|
1438
|
+
// assistant error message is preserved in agent state so the caller
|
|
1439
|
+
// can act on it.
|
|
1440
|
+
const maxDelayMs = retrySettings.maxDelayMs;
|
|
1441
|
+
if (maxDelayMs > 0 && delayMs > maxDelayMs && !switchedCredential && !switchedModel) {
|
|
1442
|
+
await this.persistTerminalEmptyErrorTurn(message);
|
|
1443
|
+
const attempt = this.#retryAttempt;
|
|
1444
|
+
this.#retryAttempt = 0;
|
|
1445
|
+
await this.#host.emitSessionEvent({
|
|
1446
|
+
type: "auto_retry_end",
|
|
1447
|
+
success: false,
|
|
1448
|
+
attempt,
|
|
1449
|
+
finalError: `Provider requested ${delayMs}ms wait, exceeds retry.maxDelayMs (${maxDelayMs}ms). Original error: ${errorMessage}`,
|
|
1450
|
+
});
|
|
1451
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
1452
|
+
this.resolveRetry();
|
|
1453
|
+
return false;
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
await this.#recordPendingRecoveredRetryError(message, id, { switchedCredential, switchedModel, delayMs });
|
|
1457
|
+
|
|
1458
|
+
await this.#host.emitSessionEvent({
|
|
1459
|
+
type: "auto_retry_start",
|
|
1460
|
+
attempt: this.#retryAttempt,
|
|
1461
|
+
maxAttempts: maxRetries,
|
|
1462
|
+
delayMs,
|
|
1463
|
+
errorMessage,
|
|
1464
|
+
errorId: message.errorId,
|
|
1465
|
+
});
|
|
1466
|
+
|
|
1467
|
+
// Resolved stream-stall tools have already emitted results. Keep that failed
|
|
1468
|
+
// turn intact so continuation cannot repeat their side effects.
|
|
1469
|
+
if (!options?.preserveFailedTurn) {
|
|
1470
|
+
this.removeAssistantMessageFromActiveContext(message, "auto-retry");
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// A thinking/response loop retried into identical context loops again. Inject a
|
|
1474
|
+
// hidden redirect so the retried turn sees a directive to break the repeated
|
|
1475
|
+
// pattern instead of re-sampling the same stalled reasoning.
|
|
1476
|
+
this.#maybeInjectThinkingLoopRedirect(id);
|
|
1477
|
+
|
|
1478
|
+
// Wait with exponential backoff (abortable).
|
|
1479
|
+
const retryAbortController = new AbortController();
|
|
1480
|
+
this.#retryAbortController?.abort();
|
|
1481
|
+
this.#retryAbortController = retryAbortController;
|
|
1482
|
+
try {
|
|
1483
|
+
await scheduler.wait(delayMs, { signal: retryAbortController.signal });
|
|
1484
|
+
} catch {
|
|
1485
|
+
if (this.#retryAbortController !== retryAbortController) {
|
|
1486
|
+
return false;
|
|
1487
|
+
}
|
|
1488
|
+
// Aborted during sleep - emit end event so UI can clean up
|
|
1489
|
+
const attempt = this.#retryAttempt;
|
|
1490
|
+
this.#retryAttempt = 0;
|
|
1491
|
+
this.#retryAbortController = undefined;
|
|
1492
|
+
await this.#host.emitSessionEvent({
|
|
1493
|
+
type: "auto_retry_end",
|
|
1494
|
+
success: false,
|
|
1495
|
+
attempt,
|
|
1496
|
+
finalError: "Retry cancelled",
|
|
1497
|
+
});
|
|
1498
|
+
this.#clearPendingRecoveredRetryErrors();
|
|
1499
|
+
this.resolveRetry();
|
|
1500
|
+
return false;
|
|
1501
|
+
}
|
|
1502
|
+
if (this.#retryAbortController === retryAbortController) {
|
|
1503
|
+
this.#retryAbortController = undefined;
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
// Retry via continue() outside the agent_end event callback chain.
|
|
1507
|
+
this.#host.scheduleAgentContinue({ delayMs: 1, generation });
|
|
1508
|
+
|
|
1509
|
+
return true;
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
/**
|
|
1513
|
+
* Inject a hidden redirect notice when a thinking/response loop is being retried, so
|
|
1514
|
+
* the retried turn carries an instruction to break the repeated pattern instead of
|
|
1515
|
+
* re-sampling the same stalled context. Injected on every {@link AIError.Flag.ThinkingLoop}
|
|
1516
|
+
* retry (the failed assistant is dropped each attempt, so the notice does not accumulate
|
|
1517
|
+
* unboundedly). No-op unless `id` carries the ThinkingLoop flag and the loop guard is
|
|
1518
|
+
* enabled. The notice is generic on purpose — the detector's detail can quote raw model
|
|
1519
|
+
* text, which must not be interpolated into a higher-priority developer message.
|
|
1520
|
+
*/
|
|
1521
|
+
#maybeInjectThinkingLoopRedirect(id: number): void {
|
|
1522
|
+
if (!AIError.is(id, AIError.Flag.ThinkingLoop)) return;
|
|
1523
|
+
if (this.#host.settings.get("model.loopGuard.enabled") !== true) return;
|
|
1524
|
+
this.#host.agent.appendMessage({
|
|
1525
|
+
role: "custom",
|
|
1526
|
+
customType: THINKING_LOOP_REDIRECT_TYPE,
|
|
1527
|
+
content: thinkingLoopRedirectTemplate,
|
|
1528
|
+
display: false,
|
|
1529
|
+
attribution: "agent",
|
|
1530
|
+
timestamp: Date.now(),
|
|
1531
|
+
});
|
|
1532
|
+
this.#host.sessionManager.appendCustomMessageEntry(
|
|
1533
|
+
THINKING_LOOP_REDIRECT_TYPE,
|
|
1534
|
+
thinkingLoopRedirectTemplate,
|
|
1535
|
+
false,
|
|
1536
|
+
undefined,
|
|
1537
|
+
"agent",
|
|
1538
|
+
);
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
/**
|
|
1542
|
+
* Cancel in-progress retry.
|
|
1543
|
+
*/
|
|
1544
|
+
abortRetry(): void {
|
|
1545
|
+
this.#retryAbortController?.abort();
|
|
1546
|
+
// Note: _retryAttempt is reset in the catch block of _autoRetry
|
|
1547
|
+
this.resolveRetry();
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
async #promptAgentWithIdleRetry(messages: AgentMessage[], options?: { toolChoice?: ToolChoice }): Promise<void> {
|
|
1551
|
+
const deadline = Date.now() + 30_000;
|
|
1552
|
+
for (;;) {
|
|
1553
|
+
try {
|
|
1554
|
+
await this.#host.agent.prompt(messages, options);
|
|
1555
|
+
return;
|
|
1556
|
+
} catch (err) {
|
|
1557
|
+
if (!(err instanceof AgentBusyError)) {
|
|
1558
|
+
throw err;
|
|
1559
|
+
}
|
|
1560
|
+
if (Date.now() >= deadline) {
|
|
1561
|
+
throw new Error("Timed out waiting for prior agent run to finish before prompting.");
|
|
1562
|
+
}
|
|
1563
|
+
await this.#host.agent.waitForIdle();
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
/** Whether auto-retry is currently in progress */
|
|
1569
|
+
get isRetrying(): boolean {
|
|
1570
|
+
return this.#retryPromise !== undefined;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/** Whether auto-retry is enabled */
|
|
1574
|
+
get autoRetryEnabled(): boolean {
|
|
1575
|
+
return this.#host.settings.get("retry.enabled") ?? true;
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
/**
|
|
1579
|
+
* Toggle auto-retry setting.
|
|
1580
|
+
*/
|
|
1581
|
+
setAutoRetryEnabled(enabled: boolean): void {
|
|
1582
|
+
this.#host.settings.set("retry.enabled", enabled);
|
|
1583
|
+
}
|
|
1584
|
+
/**
|
|
1585
|
+
* Manually retry the last failed assistant turn.
|
|
1586
|
+
* Removes the error message from active agent state when present and
|
|
1587
|
+
* re-attempts with a fresh retry budget.
|
|
1588
|
+
*
|
|
1589
|
+
* A stream that stalls or aborts mid-tool-call ends the turn with
|
|
1590
|
+
* `stopReason: "error" | "aborted"` and then appends one synthetic
|
|
1591
|
+
* {@link isSyntheticToolResultMessage tool_result} per emitted tool call to
|
|
1592
|
+
* preserve the provider's tool_use/tool_result pairing (see
|
|
1593
|
+
* `createAbortedToolResult` in `agent-loop.ts`). Those placeholders trail the
|
|
1594
|
+
* failed assistant turn, so the retry lookback walks back over them before
|
|
1595
|
+
* checking the assistant message; it strips both the placeholders and the
|
|
1596
|
+
* failed turn before re-attempting.
|
|
1597
|
+
*
|
|
1598
|
+
* A restored session deliberately omits failed assistant turns from provider
|
|
1599
|
+
* context. In that case, the persisted display transcript remains the source
|
|
1600
|
+
* of truth for whether the current branch has a retryable failed tail.
|
|
1601
|
+
*
|
|
1602
|
+
* @returns true if retry was initiated, false if no failed turn to retry or agent is busy
|
|
1603
|
+
*/
|
|
1604
|
+
async retry(): Promise<boolean> {
|
|
1605
|
+
if (this.#host.isStreaming() || this.#host.isCompacting() || this.isRetrying) return false;
|
|
1606
|
+
|
|
1607
|
+
const messages = this.#host.agent.state.messages;
|
|
1608
|
+
const activeTurnEnd = retryableAssistantTurnEnd(messages);
|
|
1609
|
+
if (activeTurnEnd !== undefined) {
|
|
1610
|
+
// Remove the failed/aborted assistant message plus its synthetic tool
|
|
1611
|
+
// results (same as auto-retry does before re-attempting).
|
|
1612
|
+
this.#host.agent.replaceMessages(messages.slice(0, activeTurnEnd - 1));
|
|
1613
|
+
} else {
|
|
1614
|
+
// A restored session already dropped the failed assistant turn (and its
|
|
1615
|
+
// paired synthetic tool results) from provider context, so the persisted
|
|
1616
|
+
// display transcript is the source of truth for a retryable failed tail.
|
|
1617
|
+
const transcriptMessages = this.#host.sessionManager.buildSessionContext({ transcript: true }).messages;
|
|
1618
|
+
if (retryableAssistantTurnEnd(transcriptMessages) === undefined) return false;
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// Reset retry budget for a fresh attempt
|
|
1622
|
+
this.#retryAttempt = 0;
|
|
1623
|
+
|
|
1624
|
+
// Re-attempt the turn
|
|
1625
|
+
this.#host.scheduleAgentContinue({ delayMs: 1 });
|
|
1626
|
+
|
|
1627
|
+
return true;
|
|
1628
|
+
}
|
|
1629
|
+
}
|