@oh-my-pi/pi-coding-agent 17.2.7 → 17.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -1
- package/dist/{CHANGELOG-1p8yvde7.md → CHANGELOG-8yy8h3bc.md} +57 -1
- package/dist/cli.js +4974 -4968
- package/dist/types/async/job-manager.d.ts +12 -0
- package/dist/types/discovery/agents.d.ts +11 -0
- package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +10 -0
- package/dist/types/hindsight/state.d.ts +0 -1
- package/dist/types/lsp/utils.d.ts +8 -0
- package/dist/types/modes/session-observer-registry.d.ts +2 -0
- package/dist/types/registry/agent-lifecycle.d.ts +1 -1
- package/dist/types/session/agent-session-types.d.ts +7 -0
- package/dist/types/session/agent-session.d.ts +5 -3
- package/dist/types/session/turn-recovery.d.ts +6 -2
- package/dist/types/task/executor.d.ts +4 -0
- package/dist/types/tools/browser/launch.d.ts +2 -0
- package/dist/types/tools/path-utils.d.ts +7 -0
- package/dist/types/tools/read.d.ts +2 -0
- package/dist/types/tools/shell-tokenize.d.ts +21 -3
- package/dist/types/utils/changelog.d.ts +3 -2
- package/dist/types/utils/late-cleanup.d.ts +2 -0
- package/dist/types/vibe/runtime.d.ts +2 -0
- package/package.json +13 -13
- package/scripts/omp +11 -2
- package/src/async/job-manager.ts +26 -0
- package/src/cli/stats-cli.ts +3 -3
- package/src/cli/update-cli.ts +2 -56
- package/src/config/config-file.ts +2 -2
- package/src/config/keybindings.ts +3 -3
- package/src/config/model-discovery.ts +36 -2
- package/src/config/model-registry.ts +4 -1
- package/src/discovery/agents.ts +71 -3
- package/src/discovery/claude.ts +8 -5
- package/src/discovery/cursor.ts +8 -5
- package/src/discovery/gemini.ts +11 -8
- package/src/discovery/vscode.ts +1 -0
- package/src/discovery/windsurf.ts +4 -2
- package/src/edit/hashline/filesystem.ts +7 -7
- package/src/eval/agent-bridge.ts +1 -3
- package/src/eval/jl/prelude.jl +4 -4
- package/src/eval/js/shared/prelude.txt +2 -2
- package/src/eval/py/prelude.py +0 -3
- package/src/eval/rb/prelude.rb +1 -2
- package/src/extensibility/legacy-pi-coding-agent-shim.ts +23 -0
- package/src/hindsight/state.ts +2 -22
- package/src/internal-urls/memory-protocol.ts +29 -0
- package/src/lsp/client.ts +2 -2
- package/src/lsp/utils.ts +29 -0
- package/src/modes/acp/acp-agent.ts +2 -1
- package/src/modes/components/agent-hub.ts +36 -20
- package/src/modes/components/agent-transcript-viewer.ts +1 -3
- package/src/modes/components/bordered-loader.ts +1 -1
- package/src/modes/components/custom-editor.ts +2 -2
- package/src/modes/components/hook-input.ts +1 -1
- package/src/modes/components/hook-selector.ts +2 -2
- package/src/modes/components/session-selector.ts +2 -2
- package/src/modes/controllers/event-controller.ts +159 -2
- package/src/modes/controllers/extension-ui-controller.ts +2 -1
- package/src/modes/print-mode.ts +29 -12
- package/src/modes/session-observer-registry.ts +5 -0
- package/src/registry/agent-lifecycle.ts +21 -3
- package/src/sdk.ts +11 -5
- package/src/session/agent-session-types.ts +8 -0
- package/src/session/agent-session.ts +156 -186
- package/src/session/session-paths.ts +68 -121
- package/src/session/settings-stream-fn.ts +7 -3
- package/src/session/turn-recovery.ts +234 -16
- package/src/session/unexpected-stop-classifier.ts +12 -3
- package/src/slash-commands/helpers/stats-dashboard.ts +2 -1
- package/src/task/agents.ts +1 -1
- package/src/task/executor.ts +105 -29
- package/src/task/isolation-runner.ts +19 -1
- package/src/task/structured-subagent.ts +17 -2
- package/src/tiny/worker.ts +1 -1
- package/src/tools/bash-interceptor.ts +8 -4
- package/src/tools/browser/launch.ts +37 -9
- package/src/tools/glob.ts +4 -2
- package/src/tools/path-utils.ts +13 -3
- package/src/tools/read.ts +6 -0
- package/src/tools/shell-tokenize.ts +38 -9
- package/src/utils/changelog.ts +5 -4
- package/src/utils/late-cleanup.ts +17 -0
- package/src/vibe/runtime.ts +197 -53
- package/src/web/scrapers/hackage.ts +1 -12
- package/src/web/search/providers/duckduckgo.ts +1 -1
- package/src/web/search/providers/ecosia.ts +1 -1
- package/src/web/search/providers/exa.ts +1 -1
- package/src/web/search/providers/firecrawl.ts +1 -1
- package/src/web/search/providers/mojeek.ts +1 -1
- package/src/web/search/providers/perplexity.ts +1 -1
- package/src/web/search/providers/public.ts +1 -1
- package/src/web/search/providers/startpage.ts +1 -1
|
@@ -2,17 +2,10 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as os from "node:os";
|
|
3
3
|
import * as path from "node:path";
|
|
4
4
|
import { getTerminalId } from "@oh-my-pi/pi-tui";
|
|
5
|
-
import {
|
|
6
|
-
getSessionsDir,
|
|
7
|
-
getTerminalSessionsDir,
|
|
8
|
-
isEnoent,
|
|
9
|
-
isRecord,
|
|
10
|
-
logger,
|
|
11
|
-
resolveEquivalentPath,
|
|
12
|
-
} from "@oh-my-pi/pi-utils";
|
|
5
|
+
import { getSessionsDir, getTerminalSessionsDir, isEnoent, logger, resolveEquivalentPath } from "@oh-my-pi/pi-utils";
|
|
13
6
|
import type { SessionStorage } from "./session-storage";
|
|
14
7
|
|
|
15
|
-
const
|
|
8
|
+
const migratedSessionRoots = new Set<string>();
|
|
16
9
|
|
|
17
10
|
/**
|
|
18
11
|
* Merge or rename a legacy session directory into its canonical target.
|
|
@@ -24,11 +17,13 @@ function migrateSessionDirPath(oldPath: string, newPath: string): void {
|
|
|
24
17
|
for (const file of fs.readdirSync(oldPath)) {
|
|
25
18
|
const src = path.join(oldPath, file);
|
|
26
19
|
const dst = path.join(newPath, file);
|
|
27
|
-
if (
|
|
28
|
-
|
|
20
|
+
if (fs.existsSync(dst)) {
|
|
21
|
+
logger.warn("Session directory migration collision; preserving legacy entry", { src, dst });
|
|
22
|
+
continue;
|
|
29
23
|
}
|
|
24
|
+
fs.renameSync(src, dst);
|
|
30
25
|
}
|
|
31
|
-
fs.
|
|
26
|
+
fs.rmdirSync(oldPath);
|
|
32
27
|
return;
|
|
33
28
|
}
|
|
34
29
|
if (existing) {
|
|
@@ -42,16 +37,12 @@ function encodeLegacyAbsoluteSessionDirName(cwd: string): string {
|
|
|
42
37
|
return `--${resolvedCwd.replace(/^[/\\]/, "").replace(/[/\\:]/g, "-")}--`;
|
|
43
38
|
}
|
|
44
39
|
|
|
45
|
-
function
|
|
40
|
+
function encodeRelativeSessionDirName(prefix: string, relative: string): string {
|
|
46
41
|
const encoded = relative.replace(/[/\\:]/g, "-");
|
|
47
42
|
return encoded ? (prefix.endsWith("-") ? `${prefix}${encoded}` : `${prefix}-${encoded}`) : prefix;
|
|
48
43
|
}
|
|
49
44
|
|
|
50
|
-
function getDefaultSessionDirName(cwd: string): {
|
|
51
|
-
encodedDirName: string;
|
|
52
|
-
legacyRelativeDirName: string | undefined;
|
|
53
|
-
resolvedCwd: string;
|
|
54
|
-
} {
|
|
45
|
+
function getDefaultSessionDirName(cwd: string): { encodedDirName: string; resolvedCwd: string } {
|
|
55
46
|
const resolvedCwd = path.resolve(cwd);
|
|
56
47
|
const canonicalCwd = resolveEquivalentPath(resolvedCwd);
|
|
57
48
|
const home = os.homedir();
|
|
@@ -60,111 +51,80 @@ function getDefaultSessionDirName(cwd: string): {
|
|
|
60
51
|
const canonicalTempRoot = resolveEquivalentPath(tempRoot);
|
|
61
52
|
const homeRelative = path.relative(canonicalHome, canonicalCwd);
|
|
62
53
|
const tempRelative = path.relative(canonicalTempRoot, canonicalCwd);
|
|
54
|
+
const encodedDirName =
|
|
55
|
+
homeRelative === "" || (!homeRelative.startsWith("..") && !path.isAbsolute(homeRelative))
|
|
56
|
+
? encodeRelativeSessionDirName("-", homeRelative)
|
|
57
|
+
: tempRelative === "" || (!tempRelative.startsWith("..") && !path.isAbsolute(tempRelative))
|
|
58
|
+
? encodeRelativeSessionDirName("-tmp", tempRelative)
|
|
59
|
+
: encodeLegacyAbsoluteSessionDirName(canonicalCwd);
|
|
60
|
+
return { encodedDirName, resolvedCwd };
|
|
61
|
+
}
|
|
63
62
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
legacyRelativeDirName = encodeLegacyRelativeSessionDirName("-tmp", tempRelative);
|
|
72
|
-
} else {
|
|
73
|
-
scope = "abs";
|
|
74
|
-
}
|
|
63
|
+
/**
|
|
64
|
+
* Migrate old `--<home-encoded>-*--` session dirs to the new `-*` format.
|
|
65
|
+
* Runs once per sessions root on first access, best-effort.
|
|
66
|
+
*/
|
|
67
|
+
function migrateHomeSessionDirs(sessionsRoot: string): void {
|
|
68
|
+
if (migratedSessionRoots.has(sessionsRoot)) return;
|
|
69
|
+
migratedSessionRoots.add(sessionsRoot);
|
|
75
70
|
|
|
76
|
-
const
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
.replace(/^-+|-+$/g, "")
|
|
81
|
-
.slice(-80);
|
|
82
|
-
const digest = Bun.SHA256.hash(normalized, "hex");
|
|
83
|
-
const encodedDirName = `${scope}-${readable || "project"}-${digest}`;
|
|
84
|
-
return { encodedDirName, legacyRelativeDirName, resolvedCwd };
|
|
85
|
-
}
|
|
71
|
+
const home = os.homedir();
|
|
72
|
+
const homeEncoded = home.replace(/^[/\\]/, "").replace(/[/\\:]/g, "-");
|
|
73
|
+
const oldPrefix = `--${homeEncoded}-`;
|
|
74
|
+
const oldExact = `--${homeEncoded}--`;
|
|
86
75
|
|
|
87
|
-
|
|
88
|
-
let descriptor: number | undefined;
|
|
76
|
+
let entries: string[];
|
|
89
77
|
try {
|
|
90
|
-
|
|
91
|
-
const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, 0);
|
|
92
|
-
const prefix = buffer.toString("utf8", 0, bytesRead);
|
|
93
|
-
for (const line of prefix.split(/\r?\n/, 3)) {
|
|
94
|
-
try {
|
|
95
|
-
const record: unknown = JSON.parse(line);
|
|
96
|
-
if (isRecord(record) && record.type === "session" && typeof record.cwd === "string") {
|
|
97
|
-
return record.cwd;
|
|
98
|
-
}
|
|
99
|
-
} catch {
|
|
100
|
-
// Ignore title slots or truncated/corrupt headers.
|
|
101
|
-
}
|
|
102
|
-
}
|
|
78
|
+
entries = fs.readdirSync(sessionsRoot);
|
|
103
79
|
} catch {
|
|
104
|
-
|
|
105
|
-
} finally {
|
|
106
|
-
if (descriptor !== undefined) fs.closeSync(descriptor);
|
|
80
|
+
return;
|
|
107
81
|
}
|
|
108
|
-
return undefined;
|
|
109
|
-
}
|
|
110
82
|
|
|
111
|
-
function moveSessionBundle(sourceDir: string, targetDir: string, sessionName: string, entries: string[]): void {
|
|
112
|
-
fs.mkdirSync(targetDir, { recursive: true });
|
|
113
|
-
const artifactsName = path.basename(sessionName, ".jsonl");
|
|
114
83
|
for (const entry of entries) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if (!existing) {
|
|
121
|
-
fs.renameSync(source, target);
|
|
122
|
-
} else if (existing.isDirectory() && fs.statSync(source).isDirectory()) {
|
|
123
|
-
migrateSessionDirPath(source, target);
|
|
84
|
+
let remainder: string;
|
|
85
|
+
if (entry === oldExact) {
|
|
86
|
+
remainder = "";
|
|
87
|
+
} else if (entry.startsWith(oldPrefix) && entry.endsWith("--")) {
|
|
88
|
+
remainder = entry.slice(oldPrefix.length, -2);
|
|
124
89
|
} else {
|
|
125
|
-
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const newName = remainder ? `-${remainder}` : "-";
|
|
94
|
+
const oldPath = path.join(sessionsRoot, entry);
|
|
95
|
+
const newPath = path.join(sessionsRoot, newName);
|
|
96
|
+
|
|
97
|
+
try {
|
|
98
|
+
migrateSessionDirPath(oldPath, newPath);
|
|
99
|
+
} catch (error) {
|
|
100
|
+
logger.warn("Failed to migrate legacy home session directory", {
|
|
101
|
+
oldPath,
|
|
102
|
+
newPath,
|
|
103
|
+
error: String(error),
|
|
104
|
+
});
|
|
126
105
|
}
|
|
127
106
|
}
|
|
128
107
|
}
|
|
129
108
|
|
|
130
|
-
function
|
|
131
|
-
|
|
132
|
-
legacyDir
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
if (!recordedCwd) continue;
|
|
143
|
-
const recordedCanonical = resolveEquivalentPath(path.resolve(recordedCwd));
|
|
144
|
-
if (
|
|
145
|
-
recordedCanonical === currentCwd ||
|
|
146
|
-
!fs.statSync(recordedCanonical, { throwIfNoEntry: false })?.isDirectory()
|
|
147
|
-
) {
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
const recordedNames = getDefaultSessionDirName(recordedCanonical);
|
|
151
|
-
const recordedLegacyName =
|
|
152
|
-
kind === "relative"
|
|
153
|
-
? recordedNames.legacyRelativeDirName
|
|
154
|
-
: encodeLegacyAbsoluteSessionDirName(recordedCanonical);
|
|
155
|
-
if (recordedLegacyName !== path.basename(legacyDir)) continue;
|
|
156
|
-
moveSessionBundle(legacyDir, path.join(sessionsRoot, recordedNames.encodedDirName), entry, entries);
|
|
109
|
+
function migrateLegacyAbsoluteSessionDir(cwd: string, sessionDir: string, sessionsRoot: string): void {
|
|
110
|
+
const legacyDir = path.join(sessionsRoot, encodeLegacyAbsoluteSessionDirName(cwd));
|
|
111
|
+
if (legacyDir === sessionDir || !fs.existsSync(legacyDir)) return;
|
|
112
|
+
|
|
113
|
+
try {
|
|
114
|
+
migrateSessionDirPath(legacyDir, sessionDir);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
logger.warn("Failed to migrate legacy session directory", {
|
|
117
|
+
oldPath: legacyDir,
|
|
118
|
+
newPath: sessionDir,
|
|
119
|
+
error: String(error),
|
|
120
|
+
});
|
|
157
121
|
}
|
|
158
122
|
}
|
|
159
123
|
|
|
160
124
|
export function resolveManagedSessionRoot(sessionDir: string, cwd: string): string | undefined {
|
|
161
125
|
const currentDirName = path.basename(sessionDir);
|
|
162
|
-
const { encodedDirName
|
|
163
|
-
if (
|
|
164
|
-
currentDirName !== encodedDirName &&
|
|
165
|
-
currentDirName !== legacyRelativeDirName &&
|
|
166
|
-
currentDirName !== encodeLegacyAbsoluteSessionDirName(cwd)
|
|
167
|
-
) {
|
|
126
|
+
const { encodedDirName } = getDefaultSessionDirName(cwd);
|
|
127
|
+
if (currentDirName !== encodedDirName && currentDirName !== encodeLegacyAbsoluteSessionDirName(cwd)) {
|
|
168
128
|
return undefined;
|
|
169
129
|
}
|
|
170
130
|
return path.dirname(sessionDir);
|
|
@@ -180,23 +140,10 @@ export function computeDefaultSessionDir(
|
|
|
180
140
|
storage: SessionStorage,
|
|
181
141
|
sessionsRoot: string = getSessionsDir(),
|
|
182
142
|
): string {
|
|
183
|
-
const { encodedDirName,
|
|
143
|
+
const { encodedDirName, resolvedCwd } = getDefaultSessionDirName(cwd);
|
|
144
|
+
migrateHomeSessionDirs(sessionsRoot);
|
|
184
145
|
const sessionDir = path.join(sessionsRoot, encodedDirName);
|
|
185
|
-
|
|
186
|
-
{ kind: "relative", name: legacyRelativeDirName },
|
|
187
|
-
{ kind: "absolute", name: encodeLegacyAbsoluteSessionDirName(resolvedCwd) },
|
|
188
|
-
];
|
|
189
|
-
for (const legacy of legacyDirs) {
|
|
190
|
-
if (!legacy.name) continue;
|
|
191
|
-
const legacyDir = path.join(sessionsRoot, legacy.name);
|
|
192
|
-
if (legacyDir === sessionDir || !fs.existsSync(legacyDir)) continue;
|
|
193
|
-
try {
|
|
194
|
-
rerouteCollidingSessions(resolvedCwd, legacyDir, sessionsRoot, legacy.kind);
|
|
195
|
-
migrateSessionDirPath(legacyDir, sessionDir);
|
|
196
|
-
} catch {
|
|
197
|
-
// Best effort
|
|
198
|
-
}
|
|
199
|
-
}
|
|
146
|
+
migrateLegacyAbsoluteSessionDir(resolvedCwd, sessionDir, sessionsRoot);
|
|
200
147
|
storage.ensureDirSync(sessionDir);
|
|
201
148
|
return sessionDir;
|
|
202
149
|
}
|
|
@@ -34,9 +34,13 @@ export function createSettingsAwareStreamFn(settings: Settings, base: StreamFn =
|
|
|
34
34
|
openrouterRoutingPreset && openrouterRoutingPreset !== "default" ? openrouterRoutingPreset : undefined;
|
|
35
35
|
const antigravityEndpointMode = settings.get("providers.antigravityEndpoint");
|
|
36
36
|
const textVerbosity =
|
|
37
|
-
model.api === "openai-codex-responses"
|
|
38
|
-
? settings.
|
|
39
|
-
|
|
37
|
+
model.api === "openai-codex-responses"
|
|
38
|
+
? settings.isConfigured("textVerbosity")
|
|
39
|
+
? settings.get("textVerbosity")
|
|
40
|
+
: undefined
|
|
41
|
+
: model.api === "openai-responses"
|
|
42
|
+
? settings.get("textVerbosity")
|
|
43
|
+
: undefined;
|
|
40
44
|
const streamFirstEventTimeoutMs = timeoutSecondsToMs(settings.get("providers.streamFirstEventTimeoutSeconds"));
|
|
41
45
|
const streamIdleTimeoutMs = timeoutSecondsToMs(settings.get("providers.streamIdleTimeoutSeconds"));
|
|
42
46
|
// Server-side fallback (opt-in): when the user enables it AND the
|
|
@@ -13,16 +13,20 @@ import type {
|
|
|
13
13
|
CodexCompactionContext,
|
|
14
14
|
Effort,
|
|
15
15
|
Model,
|
|
16
|
+
ModelUsageHealth,
|
|
16
17
|
TextContent,
|
|
18
|
+
ThinkingContent,
|
|
17
19
|
ToolChoice,
|
|
18
20
|
} from "@oh-my-pi/pi-ai";
|
|
19
21
|
import { calculateRateLimitBackoffMs, parseRateLimitReason } from "@oh-my-pi/pi-ai";
|
|
20
22
|
import * as AIError from "@oh-my-pi/pi-ai/error";
|
|
21
23
|
import { kCursorExecResolved } from "@oh-my-pi/pi-ai/utils/block-symbols";
|
|
22
24
|
import { isFireworksFastModelId, toFireworksBaseModelId } from "@oh-my-pi/pi-catalog/fireworks-model-id";
|
|
25
|
+
import { modelsAreEqual } from "@oh-my-pi/pi-catalog/models";
|
|
23
26
|
import { extractRetryHint, logger, prompt } from "@oh-my-pi/pi-utils";
|
|
24
27
|
import type { ModelRegistry } from "../config/model-registry";
|
|
25
28
|
import { formatModelStringWithRouting, resolveModelOverride } from "../config/model-resolver";
|
|
29
|
+
|
|
26
30
|
import type { Settings } from "../config/settings";
|
|
27
31
|
import type { RecoveredRetryError } from "../extensibility/shared-events";
|
|
28
32
|
import emptyStopRetryTemplate from "../prompts/system/empty-stop-retry.md" with { type: "text" };
|
|
@@ -35,7 +39,11 @@ import {
|
|
|
35
39
|
modelSupportsEffortCeiling,
|
|
36
40
|
} from "../thinking";
|
|
37
41
|
import type { AgentSessionEvent } from "./agent-session-events";
|
|
38
|
-
import type {
|
|
42
|
+
import type {
|
|
43
|
+
InitialRetryFallbackState,
|
|
44
|
+
UsageFallbackConfirmation,
|
|
45
|
+
UsageFallbackConfirmer,
|
|
46
|
+
} from "./agent-session-types";
|
|
39
47
|
import { isEmptyErrorTurn } from "./messages";
|
|
40
48
|
import {
|
|
41
49
|
type ActiveRetryFallbackState,
|
|
@@ -64,6 +72,7 @@ const UNEXPECTED_STOP_TIMEOUT_MS = 4000;
|
|
|
64
72
|
const EMPTY_STOP_MAX_RETRIES = 3;
|
|
65
73
|
const SIBLING_UNBLOCK_BUFFER_MS = 1_000;
|
|
66
74
|
const NON_WHITESPACE_RE = /\S/;
|
|
75
|
+
const USAGE_PREFLIGHT_BLOCKED_PREFIX = "Usage preflight blocked:";
|
|
67
76
|
|
|
68
77
|
function hasNonWhitespace(value: string): boolean {
|
|
69
78
|
return NON_WHITESPACE_RE.test(value);
|
|
@@ -101,6 +110,8 @@ export interface TurnRecoveryHost {
|
|
|
101
110
|
modelRegistry: ModelRegistry;
|
|
102
111
|
configWarnings: string[];
|
|
103
112
|
model(): Model | undefined;
|
|
113
|
+
/** Whether streamed text has already been committed to the active output sink. */
|
|
114
|
+
textOutputCommitted(): boolean;
|
|
104
115
|
thinkingLevel(): ThinkingLevel | undefined;
|
|
105
116
|
configuredThinkingLevel(): ConfiguredThinkingLevel | undefined;
|
|
106
117
|
setThinkingLevel(level: ConfiguredThinkingLevel | undefined): void;
|
|
@@ -172,6 +183,7 @@ export class TurnRecovery {
|
|
|
172
183
|
#retryPromise: Promise<void> | undefined;
|
|
173
184
|
#retryResolve: (() => void) | undefined;
|
|
174
185
|
#activeRetryFallback: ActiveRetryFallbackState | undefined;
|
|
186
|
+
#usageReserveApprovedSelector: string | undefined;
|
|
175
187
|
#pendingRecoveredRetryErrors: PendingRecoveredRetryError[] = [];
|
|
176
188
|
#usageLimitOutcomes = new WeakMap<AssistantMessage, Promise<UsageLimitOutcome>>();
|
|
177
189
|
#emptyStopRetryCount = 0;
|
|
@@ -298,6 +310,11 @@ export class TurnRecovery {
|
|
|
298
310
|
return this.#maybeRestoreRetryFallbackPrimary();
|
|
299
311
|
}
|
|
300
312
|
|
|
313
|
+
/** Applies model fallback policy from live usage health before a turn starts. */
|
|
314
|
+
maybeApplyUsageAwareFallback(signal: AbortSignal, confirmer?: UsageFallbackConfirmer): Promise<boolean> {
|
|
315
|
+
return this.#maybeApplyUsageAwareFallback(signal, confirmer);
|
|
316
|
+
}
|
|
317
|
+
|
|
301
318
|
/** Applies automatic retry, credential rotation, and model fallback policy. */
|
|
302
319
|
handleRetryableError(
|
|
303
320
|
message: AssistantMessage,
|
|
@@ -591,11 +608,19 @@ export class TurnRecovery {
|
|
|
591
608
|
return false;
|
|
592
609
|
}
|
|
593
610
|
|
|
594
|
-
|
|
611
|
+
let text = assistantMessage.content
|
|
595
612
|
.filter((content): content is TextContent => content.type === "text")
|
|
596
613
|
.map(content => content.text)
|
|
597
614
|
.join("\n");
|
|
598
|
-
|
|
615
|
+
// Thinking-only stops carry their signal in the thinking block (a trapped
|
|
616
|
+
// response or a truncated fragment); classify on that when there is no text.
|
|
617
|
+
if (!hasNonWhitespace(text)) {
|
|
618
|
+
text = assistantMessage.content
|
|
619
|
+
.filter((content): content is ThinkingContent => content.type === "thinking")
|
|
620
|
+
.map(content => content.thinking)
|
|
621
|
+
.join("\n");
|
|
622
|
+
}
|
|
623
|
+
if (!hasNonWhitespace(text)) {
|
|
599
624
|
this.#unexpectedStopRetryCount = 0;
|
|
600
625
|
return false;
|
|
601
626
|
}
|
|
@@ -840,6 +865,9 @@ export class TurnRecovery {
|
|
|
840
865
|
return id;
|
|
841
866
|
}
|
|
842
867
|
|
|
868
|
+
#isUsagePreflightBlocked(message: AssistantMessage): boolean {
|
|
869
|
+
return message.errorMessage?.startsWith(USAGE_PREFLIGHT_BLOCKED_PREFIX) === true;
|
|
870
|
+
}
|
|
843
871
|
/**
|
|
844
872
|
* Retry an empty, reason-less provider abort: a turn with no content that
|
|
845
873
|
* carries the generic sentinel (bare `abort()`), whether the provider
|
|
@@ -883,6 +911,7 @@ export class TurnRecovery {
|
|
|
883
911
|
*/
|
|
884
912
|
isRetryableError(message: AssistantMessage): boolean {
|
|
885
913
|
if (message.stopReason !== "error") return false;
|
|
914
|
+
if (this.#isUsagePreflightBlocked(message)) return false;
|
|
886
915
|
|
|
887
916
|
const id = this.#classifyRetryMessage(message);
|
|
888
917
|
// Context overflow is handled by compaction, not retry
|
|
@@ -890,8 +919,8 @@ export class TurnRecovery {
|
|
|
890
919
|
if (AIError.isContextOverflow(message, contextWindow)) return false;
|
|
891
920
|
|
|
892
921
|
// A classifier refusal/sensitivity stop is the model's decision, not a route
|
|
893
|
-
// failure, but only after we confirm no
|
|
894
|
-
// streamed.
|
|
922
|
+
// failure, but only after we confirm no replay-unsafe output has already
|
|
923
|
+
// streamed. Committed text, images, tool calls, or server tools must not be
|
|
895
924
|
// discarded and replayed.
|
|
896
925
|
if (this.#hasReplayUnsafeOutput(message)) return false;
|
|
897
926
|
if (this.isClassifierRefusal(message)) return true;
|
|
@@ -966,10 +995,10 @@ export class TurnRecovery {
|
|
|
966
995
|
* Thinking-only partials are safe to discard and replay: reasoning models
|
|
967
996
|
* routinely stall after long thinking with no visible output, and duplicated
|
|
968
997
|
* thinking display is materially lower harm than duplicated final text.
|
|
969
|
-
* Whitespace-only text
|
|
970
|
-
* user.
|
|
971
|
-
* are NOT safe: each has already rendered or may have side effects,
|
|
972
|
-
* the turn can duplicate user-visible output or work.
|
|
998
|
+
* Whitespace-only and buffered text are likewise safe since nothing meaningful
|
|
999
|
+
* reached the user. Committed text, generated images, server tools, and retained
|
|
1000
|
+
* tool calls are NOT safe: each has already rendered or may have side effects,
|
|
1001
|
+
* so replaying the turn can duplicate user-visible output or work.
|
|
973
1002
|
*/
|
|
974
1003
|
#hasReplayUnsafeOutput(message: AssistantMessage): boolean {
|
|
975
1004
|
return message.content.some(
|
|
@@ -977,7 +1006,7 @@ export class TurnRecovery {
|
|
|
977
1006
|
block.type === "toolCall" ||
|
|
978
1007
|
block.type === "image" ||
|
|
979
1008
|
block.type === "anthropicServerTool" ||
|
|
980
|
-
(block.type === "text" && block.text.trim().length > 0),
|
|
1009
|
+
(block.type === "text" && this.#host.textOutputCommitted() && block.text.trim().length > 0),
|
|
981
1010
|
);
|
|
982
1011
|
}
|
|
983
1012
|
|
|
@@ -1008,7 +1037,6 @@ export class TurnRecovery {
|
|
|
1008
1037
|
modelLookup: this.#host.modelRegistry,
|
|
1009
1038
|
};
|
|
1010
1039
|
}
|
|
1011
|
-
|
|
1012
1040
|
#getRetryFallbackChains(): RetryFallbackChains {
|
|
1013
1041
|
return getRetryFallbackChains(this.#host.settings);
|
|
1014
1042
|
}
|
|
@@ -1071,23 +1099,187 @@ export class TurnRecovery {
|
|
|
1071
1099
|
);
|
|
1072
1100
|
}
|
|
1073
1101
|
|
|
1102
|
+
async #maybeApplyUsageAwareFallback(signal: AbortSignal, confirmer?: UsageFallbackConfirmer): Promise<boolean> {
|
|
1103
|
+
if (!this.#host.settings.get("retry.usageAwareFallback")) return false;
|
|
1104
|
+
const currentModel = this.#host.model();
|
|
1105
|
+
if (!currentModel) return false;
|
|
1106
|
+
const currentSelector = formatRetryFallbackSelector(currentModel, this.#host.thinkingLevel());
|
|
1107
|
+
let health: ModelUsageHealth;
|
|
1108
|
+
try {
|
|
1109
|
+
health = await this.#host.modelRegistry.authStorage.getModelUsageHealth(currentModel.provider, {
|
|
1110
|
+
modelId: currentModel.id,
|
|
1111
|
+
sessionId: this.#host.sessionId(),
|
|
1112
|
+
baseUrl: currentModel.baseUrl,
|
|
1113
|
+
reserveFraction: this.#host.settings.get("retry.usageReservePct") / 100,
|
|
1114
|
+
signal,
|
|
1115
|
+
});
|
|
1116
|
+
} catch (error) {
|
|
1117
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1118
|
+
logger.debug("Usage-aware runtime preflight failed open", {
|
|
1119
|
+
provider: currentModel.provider,
|
|
1120
|
+
model: currentModel.id,
|
|
1121
|
+
error: String(error),
|
|
1122
|
+
});
|
|
1123
|
+
return false;
|
|
1124
|
+
}
|
|
1125
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1126
|
+
const selectedAccount = health.accounts.find(account => account.selected);
|
|
1127
|
+
if (health.state === "healthy") {
|
|
1128
|
+
this.#usageReserveApprovedSelector = undefined;
|
|
1129
|
+
if (
|
|
1130
|
+
selectedAccount &&
|
|
1131
|
+
selectedAccount.state !== "healthy" &&
|
|
1132
|
+
health.accounts.some(account => account.state === "healthy")
|
|
1133
|
+
) {
|
|
1134
|
+
this.#host.modelRegistry.authStorage.releaseSessionCredentialForReselection(
|
|
1135
|
+
currentModel.provider,
|
|
1136
|
+
this.#host.sessionId(),
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
return false;
|
|
1140
|
+
}
|
|
1141
|
+
if (health.state === "unknown") {
|
|
1142
|
+
this.#usageReserveApprovedSelector = undefined;
|
|
1143
|
+
return false;
|
|
1144
|
+
}
|
|
1145
|
+
if (health.state !== "reserve") this.#usageReserveApprovedSelector = undefined;
|
|
1146
|
+
|
|
1147
|
+
const reservePolicy = this.#host.settings.get("retry.usageReservePolicy");
|
|
1148
|
+
if (reservePolicy === "fail-closed") {
|
|
1149
|
+
const condition = health.state === "reserve" ? "reserve reached" : "usage depleted";
|
|
1150
|
+
throw new Error(
|
|
1151
|
+
`${USAGE_PREFLIGHT_BLOCKED_PREFIX} ${condition} for ${currentSelector}; reserve policy is fail-closed.`,
|
|
1152
|
+
);
|
|
1153
|
+
}
|
|
1154
|
+
if (
|
|
1155
|
+
reservePolicy === "confirm" &&
|
|
1156
|
+
health.state === "reserve" &&
|
|
1157
|
+
this.#usageReserveApprovedSelector === currentSelector
|
|
1158
|
+
) {
|
|
1159
|
+
return false;
|
|
1160
|
+
}
|
|
1161
|
+
if (!this.#host.settings.get("retry.modelFallback")) return false;
|
|
1162
|
+
|
|
1163
|
+
const role = this.#activeRetryFallback?.role ?? this.resolveRetryFallbackRole(currentSelector, currentModel);
|
|
1164
|
+
if (!role) return false;
|
|
1165
|
+
let fallback: { selector: RetryFallbackSelector; apiKey: string } | undefined;
|
|
1166
|
+
const ceiling = this.#host.thinkingLevelCeiling();
|
|
1167
|
+
for (const candidate of this.findRetryFallbackCandidates(role, currentSelector, currentModel)) {
|
|
1168
|
+
if (this.isRetryFallbackSelectorSuppressed(candidate)) continue;
|
|
1169
|
+
const resolved = resolveModelOverride([candidate.raw], this.#host.modelRegistry, this.#host.settings);
|
|
1170
|
+
const candidateModel = resolved.model ?? this.#host.modelRegistry.find(candidate.provider, candidate.id);
|
|
1171
|
+
if (!candidateModel || !this.#host.modelRegistry.hasConfiguredAuth(candidateModel)) continue;
|
|
1172
|
+
if (ceiling !== undefined && !modelSupportsEffortCeiling(candidateModel, ceiling)) continue;
|
|
1173
|
+
try {
|
|
1174
|
+
const candidateHealth = await this.#host.modelRegistry.authStorage.getModelUsageHealth(
|
|
1175
|
+
candidateModel.provider,
|
|
1176
|
+
{
|
|
1177
|
+
modelId: candidateModel.id,
|
|
1178
|
+
sessionId: this.#host.sessionId(),
|
|
1179
|
+
baseUrl: candidateModel.baseUrl,
|
|
1180
|
+
reserveFraction: this.#host.settings.get("retry.usageReservePct") / 100,
|
|
1181
|
+
signal,
|
|
1182
|
+
},
|
|
1183
|
+
);
|
|
1184
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1185
|
+
if (candidateHealth.state === "depleted" || candidateHealth.state === "reserve") continue;
|
|
1186
|
+
if (candidateHealth.state === "healthy") {
|
|
1187
|
+
const selected = candidateHealth.accounts.find(account => account.selected);
|
|
1188
|
+
if (
|
|
1189
|
+
selected &&
|
|
1190
|
+
selected.state !== "healthy" &&
|
|
1191
|
+
candidateHealth.accounts.some(account => account.state === "healthy")
|
|
1192
|
+
) {
|
|
1193
|
+
this.#host.modelRegistry.authStorage.releaseSessionCredentialForReselection(
|
|
1194
|
+
candidateModel.provider,
|
|
1195
|
+
this.#host.sessionId(),
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
} catch {
|
|
1200
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1201
|
+
// Unknown usage fails open for an otherwise valid fallback.
|
|
1202
|
+
}
|
|
1203
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1204
|
+
let apiKey: string | undefined;
|
|
1205
|
+
try {
|
|
1206
|
+
apiKey = await this.#host.modelRegistry.getApiKey(candidateModel, this.#host.sessionId(), { signal });
|
|
1207
|
+
} catch {
|
|
1208
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1209
|
+
continue;
|
|
1210
|
+
}
|
|
1211
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1212
|
+
if (!apiKey) continue;
|
|
1213
|
+
fallback = { selector: candidate, apiKey };
|
|
1214
|
+
break;
|
|
1215
|
+
}
|
|
1216
|
+
if (!fallback) return false;
|
|
1217
|
+
|
|
1218
|
+
let shouldFallback = health.state === "depleted" || reservePolicy === "auto" || !confirmer;
|
|
1219
|
+
if (!shouldFallback && health.state === "reserve" && confirmer) {
|
|
1220
|
+
const remainingFraction =
|
|
1221
|
+
selectedAccount?.remainingFraction ??
|
|
1222
|
+
health.accounts.reduce<number | undefined>((minimum, account) => {
|
|
1223
|
+
if (account.remainingFraction === undefined) return minimum;
|
|
1224
|
+
return minimum === undefined ? account.remainingFraction : Math.min(minimum, account.remainingFraction);
|
|
1225
|
+
}, undefined);
|
|
1226
|
+
shouldFallback = await this.#confirmUsageFallback(
|
|
1227
|
+
confirmer,
|
|
1228
|
+
{
|
|
1229
|
+
from: currentSelector,
|
|
1230
|
+
to: fallback.selector.raw,
|
|
1231
|
+
remainingPercent: remainingFraction === undefined ? undefined : Math.max(0, remainingFraction * 100),
|
|
1232
|
+
},
|
|
1233
|
+
signal,
|
|
1234
|
+
);
|
|
1235
|
+
if (signal.aborted || !modelsAreEqual(this.#host.model(), currentModel)) return false;
|
|
1236
|
+
}
|
|
1237
|
+
if (!shouldFallback) {
|
|
1238
|
+
this.#usageReserveApprovedSelector = currentSelector;
|
|
1239
|
+
return false;
|
|
1240
|
+
}
|
|
1241
|
+
this.#usageReserveApprovedSelector = undefined;
|
|
1242
|
+
return this.applyRetryFallbackCandidate(role, fallback.selector, currentSelector, {
|
|
1243
|
+
pinFallback: true,
|
|
1244
|
+
apiKey: fallback.apiKey,
|
|
1245
|
+
signal,
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
async #confirmUsageFallback(
|
|
1250
|
+
confirmer: UsageFallbackConfirmer,
|
|
1251
|
+
confirmation: UsageFallbackConfirmation,
|
|
1252
|
+
signal: AbortSignal,
|
|
1253
|
+
): Promise<boolean> {
|
|
1254
|
+
if (signal.aborted) return false;
|
|
1255
|
+
const aborted = Promise.withResolvers<boolean>();
|
|
1256
|
+
const onAbort = () => aborted.resolve(false);
|
|
1257
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1258
|
+
try {
|
|
1259
|
+
return await Promise.race([confirmer(confirmation, signal), aborted.promise]);
|
|
1260
|
+
} finally {
|
|
1261
|
+
signal.removeEventListener("abort", onAbort);
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1074
1265
|
async applyRetryFallbackCandidate(
|
|
1075
1266
|
role: string,
|
|
1076
1267
|
selector: RetryFallbackSelector,
|
|
1077
1268
|
currentSelector: string,
|
|
1078
1269
|
options?: { pinFallback?: boolean; apiKey?: string; signal?: AbortSignal },
|
|
1079
|
-
): Promise<
|
|
1270
|
+
): Promise<boolean> {
|
|
1080
1271
|
const resolved = resolveModelOverride([selector.raw], this.#host.modelRegistry, this.#host.settings);
|
|
1081
1272
|
const candidate = resolved.model ?? this.#host.modelRegistry.find(selector.provider, selector.id);
|
|
1082
1273
|
if (!candidate) {
|
|
1083
1274
|
throw new Error(`Retry fallback model not found: ${selector.raw}`);
|
|
1084
1275
|
}
|
|
1085
1276
|
const apiKey =
|
|
1086
|
-
options?.apiKey ??
|
|
1277
|
+
options?.apiKey ??
|
|
1278
|
+
(await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId(), { signal: options?.signal }));
|
|
1087
1279
|
if (!apiKey) {
|
|
1088
1280
|
throw new Error(`No API key for retry fallback ${selector.raw}`);
|
|
1089
1281
|
}
|
|
1090
|
-
if (options?.signal?.aborted) return;
|
|
1282
|
+
if (options?.signal?.aborted) return false;
|
|
1091
1283
|
|
|
1092
1284
|
// Capture the configured selector (auto-aware) so a fallback chain preserves
|
|
1093
1285
|
// `auto` instead of collapsing it to the level it resolved to this turn.
|
|
@@ -1101,7 +1293,15 @@ export class TurnRecovery {
|
|
|
1101
1293
|
? requestedThinkingLevel
|
|
1102
1294
|
: clampThinkingLevelToCeiling(candidate, requestedThinkingLevel, this.#host.thinkingLevelCeiling());
|
|
1103
1295
|
const candidateSelector = formatModelStringWithRouting(candidate);
|
|
1296
|
+
const previousModel = this.#host.model();
|
|
1104
1297
|
await this.#host.setModelWithProviderSessionReset(candidate);
|
|
1298
|
+
if (options?.signal?.aborted) {
|
|
1299
|
+
if (previousModel && this.#host.model() === candidate) {
|
|
1300
|
+
await this.#host.setModelWithProviderSessionReset(previousModel);
|
|
1301
|
+
}
|
|
1302
|
+
return false;
|
|
1303
|
+
}
|
|
1304
|
+
if (this.#host.model() !== candidate) return false;
|
|
1105
1305
|
this.#host.sessionManager.appendModelChange(candidateSelector, EPHEMERAL_MODEL_CHANGE_ROLE);
|
|
1106
1306
|
this.#host.settings.getStorage()?.recordModelUsage(candidateSelector);
|
|
1107
1307
|
this.#host.setThinkingLevel(nextThinkingLevel);
|
|
@@ -1123,6 +1323,7 @@ export class TurnRecovery {
|
|
|
1123
1323
|
to: selector.raw,
|
|
1124
1324
|
role,
|
|
1125
1325
|
});
|
|
1326
|
+
return true;
|
|
1126
1327
|
}
|
|
1127
1328
|
|
|
1128
1329
|
async #tryRetryModelFallback(currentSelector: string, options?: { pinFallback?: boolean }): Promise<boolean> {
|
|
@@ -1140,8 +1341,7 @@ export class TurnRecovery {
|
|
|
1140
1341
|
if (ceiling !== undefined && !modelSupportsEffortCeiling(candidate, ceiling)) continue;
|
|
1141
1342
|
const apiKey = await this.#host.modelRegistry.getApiKey(candidate, this.#host.sessionId());
|
|
1142
1343
|
if (!apiKey) continue;
|
|
1143
|
-
|
|
1144
|
-
return true;
|
|
1344
|
+
return this.applyRetryFallbackCandidate(role, selector, currentSelector, options);
|
|
1145
1345
|
}
|
|
1146
1346
|
|
|
1147
1347
|
return false;
|
|
@@ -1167,6 +1367,7 @@ export class TurnRecovery {
|
|
|
1167
1367
|
const model = this.#activeFireworksFastModel();
|
|
1168
1368
|
if (!model) return false;
|
|
1169
1369
|
if (message.stopReason !== "error") return false;
|
|
1370
|
+
if (this.#isUsagePreflightBlocked(message)) return false;
|
|
1170
1371
|
if (this.#hasReplayUnsafeOutput(message)) return false;
|
|
1171
1372
|
// A content refusal/sensitivity stop is the model's decision, not a route
|
|
1172
1373
|
// failure — switching to the base model would just re-trigger it.
|
|
@@ -1190,6 +1391,7 @@ export class TurnRecovery {
|
|
|
1190
1391
|
*/
|
|
1191
1392
|
isHardErrorFallbackEligible(message: AssistantMessage): boolean {
|
|
1192
1393
|
if (message.stopReason !== "error") return false;
|
|
1394
|
+
if (this.#isUsagePreflightBlocked(message)) return false;
|
|
1193
1395
|
const model = this.#host.model();
|
|
1194
1396
|
if (!model) return false;
|
|
1195
1397
|
const retrySettings = this.#host.settings.getGroup("retry");
|
|
@@ -1385,6 +1587,22 @@ export class TurnRecovery {
|
|
|
1385
1587
|
let delayMs = staleOpenAIResponsesReplayError
|
|
1386
1588
|
? 0
|
|
1387
1589
|
: calculateRetryBackoffDelayMs(retrySettings.baseDelayMs, this.#retryAttempt);
|
|
1590
|
+
// Concurrency caps shed-and-backoff (5s) rather than burning a sibling
|
|
1591
|
+
// credential, so the usage-limit rotation branch below is deliberately
|
|
1592
|
+
// skipped for them. Apply the reason-based backoff to the transient
|
|
1593
|
+
// same-model retry path too — otherwise the default exponential base
|
|
1594
|
+
// (≈500ms) re-hits the cap immediately and burns the retry budget while
|
|
1595
|
+
// the concurrency slot stays occupied. A categorical 402 billing cap whose
|
|
1596
|
+
// body merely mentions concurrency is still a usage limit (handled below),
|
|
1597
|
+
// so gate on the flag matching the rotation decision.
|
|
1598
|
+
if (
|
|
1599
|
+
!staleOpenAIResponsesReplayError &&
|
|
1600
|
+
!AIError.is(id, AIError.Flag.UsageLimit) &&
|
|
1601
|
+
parseRateLimitReason(errorMessage) === "CONCURRENT_LIMIT"
|
|
1602
|
+
) {
|
|
1603
|
+
const concurrentBackoffMs = calculateRateLimitBackoffMs("CONCURRENT_LIMIT");
|
|
1604
|
+
if (concurrentBackoffMs > delayMs) delayMs = concurrentBackoffMs;
|
|
1605
|
+
}
|
|
1388
1606
|
let switchedCredential = false;
|
|
1389
1607
|
let switchedModel = false;
|
|
1390
1608
|
// Set when a usage-limit error pinned the wait to credential
|