@bitkyc08/opencodex 2.67.0 → 2.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ocx.mjs +3 -0
- package/gui/dist/assets/App-DCBismRi.css +1 -0
- package/gui/dist/assets/App-mpsibsFj.js +51 -0
- package/gui/dist/assets/Tray-Bn8WhKBH.js +1 -0
- package/gui/dist/assets/{index-DtNmX7hW.css → index-BDUBS8PW.css} +1 -1
- package/gui/dist/assets/index-kUE8tuAK.js +86 -0
- package/gui/dist/assets/quota-summary-DL-bNuNL.js +1 -0
- package/gui/dist/index.html +21 -2
- package/package.json +1 -1
- package/src/adapters/anthropic-image-guard.ts +13 -1
- package/src/adapters/anthropic-image-normalize.ts +3 -2
- package/src/adapters/anthropic-output-schema.ts +36 -0
- package/src/adapters/anthropic.ts +10 -2
- package/src/adapters/base.ts +7 -0
- package/src/adapters/codebuddy/live-models.ts +180 -0
- package/src/adapters/coding-agent/protocol.ts +91 -12
- package/src/adapters/coding-agent/turn.ts +32 -27
- package/src/adapters/devin/cloud-direct/index.ts +1 -0
- package/src/adapters/devin/cloud-direct/stated-reset-retry.ts +18 -4
- package/src/adapters/devin.ts +9 -6
- package/src/adapters/google-http.ts +4 -4
- package/src/adapters/google.ts +93 -3
- package/src/adapters/kiro/adapter.ts +6 -2
- package/src/adapters/kiro/stream.ts +12 -2
- package/src/adapters/kiro/usage.ts +3 -1
- package/src/adapters/kiro-errors.ts +11 -1
- package/src/adapters/kiro-events.ts +27 -1
- package/src/adapters/kiro-refusal.ts +30 -0
- package/src/adapters/kiro-retry.ts +71 -31
- package/src/adapters/openai-chat/deepseek-artifact-schema.ts +45 -0
- package/src/adapters/openai-chat/messages.ts +31 -4
- package/src/adapters/openai-chat/serialized-tool-call-content.ts +52 -7
- package/src/adapters/openai-chat/tool-call-id-remint.ts +65 -0
- package/src/adapters/openai-chat/tool-schema.ts +6 -2
- package/src/adapters/openai-chat.ts +2 -2
- package/src/adapters/openai-responses/muse-tool-choice.ts +31 -0
- package/src/adapters/openai-responses/passthrough.ts +9 -3
- package/src/adapters/opencode-go-additional-tools.ts +19 -10
- package/src/adapters/physical-send.ts +10 -3
- package/src/adapters/registry.ts +2 -1
- package/src/adapters/responses-tool-schema.ts +31 -3
- package/src/adapters/run-turn-queue.ts +63 -23
- package/src/adapters/unique-tool-call-ids.ts +63 -0
- package/src/adapters/xai-web-search.ts +32 -2
- package/src/bridge/sse.ts +14 -2
- package/src/chat/inbound.ts +20 -6
- package/src/claude/agents-inject.ts +4 -0
- package/src/claude/desktop-3p.ts +53 -13
- package/src/claude/desktop-profile.ts +41 -4
- package/src/claude/inbound-model-options.ts +14 -2
- package/src/claude/inbound.ts +1 -1
- package/src/claude/intercept/connect-proxy.ts +17 -1
- package/src/claude/intercept/local-ca.ts +7 -1
- package/src/claude/message-threads.ts +28 -0
- package/src/cli/account-api.ts +13 -0
- package/src/cli/account-auth.ts +49 -2
- package/src/cli/account-extended.ts +4 -2
- package/src/cli/account.ts +4 -2
- package/src/cli/capabilities.ts +17 -0
- package/src/cli/claude.ts +3 -1
- package/src/cli/dispatch.ts +62 -5
- package/src/cli/index.ts +84 -79
- package/src/cli/minimax.ts +4 -2
- package/src/cli/opencode.ts +6 -1
- package/src/cli/restart-handoff.ts +110 -0
- package/src/cli/status.ts +51 -0
- package/src/client/connect.ts +34 -15
- package/src/client/link-ingress.ts +102 -0
- package/src/client/link-join.ts +37 -16
- package/src/client/link-relay.ts +228 -45
- package/src/client/link-state.ts +54 -0
- package/src/client/link-status.ts +39 -0
- package/src/client/link-teardown.ts +2 -2
- package/src/client/link-tunnel.ts +615 -86
- package/src/client/machine-api.ts +10 -4
- package/src/client/machine-listener.ts +57 -10
- package/src/client/runtime.ts +210 -35
- package/src/clients/config-export/constants.ts +1 -12
- package/src/clients/config-export/contracts.ts +2 -0
- package/src/clients/config-export/model-metadata.ts +9 -3
- package/src/clients/config-export/omp.ts +1 -1
- package/src/clients/config-export/zcode-store.ts +2 -2
- package/src/clients/config-export.ts +36 -3
- package/src/codex/app-server-processes.ts +32 -0
- package/src/codex/app-server-restart-service.ts +20 -3
- package/src/codex/catalog/access-programs.ts +50 -0
- package/src/codex/catalog/build-entries.ts +13 -0
- package/src/codex/catalog/derive-entry.ts +1 -0
- package/src/codex/catalog/model-hints.ts +11 -0
- package/src/codex/catalog/parsing.ts +6 -2
- package/src/codex/catalog/provider-models.ts +81 -0
- package/src/codex/catalog/retained-sync.ts +3 -0
- package/src/codex/codex-write-lock.ts +2 -2
- package/src/codex/convergence.ts +2 -0
- package/src/codex/desired-state.ts +14 -4
- package/src/codex/home.ts +21 -3
- package/src/codex/inject/restore.ts +17 -0
- package/src/codex/inject/routing-classify.ts +3 -2
- package/src/codex/inject-coordination.ts +4 -1
- package/src/codex/inject.ts +8 -1
- package/src/codex/journal.ts +6 -2
- package/src/codex/management-convergence.ts +9 -0
- package/src/codex/model-entitlements.ts +28 -5
- package/src/codex/routing/idle-window.ts +58 -0
- package/src/codex/routing-drift.ts +134 -0
- package/src/codex/routing-healer.ts +419 -0
- package/src/codex/routing.ts +10 -0
- package/src/codex/runtime.ts +188 -57
- package/src/codex/sibling-handoff.ts +71 -0
- package/src/codex/sibling-start.ts +139 -0
- package/src/codex/sync.ts +5 -3
- package/src/combos/failover.ts +32 -5
- package/src/combos/request.ts +11 -3
- package/src/combos/reset-window.ts +10 -5
- package/src/combos/resolve.ts +36 -6
- package/src/config/diagnostics.ts +2 -1
- package/src/config/load-degrade.ts +2 -0
- package/src/config/paths.ts +7 -1
- package/src/config/process-state.ts +10 -1
- package/src/config/schema/compaction-recovery.ts +15 -0
- package/src/config/schema/config-schema.ts +2 -0
- package/src/config/schema/leaf-validators.ts +9 -3
- package/src/generated/compatibility-version.json +360 -204
- package/src/images/loop.ts +47 -32
- package/src/integrations/catalog-refresh.ts +4 -0
- package/src/integrations/omp-yaml-source.ts +1 -0
- package/src/lib/claude-request-projection.ts +101 -0
- package/src/lib/local-account-switch-capability.ts +48 -0
- package/src/lib/local-upstream.ts +146 -0
- package/src/lib/package-tree-integrity.ts +25 -3
- package/src/lib/package-tree-retarget.ts +130 -0
- package/src/lib/process-control.ts +4 -1
- package/src/lib/system-restart-contract.ts +106 -0
- package/src/lib/upstream-retry.ts +57 -1
- package/src/link/ports.ts +9 -0
- package/src/link/ssh-argv.ts +16 -0
- package/src/link/ssh-runner.ts +70 -2
- package/src/link/tunnel-state.ts +58 -9
- package/src/oauth/account-quota-rank.ts +29 -18
- package/src/oauth/generic-account-failover.ts +179 -22
- package/src/oauth/index.ts +17 -4
- package/src/oauth/kiro-account-load.ts +107 -0
- package/src/oauth/kiro-device-login.ts +312 -0
- package/src/oauth/kiro-terminal-failover.ts +28 -0
- package/src/oauth/login-flow-state.ts +7 -1
- package/src/oauth/pool-settings-capability.ts +23 -14
- package/src/oauth/store.ts +56 -3
- package/src/oauth/types.ts +4 -0
- package/src/plugins/loader.ts +350 -0
- package/src/plugins/upstream-hooks.ts +119 -0
- package/src/protocols/encoders/adapter-events.ts +9 -2
- package/src/providers/account-quota-disk.ts +42 -4
- package/src/providers/codebuddy-models.ts +6 -2
- package/src/providers/command-code-efforts.ts +33 -10
- package/src/providers/kiro-account-state-disk.ts +44 -0
- package/src/providers/kiro-model-catalog.ts +162 -0
- package/src/providers/kiro-models.ts +5 -4
- package/src/providers/kiro-quota-metrics.ts +35 -0
- package/src/providers/kiro-usage.ts +118 -14
- package/src/providers/quota/account-cache.ts +43 -7
- package/src/providers/quota/antigravity.ts +8 -3
- package/src/providers/quota/kiro-account-probe.ts +12 -0
- package/src/providers/quota/vendor-probes-key.ts +31 -26
- package/src/providers/quota/vendor-probes-oauth.ts +16 -7
- package/src/providers/quota-types.ts +14 -0
- package/src/providers/quota.ts +21 -21
- package/src/providers/registry/entries-extended.ts +5 -3
- package/src/providers/request-pacing.ts +410 -12
- package/src/remote-control/workspace-codex-runtime.ts +4 -3
- package/src/responses/citation-markers.ts +132 -65
- package/src/responses/hosted-tool-policy.ts +14 -3
- package/src/responses/parser-content.ts +3 -2
- package/src/responses/parser.ts +4 -1
- package/src/responses/schema.ts +4 -0
- package/src/responses/visualization-directives.ts +182 -0
- package/src/server/chat-native-sse.ts +24 -12
- package/src/server/chat-native.ts +137 -68
- package/src/server/claude-messages.ts +147 -12
- package/src/server/index/link-listener.ts +7 -0
- package/src/server/index/optional-listeners.ts +4 -1
- package/src/server/index/package-tree-guard.ts +35 -16
- package/src/server/index/serve-options.ts +4 -2
- package/src/server/index/startup-warnings.ts +6 -0
- package/src/server/index.ts +7 -7
- package/src/server/inference/client-encoder-delivery.ts +3 -0
- package/src/server/inference/context.ts +30 -2
- package/src/server/local-account-switch-auth.ts +79 -0
- package/src/server/management/agent-settings-routes.ts +11 -3
- package/src/server/management/config-routes.ts +28 -11
- package/src/server/management/context.ts +2 -0
- package/src/server/management/link-routes.ts +123 -39
- package/src/server/management/logs-usage-routes.ts +9 -42
- package/src/server/management/model-rows.ts +1 -0
- package/src/server/management/oauth-account-routes.ts +67 -12
- package/src/server/management/route-registry.ts +5 -5
- package/src/server/management/sibling-guard.ts +60 -0
- package/src/server/management/storage-log-guard-routes.ts +72 -6
- package/src/server/management/system-restart.ts +92 -135
- package/src/server/management/system-routes.ts +1 -1
- package/src/server/management-api.ts +31 -3
- package/src/server/management-auth.ts +3 -0
- package/src/server/port-reclaim.ts +111 -1
- package/src/server/proxy-liveness.ts +16 -0
- package/src/server/relay.ts +16 -15
- package/src/server/request-log-filter.ts +28 -27
- package/src/server/request-log.ts +7 -3
- package/src/server/request-metrics.ts +28 -0
- package/src/server/responses/adapter-continuation.ts +151 -21
- package/src/server/responses/adapter-delivery.ts +21 -7
- package/src/server/responses/adapter-dispatch.ts +239 -51
- package/src/server/responses/codex-ws-pool.ts +5 -4
- package/src/server/responses/codex-ws-request.ts +4 -1
- package/src/server/responses/compact.ts +8 -4
- package/src/server/responses/compaction-recovery-policy.ts +116 -0
- package/src/server/responses/compaction-recovery.ts +330 -0
- package/src/server/responses/core-combo-failure.ts +1 -0
- package/src/server/responses/core-combo.ts +5 -1
- package/src/server/responses/core-lifetime.ts +22 -0
- package/src/server/responses/core-options.ts +16 -1
- package/src/server/responses/core.ts +14 -4
- package/src/server/responses/empty-completion-guard.ts +2 -0
- package/src/server/responses/fetch-helpers.ts +82 -31
- package/src/server/responses/passthrough-delivery.ts +3 -2
- package/src/server/responses/passthrough-dispatch.ts +12 -2
- package/src/server/responses/passthrough-execution.ts +2 -2
- package/src/server/responses/request-send-budget.ts +21 -5
- package/src/server/responses/request-transport.ts +116 -18
- package/src/server/responses/reset-replay.ts +32 -0
- package/src/server/responses/run-turn-execution.ts +136 -19
- package/src/server/responses/sidecar-execution.ts +41 -8
- package/src/server/responses/terminal-guard.ts +2 -0
- package/src/server/responses/ws-upstream.ts +44 -4
- package/src/server/restart-replacement.ts +382 -0
- package/src/server/startup-health-cache.ts +23 -4
- package/src/server/stop-teardown.ts +35 -1
- package/src/server/system-env.ts +3 -0
- package/src/service/windows-taskxml.ts +11 -7
- package/src/service/windows-wrapper-exit.ts +8 -0
- package/src/stall-timeout.ts +35 -10
- package/src/storage/policy-job.ts +4 -0
- package/src/storage/scanner.ts +177 -29
- package/src/storage/storage-mutation-coordinator.ts +17 -0
- package/src/tray/windows-tray.ps1 +50 -23
- package/src/types/config.ts +4 -0
- package/src/types/provider.ts +5 -1
- package/src/types/request.ts +13 -2
- package/src/update/index.ts +3 -1
- package/src/update/job.ts +6 -2
- package/src/update/mise-launcher-target.ts +100 -0
- package/src/update/worker-launch.ts +46 -0
- package/src/usage/log.ts +2 -0
- package/src/web-search/loop.ts +5 -2
- package/gui/dist/assets/App-BJsT8Icc.css +0 -1
- package/gui/dist/assets/App-D3pNGiN4.js +0 -51
- package/gui/dist/assets/Tray-yb05wCjb.js +0 -1
- package/gui/dist/assets/index-CouvvtMV.js +0 -86
- package/gui/dist/assets/tray-data-CYGdjCJ7.js +0 -1
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* `killAllOcxOnPort`, and successful ocx verification. A historical PID allowlist
|
|
7
7
|
* never overrides a rejected verifier result; rejected live holders stay protected.
|
|
8
8
|
*/
|
|
9
|
-
import { execFileSync } from "node:child_process";
|
|
9
|
+
import { execFile, execFileSync } from "node:child_process";
|
|
10
|
+
import { readFile, readdir, readlink } from "node:fs/promises";
|
|
11
|
+
import { promisify } from "node:util";
|
|
10
12
|
import { verifyPidIdentity } from "../config/process-state";
|
|
11
13
|
import { isProcessAlive, killProxy } from "../lib/process-control";
|
|
12
14
|
import { isPortAvailable, type WaitForPortOptions } from "./ports";
|
|
@@ -169,6 +171,114 @@ export function listListenPids(port: number): number[] {
|
|
|
169
171
|
return scan.ok ? scan.pids : [];
|
|
170
172
|
}
|
|
171
173
|
|
|
174
|
+
const execFileAsync = promisify(execFile);
|
|
175
|
+
const OWNER_LOOKUP_TIMEOUT_MS = 2_000;
|
|
176
|
+
const MAX_PROC_NET_BYTES = 4 * 1024 * 1024;
|
|
177
|
+
const MAX_PROCESS_FDS = 4_096;
|
|
178
|
+
|
|
179
|
+
/** The exact IPv4 listener the Child relay contacts; ::1 and wildcard binds are not proof. */
|
|
180
|
+
export function parseProcLoopbackListenInodes(tcp: string, tcp6: string, port: number): string[] {
|
|
181
|
+
const wantedPort = port.toString(16).toUpperCase().padStart(4, "0");
|
|
182
|
+
const inodes = new Set<string>();
|
|
183
|
+
for (const [content, wantedAddress] of [
|
|
184
|
+
[tcp, "0100007F"],
|
|
185
|
+
[tcp6, "0000000000000000FFFF00000100007F"], // ::ffff:127.0.0.1 in proc word order
|
|
186
|
+
] as const) {
|
|
187
|
+
for (const line of content.split(/\r?\n/).slice(1)) {
|
|
188
|
+
const fields = line.trim().split(/\s+/);
|
|
189
|
+
const [address, hexPort] = (fields[1] ?? "").toUpperCase().split(":");
|
|
190
|
+
if (address !== wantedAddress || hexPort !== wantedPort || fields[3] !== "0A") continue;
|
|
191
|
+
const inode = fields[9];
|
|
192
|
+
if (inode && /^[1-9]\d*$/.test(inode)) inodes.add(inode);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return [...inodes];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Parse only 127.0.0.1:port LISTEN owners; an unrelated [::1]:port must not veto it. */
|
|
199
|
+
export function parseIpv4LoopbackListenPidsFromNetstat(output: string, port: number): number[] {
|
|
200
|
+
const pids = new Set<number>();
|
|
201
|
+
for (const raw of output.split(/\r?\n/)) {
|
|
202
|
+
const fields = raw.trim().split(/\s+/);
|
|
203
|
+
if (fields[0]?.toUpperCase() !== "TCP" || fields[1] !== `127.0.0.1:${port}`) continue;
|
|
204
|
+
const state = fields[3] ?? "";
|
|
205
|
+
const foreign = fields[2] ?? "";
|
|
206
|
+
if (!/^LISTEN/i.test(state) && !["0.0.0.0:0", "[::]:0", "*:*"].includes(foreign)) continue;
|
|
207
|
+
const pid = Number(fields.at(-1));
|
|
208
|
+
if (Number.isSafeInteger(pid) && pid > 0) pids.add(pid);
|
|
209
|
+
}
|
|
210
|
+
return [...pids];
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function runOwnerLookup(file: string, args: string[], timeoutMs: number): Promise<string> {
|
|
214
|
+
const { stdout } = await execFileAsync(file, args, {
|
|
215
|
+
encoding: "utf8", timeout: timeoutMs, maxBuffer: MAX_PROC_NET_BYTES, windowsHide: true,
|
|
216
|
+
});
|
|
217
|
+
return stdout;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export interface LoopbackOwnerLookupIo {
|
|
221
|
+
platform?: NodeJS.Platform;
|
|
222
|
+
readProc?: (path: string) => Promise<string>;
|
|
223
|
+
listFds?: (path: string) => Promise<string[]>;
|
|
224
|
+
readFdLink?: (path: string) => Promise<string>;
|
|
225
|
+
run?: (file: string, args: string[], timeoutMs: number) => Promise<string>;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Async, bounded proof that this PID owns the IPv4 loopback LISTEN socket used by the relay. */
|
|
229
|
+
export async function ownsIpv4LoopbackListener(
|
|
230
|
+
port: number,
|
|
231
|
+
expectedPid: number,
|
|
232
|
+
io: LoopbackOwnerLookupIo = {},
|
|
233
|
+
): Promise<boolean> {
|
|
234
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535
|
|
235
|
+
|| !Number.isSafeInteger(expectedPid) || expectedPid < 1) return false;
|
|
236
|
+
const platform = io.platform ?? process.platform;
|
|
237
|
+
const lookup = async (): Promise<boolean> => {
|
|
238
|
+
if (platform === "linux") {
|
|
239
|
+
const readProc = io.readProc ?? (path => readFile(path, "utf8"));
|
|
240
|
+
const tcp = await readProc("/proc/net/tcp");
|
|
241
|
+
const tcp6 = await readProc("/proc/net/tcp6").catch(error => {
|
|
242
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return "";
|
|
243
|
+
throw error;
|
|
244
|
+
});
|
|
245
|
+
if (tcp.length > MAX_PROC_NET_BYTES || tcp6.length > MAX_PROC_NET_BYTES) return false;
|
|
246
|
+
const inodes = parseProcLoopbackListenInodes(tcp, tcp6, port);
|
|
247
|
+
if (inodes.length !== 1) return false;
|
|
248
|
+
const fdDir = `/proc/${expectedPid}/fd`;
|
|
249
|
+
const fds = await (io.listFds ?? readdir)(fdDir);
|
|
250
|
+
if (fds.length > MAX_PROCESS_FDS) return false;
|
|
251
|
+
const wanted = `socket:[${inodes[0]}]`;
|
|
252
|
+
const readFdLink = io.readFdLink ?? readlink;
|
|
253
|
+
for (const fd of fds) {
|
|
254
|
+
try { if (await readFdLink(`${fdDir}/${fd}`) === wanted) return true; }
|
|
255
|
+
catch { /* an fd may close while it is enumerated */ }
|
|
256
|
+
}
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
const run = io.run ?? runOwnerLookup;
|
|
260
|
+
if (platform === "darwin") {
|
|
261
|
+
const stdout = await run("/usr/sbin/lsof", ["-nP", "-a", `-iTCP@127.0.0.1:${port}`, "-sTCP:LISTEN", "-t"], OWNER_LOOKUP_TIMEOUT_MS);
|
|
262
|
+
const pids = new Set(stdout.split(/\r?\n/).map(line => Number(line.trim())).filter(pid => Number.isSafeInteger(pid) && pid > 0));
|
|
263
|
+
return pids.size === 1 && pids.has(expectedPid);
|
|
264
|
+
}
|
|
265
|
+
if (platform === "win32") {
|
|
266
|
+
const netstat = `${process.env.SystemRoot ?? "C:\\Windows"}\\System32\\netstat.exe`;
|
|
267
|
+
const stdout = await run(netstat, ["-ano", "-p", "tcp"], OWNER_LOOKUP_TIMEOUT_MS);
|
|
268
|
+
const pids = parseIpv4LoopbackListenPidsFromNetstat(stdout, port);
|
|
269
|
+
return pids.length === 1 && pids[0] === expectedPid;
|
|
270
|
+
}
|
|
271
|
+
return false;
|
|
272
|
+
};
|
|
273
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
274
|
+
const deadline = new Promise<false>(resolve => {
|
|
275
|
+
timer = setTimeout(() => resolve(false), OWNER_LOOKUP_TIMEOUT_MS);
|
|
276
|
+
timer.unref?.();
|
|
277
|
+
});
|
|
278
|
+
try { return await Promise.race([lookup().catch(() => false), deadline]); }
|
|
279
|
+
finally { if (timer) clearTimeout(timer); }
|
|
280
|
+
}
|
|
281
|
+
|
|
172
282
|
/**
|
|
173
283
|
* Wait until `port` can bind.
|
|
174
284
|
* Never kills a process unless `killOcxHolders === true` and either
|
|
@@ -182,6 +182,22 @@ export interface LiveProxy {
|
|
|
182
182
|
packageTreeFenced?: true;
|
|
183
183
|
}
|
|
184
184
|
|
|
185
|
+
/**
|
|
186
|
+
* A /healthz identity proves only that a proxy holds the port. Before a destructive orphan stop,
|
|
187
|
+
* require that listener to prove possession of this home's runtime-record secret as well.
|
|
188
|
+
*/
|
|
189
|
+
export async function proveLiveProxyOwnedByHome(live: LiveProxy, io: LivenessIo = {}): Promise<boolean> {
|
|
190
|
+
if (live.pid === null) return false;
|
|
191
|
+
return attestFencedIdentity(
|
|
192
|
+
`http://${probeHostname(live.hostname)}:${live.port}/healthz`,
|
|
193
|
+
live.port,
|
|
194
|
+
live.pid,
|
|
195
|
+
io,
|
|
196
|
+
io.fetchFn ?? directLocalHttpFetch,
|
|
197
|
+
io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
185
201
|
/**
|
|
186
202
|
* Host to probe for a given bind hostname: wildcards answer on IPv4 loopback, and raw
|
|
187
203
|
* IPv6 addresses must be bracketed or the composed URL is invalid.
|
package/src/server/relay.ts
CHANGED
|
@@ -1624,6 +1624,21 @@ export const CODEX_SAFETY_BUFFERING_HEADERS = [
|
|
|
1624
1624
|
|
|
1625
1625
|
const CODEX_SAFETY_BUFFERING_HEADER_SET: ReadonlySet<string> = new Set(CODEX_SAFETY_BUFFERING_HEADERS);
|
|
1626
1626
|
|
|
1627
|
+
const PASSTHROUGH_DROP_HEADERS: ReadonlySet<string> = new Set([
|
|
1628
|
+
"content-encoding",
|
|
1629
|
+
"content-length",
|
|
1630
|
+
"transfer-encoding",
|
|
1631
|
+
"connection",
|
|
1632
|
+
"keep-alive",
|
|
1633
|
+
"proxy-authenticate",
|
|
1634
|
+
"proxy-authorization",
|
|
1635
|
+
"set-cookie",
|
|
1636
|
+
"set-cookie2",
|
|
1637
|
+
"te",
|
|
1638
|
+
"trailer",
|
|
1639
|
+
"upgrade",
|
|
1640
|
+
]);
|
|
1641
|
+
|
|
1627
1642
|
export interface CodexSafetyBufferingFilterOptions {
|
|
1628
1643
|
/**
|
|
1629
1644
|
* Drop Codex safety-buffering hints: the `x-codex-safety-buffering-*` response
|
|
@@ -1642,24 +1657,10 @@ export function codexSafetyBufferingFilterOptions(
|
|
|
1642
1657
|
|
|
1643
1658
|
export function sanitizePassthroughHeaders(upstream: Headers, options?: CodexSafetyBufferingFilterOptions): Headers {
|
|
1644
1659
|
const dropSafetyBuffering = options?.dropCodexSafetyBuffering === true;
|
|
1645
|
-
const DROP = new Set([
|
|
1646
|
-
"content-encoding",
|
|
1647
|
-
"content-length",
|
|
1648
|
-
"transfer-encoding",
|
|
1649
|
-
"connection",
|
|
1650
|
-
"keep-alive",
|
|
1651
|
-
"proxy-authenticate",
|
|
1652
|
-
"proxy-authorization",
|
|
1653
|
-
"set-cookie",
|
|
1654
|
-
"set-cookie2",
|
|
1655
|
-
"te",
|
|
1656
|
-
"trailer",
|
|
1657
|
-
"upgrade",
|
|
1658
|
-
]);
|
|
1659
1660
|
const out = new Headers();
|
|
1660
1661
|
upstream.forEach((value, key) => {
|
|
1661
1662
|
const lower = key.toLowerCase();
|
|
1662
|
-
if (
|
|
1663
|
+
if (PASSTHROUGH_DROP_HEADERS.has(lower)) return;
|
|
1663
1664
|
if (dropSafetyBuffering && CODEX_SAFETY_BUFFERING_HEADER_SET.has(lower)) return;
|
|
1664
1665
|
out.set(key, value);
|
|
1665
1666
|
});
|
|
@@ -22,49 +22,49 @@ export function matchesProtocolMode(entry: Pick<RequestLogEntry, "protocolTrace"
|
|
|
22
22
|
return isDeliveryMode(mode) && entry.protocolTrace?.mode === mode;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
export function
|
|
26
|
-
|
|
25
|
+
export function queryRequestLogs(
|
|
26
|
+
logs: RequestLogEntry[],
|
|
27
|
+
params: URLSearchParams,
|
|
28
|
+
): { total: number; logs: RequestLogEntry[] } {
|
|
27
29
|
const provider = params.get("provider")?.trim();
|
|
28
|
-
if (provider) {
|
|
29
|
-
filtered = filtered.filter(entry => entry.provider === provider
|
|
30
|
-
|| entry.attempts?.some(attempt => attempt.provider === provider));
|
|
31
|
-
}
|
|
32
30
|
const conversationId = params.get("conversationId")?.trim() || params.get("conversation")?.trim();
|
|
33
|
-
if (conversationId) {
|
|
34
|
-
filtered = filtered.filter(entry => matchesLogConversationId(entry.conversationId, conversationId));
|
|
35
|
-
}
|
|
36
31
|
// #2704: there was no `model` clause at all, so `?model=x` was ACCEPTED and silently
|
|
37
32
|
// ignored -- worse than an error, because it yields wrong conclusions from output that
|
|
38
33
|
// looks correct. Attempts are matched for the same reason `provider` matches them: a
|
|
39
34
|
// request that failed over should be findable by the model that actually served it.
|
|
40
35
|
const model = params.get("model")?.trim();
|
|
41
|
-
if (model) {
|
|
42
|
-
filtered = filtered.filter(entry => entry.model === model
|
|
43
|
-
|| entry.attempts?.some(attempt => attempt.model === model));
|
|
44
|
-
}
|
|
45
36
|
// #4057: "which account served this request" is the first question asked when one provider
|
|
46
37
|
// holds several accounts, and until now the only way to answer it was to grep usage.jsonl by
|
|
47
38
|
// hand. Attempts are matched for the same reason `provider` and `model` match them: when a
|
|
48
39
|
// request failed over between pool accounts, a search for the account that finally served it
|
|
49
40
|
// has to find that request, not only the account that first refused it.
|
|
50
41
|
const account = params.get("account")?.trim();
|
|
51
|
-
if (account) {
|
|
52
|
-
filtered = filtered.filter(entry => entry.accountLogLabel === account
|
|
53
|
-
|| entry.attempts?.some(attempt => attempt.accountLogLabel === account));
|
|
54
|
-
}
|
|
55
42
|
const protocolMode = params.get("protocolMode")?.trim().toLowerCase();
|
|
56
|
-
if (protocolMode) filtered = filtered.filter(entry => matchesProtocolMode(entry, protocolMode));
|
|
57
43
|
const status = params.get("status")?.trim().toLowerCase();
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
44
|
+
|
|
45
|
+
const hasFilters = Boolean(provider || conversationId || model || account || protocolMode || status);
|
|
46
|
+
let filtered: RequestLogEntry[];
|
|
47
|
+
if (!hasFilters) {
|
|
48
|
+
filtered = logs;
|
|
49
|
+
} else {
|
|
50
|
+
const statusClass = status && /^[1-5]xx$/.test(status) ? Number(status[0]) : undefined;
|
|
51
|
+
filtered = [];
|
|
52
|
+
for (const entry of logs) {
|
|
53
|
+
if (provider && entry.provider !== provider && !entry.attempts?.some(attempt => attempt.provider === provider)) continue;
|
|
54
|
+
if (conversationId && !matchesLogConversationId(entry.conversationId, conversationId)) continue;
|
|
55
|
+
if (model && entry.model !== model && !entry.attempts?.some(attempt => attempt.model === model)) continue;
|
|
56
|
+
if (account && entry.accountLogLabel !== account && !entry.attempts?.some(attempt => attempt.accountLogLabel === account)) continue;
|
|
57
|
+
if (protocolMode && !matchesProtocolMode(entry, protocolMode)) continue;
|
|
58
|
+
if (status && (statusClass === undefined ? String(entry.status) !== status : Math.floor(entry.status / 100) !== statusClass)) continue;
|
|
59
|
+
filtered.push(entry);
|
|
60
|
+
}
|
|
62
61
|
}
|
|
63
62
|
const tailRaw = params.get("tail")?.trim();
|
|
64
63
|
if (tailRaw) {
|
|
65
64
|
const tail = Number.parseInt(tailRaw, 10);
|
|
66
65
|
if (Number.isFinite(tail) && tail > 0) filtered = filtered.slice(-Math.min(tail, MAX_LOG_SIZE));
|
|
67
66
|
}
|
|
67
|
+
const total = filtered.length;
|
|
68
68
|
const offsetRaw = params.get("offset")?.trim();
|
|
69
69
|
const limitRaw = params.get("limit")?.trim();
|
|
70
70
|
if (limitRaw) {
|
|
@@ -81,12 +81,13 @@ export function filterRequestLogs(logs: RequestLogEntry[], params: URLSearchPara
|
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
|
-
return filtered;
|
|
84
|
+
return { total, logs: filtered };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function filterRequestLogs(logs: RequestLogEntry[], params: URLSearchParams): RequestLogEntry[] {
|
|
88
|
+
return queryRequestLogs(logs, params).logs;
|
|
85
89
|
}
|
|
86
90
|
|
|
87
91
|
export function filteredRequestLogCount(logs: RequestLogEntry[], params: URLSearchParams): number {
|
|
88
|
-
|
|
89
|
-
withoutPagination.delete("limit");
|
|
90
|
-
withoutPagination.delete("offset");
|
|
91
|
-
return filterRequestLogs(logs, withoutPagination).length;
|
|
92
|
+
return queryRequestLogs(logs, params).total;
|
|
92
93
|
}
|
|
@@ -73,11 +73,12 @@ import {
|
|
|
73
73
|
type UsageDebugBodyKind,
|
|
74
74
|
} from "../usage/debug";
|
|
75
75
|
import { MAX_LOG_SIZE } from "./request-log-filter";
|
|
76
|
-
export { filterRequestLogs, filteredRequestLogCount } from "./request-log-filter";
|
|
76
|
+
export { filterRequestLogs, filteredRequestLogCount, queryRequestLogs } from "./request-log-filter";
|
|
77
77
|
import { enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory";
|
|
78
78
|
import { capEstimateAtContextWindow } from "../lib/token-estimate";
|
|
79
79
|
import { inferCursorContextWindow } from "../adapters/cursor/discovery";
|
|
80
80
|
import { KIRO_MODEL_CONTEXT_WINDOWS, normalizeKiroModelId } from "../providers/kiro-models";
|
|
81
|
+
import { kiroObservedContextWindow } from "../providers/kiro-model-catalog";
|
|
81
82
|
import { DEVIN_MODEL_CONTEXT_WINDOWS } from "../adapters/devin/live-models";
|
|
82
83
|
import { modelRecordValue } from "../reasoning-effort";
|
|
83
84
|
import {
|
|
@@ -1629,7 +1630,8 @@ function contextWindowForModel(adapter: string, modelId: string | undefined): nu
|
|
|
1629
1630
|
if (adapter === "kiro" || adapter.startsWith("kiro-")) {
|
|
1630
1631
|
const normalized = normalizeKiroModelId(modelId);
|
|
1631
1632
|
if (normalized === "auto") return undefined;
|
|
1632
|
-
return
|
|
1633
|
+
return kiroObservedContextWindow(modelId)
|
|
1634
|
+
?? modelRecordValue(KIRO_MODEL_CONTEXT_WINDOWS, modelId)
|
|
1633
1635
|
?? modelRecordValue(KIRO_MODEL_CONTEXT_WINDOWS, normalized);
|
|
1634
1636
|
}
|
|
1635
1637
|
if (adapter === "cursor" || adapter.startsWith("cursor-")) {
|
|
@@ -1905,7 +1907,7 @@ export function aggregateAttemptUsage(
|
|
|
1905
1907
|
|
|
1906
1908
|
const sumOptional = (
|
|
1907
1909
|
key: "cachedInputTokens" | "cacheReadInputTokens" | "cacheCreationInputTokens"
|
|
1908
|
-
| "reasoningOutputTokens",
|
|
1910
|
+
| "reasoningOutputTokens" | "providerCredits",
|
|
1909
1911
|
): number | undefined => {
|
|
1910
1912
|
const present = usages.flatMap(usage => (
|
|
1911
1913
|
typeof usage[key] === "number" ? [usage[key] as number] : []
|
|
@@ -1916,6 +1918,7 @@ export function aggregateAttemptUsage(
|
|
|
1916
1918
|
const cacheReadInputTokens = sumOptional("cacheReadInputTokens");
|
|
1917
1919
|
const cacheCreationInputTokens = sumOptional("cacheCreationInputTokens");
|
|
1918
1920
|
const reasoningOutputTokens = sumOptional("reasoningOutputTokens");
|
|
1921
|
+
const providerCredits = sumOptional("providerCredits");
|
|
1919
1922
|
const totalTokens = usages.reduce(
|
|
1920
1923
|
(sum, usage) => sum + (usageTotalTokens(usage) ?? 0),
|
|
1921
1924
|
0,
|
|
@@ -1928,6 +1931,7 @@ export function aggregateAttemptUsage(
|
|
|
1928
1931
|
...(cacheReadInputTokens !== undefined ? { cacheReadInputTokens } : {}),
|
|
1929
1932
|
...(cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens } : {}),
|
|
1930
1933
|
...(reasoningOutputTokens !== undefined ? { reasoningOutputTokens } : {}),
|
|
1934
|
+
...(providerCredits !== undefined ? { providerCredits } : {}),
|
|
1931
1935
|
...(status === "estimated" ? { estimated: true } : {}),
|
|
1932
1936
|
};
|
|
1933
1937
|
return { usage: aggregate, status, totalTokens };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ResponsesTerminalStatus } from "../bridge";
|
|
2
|
+
import type { KiroQuotaMetricRow } from "../providers/kiro-quota-metrics";
|
|
2
3
|
import type { AttemptRecoveryKind } from "../usage/log";
|
|
3
4
|
import {
|
|
4
5
|
REQUEST_FAILURE_CAUSES,
|
|
@@ -188,8 +189,34 @@ function appendHistogram(
|
|
|
188
189
|
}
|
|
189
190
|
}
|
|
190
191
|
|
|
192
|
+
function appendKiroQuotaGauges(lines: string[], rows: readonly KiroQuotaMetricRow[]): void {
|
|
193
|
+
const families = [
|
|
194
|
+
["opencodex_kiro_quota_used_credits", "Cached Kiro plan credits used.", "used"],
|
|
195
|
+
["opencodex_kiro_quota_limit_credits", "Cached Kiro plan credit limit.", "limit"],
|
|
196
|
+
["opencodex_kiro_quota_used_percent", "Cached Kiro plan percent used.", "percent"],
|
|
197
|
+
["opencodex_kiro_quota_seconds_to_reset", "Seconds until the observed Kiro reset.", "secondsToReset"],
|
|
198
|
+
] as const;
|
|
199
|
+
const labels = new Set<string>();
|
|
200
|
+
const valid = rows.slice(0, 32).filter(row => {
|
|
201
|
+
if (!/^o[0-9a-f]{6}$/.test(row.account) || labels.has(row.account)) return false;
|
|
202
|
+
if (![row.used, row.limit, row.percent].every(value => Number.isFinite(value) && value >= 0)) return false;
|
|
203
|
+
if (row.limit <= 0 || row.percent > 100) return false;
|
|
204
|
+
labels.add(row.account);
|
|
205
|
+
return true;
|
|
206
|
+
});
|
|
207
|
+
for (const [name, help, key] of families) {
|
|
208
|
+
lines.push(`# HELP ${name} ${help}`, `# TYPE ${name} gauge`);
|
|
209
|
+
for (const row of valid) {
|
|
210
|
+
const value = row[key];
|
|
211
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
|
|
212
|
+
lines.push(`${name}{account="${row.account}"} ${value}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
191
217
|
export function createRequestMetricsOwner(
|
|
192
218
|
processStartTimeSeconds = Date.now() / 1000,
|
|
219
|
+
kiroQuotaRows?: () => readonly KiroQuotaMetricRow[],
|
|
193
220
|
): RequestMetricsOwner {
|
|
194
221
|
let logicalRequests = matrix(REQUEST_METRICS_PROTOCOLS.length, REQUEST_METRICS_RESULTS.length);
|
|
195
222
|
let physicalSends = Array.from({ length: REQUEST_METRICS_PROTOCOLS.length }, () => 0);
|
|
@@ -282,6 +309,7 @@ export function createRequestMetricsOwner(
|
|
|
282
309
|
lines.push(`opencodex_ttft_missing_total${sampleLabels(protocol, result)} ${missingTtft[protocolCell(protocol)]![resultCell(result)]}`);
|
|
283
310
|
}
|
|
284
311
|
}
|
|
312
|
+
appendKiroQuotaGauges(lines, kiroQuotaRows?.() ?? []);
|
|
285
313
|
lines.push(
|
|
286
314
|
"# HELP opencodex_metrics_process_start_time_seconds Unix time when this process metrics owner started.",
|
|
287
315
|
"# TYPE opencodex_metrics_process_start_time_seconds gauge",
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
recordAttemptCredentialSource,
|
|
15
15
|
} from "../request-log";
|
|
16
16
|
import { noteAttemptRecoveryWithheld } from "../request-log";
|
|
17
|
-
import {
|
|
17
|
+
import { withProviderRequestSlot } from "../../providers/request-pacing";
|
|
18
18
|
import { providerFetch, fetchWithHeaderTimeout, safeHostLabel } from "./fetch-helpers";
|
|
19
19
|
import {
|
|
20
20
|
transientRetryPolicyFor,
|
|
@@ -39,12 +39,17 @@ import {
|
|
|
39
39
|
formatAnthropicProviderForLog,
|
|
40
40
|
} from "../../oauth/anthropic-routing";
|
|
41
41
|
import {
|
|
42
|
-
GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST,
|
|
43
42
|
hasEligibleGenericOAuthFailoverTarget,
|
|
44
43
|
isGenericOAuthFailoverEnabled,
|
|
45
44
|
rotateGenericOAuthAccountOn429,
|
|
45
|
+
rotateGenericOAuthAccountOnRefusal,
|
|
46
|
+
quarantineKiroSuspendedAccount,
|
|
46
47
|
failoverAccountSnapshot,
|
|
47
48
|
} from "../../oauth/generic-account-failover";
|
|
49
|
+
import { classifyKiroRefusal } from "../../adapters/kiro-refusal";
|
|
50
|
+
import { normalizeFinalKiroHttpError } from "../../adapters/kiro-retry";
|
|
51
|
+
import { noteKiroMonthlyRefusal } from "../../providers/kiro-usage";
|
|
52
|
+
import { persistKiroAccountState } from "../../providers/kiro-account-state-disk";
|
|
48
53
|
import { shouldAttemptImageTierRetry } from "../image-retry";
|
|
49
54
|
import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors";
|
|
50
55
|
import { isCyberPolicyCode, CYBER_POLICY_FALLBACK_MESSAGE, CYBER_POLICY_ERROR_CODE } from "../../lib/errors";
|
|
@@ -53,7 +58,7 @@ import {
|
|
|
53
58
|
readResponseStreamWithInactivity,
|
|
54
59
|
ResponseBodyInactivityError,
|
|
55
60
|
} from "../../lib/response-body-inactivity";
|
|
56
|
-
import {
|
|
61
|
+
import { resolveStallTimeoutMs } from "../../stall-timeout";
|
|
57
62
|
import { guardTerminalEventStream } from "./terminal-guard";
|
|
58
63
|
|
|
59
64
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
@@ -84,6 +89,7 @@ export function createAdapterContinuations(
|
|
|
84
89
|
| "commitResolvedOAuthSelection"
|
|
85
90
|
| "genericFailoverAccountId"
|
|
86
91
|
| "genericFailovers"
|
|
92
|
+
| "genericFailoverLimit"
|
|
87
93
|
| "applyFailoverSnapshot"
|
|
88
94
|
| "replayOAuthCredentialSnapshot"
|
|
89
95
|
| "noteRoutedAttemptSend"
|
|
@@ -107,7 +113,8 @@ export function createAdapterContinuations(
|
|
|
107
113
|
| "rateLimitPolicy"
|
|
108
114
|
| "rateLimitRetries"
|
|
109
115
|
| "stallTimeoutMs"
|
|
110
|
-
|
|
116
|
+
| "localUpstream"
|
|
117
|
+
>,
|
|
111
118
|
) {
|
|
112
119
|
const { options, logCtx, config } = requestContext;
|
|
113
120
|
const {
|
|
@@ -120,8 +127,8 @@ export function createAdapterContinuations(
|
|
|
120
127
|
} = transportState;
|
|
121
128
|
const { route, translatorBudget, inboundWire, parsed } = requestState;
|
|
122
129
|
const { routedCompaction } = sidecarState;
|
|
123
|
-
const { upstream, connectMs, rateLimitPolicy, stallTimeoutMs } = adapterExchange;
|
|
124
|
-
const bodyInactivityMs =
|
|
130
|
+
const { upstream, connectMs, rateLimitPolicy, stallTimeoutMs, localUpstream } = adapterExchange;
|
|
131
|
+
const bodyInactivityMs = resolveStallTimeoutMs(config.stallTimeoutSec, { localUpstream });
|
|
125
132
|
const {
|
|
126
133
|
adapterDispatchBudget,
|
|
127
134
|
noteAdapterPhysicalSend,
|
|
@@ -151,6 +158,8 @@ export function createAdapterContinuations(
|
|
|
151
158
|
initialRecoveryKind?: AttemptRecoveryKind,
|
|
152
159
|
): AsyncGenerator<AdapterEvent> {
|
|
153
160
|
let response: Response | undefined;
|
|
161
|
+
let kiroRefusalPendingReplay: Response | undefined;
|
|
162
|
+
let kiroReplaySetupFailed = false;
|
|
154
163
|
// One-shot recovery label for the next top-of-loop continuation send after a failover rotation.
|
|
155
164
|
let nextContinuationRecoveryKind: AttemptRecoveryKind | undefined = initialRecoveryKind;
|
|
156
165
|
/**
|
|
@@ -198,21 +207,23 @@ export function createAdapterContinuations(
|
|
|
198
207
|
try {
|
|
199
208
|
if (transportState.activeAdapter.fetchResponse) {
|
|
200
209
|
transportState.noteRoutedAttemptSend(continuationEstimate, replayKind);
|
|
201
|
-
await
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
210
|
+
return await withProviderRequestSlot(route.providerName, route.provider, nextParsed.modelId, upstream.signal, pacingSlot =>
|
|
211
|
+
transportState.activeAdapter.fetchResponse!(builtContinuationRequest, {
|
|
212
|
+
kiroPreferAccountFailover: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro"),
|
|
213
|
+
abortSignal: upstream.signal,
|
|
214
|
+
timeoutMs: connectMs,
|
|
205
215
|
sendBudget: adapterDispatchBudget,
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
+
onPhysicalSend: send => noteAdapterPhysicalSend(continuationEstimate, send),
|
|
217
|
+
onRecoveryWithheld: noteAdapterRecoveryWithheld,
|
|
218
|
+
stream: nextParsed.stream,
|
|
219
|
+
executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
|
|
220
|
+
pacingSlotAcquired: true,
|
|
221
|
+
pacingSlot,
|
|
222
|
+
dispatchOverride: oauthDispatch(builtContinuationRequest, nextParsed),
|
|
223
|
+
providerName: route.providerName,
|
|
224
|
+
modelId: nextParsed.modelId,
|
|
225
|
+
}),
|
|
226
|
+
}));
|
|
216
227
|
}
|
|
217
228
|
// Same #1851 scope guard as the initial send: transient-5xx retry only for direct
|
|
218
229
|
// Google AI Studio; every other adapter keeps reset-only semantics here.
|
|
@@ -263,13 +274,26 @@ export function createAdapterContinuations(
|
|
|
263
274
|
const recoveryKind = nextContinuationRecoveryKind;
|
|
264
275
|
nextContinuationRecoveryKind = undefined;
|
|
265
276
|
response = await fetchContinuation(recoveryKind);
|
|
277
|
+
if (kiroRefusalPendingReplay) {
|
|
278
|
+
try { void kiroRefusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
279
|
+
kiroRefusalPendingReplay = undefined;
|
|
280
|
+
sendBudgetState.pendingHopPermit = undefined;
|
|
281
|
+
}
|
|
266
282
|
} catch (error) {
|
|
283
|
+
if (kiroRefusalPendingReplay && !options.abortSignal?.aborted && !upstream.signal.aborted) {
|
|
284
|
+
sendBudgetState.pendingHopPermit?.release();
|
|
285
|
+
sendBudgetState.pendingHopPermit = undefined;
|
|
286
|
+
response = kiroRefusalPendingReplay;
|
|
287
|
+
kiroRefusalPendingReplay = undefined;
|
|
288
|
+
kiroReplaySetupFailed = true;
|
|
289
|
+
} else {
|
|
267
290
|
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
268
291
|
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
269
292
|
} else {
|
|
270
293
|
yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` };
|
|
271
294
|
}
|
|
272
295
|
return;
|
|
296
|
+
}
|
|
273
297
|
}
|
|
274
298
|
|
|
275
299
|
// Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover:
|
|
@@ -277,6 +301,7 @@ export function createAdapterContinuations(
|
|
|
277
301
|
// loop; only after the attempts are exhausted does the continuation fail over.
|
|
278
302
|
while (
|
|
279
303
|
response.status === 429
|
|
304
|
+
&& !kiroReplaySetupFailed
|
|
280
305
|
// A synthesized replay refusal is not a rate limit; replaying the continuation on
|
|
281
306
|
// it would re-send a turn whose first send may already have been processed.
|
|
282
307
|
&& !isNonReplayableResponse(response)
|
|
@@ -402,11 +427,114 @@ export function createAdapterContinuations(
|
|
|
402
427
|
// 429 stayed terminal even with failover fully active -- the same class of divergence the
|
|
403
428
|
// two sidecars already produced once. Request-local state is shared with the other arms so
|
|
404
429
|
// the per-request bound cannot be silently re-armed by reaching a different loop.
|
|
430
|
+
if (route.providerName === "kiro") {
|
|
431
|
+
if (
|
|
432
|
+
(response.status === 429 || response.status === 400 || response.status === 403)
|
|
433
|
+
&& transportState.genericFailoverAccountId
|
|
434
|
+
&& !isNonReplayableResponse(response)
|
|
435
|
+
&& !kiroReplaySetupFailed
|
|
436
|
+
) {
|
|
437
|
+
const refusal = classifyKiroRefusal(response.status,
|
|
438
|
+
await readDisplaySafeErrorText(response.clone(), upstream.signal, ""));
|
|
439
|
+
if (refusal.kind === "other") break;
|
|
440
|
+
const sent = transportState.replayOAuthCredentialSnapshot;
|
|
441
|
+
const monthlyCooldownMs = refusal.kind === "monthly_quota" && sent
|
|
442
|
+
? noteKiroMonthlyRefusal(sent.accountId, sent.generation, Date.now()) : undefined;
|
|
443
|
+
if (monthlyCooldownMs !== undefined) persistKiroAccountState();
|
|
444
|
+
if (refusal.kind === "suspended")
|
|
445
|
+
quarantineKiroSuspendedAccount(transportState.genericFailoverAccountId, sent?.generation);
|
|
446
|
+
if (transportState.genericFailovers >= transportState.genericFailoverLimit
|
|
447
|
+
|| !isGenericOAuthFailoverEnabled(config, "kiro")) break;
|
|
448
|
+
// Intersection with the shared request budget. The continuation loop re-sends the
|
|
449
|
+
// turn, so without this the per-request bound could be re-armed simply by reaching a
|
|
450
|
+
// different loop -- which is the divergence the comment above already warns about.
|
|
451
|
+
//
|
|
452
|
+
// Who settles the reservation depends on who sends the replay (#4709). An adapter that
|
|
453
|
+
// owns its ladder reserves once per physical send and would charge this replay twice;
|
|
454
|
+
// the helper path reports it back instead, which is what `countedExternally` names.
|
|
455
|
+
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
456
|
+
const hop = reserveCredentialHop(
|
|
457
|
+
"auth-recovery",
|
|
458
|
+
`${route.providerName}|${route.modelId}|continuation-oauth-429`,
|
|
459
|
+
!adapterOwnsDispatch && transientRetryPolicyFor(route.provider) !== null,
|
|
460
|
+
);
|
|
461
|
+
const nextAccountId = hop.allowed
|
|
462
|
+
? rotateGenericOAuthAccountOnRefusal(
|
|
463
|
+
config,
|
|
464
|
+
route.providerName,
|
|
465
|
+
transportState.genericFailoverAccountId,
|
|
466
|
+
refusal.kind,
|
|
467
|
+
response.headers.get("retry-after"),
|
|
468
|
+
Date.now(),
|
|
469
|
+
route.modelId,
|
|
470
|
+
monthlyCooldownMs,
|
|
471
|
+
)
|
|
472
|
+
: null;
|
|
473
|
+
// A roster quorum ignores cooldowns, so only attribute a budget refusal when the
|
|
474
|
+
// non-mutating selector confirms that an alternate account could serve this model now.
|
|
475
|
+
if (!hop.allowed && hasEligibleGenericOAuthFailoverTarget(
|
|
476
|
+
route.providerName, transportState.genericFailoverAccountId, Date.now(), route.modelId,
|
|
477
|
+
)) noteAttemptRecoveryWithheld(logCtx.activeAttempt, "rotation-send-budget");
|
|
478
|
+
if (!nextAccountId) hop.permit?.release();
|
|
479
|
+
if (nextAccountId) {
|
|
480
|
+
try {
|
|
481
|
+
// The FULL snapshot through the shared helper, never a bare bearer: Antigravity
|
|
482
|
+
// pairs an account-matched projectId with its token and Kiro carries routing
|
|
483
|
+
// metadata, so a token-only swap would mix one account's credential with another's
|
|
484
|
+
// routing data.
|
|
485
|
+
const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId);
|
|
486
|
+
const applied = await applyFailoverSnapshot(snapshot, nextParsed);
|
|
487
|
+
if (!applied) hop.permit?.release();
|
|
488
|
+
if (applied) {
|
|
489
|
+
transportState.genericFailovers += 1;
|
|
490
|
+
invalidateSameTargetRequest();
|
|
491
|
+
transportState.activeAdapter = resolveSelectionAdapter(
|
|
492
|
+
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
|
|
493
|
+
config.cacheRetention,
|
|
494
|
+
);
|
|
495
|
+
bindRouteReasoningReplayScope({
|
|
496
|
+
parsed: nextParsed,
|
|
497
|
+
providerName: route.providerName,
|
|
498
|
+
provider: route.provider,
|
|
499
|
+
adapterName: transportState.activeAdapter.name,
|
|
500
|
+
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
|
|
501
|
+
});
|
|
502
|
+
// Response persistence closes over the outer parsed request; keep its owner binding in
|
|
503
|
+
// sync with the terminal-guard clone that builds the rotated continuation request.
|
|
504
|
+
bindRouteReasoningReplayScope({
|
|
505
|
+
parsed,
|
|
506
|
+
providerName: route.providerName,
|
|
507
|
+
provider: route.provider,
|
|
508
|
+
adapterName: transportState.activeAdapter.name,
|
|
509
|
+
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
|
|
510
|
+
});
|
|
511
|
+
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
|
|
512
|
+
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
513
|
+
// The replay goes out on the next iteration. An adapter that owns its ladder
|
|
514
|
+
// reserves for that send itself, so hand this reservation down rather than let it
|
|
515
|
+
// take a second one for the same replay. A helper-routed replay needs no handoff:
|
|
516
|
+
// its reporter settles the booking made above.
|
|
517
|
+
if (adapterOwnsDispatch) sendBudgetState.pendingHopPermit = hop.permit;
|
|
518
|
+
nextContinuationRecoveryKind = "oauth-account-429";
|
|
519
|
+
kiroRefusalPendingReplay = response;
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
522
|
+
} catch {
|
|
523
|
+
// Everything in this try runs before the replay: the send happens on the next
|
|
524
|
+
// iteration, after `continue`. A throw here therefore leaves a reservation that
|
|
525
|
+
// never dispatched, and holding it would refuse a later recovery in this same
|
|
526
|
+
// request for a send that never left the process.
|
|
527
|
+
hop.permit?.release();
|
|
528
|
+
// fall through to emit continuation error below
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
} else {
|
|
405
533
|
if (
|
|
406
534
|
response.status === 429
|
|
407
535
|
&& transportState.genericFailoverAccountId
|
|
408
536
|
&& !isNonReplayableResponse(response)
|
|
409
|
-
&& transportState.genericFailovers <
|
|
537
|
+
&& transportState.genericFailovers < transportState.genericFailoverLimit
|
|
410
538
|
&& isGenericOAuthFailoverEnabled(config, route.providerName)
|
|
411
539
|
) {
|
|
412
540
|
// Intersection with the shared request budget. The continuation loop re-sends the
|
|
@@ -491,6 +619,7 @@ export function createAdapterContinuations(
|
|
|
491
619
|
}
|
|
492
620
|
}
|
|
493
621
|
}
|
|
622
|
+
}
|
|
494
623
|
if (shouldAttemptImageTierRetry({
|
|
495
624
|
status: response.status,
|
|
496
625
|
adapterName: transportState.activeAdapter.name,
|
|
@@ -507,6 +636,7 @@ export function createAdapterContinuations(
|
|
|
507
636
|
}
|
|
508
637
|
|
|
509
638
|
if (!response.ok) {
|
|
639
|
+
if (route.providerName === "kiro") response = await normalizeFinalKiroHttpError(response, upstream.signal);
|
|
510
640
|
const errorText = await readDisplaySafeErrorText(response, upstream.signal, "unknown error");
|
|
511
641
|
const normalized = normalizeUpstreamErrorText(errorText, "unknown error");
|
|
512
642
|
yield {
|