@bitkyc08/opencodex 2.88.0 → 2.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/bin/ocx.mjs +15 -4
- package/gui/dist/assets/App-BnRSujOq.js +58 -0
- package/gui/dist/assets/App-dnkJ85te.css +1 -0
- package/gui/dist/assets/{Tray-B5bC7AWL.js → Tray-7z1COnmz.js} +1 -1
- package/gui/dist/assets/{index-B5bsvxor.js → index-uchD-ngf.js} +13 -13
- package/gui/dist/assets/{quota-summary-D8y2wTsB.js → quota-summary-BY_10IEz.js} +1 -1
- package/gui/dist/index.html +1 -1
- package/package.json +1 -1
- package/src/adapters/anthropic/cache-retention.ts +117 -0
- package/src/adapters/anthropic/passthrough.ts +7 -4
- package/src/adapters/anthropic.ts +10 -3
- package/src/adapters/coding-agent/protocol.ts +6 -0
- package/src/adapters/google-http.ts +3 -2
- package/src/adapters/google-vertex-endpoint.ts +67 -0
- package/src/adapters/google.ts +35 -29
- package/src/adapters/openai-responses/compaction-search-history.ts +27 -0
- package/src/adapters/openai-responses/passthrough.ts +17 -12
- package/src/adapters/openai-responses/reasoning.ts +68 -1
- package/src/adapters/registry.ts +1 -1
- package/src/adapters/upstream-http-error.ts +3 -2
- package/src/adapters/zed.ts +7 -4
- package/src/bridge/internal.ts +11 -1
- package/src/bridge/response-json.ts +2 -2
- package/src/bridge/sse.ts +10 -4
- package/src/claude/inbound.ts +12 -26
- package/src/claude/outbound.ts +5 -0
- package/src/cli/account-api.ts +13 -1
- package/src/cli/account-extended.ts +17 -5
- package/src/cli/account-key-quota.ts +4 -1
- package/src/cli/account.ts +1 -1
- package/src/cli/capabilities-agents-routing.ts +45 -8
- package/src/cli/capabilities-base.ts +5 -5
- package/src/cli/capabilities-provider-models.ts +5 -4
- package/src/cli/combo-result.ts +8 -2
- package/src/cli/combo.ts +17 -0
- package/src/cli/help.ts +1 -1
- package/src/cli/message-command.ts +4 -1
- package/src/cli/message-remote-args.ts +69 -0
- package/src/cli/message-remote-command.ts +77 -0
- package/src/cli/provider-runtime.ts +69 -1
- package/src/cli/registry.ts +3 -3
- package/src/codex/account-pause.ts +6 -1
- package/src/codex/account-runtime-state.ts +5 -0
- package/src/codex/account-store.ts +30 -3
- package/src/codex/auth-api/account-list.ts +3 -0
- package/src/codex/auth-api/main-account-probe.ts +3 -2
- package/src/codex/auth-context.ts +9 -4
- package/src/codex/catalog/aggregation.ts +15 -0
- package/src/codex/catalog/auto-review.ts +2 -1
- package/src/codex/catalog/build-entries.ts +17 -7
- package/src/codex/catalog/derive-entry.ts +3 -1
- package/src/codex/catalog/metadata.ts +2 -0
- package/src/codex/catalog/model-visibility.ts +0 -2
- package/src/codex/catalog/ownership.ts +71 -0
- package/src/codex/catalog/parsing.ts +12 -3
- package/src/codex/catalog/retained-sync.ts +2 -1
- package/src/codex/catalog/routed-removal.ts +4 -4
- package/src/codex/convergence.ts +2 -1
- package/src/codex/low-quota-desktop-notifier.ts +125 -0
- package/src/codex/low-quota-protection.ts +57 -31
- package/src/codex/main-account-cache.ts +77 -3
- package/src/codex/main-account.ts +6 -2
- package/src/codex/native-residue.ts +2 -3
- package/src/codex/quota.ts +26 -2
- package/src/combos/failover.ts +2 -1
- package/src/combos/jev-model-backend.ts +30 -7
- package/src/combos/jev-model-request.ts +20 -0
- package/src/combos/jev-quota-activation.ts +153 -0
- package/src/combos/jev-quota-config.ts +62 -0
- package/src/combos/jev-quota-route.ts +26 -0
- package/src/combos/jev-quota.ts +88 -0
- package/src/combos/jev.ts +25 -12
- package/src/combos/types.ts +27 -0
- package/src/compatibility/index.ts +3 -0
- package/src/compatibility/openai-responses.ts +8 -1
- package/src/compatibility/opengateway-responses.ts +61 -0
- package/src/config/account-auto-pause-revisions.ts +136 -0
- package/src/config/diagnostics.ts +13 -0
- package/src/config/live-reconcile.ts +9 -0
- package/src/config/persist-unlocked.ts +10 -1
- package/src/config/removed-opencode-free.ts +135 -0
- package/src/config/schema/account-auto-pause.ts +14 -0
- package/src/config/schema/config-schema.ts +15 -0
- package/src/config/schema/leaf-validators.ts +7 -0
- package/src/config.ts +10 -3
- package/src/generated/compatibility-version.json +344 -144
- package/src/images/loop.ts +32 -9
- package/src/lib/account-auto-pause-response.ts +16 -0
- package/src/lib/bun-runtime.ts +18 -1
- package/src/lib/bun-transpiler-cache-scope.d.mts +8 -0
- package/src/lib/bun-transpiler-cache-scope.mjs +68 -0
- package/src/lib/decision-quota-sink.ts +76 -0
- package/src/lib/state-store-registrations.ts +25 -0
- package/src/link/ssh-argv.ts +3 -0
- package/src/messaging/envelope.ts +6 -3
- package/src/messaging/remote-auth.ts +83 -0
- package/src/messaging/remote-bridge.ts +270 -0
- package/src/messaging/remote-contract.ts +108 -0
- package/src/messaging/remote-enrollment-recovery.ts +59 -0
- package/src/messaging/remote-enrollment.ts +244 -0
- package/src/messaging/remote-envelope.ts +40 -0
- package/src/messaging/remote-files.ts +117 -0
- package/src/messaging/remote-owner.ts +169 -0
- package/src/messaging/remote-ports.ts +56 -0
- package/src/messaging/remote-process.ts +131 -0
- package/src/messaging/remote-replies.ts +98 -0
- package/src/messaging/remote-rpc-admission.ts +32 -0
- package/src/messaging/remote-send.ts +72 -0
- package/src/messaging/remote-store.ts +111 -0
- package/src/messaging/remote-tunnels.ts +65 -0
- package/src/messaging/rpc.ts +13 -1
- package/src/oauth/account-auto-pause-policy.ts +66 -0
- package/src/oauth/account-auto-pause.ts +171 -0
- package/src/oauth/account-quota-rank.ts +2 -1
- package/src/oauth/anthropic-account-refusal.ts +6 -0
- package/src/oauth/anthropic-model-family.ts +8 -0
- package/src/oauth/anthropic-model-quota.ts +2 -6
- package/src/oauth/api-key-pool-preserve.ts +37 -0
- package/src/oauth/index.ts +5 -35
- package/src/oauth/store.ts +35 -8
- package/src/oauth/types.ts +2 -0
- package/src/plugins/upstream-hooks.ts +14 -1
- package/src/providers/api-key-pause.ts +32 -0
- package/src/providers/api-key-resolve.ts +10 -4
- package/src/providers/api-key-selection.ts +6 -1
- package/src/providers/api-keys.ts +8 -4
- package/src/providers/claude-cli-usage.ts +5 -0
- package/src/providers/key-failover.ts +11 -11
- package/src/providers/model-rename-fields.ts +1 -0
- package/src/providers/opencode-zen-rate-limit.ts +6 -72
- package/src/providers/quota/account-cache.ts +15 -1
- package/src/providers/quota/anthropic-account-quota.ts +2 -1
- package/src/providers/quota/anthropic-cooldown-recovery.ts +22 -4
- package/src/providers/quota/anthropic-decision-windows.ts +47 -0
- package/src/providers/quota/antigravity.ts +39 -6
- package/src/providers/quota/report-cache.ts +14 -0
- package/src/providers/quota/vendor-probes-oauth.ts +3 -3
- package/src/providers/quota-decision-publication.ts +77 -0
- package/src/providers/quota-decision-snapshot.ts +145 -0
- package/src/providers/quota-routing-cache.ts +3 -0
- package/src/providers/quota-types.ts +6 -1
- package/src/providers/quota.ts +3 -2
- package/src/providers/registry/entries-extended.ts +8 -52
- package/src/providers/registry/model-seeds.ts +1 -1
- package/src/providers/registry.ts +1 -2
- package/src/providers/stale-vision-classification-migration.ts +1 -1
- package/src/responses/parser.ts +18 -10
- package/src/responses/web-search-replay.ts +138 -0
- package/src/routing/compatibility/behavior.ts +4 -1
- package/src/server/auth-cors.ts +7 -0
- package/src/server/background-lifecycle.ts +2 -1
- package/src/server/chat-native.ts +28 -13
- package/src/server/claude-messages.ts +2 -1
- package/src/server/index/optional-listeners.ts +6 -0
- package/src/server/management/combo-routes.ts +17 -2
- package/src/server/management/oauth-account-routes.ts +10 -1
- package/src/server/management/provider-completion-test.ts +289 -0
- package/src/server/management/provider-routes.ts +9 -1
- package/src/server/management/route-registry.ts +2 -0
- package/src/server/messages-native.ts +18 -4
- package/src/server/messages-shared.ts +4 -1
- package/src/server/request-log.ts +7 -2
- package/src/server/responses/account-auto-pause-recovery.ts +26 -0
- package/src/server/responses/adapter-dispatch.ts +77 -28
- package/src/server/responses/combo-session-recall.ts +25 -0
- package/src/server/responses/combo-stream-preflight.ts +3 -1
- package/src/server/responses/combo-tool-loop-pin.ts +70 -0
- package/src/server/responses/compact.ts +8 -5
- package/src/server/responses/context-overflow.ts +141 -2
- package/src/server/responses/core-codex-account.ts +7 -1
- package/src/server/responses/core-combo.ts +31 -5
- package/src/server/responses/core.ts +12 -9
- package/src/server/responses/empty-completion-guard.ts +2 -0
- package/src/server/responses/fetch-helpers.ts +35 -4
- package/src/server/responses/jev-model-invoke.ts +13 -14
- package/src/server/responses/native-steering-log.ts +2 -2
- package/src/server/responses/passthrough-delivery.ts +3 -2
- package/src/server/responses/passthrough-dispatch.ts +30 -33
- package/src/server/responses/passthrough-observations.ts +38 -0
- package/src/server/responses/request-compression.ts +98 -0
- package/src/server/responses/request-transport.ts +32 -3
- package/src/server/responses/run-turn-execution.ts +2 -0
- package/src/server/responses/sidecar-execution.ts +16 -3
- package/src/server/responses/terminal-guard.ts +2 -0
- package/src/server/startup-action-control.ts +2 -2
- package/src/server/startup-health-cache.ts +2 -2
- package/src/server/windows-tray-control.ts +2 -2
- package/src/service/definition.ts +9 -1
- package/src/types/account-auto-pause.ts +15 -0
- package/src/types/config.ts +23 -1
- package/src/types/provider.ts +9 -5
- package/src/types/request.ts +20 -2
- package/src/types.ts +1 -0
- package/src/usage/cost.ts +17 -1
- package/src/usage/expected-prices.ts +3 -1
- package/src/usage/jev-stats.ts +4 -0
- package/src/usage/log.ts +2 -0
- package/src/usage/summary.ts +7 -0
- package/src/web-search/loop.ts +37 -11
- package/src/web-search/progress-stream.ts +4 -1
- package/src/web-search/run-turn-loop.ts +9 -3
- package/src/web-search/sidecar-search-replay.ts +96 -0
- package/gui/dist/assets/App-CHC89EXQ.js +0 -58
- package/gui/dist/assets/App-wT3mXqFz.css +0 -1
|
@@ -364,6 +364,7 @@ export const MANAGEMENT_ROUTES: readonly ManagementRoute[] = [
|
|
|
364
364
|
{ method: "PUT", path: "/api/oauth/accounts/pool", module: "server/management/oauth-account-routes", mutates: true },
|
|
365
365
|
{ method: "PUT", path: "/api/providers/keys/active", module: "server/management/oauth-account-routes", mutates: true },
|
|
366
366
|
{ method: "PUT", path: "/api/providers/keys/alias", module: "server/management/oauth-account-routes", mutates: true },
|
|
367
|
+
{ method: "PUT", path: "/api/providers/keys/pause", module: "server/management/oauth-account-routes", mutates: true },
|
|
367
368
|
// server/management/provider-routes
|
|
368
369
|
{ method: "DELETE", path: "/api/providers", module: "server/management/provider-routes", mutates: true },
|
|
369
370
|
{ method: "GET", path: "/api/provider-context-caps", module: "server/management/provider-routes", mutates: false },
|
|
@@ -373,6 +374,7 @@ export const MANAGEMENT_ROUTES: readonly ManagementRoute[] = [
|
|
|
373
374
|
{ method: "GET", path: "/api/providers", module: "server/management/provider-routes", mutates: false },
|
|
374
375
|
{ method: "PATCH", path: "/api/providers", module: "server/management/provider-routes", mutates: true },
|
|
375
376
|
{ method: "POST", path: "/api/providers", module: "server/management/provider-routes", mutates: true },
|
|
377
|
+
{ method: "POST", path: "/api/providers/test-completion", module: "server/management/provider-routes", mutates: true },
|
|
376
378
|
{ method: "POST", path: "/api/providers/test", module: "server/management/provider-routes", mutates: true },
|
|
377
379
|
{ method: "PUT", path: "/api/providers", module: "server/management/provider-routes", mutates: true },
|
|
378
380
|
{ method: "PUT", path: "/api/provider-context-caps", module: "server/management/provider-routes", mutates: true },
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { captureAccountAutoPauseAttempt, wasAccountAutoPaused } from "../oauth/account-auto-pause";
|
|
1
2
|
import { createPhysicalSendReporter } from "../lib/request-execution-budget";
|
|
2
3
|
import { createInferenceSendBudget } from "./inference/context";
|
|
3
4
|
/**
|
|
@@ -120,7 +121,7 @@ import {
|
|
|
120
121
|
recordKeyWireAttemptUsage,
|
|
121
122
|
type RequestLogContext,
|
|
122
123
|
} from "./request-log";
|
|
123
|
-
import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, sendWithConnectionPolicy } from "./responses/fetch-helpers";
|
|
124
|
+
import { fetchWithHeaderTimeout, physicalSendObserverInit, providerFetch, safeHostLabel, sendWithConnectionPolicy } from "./responses/fetch-helpers";
|
|
124
125
|
import { anthropicBillingPhysicalFetch } from "../providers/anthropic-billing-ownership";
|
|
125
126
|
import { rotateAnthropicBillingKeyOnResponse } from "../providers/anthropic-billing-recovery";
|
|
126
127
|
import { linkAbortSignal } from "./responses/core-lifetime";
|
|
@@ -545,6 +546,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
545
546
|
pendingDistributedFailure = undefined;
|
|
546
547
|
record?.();
|
|
547
548
|
};
|
|
549
|
+
/** Send the native Messages request under the transient retry policy, re-validating the selected OAuth account or API key at the dispatch boundary. */
|
|
548
550
|
const send = async (recovery?: "rate-limit-429" | "oauth-401" | "oauth-account-403" | "key-429" | "key-401"): Promise<Response> => {
|
|
549
551
|
const remaining = remainingTransientSends();
|
|
550
552
|
if (requestTransientPolicy && remaining <= 0) {
|
|
@@ -616,6 +618,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
616
618
|
}
|
|
617
619
|
const releaseFamily = snapshot && oauthBinding ? anthropicModelQuotaFor(oauthBinding.instance).claimAnthropicFamilyRevalidation(snapshot.accountId, route.modelId) : () => {};
|
|
618
620
|
if (!releaseFamily) throw new AnthropicAccountCooldownError(1);
|
|
621
|
+
const observeAutoPause = captureAccountAutoPauseAttempt(route.providerName, activeProvider, { config, oauth: sendingBinding?.snapshot }, headers);
|
|
619
622
|
const selection = activeProvider._apiKeyAttempt;
|
|
620
623
|
let dispatched: Response;
|
|
621
624
|
const spendReport = createPhysicalSendReporter(physicalBudget, () => ({
|
|
@@ -641,7 +644,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
641
644
|
anthropicBillingPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, route.providerName, activeProvider),
|
|
642
645
|
route.providerName, activeProvider, wire.url, activeProvider.apiKeyTransport === "bearer" ? "authorization" : "x-api-key"),
|
|
643
646
|
wire.url,
|
|
644
|
-
applyUpstreamRecoveryInit({ ...init, method: "POST", headers, body: wire.body }, transportRecovery),
|
|
647
|
+
applyUpstreamRecoveryInit({ ...init, method: "POST", headers, body: wire.body, ...physicalSendObserverInit(observeAutoPause, wire.url) }, transportRecovery),
|
|
645
648
|
{ providerName: route.providerName, provider: activeProvider },
|
|
646
649
|
);
|
|
647
650
|
} finally {
|
|
@@ -659,7 +662,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
659
662
|
&& credentialGeneration(current.credential) === snapshot.generation
|
|
660
663
|
&& current.credential.accountId === providerAccountUuid) {
|
|
661
664
|
bindAnthropicRefusalCredentialForSend(dispatched, sendOwner, providerAccountUuid);
|
|
662
|
-
recordAnthropicAccountQuotaFromHeadersForInstance(nativeInstance!, snapshot.accountId, dispatched.headers, writerGeneration, dispatched.status, route.modelId);
|
|
665
|
+
recordAnthropicAccountQuotaFromHeadersForInstance(nativeInstance!, snapshot.accountId, dispatched.headers, writerGeneration, dispatched.status, route.modelId, snapshot.generation);
|
|
663
666
|
}
|
|
664
667
|
} catch { /* Passive observation must not fail the response. */ }
|
|
665
668
|
}
|
|
@@ -672,7 +675,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
672
675
|
? () => { recordDistributedResponseFailure(dispatched, config, route.providerName, req.signal); }
|
|
673
676
|
: undefined;
|
|
674
677
|
if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal);
|
|
675
|
-
return dispatched;
|
|
678
|
+
return observeAutoPause ? observeAutoPause(dispatched) : dispatched;
|
|
676
679
|
},
|
|
677
680
|
}),
|
|
678
681
|
),
|
|
@@ -700,9 +703,19 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
700
703
|
let billingRecoveryActive = false;
|
|
701
704
|
const keySendAvailable = () => transientSendAvailable()
|
|
702
705
|
&& (!billingRecoveryActive || Math.max(physicalSendsUsed, physicalBudget.used) < physicalBudget.policy.maxTotalModelSends);
|
|
706
|
+
let autoPauseMoves = 0;
|
|
703
707
|
const recoveryPasses = activeProvider._apiKeyAttempt?.distributed ? activeProvider.apiKeyPool?.length ?? 1 : 1;
|
|
704
708
|
for (let pass = 0; ; pass++) {
|
|
705
709
|
const priorResponse = response;
|
|
710
|
+
for (; wasAccountAutoPaused(response) && !oauthBinding && config.providers[route.providerName]?.apiKey && autoPauseMoves < 3 && keySendAvailable(); autoPauseMoves++) {
|
|
711
|
+
const fresh = resolveCurrentProviderApiKeyTransport(config, route.providerName, { ...activeProvider, _apiKeyAttempt: undefined });
|
|
712
|
+
const next = fresh && (selectProactiveApiKeyTransport(config, route.providerName, fresh) ?? fresh);
|
|
713
|
+
if (!next) break;
|
|
714
|
+
try { void response.body?.cancel().catch(() => {}); } catch { /* closed */ }
|
|
715
|
+
rebuildFor(next);
|
|
716
|
+
response = await send("key-401");
|
|
717
|
+
}
|
|
718
|
+
|
|
706
719
|
// A credential-scoped 401 on a static key pool says nothing about the sibling keys.
|
|
707
720
|
while (response.status === 401 && distributedResponseOwnsKey && hasKeyPoolFailover(activeProvider) && keySendAvailable()) {
|
|
708
721
|
const rotated = rotateProviderTransportOn401(config, route.providerName, activeProvider, {
|
|
@@ -762,6 +775,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
|
|
|
762
775
|
if (response === priorResponse || isNonReplayableResponse(response) || !keySendAvailable()) break;
|
|
763
776
|
// Billing class transitions share the physical-send ceiling; distributed-only
|
|
764
777
|
// transitions additionally retain their original bounded pool-pass allowance.
|
|
778
|
+
if (wasAccountAutoPaused(response) && autoPauseMoves < 3) continue;
|
|
765
779
|
if (distributedResponseOwnsKey && (
|
|
766
780
|
(billingRecoveryActive && [401, 429].includes(response.status))
|
|
767
781
|
|| (activeProvider.adapter === "anthropic" && hasKeyPoolFailover(activeProvider) && [400, 402].includes(response.status))
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* other.
|
|
14
14
|
*/
|
|
15
15
|
import { sniffImageDimensions } from "../adapters/anthropic-image-guard";
|
|
16
|
+
import { anthropicCacheWrite1hTokens } from "../adapters/anthropic/cache-retention";
|
|
16
17
|
import { createToolCallIdAllocator } from "../adapters/tool-call-id";
|
|
17
18
|
import { idleDeadline } from "../lib/abort";
|
|
18
19
|
import {
|
|
@@ -29,12 +30,13 @@ import type { RequestLogContext } from "./request-log";
|
|
|
29
30
|
import { recordGenerationEvent } from "./request-log-generation-window";
|
|
30
31
|
import type { OcxConfig } from "../types";
|
|
31
32
|
|
|
32
|
-
export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: number; outputTokens: number; cachedInputTokens?: number; cacheReadInputTokens?: number; cacheCreationInputTokens?: number } | undefined {
|
|
33
|
+
export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: number; outputTokens: number; cachedInputTokens?: number; cacheReadInputTokens?: number; cacheCreationInputTokens?: number; cacheCreation1hInputTokens?: number } | undefined {
|
|
33
34
|
if (!usage) return undefined;
|
|
34
35
|
const num = (v: unknown) => typeof v === "number" ? v : 0;
|
|
35
36
|
const hasCache = usage.cache_read_input_tokens !== undefined || usage.cache_creation_input_tokens !== undefined;
|
|
36
37
|
const read = num(usage.cache_read_input_tokens);
|
|
37
38
|
const write = num(usage.cache_creation_input_tokens);
|
|
39
|
+
const write1h = anthropicCacheWrite1hTokens(usage, write);
|
|
38
40
|
// Anthropic input_tokens excludes cache read/write; normalize to the canonical
|
|
39
41
|
// inclusive convention (types.ts OcxUsage / devlog 070). cached = READS only.
|
|
40
42
|
return {
|
|
@@ -44,6 +46,7 @@ export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: numb
|
|
|
44
46
|
cachedInputTokens: read,
|
|
45
47
|
cacheReadInputTokens: read,
|
|
46
48
|
cacheCreationInputTokens: write,
|
|
49
|
+
...(write1h !== undefined ? { cacheCreation1hInputTokens: write1h } : {}),
|
|
47
50
|
} : {}),
|
|
48
51
|
};
|
|
49
52
|
}
|
|
@@ -982,7 +982,7 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
|
|
|
982
982
|
const raw = usage as {
|
|
983
983
|
input_tokens?: unknown;
|
|
984
984
|
output_tokens?: unknown;
|
|
985
|
-
input_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown };
|
|
985
|
+
input_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown; cache_write_1h_tokens?: unknown };
|
|
986
986
|
output_tokens_details?: { reasoning_tokens?: unknown };
|
|
987
987
|
total_tokens?: unknown;
|
|
988
988
|
prompt_tokens?: unknown;
|
|
@@ -1004,6 +1004,9 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
|
|
|
1004
1004
|
...(typeof raw.input_tokens_details?.cache_write_tokens === "number"
|
|
1005
1005
|
? { cacheCreationInputTokens: raw.input_tokens_details.cache_write_tokens }
|
|
1006
1006
|
: {}),
|
|
1007
|
+
...(typeof raw.input_tokens_details?.cache_write_1h_tokens === "number"
|
|
1008
|
+
? { cacheCreation1hInputTokens: raw.input_tokens_details.cache_write_1h_tokens }
|
|
1009
|
+
: {}),
|
|
1007
1010
|
...(typeof raw.output_tokens_details?.reasoning_tokens === "number"
|
|
1008
1011
|
? { reasoningOutputTokens: raw.output_tokens_details.reasoning_tokens }
|
|
1009
1012
|
: {}),
|
|
@@ -1864,7 +1867,7 @@ export function aggregateAttemptUsage(
|
|
|
1864
1867
|
if (usages.length === 0) return { status };
|
|
1865
1868
|
|
|
1866
1869
|
const sumOptional = (
|
|
1867
|
-
key: "cachedInputTokens" | "cacheReadInputTokens" | "cacheCreationInputTokens"
|
|
1870
|
+
key: "cachedInputTokens" | "cacheReadInputTokens" | "cacheCreationInputTokens" | "cacheCreation1hInputTokens"
|
|
1868
1871
|
| "reasoningOutputTokens" | "providerCredits",
|
|
1869
1872
|
): number | undefined => {
|
|
1870
1873
|
const present = usages.flatMap(usage => (
|
|
@@ -1875,6 +1878,7 @@ export function aggregateAttemptUsage(
|
|
|
1875
1878
|
const cachedInputTokens = sumOptional("cachedInputTokens");
|
|
1876
1879
|
const cacheReadInputTokens = sumOptional("cacheReadInputTokens");
|
|
1877
1880
|
const cacheCreationInputTokens = sumOptional("cacheCreationInputTokens");
|
|
1881
|
+
const cacheCreation1hInputTokens = sumOptional("cacheCreation1hInputTokens");
|
|
1878
1882
|
const reasoningOutputTokens = sumOptional("reasoningOutputTokens");
|
|
1879
1883
|
const providerCredits = sumOptional("providerCredits");
|
|
1880
1884
|
const totalTokens = usages.reduce(
|
|
@@ -1888,6 +1892,7 @@ export function aggregateAttemptUsage(
|
|
|
1888
1892
|
...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
|
|
1889
1893
|
...(cacheReadInputTokens !== undefined ? { cacheReadInputTokens } : {}),
|
|
1890
1894
|
...(cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens } : {}),
|
|
1895
|
+
...(cacheCreation1hInputTokens !== undefined ? { cacheCreation1hInputTokens } : {}),
|
|
1891
1896
|
...(reasoningOutputTokens !== undefined ? { reasoningOutputTokens } : {}),
|
|
1892
1897
|
...(providerCredits !== undefined ? { providerCredits } : {}),
|
|
1893
1898
|
...(status === "estimated" ? { estimated: true } : {}),
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ResponsesSendBudget } from "./request-send-budget";
|
|
2
|
+
|
|
3
|
+
const moves = new WeakMap<object, number>();
|
|
4
|
+
/** One same-provider credential move, funded by the existing recovery owner. */
|
|
5
|
+
export async function recoverAutoPausedAccount<T>(options: {
|
|
6
|
+
requestKey: object;
|
|
7
|
+
remaining?: number;
|
|
8
|
+
reserve: ResponsesSendBudget["reserveCredentialHop"];
|
|
9
|
+
refresh: () => Promise<boolean>;
|
|
10
|
+
refetch: () => Promise<T>;
|
|
11
|
+
setPermit: (permit: ResponsesSendBudget["pendingHopPermit"]) => void;
|
|
12
|
+
}): Promise<T | undefined> {
|
|
13
|
+
const count = moves.get(options.requestKey) ?? 0;
|
|
14
|
+
if (count >= 3 || options.remaining !== undefined && options.remaining <= 0) return;
|
|
15
|
+
const hop = options.reserve("auth-recovery", "account-auto-pause", true);
|
|
16
|
+
if (!hop.allowed) return;
|
|
17
|
+
try {
|
|
18
|
+
if (!await options.refresh()) return;
|
|
19
|
+
moves.set(options.requestKey, count + 1);
|
|
20
|
+
options.setPermit(hop.permit);
|
|
21
|
+
return await options.refetch();
|
|
22
|
+
} finally {
|
|
23
|
+
options.setPermit(undefined);
|
|
24
|
+
hop.permit?.release();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { recoverAutoPausedAccount } from "./account-auto-pause-recovery";
|
|
2
|
+
import { wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause";
|
|
1
3
|
import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo";
|
|
2
4
|
import { deferPhysicalSendReporter } from "./deferred-send-reporter";
|
|
5
|
+
import { usesApiKeyAccount } from "../../providers/label";
|
|
3
6
|
import { spendLedgerStorageErrorResponse } from "./spend-storage-error";
|
|
4
7
|
import { classifyAnthropic429 } from "../../oauth/anthropic-rate-limit-policy";
|
|
5
8
|
import { rotateAnthropicAccountOnResponseForInstance } from "../../oauth/anthropic-account-refusal";
|
|
@@ -141,6 +144,7 @@ export async function prepareAdapterExchange(
|
|
|
141
144
|
| "sameTargetToken"
|
|
142
145
|
| "transportToken"
|
|
143
146
|
| "oauthDispatch"
|
|
147
|
+
| "recoverAutoPausedSelection"
|
|
144
148
|
| "imageTierBias"
|
|
145
149
|
| "oauth401ReplayAttempted"
|
|
146
150
|
| "isOAuth401ReplayProvider"
|
|
@@ -394,11 +398,12 @@ export async function prepareAdapterExchange(
|
|
|
394
398
|
upstreamResponse = new Response(text, { status: original.status, statusText: original.statusText, headers });
|
|
395
399
|
return upstreamResponse;
|
|
396
400
|
};
|
|
401
|
+
const autoPauseEnabled = config.providers[route.providerName]?.accountAutoPause?.enabled === true;
|
|
397
402
|
const billingPoolRequest = isAnthropicBillingKeyPool(route.provider);
|
|
398
403
|
let billingRecoveryActive = false;
|
|
399
|
-
const genericRefetchCountsSends = () => sendBudgetState.adapterSendBudget?.spendEnforced === true
|
|
404
|
+
const genericRefetchCountsSends = () => options.comboAttempt === true || sendBudgetState.adapterSendBudget?.spendEnforced === true
|
|
400
405
|
|| transientRetryPolicyFor(route.provider) !== null || resetReplayPolicyFor(route.provider) !== null
|
|
401
|
-
|| billingPoolRequest || billingRecoveryActive;
|
|
406
|
+
|| billingPoolRequest || billingRecoveryActive || autoPauseEnabled;
|
|
402
407
|
let handoffPermit: ResponsesSendBudget["pendingHopPermit"];
|
|
403
408
|
try {
|
|
404
409
|
if (transportState.activeAdapter.fetchResponse) {
|
|
@@ -520,10 +525,10 @@ export async function prepareAdapterExchange(
|
|
|
520
525
|
label: safeHostLabel(builtInitialRequest.url),
|
|
521
526
|
claimAmbiguousResend: claimPreHeaderResend,
|
|
522
527
|
...(sendBudgetState.adapterSendBudget?.spendEnforced || options.comboDispatchPermit || transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt
|
|
523
|
-
|| billingPoolRequest
|
|
528
|
+
|| billingPoolRequest || autoPauseEnabled
|
|
524
529
|
? {
|
|
525
530
|
// A pending compaction permit already booked this leg's first physical send.
|
|
526
|
-
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt
|
|
531
|
+
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt || autoPauseEnabled
|
|
527
532
|
? {
|
|
528
533
|
attempts: Math.min(initialSendCap,
|
|
529
534
|
sendBudgetState.initialSendAllowance(initialSendCap) + (compactPrepaid ? 1 : 0)),
|
|
@@ -758,7 +763,9 @@ export async function prepareAdapterExchange(
|
|
|
758
763
|
: undefined;
|
|
759
764
|
let refetchPermitUsed = false;
|
|
760
765
|
const refetchReporter = refetchAllowance ? transientSendReporter(refetchAllowance.permit) : undefined;
|
|
761
|
-
const deferredRefetch = billingPoolRequest
|
|
766
|
+
const deferredRefetch = (billingPoolRequest || options.comboAttempt) && refetchReporter
|
|
767
|
+
? deferPhysicalSendReporter(refetchReporter) : undefined;
|
|
768
|
+
const pendingKeyMetadata = deferredRefetch && usesApiKeyAccount(route.provider);
|
|
762
769
|
try {
|
|
763
770
|
return await refetchWithPolicy(
|
|
764
771
|
recoveryKind => {
|
|
@@ -767,14 +774,14 @@ export async function prepareAdapterExchange(
|
|
|
767
774
|
});
|
|
768
775
|
// Prepare key metadata before the guarded transport commits its attempt;
|
|
769
776
|
// counters and permits still wait for the physical executor after pacing.
|
|
770
|
-
if (
|
|
777
|
+
if (pendingKeyMetadata) transportState.noteRoutedAttemptSend(retryEstimate, recoveryKind ?? recovery);
|
|
771
778
|
const confirmDispatch = () => {
|
|
772
779
|
if (refetchAllowance?.permit && !refetchPermitUsed) {
|
|
773
780
|
if (!refetchAllowance.permit.use()) throw new SendBudgetExhaustedError(safeHostLabel(retryRequest.url));
|
|
774
781
|
refetchPermitUsed = true;
|
|
775
782
|
}
|
|
776
|
-
if (!
|
|
777
|
-
//
|
|
783
|
+
if (!pendingKeyMetadata) transportState.noteRoutedAttemptSend(retryEstimate, recoveryKind ?? recovery);
|
|
784
|
+
// Counted Combo/pool sends confirm at the physical executor; legacy
|
|
778
785
|
// helper paths retain their existing thunk admission boundary.
|
|
779
786
|
onDispatch?.();
|
|
780
787
|
if (preserveFailureResponse) replacementAdmitted = true;
|
|
@@ -887,7 +894,7 @@ export async function prepareAdapterExchange(
|
|
|
887
894
|
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
888
895
|
const hop = reserveCredentialHop("auth-recovery",
|
|
889
896
|
`${route.providerName}|${route.modelId}|antigravity-auth`,
|
|
890
|
-
!adapterOwnsDispatch && (
|
|
897
|
+
!adapterOwnsDispatch && genericRefetchCountsSends());
|
|
891
898
|
if (!hop.allowed) return null;
|
|
892
899
|
try {
|
|
893
900
|
const snapshot = await failoverAccountSnapshot(route.providerName, nextId);
|
|
@@ -923,6 +930,15 @@ export async function prepareAdapterExchange(
|
|
|
923
930
|
};
|
|
924
931
|
// Keep recovery kinds in sync with the native Responses `passthroughRecovery:` loop above.
|
|
925
932
|
recovery: for (;;) {
|
|
933
|
+
if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") {
|
|
934
|
+
const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop,
|
|
935
|
+
remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed),
|
|
936
|
+
refresh: () => transportState.recoverAutoPausedSelection(parsed), refetch: () => rebuildAndRefetch("oauth-account-429"), setPermit: permit => { sendBudgetState.pendingHopPermit = permit; } });
|
|
937
|
+
if (!recovered) break;
|
|
938
|
+
if ("failed" in recovered) return recovered.failed;
|
|
939
|
+
upstreamResponse = recovered; continue recovery;
|
|
940
|
+
}
|
|
941
|
+
|
|
926
942
|
// Preserve the terminal verdict through adapter and combo error formatting.
|
|
927
943
|
// This also covers a reset reached by a 401/429/413 recovery refetch.
|
|
928
944
|
if (isNonReplayableResponse(upstreamResponse)) {
|
|
@@ -942,7 +958,7 @@ export async function prepareAdapterExchange(
|
|
|
942
958
|
continue recovery;
|
|
943
959
|
}
|
|
944
960
|
if (
|
|
945
|
-
upstreamResponse.status === 401
|
|
961
|
+
upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
|
|
946
962
|
&& isOAuth401ReplayProvider
|
|
947
963
|
&& transportState.sentOAuthSnapshot
|
|
948
964
|
&& !oauth401ReplayAttempted
|
|
@@ -978,7 +994,7 @@ export async function prepareAdapterExchange(
|
|
|
978
994
|
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
979
995
|
const hop = reserveCredentialHop("auth-recovery",
|
|
980
996
|
`${route.providerName}|${route.modelId}|terminal-refresh-account`,
|
|
981
|
-
!adapterOwnsDispatch && (
|
|
997
|
+
!adapterOwnsDispatch && genericRefetchCountsSends());
|
|
982
998
|
if (hop.allowed) {
|
|
983
999
|
try {
|
|
984
1000
|
const admitted = await applyFailoverSnapshot(alternate);
|
|
@@ -1062,7 +1078,7 @@ export async function prepareAdapterExchange(
|
|
|
1062
1078
|
continue recovery;
|
|
1063
1079
|
}
|
|
1064
1080
|
|
|
1065
|
-
if (route.providerName === "google-antigravity" && upstreamResponse.status === 401
|
|
1081
|
+
if (route.providerName === "google-antigravity" && upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
|
|
1066
1082
|
&& oauth401ReplayAttempted && !antigravityAuthRotationAttempted
|
|
1067
1083
|
&& !isNonReplayableResponse(upstreamResponse)) {
|
|
1068
1084
|
antigravityAuthRotationAttempted = true;
|
|
@@ -1348,12 +1364,31 @@ export async function prepareAdapterExchange(
|
|
|
1348
1364
|
&& anthropicInstance
|
|
1349
1365
|
&& transportState.anthropicPoolAccountId
|
|
1350
1366
|
) {
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1367
|
+
// Same admission as the other rotation ladders: the roster cap cannot see the shared
|
|
1368
|
+
// send budget. A canRetry term, not a break -- the call also records the refusal.
|
|
1369
|
+
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
1370
|
+
const hop = reserveCredentialHop(
|
|
1371
|
+
"auth-recovery",
|
|
1372
|
+
`${route.providerName}|${route.modelId}|anthropic-pool-rotation`,
|
|
1373
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1374
|
+
);
|
|
1375
|
+
let nextAccountId: string | null;
|
|
1376
|
+
try {
|
|
1377
|
+
nextAccountId = await rotateAnthropicAccountOnResponseForInstance(anthropicInstance, upstreamResponse, {
|
|
1378
|
+
config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey,
|
|
1379
|
+
model: route.modelId, requestKey: transportState, decision: transportState.anthropicRouteDecision, currentDecision: currentAnthropicRouteDecision, signal: upstream.signal,
|
|
1380
|
+
// A denied reservation must not rotate on base headroom its bounds cannot see.
|
|
1381
|
+
canRetry: hop.allowed && (hop.permit !== undefined || !sendBudgetExhausted())
|
|
1382
|
+
&& transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
|
|
1383
|
+
});
|
|
1384
|
+
} catch (rotationError) {
|
|
1385
|
+
hop.permit?.release();
|
|
1386
|
+
throw rotationError;
|
|
1387
|
+
}
|
|
1388
|
+
if (!nextAccountId) {
|
|
1389
|
+
hop.permit?.release();
|
|
1390
|
+
break;
|
|
1391
|
+
}
|
|
1357
1392
|
try {
|
|
1358
1393
|
const admitted = await commitResolvedOAuthSelection(await anthropicRoutingFor(anthropicInstance).getAnthropicPoolAccessSnapshot(nextAccountId));
|
|
1359
1394
|
if (!admitted) throw new Error("OAuth selection changed during recovery");
|
|
@@ -1376,12 +1411,26 @@ export async function prepareAdapterExchange(
|
|
|
1376
1411
|
});
|
|
1377
1412
|
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
|
|
1378
1413
|
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1414
|
+
// The replay IS this hop's send; the dispatching layer settles the reservation.
|
|
1415
|
+
sendBudgetState.pendingHopPermit = hop.permit;
|
|
1416
|
+
let result: Response | { failed: Response };
|
|
1417
|
+
try {
|
|
1418
|
+
result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401"
|
|
1419
|
+
: upstreamResponse.status === 403 ? "oauth-account-403" : "anthropic-oauth-429",
|
|
1420
|
+
() => { if (!adapterOwnsDispatch) hop.permit?.use(); });
|
|
1421
|
+
} finally {
|
|
1422
|
+
sendBudgetState.pendingHopPermit = undefined;
|
|
1423
|
+
}
|
|
1424
|
+
if ("failed" in result) {
|
|
1425
|
+
// A no-op once the boundary was reached; a refund when the rebuild died first.
|
|
1426
|
+
hop.permit?.release();
|
|
1427
|
+
return result.failed;
|
|
1428
|
+
}
|
|
1429
|
+
upstreamResponse = result;
|
|
1383
1430
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
1384
|
-
|
|
1431
|
+
} catch {
|
|
1432
|
+
// release() refunds a pre-send hop and is a no-op once the send settled it.
|
|
1433
|
+
hop.permit?.release();
|
|
1385
1434
|
break;
|
|
1386
1435
|
}
|
|
1387
1436
|
}
|
|
@@ -1421,9 +1470,9 @@ export async function prepareAdapterExchange(
|
|
|
1421
1470
|
const hop = reserveCredentialHop(
|
|
1422
1471
|
"auth-recovery",
|
|
1423
1472
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-429`,
|
|
1424
|
-
//
|
|
1473
|
+
// Counted generic helpers report physical sends; an adapter-owned
|
|
1425
1474
|
// ladder settles its own booking.
|
|
1426
|
-
!adapterOwnsDispatch && (
|
|
1475
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1427
1476
|
);
|
|
1428
1477
|
if (!hop.allowed) break;
|
|
1429
1478
|
const nextAccountId = rotateGenericOAuthAccountOnRefusal(
|
|
@@ -1525,9 +1574,9 @@ export async function prepareAdapterExchange(
|
|
|
1525
1574
|
const hop = reserveCredentialHop(
|
|
1526
1575
|
"auth-recovery",
|
|
1527
1576
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-429`,
|
|
1528
|
-
//
|
|
1577
|
+
// Counted generic helpers report physical sends; an adapter-owned
|
|
1529
1578
|
// ladder settles its own booking.
|
|
1530
|
-
!adapterOwnsDispatch && (
|
|
1579
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1531
1580
|
);
|
|
1532
1581
|
if (!hop.allowed) break;
|
|
1533
1582
|
const nextAccountId = rotateGenericOAuthAccountOn429(
|
|
@@ -1646,7 +1695,7 @@ export async function prepareAdapterExchange(
|
|
|
1646
1695
|
const hop = reserveCredentialHop(
|
|
1647
1696
|
"auth-recovery",
|
|
1648
1697
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-verify`,
|
|
1649
|
-
!adapterOwnsDispatch && (
|
|
1698
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1650
1699
|
);
|
|
1651
1700
|
if (!hop.allowed) break;
|
|
1652
1701
|
// Auth-refusal rotation, not the rate-limit one: a verification refusal
|
|
@@ -110,6 +110,31 @@ export function recallComboForLane(
|
|
|
110
110
|
return entry.responseModel === model ? entry.comboId : undefined;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
+
/**
|
|
114
|
+
* The concrete target that completed the lane's last turn for `comboId`, under the same
|
|
115
|
+
* freshness and ownership rules as `recallComboForLane`. A lane answered by another combo, or
|
|
116
|
+
* whose target left the combo or was disabled, recalls nothing.
|
|
117
|
+
*/
|
|
118
|
+
export function recallComboTargetForLane(
|
|
119
|
+
config: OcxConfig,
|
|
120
|
+
lane: string | undefined,
|
|
121
|
+
comboId: string,
|
|
122
|
+
): Pick<OcxComboTarget, "provider" | "model"> | undefined {
|
|
123
|
+
if (!lane || !comboId) return undefined;
|
|
124
|
+
const entry = recall.get(lane);
|
|
125
|
+
if (!entry || entry.comboId !== comboId) return undefined;
|
|
126
|
+
const combo = getCombo(config, entry.comboId);
|
|
127
|
+
const provider = config.providers[entry.target.provider];
|
|
128
|
+
if (Date.now() - entry.at >= RECALL_TTL_MS
|
|
129
|
+
|| !Object.hasOwn(config.providers, entry.target.provider)
|
|
130
|
+
|| !provider || provider.disabled === true
|
|
131
|
+
|| !combo?.targets.some(target => targetKey(target) === targetKey(entry.target))) {
|
|
132
|
+
deleteEntry(lane);
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
return { provider: entry.target.provider, model: entry.target.model };
|
|
136
|
+
}
|
|
137
|
+
|
|
113
138
|
/**
|
|
114
139
|
* Periodic expiry. Without it a lane that is never read again and never touched by a config
|
|
115
140
|
* reconciliation holds its entry for the life of the process: the existing TTL is only
|
|
@@ -115,12 +115,14 @@ function retryableZeroOutputTerminal(payload: unknown): boolean {
|
|
|
115
115
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false;
|
|
116
116
|
const event = payload as {
|
|
117
117
|
type?: unknown;
|
|
118
|
-
response?: { incomplete_details?: { reason?: unknown } };
|
|
118
|
+
response?: { incomplete_details?: { reason?: unknown }; output?: unknown };
|
|
119
119
|
};
|
|
120
120
|
if (bareErrorIsRetryable(event)) return true;
|
|
121
121
|
if (event.type === "response.failed") return true;
|
|
122
122
|
if (event.type !== "response.incomplete") return false;
|
|
123
123
|
const reason = event.response?.incomplete_details?.reason;
|
|
124
|
+
// A terminal-only upstream can carry output without preceding delta/item events.
|
|
125
|
+
if (reason === undefined) return !Array.isArray(event.response?.output) || event.response.output.length === 0;
|
|
124
126
|
return typeof reason === "string" && RETRYABLE_ZERO_OUTPUT_INCOMPLETE_REASONS.has(reason);
|
|
125
127
|
}
|
|
126
128
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Combo target pinning for an in-flight tool loop (#7298).
|
|
3
|
+
*
|
|
4
|
+
* A combo picks a target per request. A tool loop is several requests that together form one
|
|
5
|
+
* assistant turn: the model emits a call, the client runs it, and the next request carries the
|
|
6
|
+
* result back. Re-picking between those requests hands the second half of a turn to a model that
|
|
7
|
+
* did not produce the first half. That is wasteful at best (a cold prompt cache, a different
|
|
8
|
+
* plan) and a hard failure at worst: DeepSeek's thinking mode refuses a tool continuation whose
|
|
9
|
+
* call carries no reasoning it can read, and a GPT turn never leaves one.
|
|
10
|
+
*
|
|
11
|
+
* So a continuation whose input ends in a tool output keeps the target that completed the lane's
|
|
12
|
+
* previous turn, as long as that target is still selectable right now: enabled, still in the
|
|
13
|
+
* combo, out of cooldown and passing every per-request eligibility filter the caller supplies.
|
|
14
|
+
* When it is not, normal selection (and normal failover) runs, and the strict serializers make
|
|
15
|
+
* the foreign history acceptable instead: see `backfillBareToolCallTurnReasoning` and the Chat
|
|
16
|
+
* adapter's reasoning placeholder.
|
|
17
|
+
*
|
|
18
|
+
* A new user message is never pinned; re-deciding per user turn is what a combo is for.
|
|
19
|
+
*/
|
|
20
|
+
import { pickComboTarget, targetKey, type ComboPick } from "../../combos";
|
|
21
|
+
import type { NormalizedComboTarget } from "../../combos/types";
|
|
22
|
+
import type { OcxConfig } from "../../types";
|
|
23
|
+
import { recallComboTargetForLane } from "./combo-session-recall";
|
|
24
|
+
|
|
25
|
+
export const COMBO_TOOL_LOOP_PIN_REASON = "combo-tool-loop-pin";
|
|
26
|
+
|
|
27
|
+
function isToolOutputItem(item: unknown): boolean {
|
|
28
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return false;
|
|
29
|
+
const type = (item as { type?: unknown }).type;
|
|
30
|
+
return typeof type === "string" && type.endsWith("_output");
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Whether this request resumes an assistant turn by returning tool results: its Responses
|
|
35
|
+
* `input` (or Chat `messages`) ends with a tool output.
|
|
36
|
+
*/
|
|
37
|
+
export function isToolLoopContinuation(body: unknown): boolean {
|
|
38
|
+
if (!body || typeof body !== "object" || Array.isArray(body)) return false;
|
|
39
|
+
const { input, messages } = body as { input?: unknown; messages?: unknown };
|
|
40
|
+
if (Array.isArray(input)) return input.length > 0 && isToolOutputItem(input[input.length - 1]);
|
|
41
|
+
if (Array.isArray(messages) && messages.length > 0) {
|
|
42
|
+
const last = messages[messages.length - 1];
|
|
43
|
+
return !!last && typeof last === "object" && (last as { role?: unknown }).role === "tool";
|
|
44
|
+
}
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The pick that keeps a tool loop on its previous target, or undefined to select normally.
|
|
50
|
+
* Uses a non-mutating preview so a pinned request never disturbs round-robin rotation state.
|
|
51
|
+
*/
|
|
52
|
+
export function toolLoopPinnedPick(args: {
|
|
53
|
+
config: OcxConfig;
|
|
54
|
+
comboId: string;
|
|
55
|
+
lane: string | undefined;
|
|
56
|
+
body: unknown;
|
|
57
|
+
eligible: (target: NormalizedComboTarget) => boolean;
|
|
58
|
+
now?: number;
|
|
59
|
+
}): ComboPick | undefined {
|
|
60
|
+
if (!isToolLoopContinuation(args.body)) return undefined;
|
|
61
|
+
const previous = recallComboTargetForLane(args.config, args.lane, args.comboId);
|
|
62
|
+
if (!previous) return undefined;
|
|
63
|
+
const previousKey = targetKey(previous);
|
|
64
|
+
const pick = pickComboTarget(args.config, args.comboId, {
|
|
65
|
+
eligible: target => targetKey(target) === previousKey && args.eligible(target),
|
|
66
|
+
now: args.now,
|
|
67
|
+
preview: true,
|
|
68
|
+
});
|
|
69
|
+
return pick ? { ...pick, attempted: [previousKey] } : undefined;
|
|
70
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { captureAccountAutoPauseAttempt, wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause";
|
|
1
2
|
import { spendLedgerStorageErrorResponse } from "./spend-storage-error";
|
|
2
3
|
import { createPhysicalSendReporter, rebindPhysicalSend } from "../../lib/request-execution-budget";
|
|
3
4
|
import { createInferenceSendBudget } from "../inference/context";
|
|
@@ -1058,6 +1059,7 @@ export async function handleResponsesCompact(
|
|
|
1058
1059
|
) => {
|
|
1059
1060
|
// A gateway response is not evidence about any Codex account.
|
|
1060
1061
|
if (gatewayTransport || !usesCodexForwardPoolAuth(ctx, route.provider)) return;
|
|
1062
|
+
if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return;
|
|
1061
1063
|
recordCodexUpstreamOutcome(config, ctx.accountId, outcome, {
|
|
1062
1064
|
...meta,
|
|
1063
1065
|
threadId: ctx.kind === "pool" || ctx.kind === "main-pool" ? ctx.affinityKey : undefined,
|
|
@@ -1166,6 +1168,7 @@ export async function handleResponsesCompact(
|
|
|
1166
1168
|
providerFetch(sendProvider, undefined, {
|
|
1167
1169
|
providerName: route.providerName,
|
|
1168
1170
|
modelId: route.modelId,
|
|
1171
|
+
accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: sendAuthCtx }, headers),
|
|
1169
1172
|
beforeDispatch: isCanonicalOpenAiForwardProvider(sendProvider)
|
|
1170
1173
|
? createCodexAuthDispatchGuard(sendAuthCtx, config, selectedModelId, admission) : undefined,
|
|
1171
1174
|
dispatchOverride: sendProvider.authMode !== "forward" && sendProvider.apiKey
|
|
@@ -1287,7 +1290,7 @@ export async function handleResponsesCompact(
|
|
|
1287
1290
|
|
|
1288
1291
|
recordDistributedResponseFailure(upstream, config, route.providerName, req.signal);
|
|
1289
1292
|
if (
|
|
1290
|
-
upstream.status === 401
|
|
1293
|
+
upstream.status === 401 && !wasPermanentAccountRefusal(upstream)
|
|
1291
1294
|
&& (authCtx.kind === "main-pool" || authCtx.kind === "pool")
|
|
1292
1295
|
&& !gatewayTransport
|
|
1293
1296
|
&& usesCodexForwardPoolAuth(authCtx, compactProvider)
|
|
@@ -1411,12 +1414,12 @@ export async function handleResponsesCompact(
|
|
|
1411
1414
|
// surfaces to the client, which retries the compact task OUTSIDE the logical request
|
|
1412
1415
|
// — reporting exhausted retries while another pool account sat idle (#913).
|
|
1413
1416
|
if (
|
|
1414
|
-
(upstream.status === 429 || upstream.status === 402)
|
|
1417
|
+
(upstream.status === 429 || upstream.status === 402 || wasAccountAutoPaused(upstream))
|
|
1415
1418
|
// A replay refusal this proxy synthesized carries 429 for the client's benefit only.
|
|
1416
1419
|
// It is not pool quota evidence, and the alternate account below is another send of a
|
|
1417
1420
|
// compact turn that may already have been processed.
|
|
1418
1421
|
&& !isNonReplayableResponse(upstream)
|
|
1419
|
-
&& !storedPool401ReplayAttempted
|
|
1422
|
+
&& (!storedPool401ReplayAttempted || wasAccountAutoPaused(upstream))
|
|
1420
1423
|
&& !gatewayTransport
|
|
1421
1424
|
&& usesCodexForwardPoolAuth(authCtx, route.provider)
|
|
1422
1425
|
&& !authCtx.fixedAccount
|
|
@@ -1521,7 +1524,7 @@ export async function handleResponsesCompact(
|
|
|
1521
1524
|
// the same guards as the quota move, and a 5xx whose body confirms quota keeps today's path.
|
|
1522
1525
|
if (
|
|
1523
1526
|
shouldRetryCodexPoolAccountTransient(upstream)
|
|
1524
|
-
&& !storedPool401ReplayAttempted
|
|
1527
|
+
&& (!storedPool401ReplayAttempted || wasAccountAutoPaused(upstream))
|
|
1525
1528
|
&& !gatewayTransport
|
|
1526
1529
|
&& usesCodexForwardPoolAuth(authCtx, route.provider)
|
|
1527
1530
|
&& !authCtx.fixedAccount
|
|
@@ -1642,7 +1645,7 @@ export async function handleResponsesCompact(
|
|
|
1642
1645
|
inspectResponseLogJson(logCtx, compactText);
|
|
1643
1646
|
if (!options.compactionRoutingOverride) forgetCompactHandoffRoute(req, admission);
|
|
1644
1647
|
rememberServingConversationStateIssuer(outcomeCtx, codexPoolAffinityKey(req.headers));
|
|
1645
|
-
} else if (!options.compactionRoutingOverride && quotaFailure && !storedPool401ReplayAttempted) {
|
|
1648
|
+
} else if (!options.compactionRoutingOverride && quotaFailure && (!storedPool401ReplayAttempted || wasAccountAutoPaused(upstream))) {
|
|
1646
1649
|
const fallbackModel = compactHandoffRoute(req, admission, raw.model);
|
|
1647
1650
|
if (fallbackModel && !req.signal.aborted) {
|
|
1648
1651
|
const fallbackReq = new Request(req.url, {
|