@bitkyc08/opencodex 2.86.0 → 2.87.0

package/src/server/messages-native.ts

@@ -61,7 +61,9 @@ import {
   type UpstreamSendRecovery,
   UPSTREAM_RESET_REPLAY_REFUSED_CODE,
 } from "../lib/upstream-retry";
+import { distributedKeyPhysicalFetch, distributedKeyResponseOwner, recordDistributedResponseFailure } from "../providers/api-key-response-ownership";
 import { providerApiKeySelectionIsCurrent, resolveCurrentProviderApiKeyTransport } from "../providers/api-key-selection";
+import { RequestPacingQueueOverloadError } from "../providers/request-pacing";
 import {
   hasKeyPoolFailover,
   rateLimitRetryDelayMs,
@@ -91,12 +93,14 @@ import {
   resolvePassthroughBodyGuard,
   sanitizePassthroughToolCallIds,
   tapAnthropicSseForLog,
-} from "./claude-messages";
+} from "./messages-shared";
 import { beginInferenceAttempt } from "./inference/attempt";
 import { createFinalRequestLog, type FinalRequestLogMeta } from "./inference/final-log";
 import { registerTurn, unregisterTurn } from "./lifecycle";
 import { nativeMessagesDeclineReason, type NativeMessagesSelector } from "./messages-native-eligibility";
 import { messagesSecondaryInstanceUnavailable } from "./messages-native-selector";
+import { guardNativeMessagesEmptyToolCatalog, nativeMessagesClientToolName, nativeMessagesInlineToolNames } from "./messages-native-tool-guard";
+import { UNDECLARED_TOOL_CALL_ERROR_CODE, undeclaredToolCallMessage } from "./responses-undeclared-tool-guard";
 import {
   nativeOAuthBindingIsCurrent,
   NativeOAuthSelectionChangedError,
@@ -315,6 +319,8 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
   const { req, config, logCtx, logIds, route, body, requestedModel, translatorBudget } = options;
   const nativeInstance = configuredAnthropicInstance(config, route.providerName);
   const requestedStream = body.stream === true;
+  const clientDeclaredEmptyToolCatalog = Array.isArray(body.tools) && body.tools.length === 0;
+  const declaredInlineTools = clientDeclaredEmptyToolCatalog ? nativeMessagesInlineToolNames(body) : undefined;
   logCtx.inboundProtocol = "messages";
   logCtx.model = route.modelId;
   logCtx.provider = route.providerName;
@@ -436,8 +442,16 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
     }
   } else {
     // Same pre-dispatch key preference every direct send path applies (see chat-native.ts).
-    const proactiveKeyProvider = selectProactiveApiKeyTransport(config, route.providerName, route.provider);
-    if (proactiveKeyProvider) route.provider = proactiveKeyProvider;
+    try {
+      const proactiveKeyProvider = selectProactiveApiKeyTransport(config, route.providerName, route.provider);
+      if (proactiveKeyProvider) route.provider = proactiveKeyProvider;
+    } catch (error) {
+      cleanupAbort(); upstream.abort();
+      if (!(error instanceof RequestPacingQueueOverloadError)) throw error;
+      const refusal = fail(429, error.message, "rate_limit_error");
+      refusal.headers.set("Retry-After", String(error.retryAfterSeconds));
+      return refusal;
+    }
   }
   let activeProvider: OcxProviderConfig = oauthBinding ? oauthProvider(oauthBinding) : route.provider;
   stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider);
@@ -521,6 +535,15 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
       && nativeMessagesDeclineReason({ ...route, provider: routed }, body, config, selector) === undefined;
   };
 
+  // Keep the actual failed wire identity if a later recovery is refused before sending.
+  // A 429 remains eligible for explicit same-key retries until that ladder is finished.
+  let pendingDistributedFailure: (() => void) | undefined;
+  let distributedResponseOwnsKey = true;
+  const settleDistributedFailure = () => {
+    const record = pendingDistributedFailure;
+    pendingDistributedFailure = undefined;
+    record?.();
+  };
   const send = async (recovery?: "rate-limit-429" | "oauth-401" | "oauth-account-403" | "key-429" | "key-401"): Promise<Response> => {
     const remaining = remainingTransientSends();
     if (requestTransientPolicy && remaining <= 0) {
@@ -538,7 +561,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
         providerFetch(activeProvider, undefined, {
           providerName: route.providerName,
           modelId: route.modelId,
-          dispatchOverride: async (_input, init, execute) => {
+          dispatchOverride: async (_input, init, execute, reacquirePacing) => {
             if (oauthBinding) {
               if (!nativeOAuthRouteIsCurrent(oauthBinding)) throw new NativeOAuthSelectionChangedError();
               // The OAuth twin of the key check below: re-resolve through the same selection
@@ -555,12 +578,25 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
                 rebuildFor(oauthProvider(oauthBinding));
                 logCtx.provider = formatAnthropicProviderForLog(route.providerName, oauthBinding.snapshot.accountId, config);
               }
-            } else if (!providerApiKeySelectionIsCurrent(config, route.providerName, activeProvider)) {
-              const current = resolveCurrentProviderApiKeyTransport(config, route.providerName, activeProvider);
-              if (!current || nativeMessagesDeclineReason({ ...route, provider: current }, body, config, selector) !== undefined) {
-                throw new Error("Provider key selection is no longer available for native Messages");
+            } else {
+              for (let attempt = 0; !providerApiKeySelectionIsCurrent(config, route.providerName, activeProvider); attempt++) {
+                if (attempt >= 3) throw new Error("Provider key selection changed repeatedly before native Messages dispatch");
+                let current = resolveCurrentProviderApiKeyTransport(config, route.providerName, activeProvider);
+                if (!current && activeProvider.requestPacing?.enabled
+                  && (activeProvider.requestPacing.keys || activeProvider.requestPacing.keySelection === "round-robin")) {
+                  const fresh = resolveCurrentProviderApiKeyTransport(config, route.providerName, { ...activeProvider, _apiKeyAttempt: undefined });
+                  if (fresh) current = selectProactiveApiKeyTransport(config, route.providerName, fresh) ?? fresh;
+                }
+                if (!current || nativeMessagesDeclineReason({ ...route, provider: current }, body, config, selector) !== undefined) {
+                  throw new Error("Provider key selection is no longer available for native Messages");
+                }
+                rebuildFor(current);
+                if (current.requestPacing?.enabled
+                  && (current.requestPacing.keys || current.requestPacing.keySelection === "round-robin")) {
+                  if (!reacquirePacing) throw new Error("Refreshed native Messages dispatch requires pacing readmission");
+                  await reacquirePacing(current, route.modelId);
+                }
               }
-              rebuildFor(current);
             }
             // The retry closure may hold a pre-reselection request: send the current one whole.
             const wire = activeRequest;
@@ -579,6 +615,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
             }
             const releaseFamily = snapshot && oauthBinding ? anthropicModelQuotaFor(oauthBinding.instance).claimAnthropicFamilyRevalidation(snapshot.accountId, route.modelId) : () => {};
             if (!releaseFamily) throw new AnthropicAccountCooldownError(1);
+            const selection = activeProvider._apiKeyAttempt;
             let dispatched: Response;
             const spendReport = createPhysicalSendReporter(physicalBudget, () => ({
               poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel,
@@ -599,7 +636,9 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
               if (snapshot && !sendOwner) throw new NativeOAuthSelectionChangedError();
               physicalSendsUsed++;
               dispatched = await sendWithConnectionPolicy(
-                anthropicBillingPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, route.providerName, activeProvider),
+                distributedKeyPhysicalFetch(
+                  anthropicBillingPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, route.providerName, activeProvider),
+                  route.providerName, activeProvider, wire.url, activeProvider.apiKeyTransport === "bearer" ? "authorization" : "x-api-key"),
                 wire.url,
                 applyUpstreamRecoveryInit({ ...init, method: "POST", headers, body: wire.body }, transportRecovery),
                 { providerName: route.providerName, provider: activeProvider },
@@ -623,6 +662,14 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
                 }
               } catch { /* Passive observation must not fail the response. */ }
             }
+            const ownsDistributedKey = !selection?.distributed
+              || distributedKeyResponseOwner(dispatched, config, route.providerName) !== null;
+            distributedResponseOwnsKey = ownsDistributedKey;
+            const failureStatus = dispatched.status;
+            pendingDistributedFailure = selection?.distributed && ownsDistributedKey && (failureStatus === 401 || failureStatus === 429)
+              && !isNonReplayableResponse(dispatched)
+              ? () => { recordDistributedResponseFailure(dispatched, config, route.providerName, req.signal); }
+              : undefined;
             if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal);
             return dispatched;
           },
@@ -647,25 +694,28 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
   let response: Response;
   try {
     response = await send();
-    const retryPolicy = oauthBinding ? undefined : rateLimitRetryPolicyFor(activeProvider);
+    let retryPolicy: ReturnType<typeof rateLimitRetryPolicyFor> | undefined;
     let retries = 0;
     let billingRecoveryActive = false;
     const keySendAvailable = () => transientSendAvailable()
       && (!billingRecoveryActive || Math.max(physicalSendsUsed, physicalBudget.used) < physicalBudget.policy.maxTotalModelSends);
-    for (;;) {
+    const recoveryPasses = activeProvider._apiKeyAttempt?.distributed ? activeProvider.apiKeyPool?.length ?? 1 : 1;
+    for (let pass = 0; ; pass++) {
       const priorResponse = response;
       // A credential-scoped 401 on a static key pool says nothing about the sibling keys.
-      while (response.status === 401 && hasKeyPoolFailover(activeProvider) && keySendAvailable()) {
+      while (response.status === 401 && distributedResponseOwnsKey && hasKeyPoolFailover(activeProvider) && keySendAvailable()) {
         const rotated = rotateProviderTransportOn401(config, route.providerName, activeProvider, {
           now: Date.now(),
           attemptedKey: activeProvider.apiKey,
+          attemptedSelection: activeProvider._apiKeyAttempt,
         });
+        pendingDistributedFailure = undefined; // Rotation already records this physical failure.
         if (!rotated) break;
         discard(response);
         rebuildFor(rotated);
         response = await send("key-401");
       }
-      while ((response.status === 400 || response.status === 402) && hasKeyPoolFailover(activeProvider)) {
+      while ((response.status === 400 || response.status === 402) && distributedResponseOwnsKey && hasKeyPoolFailover(activeProvider)) {
         const rotated = await rotateAnthropicBillingKeyOnResponse(response, config, route.providerName, activeProvider, { signal: upstream.signal });
         if (!rotated) break;
         billingRecoveryActive = true;
@@ -674,9 +724,11 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
         rebuildFor(rotated);
         response = await send("key-401");
       }
+      if (pass === 0) retryPolicy = oauthBinding ? undefined : rateLimitRetryPolicyFor(activeProvider);
       // A refusal this proxy synthesized for a reset replay is not a provider rate limit.
       while (
         response.status === 429
+        && distributedResponseOwnsKey
         && !isNonReplayableResponse(response)
         && retryPolicy
         && retries < retryPolicy.attempts
@@ -691,13 +743,14 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
         if (upstream.signal.aborted) throw upstream.signal.reason;
         response = await send("rate-limit-429");
       }
-      while (response.status === 429 && !isNonReplayableResponse(response) && hasKeyPoolFailover(activeProvider)) {
+      while (response.status === 429 && distributedResponseOwnsKey && !isNonReplayableResponse(response) && hasKeyPoolFailover(activeProvider)) {
         const rotated = rotateProviderTransportOn429(config, route.providerName, activeProvider, {
           retryAfter: response.headers.get("retry-after"),
           now: Date.now(),
           attemptedKey: activeProvider.apiKey,
           attemptedSelection: activeProvider._apiKeyAttempt,
         });
+        pendingDistributedFailure = undefined; // Rotation already records this physical failure.
         if (!rotated) break;
         // Rotation already recorded the cooldown; keep the terminal 429 when no send remains.
         if (!keySendAvailable()) break;
@@ -705,13 +758,17 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
         rebuildFor(rotated);
         response = await send("key-429");
       }
-      // A sibling's next refusal may change class; the same counters survive that transition.
-      if (response !== priorResponse && (
+      if (response === priorResponse || isNonReplayableResponse(response) || !keySendAvailable()) break;
+      // Billing class transitions share the physical-send ceiling; distributed-only
+      // transitions additionally retain their original bounded pool-pass allowance.
+      if (distributedResponseOwnsKey && (
         (billingRecoveryActive && [401, 429].includes(response.status))
         || (activeProvider.adapter === "anthropic" && hasKeyPoolFailover(activeProvider) && [400, 402].includes(response.status))
       )) continue;
-      break;
+      if (!activeProvider._apiKeyAttempt?.distributed || !distributedResponseOwnsKey
+        || (response.status !== 401 && response.status !== 429) || pass + 1 >= recoveryPasses) break;
     }
+    settleDistributedFailure();
     const oauthRetryKey = {};
     const triedAccountIds = new Set<string>();
     let oauthFailovers = 0;
@@ -749,7 +806,13 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
     cleanupAbort();
     upstream.abort();
     if (req.signal.aborted) return fail(499, "Client cancelled request", "api_error");
+    settleDistributedFailure();
     const sendError = error instanceof UpstreamRetryEvidenceError ? error.cause : error;
+    if (sendError instanceof RequestPacingQueueOverloadError) {
+      const refusal = fail(429, sendError.message, "rate_limit_error");
+      refusal.headers.set("Retry-After", String(sendError.retryAfterSeconds));
+      return refusal;
+    }
     if (sendError instanceof SendBudgetExhaustedError) return fail(429, sendError.message, "send_budget_exhausted", "send_budget_exhausted");
     if (sendError instanceof NativeMessagesSpendRefusal) {
       const refusal = workflowRefusalResponse("workflow-spend-exhausted", logCtx);
@@ -794,15 +857,22 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
     const renamed = activeRequest.oauthToolNames
       ? restoreOAuthToolNamesInSse(observed, activeRequest.oauthToolNames, translatorBudget)
       : observed;
-    const source = echoRequestedModel(renamed, requestedModel);
+    const echoed = echoRequestedModel(renamed, requestedModel);
+    let toolRefusal: string | undefined;
+    const source = clientDeclaredEmptyToolCatalog
+      ? guardNativeMessagesEmptyToolCatalog(echoed, translatorBudget, message => {
+        toolRefusal = message;
+        logCtx.errorCode = UNDECLARED_TOOL_CALL_ERROR_CODE;
+      }, declaredInlineTools)
+      : echoed;
     if (requestedStream) {
       transferTurnToStream();
       const relayed = tapAnthropicSseForLog(source, logCtx, (status, meta) => {
         try {
           cleanupAbort();
           bindUsage(logCtx.usage);
-          finishLog(status, undefined, meta);
-          if (meta.closeReason !== "terminal") upstream.abort();
+          finishLog(toolRefusal ? 502 : status, toolRefusal, toolRefusal ? { ...meta, terminalStatus: "failed" } : meta);
+          if (toolRefusal || meta.closeReason !== "terminal") upstream.abort();
         } finally {
           releaseStreamTurn();
         }
@@ -833,6 +903,11 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
       if (message.type === "error") {
         const error = isRec(message.error) ? message.error : {};
         const text = typeof error.message === "string" ? error.message : "upstream stream failed";
+        if (toolRefusal) {
+          upstream.abort();
+          finishLog(502, toolRefusal, { closeReason: "terminal", terminalStatus: "failed" });
+          return fail(502, toolRefusal, "api_error", UNDECLARED_TOOL_CALL_ERROR_CODE);
+        }
         // The tap's own verdict (a reset, a stall, the byte cap) closes this row with the same meta
         // as the streaming lane's row; a plain upstream error event keeps the non_stream row.
         const tapMeta = tapState.meta;
@@ -884,6 +959,11 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions)
   const message = activeRequest.oauthToolNames
     ? restoreOAuthToolNamesInMessage(parsedMessage, activeRequest.oauthToolNames)
     : parsedMessage;
+  const undeclaredTool = clientDeclaredEmptyToolCatalog ? nativeMessagesClientToolName(message, declaredInlineTools) : undefined;
+  if (undeclaredTool !== undefined) {
+    logCtx.errorCode = UNDECLARED_TOOL_CALL_ERROR_CODE;
+    return fail(502, undeclaredToolCallMessage(undeclaredTool), "api_error", UNDECLARED_TOOL_CALL_ERROR_CODE);
+  }
   bindUsage(anthropicUsageToOcx(isRec(message.usage) ? message.usage : undefined));
   if (logIds) recordFirstOutput(logCtx, logIds.start);
   try {