cartethyia 2.5.4 → 2.5.6

package/src/transport/dispatch/stream-bridge.ts

@@ -44,9 +44,7 @@ export function createDispatchStreamEncoder(
     return {
       push: (event) => encoder.push(event).map((wire) => formatSseEvent(wire.type, wire)),
       finish: () => encoder.finish().map((wire) => formatSseEvent(wire.type, wire)),
-      encodeError: ({ origin, code, message, details }) => [
-        formatSseEvent("response.error", { type: "response.error", origin, code, message, details }),
-      ],
+      encodeError: (input) => encoder.encodeError(input).map((wire) => formatSseEvent(wire.type, wire)),
     };
   }
   if (surface === "messages") {

package/src/transport/dispatch/streaming/finalize.ts

@@ -154,7 +154,7 @@ export class StreamFinalizer {
    *
    * Once the upstream has produced anything — even surface-invisible
    * reasoning deltas — it is alive: the stall bound governs and the tighter
-   * first-chunk bound no longer applies.
+   * first-chunk bound does not apply.
    *
    * The request deadline is a fixed ceiling measured from ingress, and a
    * stream that keeps sending past it is not stalled. A live WorkBuddy stream

package/src/transport/dispatch/streaming/stream.ts

@@ -232,10 +232,9 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
   });
   // `ProviderAdapter.dispatch` is declared to return a non-nullable
   // `AsyncIterable`, so an adapter that returned `undefined` would be a
-  // type violation rather than a state this code could handle. The guard
-  // that used to sit here (`if (!iterable) throw ...`) was therefore
-  // unreachable, and it advertised a contract the interface does not
-  // have. Enforcing it belongs at the adapter boundary, not here.
+  // type violation rather than a state this code could handle — no null check
+  // is needed here, and adding one would advertise a contract the interface
+  // does not have. Enforcing it belongs at the adapter boundary.
   const createUpstreamIterable = () =>
     adapter.dispatch(
       dispatchRequest,
@@ -375,8 +374,8 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
           // reasoning item when the client is Chat Completions). Keep
           // pulling until at least one wire chunk is enqueued; returning
           // from pull with an empty queue can leave Web Streams waiting
-          // forever, which is why real clients previously saw one
-          // assistant header and then a 60s socket reset.
+          // forever — a client then sees one assistant header and then a
+          // 60s socket reset.
           for (;;) {
             // Re-arm on every upstream read: the watchdog fires only
             // when the provider stays silent longer than the bound.

package/src/transport/failure-policy.ts

@@ -229,6 +229,35 @@ export function extractUpstreamMessage(body: unknown): string {
  * Reading only a nested string dropped the code for the second family
  * entirely, so the operator saw a bare message with no provider code.
  */
+/**
+ * The provider code hidden inside one string value, or `undefined`.
+ *
+ * Recognizes both a nested JSON envelope written as an escaped string
+ * (`"{\"code\":\"11-128\"}"`) and the bare escaped form (`"\"code\\\":\\\"11-128\""`).
+ * Cheap by construction: the scan is a substring test, and only a string that
+ * looks like it carries a code is handed to `JSON.parse`, so a large message
+ * body costs one `includes` rather than a parse.
+ */
+function parseEmbeddedCode(value: string): string | undefined {
+  if (!value.includes("code")) return undefined;
+  // An escaped envelope: try it as JSON first, then with its escapes collapsed.
+  for (const candidate of [value, value.replace(/\\+"/g, '"')]) {
+    const trimmed = candidate.trim();
+    if (!trimmed.startsWith("{") && !trimmed.startsWith('"')) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    const code = upstreamProviderCode(parsed);
+    if (code !== undefined) return code;
+  }
+  // A bare `"code": "11-128"` fragment inside a longer message.
+  const match = value.match(/\\*["']?code\\*["']?\s*:\s*\\*["']?([A-Za-z0-9_.:-]{3,32})/i);
+  return match?.[1];
+}
+
 export function upstreamProviderCode(body: unknown): string | undefined {
   if (!isRecord(body)) return undefined;
   const nested = isRecord(body.error) ? body.error : undefined;
@@ -236,6 +265,20 @@ export function upstreamProviderCode(body: unknown): string | undefined {
   for (const candidate of [ext?.code, nested?.code, body.code]) {
     if (typeof candidate === "string" && candidate.length > 0) return candidate;
   }
+  // A provider may embed its error envelope as an *escaped string* rather than a
+  // nested object — `"error": "{\"code\":\"11-128\",...}"` — so none of the
+  // candidates above is a record and the code is invisible. That is the shape
+  // the reference gateways detect, and it matters because the channel code is
+  // what decides whether an account is cooled: missing it, a blocked account
+  // stays in rotation failing every request.
+  for (const candidate of [ext, nested, body]) {
+    if (!isRecord(candidate)) continue;
+    for (const value of Object.values(candidate)) {
+      if (typeof value !== "string") continue;
+      const embedded = parseEmbeddedCode(value);
+      if (embedded !== undefined) return embedded;
+    }
+  }
   // A numeric code is still the provider's identifier; keep it as text so an
   // operator can search for it. The nested string above wins when both exist.
   const numeric = ext?.code ?? nested?.code ?? body.code;
@@ -364,6 +407,14 @@ export function classifyUpstreamFailure(error: unknown): UpstreamFailurePolicy {
     error.code === "model_not_found" ||
     error.code === "admission_unavailable" ||
     error.code === "capacity_exhausted" ||
+    // A provider max-inflight rejection: the plan's next sibling account may
+    // have a free slot, exactly as with an API-key concurrency rejection.
+    error.code === "provider_capacity_exhausted" ||
+    // The upstream accepted the request and is holding it in a queue. Another
+    // account of the same provider is very likely not queued, so failover is
+    // the faster answer — and if every account is queued, the caller still has
+    // the stated interval to wait for.
+    error.code === "upstream_queued" ||
     error.code === "accounts_unavailable" ||
     error.code === "accounts_rate_limited" ||
     error.code === "quota_exceeded" ||
@@ -499,8 +550,8 @@ export interface TerminalOutcome {
  * Derives a failed attempt's status, error category, and error origin **from
  * one decision**, so the three can never contradict each other.
  *
- * They used to be computed independently at each dispatch call site, and they
- * drifted: a request that failed on an upstream condition could be recorded
+ * Computing them independently at a dispatch call site lets them
+ * drift: a request that failed on an upstream condition could be recorded
  * `cancelled` (499) while its category said `transport_unavailable` and its
  * http status said 502 — a row no operator could trust. Bundling them here
  * makes the inconsistency unrepresentable: a cancel always carries
@@ -591,6 +642,88 @@ export function isContextLengthFailure(body: unknown): boolean {
   }
   return false;
 }
+/**
+ * How long an upstream asked the caller to wait before retrying, in
+ * milliseconds, or `undefined` when the body states no duration.
+ *
+ * `retryAfterSeconds` is the field the queue-shaped refusals carry. It is read
+ * from a few plausible spellings because the upstreams that queue a request do
+ * not agree on one, and a queue with no duration still has to be reported as a
+ * queue — the caller's wait is then bounded by the caller, not by us.
+ */
+export function queueRetryAfterMs(body: unknown): number | undefined {
+  if (!isRecord(body)) return undefined;
+  const envelope = isRecord(body.error) ? body.error : body;
+  for (const source of [envelope, body]) {
+    for (const key of ["retryAfterSeconds", "retry_after_seconds", "retryAfterMs", "retry_after"]) {
+      const raw = source[key];
+      if (typeof raw === "number" && Number.isFinite(raw) && raw > 0) {
+        return key.toLowerCase().endsWith("ms") ? Math.round(raw) : Math.round(raw * 1000);
+      }
+      if (typeof raw === "string") {
+        const value = Number(raw.trim());
+        if (Number.isFinite(value) && value > 0) {
+          return key.toLowerCase().endsWith("ms") ? Math.round(value) : Math.round(value * 1000);
+        }
+      }
+    }
+  }
+  return undefined;
+}
+
+/**
+ * True when an upstream error body says the request is **queued**, not refused.
+ *
+ * Qoder answers `403` with `{"code":"10605", "isQueued":true,
+ * "retryAfterSeconds":N}`. That is not a rejection — the request is accepted and
+ * will be served once capacity frees up — but it arrives on a status the
+ * gateway treats as a failure, so without this it surfaced as a 403 the client
+ * read as "you may not do this" and the turn ended. Reporting it as a queue
+ * lets the caller wait for the stated interval instead.
+ *
+ * Recognized conservatively: an explicit `isQueued` flag, the `10605` code, or
+ * a message naming a queue. A bare word like "queue" appearing anywhere in a
+ * long body is not enough on its own, which is why the message test requires
+ * the queue to be what the message is *about* rather than a word it contains.
+ */
+export function isUpstreamQueued(body: unknown): boolean {
+  if (!isRecord(body)) return false;
+  const envelope = isRecord(body.error) ? body.error : body;
+  if (envelope.isQueued === true || body.isQueued === true) return true;
+  if (envelope.queued === true || body.queued === true) return true;
+  for (const candidate of [envelope.code, body.code]) {
+    if (typeof candidate === "string" && candidate.trim() === "10605") return true;
+  }
+  for (const candidate of [envelope.message, envelope.msg, body.message, body.msg]) {
+    if (typeof candidate !== "string") continue;
+    const lower = candidate.toLowerCase();
+    // "is queued" / "queued, retry in Ns" — the queue as the message's subject.
+    if (/\bqueued\b|\bis queue\b|resource_limit/.test(lower)) return true;
+  }
+  return false;
+}
+
+/** The error a queued upstream request becomes: retryable, with its wait. */
+export function upstreamQueuedError(
+  providerId: string,
+  retryAfterMs: number | undefined,
+  detail: string,
+): GatewayError {
+  return new GatewayError(
+    "upstream_queued",
+    503,
+    detail.length > 0
+      ? detail
+      : "The upstream has queued this request. Retry after the stated interval.",
+    {
+      providerId,
+      reason: "upstream-queued",
+      ...(retryAfterMs === undefined ? {} : { retryAfterMs, cooldownMs: retryAfterMs }),
+    },
+    "upstream",
+  );
+}
+
 const QUOTA_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "rate_limit_exceeded",
   "rate_limit_error",
@@ -815,12 +948,31 @@ export async function mapUpstreamHttpError(response: Response, providerId: strin
     parsedBody = undefined;
   }
   const identifier = upstreamErrorIdentifier(parsedBody) ?? providerCode;
-  const classification = isContextLengthFailure(parsedBody)
-    ? { code: "context_length_exceeded" as const, status: 413, origin: "upstream" as const }
-    : classifyUpstreamError(response.status, identifier);
+  // A queued request is checked *before* the status table: the upstream accepted
+  // the request and will serve it, but says so on a failure status (Qoder's 403
+  // + 10605). Letting the table classify it first turned "wait, then retry"
+  // into a refusal the client could not distinguish from a permissions error.
+  const upstreamQueued = isUpstreamQueued(parsedBody);
+  // The body's stated interval wins; the header is only a fallback, because an
+  // upstream that queues a request usually states the wait in the body and
+  // leaves `Retry-After` off entirely.
+  const queuedWaitMs = upstreamQueued
+    ? (queueRetryAfterMs(parsedBody) ??
+      (parseUpstreamBackoff(response.headers) ?? undefined))
+    : undefined;
+  const classification = upstreamQueued
+    ? { code: "upstream_queued" as const, status: 503, origin: "upstream" as const }
+    : isContextLengthFailure(parsedBody)
+      ? { code: "context_length_exceeded" as const, status: 413, origin: "upstream" as const }
+      : classifyUpstreamError(response.status, identifier);
   const extracted = extractUpstreamMessage(parsedBody ?? text);
   const message = (extracted.length > 0 ? extracted : text.trim() || `Provider returned HTTP ${response.status}.`).slice(0, MAX_UPSTREAM_ERROR_BYTES).replace(/[\r\n]+/g, " ");
-  const retryAfterMs = parseUpstreamBackoff(response.headers);
+  const headerWaitMs = parseUpstreamBackoff(response.headers);
+  // A queue carries its own interval, which is the one the caller must honour:
+  // it is what the upstream promised, while `Retry-After` may be absent or may
+  // describe a different window. `cooldownMs` rides along so the account-health
+  // layer can also read it.
+  const retryAfterMs = queuedWaitMs ?? headerWaitMs;
   const requestId = upstreamRequestId(response.headers);
   const reportedProviderCode = providerCode ?? identifier;
   throw new GatewayError(classification.code, classification.status, message, {
@@ -830,6 +982,9 @@ export async function mapUpstreamHttpError(response: Response, providerId: strin
     ...(requestId ? { upstreamRequestId: requestId } : {}),
     raw: text.slice(0, MAX_UPSTREAM_ERROR_BYTES),
     ...(retryAfterMs === null || retryAfterMs === undefined ? {} : { retryAfterMs }),
+    ...(queuedWaitMs !== undefined
+      ? { reason: "upstream-queued", cooldownMs: queuedWaitMs }
+      : {}),
     ...(classification.rateLimitScope
       ? { rateLimitScope: classification.rateLimitScope }
       : {}),
@@ -895,8 +1050,7 @@ export function isRetryableFailure(error: unknown): boolean {
  * timeout.
  *
  * Owned here so the attempt loop, the empty-stream replay, and the Codex
- * retry budget cannot drift apart; each previously declared its own copy of
- * the constant.
+ * retry budget cannot drift apart.
  */
 export const MIN_RETRY_BUDGET_MS = 15_000;
 

package/src/transport/guards/auth-guard.ts

@@ -90,14 +90,17 @@ export function createApiKeyAuthenticationMiddleware(deps: {
         );
       // Abuse ban: refuse before parse/prepare and before authorization is
       // recorded, so a banned caller cannot keep producing failed rows. The ban
-        // is keyed on the client address, not the key: one key can be shared by
-        // every recipient of a share link, so refusing the key would punish
-        // callers that did nothing.
+      // is keyed on the client address *and* the API key: one key can be shared
+      // by every recipient of a share link, so refusing the address alone would
+      // punish callers that did nothing, while refusing the key alone would let
+      // the caller that probed mint a fresh one.
       if (deps.modelStrikes) {
         const state = deps.stateStore.get(request);
         const ip = state?.clientIdentity?.address;
         if (ip !== undefined) {
-          const banned = await deps.modelStrikes.check({ ip }).catch(() => false);
+          const banned = await deps.modelStrikes
+            .check({ ip, apiKeyId: authorization.id })
+            .catch(() => false);
           if (banned) throw modelAbuseBannedError();
         }
       }

package/src/transport/guards/concurrency-guard.ts

@@ -22,11 +22,6 @@ const PROCESS_CONCURRENCY_RETRY_MS = 3_000;
 
 let inFlight = 0;
 
-/** The current in-flight count. Exposed for tests; not a control surface. */
-export function processInFlight(): number {
-  return inFlight;
-}
-
 export function createProcessConcurrencyMiddleware(): Elysia {
   return new Elysia()
     .beforeHandle(({ request }) => {

package/src/transport/guards/error-lifecycle.ts

@@ -12,6 +12,7 @@ import { computeTokensPerSec } from "../../observability/token-speed";
 import { pushStructuredConsoleLog } from "../../observability/log-ring";
 import type { ConsoleLogLevel, ConsoleLogMetadata } from "../../observability/log-ring";
 import { isProxyDispatchRoute } from "./body-policy";
+import { notifyRequestCompleted } from "../../observability/request-notify";
 
 interface ElysiaBuiltinError {
   readonly status?: number;
@@ -217,12 +218,12 @@ export function finalizeRequestTelemetry(
     return;
   }
 
-  // An early rejection never reaches `completeAttempt`, which is the only
-  // caller that used to write a payload frame — so a 400 from ingress, auth,
-  // parse, or the preparer produced a request row whose four capture panels
-  // were all empty, and the one case an operator most wants to read (what did
-  // the client actually send?) had nothing to read. Capture the ingress body
-  // and the error the client saw, keyed to the same request id as the row.
+  // An early rejection never reaches `completeAttempt`, the caller that writes
+  // a payload frame — so a 400 from ingress, auth, parse, or the preparer would
+  // produce a request row whose four capture panels were all empty, and the one
+  // case an operator most wants to read (what did the client actually send?)
+  // would have nothing to read. Capture the ingress body and the error the
+  // client saw, keyed to the same request id as the row.
   if (requestData === undefined && state.ingressBody !== undefined) {
     void captureEarlyRejectionPayload(state, authorization);
   }
@@ -235,6 +236,12 @@ export function finalizeRequestTelemetry(
       tokensPerSec,
     }),
   );
+  notifyRequestCompleted({
+    tenantId: authorization.tenantId,
+    requestId: state.requestId,
+    status,
+    httpStatus: requestStatus,
+  });
 }
 
 /** Console-log severity for a terminal request status. */

package/src/transport/guards/pipeline.ts

@@ -28,7 +28,12 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import type { RedisClient } from "../../persistence/redis";
 import type { TelemetryBatchBuffer } from "../../observability/telemetry-buffer";
 import type { ModelStrikeService } from "../../security/model-abuse";
-import { TelemetryPayloadCapture, payloadCaptureKeepsBodies } from "../../observability/payload-capture";
+import {
+  TelemetryPayloadCapture,
+  payloadCaptureKeepsBodies,
+  payloadRetentionMs,
+  resolvePayloadCapture,
+} from "../../observability/payload-capture";
 
 /** Shared state/context carried by every ordered transport stage. */
 export interface TransportPipelineContext {
@@ -167,8 +172,15 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
         stateStore: context.stateStore,
         telemetryBuffer: context.telemetry,
         // One capture for the process, shared with every request state so an
-        // early rejection can record its payload from the finalize hook.
-        payloadCapture: new TelemetryPayloadCapture(context.db),
+        // early rejection can record its payload from the finalize hook. It
+        // resolves the tenant's retention itself because an early rejection has
+        // no preference memo on its state — nothing resolved one before the
+        // request was refused.
+        payloadCapture: new TelemetryPayloadCapture(context.db, (tenantId) =>
+          resolvePayloadCapture(context.db, tenantId).then(
+            (resolved) => (resolved.ok ? resolved.retentionMs : payloadRetentionMs()),
+          ),
+        ),
       });
     } else {
       registerRequestCleanup(rootApp, { stateStore: context.stateStore });

package/src/transport/guards/request-context.ts

@@ -269,7 +269,13 @@ export function createProxyRoutePreparationMiddleware(deps: {
         throw new GatewayError("admission_unavailable", 503, "The request was not authenticated and parsed, so this route cannot be served. Resend the request.");
       const strikes = deps.modelStrikes;
       const ip = state.clientIdentity?.address;
-      const strikeIdentity = strikes && ip !== undefined ? { strikes, ip } : undefined;
+      // The key rides along so a strike is charged to the caller that made it.
+      // Without it, one recipient of a shared link could escalate a ban onto
+      // every other caller behind the same address.
+      const strikeIdentity =
+        strikes && ip !== undefined
+          ? { strikes, ip, apiKeyId: state.authorization?.id ?? null }
+          : undefined;
       try {
         state.preparedRequest = await deps.preparer.prepare({
           canonicalRequest: state.canonicalRequest,
@@ -286,7 +292,7 @@ export function createProxyRoutePreparationMiddleware(deps: {
         // request, or any other failure is not the caller probing for models.
         if (strikeIdentity && isModelRejection(error)) {
           const outcome = await strikeIdentity.strikes
-            .noteInvalid({ ip: strikeIdentity.ip })
+            .noteInvalid({ ip: strikeIdentity.ip, apiKeyId: strikeIdentity.apiKeyId })
             .catch(() => null);
           if (outcome?.banned === true) throw modelAbuseBannedError();
           if (outcome) {

package/src/transport/request/inflight.ts

@@ -4,12 +4,11 @@
  * provider-dispatch leases, keyed by `requestId`, carrying the client IP
  * captured at ingress and the hard deadline the flight must not outlive.
  *
- * This used to be a process-global module that kept its own counter alongside
- * the request state store's `liveControllers` map — two registries that could
- * disagree, and did: a flight the store had already torn down could stay
- * counted here forever because nothing tied the two together. The registry is
- * now *owned* by the `ProxyRequestStateStore` (`state.ts`), which is also the
- * one place that evicts a request, so a flight can no longer outlive its state.
+ * The registry is *owned* by the `ProxyRequestStateStore` (`state.ts`), which
+ * is also the one place that evicts a request, so a flight cannot outlive its
+ * state. A separate process-global counter alongside the store's
+ * `liveControllers` map would be a second registry able to disagree with it: a
+ * flight the store had already torn down would stay counted there forever.
  *
  * The store hands each entry an `expiresAtMs` (the request's hard deadline). A
  * periodic backstop sweep reads `overdue()` and force-releases any flight whose

package/src/transport/request/preparer.ts

@@ -241,8 +241,8 @@ export class ProxyRequestPreparer {
     // than dispatched: a provider rejects the whole request for one corrupt
     // attachment, and the buddy family reports it with no field named, so the
     // caller gets a 400 it cannot act on. Running this before capability
-    // derivation also means a request whose only image was defective no longer
-    // requires the `image` capability — it plans like the text request it has
+    // derivation also means a request whose only image was defective does not
+    // require the `image` capability — it plans like the text request it has
     // become, instead of being routed to a vision route for nothing.
     const requestWithAttachments = dropCorruptAttachments(request).request;
     // Continuation affinity is resolved before anything else: a request

package/src/transport/request/state.ts

@@ -100,22 +100,26 @@ export interface ProxyRequestState {
    * Remote-diagnosis field: the drawer reads empty for reasons invisible from
    * the client — the tenant's capture mode, a missing tenant, a non-terminal
    * attempt — and without the reason an operator cannot tell "not configured"
-   * from "broken". The four panels are the only surface that could previously
-   * hint at it, and they look identical in both cases.
+   * from "broken". The four panels are the only surface that could hint at it,
+   * and they look identical in both cases.
    */
   payloadCaptureState?: PayloadCaptureState;
   /**
    * This request's resolved payload-capture preference, memoized on the request.
    *
    * The preference is a tenant setting, so it cannot change between attempts of
-   * one request. It was resolved twice — once per attempt to decide whether to
-   * tee the provider body, and again on the terminal attempt to decide what to
-   * store — so a request with several attempts paid for it several times.
-   * Memoized here, the second read is free and the two cannot disagree about
-   * whether capture is on.
+   * one request. It is read twice — once per attempt to decide whether to tee
+   * the provider body, and again on the terminal attempt to decide what to
+   * store — so memoizing it here makes the second read free and keeps the two
+   * from disagreeing about whether capture is on.
    */
   payloadCapturePreference?:
-    | { readonly ok: true; readonly mode: TelemetryPayloadMode; readonly depth: TelemetryPayloadDepth }
+    | {
+        readonly ok: true;
+        readonly mode: TelemetryPayloadMode;
+        readonly depth: TelemetryPayloadDepth;
+        readonly retentionMs: number;
+      }
     | { readonly ok: false };
   /**
    * This request's resolved inline-attachment ceiling, memoized for the same

package/src/transport/routing/admission.ts

@@ -60,6 +60,10 @@ function admissionDirective(candidate: RouteCandidate): AdmissionDirective {
 /** The one shape of an admission rejection: the metric, then the decision. */
 function capacityRejected(): AdmissionDecision {
   metrics.proxy_admission_total.inc(1, { reason: "provider_capacity" });
+  // `reason` is the routing-internal label, not the public code. It stays the
+  // string the router and its tests match on; the envelope code is
+  // `provider_capacity_exhausted` so this layer cannot be confused with an
+  // API-key `max_concurrent` rejection (which owns `capacity_exhausted`).
   return { admitted: false, reason: "capacity_exhausted" };
 }
 

package/src/transport/routing/route-catalog.ts

@@ -404,7 +404,7 @@ class RouteCatalogRepository {
         // The cooldown override is a console_settings preference, read inside
         // the same fan-out rather than per request: one more row set on a
         // query that already runs, instead of a Postgres round trip on every
-        // admitted request the way `tenantConcurrencyLimit` once did.
+        // admitted request.
         this.db
           .select({ tenantId: consoleSettings.tenantId, preferences: consoleSettings.preferences })
           .from(consoleSettings),
@@ -747,6 +747,13 @@ class RouteCatalogRepository {
               // The account is still cooling — this only stops it from being
               // excluded. `plan()` reads `cooldown_until` to answer the
               // all-cooling case, so it stays populated either way.
+              //
+              // Neither `health_status` nor `cooldown_kind` is set: the
+              // evaluator excludes a hard cooldown on `health_status ===
+              // "cooldown"` alone, so stamping it here would keep excluding
+              // the very accounts the override exists to reach — a spent quota
+              // or a policy block, which are the ones an operator actually
+              // wants to override.
               candidate.cooldown_until = account.cooldownUntil.getTime();
             } else {
               candidate.health_status = "cooldown";

package/src/transport/routing/route-model.ts

@@ -255,8 +255,8 @@ export interface AdmissionController {
  * Translates the evaluator's internal reason codes into operator English.
  *
  * These names are the vocabulary the routing layers use with each other —
- * `model_cooldown`, `cooldown_hard`, `locked` — and they used to be printed
- * verbatim into the public message, so the client read an enum instead of an
+ * `model_cooldown`, `cooldown_hard`, `locked` — not something to print into the
+ * public message, where the client would read an enum instead of an
  * explanation. The codes stay in the envelope's `reasons` field for anyone who
  * wants them; the sentence carries what they mean.
  *
@@ -324,11 +324,43 @@ export function capabilityUnsupportedError(capability: string): GatewayError {
   );
 }
 
-export function capacityExhaustedError(): GatewayError {
+/**
+ * Every candidate in the plan refused admission: each account is already
+ * serving its configured max-inflight.
+ *
+ * Deliberately NOT `capacity_exhausted`. That code also belongs to the API-key
+ * admission layer (`security/admission`, reason `concurrency-limit`), which
+ * enforces a key's own `max_concurrent`. Sharing one code made the two
+ * indistinguishable on the wire: an operator who raised the provider's
+ * max-inflight saw no change because the rejection was the *key's* ceiling, and
+ * vice versa. Distinct codes let each be read off the envelope alone.
+ *
+ * The counts are carried in `details` because the interesting question is never
+ * "was it rejected" but "was every account really full, or did the plan only
+ * carry one candidate" — the latter is a routing/eligibility problem that
+ * raising the ceiling will not fix.
+ */
+export function capacityExhaustedError(context?: {
+  readonly providerId?: string;
+  readonly model?: string;
+  readonly candidateCount?: number;
+  readonly maxInflight?: number;
+}): GatewayError {
+  const onlyOne = context?.candidateCount === 1;
   return new GatewayError(
-    "capacity_exhausted",
+    "provider_capacity_exhausted",
     429,
-    "Every account is at its concurrency limit. Retry shortly; if it persists, raise the provider's max-inflight or add an account.",
+    onlyOne
+      ? "The only account routing this model is at its concurrency limit. Retry shortly; if it persists, raise the provider's max-inflight or add another account."
+      : "Every account is at its concurrency limit. Retry shortly; if it persists, raise the provider's max-inflight or add an account.",
+    {
+      ...(context?.providerId !== undefined ? { providerId: context.providerId } : {}),
+      ...(context?.model !== undefined ? { model: context.model } : {}),
+      ...(context?.candidateCount !== undefined
+        ? { candidate_count: context.candidateCount }
+        : {}),
+      ...(context?.maxInflight !== undefined ? { maxInflight: context.maxInflight } : {}),
+    },
   );
 }
 
@@ -423,10 +455,10 @@ export class InMemoryRouteSnapshotService implements RouteSnapshotService {
    *
    * The tag is what keeps a mutation from being lost. A build reads the catalog
    * over ten Postgres round-trips, so a dashboard write can land while one is
-   * in flight. Without the tag two things went wrong: a reader that arrived
-   * *after* the mutation was handed the still-running pre-mutation build, and
-   * that build then overwrote the cache the mutation had just cleared — so the
-   * data plane served the old routing until some later write invalidated it
+   * in flight. Without the tag two things go wrong: a reader that arrives
+   * *after* the mutation is handed the still-running pre-mutation build, and
+   * that build then overwrites the cache the mutation has just cleared — so the
+   * data plane serves stale routing until some later write invalidates it
    * again. A build may only be reused by readers at its own revision, and may
    * only populate the cache if no mutation happened while it ran.
    */
@@ -491,9 +523,9 @@ export class InMemoryRouteSnapshotService implements RouteSnapshotService {
           ]),
         )
       : undefined;
-    // One literal for both routing modes. These used to be two verbatim copies
-    // that differed only by the `providerRouting` key, so a snapshot field added
-    // to one branch would silently vanish on the other.
+    // One literal for both routing modes: two verbatim copies differing only by
+    // the `providerRouting` key would mean a snapshot field added to one branch
+    // silently vanishes on the other.
     const snapshot = deepFreeze({
       revision,
       candidates: Object.freeze([...built.candidates]) as readonly RouteCandidate[],
@@ -508,8 +540,8 @@ export class InMemoryRouteSnapshotService implements RouteSnapshotService {
     });
     // A mutation during the build means this snapshot describes a catalog that
     // has already been superseded. Returning it is fine — the caller asked
-    // before the mutation — but caching it would strand the data plane on the
-    // old routing until the next write.
+    // before the mutation — but caching it would strand the data plane on stale
+    // routing until the next write.
     if (revision === this.revision) this.snapshot = snapshot;
     return snapshot;
   }