@opengeni/runtime 4.0.1 → 4.0.3-canary.36199476632001

package/src/lazy-tool-transport.ts

@@ -38,6 +38,10 @@ const ALWAYS_VISIBLE_BASE_TOOL_NAMES: ReadonlySet<string> = new Set([
   "repository_skill_read",
   "request_human_input",
   "list_models",
+  // Jev-backed code search. Present only when the deployment and workspace
+  // enable it; hiding it behind search would spend a model round trip to find
+  // the tool that exists to save round trips.
+  "code_search",
 ]);
 const DISPATCH_MARKER_KEY = "opengeni.lazy_dispatch.v1";
 const SEARCH_MARKER_KEY = "opengeni.lazy_search.v1";

package/src/metrics.ts

@@ -15,6 +15,23 @@ export const MCP_LIFECYCLE_PHASES = ["connect", "close"] as const;
 export const MCP_LIFECYCLE_POLICIES = ["strict", "best_effort"] as const;
 export const MCP_LIFECYCLE_OUTCOMES = ["completed", "failed"] as const;
 
+/**
+ * Closed outcomes of one fresh-box command-readiness replacement decision:
+ * the replacement became ready, it also missed readiness, it failed for another
+ * typed reason, cancellation won the pause or the replacement, or the turn
+ * attempt had already spent its single replacement.
+ */
+export const SANDBOX_READINESS_REPLACEMENT_OUTCOMES = [
+  "replaced",
+  "failed_again",
+  "replacement_failed",
+  "cancelled",
+  "budget_spent",
+] as const;
+
+export type SandboxReadinessReplacementOutcome =
+  (typeof SANDBOX_READINESS_REPLACEMENT_OUTCOMES)[number];
+
 export type McpLifecyclePhase = (typeof MCP_LIFECYCLE_PHASES)[number];
 export type McpLifecyclePolicy = (typeof MCP_LIFECYCLE_POLICIES)[number];
 export type McpLifecycleOutcome = (typeof MCP_LIFECYCLE_OUTCOMES)[number];
@@ -35,6 +52,15 @@ export type RuntimeMetricsHooks = {
     backend: string;
     stage: "exec_readiness" | "sibling_warming";
   }) => void;
+  /**
+   * One fresh-box readiness replacement decision. The first readiness miss is
+   * still counted by `onSandboxWarmingTimeout`; this hook says whether the
+   * single per-turn-attempt replacement happened and whether it worked.
+   */
+  onSandboxReadinessReplacement?: (input: {
+    backend: string;
+    outcome: SandboxReadinessReplacementOutcome;
+  }) => void;
   onSandboxProviderApiThrottle?: (input: {
     backend: string;
     operation: "create" | "renew";

package/src/model-context-inspector.ts

@@ -101,6 +101,7 @@ const LAYER_TITLES: Record<ModelContextInstructionLayerId, string> = {
   session_instructions: "Session instructions",
   workspace_memory: "Workspace memory",
   codemode: "Codemode",
+  code_search: "Code search",
   git_bindings: "Git credential bindings",
   genesis_title: "Missing-title directive",
   sdk_capability_instructions: "SDK capability instructions",

package/src/model-provider-client.ts

@@ -28,6 +28,7 @@ import { isModelCallFetch, vercelGatewayRoutingFetch } from "./model-provider-tr
 import { ReplayableJsonOpenAI } from "./replayable-json-body";
 import { recordModelTransportStarted } from "./model-preparation-diagnostics";
 import { captureProviderRequestBody } from "./model-request-capture";
+import { withoutQuotaExhaustedRetries } from "./provider-quota";
 
 let runtimeMetricsHooks: RuntimeMetricsHooks | null = null;
 
@@ -85,7 +86,10 @@ export function buildOpenAIClientFromSettings(
           settings.azureOpenaiAdToken && !settings.azureOpenaiApiKey
             ? { Authorization: `Bearer ${settings.azureOpenaiAdToken}` }
             : undefined,
-        fetch: instrumentedModelFetch(providerId, globalThis.fetch),
+        fetch: sdkRetryingModelFetch(
+          settings.openaiMaxRetries,
+          instrumentedModelFetch(providerId, globalThis.fetch),
+        ),
       },
       { modelRequestPolicy: azureModelRequestPolicy },
     );
@@ -94,10 +98,23 @@ export function buildOpenAIClientFromSettings(
     apiKey: settings.openaiApiKey ?? process.env.OPENAI_API_KEY,
     ...(settings.openaiBaseUrl ? { baseURL: settings.openaiBaseUrl } : {}),
     maxRetries: settings.openaiMaxRetries,
-    fetch: instrumentedModelFetch(providerId, globalThis.fetch),
+    fetch: sdkRetryingModelFetch(
+      settings.openaiMaxRetries,
+      instrumentedModelFetch(providerId, globalThis.fetch),
+    ),
   });
 }
 
+/**
+ * A client that lets the OpenAI SDK retry 429s must not replay an exhausted
+ * provider quota: the refusal cannot clear within the SDK's backoff, and each
+ * retry only delays the typed turn failure. Clients with SDK retries disabled
+ * need no veto.
+ */
+function sdkRetryingModelFetch(maxRetries: number, inner: typeof fetch): typeof fetch {
+  return maxRetries > 0 ? withoutQuotaExhaustedRetries(inner) : inner;
+}
+
 /**
  * One OpenAI client per deployment-scoped provider configuration, built lazily
  * and cached for the process. Workspace-connection providers are deliberately
@@ -282,6 +299,11 @@ export function buildProviderClient(provider: ResolvedModelProvider, settings: S
     throw new WorkspaceGatewayUnavailableError();
   }
   const anonymousProvider = provider.kind === "anonymous";
+  // Gateway and OpenRouter requests can incur upstream work or charges
+  // before a retryable response failure reaches this process. Neither
+  // transport has a provider idempotency key tied to our durable call,
+  // so never let the SDK replay them blindly.
+  const registryMaxRetries = gatewayProvider || openRouterProvider ? 0 : settings.openaiMaxRetries;
   const client = provider.builtin
     ? buildOpenAIClientFromSettings(settings, provider.id)
     : provider.kind === "codex-subscription"
@@ -342,24 +364,23 @@ export function buildProviderClient(provider: ResolvedModelProvider, settings: S
                   ? { apiKey: provider.apiKey }
                   : {}),
               ...(provider.baseUrl ? { baseURL: provider.baseUrl } : {}),
-              // Gateway and OpenRouter requests can incur upstream work or charges
-              // before a retryable response failure reaches this process. Neither
-              // transport has a provider idempotency key tied to our durable call,
-              // so never let the SDK replay them blindly.
-              maxRetries: gatewayProvider || openRouterProvider ? 0 : settings.openaiMaxRetries,
+              maxRetries: registryMaxRetries,
               ...(provider.defaultQuery ? { defaultQuery: provider.defaultQuery } : {}),
               ...(provider.defaultHeaders ? { defaultHeaders: provider.defaultHeaders } : {}),
-              fetch: anonymousProvider
-                ? withoutAuthenticationHeaders(
-                    instrumentedModelFetch(provider.id, globalThis.fetch),
-                  )
-                : gatewayProvider
-                  ? vercelGatewayRoutingFetch(
-                      provider.kind as "vercel-gateway-managed" | "vercel-gateway-workspace",
+              fetch: sdkRetryingModelFetch(
+                registryMaxRetries,
+                anonymousProvider
+                  ? withoutAuthenticationHeaders(
                       instrumentedModelFetch(provider.id, globalThis.fetch),
-                      gatewayPolicies,
                     )
-                  : instrumentedModelFetch(provider.id, globalThis.fetch),
+                  : gatewayProvider
+                    ? vercelGatewayRoutingFetch(
+                        provider.kind as "vercel-gateway-managed" | "vercel-gateway-workspace",
+                        instrumentedModelFetch(provider.id, globalThis.fetch),
+                        gatewayPolicies,
+                      )
+                    : instrumentedModelFetch(provider.id, globalThis.fetch),
+              ),
             },
             { modelRequestPolicy: modelRequestPolicyForProvider(provider, gatewayPolicies) },
           );

package/src/model-provider-routing.ts

@@ -163,6 +163,16 @@ export class MultiProviderModelProvider implements ModelProvider {
   constructor(private readonly settings: Settings) {}
 
   async getModel(modelName?: string): Promise<Model> {
+    return this.resolveBinding(modelName).model;
+  }
+
+  /**
+   * The provider, owned client, and upstream model id that `getModel` binds.
+   * Standalone requests outside an agent run (the session-title sidecar) use
+   * this to call the provider directly instead of the runner-facing
+   * `Model.getResponse()`, which requires an active trace.
+   */
+  resolveBinding(modelName?: string): ModelProviderBinding {
     if (modelName) {
       const resolved = resolveTurnModel(
         settingsForRunScopedModelResolution(this.settings, modelName),
@@ -189,7 +199,12 @@ export class MultiProviderModelProvider implements ModelProvider {
         ) {
           throw new XaiSubscriptionUnavailableError(modelName);
         }
-        return resolved.model;
+        return {
+          provider: resolved.provider,
+          client: resolved.client,
+          model: resolved.model,
+          modelId: resolved.configured.upstreamModelId,
+        };
       }
       // A `codex/<slug>` id only resolves when the per-workspace worker overlay
       // (settingsWithCodexCredential) has injected the synthetic codex-subscription
@@ -215,14 +230,25 @@ export class MultiProviderModelProvider implements ModelProvider {
     // JSON parse/stringify transport wrapper on the fallback path.
     const builtin = configuredProviders(this.settings)[0];
     if (!builtin) throw new Error("Built-in model provider is unavailable");
-    return new OpenGeniResponsesModel(
-      buildProviderClient(builtin, this.settings),
-      modelName ?? this.settings.openaiModel,
-      builtin,
-    );
+    const client = buildProviderClient(builtin, this.settings);
+    const modelId = modelName ?? this.settings.openaiModel;
+    return {
+      provider: builtin,
+      client,
+      model: new OpenGeniResponsesModel(client, modelId, builtin),
+      modelId,
+    };
   }
 }
 
+export type ModelProviderBinding = {
+  provider: ResolvedModelProvider;
+  client: OpenAI;
+  model: Model;
+  /** The id sent on the provider wire (the upstream id for a registry model). */
+  modelId: string;
+};
+
 function settingsForRunScopedModelResolution(settings: Settings, modelName: string): Settings {
   if (modelName !== settings.openaiModel) {
     return settings;

package/src/operational-instructions.ts

@@ -156,7 +156,7 @@ Skills are reusable instructions supplied dynamically for the current session. W
 
 # Integration setup
 
-Use available integration tools directly. If the task needs an integration you cannot access, discover it with \`capability_catalog_search\` before declaring a setup blocker. For a suitable match with \`setup.nextAction\`, call \`capability_authorization_request\` with the returned capability ID and a brief task-specific rationale to show its Connect card in chat. Requesting the card does not need integration-management permission; the authenticated human must authorize setup. A card is for an integration required by the authorized design, including established or delegated choices. Resolve out-of-scope architecture choices before requesting setup. After setup, rediscover the tools and continue the task, verifying the access it needs. If access remains blocked, explain the specific blocker from the returned facts. If either setup tool is unavailable, report the missing setup path.
+Use available integration tools directly. If access is missing, search \`capability_catalog_search\` first. For a suitable match with \`setup.nextAction\`, call \`capability_authorization_request\` with its ID and a task-specific rationale. If no match exists and the task needs a remote MCP whose exact HTTPS URL the user supplied or reliable documentation establishes, call \`custom_mcp_setup_request\` with its name, URL, and rationale. Never invent URLs or request credentials in chat. Showing either card does not need integration-management permission and grants no access; the authenticated human must authorize setup. A card is for an integration required by the authorized design, including established or delegated choices. Resolve out-of-scope architecture choices before requesting setup. After setup, rediscover tools and verify access. If blocked or a setup tool is unavailable, explain the specific gap.
 
 # Session coordination
 

package/src/provider-quota.ts

@@ -0,0 +1,412 @@
+import { APIError } from "openai";
+
+/**
+ * Provider quota exhaustion versus ordinary rate limiting.
+ *
+ * Model providers use HTTP 429 for two different conditions. An ordinary rate
+ * limit (requests or tokens per minute) clears within seconds, so pacing the
+ * same request again is correct. An exhausted quota (a daily or monthly
+ * allowance, a free-tier day cap, or an account with no credits left) cannot
+ * clear inside a bounded retry budget, so every retry only delays the failure.
+ *
+ * This module is the single classifier for API-key provider responses. It
+ * reads only provider-owned evidence: the HTTP status, provider message/code
+ * strings, and the provider's own retry hint. The OpenAI SDK retry veto and the
+ * worker's turn classification both read that evidence from the same thrown
+ * SDK error shape (`classifyProviderQuotaError`), so one response can never be
+ * classified two ways. Subscription transports (Codex, SuperGrok) own their
+ * quota semantics through credential rotation and durable capacity waits and
+ * must not route through it.
+ */
+
+export const PROVIDER_QUOTA_EXHAUSTED_CODE = "provider_quota_exhausted";
+
+/** What ran out. Chooses plain-language copy only; every scope is non-retryable. */
+export type ProviderQuotaScope = "daily" | "monthly" | "credits" | "quota";
+
+export type ProviderQuotaExhaustion = { scope: ProviderQuotaScope };
+
+export type ProviderQuotaEvidence = {
+  /** HTTP status when one survived the error wrappers; null when statusless. */
+  status: number | null;
+  /** Provider-authored message, code and type strings, in any order. */
+  texts: readonly string[];
+  /** Provider retry hint from a header or structured field, in milliseconds. */
+  retryAfterMs: number | null;
+};
+
+/** A provider-stated wait at or below this is an ordinary short rate limit. */
+export const PROVIDER_SHORT_RETRY_HINT_MS = 60_000;
+
+/**
+ * Without any quota wording, only a provider-stated wait beyond this proves a
+ * quota window rather than per-minute pacing.
+ */
+export const PROVIDER_QUOTA_RETRY_HINT_MS = 15 * 60_000;
+
+// Bound the text inspected per field and in total. Provider errors are short;
+// a pathological body must not turn failure classification into a hot loop.
+const MAX_FIELD_CHARS = 4_096;
+const MAX_FIELDS = 64;
+
+// Account-level exhaustion that nothing clears in seconds. Structured codes are
+// matched exactly; wording is matched as a phrase. OpenAI's
+// `billing_hard_limit_reached` is the organization's own spending cap.
+const CREDIT_CODES = new Set([
+  "insufficient_credits",
+  "insufficient_balance",
+  "billing_hard_limit_reached",
+]);
+const QUOTA_CODES = new Set(["insufficient_quota"]);
+const CREDIT_TEXT =
+  /\binsufficient[ _](?:credits?|balance|funds)\b|\bcredit balance is too low\b|\brequires more credits\b|\bout of credits\b|\bused all (?:of )?(?:your |the )?(?:available )?credits\b|\bspending limit\b|\bbilling[ _]hard[ _]limit\b|\breached your specified api usage limits?\b/;
+
+// Generic quota wording, including a reached "usage limit". Google reports
+// per-minute limits with the same "exceeded your current quota" sentence, so
+// this alone is not decisive: an explicit per-minute scope or a short provider
+// retry hint keeps it retryable.
+// Google's `RESOURCE_EXHAUSTED` status is deliberately absent: it accompanies
+// every Google 429, including short dynamic-shared-quota capacity refusals, so
+// it only marks the response as rate-shaped.
+const QUOTA_TEXT =
+  /\bexceeded (?:your |the )?(?:current |allotted |daily |monthly )?quota\b|\bquota (?:has been |was |is )?(?:exceeded|exhausted|reached)\b|\bout of (?:call volume )?quota\b|\bquota[ _]?exceeded\b|\b(?:hit|reached|exceeded) (?:your |the |its )?(?:[a-z]+ )?usage limits?\b/;
+
+// Explicit limit windows. "free-models-per-min" is OpenRouter's per-minute
+// free tier; "free-models-per-day" its daily cap. GitHub/Azure AI inference
+// name windows like "UserByModelByDay" or "per 86400s", and Google/Vertex
+// metric ids join words with underscores
+// ("generate_content_requests_per_minute_per_project"). Letter lookarounds
+// rather than `\b` keep snake_case ids matching, since `_` is a word character.
+const MINUTE_SCOPE =
+  /(?<![a-z])per[ _-]?min(?:ute)?s?(?![a-z])|perminute|byminute|(?<![a-z])(?:rpm|tpm)(?![a-z])|\/min(?![a-z])|\bper 60 ?s\b|(?<![a-z])per[ _-]?second(?![a-z])/;
+const DAY_SCOPE =
+  /(?<![a-z])per[ _-]?day(?![a-z])|perday|byday|(?<![a-z])(?:rpd|tpd)(?![a-z])|(?<![a-z])daily(?![a-z])|\bper 86400 ?s\b/;
+const MONTH_SCOPE = /(?<![a-z])per[ _-]?month(?![a-z])|permonth|bymonth|(?<![a-z])monthly(?![a-z])/;
+
+const RATE_TEXT = /too many requests|rate.?limit|\b429\b|\bresource[ _]exhausted\b/;
+
+const UNIT_MS: Record<string, number> = {
+  ms: 1,
+  millisecond: 1,
+  milliseconds: 1,
+  s: 1_000,
+  sec: 1_000,
+  secs: 1_000,
+  second: 1_000,
+  seconds: 1_000,
+  m: 60_000,
+  min: 60_000,
+  mins: 60_000,
+  minute: 60_000,
+  minutes: 60_000,
+  h: 3_600_000,
+  hr: 3_600_000,
+  hrs: 3_600_000,
+  hour: 3_600_000,
+  hours: 3_600_000,
+  d: 86_400_000,
+  day: 86_400_000,
+  days: 86_400_000,
+};
+
+const DURATION_TOKEN =
+  /^\s*(\d{1,9}(?:\.\d{1,12})?)\s*(milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)(?![a-z])/i;
+
+/** Parse a leading compound duration such as "16m32.64s", "6 seconds" or "7 days". */
+function leadingDurationMs(text: string): number | null {
+  let rest = text;
+  let total = 0;
+  let matched = false;
+  for (let index = 0; index < 4; index += 1) {
+    const token = DURATION_TOKEN.exec(rest);
+    if (!token) break;
+    const unit = UNIT_MS[token[2]!.toLowerCase()];
+    if (unit === undefined) break;
+    total += Number(token[1]) * unit;
+    matched = true;
+    rest = rest.slice(token[0].length);
+  }
+  return matched && Number.isFinite(total) ? Math.ceil(total) : null;
+}
+
+/**
+ * Read a retry hint a provider wrote into its message ("Please retry after 6
+ * seconds", "Please try again in 16m32.64s", "Please wait 42567 seconds before
+ * retrying"). Used only to tell a short limit from an exhausted quota; it never
+ * paces a retry.
+ */
+export function providerMessageRetryHintMs(text: string): number | null {
+  let longest: number | null = null;
+  const prefixes = /\b(?:retry|try again)\s+(?:after|in)\s+|\bwait\s+/gi;
+  for (const match of text.slice(0, MAX_FIELD_CHARS * 4).matchAll(prefixes)) {
+    const start = (match.index ?? 0) + match[0].length;
+    const duration = leadingDurationMs(text.slice(start, start + 64));
+    if (duration !== null) longest = longest === null ? duration : Math.max(longest, duration);
+  }
+  return longest;
+}
+
+/**
+ * Decide whether a provider refusal is an exhausted quota. Returns null for an
+ * ordinary short rate limit and for anything that is not provider quota
+ * evidence at all (a sandbox "disk quota exceeded", an OpenGeni credit
+ * refusal, a 5xx), so those keep their existing classification.
+ */
+export function classifyProviderQuotaExhaustion(
+  evidence: ProviderQuotaEvidence,
+): ProviderQuotaExhaustion | null {
+  const fields = evidence.texts
+    .slice(0, MAX_FIELDS)
+    .filter((value): value is string => typeof value === "string" && value.length > 0)
+    .map((value) => value.slice(0, MAX_FIELD_CHARS).toLowerCase());
+  const text = fields.join("\n");
+  if (text.includes("opengeni credits")) return null;
+
+  const explicitStatus =
+    evidence.status !== null && Number.isInteger(evidence.status) ? evidence.status : null;
+  // SDK wrappers sometimes keep only the "429 ..." message prefix.
+  const prefixedStatus = fields
+    .map((value) => /^\s*(4\d\d)\b/.exec(value)?.[1])
+    .find((value) => value !== undefined);
+  const status = explicitStatus ?? (prefixedStatus ? Number(prefixedStatus) : null);
+  // Only a rate-limit or payment refusal can be quota exhaustion. Every other
+  // status keeps its existing classification (a 5xx is transient, other 4xx
+  // are already terminal request faults).
+  if (status !== null && status !== 429 && status !== 402) return null;
+
+  const hasCode = (codes: ReadonlySet<string>) => fields.some((value) => codes.has(value.trim()));
+  const creditCode = hasCode(CREDIT_CODES);
+  const quotaCode = hasCode(QUOTA_CODES);
+  const rateShaped = status === 429 || RATE_TEXT.test(text);
+  // A statusless error must still look like a provider refusal; free text such
+  // as a sandbox "Disk quota exceeded" is not model-provider evidence.
+  if (status === null && !rateShaped && !creditCode && !quotaCode) return null;
+
+  // Payment Required and account credit/billing exhaustion never clear by
+  // waiting, whatever retry hint accompanies them.
+  if (status === 402 || creditCode || CREDIT_TEXT.test(text)) return { scope: "credits" };
+  if (quotaCode) return { scope: "quota" };
+
+  const hint = evidence.retryAfterMs ?? providerMessageRetryHintMs(text);
+  const minute = MINUTE_SCOPE.test(text);
+  const day = DAY_SCOPE.test(text);
+  const month = MONTH_SCOPE.test(text);
+  const quotaWording = QUOTA_TEXT.test(text);
+
+  if (!day && !month && !quotaWording) {
+    // No quota wording: only a provider wait far beyond per-minute pacing
+    // proves an exhausted window.
+    return hint !== null && hint > PROVIDER_QUOTA_RETRY_HINT_MS ? { scope: "quota" } : null;
+  }
+  // An explicit per-minute window is ordinary pacing even when the provider's
+  // message also advertises its daily allowance.
+  if (minute) return null;
+  // The provider itself says the limit clears within a minute.
+  if (hint !== null && hint <= PROVIDER_SHORT_RETRY_HINT_MS) return null;
+  if (day) return { scope: "daily" };
+  if (month) return { scope: "monthly" };
+  return { scope: "quota" };
+}
+
+/** Plain-language turn failure copy for an exhausted provider quota. */
+export function providerQuotaExhaustedMessage(scope: ProviderQuotaScope): string {
+  switch (scope) {
+    case "daily":
+      return "This model's daily limit at the model provider has been reached, so automatic retries stopped. Choose another model, or try again after the limit resets.";
+    case "monthly":
+      return "This model's monthly limit at the model provider has been reached, so automatic retries stopped. Choose another model, or try again after the limit resets.";
+    case "credits":
+      return "The model provider account for this model is out of credits, so automatic retries stopped. Choose another model, or add credits with the provider and try again.";
+    case "quota":
+      return "The model provider's usage quota for this model is used up, so automatic retries stopped. Choose another model, or try again after the quota resets.";
+  }
+}
+
+const MAX_RESPONSE_BODY_BYTES = 64 * 1024;
+
+/** Read at most `maxBytes` of a cloned body; the original stays unread. */
+async function boundedCloneText(response: Response, maxBytes: number): Promise<string | null> {
+  const body = response.clone().body;
+  if (!body) return "";
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let text = "";
+  let bytes = 0;
+  try {
+    for (;;) {
+      const chunk = await reader.read();
+      if (chunk.done) break;
+      bytes += chunk.value.byteLength;
+      if (bytes > maxBytes) return null;
+      text += decoder.decode(chunk.value, { stream: true });
+    }
+    return text + decoder.decode();
+  } finally {
+    await reader.cancel().catch(() => undefined);
+  }
+}
+
+function statusOf(value: Record<string, unknown>): number | null {
+  const body =
+    value.error && typeof value.error === "object"
+      ? (value.error as Record<string, unknown>)
+      : null;
+  const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
+  return Number.isInteger(status) && status >= 100 && status <= 599 ? status : null;
+}
+
+/** The first HTTP status along an error and its cause chain. */
+function errorHttpStatus(error: unknown): number | null {
+  let current: unknown = error;
+  for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
+    const status = statusOf(current as Record<string, unknown>);
+    if (status !== null) return status;
+    current = (current as Record<string, unknown>).cause;
+  }
+  return null;
+}
+
+/**
+ * Provider-authored strings on an SDK error and its wrappers: message, code,
+ * type, name and param, recursing through `error`, `cause`, `response` and
+ * `data`. These are exactly the fields the worker's other failure classifiers
+ * read, so quota evidence never depends on a key only one reader sees.
+ */
+function errorTexts(value: unknown, out: string[], seen: WeakSet<object>): void {
+  if (out.length >= MAX_FIELDS) return;
+  if (typeof value === "string") {
+    out.push(value);
+    return;
+  }
+  if (!value || typeof value !== "object" || seen.has(value)) return;
+  seen.add(value);
+  const record = value as Record<string, unknown>;
+  for (const key of ["message", "code", "type", "name", "param"]) {
+    const field = record[key];
+    if (typeof field === "string" && field.length > 0) out.push(field);
+  }
+  for (const key of ["error", "cause", "response", "data"]) errorTexts(record[key], out, seen);
+}
+
+function headerValue(headers: unknown, name: string): string | null {
+  if (!headers || typeof headers !== "object") return null;
+  const getter = (headers as { get?: unknown }).get;
+  if (typeof getter === "function") {
+    const value = getter.call(headers, name);
+    return typeof value === "string" ? value : null;
+  }
+  const entry = Object.entries(headers as Record<string, unknown>).find(
+    ([key, value]) => key.toLowerCase() === name && typeof value === "string",
+  );
+  return typeof entry?.[1] === "string" ? entry[1] : null;
+}
+
+/** A provider retry hint (`retry-after-ms`, `retry-after`, or a structured field) in milliseconds. */
+function errorRetryAfterMs(error: unknown, nowMs: number): number | null {
+  let current: unknown = error;
+  for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
+    const value = current as Record<string, unknown>;
+    const body =
+      value.error && typeof value.error === "object"
+        ? (value.error as Record<string, unknown>)
+        : null;
+    for (const headers of [value.headers, value.responseHeaders, body?.headers]) {
+      const millis = Number(headerValue(headers, "retry-after-ms") ?? Number.NaN);
+      if (Number.isFinite(millis) && millis > 0) return Math.ceil(millis);
+      const header = headerValue(headers, "retry-after");
+      if (header === null) continue;
+      const seconds = Number(header);
+      if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
+      const date = Date.parse(header);
+      if (Number.isFinite(date) && date > nowMs) return Math.ceil(date - nowMs);
+    }
+    const seconds = Number(
+      value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
+    );
+    if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
+    current = value.cause;
+  }
+  return null;
+}
+
+/**
+ * Classify a thrown provider error: an OpenAI SDK `APIError`, or any wrapper
+ * whose `error`/`cause` chain reaches one. This is the one evidence reader for
+ * both the SDK retry veto and the worker's turn failure.
+ */
+export function classifyProviderQuotaError(
+  error: unknown,
+  nowMs = Date.now(),
+): ProviderQuotaExhaustion | null {
+  try {
+    const texts: string[] = [];
+    errorTexts(error, texts, new WeakSet());
+    return classifyProviderQuotaExhaustion({
+      status: errorHttpStatus(error),
+      texts,
+      retryAfterMs: errorRetryAfterMs(error, nowMs),
+    });
+  } catch {
+    // A hostile getter is not quota evidence; callers keep their existing path.
+    return null;
+  }
+}
+
+/**
+ * Classify a raw provider HTTP response without consuming its body. It builds
+ * the exact error the OpenAI SDK throws for this response (the same
+ * `APIError.generate` inputs as the SDK's own failure path) and classifies
+ * that, so the veto and the worker read identical evidence.
+ */
+export async function classifyProviderQuotaResponse(
+  response: Response,
+  nowMs = Date.now(),
+): Promise<ProviderQuotaExhaustion | null> {
+  if (response.status !== 429) return null;
+  const text = await boundedCloneText(response, MAX_RESPONSE_BODY_BYTES);
+  if (text === null) return null;
+  let json: unknown;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    json = undefined;
+  }
+  const sdkError = APIError.generate(
+    response.status,
+    json as object | undefined,
+    json ? undefined : text,
+    response.headers,
+  );
+  return classifyProviderQuotaError(sdkError, nowMs);
+}
+
+/**
+ * Tell the OpenAI SDK not to replay an exhausted-quota 429. The SDK otherwise
+ * retries every 429 up to `maxRetries` times, which only delays a refusal that
+ * cannot clear. The SDK honors the standard-for-it `x-should-retry` header, so
+ * the exact response (status, headers, body) is passed through with only that
+ * header added. Ordinary rate limits and every other response are untouched.
+ */
+export function withoutQuotaExhaustedRetries(inner: typeof fetch): typeof fetch {
+  return (async (input: Parameters<typeof fetch>[0], init?: Parameters<typeof fetch>[1]) => {
+    const response = await inner(input, init);
+    if (response.status !== 429 || response.headers.has("x-should-retry")) return response;
+    let exhausted: ProviderQuotaExhaustion | null = null;
+    try {
+      exhausted = await classifyProviderQuotaResponse(response);
+    } catch {
+      // Classification is advisory; the SDK keeps its ordinary retry decision.
+    }
+    if (!exhausted) return response;
+    const headers = new Headers(response.headers);
+    headers.set("x-should-retry", "false");
+    const vetoed = new Response(response.body, {
+      status: response.status,
+      statusText: response.statusText,
+      headers,
+    });
+    // Keep the request URL the SDK reports in its request logs.
+    if (response.url) Object.defineProperty(vetoed, "url", { value: response.url });
+    return vetoed;
+  }) as typeof fetch;
+}