@providerkit/core 0.22.0 → 0.23.0

package/src/providers/responses.ts

@@ -10,8 +10,11 @@
 // The event names arrive on the SSE `event:` line and are repeated inside each
 // payload's own `type`. The transport yields only `data:` payloads, so this
 // adapter reads `type` — which is what survives, and what gateways agree on.
+import { repairHistory } from "../context.ts";
 import { fileRefused, ProviderError, streamCut, streamError } from "../errors.ts";
-import { streamSse, apiUrl } from "../transport.ts";
+import { streamSse, apiUrl, parseFrame } from "../transport.ts";
+import { parseRateLimitReset } from "../rate-limit.ts";
+import { settleToolCalls } from "../tool-args.ts";
 import type {
   ChatMessage,
   ContentPart,
@@ -82,13 +85,26 @@ function reasoningEffort(model: string, effort: Effort): string {
 }
 
 /**
- * `response.incomplete` means the turn was cut short, and the seam has one word
- * for that: "length". `content_filter` is the only other reason this shape
- * documents; anything new stays on "length" rather than reporting a clean stop,
- * because a caller that believes a truncated answer finished will act on it.
+ * The finish an `incomplete` turn stands for. This shape documents two
+ * reasons, and each has its own finish: `max_output_tokens` is the output cap
+ * (`length`) and `content_filter` the filter.
+ *
+ * Any other reason, or none, says only that the turn did not end on purpose.
+ * It used to read as `length`, so an empty turn was told to raise a maxTokens
+ * it never hit. It is thrown as `overload` instead, the transient fault
+ * cc-proxy answers it with (a 503). Nothing about the request is wrong, so it
+ * is retried if nothing was emitted yet; if something was, the error reaches
+ * the caller after it (invariant 1). It is read before reasoning and usage go
+ * out, like a failed terminal, so that retry stays open.
  */
-function mapIncompleteReason(reason: string | undefined): FinishReason {
-  return reason === "content_filter" ? "content_filter" : "length";
+function incompleteFinish(provider: string, reason: string | undefined): FinishReason {
+  if (reason === "max_output_tokens") return "length";
+  if (reason === "content_filter") return "content_filter";
+  throw new ProviderError(
+    provider,
+    "overload",
+    `${provider}: the turn stopped early, reason: ${reason ?? "none given"}`,
+  );
 }
 
 // ── input items ───────────────────────────────────────────────────────────
@@ -282,8 +298,11 @@ interface ResponsesEvent {
   headers?: Record<string, unknown>;
 }
 
+/** One window of the snapshot: the `x-codex-<slot>-*` header numbers as JSON. */
 interface CodexWindow {
   used_percent?: number;
+  window_minutes?: number;
+  reset_at?: number;
   reset_after_seconds?: number;
 }
 
@@ -301,24 +320,42 @@ interface CodexRateLimits {
  * A full window the account's credits cover, or that the backend still
  * `allowed`, lets the turn proceed. It is only the reason when the stream then
  * closes without a response: that close is a spent window, not a dropped
- * socket, and retrying it walks every backup into the same wall. The fuller
- * window binds; cc-proxy took the primary's clock first and told a weekly wall
- * to come back in a few hours.
+ * socket, and retrying it walks every backup into the same wall.
+ *
+ * Which window, and when it lifts, comes from the same parser as the headers
+ * the snapshot mirrors: the fuller window binds, and the later reset when both
+ * are full. cc-proxy took the primary's clock first and told a weekly wall to
+ * come back in a few hours. The window is named so a caller can say "weekly"
+ * instead of "the limit", which reads as the 5-hour one.
  */
 function quotaWall(event: ResponsesEvent, provider: string): ProviderError | undefined {
   const limits = event.rate_limits;
   if (!limits?.limit_reached || limits.allowed) return undefined;
   if (event.credits?.has_credits || event.credits?.unlimited) return undefined;
-  const windows = [limits.primary, limits.secondary].filter((w): w is CodexWindow => !!w);
-  const binding = windows.sort((a, b) => (b.used_percent ?? 0) - (a.used_percent ?? 0))[0];
-  const after = binding?.reset_after_seconds;
-  return new ProviderError(provider, "quota", `${provider}: the usage limit has been reached`, {
-    ...(after !== undefined && after >= 0
-      ? { retryAfterMs: after * 1000, resetAtMs: Date.now() + after * 1000 }
-      : {}),
+  const now = Date.now();
+  const { window, resetAtMs } = parseRateLimitReset(codexHeaders(limits), now);
+  const named = window ? ` (${window} window)` : "";
+  const message = `${provider}: the usage limit has been reached${named}`;
+  return new ProviderError(provider, "quota", message, {
+    ...(resetAtMs !== undefined ? { retryAfterMs: resetAtMs - now, resetAtMs } : {}),
+    window,
   });
 }
 
+/** The snapshot as the `x-codex-*` headers it mirrors, rebuilt the way
+ *  cc-proxy's `event_headers` does, so one parser reads both. */
+function codexHeaders(limits: CodexRateLimits): Headers {
+  const headers = new Headers();
+  for (const slot of ["primary", "secondary"] as const) {
+    for (const [field, value] of Object.entries(limits[slot] ?? {})) {
+      if (typeof value === "number") {
+        headers.set(`x-codex-${slot}-${field.replaceAll("_", "-")}`, String(value));
+      }
+    }
+  }
+  return headers;
+}
+
 function usageChunk(usage: ResponsesUsage): ProviderChunk {
   return {
     type: "usage",
@@ -373,8 +410,11 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
       const model = opts.model ?? config.model;
       const museContributor = isMuseSparkContributor(model);
       const requestTools = museContributor && opts.toolChoice === "none" ? [] : tools;
+      // Repaired before the images are gated: an orphaned result becomes a user
+      // note that keeps its images, and those must meet the limits too.
+      const sent = repairHistory(messages).messages;
       const { instructions, input } = toResponsesInput(
-        config.imageLimits ? gateImages(messages, config.imageLimits) : messages,
+        config.imageLimits ? gateImages(sent, config.imageLimits) : sent,
         id,
       );
 
@@ -457,9 +497,10 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
       const pendingFor = (itemId?: string, callId?: string) =>
         (itemId ? pending.get(itemId) : undefined) ?? (callId ? pending.get(callId) : undefined);
       let nextIndex = 0;
-      // This shape never states a stop reason on a clean finish, so it is
-      // inferred from whether the turn produced a function call.
-      let sawToolCall = false;
+      // Every call the turn opened, kept past its `.done`. This shape never
+      // states a stop reason on a clean finish, so it is inferred from whether
+      // there are any, and the terminal event checks each one can run.
+      const calls: PendingCall[] = [];
       let wall: ProviderError | undefined;
       let produced = false;
       const reasoningItems: unknown[] = [];
@@ -482,13 +523,9 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
         ...(opts.signal ? { signal: opts.signal } : {}),
         ...(config.fetchImpl ? { fetchImpl: config.fetchImpl } : {}),
         ...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
+        ...(opts.onRateLimit ? { onRateLimit: opts.onRateLimit } : {}),
       })) {
-        let event: ResponsesEvent;
-        try {
-          event = JSON.parse(data) as ResponsesEvent;
-        } catch {
-          continue; // a keep-alive or a frame we do not model
-        }
+        const event = parseFrame<ResponsesEvent>(data, id);
         if (event.type?.endsWith(".delta") || event.type === "response.output_item.added") {
           produced = true;
         }
@@ -515,7 +552,6 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
             // turn came back as a nameless call with empty arguments.
             const key = item.id ?? item.call_id;
             if (!key) break;
-            sawToolCall = true;
             const index = nextIndex++;
             // Normally empty here, but a backend that already has the whole
             // call sends it in the skeleton.
@@ -526,6 +562,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
               name: item.name ?? "",
               streamed: seeded,
             };
+            calls.push(call);
             pending.set(key, call);
             if (item.call_id) pending.set(item.call_id, call);
             // Only the fields the skeleton actually states. An empty `id` here
@@ -582,20 +619,24 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
 
             if (!known) {
               // A backend that emits neither the skeleton nor the deltas — the
-              // whole call arrives here or not at all. Without a `call_id`
-              // there is nothing to answer it with: the caller's
-              // `function_call_output` would quote `""` and take a 400 reading
-              // "No tool output found for function call" one turn later, so the
-              // call is dropped rather than handed over unrunnable. Dropping is
-              // only possible here, where nothing has been streamed for it yet.
-              if (!item.call_id) break;
-              sawToolCall = true;
+              // whole call arrives here or not at all. A call with no `call_id`
+              // used to be dropped, because the caller's `function_call_output`
+              // would quote `""` and take a 400 one turn later. Dropped, the
+              // model's call silently never happened. It gets an id at the
+              // terminal event instead (`settleToolCalls`).
+              const call: PendingCall = {
+                index: nextIndex++,
+                id: item.call_id ?? "",
+                name: item.name ?? "",
+                streamed: snapshot,
+              };
+              calls.push(call);
               yield {
                 type: "delta",
                 toolCalls: [
                   {
-                    index: nextIndex++,
-                    id: item.call_id,
+                    index: call.index,
+                    ...(item.call_id ? { id: item.call_id } : {}),
                     ...(item.name ? { name: item.name } : {}),
                     arguments: snapshot,
                   },
@@ -617,6 +658,8 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
               ...(item.name && item.name !== known.name ? { name: item.name } : {}),
               ...(tail ? { arguments: tail } : {}),
             };
+            known.id = item.call_id || known.id;
+            known.name = item.name || known.name;
             if (Object.keys(restated).length > 0) {
               yield { type: "delta", toolCalls: [{ index: known.index, ...restated }] };
             }
@@ -635,16 +678,27 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
             if (response?.status === "failed" || response?.error) {
               throw streamError(id, response.error ?? { message: "response failed" });
             }
+            // Any other status (`cancelled`, `queued`, one added later) says
+            // nothing about the answer being whole. Read as a clean stop, it
+            // was a finished turn nobody retried. Theirs, so `overload`.
+            const status = response?.status;
+            if (status !== undefined && status !== "completed" && status !== "incomplete") {
+              throw new ProviderError(id, "overload", `${id} ended the turn as "${status}"`, {
+                code: status,
+              });
+            }
+            let finishReason: FinishReason = calls.length > 0 ? "tool_calls" : "stop";
+            if (status === "incomplete") {
+              finishReason = incompleteFinish(id, response?.incomplete_details?.reason);
+            } else {
+              // Checked on a whole turn only: a cut one may have lost a call's
+              // name to the cut, and nobody runs a cut turn's calls.
+              const minted = settleToolCalls(id, calls);
+              if (minted.length > 0) yield { type: "delta", toolCalls: minted };
+            }
             yield* reasoningChunk();
             if (response?.usage) yield usageChunk(response.usage);
-            if (response?.status === "incomplete") {
-              yield {
-                type: "finish",
-                finishReason: mapIncompleteReason(response.incomplete_details?.reason),
-              };
-              return;
-            }
-            yield { type: "finish", finishReason: sawToolCall ? "tool_calls" : "stop" };
+            yield { type: "finish", finishReason };
             return;
           }
 
@@ -655,13 +709,11 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
           case "response.incomplete": {
             // A turn that ran out of output tokens still billed for its input,
             // and this event carries usage in the same shape as `completed`.
+            const finishReason = incompleteFinish(id, event.response?.incomplete_details?.reason);
             yield* reasoningChunk();
             const usage = event.response?.usage;
             if (usage) yield usageChunk(usage);
-            yield {
-              type: "finish",
-              finishReason: mapIncompleteReason(event.response?.incomplete_details?.reason),
-            };
+            yield { type: "finish", finishReason };
             return;
           }
 

package/src/rate-limit.ts

@@ -36,8 +36,16 @@ export interface RateLimitReset {
   retryAfterMs?: number;
   /** Which subscription window bound, when it could be named at all. */
   window?: RateLimitWindow;
+  /** How full the binding window is, as the headers report it (1 is full).
+   *  Set only when a window's fill rode the response: Anthropic's unified
+   *  5h/7d headers, or the ChatGPT backend's `x-codex-*` ones. */
+  utilization?: number;
 }
 
+/** A window's fill off a response that reported one: what
+ *  `StreamOptions.onRateLimit` receives. */
+export type RateLimitReading = RateLimitReset & { utilization: number; resetAtMs: number };
+
 const RETRY_AFTER = "retry-after";
 const RETRY_AFTER_MS = "retry-after-ms";
 const UNIFIED_5H = "anthropic-ratelimit-unified-5h-";
@@ -66,11 +74,23 @@ const RESET_HEADERS = [
  */
 const CODEX_SLOTS = ["primary", "secondary"] as const;
 
+/** One window as a response reports it: how full, and when it resets. */
+interface WindowReading {
+  window?: RateLimitWindow;
+  utilization?: number;
+  reset?: number;
+}
+type Dated = WindowReading & { utilization: number; reset: number };
+
 function windowForMinutes(minutes: number | undefined): RateLimitWindow | undefined {
   return minutes !== undefined && minutes > 0 ? windowForMs(minutes * 60_000) : undefined;
 }
 
-function codexWindow(headers: Headers, slot: (typeof CODEX_SLOTS)[number], now: number) {
+function codexWindow(
+  headers: Headers,
+  slot: (typeof CODEX_SLOTS)[number],
+  now: number,
+): WindowReading {
   const name = (suffix: string) => headers.get(`x-codex-${slot}-${suffix}`);
   const percent = numeric(name("used-percent"));
   const after = numeric(name("reset-after-seconds"));
@@ -185,10 +205,18 @@ export function parseRateLimitReset(headers: Headers, now = Date.now()): RateLim
       reset: unixSecondsToMs(headers.get(`${UNIFIED_7D}reset`)),
     },
     ...CODEX_SLOTS.map((slot) => codexWindow(headers, slot, now)),
-  ].filter((w) => w.utilization !== undefined && w.reset !== undefined && w.reset > now);
-  const binding = windows.sort((a, b) => (b.utilization ?? 0) - (a.utilization ?? 0))[0];
+  ].filter(
+    (w): w is Dated => w.utilization !== undefined && w.reset !== undefined && w.reset > now,
+  );
+  // The fuller window binds. Between two as full as each other (both spent,
+  // usually) the later reset does: the request passes only once both have
+  // cleared, and waiting out the earlier one walks into the other wall. On a
+  // dead heat the later slot wins (the sort is stable), as cc-proxy's `max_by`
+  // does.
+  const binding = windows.sort((a, b) => a.utilization - b.utilization || a.reset - b.reset).at(-1);
   if (binding) {
     if (binding.window) result.window = binding.window;
+    result.utilization = binding.utilization;
     // The window's own reset outranks retry-after: it names the real horizon,
     // where retry-after names the next polite attempt.
     if (binding.reset !== undefined) result.resetAtMs = binding.reset;

package/src/retry.ts

@@ -28,6 +28,17 @@ export interface RetryOptions {
   /** Maximum acceptable wait. A longer server deadline ends retry so an outer
    *  fallback can switch provider; it is never shortened to retry too early. */
   maxDelayMs?: number;
+  /**
+   * How long recovery may go on, counted from the FIRST failure. A retry
+   * starts only if it would start before then, its wait included. An attempt
+   * already running is never cut. Default 60s; `Infinity` turns it off.
+   *
+   * Without it the worst case is every attempt in full plus every wait: three
+   * stalled streams at the watchdog's 300s each is a quarter of an hour before
+   * the caller hears anything. The clock starts at the first failure, not at
+   * the call, so a long healthy run that then fails still gets its recovery.
+   */
+  budgetMs?: number;
   /** Aborts the wait as well as the work, so Stop lands promptly. */
   signal?: AbortSignal;
   /** Decide retryability. Default: transient kinds only. */
@@ -40,6 +51,7 @@ export interface RetryOptions {
 const DEFAULT_MAX_ATTEMPTS = 3;
 const DEFAULT_BASE_DELAY_MS = 1_000;
 const DEFAULT_MAX_DELAY_MS = 30_000;
+const DEFAULT_BUDGET_MS = 60_000;
 
 /**
  * Full-jitter exponential backoff: a random delay in
@@ -92,6 +104,29 @@ function delayFor(err: unknown, attempt: number, opts: RetryOptions): number | n
 
 const defaultShouldRetry = (err: unknown, _attempt: number): boolean => isRetryable(err);
 
+/** The moment the recovery budget runs out, fixed at the first failure. */
+const budgetEnd = (opts: RetryOptions): number => Date.now() + (opts.budgetMs ?? DEFAULT_BUDGET_MS);
+
+/**
+ * Wait out the delay before the next attempt, or rethrow `err` unchanged when
+ * there is none: the server's deadline is past `maxDelayMs`, or the next
+ * attempt would start past the budget.
+ */
+async function waitToRetry(
+  err: unknown,
+  attempt: number,
+  opts: RetryOptions,
+  budgetEndsAt: number,
+): Promise<void> {
+  const delayMs = delayFor(err, attempt, opts);
+  if (delayMs === null || Date.now() + delayMs >= budgetEndsAt) throw err;
+  opts.onRetry?.({ error: err, attempt, delayMs });
+  await (opts.sleep ?? sleep)(delayMs, opts.signal);
+  // A timer can wake long after it was due (a suspended worker, a throttled
+  // tab). The budget is about when the attempt starts, not when it was meant to.
+  if (Date.now() >= budgetEndsAt) throw err;
+}
+
 /**
  * Run `fn`, retrying transient failures with backoff. For one-shot calls such
  * as a title, a summary or a compaction pass.
@@ -102,7 +137,7 @@ export async function withRetry<T>(
 ): Promise<T> {
   const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
   const shouldRetry = opts.shouldRetry ?? defaultShouldRetry;
-  const nap = opts.sleep ?? sleep;
+  let budgetEndsAt: number | undefined;
 
   let lastError: unknown;
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -114,10 +149,8 @@ export async function withRetry<T>(
       // A caller's Stop is never a transient failure, whatever it looks like.
       if (opts.signal?.aborted) throw err;
       if (attempt >= maxAttempts || !shouldRetry(err, attempt)) throw err;
-      const delayMs = delayFor(err, attempt, opts);
-      if (delayMs === null) throw err;
-      opts.onRetry?.({ error: err, attempt, delayMs });
-      await nap(delayMs, opts.signal);
+      budgetEndsAt ??= budgetEnd(opts);
+      await waitToRetry(err, attempt, opts, budgetEndsAt);
     }
   }
   throw lastError;
@@ -138,7 +171,7 @@ export async function* withStreamRetry<T>(
 ): AsyncGenerator<T> {
   const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
   const shouldRetry = opts.shouldRetry ?? defaultShouldRetry;
-  const nap = opts.sleep ?? sleep;
+  let budgetEndsAt: number | undefined;
 
   let lastError: unknown;
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -163,10 +196,8 @@ export async function* withStreamRetry<T>(
       if (opts.signal?.aborted) throw err;
       // Rule 2: past the first chunk there is no going back.
       if (emitted || attempt >= maxAttempts || !shouldRetry(err, attempt)) throw err;
-      const delayMs = delayFor(err, attempt, opts);
-      if (delayMs === null) throw err;
-      opts.onRetry?.({ error: err, attempt, delayMs });
-      await nap(delayMs, opts.signal);
+      budgetEndsAt ??= budgetEnd(opts);
+      await waitToRetry(err, attempt, opts, budgetEndsAt);
     } finally {
       opts.signal?.removeEventListener("abort", onOuterAbort);
       // Abandoning mid-iteration (the consumer broke out, or we are retrying)

package/src/tool-args.ts

@@ -12,6 +12,8 @@
 //     `\uXXXX`, which JSON.parse decodes correctly. Some escape it twice and
 //     emit `\\u00e7`, so even a clean parse leaves six literal characters
 //     standing and a Portuguese answer reaches the user as `atenção`.
+import { ProviderError } from "./errors.ts";
+import type { ToolCallDelta } from "./types.ts";
 
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -129,6 +131,48 @@ export function normalizeToolId(id: string | undefined | null, prefix = "call_pk
   return clean;
 }
 
+/** A tool call's identity as a stream stated it: the last id and name it gave. */
+export interface ToolCallIdentity {
+  index: number;
+  id: string;
+  name: string;
+}
+
+/**
+ * Make a turn's tool calls runnable before its finish goes out, or fail the turn.
+ *
+ * A call with no name can't run: nothing says which tool to dispatch. It used to
+ * pass through (the Messages reader wrote `""` for both fields), and the loop
+ * failed one step later with nothing to say why. It throws `overload` now, a
+ * broken turn the same request usually does not repeat; cc-proxy answers it
+ * with a 502.
+ *
+ * A call with no id still can run. It gets one from `normalizeToolId`, written
+ * back into `calls` and returned as the deltas to send, so the result has an id
+ * to quote. An id the vendor did send is never rewritten: the four wires must
+ * agree on it (invariant 10), and repeated ids are history repair's job.
+ */
+export function settleToolCalls(
+  provider: string,
+  calls: Iterable<ToolCallIdentity>,
+): ToolCallDelta[] {
+  const minted: ToolCallDelta[] = [];
+  for (const call of calls) {
+    if (!call.name.trim()) {
+      throw new ProviderError(
+        provider,
+        "overload",
+        `${provider}: tool call ${call.index} came with no name, so no tool can run it`,
+      );
+    }
+    if (!call.id.trim()) {
+      call.id = normalizeToolId(call.id);
+      minted.push({ index: call.index, id: call.id });
+    }
+  }
+  return minted;
+}
+
 /**
  * Parse a tool call's raw argument string into an object, salvaging what a
  * truncated stream left behind, resolving concatenated JSON, and healing