@providerkit/core 0.22.0 → 0.23.0

package/src/errors.ts

@@ -343,6 +343,12 @@ const CONTEXT_PATTERNS: readonly RegExp[] = [
   // "input length and `max_tokens` exceed context limit: 188059 + 20000 > 200000"
   /input length and `?max_tokens`? exceed context limit/i,
   /exceed(?:s|ed)? (?:the )?context limit/i,
+  // GitHub Copilot: "prompt token count of 270000 exceeds the limit of 128000",
+  // code `model_max_prompt_tokens_exceeded`. No other wording here reads it,
+  // so it landed on `invalid` and the conversation never compacted. cc-proxy
+  // shipped the same two patterns for the same reason (0.1.51).
+  /model_max_prompt_tokens_exceeded/i,
+  /prompt token count of \d+ exceeds the limit/i,
   // Scoped to the thing that overflowed: a bare "exceeds the maximum" also
   // covers image counts and per-minute token rates, which compaction can't fix.
   /exceeds? the (?:model'?s? )?maximum (?:input |prompt |context )?(?:tokens?|length|context)/i,
@@ -652,6 +658,25 @@ export function parseContextOverflow(err: unknown, body?: string): ContextOverfl
   return undefined;
 }
 
+/**
+ * The HTTP status an in-band error's type stands for when it states none:
+ * Anthropic's types, then OpenAI's, read from `type` and then from a slug
+ * `code`. cc-proxy's table. A Map, so a type named after an Object method
+ * finds nothing rather than a function.
+ */
+const STATUS_BY_TYPE: ReadonlyMap<string, number> = new Map([
+  ["invalid_request_error", 400],
+  ["authentication_error", 401],
+  ["permission_error", 403],
+  ["not_found_error", 404],
+  ["request_too_large", 413],
+  ["rate_limit_error", 429],
+  ["api_error", 500],
+  ["server_error", 500],
+  ["internal_server_error", 500],
+  ["overloaded_error", 529],
+]);
+
 /**
  * A failure the backend reports INSIDE an already-200 stream.
  *
@@ -669,19 +694,34 @@ export function parseContextOverflow(err: unknown, body?: string): ContextOverfl
  * `code`/`type`, the canonical `status` name. The whole thing serializes into
  * the searchable body, so RetryInfo's `retryDelay` is honoured wherever it sits.
  *
+ * A number is a status only from 400 to 599. Vendors put their own codes in
+ * the same field: Z.ai's 1113 is an empty balance, and read as a status it was
+ * past 500, which skips the 4xx body patterns that name it quota and lands on
+ * overload. With no status stated, the error's type says which one it stands
+ * for (`STATUS_BY_TYPE`), and the body still outranks it (invariant 3).
+ *
  * `unknown` is floored to "overload" rather than kept: a stream that dies after
  * its headers is by construction a transient upstream fault, and "unknown" is
- * never retried. Everything the body does name keeps its own kind, which is
- * what tells the caller to wait, rotate a key, or compact instead of retrying
- * a failure that will repeat.
+ * never retried. Only an error with no status and no type it knows gets there.
+ * Before the type was read, one `invalid_request_error` in words no pattern
+ * knew was floored too: retried, then walked across every backup, cooling each
+ * for 30s, on a request that fails the same way everywhere. Everything the
+ * body does name keeps its own kind, which is what tells the caller to wait,
+ * rotate a key, or compact instead of retrying a failure that will repeat.
  */
 export function streamError(provider: string, error: unknown): ProviderError {
   const body = JSON.stringify(error ?? {}) ?? "";
-  const status =
-    readNumber(error, "code") ?? readNumber(error, "status") ?? readNumber(error, "status_code");
+  const status = [
+    readNumber(error, "code"),
+    readNumber(error, "status"),
+    readNumber(error, "status_code"),
+  ].find((value) => value !== undefined && value >= 400 && value < 600);
   const code =
     readString(error, "code") ?? readString(error, "status") ?? readString(error, "type");
-  const kind = classifyHttp(status, body);
+  const typeStatus =
+    STATUS_BY_TYPE.get(readString(error, "type") ?? "") ??
+    STATUS_BY_TYPE.get(readString(error, "code") ?? "");
+  const kind = classifyHttp(status ?? typeStatus, body);
   return new ProviderError(
     provider,
     kind === "unknown" ? "overload" : kind,

package/src/fallback.ts

@@ -1,8 +1,9 @@
 import { classify, parseRetryAfterMs, ProviderError, type ErrorKind } from "./errors.ts";
 import type { RateLimitWindow } from "./rate-limit.ts";
 import type { ProviderPresetId } from "./presets.ts";
+import { sleep } from "./retry.ts";
 import type { Provider, ProviderChunk } from "./types.ts";
-import { withWatchdog, type WatchdogOptions } from "./watchdog.ts";
+import { STREAM_PROGRESS_MS, withWatchdog, type WatchdogOptions } from "./watchdog.ts";
 import { createPresetProvider, type PresetProviderConfig } from "./providers/factory.ts";
 
 /** Retry intervals when the server supplies no deadline, not estimates of when
@@ -30,6 +31,20 @@ export interface FallbackOptions<T> {
     retryAtMs: number;
     window?: RateLimitWindow;
   }) => void;
+  /** Where cooldowns live; a private Map when omitted. Hand one store to every
+   *  pool that holds the same candidate and a lockout learned in one chain
+   *  benches it in all of them, instead of each chain spending a call to find
+   *  the same weekly wall. Pools sharing a store must share `cooldownMs` too: a
+   *  bench set by one pool is honoured by the others, so a kind one of them
+   *  turned off (`null`) would still divert its calls whenever another pool
+   *  benched the candidate for it. */
+  cooldowns?: CooldownStore<T>;
+  /** How long a recovery probe may go without a first chunk before another
+   *  call may replace it. A probe that hung used to hold its candidate out of
+   *  service for good. Defaults to the watchdog's progress clock
+   *  (`STREAM_PROGRESS_MS`): a call silent that long is one the watchdog
+   *  would already have killed. */
+  probeStaleMs?: number;
   /** Inject a clock for deterministic tests. */
   now?: () => number;
 }
@@ -93,11 +108,53 @@ export function withConfiguredFallbacks(
   );
 }
 
-interface Cooldown {
+/** One candidate's bench, as a pool records it. */
+export interface Cooldown {
   retryAtMs: number;
   kind: ErrorKind;
   window?: RateLimitWindow;
-  probing: boolean;
+  /** The one call let through once the bench expired. Only that call clears
+   *  it, and `settled` resolves at its first chunk or its end. */
+  probe?: { since: number; settled: Promise<void>; settle(): void };
+}
+
+/** Where a pool keeps its benches. A `Map` fits, and so does a `WeakMap` keyed
+ *  by providers, which lets a bench go when its provider does. `get` must
+ *  return the very object `set` was given: the probe is tracked on it. */
+export interface CooldownStore<T> {
+  get(candidate: T): Cooldown | undefined;
+  set(candidate: T, cooldown: Cooldown): void;
+  delete(candidate: T): unknown;
+}
+
+function openProbe(since: number): NonNullable<Cooldown["probe"]> {
+  let settle = (): void => {};
+  const settled = new Promise<void>((resolve) => {
+    settle = resolve;
+  });
+  return { since, settled, settle };
+}
+
+/** How many times a call that found nothing free waits on a probe before it
+ *  fails. The first wait covers a probe that answers or fails. The second
+ *  covers one that ended with neither (its own caller stopped it) and was
+ *  taken over by another waiter. Past that the caller's own retry decides. */
+const PROBE_WAITS = 2;
+
+/** Resolve when any probe settles or `ms` passes. Rejects with the caller's
+ *  reason on abort, and leaves no timer running once a probe wins. */
+async function waitForProbe(
+  settled: readonly Promise<void>[],
+  ms: number,
+  signal?: AbortSignal,
+): Promise<void> {
+  const done = new AbortController();
+  const wake = signal ? AbortSignal.any([signal, done.signal]) : done.signal;
+  try {
+    await Promise.race([...settled, sleep(Math.max(0, ms), wake)]);
+  } finally {
+    done.abort();
+  }
 }
 
 /** The whole chain is unavailable. Fail promptly instead of holding a request
@@ -126,7 +183,8 @@ export class NoAvailableProviderError extends ProviderError {
  *  a background timer, vendor SDK, database or global singleton. */
 export class FallbackPool<T> {
   private readonly candidates: readonly T[];
-  private readonly cooldowns = new Map<T, Cooldown>();
+  private readonly cooldowns: CooldownStore<T>;
+  private readonly probeStaleMs: number;
   private readonly now: () => number;
 
   constructor(
@@ -146,14 +204,21 @@ export class FallbackPool<T> {
       }
     }
     this.candidates = [...candidates];
+    this.cooldowns = options.cooldowns ?? new Map();
+    this.probeStaleMs = options.probeStaleMs ?? STREAM_PROGRESS_MS;
+    if (!Number.isFinite(this.probeStaleMs) || this.probeStaleMs < 0) {
+      throw new Error("providerkit: probeStaleMs must be finite, non-negative milliseconds");
+    }
     this.now = options.now ?? Date.now;
   }
 
   /** Clear a cooldown after replacing a key or fixing configuration. Omit the
-   *  candidate to clear all. Does not interrupt calls already in flight. */
+   *  candidate to clear all of this pool's own; a shared store keeps the benches
+   *  of candidates only other pools hold. Does not interrupt calls in flight. */
   reset(...selection: [] | [T]): void {
-    if (selection.length === 0) this.cooldowns.clear();
-    else this.cooldowns.delete(selection[0]);
+    for (const candidate of selection.length === 0 ? this.candidates : selection) {
+      this.cooldowns.delete(candidate);
+    }
   }
 
   status(): Array<{
@@ -163,12 +228,12 @@ export class FallbackPool<T> {
     kind?: ErrorKind;
     window?: RateLimitWindow;
   }> {
-    return this.candidates.map((candidate) => ({
-      candidate,
-      retryAtMs: 0,
-      probing: false,
-      ...this.cooldowns.get(candidate),
-    }));
+    return this.candidates.map((candidate) => {
+      const cooldown = this.cooldowns.get(candidate);
+      if (!cooldown) return { candidate, retryAtMs: 0, probing: false };
+      const { probe, ...bench } = cooldown;
+      return { candidate, ...bench, probing: probe !== undefined };
+    });
   }
 
   /** Buffered calls only: do not wrap an agent turn that has run side effects. */
@@ -192,74 +257,107 @@ export class FallbackPool<T> {
     signal?.throwIfAborted();
     let hasFailed = false;
     let lastError: unknown;
-    for (const candidate of this.candidates) {
-      signal?.throwIfAborted();
-      const previous = this.cooldowns.get(candidate);
-      if (previous && (previous.retryAtMs > this.now() || previous.probing)) continue;
-      // One recovery probe after expiry. Healthy endpoints still allow normal
-      // concurrency; only a known failed endpoint gets a single probe.
-      if (previous) previous.probing = true;
-      const controller = new AbortController();
-      const inner = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
-      let emitted = false;
-      let attemptFailed = false;
-      const startedWithCooldown = previous !== undefined;
-      try {
-        for await (const chunk of attempt(candidate, inner)) {
+    for (let round = 0; ; round++) {
+      const probing: Array<NonNullable<Cooldown["probe"]>> = [];
+      for (const candidate of this.candidates) {
+        signal?.throwIfAborted();
+        const previous = this.cooldowns.get(candidate);
+        if (previous && previous.retryAtMs > this.now()) continue;
+        if (previous?.probe && this.now() - previous.probe.since < this.probeStaleMs) {
+          probing.push(previous.probe);
+          continue;
+        }
+        // One recovery probe after expiry. Healthy endpoints still allow normal
+        // concurrency; only a known failed endpoint gets a single probe. A stale
+        // one is replaced: it hung, and holding the candidate for it is an outage.
+        const probe = previous ? openProbe(this.now()) : undefined;
+        if (previous) previous.probe = probe;
+        const controller = new AbortController();
+        const inner = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+        let emitted = false;
+        try {
+          for await (const chunk of attempt(candidate, inner)) {
+            inner.throwIfAborted();
+            if (!emitted) {
+              this.recovered(candidate, previous);
+              probe?.settle();
+            }
+            emitted = true;
+            yield chunk;
+          }
           inner.throwIfAborted();
-          emitted = true;
-          // Only a call that was admitted as a recovery probe (started while on cooldown)
-          // is authorized to clear that cooldown upon succeeding.
-          if (startedWithCooldown && this.cooldowns.get(candidate) === previous) {
-            this.cooldowns.delete(candidate);
+          if (!emitted) this.recovered(candidate, previous);
+          return;
+        } catch (error) {
+          signal?.throwIfAborted();
+          const kind = classify(error);
+          if (kind === "aborted") throw error;
+          const configured = this.options.cooldownMs?.[kind];
+          const interval = configured === undefined ? DEFAULT_FALLBACK_COOLDOWNS[kind] : configured;
+          if (interval === undefined || interval === null) throw error;
+          const now = this.now();
+          const after = parseRetryAfterMs(error);
+          const reset = error instanceof ProviderError ? error.resetAtMs : undefined;
+          const hints = [reset, after === undefined ? undefined : now + after].filter(
+            (value): value is number => value !== undefined && Number.isFinite(value),
+          );
+          const retryAtMs = Math.max(
+            now,
+            ...(hints.length ? hints : [now + interval]),
+            this.cooldowns.get(candidate)?.retryAtMs ?? 0,
+          );
+          const window = error instanceof ProviderError ? error.window : undefined;
+          this.cooldowns.set(candidate, { retryAtMs, kind, window });
+          this.options.onCooldown?.({ candidate, error, kind, retryAtMs, window });
+          hasFailed = true;
+          lastError = error;
+          if (emitted) throw error;
+        } finally {
+          controller.abort();
+          // Only the owner releases its probe. Clearing whatever probe was set
+          // let a stale probe that ended late free the one that replaced it, and
+          // any call that did not fail let a second probe in.
+          if (probe) {
+            if (previous?.probe === probe) previous.probe = undefined;
+            probe.settle();
           }
-          yield chunk;
-        }
-        inner.throwIfAborted();
-        if (startedWithCooldown && this.cooldowns.get(candidate) === previous) {
-          this.cooldowns.delete(candidate);
         }
-        return;
-      } catch (error) {
-        signal?.throwIfAborted();
-        const kind = classify(error);
-        if (kind === "aborted") throw error;
-        const configured = this.options.cooldownMs?.[kind];
-        const interval = configured === undefined ? DEFAULT_FALLBACK_COOLDOWNS[kind] : configured;
-        if (interval === undefined || interval === null) throw error;
-        const now = this.now();
-        const after = parseRetryAfterMs(error);
-        const reset = error instanceof ProviderError ? error.resetAtMs : undefined;
-        const hints = [reset, after === undefined ? undefined : now + after].filter(
-          (value): value is number => value !== undefined && Number.isFinite(value),
-        );
-        const retryAtMs = Math.max(
-          now,
-          ...(hints.length ? hints : [now + interval]),
-          this.cooldowns.get(candidate)?.retryAtMs ?? 0,
+      }
+      if (hasFailed) throw lastError;
+      // Nothing was free, and a probe is deciding whether something will be.
+      // Failing here answered a 0 ms retry, which `withRetry` spent at once: at
+      // 70 concurrent calls every one but the probe failed. Wait for a probe to
+      // answer or end, or until the oldest goes stale, then look again.
+      if (probing.length > 0 && round < PROBE_WAITS) {
+        const staleAt = Math.min(...probing.map((probe) => probe.since + this.probeStaleMs));
+        await waitForProbe(
+          probing.map((probe) => probe.settled),
+          staleAt - this.now(),
+          signal,
         );
-        const window = error instanceof ProviderError ? error.window : undefined;
-        this.cooldowns.set(candidate, { retryAtMs, kind, window, probing: false });
-        this.options.onCooldown?.({ candidate, error, kind, retryAtMs, window });
-        attemptFailed = true;
-        hasFailed = true;
-        lastError = error;
-        if (emitted) throw error;
-      } finally {
-        controller.abort();
-        const current = this.cooldowns.get(candidate);
-        if (current && current.probing && !attemptFailed) {
-          current.probing = false;
-        }
+        continue;
       }
+      // Every candidate is cooling, or its probe outlasted the waits; a probe
+      // still in flight makes the next opportunity now.
+      const retryAtMs = Math.min(
+        ...this.status().map((entry) => Math.max(this.now(), entry.retryAtMs)),
+      );
+      throw new NoAvailableProviderError(retryAtMs, this.now());
+    }
+  }
+
+  /** An answer proves the candidate serves again. It lifts the bench this call
+   *  was admitted under as the probe, and any bench a sibling set while it ran:
+   *  a call that started before one stray 401 and then answered used to leave
+   *  a busy primary benched for an hour. Rate and quota stay. A call that
+   *  started before the throttle proves nothing about the window, and lifting
+   *  it walks every caller back into the 429. */
+  private recovered(candidate: T, admittedUnder: Cooldown | undefined): void {
+    const current = this.cooldowns.get(candidate);
+    if (!current) return;
+    if (current === admittedUnder || (current.kind !== "rate" && current.kind !== "quota")) {
+      this.cooldowns.delete(candidate);
     }
-    if (hasFailed) throw lastError;
-    // If all expired candidates are already being probed, the next opportunity
-    // is now. Other callers need not wait for those probes or start duplicates.
-    const retryAtMs = Math.min(
-      ...this.status().map((entry) => Math.max(this.now(), entry.retryAtMs)),
-    );
-    throw new NoAvailableProviderError(retryAtMs, this.now());
   }
 }
 

package/src/key-pool.ts

@@ -55,12 +55,20 @@ export interface KeyPoolOptions {
   now?: () => number;
 }
 
-/** No key can serve right now. `retryAtMs` = when the soonest one is back. */
-export class NoAvailableKeyError extends Error {
+/** No key can serve right now. `retryAtMs` = when the soonest one is back.
+ *
+ *  A `ProviderError` of kind `rate` that carries that moment as `resetAtMs` and
+ *  `retryAfterMs`. As a plain `Error` it classified as `rate` only by its
+ *  wording and carried no deadline anyone read, so a FallbackPool around the
+ *  key pool benched it for its default minute, whatever the keys had said. */
+export class NoAvailableKeyError extends ProviderError {
   readonly retryAtMs: number;
 
-  constructor(label: string, retryAtMs: number) {
-    super(`No ${label} API key available (all rate-limited or denied)`);
+  constructor(label: string, retryAtMs: number, now: number) {
+    super(label, "rate", `No ${label} API key available (all rate-limited or denied)`, {
+      resetAtMs: retryAtMs,
+      retryAfterMs: Math.max(0, retryAtMs - now),
+    });
     this.name = "NoAvailableKeyError";
     this.retryAtMs = retryAtMs;
   }
@@ -101,7 +109,7 @@ export class KeyPool {
    */
   async with<T>(fn: (apiKey: string, tier: KeyTier) => Promise<T>): Promise<T> {
     const candidates = this.candidates(this.now());
-    if (candidates.length === 0) throw new NoAvailableKeyError(this.label, this.nextAvailableAt());
+    if (candidates.length === 0) throw this.noneAvailable();
 
     let last: unknown;
     for (const key of candidates) {
@@ -112,7 +120,7 @@ export class KeyPool {
         // The single-key rule: with nothing to rotate to, benching the only key
         // answers NoAvailableKeyError to every call for the next minute — for a
         // failure the caller's own retry would have absorbed.
-        const cooldown = this.size > 1 ? cooldownFor(err) : null;
+        const cooldown = this.size > 1 ? cooldownFor(err, this.now()) : null;
         if (cooldown === null) throw err;
         key.evictedUntil = this.now() + cooldown.forMs;
         this.onEvict?.({ tier: key.tier, kind: cooldown.kind, forMs: cooldown.forMs });
@@ -120,8 +128,7 @@ export class KeyPool {
     }
 
     // Every candidate was evicted during this call.
-    if (this.candidates(this.now()).length === 0)
-      throw new NoAvailableKeyError(this.label, this.nextAvailableAt());
+    if (this.candidates(this.now()).length === 0) throw this.noneAvailable();
     throw last;
   }
 
@@ -138,9 +145,10 @@ export class KeyPool {
     return out;
   }
 
-  private nextAvailableAt(): number {
+  private noneAvailable(): NoAvailableKeyError {
     const all = this.paid ? [...this.free, this.paid] : this.free;
-    return Math.min(...all.map((key) => key.evictedUntil));
+    const retryAtMs = Math.min(...all.map((key) => key.evictedUntil));
+    return new NoAvailableKeyError(this.label, retryAtMs, this.now());
   }
 }
 
@@ -149,15 +157,24 @@ export class KeyPool {
  * key's fault. Nothing is gained by rotating, and benching would spend the
  * pool on a request that fails the same way everywhere.
  */
-function cooldownFor(err: unknown): { kind: ErrorKind; forMs: number } | null {
+function cooldownFor(err: unknown, now: number): { kind: ErrorKind; forMs: number } | null {
   if (!(err instanceof ProviderError)) return null;
   const clamp = (ms: number) => Math.min(Math.max(ms, MIN_EVICTION_MS), MAX_EVICTION_MS);
   switch (err.kind) {
     // The three kinds that are about THIS key.
     case "rate":
       return { kind: err.kind, forMs: clamp(err.retryAfterMs ?? RATE_COOLDOWN_MS) };
-    case "quota":
-      return { kind: err.kind, forMs: clamp(err.retryAfterMs ?? QUOTA_COOLDOWN_MS) };
+    case "quota": {
+      // A spent window names its reset, and that outranks Retry-After: Anthropic
+      // sends `retry-after: 60` beside a weekly window, and a key benched for
+      // the minute is dialled again into the same wall.
+      const reset = err.resetAtMs;
+      const forMs =
+        reset !== undefined && Number.isFinite(reset)
+          ? reset - now
+          : (err.retryAfterMs ?? QUOTA_COOLDOWN_MS);
+      return { kind: err.kind, forMs: clamp(forMs) };
+    }
     case "auth":
       // 12 h, not forever: a key is also refused while a billing account is
       // reinstated, and a pool that drops keys permanently ends up empty.

package/src/presets.ts

@@ -373,8 +373,9 @@ export const PROVIDER_PRESETS = {
    * Go serves three wires, and `routes` picks one from the model id (cc-proxy's
    * table, from Go's own docs): `minimax-` and `qwen` on Anthropic /messages
    * (key in `x-api-key`), `gpt-`, `grok-` and `muse-spark-` on /responses, and
-   * everything else on chat completions. Each model in a chain resolves its
-   * own route, so one preset serves a chain that crosses wires.
+   * everything else on chat completions, `grok-4.5` included. Each model in a
+   * chain resolves its own route, so one preset serves a chain that crosses
+   * wires.
    *
    * Muse Contributor, measured 2026-09-28 on /responses: text, images, strict
    * JSON Schema, parallel tools and tool-result round trips all passed. It
@@ -408,6 +409,15 @@ export const PROVIDER_PRESETS = {
         baseUrl: "https://opencode.ai/zen/go",
         auth: "key",
       },
+      {
+        // The one Grok Go serves on chat completions, while 4.6 and 4.7 are on
+        // /responses: cc-proxy's catalog pins it there as an override of the
+        // family rule. It sits above the `grok-` route because the first match
+        // wins.
+        prefixes: ["grok-4.5"],
+        shape: "openai",
+        baseUrl: "https://opencode.ai/zen/go/v1",
+      },
       {
         prefixes: ["gpt-", "grok-", "muse-spark-"],
         shape: "responses",