@providerkit/core 0.22.0 → 0.24.0

package/src/context.ts

@@ -1,9 +1,10 @@
-// Deciding when a conversation no longer fits, and where to cut it.
+// Deciding when a conversation no longer fits, where to cut it, and how to
+// make a saved one valid to send again.
 //
 // Everything here is pure and testable. The model call that writes the summary
 // and the row that stores it belong to the caller — what is decidable without
 // I/O is decided here.
-import type { ChatMessage } from "./types.ts";
+import type { ChatMessage, ToolCall } from "./types.ts";
 
 /**
  * What the answer needs after the prompt: an output budget, plus the tools'
@@ -153,6 +154,140 @@ export function applyCompaction(
   ];
 }
 
+type ToolResult = Extract<ChatMessage, { role: "tool" }>;
+
+/** What a call with no result answers, by whether running it again is safe.
+ *  The model reads this and decides whether to call again, so the two must
+ *  differ on exactly that. */
+const INTERRUPTED_READ_ONLY =
+  "This call was interrupted before it returned. It changed nothing, so it is safe to call again.";
+const INTERRUPTED =
+  "This call was interrupted before it returned. It may have partly run, so check its effect before calling it again.";
+
+/** `id`, or `id_2`, `id_3`… whichever is free first. Only ids already handed
+ *  out are consulted, so an id never depends on what comes after it. */
+function freshId(id: string, used: Set<string>): string {
+  let fresh = id;
+  for (let k = 2; used.has(fresh); k++) fresh = `${id}_${k}`;
+  used.add(fresh);
+  return fresh;
+}
+
+interface Span {
+  calls: ToolCall[];
+  vendorIds: string[];
+  results: (ToolResult | undefined)[];
+  rest: ChatMessage[];
+}
+
+/** The first call in the span asked under `id` that has no result yet, or -1.
+ *  First unanswered rather than first: two calls in one reply can share an id,
+ *  and their results then pair in order. */
+function openSlot(span: Span, id: string): number {
+  return span.vendorIds.findIndex((asked, i) => asked === id && !span.results[i]);
+}
+
+/** A result with no call to answer, kept as words the model can still read
+ *  instead of a `tool` message every provider answers with a 400. */
+function orphanNote(result: ToolResult): ChatMessage {
+  const text = `[Tool result for ${result.toolCallId}; the tool call is missing or already has a result]\n${result.content}`;
+  return {
+    role: "user",
+    content: result.images?.length ? [{ type: "text", text }, ...result.images] : text,
+  };
+}
+
+/**
+ * Make a saved, resumed or interrupted history valid to send: every call has
+ * exactly one result, right after it, under an id no other call uses.
+ *
+ * A span runs from an assistant message with tool calls to the next assistant
+ * message. Its results move to just after the calls, in call order, and the
+ * rest of the span (a steer the user typed while the tools ran) keeps its
+ * order after them. Then:
+ *
+ * - A call with no result in its span gets one under the same id, saying it was
+ *   interrupted. `isReadOnly` picks the wording; without it every tool may
+ *   have partly run. These calls come back as `interrupted`, under the ids
+ *   the request carries.
+ * - A result whose call is not in its span becomes a user note, images kept.
+ *   Paired against the span only, because chat and Messages both require the
+ *   result in the turn right after its call: a looser pairing still 400s.
+ * - A repeated id becomes `{id}_{k}`, the smallest free k, call and result
+ *   together. Kimi and Gemini restart their ids every reply
+ *   (`functions.search:0`, `call_0`), and Anthropic and Responses reject the
+ *   second use.
+ * - An empty id becomes `call_{message}_{call}`, never `normalizeToolId`'s
+ *   random fallback.
+ * - An assistant message with no calls and only whitespace is dropped, as if
+ *   it were never there, so a placeholder saved before the tools ran orphans
+ *   nothing.
+ *
+ * No side effects, and the same input always gives the same bytes: a span's
+ * output and every id depend only on what came before, so repairing a longer
+ * history keeps the shorter one's bytes and the prompt cache keeps hitting.
+ * Repairing a repaired history changes nothing. Every adapter here runs this
+ * on what it sends; call it yourself to know what `interrupted` holds.
+ */
+export function repairHistory(
+  messages: readonly ChatMessage[],
+  opts: { isReadOnly?: (toolName: string) => boolean } = {},
+): { messages: ChatMessage[]; interrupted: ToolCall[] } {
+  const out: ChatMessage[] = [];
+  const interrupted: ToolCall[] = [];
+  const used = new Set<string>();
+  // The open span: its calls as sent, the id the vendor gave each, the result
+  // each has found so far, and everything else in arrival order.
+  let span: Span | undefined;
+
+  const close = () => {
+    if (!span) return;
+    for (const [i, call] of span.calls.entries()) {
+      const result = span.results[i];
+      if (result) {
+        out.push(result);
+        continue;
+      }
+      interrupted.push(call);
+      const content = opts.isReadOnly?.(call.name) ? INTERRUPTED_READ_ONLY : INTERRUPTED;
+      out.push({ role: "tool", toolCallId: call.id, name: call.name, content });
+    }
+    out.push(...span.rest);
+    span = undefined;
+  };
+
+  for (const [index, message] of messages.entries()) {
+    if (message.role === "assistant") {
+      const calls = message.toolCalls ?? [];
+      if (calls.length === 0 && !message.content.trim()) continue;
+      close();
+      if (calls.length === 0) {
+        out.push(message);
+        continue;
+      }
+      const sent = calls.map((call, c) => {
+        const id = freshId(call.id.trim() ? call.id : `call_${index}_${c}`, used);
+        return id === call.id ? call : { ...call, id };
+      });
+      const renamed = sent.some((call, c) => call !== calls[c]);
+      out.push(renamed ? { ...message, toolCalls: sent } : message);
+      span = { calls: sent, vendorIds: calls.map((call) => call.id), results: [], rest: [] };
+    } else if (message.role === "tool") {
+      const slot = span ? openSlot(span, message.toolCallId) : -1;
+      if (span && slot !== -1) {
+        const id = span.calls[slot]!.id;
+        span.results[slot] = id === message.toolCallId ? message : { ...message, toolCallId: id };
+      } else {
+        (span?.rest ?? out).push(orphanNote(message));
+      }
+    } else {
+      (span?.rest ?? out).push(message);
+    }
+  }
+  close();
+  return { messages: out, interrupted };
+}
+
 /**
  * The context window for a model, when nothing volunteered one.
  *

package/src/errors.ts

@@ -343,6 +343,12 @@ const CONTEXT_PATTERNS: readonly RegExp[] = [
   // "input length and `max_tokens` exceed context limit: 188059 + 20000 > 200000"
   /input length and `?max_tokens`? exceed context limit/i,
   /exceed(?:s|ed)? (?:the )?context limit/i,
+  // GitHub Copilot: "prompt token count of 270000 exceeds the limit of 128000",
+  // code `model_max_prompt_tokens_exceeded`. No other wording here reads it,
+  // so it landed on `invalid` and the conversation never compacted. cc-proxy
+  // shipped the same two patterns for the same reason (0.1.51).
+  /model_max_prompt_tokens_exceeded/i,
+  /prompt token count of \d+ exceeds the limit/i,
   // Scoped to the thing that overflowed: a bare "exceeds the maximum" also
   // covers image counts and per-minute token rates, which compaction can't fix.
   /exceeds? the (?:model'?s? )?maximum (?:input |prompt |context )?(?:tokens?|length|context)/i,
@@ -652,6 +658,25 @@ export function parseContextOverflow(err: unknown, body?: string): ContextOverfl
   return undefined;
 }
 
+/**
+ * The HTTP status an in-band error's type stands for when it states none:
+ * Anthropic's types, then OpenAI's, read from `type` and then from a slug
+ * `code`. cc-proxy's table. A Map, so a type named after an Object method
+ * finds nothing rather than a function.
+ */
+const STATUS_BY_TYPE: ReadonlyMap<string, number> = new Map([
+  ["invalid_request_error", 400],
+  ["authentication_error", 401],
+  ["permission_error", 403],
+  ["not_found_error", 404],
+  ["request_too_large", 413],
+  ["rate_limit_error", 429],
+  ["api_error", 500],
+  ["server_error", 500],
+  ["internal_server_error", 500],
+  ["overloaded_error", 529],
+]);
+
 /**
  * A failure the backend reports INSIDE an already-200 stream.
  *
@@ -669,19 +694,34 @@ export function parseContextOverflow(err: unknown, body?: string): ContextOverfl
  * `code`/`type`, the canonical `status` name. The whole thing serializes into
  * the searchable body, so RetryInfo's `retryDelay` is honoured wherever it sits.
  *
+ * A number is a status only from 400 to 599. Vendors put their own codes in
+ * the same field: Z.ai's 1113 is an empty balance, and read as a status it was
+ * past 500, which skips the 4xx body patterns that name it quota and lands on
+ * overload. With no status stated, the error's type says which one it stands
+ * for (`STATUS_BY_TYPE`), and the body still outranks it (invariant 3).
+ *
  * `unknown` is floored to "overload" rather than kept: a stream that dies after
  * its headers is by construction a transient upstream fault, and "unknown" is
- * never retried. Everything the body does name keeps its own kind, which is
- * what tells the caller to wait, rotate a key, or compact instead of retrying
- * a failure that will repeat.
+ * never retried. Only an error with no status and no type it knows gets there.
+ * Before the type was read, one `invalid_request_error` in words no pattern
+ * knew was floored too: retried, then walked across every backup, cooling each
+ * for 30s, on a request that fails the same way everywhere. Everything the
+ * body does name keeps its own kind, which is what tells the caller to wait,
+ * rotate a key, or compact instead of retrying a failure that will repeat.
  */
 export function streamError(provider: string, error: unknown): ProviderError {
   const body = JSON.stringify(error ?? {}) ?? "";
-  const status =
-    readNumber(error, "code") ?? readNumber(error, "status") ?? readNumber(error, "status_code");
+  const status = [
+    readNumber(error, "code"),
+    readNumber(error, "status"),
+    readNumber(error, "status_code"),
+  ].find((value) => value !== undefined && value >= 400 && value < 600);
   const code =
     readString(error, "code") ?? readString(error, "status") ?? readString(error, "type");
-  const kind = classifyHttp(status, body);
+  const typeStatus =
+    STATUS_BY_TYPE.get(readString(error, "type") ?? "") ??
+    STATUS_BY_TYPE.get(readString(error, "code") ?? "");
+  const kind = classifyHttp(status ?? typeStatus, body);
   return new ProviderError(
     provider,
     kind === "unknown" ? "overload" : kind,

package/src/fallback.ts

@@ -1,8 +1,9 @@
 import { classify, parseRetryAfterMs, ProviderError, type ErrorKind } from "./errors.ts";
 import type { RateLimitWindow } from "./rate-limit.ts";
 import type { ProviderPresetId } from "./presets.ts";
+import { sleep } from "./retry.ts";
 import type { Provider, ProviderChunk } from "./types.ts";
-import { withWatchdog, type WatchdogOptions } from "./watchdog.ts";
+import { STREAM_PROGRESS_MS, withWatchdog, type WatchdogOptions } from "./watchdog.ts";
 import { createPresetProvider, type PresetProviderConfig } from "./providers/factory.ts";
 
 /** Retry intervals when the server supplies no deadline, not estimates of when
@@ -30,6 +31,20 @@ export interface FallbackOptions<T> {
     retryAtMs: number;
     window?: RateLimitWindow;
   }) => void;
+  /** Where cooldowns live; a private Map when omitted. Hand one store to every
+   *  pool that holds the same candidate and a lockout learned in one chain
+   *  benches it in all of them, instead of each chain spending a call to find
+   *  the same weekly wall. Pools sharing a store must share `cooldownMs` too: a
+   *  bench set by one pool is honoured by the others, so a kind one of them
+   *  turned off (`null`) would still divert its calls whenever another pool
+   *  benched the candidate for it. */
+  cooldowns?: CooldownStore<T>;
+  /** How long a recovery probe may go without a first chunk before another
+   *  call may replace it. A probe that hung used to hold its candidate out of
+   *  service for good. Defaults to the watchdog's progress clock
+   *  (`STREAM_PROGRESS_MS`): a call silent that long is one the watchdog
+   *  would already have killed. */
+  probeStaleMs?: number;
   /** Inject a clock for deterministic tests. */
   now?: () => number;
 }
@@ -93,11 +108,53 @@ export function withConfiguredFallbacks(
   );
 }
 
-interface Cooldown {
+/** One candidate's bench, as a pool records it. */
+export interface Cooldown {
   retryAtMs: number;
   kind: ErrorKind;
   window?: RateLimitWindow;
-  probing: boolean;
+  /** The one call let through once the bench expired. Only that call clears
+   *  it, and `settled` resolves at its first chunk or its end. */
+  probe?: { since: number; settled: Promise<void>; settle(): void };
+}
+
+/** Where a pool keeps its benches. A `Map` fits, and so does a `WeakMap` keyed
+ *  by providers, which lets a bench go when its provider does. `get` must
+ *  return the very object `set` was given: the probe is tracked on it. */
+export interface CooldownStore<T> {
+  get(candidate: T): Cooldown | undefined;
+  set(candidate: T, cooldown: Cooldown): void;
+  delete(candidate: T): unknown;
+}
+
+function openProbe(since: number): NonNullable<Cooldown["probe"]> {
+  let settle = (): void => {};
+  const settled = new Promise<void>((resolve) => {
+    settle = resolve;
+  });
+  return { since, settled, settle };
+}
+
+/** How many times a call that found nothing free waits on a probe before it
+ *  fails. The first wait covers a probe that answers or fails. The second
+ *  covers one that ended with neither (its own caller stopped it) and was
+ *  taken over by another waiter. Past that the caller's own retry decides. */
+const PROBE_WAITS = 2;
+
+/** Resolve when any probe settles or `ms` passes. Rejects with the caller's
+ *  reason on abort, and leaves no timer running once a probe wins. */
+async function waitForProbe(
+  settled: readonly Promise<void>[],
+  ms: number,
+  signal?: AbortSignal,
+): Promise<void> {
+  const done = new AbortController();
+  const wake = signal ? AbortSignal.any([signal, done.signal]) : done.signal;
+  try {
+    await Promise.race([...settled, sleep(Math.max(0, ms), wake)]);
+  } finally {
+    done.abort();
+  }
 }
 
 /** The whole chain is unavailable. Fail promptly instead of holding a request
@@ -126,7 +183,8 @@ export class NoAvailableProviderError extends ProviderError {
  *  a background timer, vendor SDK, database or global singleton. */
 export class FallbackPool<T> {
   private readonly candidates: readonly T[];
-  private readonly cooldowns = new Map<T, Cooldown>();
+  private readonly cooldowns: CooldownStore<T>;
+  private readonly probeStaleMs: number;
   private readonly now: () => number;
 
   constructor(
@@ -146,14 +204,21 @@ export class FallbackPool<T> {
       }
     }
     this.candidates = [...candidates];
+    this.cooldowns = options.cooldowns ?? new Map();
+    this.probeStaleMs = options.probeStaleMs ?? STREAM_PROGRESS_MS;
+    if (!Number.isFinite(this.probeStaleMs) || this.probeStaleMs < 0) {
+      throw new Error("providerkit: probeStaleMs must be finite, non-negative milliseconds");
+    }
     this.now = options.now ?? Date.now;
   }
 
   /** Clear a cooldown after replacing a key or fixing configuration. Omit the
-   *  candidate to clear all. Does not interrupt calls already in flight. */
+   *  candidate to clear all of this pool's own; a shared store keeps the benches
+   *  of candidates only other pools hold. Does not interrupt calls in flight. */
   reset(...selection: [] | [T]): void {
-    if (selection.length === 0) this.cooldowns.clear();
-    else this.cooldowns.delete(selection[0]);
+    for (const candidate of selection.length === 0 ? this.candidates : selection) {
+      this.cooldowns.delete(candidate);
+    }
   }
 
   status(): Array<{
@@ -163,12 +228,12 @@ export class FallbackPool<T> {
     kind?: ErrorKind;
     window?: RateLimitWindow;
   }> {
-    return this.candidates.map((candidate) => ({
-      candidate,
-      retryAtMs: 0,
-      probing: false,
-      ...this.cooldowns.get(candidate),
-    }));
+    return this.candidates.map((candidate) => {
+      const cooldown = this.cooldowns.get(candidate);
+      if (!cooldown) return { candidate, retryAtMs: 0, probing: false };
+      const { probe, ...bench } = cooldown;
+      return { candidate, ...bench, probing: probe !== undefined };
+    });
   }
 
   /** Buffered calls only: do not wrap an agent turn that has run side effects. */
@@ -192,74 +257,107 @@ export class FallbackPool<T> {
     signal?.throwIfAborted();
     let hasFailed = false;
     let lastError: unknown;
-    for (const candidate of this.candidates) {
-      signal?.throwIfAborted();
-      const previous = this.cooldowns.get(candidate);
-      if (previous && (previous.retryAtMs > this.now() || previous.probing)) continue;
-      // One recovery probe after expiry. Healthy endpoints still allow normal
-      // concurrency; only a known failed endpoint gets a single probe.
-      if (previous) previous.probing = true;
-      const controller = new AbortController();
-      const inner = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
-      let emitted = false;
-      let attemptFailed = false;
-      const startedWithCooldown = previous !== undefined;
-      try {
-        for await (const chunk of attempt(candidate, inner)) {
+    for (let round = 0; ; round++) {
+      const probing: Array<NonNullable<Cooldown["probe"]>> = [];
+      for (const candidate of this.candidates) {
+        signal?.throwIfAborted();
+        const previous = this.cooldowns.get(candidate);
+        if (previous && previous.retryAtMs > this.now()) continue;
+        if (previous?.probe && this.now() - previous.probe.since < this.probeStaleMs) {
+          probing.push(previous.probe);
+          continue;
+        }
+        // One recovery probe after expiry. Healthy endpoints still allow normal
+        // concurrency; only a known failed endpoint gets a single probe. A stale
+        // one is replaced: it hung, and holding the candidate for it is an outage.
+        const probe = previous ? openProbe(this.now()) : undefined;
+        if (previous) previous.probe = probe;
+        const controller = new AbortController();
+        const inner = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+        let emitted = false;
+        try {
+          for await (const chunk of attempt(candidate, inner)) {
+            inner.throwIfAborted();
+            if (!emitted) {
+              this.recovered(candidate, previous);
+              probe?.settle();
+            }
+            emitted = true;
+            yield chunk;
+          }
           inner.throwIfAborted();
-          emitted = true;
-          // Only a call that was admitted as a recovery probe (started while on cooldown)
-          // is authorized to clear that cooldown upon succeeding.
-          if (startedWithCooldown && this.cooldowns.get(candidate) === previous) {
-            this.cooldowns.delete(candidate);
+          if (!emitted) this.recovered(candidate, previous);
+          return;
+        } catch (error) {
+          signal?.throwIfAborted();
+          const kind = classify(error);
+          if (kind === "aborted") throw error;
+          const configured = this.options.cooldownMs?.[kind];
+          const interval = configured === undefined ? DEFAULT_FALLBACK_COOLDOWNS[kind] : configured;
+          if (interval === undefined || interval === null) throw error;
+          const now = this.now();
+          const after = parseRetryAfterMs(error);
+          const reset = error instanceof ProviderError ? error.resetAtMs : undefined;
+          const hints = [reset, after === undefined ? undefined : now + after].filter(
+            (value): value is number => value !== undefined && Number.isFinite(value),
+          );
+          const retryAtMs = Math.max(
+            now,
+            ...(hints.length ? hints : [now + interval]),
+            this.cooldowns.get(candidate)?.retryAtMs ?? 0,
+          );
+          const window = error instanceof ProviderError ? error.window : undefined;
+          this.cooldowns.set(candidate, { retryAtMs, kind, window });
+          this.options.onCooldown?.({ candidate, error, kind, retryAtMs, window });
+          hasFailed = true;
+          lastError = error;
+          if (emitted) throw error;
+        } finally {
+          controller.abort();
+          // Only the owner releases its probe. Clearing whatever probe was set
+          // let a stale probe that ended late free the one that replaced it, and
+          // any call that did not fail let a second probe in.
+          if (probe) {
+            if (previous?.probe === probe) previous.probe = undefined;
+            probe.settle();
           }
-          yield chunk;
-        }
-        inner.throwIfAborted();
-        if (startedWithCooldown && this.cooldowns.get(candidate) === previous) {
-          this.cooldowns.delete(candidate);
         }
-        return;
-      } catch (error) {
-        signal?.throwIfAborted();
-        const kind = classify(error);
-        if (kind === "aborted") throw error;
-        const configured = this.options.cooldownMs?.[kind];
-        const interval = configured === undefined ? DEFAULT_FALLBACK_COOLDOWNS[kind] : configured;
-        if (interval === undefined || interval === null) throw error;
-        const now = this.now();
-        const after = parseRetryAfterMs(error);
-        const reset = error instanceof ProviderError ? error.resetAtMs : undefined;
-        const hints = [reset, after === undefined ? undefined : now + after].filter(
-          (value): value is number => value !== undefined && Number.isFinite(value),
-        );
-        const retryAtMs = Math.max(
-          now,
-          ...(hints.length ? hints : [now + interval]),
-          this.cooldowns.get(candidate)?.retryAtMs ?? 0,
+      }
+      if (hasFailed) throw lastError;
+      // Nothing was free, and a probe is deciding whether something will be.
+      // Failing here answered a 0 ms retry, which `withRetry` spent at once: at
+      // 70 concurrent calls every one but the probe failed. Wait for a probe to
+      // answer or end, or until the oldest goes stale, then look again.
+      if (probing.length > 0 && round < PROBE_WAITS) {
+        const staleAt = Math.min(...probing.map((probe) => probe.since + this.probeStaleMs));
+        await waitForProbe(
+          probing.map((probe) => probe.settled),
+          staleAt - this.now(),
+          signal,
         );
-        const window = error instanceof ProviderError ? error.window : undefined;
-        this.cooldowns.set(candidate, { retryAtMs, kind, window, probing: false });
-        this.options.onCooldown?.({ candidate, error, kind, retryAtMs, window });
-        attemptFailed = true;
-        hasFailed = true;
-        lastError = error;
-        if (emitted) throw error;
-      } finally {
-        controller.abort();
-        const current = this.cooldowns.get(candidate);
-        if (current && current.probing && !attemptFailed) {
-          current.probing = false;
-        }
+        continue;
       }
+      // Every candidate is cooling, or its probe outlasted the waits; a probe
+      // still in flight makes the next opportunity now.
+      const retryAtMs = Math.min(
+        ...this.status().map((entry) => Math.max(this.now(), entry.retryAtMs)),
+      );
+      throw new NoAvailableProviderError(retryAtMs, this.now());
+    }
+  }
+
+  /** An answer proves the candidate serves again. It lifts the bench this call
+   *  was admitted under as the probe, and any bench a sibling set while it ran:
+   *  a call that started before one stray 401 and then answered used to leave
+   *  a busy primary benched for an hour. Rate and quota stay. A call that
+   *  started before the throttle proves nothing about the window, and lifting
+   *  it walks every caller back into the 429. */
+  private recovered(candidate: T, admittedUnder: Cooldown | undefined): void {
+    const current = this.cooldowns.get(candidate);
+    if (!current) return;
+    if (current === admittedUnder || (current.kind !== "rate" && current.kind !== "quota")) {
+      this.cooldowns.delete(candidate);
     }
-    if (hasFailed) throw lastError;
-    // If all expired candidates are already being probed, the next opportunity
-    // is now. Other callers need not wait for those probes or start duplicates.
-    const retryAtMs = Math.min(
-      ...this.status().map((entry) => Math.max(this.now(), entry.retryAtMs)),
-    );
-    throw new NoAvailableProviderError(retryAtMs, this.now());
   }
 }