@providerkit/core 0.22.0 → 0.23.0

package/src/transport.ts

@@ -4,8 +4,12 @@
 //
 // Adapters keep only their per-event mapping; everything about being an HTTP
 // client lives here once.
-import { classifyHttp, isTransportFailure, parseRetryAfterMs, ProviderError } from "./errors.ts";
-import { parseRateLimitReset, parseRateLimitResponse } from "./rate-limit.ts";
+import { classifyHttp, parseRetryAfterMs, ProviderError } from "./errors.ts";
+import {
+  parseRateLimitReset,
+  parseRateLimitResponse,
+  type RateLimitReading,
+} from "./rate-limit.ts";
 
 export interface RequestInit_ {
   url: string;
@@ -21,6 +25,11 @@ export interface RequestInit_ {
   onActivity?: () => void;
   /** Called when the body states `[DONE]`. See `SseHooks.onDone`. */
   onDone?: () => void;
+  /** Called when the response headers report how full a subscription window
+   *  is. See `StreamOptions.onRateLimit`. */
+  onRateLimit?: (reading: RateLimitReading) => void;
+  /** Whether the turn already stated its finish. See `SseHooks.finished`. */
+  finished?: () => boolean;
 }
 
 export interface SseHooks {
@@ -33,6 +42,18 @@ export interface SseHooks {
    * was not simply cut off.
    */
   onDone?: () => void;
+  /**
+   * Whether the turn already stated its finish, so only trailing frames (the
+   * usage record, `[DONE]`) are left. Once this says true, a read that fails
+   * ends the stream instead of throwing. Kimi, GLM and OpenCode Go drop the
+   * socket there, and the failure threw away an answer the reader had already
+   * been shown (cc-proxy, measured on live traffic). The usage may be lost;
+   * the answer is not.
+   */
+  finished?: () => boolean;
+  /** The request's signal. A caller's Stop is never swallowed, finished or not:
+   *  a loop that sees a clean end goes on to run the turn's tools. */
+  signal?: AbortSignal;
   /** Largest partial frame held before giving up. Defaults to 8 MiB. */
   maxFrameChars?: number;
   /** Names the provider on the oversized-frame error. */
@@ -120,18 +141,25 @@ async function errorFor(provider: string, res: Response): Promise<ProviderError>
 
 async function send(opts: RequestInit_): Promise<Response> {
   const doFetch = opts.fetchImpl ?? globalThis.fetch;
+  // Serialized outside the catch below: a body that cannot become JSON is a
+  // bug of ours, and must not be dressed as a network blip and retried.
+  const body = typeof opts.body === "string" ? opts.body : JSON.stringify(opts.body);
   try {
     return await doFetch(opts.url, {
       method: "POST",
       headers: { "content-type": "application/json", ...opts.headers },
-      body: typeof opts.body === "string" ? opts.body : JSON.stringify(opts.body),
+      body,
       signal: opts.signal,
     });
   } catch (err) {
-    // A stopped run rejects here too, and that is not a failure — let it pass
-    // through untouched. Anything that is not a recognizable transport
-    // rejection is a bug of ours and keeps its own loud shape.
-    if (opts.signal?.aborted || !isTransportFailure(err)) throw err;
+    // An aborted signal passes through untouched: the caller's Stop stays
+    // `aborted`, and the watchdog re-issues its own abort as `timeout`
+    // (invariant 2). Every other rejection means no response came back, so it
+    // is a transport failure whatever its wording. Matching known wordings
+    // missed each runtime's next one, and a missed one was never retried: a
+    // long run died on its first blip. cc-proxy reads every failed send the
+    // same way.
+    if (opts.signal?.aborted) throw err;
     throw new ProviderError(opts.provider, "network", `no response from ${opts.url}`, {
       cause: err,
     });
@@ -173,10 +201,11 @@ function payloadOf(frame: string): string | null {
  * (see `payloadOf`).
  *
  * Frames are split on the blank line the spec requires, so a payload containing
- * a bare newline survives; `[DONE]` is swallowed here rather than in every
- * adapter, and reported through `hooks.onDone`. CRLF and bare CR are
- * normalized: some gateways send them, and a `\r` left on the end of a JSON
- * payload is a parse error nobody enjoys debugging.
+ * a bare newline survives. `[DONE]` is swallowed here rather than in every
+ * adapter, reported through `hooks.onDone`, and ends the stream: it is the
+ * body's last word, and a read after it can only fail a finished turn. CRLF and
+ * bare CR are normalized: some gateways send them, and a `\r` left on the end
+ * of a JSON payload is a parse error nobody enjoys debugging.
  *
  * Exported apart from `streamSse` because the envelope above it is the half an
  * adopting app most often cannot take: an app with its own translated error
@@ -195,17 +224,36 @@ export async function* parseSseStream(
   // A CR at the very end of a read may be the first half of a CRLF; it waits
   // for the next read before it is turned into a line break.
   let heldCR = false;
+  // Whether the body was read to its end. Any other exit (`[DONE]`, a failed
+  // read, a consumer that stopped at its own terminal event) cancels it, so the
+  // connection is let go now rather than whenever the body is collected.
+  let drained = false;
 
-  function* emit(frame: string): Generator<string> {
+  /** Yields the frame's payload, and returns true when the frame was `[DONE]`. */
+  function* emit(frame: string): Generator<string, boolean> {
     const payload = payloadOf(frame);
-    if (payload === "[DONE]") hooks.onDone?.();
-    else if (payload !== null) yield payload;
+    if (payload === "[DONE]") {
+      hooks.onDone?.();
+      return true;
+    }
+    if (payload !== null) yield payload;
+    return false;
   }
 
   try {
     for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
+      let read: ReadableStreamReadResult<Uint8Array>;
+      try {
+        read = await reader.read();
+      } catch (err) {
+        if (hooks.finished?.() && !hooks.signal?.aborted) return;
+        throw err;
+      }
+      const { done, value } = read;
+      if (done) {
+        drained = true;
+        break;
+      }
       hooks.onActivity?.();
       let text: string = (heldCR ? "\r" : "") + decoder.decode(value, { stream: true });
       heldCR = text.endsWith("\r");
@@ -222,7 +270,7 @@ export async function* parseSseStream(
 
       let boundary = buffer.indexOf("\n\n", from);
       while (boundary !== -1) {
-        yield* emit(buffer.slice(0, boundary));
+        if (yield* emit(buffer.slice(0, boundary))) return;
         buffer = buffer.slice(boundary + 2);
         from = 0;
         boundary = buffer.indexOf("\n\n");
@@ -240,10 +288,32 @@ export async function* parseSseStream(
     // hand — dropping it loses the last delta, or the usage record.
     yield* emit(buffer + decoder.decode() + (heldCR ? "\n" : ""));
   } finally {
+    if (!drained) reader.cancel().catch(() => {});
     reader.releaseLock();
   }
 }
 
+/**
+ * Parse one SSE payload as the JSON event it must be.
+ *
+ * A frame that is not JSON used to be skipped as "a keep-alive or a frame we do
+ * not model". But `parseSseStream` already drops comments and empty frames,
+ * which is every keep-alive the vendors send, so what reaches here unparsable
+ * is a broken stream: a delta, a tool call or the finish itself, gone, with the
+ * turn still reading as whole. It fails as `network`, the same as a socket that
+ * dies mid-read (cc-proxy fails these too).
+ */
+export function parseFrame<T>(data: string, provider: string): T {
+  try {
+    return JSON.parse(data) as T;
+  } catch (err) {
+    throw new ProviderError(provider, "network", `${provider}: a stream frame was not JSON`, {
+      body: data.slice(0, 2_000),
+      cause: err,
+    });
+  }
+}
+
 /**
  * POST an SSE request and stream its payloads: the envelope (auth, the
  * classified error, the retry hints) plus `parseSseStream`.
@@ -254,6 +324,16 @@ export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
   // the POST: a backend that withholds them until its first output token (the
   // ChatGPT backend does, for minutes on a large high-effort turn) is working.
   opts.onActivity?.();
+  // The window's fill rides every response, the ones that go through too.
+  // Read only off a refusal, a caller learns the window was filling once it
+  // is already full.
+  if (opts.onRateLimit) {
+    const reading = parseRateLimitReset(res.headers);
+    const { utilization, resetAtMs } = reading;
+    if (utilization !== undefined && resetAtMs !== undefined) {
+      opts.onRateLimit({ ...reading, utilization, resetAtMs });
+    }
+  }
   if (!res.ok) throw await errorFor(opts.provider, res);
   // A 2xx with no body at all is an upstream anomaly, not a request we got
   // wrong — worth the same retry a 5xx gets.

package/src/types.ts

@@ -6,6 +6,7 @@
 // the lingua franca forces the other adapters to round-trip through a dialect
 // that isn't theirs, and every quirk of that dialect then leaks into callers
 // who never asked for it.
+import type { RateLimitReading } from "./rate-limit.ts";
 
 /**
  * How hard the model thinks before answering. Absent = the provider's own
@@ -397,6 +398,18 @@ export interface StreamOptions {
    * works, on the watchdog's progress clock alone.
    */
   onActivity?: () => void;
+  /**
+   * Called once per response whose headers report how full a subscription
+   * window is, whatever its status: `utilization` (1 is full), `resetAtMs`,
+   * and `window` when the headers name it. A caller sees the window fill
+   * turn by turn, instead of learning about it from the 429 that ends a run.
+   *
+   * Every adapter forwards it. Two backends send the headers today: Claude
+   * subscriptions (Anthropic's unified 5h/7d headers) and the ChatGPT backend
+   * (`x-codex-*`). A response that reports no window never calls it, which
+   * covers API-key accounts on every vendor and all of Gemini.
+   */
+  onRateLimit?: (reading: RateLimitReading) => void;
   /**
    * `false` asks for at most one tool call per turn, for tools that must run
    * in order, or a backend lane that refuses parallel calls. Absent or `true`

package/src/watchdog.ts

@@ -24,10 +24,14 @@
 //   scratch, which re-ran the reasoning into the same deadline.
 //
 // When either fires, the watchdog aborts ITS OWN controller and the caller's
-// signal is only bridged in — which is what keeps a person's Stop
-// distinguishable from our timeout. One is their cancel and is never retried;
-// the other is ours, is transient, and fires while nothing has streamed yet,
-// so the retry is always safe.
+// signal is only bridged in, which is what keeps a person's Stop
+// distinguishable from our timeout. One is their cancel and is never retried.
+// The other is ours and transient, with one exception: a progress timeout
+// before the response started carries `shouldRetry: false`. That wait was a
+// backend thinking with its headers held back, and a replay sends the same
+// prompt into the same 300s and pays for the reasoning again, up to three
+// times (cc-proxy never replays slow headers). Its kind stays `timeout`, so a
+// fallback pool still moves on to its next candidate.
 import { ProviderError } from "./errors.ts";
 import type {
   ChatMessage,
@@ -91,6 +95,9 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   let firstChunk: number | null = null;
   let fired: ProviderError | null = null;
   let disposed = false;
+  // The first sign of life (the headers, a keep-alive, a chunk). The idle
+  // clock waits for it, and a timeout that fires before it is not replayed.
+  let responded = false;
 
   // The bridge is structural rather than an event listener: AbortSignal.any
   // aborts synchronously when an input is ALREADY aborted, which is the race
@@ -101,7 +108,13 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
 
   function deadline(ms: number, what: string): ReturnType<typeof setTimeout> {
     const timer = setTimeout(() => {
-      fired = new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`);
+      // Only the progress clock runs before the response starts, so this is
+      // the backend that thinks with its headers held back (see the header).
+      fired = responded
+        ? new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`)
+        : new ProviderError(provider, "timeout", `no response in ${ms / 1000}s`, {
+            shouldRetry: false,
+          });
       timeout.abort(fired);
     }, ms);
     // An orphaned watch — its consumer gone, dispose never called — must not
@@ -117,6 +130,7 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   const live = () => !disposed && !signal.aborted;
 
   function sawActivity() {
+    responded = true;
     clearTimeout(idle);
     if (live()) idle = deadline(idleMs, "a byte");
   }
@@ -134,7 +148,10 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
     classify(err: unknown) {
       // Our deadline, not theirs — and not the caller's Stop.
       if (fired && !(callerSignal?.aborted ?? false)) {
-        return new ProviderError(fired.provider, "timeout", fired.message, { cause: err });
+        return new ProviderError(fired.provider, "timeout", fired.message, {
+          shouldRetry: fired.shouldRetry,
+          cause: err,
+        });
       }
       return err;
     },
@@ -278,6 +295,8 @@ export interface WatchdogOptions {
  * copies. The result is still a `Provider`, so it composes unchanged with
  * `withStreamRetry` and `streamWithBackupModels`. Timeouts and empty turns
  * without a `length` finish are transient, so wrapping it in a retry is safe.
+ * The one timeout a replay would make worse, no response at all within the
+ * progress window, says so itself with `shouldRetry: false`.
  */
 export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Provider {
   return {