@providerkit/core 0.15.0 → 0.16.0

package/src/transport.ts

@@ -5,7 +5,7 @@
 // Adapters keep only their per-event mapping; everything about being an HTTP
 // client lives here once.
 import { classifyHttp, isTransportFailure, parseRetryAfterMs, ProviderError } from "./errors.ts";
-import { parseRateLimitResponse } from "./rate-limit.ts";
+import { parseRateLimitReset, parseRateLimitResponse } from "./rate-limit.ts";
 
 export interface RequestInit_ {
   url: string;
@@ -16,8 +16,36 @@ export interface RequestInit_ {
   signal?: AbortSignal;
   /** Swapped in tests, or to route through a proxy. Defaults to global fetch. */
   fetchImpl?: typeof fetch;
+  /** Called when the response headers arrive and on every body read after —
+   *  keep-alives included. What the watchdog's idle clock runs on. */
+  onActivity?: () => void;
+  /** Called when the body states `[DONE]`. See `SseHooks.onDone`. */
+  onDone?: () => void;
 }
 
+export interface SseHooks {
+  /** Every read of the body, keep-alives and partial frames included: the
+   *  stream is alive even when no event is complete yet. */
+  onActivity?: () => void;
+  /**
+   * The body said `[DONE]`. The payload itself is swallowed, but its arrival is
+   * the OpenAI dialect's only proof that a stream which stated no finish reason
+   * was not simply cut off.
+   */
+  onDone?: () => void;
+  /** Largest partial frame held before giving up. Defaults to 8 MiB. */
+  maxFrameChars?: number;
+  /** Names the provider on the oversized-frame error. */
+  provider?: string;
+}
+
+/**
+ * One frame larger than this is not a frame. Without a cap a malformed or
+ * runaway upstream grows the buffer until the runtime dies — an MV3 worker
+ * first. cc-proxy caps the ChatGPT backend at the same 8 MiB.
+ */
+export const MAX_SSE_FRAME_CHARS = 8 * 1024 * 1024;
+
 /** Join a base URL and a path without doubling or dropping the slash. */
 export function apiUrl(baseUrl: string, path: string): string {
   return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
@@ -30,12 +58,12 @@ export function apiUrl(baseUrl: string, path: string): string {
  * per-minute throttle, because "try again in a moment" is a lie for those.
  */
 export function retryAfterFromHeaders(headers: Headers, now = Date.now()): number | undefined {
-  const retryAfter = headers.get("retry-after");
-  if (retryAfter) {
-    if (/^\d+$/.test(retryAfter)) return Number(retryAfter) * 1000;
-    const date = Date.parse(retryAfter);
-    if (!Number.isNaN(date)) return Math.max(0, date - now);
-  }
+  // `retry-after-ms`, then `Retry-After` as seconds (fractions included), a
+  // duration or an HTTP-date — one parser for both modules. This one used to
+  // test for digits and hand anything else to `Date.parse`, which reads "1.5"
+  // as a day in 2001: a wait of zero, and an immediate retry into the throttle.
+  const asked = parseRateLimitReset(headers, now).retryAfterMs;
+  if (asked !== undefined) return asked;
   // Anthropic and several gateways publish an epoch-seconds reset instead.
   for (const name of [
     "anthropic-ratelimit-unified-reset",
@@ -64,7 +92,17 @@ export function retryAfterFromHeaders(headers: Headers, now = Date.now()): numbe
 async function errorFor(provider: string, res: Response): Promise<ProviderError> {
   const text = await res.text().catch(() => "");
   const kind = classifyHttp(res.status, text);
-  const reset = parseRateLimitResponse(res.headers, text);
+  // A window-reset header describes the ACCOUNT, not this failure, and only a
+  // throttle or a quota answer is about the window. Claude OAuth stamps the
+  // unified reset on every response, so read on any status a 529 said "lifts in
+  // three days": past every retry budget, and the backup walker benched the
+  // model until the weekly window rolled — the overload that most wants another
+  // attempt got none. A server's own Retry-After is honoured on any status.
+  const aboutWindow = kind === "rate" || kind === "quota";
+  const reset = aboutWindow ? parseRateLimitResponse(res.headers, text) : {};
+  const asked = aboutWindow
+    ? retryAfterFromHeaders(res.headers)
+    : parseRateLimitReset(res.headers).retryAfterMs;
   const message = text
     ? `${provider} ${res.status}: ${text.slice(0, 500)}`
     : `${provider} ${res.status} ${res.statusText}`;
@@ -75,7 +113,7 @@ async function errorFor(provider: string, res: Response): Promise<ProviderError>
     status: res.status,
     ...reset,
     shouldRetry,
-    retryAfterMs: retryAfterFromHeaders(res.headers) ?? parseRetryAfterMs({}, text),
+    retryAfterMs: asked ?? parseRetryAfterMs({}, text),
     body: text.slice(0, 2_000) || undefined,
   });
 }
@@ -136,8 +174,9 @@ function payloadOf(frame: string): string | null {
  *
  * Frames are split on the blank line the spec requires, so a payload containing
  * a bare newline survives; `[DONE]` is swallowed here rather than in every
- * adapter. CRLF is normalized — some gateways send it, and a `\r` left on the
- * end of a JSON payload is a parse error nobody enjoys debugging.
+ * adapter, and reported through `hooks.onDone`. CRLF and bare CR are
+ * normalized — some gateways send them, and a `\r` left on the end of a JSON
+ * payload is a parse error nobody enjoys debugging.
  *
  * Exported apart from `streamSse` because the envelope above it is the half an
  * adopting app most often cannot take: an app with its own translated error
@@ -145,33 +184,61 @@ function payloadOf(frame: string): string | null {
  * request and reading the failure itself. Framing is the half nobody should
  * write twice — hand it a `res.body` and keep your own envelope.
  */
-export async function* parseSseStream(body: ReadableStream<Uint8Array>): AsyncGenerator<string> {
+export async function* parseSseStream(
+  body: ReadableStream<Uint8Array>,
+  hooks: SseHooks = {},
+): AsyncGenerator<string> {
   const reader = body.getReader();
   const decoder = new TextDecoder();
+  const maxFrame = hooks.maxFrameChars ?? MAX_SSE_FRAME_CHARS;
   let buffer = "";
+  // A CR at the very end of a read may be the first half of a CRLF; it waits
+  // for the next read before it is turned into a line break.
+  let heldCR = false;
+
+  function* emit(frame: string): Generator<string> {
+    const payload = payloadOf(frame);
+    if (payload === "[DONE]") hooks.onDone?.();
+    else if (payload !== null) yield payload;
+  }
+
   try {
     for (;;) {
       const { done, value } = await reader.read();
       if (done) break;
-      // Normalized on the BUFFER, not on the decoded read: a network read can
-      // end between the CR and the LF, and normalizing each read separately
-      // leaves that CR stranded on the end of a payload — a JSON parse error
-      // that only ever reproduces under a particular packet split.
-      buffer = (buffer + decoder.decode(value, { stream: true })).replace(/\r\n/g, "\n");
+      hooks.onActivity?.();
+      let text: string = (heldCR ? "\r" : "") + decoder.decode(value, { stream: true });
+      heldCR = text.endsWith("\r");
+      if (heldCR) text = text.slice(0, -1);
+      // Line endings are normalized on the new text only, CRLF and the bare CR
+      // the spec also allows. Normalizing the whole buffer on every read made a
+      // multi-megabyte frame quadratic; normalizing each read on its own, with
+      // no carry, strands the CR of a CRLF split across reads — a JSON parse
+      // error that only reproduces under one packet split.
+      text = text.replace(/\r\n?/g, "\n");
+      // The boundary may straddle the join, so the search starts one char back.
+      let from = Math.max(0, buffer.length - 1);
+      buffer += text;
 
-      let boundary = buffer.indexOf("\n\n");
+      let boundary = buffer.indexOf("\n\n", from);
       while (boundary !== -1) {
-        const frame = buffer.slice(0, boundary);
+        yield* emit(buffer.slice(0, boundary));
         buffer = buffer.slice(boundary + 2);
-        const payload = payloadOf(frame);
-        if (payload !== null && payload !== "[DONE]") yield payload;
+        from = 0;
         boundary = buffer.indexOf("\n\n");
       }
+      if (buffer.length > maxFrame) {
+        throw new ProviderError(
+          hooks.provider ?? "sse",
+          "overload",
+          `an SSE frame grew past ${maxFrame} characters without ending`,
+          { shouldRetry: false },
+        );
+      }
     }
     // A stream that ends without its final blank line still has an event in
     // hand — dropping it loses the last delta, or the usage record.
-    const tail = payloadOf(buffer);
-    if (tail !== null && tail !== "[DONE]") yield tail;
+    yield* emit(buffer + decoder.decode() + (heldCR ? "\n" : ""));
   } finally {
     reader.releaseLock();
   }
@@ -183,6 +250,10 @@ export async function* parseSseStream(body: ReadableStream<Uint8Array>): AsyncGe
  */
 export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
   const res = await send(opts);
+  // Headers are the first sign of life — the idle clock starts here, not at
+  // the POST: a backend that withholds them until its first output token (the
+  // ChatGPT backend does, for minutes on a large high-effort turn) is working.
+  opts.onActivity?.();
   if (!res.ok) throw await errorFor(opts.provider, res);
   // A 2xx with no body at all is an upstream anomaly, not a request we got
   // wrong — worth the same retry a 5xx gets.
@@ -191,5 +262,5 @@ export async function* streamSse(opts: RequestInit_): AsyncGenerator<string> {
       status: res.status,
     });
   }
-  yield* parseSseStream(res.body);
+  yield* parseSseStream(res.body, opts);
 }

package/src/types.ts

@@ -151,10 +151,19 @@ export interface TokenUsage {
    *  prefixes and bill the hit portion far cheaper, so it must be tracked
    *  separately to cost a turn correctly. 0 when the provider reports none. */
   cachedInputTokens: number;
-  /** Tokens WRITTEN to cache. Anthropic bills these above the input rate;
-   *  the OpenAI-shape auto-cachers bill them at it. 0 when not reported. */
+  /** Tokens WRITTEN to cache — also a subset of `inputTokens`, so the total
+   *  stays the whole prompt the window has to hold. Anthropic bills these
+   *  above the input rate; the OpenAI-shape auto-cachers bill them at it.
+   *  0 when not reported. */
   cacheWriteTokens?: number;
   outputTokens: number;
+  /**
+   * The part of `outputTokens` spent thinking, when the provider reports it —
+   * already INSIDE `outputTokens`, so never add it again. It is the number
+   * that explains a turn that came back empty with a length finish: the
+   * reasoning ate the budget. Anthropic does not report it; absent there.
+   */
+  reasoningTokens?: number;
   /**
    * What the provider says this call cost, in USD: its bill, not our
    * arithmetic. Only OpenRouter sends one today. It is the one number a rate
@@ -363,6 +372,21 @@ export interface StreamOptions {
    * caches by it); every other endpoint ignores it.
    */
   sessionId?: string;
+  /**
+   * Called when the response headers arrive and on every read of the body
+   * after, keep-alives included. `withWatchdog` sets it: a stream that is only
+   * sending keep-alives is alive, which no chunk an adapter yields can show.
+   * Every adapter here forwards it; a custom Provider that does not still
+   * works, on the watchdog's progress clock alone.
+   */
+  onActivity?: () => void;
+  /**
+   * `false` asks for at most one tool call per turn — for tools that must run
+   * in order, or a backend lane that refuses parallel calls. Absent or `true`
+   * is every vendor's default and sends nothing. Gemini has no such switch,
+   * so it refuses `false` rather than silently ignore it.
+   */
+  parallelToolCalls?: boolean;
 }
 
 export interface Provider {

package/src/usage.ts

@@ -42,6 +42,9 @@ export function addUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
     cachedInputTokens: a.cachedInputTokens + b.cachedInputTokens,
     cacheWriteTokens: (a.cacheWriteTokens ?? 0) + (b.cacheWriteTokens ?? 0),
     outputTokens: a.outputTokens + b.outputTokens,
+    ...(a.reasoningTokens !== undefined || b.reasoningTokens !== undefined
+      ? { reasoningTokens: (a.reasoningTokens ?? 0) + (b.reasoningTokens ?? 0) }
+      : {}),
   };
 }
 
@@ -55,6 +58,9 @@ export function subtractUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
     cachedInputTokens: Math.max(0, a.cachedInputTokens - b.cachedInputTokens),
     cacheWriteTokens: Math.max(0, (a.cacheWriteTokens ?? 0) - (b.cacheWriteTokens ?? 0)),
     outputTokens: Math.max(0, a.outputTokens - b.outputTokens),
+    ...(a.reasoningTokens !== undefined
+      ? { reasoningTokens: Math.max(0, a.reasoningTokens - (b.reasoningTokens ?? 0)) }
+      : {}),
   };
 }
 
@@ -64,17 +70,20 @@ export function subtractUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
  * A cost the provider reported wins outright, including a reported 0: a free
  * call priced at the rate would be billed for something nobody charged.
  *
- * Cached tokens are a SUBSET of input, not an addition to it: the miss part
- * bills at the full rate and the hit part at the cache rate. `cached` is
- * clamped to `input` so an over-reporting provider can neither drive the miss
- * count negative nor bill for more prompt than it was sent.
+ * Cached and written tokens are SUBSETS of input, not additions to it: the
+ * miss part bills at the full rate, the hit part at the cache rate, the write
+ * part at the write rate. `cached` is clamped to `input` so an over-reporting
+ * provider can neither drive the miss count negative nor bill for more prompt
+ * than it was sent.
  */
 export function costUsd(usage: TokenUsage, rate: ModelRate): number {
   if (usage.reportedCostUsd !== undefined) return usage.reportedCostUsd;
   const input = Math.max(0, usage.inputTokens);
   const cached = Math.min(Math.max(0, usage.cachedInputTokens), input);
   const written = Math.max(0, usage.cacheWriteTokens ?? 0);
-  const miss = input - cached;
+  // Writes are inside `input` too. Left in the miss count they billed twice —
+  // once at the input rate, again at the write rate: 2.25× on Claude.
+  const miss = Math.max(0, input - cached - written);
   const writeRate = rate.cacheWrite ?? rate.input;
   return (
     (miss * rate.input +

package/src/watchdog.ts

@@ -1,17 +1,30 @@
 // The two ways a stream fails without failing: it goes silent, or it ends
 // having said nothing at all.
 //
-// The stream-idle watchdog.
+// The stream watchdog.
 //
 // A provider that stops sending bytes is indistinguishable from a long prefill
 // — except that it never ends, and every SDK's default is to wait forever. A
 // queued route or a wedged prefill upstream hangs the caller indefinitely, and
 // the symptom is the worst kind: nothing. No error, no log, no timeout.
 //
-// So the seam gives itself a deadline. Any byte of any kind re-arms it
-// (reasoning models emit thinking deltas continuously, so silence really is
-// silence). When it fires, the watchdog aborts ITS OWN controller and the
-// caller's signal is only bridged in — which is what keeps a person's Stop
+// So the seam gives itself two deadlines, because "silent" means two things:
+//
+// - IDLE: no byte at all, keep-alives included, for a minute after the
+//   response started. A dead socket. Any read re-arms it — a keep-alive is
+//   proof of life no chunk can carry (cc-proxy pings every 15s while it
+//   buffers a long tool call, and the old chunk-only clock killed those turns).
+// - PROGRESS: no chunk for five minutes, counted from the POST. A stream that
+//   is alive and going nowhere — a route parked behind keep-alives — and the
+//   wait for the response to START, which is the one some backends take
+//   minutes over: the ChatGPT backend withholds its headers until the model's
+//   first output, so a high-effort turn on a large prompt sends nothing at all
+//   for minutes and is healthy the whole time (cc-proxy defaults that wait to
+//   300s, measured). On a one-minute clock it was aborted and retried from
+//   scratch, which re-ran the reasoning into the same deadline.
+//
+// When either fires, the watchdog aborts ITS OWN controller and the caller's
+// signal is only bridged in — which is what keeps a person's Stop
 // distinguishable from our timeout. One is their cancel and is never retried;
 // the other is ours, is transient, and fires while nothing has streamed yet,
 // so the retry is always safe.
@@ -24,44 +37,54 @@ import type {
   ToolDefinition,
 } from "./types.ts";
 
-/** No byte at all for this long and the stream is considered wedged. */
+/** No byte at all for this long, once the response has started, and the
+ *  stream is dead. */
 export const STREAM_IDLE_MS = 60_000;
 
+/** No chunk for this long — the wait for the response to start included —
+ *  and the stream is going nowhere, keep-alives or not. */
+export const STREAM_PROGRESS_MS = 300_000;
+
 export interface StreamWatch {
   /** Hand this to the provider in place of the caller's signal. */
   readonly signal: AbortSignal;
-  /** A byte arrived: re-arm the deadline, and mark TTFT if it was the first. */
+  /** A chunk arrived: re-arm both deadlines, and mark TTFT if it was the first. */
   sawByte(): void;
+  /** The response showed life without a chunk — its headers, a keep-alive, a
+   *  partial frame. Starts or re-arms the idle deadline only. */
+  sawActivity(): void;
   /**
-   * Milliseconds from the call opening to its first byte of any kind — the
+   * Milliseconds from the call opening to its first chunk of any kind — the
    * wait a person actually experiences, and the number a prompt-cache pin
    * exists to shrink. Null until something arrives.
    */
   firstChunkMs(): number | null;
   /**
-   * Re-issue a provider failure as the idle timeout when — and only when — it
-   * was our deadline that aborted. A caller's Stop passes through untouched.
+   * Re-issue a provider failure as the timeout when — and only when — it was
+   * one of our deadlines that aborted. A caller's Stop passes through untouched.
    */
   classify(err: unknown): unknown;
-  /** Clear the deadline timer. Safe to call more than once. */
+  /** Clear the deadline timers. Safe to call more than once. */
   dispose(): void;
 }
 
 export interface StreamWatchOptions {
   provider?: string;
   idleMs?: number;
+  progressMs?: number;
   signal?: AbortSignal;
 }
 
 export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   const provider = opts.provider ?? "provider";
   const idleMs = opts.idleMs ?? STREAM_IDLE_MS;
+  const progressMs = opts.progressMs ?? STREAM_PROGRESS_MS;
   const callerSignal = opts.signal;
   const started = Date.now();
   const timeout = new AbortController();
 
   let firstChunk: number | null = null;
-  let idle = false;
+  let fired: ProviderError | null = null;
   let disposed = false;
 
   // The bridge is structural rather than an event listener: AbortSignal.any
@@ -71,40 +94,49 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   // nicety: every runtime this package targets has had it for years.
   const signal = callerSignal ? AbortSignal.any([callerSignal, timeout.signal]) : timeout.signal;
 
-  const idleError = (cause?: unknown) =>
-    new ProviderError(provider, "timeout", `stream went ${idleMs / 1000}s without a byte`, {
-      cause,
-    });
-
-  function arm(): ReturnType<typeof setTimeout> {
+  function deadline(ms: number, what: string): ReturnType<typeof setTimeout> {
     const timer = setTimeout(() => {
-      idle = true;
-      timeout.abort(idleError());
-    }, idleMs);
+      fired = new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`);
+      timeout.abort(fired);
+    }, ms);
     // An orphaned watch — its consumer gone, dispose never called — must not
     // hold a Node event loop open for a full deadline.
     (timer as { unref?: () => void }).unref?.();
     return timer;
   }
 
-  let timer = arm();
+  // The idle clock waits for the response to start; until then only the
+  // progress clock runs (see the header comment).
+  let idle: ReturnType<typeof setTimeout> | undefined;
+  let progress = deadline(progressMs, "a chunk");
+  const live = () => !disposed && !signal.aborted;
+
+  function sawActivity() {
+    clearTimeout(idle);
+    if (live()) idle = deadline(idleMs, "a byte");
+  }
 
   return {
     signal,
     sawByte() {
       firstChunk ??= Date.now() - started;
-      clearTimeout(timer);
-      if (!disposed && !signal.aborted) timer = arm();
+      sawActivity();
+      clearTimeout(progress);
+      if (live()) progress = deadline(progressMs, "a chunk");
     },
+    sawActivity,
     firstChunkMs: () => firstChunk,
     classify(err: unknown) {
       // Our deadline, not theirs — and not the caller's Stop.
-      if (idle && !(callerSignal?.aborted ?? false)) return idleError(err);
+      if (fired && !(callerSignal?.aborted ?? false)) {
+        return new ProviderError(fired.provider, "timeout", fired.message, { cause: err });
+      }
       return err;
     },
     dispose() {
       disposed = true;
-      clearTimeout(timer);
+      clearTimeout(idle);
+      clearTimeout(progress);
     },
   };
 }
@@ -148,11 +180,14 @@ export async function* watchChunks<T>(
 export async function* requireContent<T extends ProviderChunk>(
   provider: string,
   chunks: AsyncIterable<T>,
+  opts: { afterToolResult?: boolean } = {},
 ): AsyncGenerator<T> {
   const held: T[] = [];
   let usable = false;
+  let finish: ProviderChunk["finishReason"];
 
   for await (const chunk of chunks) {
+    if (chunk.type === "finish") finish = chunk.finishReason;
     if (!usable) {
       usable = Boolean(chunk.content || chunk.reasoning || chunk.toolCalls?.length);
       // Held rather than forwarded: once a chunk is out, the stream is
@@ -169,14 +204,26 @@ export async function* requireContent<T extends ProviderChunk>(
     yield chunk;
   }
 
-  if (!usable) {
-    throw new ProviderError(provider, "overload", `${provider}: completed with no content`);
+  if (usable) return;
+  // The one empty turn that is an answer: the model read a tool result and
+  // chose to stop. An agent whose reply IS a tool (a chat bot's send_message)
+  // ends every turn this way — retried, cc-proxy measured one such bot
+  // failing ~68% of real turns (11 retries, ~3 minutes, then an error). A
+  // `length` stop or no finish at all is still the failure this guard is for.
+  if (opts.afterToolResult && finish === "stop") {
+    yield* held;
+    return;
   }
+  throw new ProviderError(provider, "overload", `${provider}: completed with no content`);
 }
 
 export interface WatchdogOptions {
-  /** Silence this long and the stream is wedged. Defaults to `STREAM_IDLE_MS`. */
+  /** No byte this long after the response starts and the stream is dead.
+   *  Defaults to `STREAM_IDLE_MS`. */
   idleMs?: number;
+  /** No chunk this long, from the POST on, and the stream is going nowhere.
+   *  Defaults to `STREAM_PROGRESS_MS`. */
+  progressMs?: number;
   /**
    * Reject a turn that ends having said nothing, as `requireContent` does. On
    * by default: an empty completion is a failure in every loop, and the one
@@ -214,20 +261,26 @@ export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Pr
       // Armed on first read, not here: a stream built now and iterated later
       // must not spend its deadline sitting in a variable.
       async function* watched(): AsyncGenerator<ProviderChunk> {
-        // idleMs and signal both default inside streamWatch.
+        // The deadlines and signal all default inside streamWatch.
         const watch = streamWatch({
           provider: provider.id,
           idleMs: opts.idleMs,
+          progressMs: opts.progressMs,
           signal: streamOpts.signal,
         });
+        const callerActivity = streamOpts.onActivity;
         const source = provider.createStream(messages, tools, {
           ...streamOpts,
           signal: watch.signal,
+          onActivity: () => {
+            watch.sawActivity();
+            callerActivity?.();
+          },
         });
         let reported = false;
         for await (const chunk of watchChunks(watch, source)) {
           // Before `requireContent` holds anything back — TTFT is the first
-          // byte of any kind, not the first byte worth showing.
+          // chunk of any kind, not the first one worth showing.
           if (!reported) {
             reported = true;
             opts.onFirstChunk?.(watch.firstChunkMs() ?? 0);
@@ -236,7 +289,11 @@ export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Pr
         }
       }
 
-      return opts.requireContent === false ? watched() : requireContent(provider.id, watched());
+      return opts.requireContent === false
+        ? watched()
+        : requireContent(provider.id, watched(), {
+            afterToolResult: messages.at(-1)?.role === "tool",
+          });
     },
   };
 }