@providerkit/core 0.21.1 → 0.23.0

package/src/watchdog.ts

@@ -24,10 +24,14 @@
 //   scratch, which re-ran the reasoning into the same deadline.
 //
 // When either fires, the watchdog aborts ITS OWN controller and the caller's
-// signal is only bridged in — which is what keeps a person's Stop
-// distinguishable from our timeout. One is their cancel and is never retried;
-// the other is ours, is transient, and fires while nothing has streamed yet,
-// so the retry is always safe.
+// signal is only bridged in, which is what keeps a person's Stop
+// distinguishable from our timeout. One is their cancel and is never retried.
+// The other is ours and transient, with one exception: a progress timeout
+// before the response started carries `shouldRetry: false`. That wait was a
+// backend thinking with its headers held back, and a replay sends the same
+// prompt into the same 300s and pays for the reasoning again, up to three
+// times (cc-proxy never replays slow headers). Its kind stays `timeout`, so a
+// fallback pool still moves on to its next candidate.
 import { ProviderError } from "./errors.ts";
 import type {
   ChatMessage,
@@ -42,7 +46,7 @@ import type {
  *  stream is dead. */
 export const STREAM_IDLE_MS = 60_000;
 
-/** No chunk for this long — the wait for the response to start included —
+/** No chunk for this long, including the wait for the response to start,
  *  and the stream is going nowhere, keep-alives or not. */
 export const STREAM_PROGRESS_MS = 300_000;
 
@@ -51,17 +55,17 @@ export interface StreamWatch {
   readonly signal: AbortSignal;
   /** A chunk arrived: re-arm both deadlines, and mark TTFT if it was the first. */
   sawByte(): void;
-  /** The response showed life without a chunk — its headers, a keep-alive, a
+  /** The response showed life without a chunk: its headers, a keep-alive, a
    *  partial frame. Starts or re-arms the idle deadline only. */
   sawActivity(): void;
   /**
-   * Milliseconds from the call opening to its first chunk of any kind — the
+   * Milliseconds from the call opening to its first chunk of any kind. The
    * wait a person actually experiences, and the number a prompt-cache pin
    * exists to shrink. Null until something arrives.
    */
   firstChunkMs(): number | null;
   /**
-   * Re-issue a provider failure as the timeout when — and only when — it was
+   * Re-issue a provider failure as the timeout only when it was
    * one of our deadlines that aborted. A caller's Stop passes through untouched.
    */
   classify(err: unknown): unknown;
@@ -91,6 +95,9 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   let firstChunk: number | null = null;
   let fired: ProviderError | null = null;
   let disposed = false;
+  // The first sign of life (the headers, a keep-alive, a chunk). The idle
+  // clock waits for it, and a timeout that fires before it is not replayed.
+  let responded = false;
 
   // The bridge is structural rather than an event listener: AbortSignal.any
   // aborts synchronously when an input is ALREADY aborted, which is the race
@@ -101,7 +108,13 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
 
   function deadline(ms: number, what: string): ReturnType<typeof setTimeout> {
     const timer = setTimeout(() => {
-      fired = new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`);
+      // Only the progress clock runs before the response starts, so this is
+      // the backend that thinks with its headers held back (see the header).
+      fired = responded
+        ? new ProviderError(provider, "timeout", `stream went ${ms / 1000}s without ${what}`)
+        : new ProviderError(provider, "timeout", `no response in ${ms / 1000}s`, {
+            shouldRetry: false,
+          });
       timeout.abort(fired);
     }, ms);
     // An orphaned watch — its consumer gone, dispose never called — must not
@@ -117,6 +130,7 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
   const live = () => !disposed && !signal.aborted;
 
   function sawActivity() {
+    responded = true;
     clearTimeout(idle);
     if (live()) idle = deadline(idleMs, "a byte");
   }
@@ -134,7 +148,10 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
     classify(err: unknown) {
       // Our deadline, not theirs — and not the caller's Stop.
       if (fired && !(callerSignal?.aborted ?? false)) {
-        return new ProviderError(fired.provider, "timeout", fired.message, { cause: err });
+        return new ProviderError(fired.provider, "timeout", fired.message, {
+          shouldRetry: fired.shouldRetry,
+          cause: err,
+        });
       }
       return err;
     },
@@ -148,7 +165,7 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
 
 /**
  * Wrap a stream so every chunk re-arms `watch`, and a failure is re-classified
- * through it. Disposes on any exit — completion, throw, or the consumer
+ * through it. Disposes on any exit: completion, throw, or the consumer
  * breaking out of the loop.
  */
 export async function* watchChunks<T>(
@@ -173,7 +190,7 @@ export async function* watchChunks<T>(
  * A stream that ends with no text and no tool call is a failure wearing a
  * success's clothes: `stop_reason: end_turn` with zero content blocks, which
  * the vendors emit under load, or a thinking block that ate the whole
- * `max_tokens`. Nothing throws, so nothing retries — the caller simply shows a
+ * `max_tokens`. Nothing throws, so nothing retries. The caller simply shows a
  * person an empty answer, and the only trace is a bill.
  *
  * Reasoning is not an answer. It still streams the moment it arrives, so a
@@ -185,12 +202,12 @@ export async function* watchChunks<T>(
  *
  * Classified by what fixes it. A `length` finish is the caller's own cap:
  * `invalid`, so nothing retries it into the same cap and no backup model is
- * cooled down for it — the fix is a larger `maxTokens` or a lower effort. Any
+ * cooled down for it. The fix is a larger `maxTokens` or a lower effort. Any
  * other empty turn is `overload`, because that is both true and useful: it is
  * theirs and temporary, so it is transient (the same model, retried, usually
  * answers) and backup-eligible (a model that keeps doing it should be walked
  * away from). Until something streams, the empty frames are held back, so the
- * retry rule that matters — retry only while nothing was emitted — still holds.
+ * retry rule still holds: retry only while nothing was emitted.
  */
 export async function* requireContent<T extends ProviderChunk>(
   provider: string,
@@ -240,7 +257,7 @@ export async function* requireContent<T extends ProviderChunk>(
     throw new ProviderError(
       provider,
       "invalid",
-      `${provider}: the output cap ran out before any answer${spent} — raise maxTokens or lower effort`,
+      `${provider}: the output cap ran out before any answer${spent}. Raise maxTokens or lower effort.`,
     );
   }
   throw new ProviderError(provider, "overload", `${provider}: completed with no content`);
@@ -267,8 +284,8 @@ export interface WatchdogOptions {
  * A provider with both silent failures already handled.
  *
  * Every consumer of this package wrote the same three lines around every
- * `createStream` — build a watch, hand the provider the WATCH's signal, wrap
- * the chunks — and the middle one is the trap. Pass the caller's signal
+ * `createStream`: build a watch, hand the provider the watch's signal, wrap
+ * the chunks. The middle one is the trap. Pass the caller's signal
  * instead and everything still compiles, still streams, still passes the
  * tests: the watchdog simply never aborts anything, because the request it was
  * meant to cancel was never told about it. The failure has no symptom until
@@ -276,8 +293,10 @@ export interface WatchdogOptions {
  *
  * So the composition belongs here rather than in a docs snippet each app
  * copies. The result is still a `Provider`, so it composes unchanged with
- * `withStreamRetry` and `streamWithBackupModels` — and both of the failures it
- * catches are transient, which is what makes wrapping it in a retry correct.
+ * `withStreamRetry` and `streamWithBackupModels`. Timeouts and empty turns
+ * without a `length` finish are transient, so wrapping it in a retry is safe.
+ * The one timeout a replay would make worse, no response at all within the
+ * progress window, says so itself with `shouldRetry: false`.
  */
 export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Provider {
   return {

package/src/zod.ts

@@ -12,8 +12,8 @@ import type { JsonObjectSchema } from "./types.ts";
  * A zod schema as the JSON Schema every provider's tool contract wants.
  *
  * `$schema` is dropped. zod emits the dialect URI at the root, and a provider
- * validating a tool's `parameters` against its own supported subset — OpenAI
- * under `strict: true`, Gemini's `parametersJsonSchema` — rejects the whole
+ * validating a tool's `parameters` against its own supported subset (OpenAI
+ * under `strict: true`, Gemini's `parametersJsonSchema`) rejects the whole
  * tool over that one key, with a message that names neither zod nor the field.
  */
 export function toJsonObjectSchema(schema: z.ZodType, label = "schema"): JsonObjectSchema {
@@ -46,14 +46,14 @@ export interface ZodToolSpec<I, O> {
    * rejecting the call. Worth it for a TERMINAL tool, which gets no second
    * chance: a forced-submit salvage turn runs exactly once, and discarding an
    * otherwise-valid answer over a few extra characters loses the whole run.
-   * Off by default — an ordinary tool can simply be called again.
+   * Off by default: an ordinary tool can simply be called again.
    */
   clampOverflow?: boolean;
 }
 
 /**
  * A tool whose arguments are validated by zod, with the failure reported to
- * the MODEL in words it can act on — `topic: expected string, received number`
+ * the model in words it can act on. `topic: expected string, received number`
  * beats a stack trace it cannot read.
  */
 export function zodTool<I, O>(spec: ZodToolSpec<I, O>): Tool<I, O> {