@providerkit/core 0.3.0 → 0.4.0

package/src/watchdog.ts

@@ -1,3 +1,6 @@
+// The two ways a stream fails without failing: it goes silent, or it ends
+// having said nothing at all.
+//
 // The stream-idle watchdog.
 //
 // A provider that stops sending bytes is indistinguishable from a long prefill
@@ -13,6 +16,13 @@
 // the other is ours, is transient, and fires while nothing has streamed yet,
 // so the retry is always safe.
 import { ProviderError } from "./errors.ts";
+import type {
+  ChatMessage,
+  Provider,
+  ProviderChunk,
+  StreamOptions,
+  ToolDefinition,
+} from "./types.ts";
 
 /** No byte at all for this long and the stream is considered wedged. */
 export const STREAM_IDLE_MS = 60_000;
@@ -56,7 +66,9 @@ export function streamWatch(opts: StreamWatchOptions = {}): StreamWatch {
 
   // The bridge is structural rather than an event listener: AbortSignal.any
   // aborts synchronously when an input is ALREADY aborted, which is the race
-  // no listener can catch (the event fired before we subscribed).
+  // no listener can catch (the event fired before we subscribed). It is also
+  // the package's runtime floor — see `engines` — rather than a polyfilled
+  // nicety: every runtime this package targets has had it for years.
   const signal = callerSignal ? AbortSignal.any([callerSignal, timeout.signal]) : timeout.signal;
 
   const idleError = (cause?: unknown) =>
@@ -117,3 +129,114 @@ export async function* watchChunks<T>(
     watch.dispose();
   }
 }
+
+/**
+ * Reject a turn that completed but produced nothing usable.
+ *
+ * A stream that ends with no text, no reasoning and no tool call is a failure
+ * wearing a success's clothes: `stop_reason: end_turn` with zero content
+ * blocks, which the vendors emit under load and after a thinking block eats
+ * the whole `max_tokens`. Nothing throws, so nothing retries — the caller
+ * simply shows a person an empty answer, and the only trace is a bill.
+ *
+ * Classified `overload` because that is both true and useful: it is theirs and
+ * temporary, so it is transient (the same model, retried, usually answers) and
+ * backup-eligible (a model that keeps doing it should be walked away from).
+ * The throw lands before any chunk is yielded downstream, so the retry rule
+ * that matters — retry only while nothing was emitted — still holds.
+ */
+export async function* requireContent<T extends ProviderChunk>(
+  provider: string,
+  chunks: AsyncIterable<T>,
+): AsyncGenerator<T> {
+  const held: T[] = [];
+  let usable = false;
+
+  for await (const chunk of chunks) {
+    if (!usable) {
+      usable = Boolean(chunk.content || chunk.reasoning || chunk.toolCalls?.length);
+      // Held rather than forwarded: once a chunk is out, the stream is
+      // committed and the retry this guard exists to trigger can no longer
+      // fire. Nothing content-bearing has arrived yet, so there is nothing to
+      // hold back but the empty frames.
+      if (!usable) {
+        held.push(chunk);
+        continue;
+      }
+      yield* held;
+      held.length = 0;
+    }
+    yield chunk;
+  }
+
+  if (!usable) {
+    throw new ProviderError(provider, "overload", `${provider}: completed with no content`);
+  }
+}
+
+export interface WatchdogOptions {
+  /** Silence this long and the stream is wedged. Defaults to `STREAM_IDLE_MS`. */
+  idleMs?: number;
+  /**
+   * Reject a turn that ends having said nothing, as `requireContent` does. On
+   * by default: an empty completion is a failure in every loop, and the one
+   * shaped like a success is the one nobody catches.
+   */
+  requireContent?: boolean;
+  /** Time to first byte for this call, reported once the byte arrives. */
+  onFirstChunk?: (ms: number) => void;
+}
+
+/**
+ * A provider with both silent failures already handled.
+ *
+ * Every consumer of this package wrote the same three lines around every
+ * `createStream` — build a watch, hand the provider the WATCH's signal, wrap
+ * the chunks — and the middle one is the trap. Pass the caller's signal
+ * instead and everything still compiles, still streams, still passes the
+ * tests: the watchdog simply never aborts anything, because the request it was
+ * meant to cancel was never told about it. The failure has no symptom until
+ * production, where it is the exact hang the watchdog was added to end.
+ *
+ * So the composition belongs here rather than in a docs snippet each app
+ * copies. The result is still a `Provider`, so it composes unchanged with
+ * `withStreamRetry` and `streamWithBackupModels` — and both of the failures it
+ * catches are transient, which is what makes wrapping it in a retry correct.
+ */
+export function withWatchdog(provider: Provider, opts: WatchdogOptions = {}): Provider {
+  return {
+    ...provider,
+    createStream(
+      messages: ChatMessage[],
+      tools: ToolDefinition[],
+      streamOpts: StreamOptions = {},
+    ): AsyncIterable<ProviderChunk> {
+      // Armed on first read, not here: a stream built now and iterated later
+      // must not spend its deadline sitting in a variable.
+      async function* watched(): AsyncGenerator<ProviderChunk> {
+        // idleMs and signal both default inside streamWatch.
+        const watch = streamWatch({
+          provider: provider.id,
+          idleMs: opts.idleMs,
+          signal: streamOpts.signal,
+        });
+        const source = provider.createStream(messages, tools, {
+          ...streamOpts,
+          signal: watch.signal,
+        });
+        let reported = false;
+        for await (const chunk of watchChunks(watch, source)) {
+          // Before `requireContent` holds anything back — TTFT is the first
+          // byte of any kind, not the first byte worth showing.
+          if (!reported) {
+            reported = true;
+            opts.onFirstChunk?.(watch.firstChunkMs() ?? 0);
+          }
+          yield chunk;
+        }
+      }
+
+      return opts.requireContent === false ? watched() : requireContent(provider.id, watched());
+    },
+  };
+}