@procedures/llm 0.4.0 → 0.6.0

package/src/anthropic.ts

@@ -2,16 +2,19 @@
  * The Anthropic Messages API over `fetch` (roadmap R46).
  *
  * The procedures send each request with the `fetch` of the module settings, not through
- * `http.request`: the API key goes in a header, and the input of a nested call is visible to the
+ * `http.request`. The API key goes in a header. The input of a nested call is visible to the
  * interceptors and to the traces of the host. The key comes from an environment variable that the
  * module names. An input cannot give a key.
  *
- * A call stops at the earliest of its time limit, the deadline of its capability and the abort of
- * the caller. A stop is an `InvocationError` with the code `TIMEOUT` or `ABORTED`.
+ * A request stops at the earliest of its time limit, the deadline of its capability and the abort
+ * of the caller. A stop is an `InvocationError` with the code `TIMEOUT` or `ABORTED`. A request
+ * that fails with a retryable error gets new tries (`retry.ts`).
  */
 
+import type { Backoff } from "@mark1russell7/service/backoff";
 import { InvocationError, type ProcedureContext, type ProcedurePath } from "@procedures/client";
 import { LlmError } from "./errors.js";
+import { DEFAULT_BACKOFF, DEFAULT_REQUEST_RETRIES, deadlineLeft, retryAfterMs, withRetries } from "./retry.js";
 import type { LlmEffort, LlmUsage } from "./types.js";
 
 /** The variables that the module reads: a name and a value, or undefined for a variable that is not set. */
@@ -28,7 +31,7 @@ export const DEFAULT_API_KEY_ENV = "ANTHROPIC_API_KEY";
 export const DEFAULT_BASE_URL = "https://api.anthropic.com";
 /** The default of `maxTokens`. */
 export const DEFAULT_MAX_TOKENS = 16_000;
-/** The default time limit of a call: 10 minutes. */
+/** The default time limit of a request: 10 minutes. */
 export const DEFAULT_TIMEOUT_MS = 600_000;
 /** The version header of the Messages API. */
 export const ANTHROPIC_VERSION = "2023-06-01";
@@ -47,6 +50,10 @@ export interface LlmSettings {
   readonly defaultModel: string;
   /** The default of the `fallback` input. */
   readonly fallback: boolean;
+  /** The default number of new tries after a retryable error. */
+  readonly retries: number;
+  /** The backoff before each new try, when the API does not give `retry-after`. */
+  readonly backoff: Backoff;
 }
 
 /** The settings of the default module. */
@@ -57,9 +64,11 @@ export const DEFAULT_LLM_SETTINGS: LlmSettings = Object.freeze({
   baseUrl: DEFAULT_BASE_URL,
   defaultModel: DEFAULT_MODEL,
   fallback: true,
+  retries: DEFAULT_REQUEST_RETRIES,
+  backoff: DEFAULT_BACKOFF,
 });
 
-/** One content block of a message. The procedures read only the text blocks. */
+/** One content block of a message. The procedures read the text blocks and the `tool_use` blocks. */
 export interface ContentBlock {
   readonly type: string;
   readonly text?: string;
@@ -72,6 +81,21 @@ export interface MessageParam {
   readonly content: string | readonly ContentBlock[];
 }
 
+/** A tool definition as the API gets it. */
+export interface ToolParam {
+  readonly name: string;
+  readonly description?: string;
+  readonly input_schema: Record<string, unknown>;
+  readonly strict?: boolean;
+}
+
+/** The `tool_choice` field as the API gets it. */
+export interface ToolChoiceParam {
+  readonly type: "auto" | "any" | "tool" | "none";
+  readonly name?: string;
+  readonly disable_parallel_tool_use?: boolean;
+}
+
 /** The request body of the Messages API, as the procedures send it. */
 export interface MessagesRequest {
   model: string;
@@ -81,6 +105,9 @@ export interface MessagesRequest {
   temperature?: number;
   output_config?: { effort?: LlmEffort; format?: { type: "json_schema"; schema: unknown } };
   fallbacks?: "default";
+  tools?: ToolParam[];
+  tool_choice?: ToolChoiceParam;
+  stream?: true;
 }
 
 /** The parts of an answer that the procedures use. */
@@ -131,6 +158,27 @@ export const NO_USAGE: LlmUsage = Object.freeze({ inputTokens: 0, outputTokens:
 
 const count = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0);
 
+/** The four fields of the usage of the API and their names in `LlmUsage`. */
+const USAGE_FIELDS = [
+  ["input_tokens", "inputTokens"],
+  ["output_tokens", "outputTokens"],
+  ["cache_read_input_tokens", "cacheReadInputTokens"],
+  ["cache_creation_input_tokens", "cacheCreationInputTokens"],
+] as const;
+
+/**
+ * This function gives the usage of a `usage` object of the API. A field that the object does not
+ * have keeps its value of `base`: the usage of a `message_delta` event changes only some fields.
+ */
+export function usageOf(raw: unknown, base: LlmUsage = NO_USAGE): LlmUsage {
+  const record = raw !== null && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
+  const usage = { ...base };
+  for (const [from, to] of USAGE_FIELDS) {
+    if (typeof record[from] === "number") usage[to] = count(record[from]);
+  }
+  return usage;
+}
+
 /** The answer of the API, checked: an answer of another form is a `BAD_RESPONSE`. */
 function parseResponse(text: string): MessagesResponse {
   let body: unknown;
@@ -143,24 +191,27 @@ function parseResponse(text: string): MessagesResponse {
   if (record === null || typeof record !== "object" || !Array.isArray(record["content"]) || typeof record["stop_reason"] !== "string") {
     throw new LlmError("BAD_RESPONSE", "llm: the answer of the API is not a message with content and stop_reason");
   }
-  const usage = (record["usage"] ?? {}) as Record<string, unknown>;
   const details = record["stop_details"];
   return {
     model: typeof record["model"] === "string" ? record["model"] : "",
     content: (record["content"] as unknown[]).filter((block): block is ContentBlock => block !== null && typeof block === "object" && typeof (block as ContentBlock).type === "string"),
     stopReason: record["stop_reason"],
     stopDetails: details !== null && typeof details === "object" ? (details as MessagesResponse["stopDetails"]) : undefined,
-    usage: {
-      inputTokens: count(usage["input_tokens"]),
-      outputTokens: count(usage["output_tokens"]),
-      cacheReadInputTokens: count(usage["cache_read_input_tokens"]),
-      cacheCreationInputTokens: count(usage["cache_creation_input_tokens"]),
-    },
+    usage: usageOf(record["usage"]),
   };
 }
 
-/** The error of an answer with an error status. The message has the type and the message of the API. */
-function apiError(status: number, text: string, key: string): LlmError {
+/** True for a status that the same request can pass later: 408, 409, 429 and 5xx, as in the Anthropic SDKs. */
+function retryableStatus(status: number): boolean {
+  return status === 408 || status === 409 || status === 429 || status >= 500;
+}
+
+/**
+ * The error of an answer with an error status. The message has the type and the message of the
+ * API. The header `x-should-retry` of the API overrides the rule of the status.
+ */
+function apiError(response: Response, text: string, key: string): LlmError {
+  const { status } = response;
   let detail = text.slice(0, 500);
   try {
     const error = (JSON.parse(text) as { error?: { type?: unknown; message?: unknown } }).error;
@@ -170,14 +221,9 @@ function apiError(status: number, text: string, key: string): LlmError {
   }
   // The API does not repeat the key, but the message must not carry it in any case
   const safe = detail.split(key).join("[key]");
-  const retryable = status === 408 || status === 409 || status === 429 || status >= 500;
-  return new LlmError("API_ERROR", `llm: the API answered ${status}: ${safe}`, { status, retryable });
-}
-
-/** The time that is left before the deadline of the capability, or undefined. */
-function deadlineLeft(ctx: CallLimits["ctx"]): number | undefined {
-  const deadline = ctx?.capability?.deadline;
-  return typeof deadline === "number" ? deadline - Date.now() : undefined;
+  const should = response.headers.get("x-should-retry");
+  const retryable = should === "true" ? true : should === "false" ? false : retryableStatus(status);
+  return new LlmError("API_ERROR", `llm: the API answered ${status}: ${safe}`, { status, retryable, retryAfterMs: retryAfterMs(response.headers) });
 }
 
 /** The body of an answer as text, with a size limit. */
@@ -190,10 +236,25 @@ async function readText(response: Response): Promise<string> {
 }
 
 /**
- * This function sends one request to the Messages API and gives the checked answer. It throws a
- * `REFUSAL` error when a safety classifier declined the request.
+ * The scope of one request: the key, the signal of the `fetch` and the stop. The signal aborts at
+ * the time limit, at the deadline and at the abort of the caller.
  */
-export async function sendMessages(request: MessagesRequest, settings: LlmSettings, limits: CallLimits): Promise<MessagesResponse> {
+export interface RequestScope {
+  readonly key: string;
+  readonly signal: AbortSignal;
+  /** The reason of a stop, or undefined while the request runs. */
+  readonly stopped: () => "timeout" | "caller" | undefined;
+  /** This function gives the error of a stop for an error of the request, or the error unchanged. */
+  readonly translate: (error: unknown) => unknown;
+  /** This function stops the timer and removes the listener of the abort. */
+  readonly close: () => void;
+}
+
+/**
+ * This function opens the scope of one request. It throws `NO_API_KEY` without a key, `TIMEOUT`
+ * when the deadline passed and `ABORTED` when the caller aborted, before a request goes out.
+ */
+export function openScope(settings: LlmSettings, limits: CallLimits): RequestScope {
   const key = apiKey(settings);
   const { path, ctx } = limits;
   const left = deadlineLeft(ctx);
@@ -212,44 +273,83 @@ export async function sendMessages(request: MessagesRequest, settings: LlmSettin
     controller.abort();
   };
   ctx?.signal?.addEventListener("abort", onAbort, { once: true });
+  return {
+    key,
+    signal: controller.signal,
+    stopped: () => stop,
+    translate: (error) => {
+      if (stop === "timeout") return new InvocationError("TIMEOUT", `${path.join(".")} passed its time limit of ${timeoutMs} ms`, path);
+      if (stop === "caller") return new InvocationError("ABORTED", `Procedure was aborted: ${path.join(".")}`, path);
+      return error;
+    },
+    close: () => {
+      clearTimeout(timer);
+      ctx?.signal?.removeEventListener("abort", onAbort);
+    },
+  };
+}
 
+/** The message of an error, without the key. */
+export function safeMessage(error: unknown, key: string): string {
+  const message = error instanceof Error ? error.message : String(error);
+  return message.split(key).join("[key]");
+}
+
+/**
+ * This function sends a request to the Messages API in a scope, and gives the answer with a
+ * status from 200 to 299. It throws `CONNECTION` for a failed connection and `API_ERROR` for an
+ * error status.
+ */
+export async function postMessages(request: MessagesRequest, settings: LlmSettings, scope: RequestScope): Promise<Response> {
   const headers: Record<string, string> = {
     "content-type": "application/json",
-    "x-api-key": key,
+    "x-api-key": scope.key,
     "anthropic-version": ANTHROPIC_VERSION,
   };
   if (request.fallbacks !== undefined) headers["anthropic-beta"] = FALLBACK_BETA;
-
+  let response: Response;
   try {
-    let response: Response;
-    try {
-      response = await settings.fetch(`${settings.baseUrl.replace(/\/+$/, "")}/v1/messages`, {
-        method: "POST",
-        headers,
-        body: JSON.stringify(request),
-        signal: controller.signal,
-      });
-    } catch (error) {
-      if (stop === undefined) {
-        const message = error instanceof Error ? error.message : String(error);
-        throw new LlmError("CONNECTION", `llm: the connection to the API failed: ${message.split(key).join("[key]")}`, { retryable: true });
-      }
-      throw error;
+    response = await settings.fetch(`${settings.baseUrl.replace(/\/+$/, "")}/v1/messages`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(request),
+      signal: scope.signal,
+    });
+  } catch (error) {
+    if (scope.stopped() === undefined) {
+      throw new LlmError("CONNECTION", `llm: the connection to the API failed: ${safeMessage(error, scope.key)}`, { retryable: true });
     }
-    const text = await readText(response);
-    if (!response.ok) throw apiError(response.status, text, key);
-    const answer = parseResponse(text);
+    throw error;
+  }
+  if (!response.ok) throw apiError(response, await readText(response), scope.key);
+  return response;
+}
+
+/**
+ * This function sends one request to the Messages API and gives the checked answer. It throws a
+ * `REFUSAL` error when a safety classifier declined the request.
+ */
+export async function sendMessages(request: MessagesRequest, settings: LlmSettings, limits: CallLimits): Promise<MessagesResponse> {
+  const scope = openScope(settings, limits);
+  try {
+    const response = await postMessages(request, settings, scope);
+    const answer = parseResponse(await readText(response));
     if (answer.stopReason === "refusal") {
       const category = answer.stopDetails?.category ?? "not given";
       throw new LlmError("REFUSAL", `llm: a safety classifier declined the request (category: ${category})`);
     }
     return answer;
   } catch (error) {
-    if (stop === "timeout") throw new InvocationError("TIMEOUT", `${path.join(".")} passed its time limit of ${timeoutMs} ms`, path);
-    if (stop === "caller") throw new InvocationError("ABORTED", `Procedure was aborted: ${path.join(".")}`, path);
-    throw error;
+    throw scope.translate(error);
   } finally {
-    clearTimeout(timer);
-    ctx?.signal?.removeEventListener("abort", onAbort);
+    scope.close();
   }
 }
+
+/**
+ * This function sends a request with `sendMessages`, and sends it again after a retryable
+ * error, at most `retries` more times.
+ */
+export function sendWithRetries(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number): Promise<MessagesResponse> {
+  return withRetries(() => sendMessages(request, settings, limits), { backoff: settings.backoff, retries }, limits);
+}

package/src/errors.ts

@@ -7,32 +7,44 @@
 export type LlmErrorCode =
   /** The environment variable of the API key has no value. */
   | "NO_API_KEY"
-  /** The API answered with an error status. `status` has it. */
+  /** The API answered with an error status, or sent an error event in a stream. */
   | "API_ERROR"
-  /** The connection to the API failed. */
+  /** The connection to the API failed, or a stream ended before its end event. */
   | "CONNECTION"
-  /** The answer of the API does not have the form of a message. */
+  /** The answer of the API does not have the form of a message or of a stream of events. */
   | "BAD_RESPONSE"
   /** A safety classifier declined the request (`stop_reason: "refusal"`). */
   | "REFUSAL"
+  /** A tool definition is not valid: a name that two tools have, or a procedure without an object input. */
+  | "BAD_TOOL"
   /** `llm.structured`: the output stopped at `maxTokens`, so the JSON is not complete. */
   | "MAX_TOKENS"
   /** `llm.structured`: no answer gave a value that the schema accepts. */
   | "INVALID_OUTPUT";
 
+/** The options of an `LlmError`. */
+export interface LlmErrorOptions {
+  readonly status?: number | undefined;
+  readonly retryable?: boolean | undefined;
+  readonly retryAfterMs?: number | undefined;
+}
+
 /** An error of a call to the model. */
 export class LlmError extends Error {
   override readonly name = "LlmError";
   readonly code: LlmErrorCode;
-  /** The HTTP status of an `API_ERROR`. */
+  /** The HTTP status of an `API_ERROR`. An error event of a stream has no status. */
   readonly status: number | undefined;
   /** True when the same call can succeed later: a rate limit, an overload, a server error or a connection error. */
   readonly retryable: boolean;
+  /** The wait that the API asked for with `retry-after-ms` or `retry-after`, in milliseconds. */
+  readonly retryAfterMs: number | undefined;
 
-  constructor(code: LlmErrorCode, message: string, options: { status?: number | undefined; retryable?: boolean | undefined } = {}) {
+  constructor(code: LlmErrorCode, message: string, options: LlmErrorOptions = {}) {
     super(message);
     this.code = code;
     this.status = options.status;
     this.retryable = options.retryable ?? false;
+    this.retryAfterMs = options.retryAfterMs;
   }
 }

package/src/events.ts

@@ -0,0 +1,218 @@
+/**
+ * The streaming Messages API (`"stream": true`): the server-sent events of an answer as stream items.
+ *
+ * `sseEvents` of `@procedures/http` parses the event stream. The first event of the API is
+ * `message_start`. Then each content block comes (`content_block_start`, its
+ * `content_block_delta` events, `content_block_stop`), then one or more `message_delta` events, and
+ * at the end `message_stop`. A `ping` event can come at each time, and an `error` event stops the
+ * stream. The usage of a `message_delta` event is cumulative. Other event types are ignored, as
+ * the versioning policy of the API asks.
+ *
+ * A stream gets a new try only before its first item. After the first item, an error goes to the
+ * caller.
+ */
+
+import { HttpError, sseEvents, type ServerSentEvent } from "@procedures/http";
+import { InvocationError } from "@procedures/client";
+import { openScope, postMessages, safeMessage, usageOf, NO_USAGE, type CallLimits, type LlmSettings, type MessagesRequest } from "./anthropic.js";
+import { LlmError } from "./errors.js";
+import { waitToRetry } from "./retry.js";
+import type { LlmStreamOutput, LlmToolUse, LlmUsage } from "./types.js";
+
+/** The largest event of a stream, in bytes. */
+const MAX_EVENT_BYTES = 8 * 1024 * 1024;
+
+/** The error types of an `error` event that the same request can pass later. */
+const RETRYABLE_ERROR_TYPES: ReadonlySet<string> = new Set(["overloaded_error", "api_error", "rate_limit_error", "timeout_error"]);
+
+/** The chunks of a body. A stop of the reader cancels the body. */
+async function* chunksOf(body: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
+  const reader = body.getReader();
+  let done = false;
+  try {
+    for (;;) {
+      const step = await reader.read();
+      if (step.done) {
+        done = true;
+        return;
+      }
+      yield step.value;
+    }
+  } finally {
+    if (!done) reader.cancel().catch(() => undefined);
+  }
+}
+
+/** The data of an event as an object, or a `BAD_RESPONSE` error. */
+function eventData(event: ServerSentEvent): Record<string, unknown> {
+  let data: unknown;
+  try {
+    data = JSON.parse(event.data);
+  } catch {
+    throw new LlmError("BAD_RESPONSE", `llm: the data of the event ${event.event} is not JSON: ${event.data.slice(0, 200)}`);
+  }
+  if (data === null || typeof data !== "object") throw new LlmError("BAD_RESPONSE", `llm: the data of the event ${event.event} is not an object`);
+  return data as Record<string, unknown>;
+}
+
+const recordOf = (value: unknown): Record<string, unknown> => (value !== null && typeof value === "object" ? (value as Record<string, unknown>) : {});
+const stringOf = (value: unknown): string | undefined => (typeof value === "string" ? value : undefined);
+
+/** The error of an `error` event. An overload, a rate limit or a server error is retryable. */
+function streamError(data: Record<string, unknown>): LlmError {
+  const error = recordOf(data["error"]);
+  const type = stringOf(error["type"]) ?? "error";
+  const message = stringOf(error["message"]) ?? "no message";
+  return new LlmError("API_ERROR", `llm: the API sent an error event: ${type}: ${message}`, { retryable: RETRYABLE_ERROR_TYPES.has(type) });
+}
+
+/** One content block of a stream while its deltas come. */
+interface OpenBlock {
+  readonly type: string;
+  readonly id: string;
+  readonly name: string;
+  json: string;
+}
+
+/** The state of one stream: it turns each event into the items of the procedure. */
+class StreamState {
+  private model: string;
+  private id = "";
+  private usage: LlmUsage = NO_USAGE;
+  private stopReason: string | undefined;
+  private category: string | undefined;
+  private text = "";
+  private readonly blocks = new Map<number, OpenBlock>();
+  private readonly toolUses: LlmToolUse[] = [];
+  /** True after the event `message_stop`. */
+  stopped = false;
+
+  constructor(model: string) {
+    this.model = model;
+  }
+
+  /** The items of one event. The event `error` and a refusal at the end throw. */
+  apply(event: ServerSentEvent): LlmStreamOutput[] {
+    if (event.event === "ping") return [];
+    const data = eventData(event);
+    const index = typeof data["index"] === "number" ? data["index"] : -1;
+    switch (stringOf(data["type"]) ?? event.event) {
+      case "message_start": {
+        const message = recordOf(data["message"]);
+        this.id = stringOf(message["id"]) ?? "";
+        this.model = stringOf(message["model"]) || this.model;
+        this.usage = usageOf(message["usage"]);
+        return [{ type: "start", id: this.id, model: this.model, usage: this.usage }];
+      }
+      case "content_block_start": {
+        const block = recordOf(data["content_block"]);
+        const type = stringOf(block["type"]) ?? "";
+        // A server-side fallback puts a fallback block at the change of the model
+        const to = stringOf(recordOf(block["to"])["model"]);
+        if (type === "fallback" && to) this.model = to;
+        this.blocks.set(index, { type, id: stringOf(block["id"]) ?? "", name: stringOf(block["name"]) ?? "", json: "" });
+        const text = stringOf(block["text"]);
+        if (type === "text" && text) {
+          this.text += text;
+          return [{ type: "text", index, text }];
+        }
+        return [];
+      }
+      case "content_block_delta": {
+        const delta = recordOf(data["delta"]);
+        const deltaType = stringOf(delta["type"]);
+        if (deltaType === "text_delta") {
+          const text = stringOf(delta["text"]) ?? "";
+          this.text += text;
+          return text === "" ? [] : [{ type: "text", index, text }];
+        }
+        if (deltaType === "input_json_delta") {
+          const block = this.blocks.get(index);
+          if (block) block.json += stringOf(delta["partial_json"]) ?? "";
+        }
+        return [];
+      }
+      case "content_block_stop": {
+        const block = this.blocks.get(index);
+        this.blocks.delete(index);
+        if (block?.type !== "tool_use") return [];
+        let input: unknown = {};
+        let partialJson: string | undefined;
+        if (block.json.trim() !== "") {
+          try {
+            input = JSON.parse(block.json);
+          } catch {
+            input = null;
+            partialJson = block.json;
+          }
+        }
+        this.toolUses.push({ id: block.id, name: block.name, input });
+        return [{ type: "tool_use", index, id: block.id, name: block.name, input, ...(partialJson !== undefined ? { partialJson } : {}) }];
+      }
+      case "message_delta": {
+        const delta = recordOf(data["delta"]);
+        this.stopReason = stringOf(delta["stop_reason"]) ?? this.stopReason;
+        this.category = stringOf(recordOf(delta["stop_details"] ?? data["stop_details"])["category"]) ?? this.category;
+        this.usage = usageOf(data["usage"], this.usage);
+        return [];
+      }
+      case "message_stop": {
+        if (this.stopReason === undefined) throw new LlmError("BAD_RESPONSE", "llm: the stream stopped without a stop reason");
+        if (this.stopReason === "refusal") {
+          throw new LlmError("REFUSAL", `llm: a safety classifier declined the request (category: ${this.category ?? "not given"})`);
+        }
+        this.stopped = true;
+        return [{ type: "stop", model: this.model, stopReason: this.stopReason, usage: this.usage, text: this.text, toolUses: [...this.toolUses] }];
+      }
+      case "error":
+        throw streamError(data);
+      default:
+        return [];
+    }
+  }
+}
+
+/** The items of one try of a stream, in one request scope. */
+async function* streamOnce(request: MessagesRequest, settings: LlmSettings, limits: CallLimits): AsyncGenerator<LlmStreamOutput, void, undefined> {
+  const scope = openScope(settings, limits);
+  try {
+    const response = await postMessages(request, settings, scope);
+    if (response.body === null) throw new LlmError("BAD_RESPONSE", "llm: the answer of the API has no body");
+    const state = new StreamState(request.model);
+    try {
+      for await (const event of sseEvents(chunksOf(response.body), MAX_EVENT_BYTES, limits.path.join("."))) {
+        for (const item of state.apply(event)) yield item;
+        if (state.stopped) return;
+      }
+    } catch (error) {
+      if (error instanceof LlmError || error instanceof InvocationError || scope.stopped() !== undefined) throw error;
+      if (error instanceof HttpError) throw new LlmError("BAD_RESPONSE", `llm: ${error.message}`);
+      throw new LlmError("CONNECTION", `llm: the stream of the API failed: ${safeMessage(error, scope.key)}`, { retryable: true });
+    }
+    throw new LlmError("CONNECTION", "llm: the stream of the API ended before the event message_stop", { retryable: true });
+  } catch (error) {
+    throw scope.translate(error);
+  } finally {
+    scope.close();
+  }
+}
+
+/**
+ * This function sends a request with `"stream": true` and gives each item of the answer. Before
+ * the first item, a retryable error gets a new try, at most `retries` times.
+ */
+export async function* streamMessages(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number): AsyncGenerator<LlmStreamOutput, void, undefined> {
+  const body: MessagesRequest = { ...request, stream: true };
+  for (let retry = 1; ; retry++) {
+    let started = false;
+    try {
+      for await (const item of streamOnce(body, settings, limits)) {
+        started = true;
+        yield item;
+      }
+      return;
+    } catch (error) {
+      if (started || !(await waitToRetry(error, retry, { backoff: settings.backoff, retries }, limits))) throw error;
+    }
+  }
+}

package/src/index.ts

@@ -1,7 +1,7 @@
 // The root registers nothing: import "<package>/register", or give the module to createRegistry()
 export * from "./module.js";
 export * from "./types.js";
-export { llmComplete, llmStructured, MAX_RETRIES } from "./procedures/llm/index.js";
+export { llmComplete, llmStream, llmStructured, MAX_RETRIES } from "./procedures/llm/index.js";
 // The provider (roadmap R46)
 export {
   ANTHROPIC_VERSION,
@@ -16,5 +16,8 @@ export {
   type FetchFunction,
   type LlmSettings,
 } from "./anthropic.js";
-export { LlmError, type LlmErrorCode } from "./errors.js";
+export { LlmError, type LlmErrorCode, type LlmErrorOptions } from "./errors.js";
 export { constrainedSchema, parseAnswer, UNSUPPORTED_KEYWORDS } from "./schema.js";
+// The retry of a request, the procedure tools
+export { DEFAULT_BACKOFF, DEFAULT_REQUEST_RETRIES, MAX_REQUEST_RETRIES, MAX_RETRY_AFTER_MS, retryAfterMs } from "./retry.js";
+export { DEFAULT_MAX_ROUNDS, MAX_TOOL_RESULT_CHARS, toolNameOf, type ToolContext } from "./tools.js";