@procedures/llm 0.4.0 → 0.6.0

package/src/module.ts

@@ -5,26 +5,32 @@
  *
  * `llmModule` sends each request to the Anthropic Messages API with the key of the environment
  * variable `ANTHROPIC_API_KEY`. `createLlmModule()` gives the same procedures with another
- * variable, another default model, another base URL or another `fetch` (roadmap R46).
+ * variable, another default model, another base URL, another `fetch`, another number of new
+ * tries or another backoff (roadmap R46).
  */
 
 import { z } from "zod";
+import { backoffProblems, type Backoff } from "@mark1russell7/service/backoff";
 import {
   defineModule,
   registerModules,
   PROCEDURE_REGISTRY,
   createProcedure,
+  markDataDriven,
   zodAdapter,
   type AnyProcedure,
   type Effects,
   type ProcedureContext,
+  type ProcedureHandler,
   type ProcedureModule,
   type ProcedureRegistry,
 } from "@procedures/client";
 import { llmComplete } from "./procedures/llm/complete.js";
 import { llmStructured, MAX_RETRIES } from "./procedures/llm/structured.js";
+import { llmStream } from "./procedures/llm/stream.js";
 import { DEFAULT_LLM_SETTINGS, type EnvSource, type FetchFunction, type LlmSettings } from "./anthropic.js";
-import type { LlmCompleteInput, LlmCompleteOutput, LlmStructuredInput, LlmStructuredOutput } from "./types.js";
+import { MAX_REQUEST_RETRIES } from "./retry.js";
+import type { LlmCompleteInput, LlmCompleteOutput, LlmStreamInput, LlmStreamOutput, LlmStructuredInput, LlmStructuredOutput } from "./types.js";
 
 // =============================================================================
 // Schemas
@@ -43,7 +49,7 @@ const requestFields = {
   maxTokens: z.number().int().min(1).max(64_000).optional().describe("The most tokens of the output, with the thinking (default 16000)"),
   temperature: z.number().min(0).max(1).optional().describe("The temperature, 0 to 1. Some models accept only the default"),
   effort: z.enum(["low", "medium", "high", "xhigh", "max"]).optional().describe("The effort of the model (default: the default of the model)"),
-  timeoutMs: z.number().int().positive().max(3_600_000).optional().describe("The time limit of the call, in milliseconds (default 600000)"),
+  timeoutMs: z.number().int().positive().max(3_600_000).optional().describe("The time limit of each request, in milliseconds (default 600000)"),
   fallback: z.boolean().optional().describe("True: the API runs a declined request again on its fallback model (default: the option of the module, true)"),
 };
 
@@ -56,13 +62,74 @@ const usage = z
   })
   .describe("The tokens of the call");
 
-const LlmCompleteInputSchema = z.object(requestFields);
+const TOOL_NAME = /^[a-zA-Z0-9_-]{1,128}$/;
+
+const tool = z.object({
+  name: z.string().regex(TOOL_NAME, "1 to 128 letters, digits, _ and -").describe("The name of the tool"),
+  description: z.string().max(100_000).optional().describe("What the tool does and when the model uses it"),
+  inputSchema: z.record(z.string(), z.unknown()).describe("The JSON Schema of the input of the tool, with type object"),
+  strict: z.boolean().optional().describe("True: the API makes each input agree with the schema (strict tool use)"),
+});
+
+const parallel = z.boolean().optional().describe("True: the model uses at most one tool in an answer");
+
+const toolChoice = z
+  .discriminatedUnion("type", [
+    z.object({ type: z.literal("auto"), disableParallelToolUse: parallel }),
+    z.object({ type: z.literal("any"), disableParallelToolUse: parallel }),
+    z.object({ type: z.literal("tool"), name: z.string().regex(TOOL_NAME).describe("The tool that the model uses"), disableParallelToolUse: parallel }),
+    z.object({ type: z.literal("none") }),
+  ])
+  .describe("How the model uses the tools: auto (default), any, tool or none. Some models refuse any and tool");
+
+const toolFields = {
+  tools: z.array(tool).max(128).optional().describe("The tools that the model can use. The output gives each tool_use block"),
+  toolChoice: toolChoice.optional(),
+  retries: z
+    .number()
+    .int()
+    .min(0)
+    .max(MAX_REQUEST_RETRIES)
+    .optional()
+    .describe("The new tries after a retryable error (429, 529, 5xx, a connection error), 0 to 10 (default: the option of the module, 2)"),
+};
+
+const toolUse = z.object({
+  id: z.string().describe("The ID of the tool_use block"),
+  name: z.string().describe("The name of the tool"),
+  input: z.unknown().describe("The input of the call"),
+});
+
+const toolCall = z.object({
+  id: z.string().describe("The ID of the tool_use block"),
+  name: z.string().describe("The name of the tool"),
+  procedure: z.string().describe("The key of the procedure"),
+  input: z.unknown().describe("The input that the model gave"),
+  isError: z.boolean().describe("True when the procedure failed: the model got the error"),
+  output: z.unknown().optional().describe("The output of the procedure"),
+  error: z.string().optional().describe("The error of the procedure"),
+});
+
+const LlmCompleteInputSchema = z.object({
+  ...requestFields,
+  ...toolFields,
+  procedures: z
+    .array(z.string().regex(/^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/, "A procedure key, for example weather.current"))
+    .max(64)
+    .optional()
+    .describe("The procedures that the model can use as tools: llm.complete calls them and sends the results back"),
+  maxRounds: z.number().int().min(1).max(64).optional().describe("The most requests to the model in one call, 1 to 64 (default 8)"),
+});
 
 const LlmCompleteOutputSchema = z.object({
-  text: z.string().describe("The text of the answer"),
+  text: z.string().describe("The text of the last answer"),
   model: z.string().describe("The model that answered"),
-  stopReason: z.string().describe("Why the model stopped: end_turn, max_tokens, stop_sequence or another reason"),
+  stopReason: z.string().describe("Why the model stopped: end_turn, max_tokens, stop_sequence, tool_use or another reason"),
   usage,
+  toolUses: z.array(toolUse).describe("The tool_use blocks of the last answer that no procedure ran"),
+  toolCalls: z.array(toolCall).describe("The calls of the procedure tools, in their sequence"),
+  rounds: z.number().int().describe("The number of requests to the model"),
+  roundLimit: z.boolean().describe("True when maxRounds stopped the loop before the model stopped"),
 });
 
 const LlmStructuredInputSchema = z.object({
@@ -70,6 +137,7 @@ const LlmStructuredInputSchema = z.object({
   schema: z.record(z.string(), z.unknown()).describe("The JSON Schema of the value"),
   constrained: z.boolean().optional().describe("True (default): the API constrains the output to the schema. False: the system text gives the schema"),
   retries: z.number().int().min(0).max(MAX_RETRIES).optional().describe("The new tries after an answer that is not valid, 0 to 2 (default 2)"),
+  requestRetries: toolFields.retries,
 });
 
 const LlmStructuredOutputSchema = z.object({
@@ -80,6 +148,33 @@ const LlmStructuredOutputSchema = z.object({
   unchecked: z.array(z.string()).describe("The keywords of the schema that the check does not examine fully"),
 });
 
+const LlmStreamInputSchema = z.object({ ...requestFields, ...toolFields });
+
+const index = z.number().int().describe("The index of the content block");
+
+const LlmStreamOutputSchema = z
+  .discriminatedUnion("type", [
+    z.object({ type: z.literal("start"), id: z.string().describe("The ID of the message"), model: z.string().describe("The model that answers"), usage }),
+    z.object({ type: z.literal("text"), index, text: z.string().describe("The new text") }),
+    z.object({
+      type: z.literal("tool_use"),
+      index,
+      id: z.string().describe("The ID of the tool_use block"),
+      name: z.string().describe("The name of the tool"),
+      input: z.unknown().describe("The input of the call, or null when its JSON is not complete"),
+      partialJson: z.string().optional().describe("The JSON text of an input that does not parse"),
+    }),
+    z.object({
+      type: z.literal("stop"),
+      model: z.string().describe("The model that answered"),
+      stopReason: z.string().describe("Why the model stopped"),
+      usage,
+      text: z.string().describe("The text of the answer: all the text deltas, joined"),
+      toolUses: z.array(toolUse).describe("The tool_use blocks of the answer"),
+    }),
+  ])
+  .describe("One item of the stream: start, text, tool_use or stop");
+
 // =============================================================================
 // Procedures
 // =============================================================================
@@ -87,19 +182,20 @@ const LlmStructuredOutputSchema = z.object({
 /**
  * The effects of each procedure: it sends data to a system outside the host and gets an answer.
  * A call costs money and can change the state of an account, so it declares a write. It deletes
- * nothing and a second call is a new call.
+ * nothing and a second call is a new call. The procedure tools of `llm.complete` are nested calls:
+ * the admission of the host examines each one.
  */
 const SENDS_TO_MODEL: Effects = Object.freeze({ reads: Object.freeze(["network"]), writes: Object.freeze(["network"]), destructive: false, idempotent: false });
 
-/** The two procedures, with handlers that use the settings. */
+/** The three procedures, with handlers that use the settings. */
 function llmProcedures(settings: LlmSettings): AnyProcedure[] {
   const llmCompleteProcedure = createProcedure()
     .path(["llm", "complete"])
     .input(zodAdapter<LlmCompleteInput>(LlmCompleteInputSchema))
     .output(zodAdapter<LlmCompleteOutput>(LlmCompleteOutputSchema))
     .meta({
-      description: "Send a prompt to a language model and give the text and the usage",
-      tags: ["llm", "anthropic"],
+      description: "Send a prompt to a language model and give the text and the usage. The model can use tools and procedures",
+      tags: ["llm", "anthropic", "tools"],
       args: ["prompt"],
       shorts: { model: "m", system: "s" },
       output: "json",
@@ -107,6 +203,10 @@ function llmProcedures(settings: LlmSettings): AnyProcedure[] {
     })
     .handler(async (input: LlmCompleteInput, ctx: ProcedureContext): Promise<LlmCompleteOutput> => llmComplete(input, ctx, settings))
     .build();
+  // The input names the procedures that the model can call, so the procedure is data-driven: the
+  // host examines each nested call with its capability, and permits only the procedures that it
+  // exposes. It is not "runs-refs": the refs of its input still run before the call.
+  markDataDriven(llmCompleteProcedure);
 
   const llmStructuredProcedure = createProcedure()
     .path(["llm", "structured"])
@@ -123,7 +223,25 @@ function llmProcedures(settings: LlmSettings): AnyProcedure[] {
     .handler(async (input: LlmStructuredInput, ctx: ProcedureContext): Promise<LlmStructuredOutput> => llmStructured(input, ctx, settings))
     .build();
 
-  return [llmCompleteProcedure, llmStructuredProcedure];
+  const llmStreamProcedure = createProcedure()
+    .path(["llm", "stream"])
+    .input(zodAdapter<LlmStreamInput>(LlmStreamInputSchema))
+    .output(zodAdapter<LlmStreamOutput>(LlmStreamOutputSchema))
+    .meta({
+      description: "Send a prompt to a language model and stream the events of the answer",
+      tags: ["llm", "anthropic", "stream"],
+      args: ["prompt"],
+      shorts: { model: "m", system: "s" },
+      output: "json",
+      streaming: true,
+      // The stream ends with the answer: maxTokens and timeoutMs limit it (roadmap R15)
+      stream: { unbounded: false },
+      effects: SENDS_TO_MODEL,
+    })
+    .handler(((input: LlmStreamInput, ctx: ProcedureContext) => llmStream(input, ctx, settings)) as ProcedureHandler<LlmStreamInput, LlmStreamOutput>)
+    .build();
+
+  return [llmCompleteProcedure, llmStructuredProcedure, llmStreamProcedure];
 }
 
 /**
@@ -148,12 +266,16 @@ export interface LlmModuleOptions {
   readonly fetch?: FetchFunction | undefined;
   /** The variables: a record, or a function that gives them at each call. The default is `process.env`. */
   readonly env?: EnvSource | (() => EnvSource) | undefined;
+  /** The default number of new tries after a retryable error: 0 to 10. The default is 2. */
+  readonly retries?: number | undefined;
+  /** The backoff before a new try, in milliseconds. The default starts at 500 and stops at 8000. */
+  readonly backoff?: Backoff | undefined;
 }
 
 /**
  * This function gives the module of this package with other settings. Without options, it gives
  * `llmModule`. A host gives it to `createRegistry()` before its bundles, so it replaces the
- * `llmModule` that a bundle requires (the first module of a name counts). A bad option throws here.
+ * `llmModule` of a bundle (the first module of a name counts). A bad option throws here.
  *
  * ```ts
  * createRegistry(createLlmModule({ apiKeyEnv: "WORK_ANTHROPIC_KEY", defaultModel: "claude-haiku-5-5" }));
@@ -165,6 +287,13 @@ export function createLlmModule(options: LlmModuleOptions = {}): ProcedureModule
   if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(apiKeyEnv)) throw new TypeError(`createLlmModule: apiKeyEnv must be the name of a variable, not ${JSON.stringify(apiKeyEnv)}`);
   const baseUrl = options.baseUrl ?? DEFAULT_LLM_SETTINGS.baseUrl;
   if (!/^https?:\/\//.test(baseUrl)) throw new TypeError("createLlmModule: baseUrl must be an http or https URL");
+  const retries = options.retries ?? DEFAULT_LLM_SETTINGS.retries;
+  if (!Number.isInteger(retries) || retries < 0 || retries > MAX_REQUEST_RETRIES) {
+    throw new TypeError(`createLlmModule: retries must be a whole number from 0 to ${MAX_REQUEST_RETRIES}`);
+  }
+  const backoff = options.backoff ?? DEFAULT_LLM_SETTINGS.backoff;
+  const problems = backoffProblems(backoff);
+  if (problems.length > 0) throw new TypeError(`createLlmModule: the backoff is not valid: ${problems.join("; ")}`);
   const env = options.env;
   const settings: LlmSettings = {
     apiKeyEnv,
@@ -173,6 +302,8 @@ export function createLlmModule(options: LlmModuleOptions = {}): ProcedureModule
     baseUrl,
     defaultModel: options.defaultModel ?? DEFAULT_LLM_SETTINGS.defaultModel,
     fallback: options.fallback ?? DEFAULT_LLM_SETTINGS.fallback,
+    retries,
+    backoff,
   };
   return defineModule("@procedures/llm", llmProcedures(settings));
 }

package/src/procedures/llm/complete.ts

@@ -2,25 +2,66 @@
  * llm.complete procedure
  *
  * Send a prompt to a language model and give the text and the usage. The text is the text
- * blocks of the answer, joined: the thinking blocks are not in it.
+ * blocks of the last answer, joined: the thinking blocks are not in it.
+ *
+ * With `tools`, the output gives each `tool_use` block of the answer, and the caller starts them.
+ * With `procedures`, the procedure starts the tools. It calls each procedure that the model asks
+ * for through `ctx.client` and sends the results back. Then it asks again, until the model stops
+ * or `maxRounds` requests went out.
  */
 
-import { DEFAULT_LLM_SETTINGS, sendMessages, textOf, type CallLimits, type LlmSettings } from "../../anthropic.js";
-import { baseRequest, timeoutOf } from "../../request.js";
-import type { LlmCompleteInput, LlmCompleteOutput } from "../../types.js";
+import { addUsage, DEFAULT_LLM_SETTINGS, NO_USAGE, sendWithRetries, textOf, type CallLimits, type ContentBlock, type LlmSettings } from "../../anthropic.js";
+import { addTools, baseRequest, retriesOf, timeoutOf, toolParam } from "../../request.js";
+import { DEFAULT_MAX_ROUNDS, procedureTools, runToolUse, toolUsesOf, type ToolContext } from "../../tools.js";
+import type { LlmCompleteInput, LlmCompleteOutput, LlmToolCall, LlmUsage } from "../../types.js";
 
 const PATH = ["llm", "complete"];
 
 /**
  * Send a prompt to a language model and give the text and the usage
  */
-export async function llmComplete(input: LlmCompleteInput, ctx?: CallLimits["ctx"], settings: LlmSettings = DEFAULT_LLM_SETTINGS): Promise<LlmCompleteOutput> {
+export async function llmComplete(input: LlmCompleteInput, ctx?: ToolContext, settings: LlmSettings = DEFAULT_LLM_SETTINGS): Promise<LlmCompleteOutput> {
   const request = baseRequest(input, settings, input.system);
-  const answer = await sendMessages(request, settings, { path: PATH, ctx, timeoutMs: timeoutOf(input) });
-  return {
-    text: textOf(answer.content),
-    model: answer.model || request.model,
-    stopReason: answer.stopReason,
-    usage: answer.usage,
-  };
+  const procedures = procedureTools(input.procedures ?? [], ctx, PATH);
+  addTools(request, [...(input.tools ?? []).map(toolParam), ...procedures.map((tool) => tool.param)], input.toolChoice);
+  const byName = new Map(procedures.map((tool) => [tool.param.name, tool]));
+  const limits: CallLimits = { path: PATH, ctx, timeoutMs: timeoutOf(input) };
+  const retries = retriesOf(input.retries, settings);
+  const maxRounds = input.maxRounds ?? DEFAULT_MAX_ROUNDS;
+
+  let usage: LlmUsage = NO_USAGE;
+  const toolCalls: LlmToolCall[] = [];
+  for (let round = 1; ; round++) {
+    const answer = await sendWithRetries(request, settings, limits, retries);
+    usage = addUsage(usage, answer.usage);
+    const uses = toolUsesOf(answer.content);
+    // The procedure runs the tools only when each tool of the answer is a procedure tool
+    const runnable = answer.stopReason === "tool_use" && uses.length > 0 && uses.every((use) => byName.has(use.name));
+    if (!runnable || round >= maxRounds) {
+      return {
+        text: textOf(answer.content),
+        model: answer.model || request.model,
+        stopReason: answer.stopReason,
+        usage,
+        toolUses: uses,
+        toolCalls,
+        rounds: round,
+        roundLimit: runnable,
+      };
+    }
+    // One at a time, in the sequence of the answer: a procedure can change data that the next one reads
+    const results: ContentBlock[] = [];
+    for (const use of uses) {
+      const { call, result } = await runToolUse(use, byName.get(use.name)!, ctx!);
+      toolCalls.push(call);
+      results.push(result);
+    }
+    // The next request continues the conversation: the answer as it came, then the results
+    request.messages = [...request.messages, { role: "assistant", content: answer.content }, { role: "user", content: results }];
+    // A forced tool_choice applies to the first request only: then the model can stop
+    if (request.tool_choice !== undefined && (request.tool_choice.type === "any" || request.tool_choice.type === "tool")) {
+      const { disable_parallel_tool_use } = request.tool_choice;
+      request.tool_choice = { type: "auto", ...(disable_parallel_tool_use !== undefined ? { disable_parallel_tool_use } : {}) };
+    }
+  }
 }

package/src/procedures/llm/index.ts

@@ -1,2 +1,3 @@
 export { llmComplete } from "./complete.js";
 export { llmStructured, MAX_RETRIES } from "./structured.js";
+export { llmStream } from "./stream.js";

package/src/procedures/llm/stream.ts

@@ -0,0 +1,23 @@
+/**
+ * llm.stream procedure
+ *
+ * Send a prompt to a language model and stream the events of the answer. The request has
+ * `"stream": true`. The items are the start of the message, the text deltas and each complete
+ * `tool_use` block. The last item has the stop reason, the usage and the full text.
+ */
+
+import { DEFAULT_LLM_SETTINGS, type CallLimits, type LlmSettings } from "../../anthropic.js";
+import { streamMessages } from "../../events.js";
+import { addTools, baseRequest, retriesOf, timeoutOf, toolParam } from "../../request.js";
+import type { LlmStreamInput, LlmStreamOutput } from "../../types.js";
+
+const PATH = ["llm", "stream"];
+
+/**
+ * Send a prompt to a language model and stream the events of the answer
+ */
+export async function* llmStream(input: LlmStreamInput, ctx?: CallLimits["ctx"], settings: LlmSettings = DEFAULT_LLM_SETTINGS): AsyncGenerator<LlmStreamOutput, void, undefined> {
+  const request = baseRequest(input, settings, input.system);
+  addTools(request, (input.tools ?? []).map(toolParam), input.toolChoice);
+  yield* streamMessages(request, settings, { path: PATH, ctx, timeoutMs: timeoutOf(input) }, retriesOf(input.retries, settings));
+}

package/src/procedures/llm/structured.ts

@@ -2,14 +2,14 @@
  * llm.structured procedure
  *
  * Send a prompt and a JSON Schema to a language model and give a value that the schema accepts.
- * An answer that is not JSON, or that the schema refuses, gets a new try: the request then has
+ * An answer that is not JSON, or that the schema refuses, gets a new try. The next request has
  * the answer (its content, unchanged) and a user turn with the problems. At most 2 new tries.
  */
 
 import { validateJson, type JsonValue } from "@procedures/json";
 import { LlmError } from "../../errors.js";
-import { addUsage, DEFAULT_LLM_SETTINGS, NO_USAGE, sendMessages, textOf, type CallLimits, type LlmSettings, type MessagesRequest } from "../../anthropic.js";
-import { baseRequest, timeoutOf } from "../../request.js";
+import { addUsage, DEFAULT_LLM_SETTINGS, NO_USAGE, sendWithRetries, textOf, type CallLimits, type LlmSettings, type MessagesRequest } from "../../anthropic.js";
+import { baseRequest, retriesOf, timeoutOf } from "../../request.js";
 import { constrainedSchema, parseAnswer } from "../../schema.js";
 import type { LlmStructuredInput, LlmStructuredOutput, LlmUsage } from "../../types.js";
 
@@ -42,6 +42,7 @@ function problemsOf(value: unknown, schema: Record<string, unknown>): { problems
 export async function llmStructured(input: LlmStructuredInput, ctx?: CallLimits["ctx"], settings: LlmSettings = DEFAULT_LLM_SETTINGS): Promise<LlmStructuredOutput> {
   const constrained = input.constrained ?? true;
   const retries = Math.min(Math.max(input.retries ?? MAX_RETRIES, 0), MAX_RETRIES);
+  const requestRetries = retriesOf(input.requestRetries, settings);
   const system = [input.system, instruction(input, constrained)].filter((part) => part !== undefined && part !== "").join("\n\n");
   const request: MessagesRequest = baseRequest(input, settings, system);
   if (constrained) {
@@ -51,7 +52,8 @@ export async function llmStructured(input: LlmStructuredInput, ctx?: CallLimits[
   let usage: LlmUsage = NO_USAGE;
   let lastProblem = "";
   for (let attempt = 1; attempt <= retries + 1; attempt++) {
-    const answer = await sendMessages(request, settings, { path: PATH, ctx, timeoutMs: timeoutOf(input) });
+    // A retryable error of the API gets new tries here: they do not count as attempts
+    const answer = await sendWithRetries(request, settings, { path: PATH, ctx, timeoutMs: timeoutOf(input) }, requestRetries);
     usage = addUsage(usage, answer.usage);
     if (answer.stopReason === "max_tokens") {
       throw new LlmError("MAX_TOKENS", `llm.structured: the answer stopped at maxTokens (${request.max_tokens}), so the JSON is not complete`);

package/src/request.ts

@@ -1,9 +1,11 @@
 /**
- * The request of the fields that both procedures take (`LlmRequestInput`).
+ * The request of the fields that the procedures take (`LlmRequestInput`), and the tool fields.
  */
 
-import { DEFAULT_MAX_TOKENS, DEFAULT_TIMEOUT_MS, type LlmSettings, type MessagesRequest } from "./anthropic.js";
-import type { LlmRequestInput } from "./types.js";
+import { DEFAULT_MAX_TOKENS, DEFAULT_TIMEOUT_MS, type LlmSettings, type MessagesRequest, type ToolChoiceParam, type ToolParam } from "./anthropic.js";
+import { LlmError } from "./errors.js";
+import { MAX_REQUEST_RETRIES } from "./retry.js";
+import type { LlmRequestInput, LlmTool, LlmToolChoice } from "./types.js";
 
 /** This function gives the body of a request: the model, the limits, the system text and the user text. */
 export function baseRequest(input: LlmRequestInput, settings: LlmSettings, system: string | undefined): MessagesRequest {
@@ -19,7 +21,48 @@ export function baseRequest(input: LlmRequestInput, settings: LlmSettings, syste
   return request;
 }
 
-/** The time limit of a call. */
+/** The time limit of a request. */
 export function timeoutOf(input: LlmRequestInput): number {
   return input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 }
+
+/** The number of new tries after a retryable error: the input, or the default of the module. */
+export function retriesOf(given: number | undefined, settings: LlmSettings): number {
+  return Math.min(Math.max(given ?? settings.retries, 0), MAX_REQUEST_RETRIES);
+}
+
+/** The definition of a tool of the input, as the API gets it. */
+export function toolParam(tool: LlmTool): ToolParam {
+  return {
+    name: tool.name,
+    ...(tool.description !== undefined ? { description: tool.description } : {}),
+    input_schema: tool.inputSchema,
+    ...(tool.strict !== undefined ? { strict: tool.strict } : {}),
+  };
+}
+
+/** The `tool_choice` of the input, as the API gets it. */
+export function toolChoiceParam(choice: LlmToolChoice): ToolChoiceParam {
+  if (choice.type === "none") return { type: "none" };
+  return {
+    type: choice.type,
+    ...(choice.type === "tool" ? { name: choice.name } : {}),
+    ...(choice.disableParallelToolUse !== undefined ? { disable_parallel_tool_use: choice.disableParallelToolUse } : {}),
+  };
+}
+
+/**
+ * This function puts the tools and the `tool_choice` into a request. It throws `BAD_TOOL` for two
+ * tools with the same name, and for a `toolChoice` without tools.
+ */
+export function addTools(request: MessagesRequest, tools: readonly ToolParam[], choice: LlmToolChoice | undefined): void {
+  const names = new Set<string>();
+  for (const tool of tools) {
+    if (names.has(tool.name)) throw new LlmError("BAD_TOOL", `llm: two tools have the name ${tool.name}`);
+    names.add(tool.name);
+  }
+  if (choice !== undefined && tools.length === 0) throw new LlmError("BAD_TOOL", "llm: toolChoice needs tools");
+  if (choice?.type === "tool" && !names.has(choice.name)) throw new LlmError("BAD_TOOL", `llm: toolChoice names the tool ${choice.name}, and no tool has that name`);
+  if (tools.length > 0) request.tools = [...tools];
+  if (choice !== undefined) request.tool_choice = toolChoiceParam(choice);
+}

package/src/retry.ts

@@ -0,0 +1,110 @@
+/**
+ * The automatic retry of a request to the Messages API.
+ *
+ * A request that fails with an `LlmError` with `retryable: true` (429, 529 and the other 5xx
+ * statuses, 408, 409, a connection error, an error event at the start of a stream) gets a new
+ * try. The wait before a try is the `retry-after-ms` or `retry-after` header of the API, or the
+ * backoff of `@mark1russell7/service/backoff`. A wait that ends after the deadline of the call
+ * gives the error at once. The abort of the caller stops a wait with `ABORTED`.
+ */
+
+import { backoffDelay, type Backoff } from "@mark1russell7/service/backoff";
+import { InvocationError } from "@procedures/client";
+import { LlmError } from "./errors.js";
+import type { CallLimits } from "./anthropic.js";
+
+/** The default number of new tries after a retryable error, as in the Anthropic SDKs. */
+export const DEFAULT_REQUEST_RETRIES = 2;
+/** The most new tries that an input or the module can ask for. */
+export const MAX_REQUEST_RETRIES = 10;
+/** The longest wait that a `retry-after` header can ask for: 60 seconds. A longer wait gives the error at once. */
+export const MAX_RETRY_AFTER_MS = 60_000;
+
+/**
+ * The default backoff: 500 ms before the first new try, then two times more each time, at most
+ * 8 seconds. A jitter removes up to a quarter of each wait.
+ */
+export const DEFAULT_BACKOFF: Backoff = Object.freeze({
+  base: 500,
+  factor: 2,
+  cap: 8_000,
+  jitter: Object.freeze({ kind: "subtract", fraction: 0.25 }),
+});
+
+/** What a retry uses: the backoff policy and the number of new tries. */
+export interface RetryPolicy {
+  readonly backoff: Backoff;
+  readonly retries: number;
+}
+
+/**
+ * This function gives the wait that the headers of an answer ask for, in milliseconds, or
+ * undefined. It reads `retry-after-ms`, then `retry-after` as seconds or as an HTTP date.
+ */
+export function retryAfterMs(headers: Pick<Headers, "get">, now: number = Date.now()): number | undefined {
+  const milliseconds = headers.get("retry-after-ms");
+  if (milliseconds !== null && milliseconds.trim() !== "") {
+    const value = Number(milliseconds);
+    if (Number.isFinite(value) && value >= 0) return value;
+  }
+  const after = headers.get("retry-after");
+  if (after === null || after.trim() === "") return undefined;
+  const seconds = Number(after);
+  if (Number.isFinite(seconds)) return seconds >= 0 ? seconds * 1000 : undefined;
+  const date = Date.parse(after);
+  return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+}
+
+/** The time that is left before the deadline of the capability, or undefined. */
+export function deadlineLeft(ctx: CallLimits["ctx"]): number | undefined {
+  const deadline = ctx?.capability?.deadline;
+  return typeof deadline === "number" ? deadline - Date.now() : undefined;
+}
+
+/** This function waits `ms` milliseconds. The abort of the signal stops the wait with `ABORTED`. */
+function sleep(ms: number, limits: CallLimits): Promise<void> {
+  const signal = limits.ctx?.signal;
+  const aborted = (): InvocationError => new InvocationError("ABORTED", `Procedure was aborted: ${limits.path.join(".")}`, limits.path);
+  if (signal?.aborted) return Promise.reject(aborted());
+  return new Promise((resolve, reject) => {
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(aborted());
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+/**
+ * This function decides about a new try after an error. It waits and gives true, or it gives
+ * false at once. `retry` is the number of the new try: the first new try is 1.
+ */
+export async function waitToRetry(error: unknown, retry: number, policy: RetryPolicy, limits: CallLimits): Promise<boolean> {
+  if (retry > policy.retries) return false;
+  if (!(error instanceof LlmError) || !error.retryable) return false;
+  const after = error.retryAfterMs;
+  if (after !== undefined && after > MAX_RETRY_AFTER_MS) return false;
+  const wait = after ?? backoffDelay(policy.backoff, retry);
+  const left = deadlineLeft(limits.ctx);
+  if (left !== undefined && wait >= left) return false;
+  await sleep(wait, limits);
+  return true;
+}
+
+/**
+ * This function runs `attempt` and gives its result. After a retryable error, it waits and
+ * runs `attempt` again, at most `policy.retries` more times. The last error goes to the caller.
+ */
+export async function withRetries<T>(attempt: () => Promise<T>, policy: RetryPolicy, limits: CallLimits): Promise<T> {
+  for (let retry = 1; ; retry++) {
+    try {
+      return await attempt();
+    } catch (error) {
+      if (!(await waitToRetry(error, retry, policy, limits))) throw error;
+    }
+  }
+}