@procedures/llm 0.7.0 → 0.8.0

package/src/events.ts

@@ -10,12 +10,16 @@
  *
  * A stream gets a new try only before its first item. After the first item, an error goes to the
  * caller.
+ *
+ * The stream of one request gives the `start`, `text` and `tool_use` items. Its return value is
+ * the answer with its content blocks. `llm.stream` gives the `stop` item at the end of its tool loop.
  */
 
 import { HttpError, sseEvents, type ServerSentEvent } from "@procedures/http";
 import { InvocationError } from "@procedures/client";
-import { openScope, postMessages, safeMessage, usageOf, NO_USAGE, type CallLimits, type LlmSettings, type MessagesRequest } from "./anthropic.js";
+import { openScope, postMessages, safeMessage, usageOf, NO_USAGE, type CallLimits, type ContentBlock, type LlmSettings, type MessagesRequest } from "./anthropic.js";
 import { LlmError } from "./errors.js";
+import type { TurnAnswer } from "./loop.js";
 import { waitToRetry } from "./retry.js";
 import type { LlmStreamOutput, LlmToolUse, LlmUsage } from "./types.js";
 
@@ -66,15 +70,39 @@ function streamError(data: Record<string, unknown>): LlmError {
   return new LlmError("API_ERROR", `llm: the API sent an error event: ${type}: ${message}`, { retryable: RETRYABLE_ERROR_TYPES.has(type) });
 }
 
-/** One content block of a stream while its deltas come. */
+/**
+ * One content block of a stream while its deltas come. `block` is the block as the event
+ * `content_block_start` gave it, with the deltas added.
+ */
 interface OpenBlock {
-  readonly type: string;
-  readonly id: string;
-  readonly name: string;
+  readonly block: Record<string, unknown>;
+  /** The parts of `input_json_delta`, joined. */
   json: string;
 }
 
-/** The state of one stream: it turns each event into the items of the procedure. */
+/** The type of block that each delta changes. An `input_json_delta` changes each block with an input. */
+const DELTA_TARGETS: ReadonlyMap<string, string> = new Map([
+  ["text_delta", "text"],
+  ["citations_delta", "text"],
+  ["thinking_delta", "thinking"],
+  ["signature_delta", "thinking"],
+]);
+
+/** The `input` of a block: the JSON of its deltas, parsed. A JSON that does not parse gives null and the text. */
+function inputOf(block: OpenBlock): { input: unknown; partialJson?: string } {
+  if (block.json.trim() === "") return { input: block.block["input"] ?? {} };
+  try {
+    return { input: JSON.parse(block.json) as unknown };
+  } catch {
+    return { input: null, partialJson: block.json };
+  }
+}
+
+/**
+ * The state of one stream. It turns each event into the items of the procedure. It also puts
+ * the content blocks of the answer together, as the JSON answer has them. The next request of the
+ * tool loop sends them back.
+ */
 class StreamState {
   private model: string;
   private id = "";
@@ -83,6 +111,7 @@ class StreamState {
   private category: string | undefined;
   private text = "";
   private readonly blocks = new Map<number, OpenBlock>();
+  private readonly content: Array<readonly [number, ContentBlock]> = [];
   private readonly toolUses: LlmToolUse[] = [];
   /** True after the event `message_stop`. */
   stopped = false;
@@ -91,6 +120,49 @@ class StreamState {
     this.model = model;
   }
 
+  /** The answer, after the event `message_stop`. */
+  answer(): TurnAnswer {
+    return {
+      model: this.model,
+      content: [...this.content].sort((a, b) => a[0] - b[0]).map(([, block]) => block),
+      stopReason: this.stopReason!,
+      usage: this.usage,
+      text: this.text,
+      toolUses: [...this.toolUses],
+    };
+  }
+
+  /**
+   * This function adds a delta to its open block. A delta for a block of another type, and a
+   * delta of a type that the procedure does not know, change nothing.
+   */
+  private addDelta(index: number, delta: Record<string, unknown>): void {
+    const open = this.blocks.get(index);
+    const type = stringOf(delta["type"]) ?? "";
+    const target = DELTA_TARGETS.get(type);
+    if (open === undefined || (target !== undefined && open.block["type"] !== target)) return;
+    const { block } = open;
+    switch (type) {
+      case "text_delta":
+        block["text"] = (stringOf(block["text"]) ?? "") + (stringOf(delta["text"]) ?? "");
+        return;
+      case "input_json_delta":
+        open.json += stringOf(delta["partial_json"]) ?? "";
+        return;
+      case "thinking_delta":
+        block["thinking"] = (stringOf(block["thinking"]) ?? "") + (stringOf(delta["thinking"]) ?? "");
+        return;
+      case "signature_delta":
+        block["signature"] = stringOf(delta["signature"]) ?? "";
+        return;
+      case "citations_delta":
+        block["citations"] = [...(Array.isArray(block["citations"]) ? (block["citations"] as unknown[]) : []), delta["citation"]];
+        return;
+      default:
+        return;
+    }
+  }
+
   /** The items of one event. The event `error` and a refusal at the end throw. */
   apply(event: ServerSentEvent): LlmStreamOutput[] {
     if (event.event === "ping") return [];
@@ -105,12 +177,12 @@ class StreamState {
         return [{ type: "start", id: this.id, model: this.model, usage: this.usage }];
       }
       case "content_block_start": {
-        const block = recordOf(data["content_block"]);
+        const block = { ...recordOf(data["content_block"]) };
         const type = stringOf(block["type"]) ?? "";
         // A server-side fallback puts a fallback block at the change of the model
         const to = stringOf(recordOf(block["to"])["model"]);
         if (type === "fallback" && to) this.model = to;
-        this.blocks.set(index, { type, id: stringOf(block["id"]) ?? "", name: stringOf(block["name"]) ?? "", json: "" });
+        this.blocks.set(index, { block, json: "" });
         const text = stringOf(block["text"]);
         if (type === "text" && text) {
           this.text += text;
@@ -120,34 +192,28 @@ class StreamState {
       }
       case "content_block_delta": {
         const delta = recordOf(data["delta"]);
-        const deltaType = stringOf(delta["type"]);
-        if (deltaType === "text_delta") {
+        this.addDelta(index, delta);
+        if (stringOf(delta["type"]) === "text_delta") {
           const text = stringOf(delta["text"]) ?? "";
           this.text += text;
           return text === "" ? [] : [{ type: "text", index, text }];
         }
-        if (deltaType === "input_json_delta") {
-          const block = this.blocks.get(index);
-          if (block) block.json += stringOf(delta["partial_json"]) ?? "";
-        }
         return [];
       }
       case "content_block_stop": {
-        const block = this.blocks.get(index);
+        const open = this.blocks.get(index);
+        if (open === undefined) return [];
         this.blocks.delete(index);
-        if (block?.type !== "tool_use") return [];
-        let input: unknown = {};
-        let partialJson: string | undefined;
-        if (block.json.trim() !== "") {
-          try {
-            input = JSON.parse(block.json);
-          } catch {
-            input = null;
-            partialJson = block.json;
-          }
-        }
-        this.toolUses.push({ id: block.id, name: block.name, input });
-        return [{ type: "tool_use", index, id: block.id, name: block.name, input, ...(partialJson !== undefined ? { partialJson } : {}) }];
+        const { block } = open;
+        const { input, partialJson } = inputOf(open);
+        // A tool_use block and a server_tool_use block get their input from the deltas
+        if (open.json !== "" || block["type"] === "tool_use") block["input"] = input;
+        this.content.push([index, block as ContentBlock]);
+        if (block["type"] !== "tool_use") return [];
+        const id = stringOf(block["id"]) ?? "";
+        const name = stringOf(block["name"]) ?? "";
+        this.toolUses.push({ id, name, input });
+        return [{ type: "tool_use", index, id, name, input, ...(partialJson !== undefined ? { partialJson } : {}) }];
       }
       case "message_delta": {
         const delta = recordOf(data["delta"]);
@@ -162,7 +228,7 @@ class StreamState {
           throw new LlmError("REFUSAL", `llm: a safety classifier declined the request (category: ${this.category ?? "not given"})`);
         }
         this.stopped = true;
-        return [{ type: "stop", model: this.model, stopReason: this.stopReason, usage: this.usage, text: this.text, toolUses: [...this.toolUses] }];
+        return [];
       }
       case "error":
         throw streamError(data);
@@ -172,8 +238,13 @@ class StreamState {
   }
 }
 
-/** The items of one try of a stream, in one request scope. */
-async function* streamOnce(request: MessagesRequest, settings: LlmSettings, limits: CallLimits): AsyncGenerator<LlmStreamOutput, void, undefined> {
+/** The progress of one try: true after its first item. */
+interface TryProgress {
+  started: boolean;
+}
+
+/** The items of one try of a stream, in one request scope. The return value is the answer. */
+async function* streamOnce(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, progress: TryProgress): AsyncGenerator<LlmStreamOutput, TurnAnswer, undefined> {
   const scope = openScope(settings, limits);
   try {
     const response = await postMessages(request, settings, scope);
@@ -181,8 +252,11 @@ async function* streamOnce(request: MessagesRequest, settings: LlmSettings, limi
     const state = new StreamState(request.model);
     try {
       for await (const event of sseEvents(chunksOf(response.body), MAX_EVENT_BYTES, limits.path.join("."))) {
-        for (const item of state.apply(event)) yield item;
-        if (state.stopped) return;
+        for (const item of state.apply(event)) {
+          progress.started = true;
+          yield item;
+        }
+        if (state.stopped) return state.answer();
       }
     } catch (error) {
       if (error instanceof LlmError || error instanceof InvocationError || scope.stopped() !== undefined) throw error;
@@ -198,21 +272,18 @@ async function* streamOnce(request: MessagesRequest, settings: LlmSettings, limi
 }
 
 /**
- * This function sends a request with `"stream": true` and gives each item of the answer. Before
- * the first item, a retryable error gets a new try, at most `retries` times.
+ * This function sends a request with `"stream": true` and gives each item of the answer: the
+ * `start`, `text` and `tool_use` items. The return value is the answer. Before the first item, a
+ * retryable error gets a new try, at most `retries` times.
  */
-export async function* streamMessages(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number): AsyncGenerator<LlmStreamOutput, void, undefined> {
+export async function* streamMessages(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number): AsyncGenerator<LlmStreamOutput, TurnAnswer, undefined> {
   const body: MessagesRequest = { ...request, stream: true };
   for (let retry = 1; ; retry++) {
-    let started = false;
+    const progress: TryProgress = { started: false };
     try {
-      for await (const item of streamOnce(body, settings, limits)) {
-        started = true;
-        yield item;
-      }
-      return;
+      return yield* streamOnce(body, settings, limits, progress);
     } catch (error) {
-      if (started || !(await waitToRetry(error, retry, { backoff: settings.backoff, retries }, limits))) throw error;
+      if (progress.started || !(await waitToRetry(error, retry, { backoff: settings.backoff, retries }, limits))) throw error;
     }
   }
 }

package/src/loop.ts

@@ -0,0 +1,159 @@
+/**
+ * The tool loop of `llm.complete` and `llm.stream`.
+ *
+ * The loop sends a request and reads the answer. When the answer asks only for procedure tools,
+ * the loop starts them, sends the answer and the results back, and asks again. It stops when the
+ * model stops, or when `maxRounds` requests went out. The two procedures differ only in one
+ * step: `llm.complete` gets each answer as JSON, and `llm.stream` reads the event stream of each
+ * answer. A `SendTurn` function does that step.
+ */
+
+import type { ProcedurePath } from "@procedures/client";
+import { addUsage, NO_USAGE, type CallLimits, type ContentBlock, type LlmSettings, type MessagesRequest } from "./anthropic.js";
+import { addTools, baseRequest, retriesOf, timeoutOf, toolParam } from "./request.js";
+import { DEFAULT_MAX_ROUNDS, procedureTools, runToolUse, type ToolContext } from "./tools.js";
+import type { LlmRequestInput, LlmStreamToolCall, LlmStreamToolResult, LlmToolCall, LlmToolInput, LlmToolUse, LlmUsage } from "./types.js";
+
+/** One answer of the model, as the loop reads it. */
+export interface TurnAnswer {
+  /** The model that answered. */
+  readonly model: string;
+  /** The content blocks of the answer. The next request sends them back, after `continuationOf()`. */
+  readonly content: readonly ContentBlock[];
+  /** Why the model stopped. */
+  readonly stopReason: string;
+  /** The tokens of the request. */
+  readonly usage: LlmUsage;
+  /** The text of the answer. */
+  readonly text: string;
+  /** The `tool_use` blocks of the answer. */
+  readonly toolUses: readonly LlmToolUse[];
+}
+
+/**
+ * One request of the loop. The function sends the request, with at most `retries` new tries.
+ * It gives the items of the answer. Its return value is the answer.
+ */
+export type SendTurn<T> = (request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number) => AsyncGenerator<T, TurnAnswer, undefined>;
+
+/** An item of the loop: the start or the end of a procedure tool. */
+export type ToolEvent = LlmStreamToolCall | LlmStreamToolResult;
+
+/** The end of the loop. */
+export interface LoopEnd {
+  /** The last answer. */
+  readonly answer: TurnAnswer;
+  /** The model of the last answer. Without one, it is the model of the request. */
+  readonly model: string;
+  /** The tokens of all the requests, added together. */
+  readonly usage: LlmUsage;
+  /** The text of all the answers, joined. */
+  readonly text: string;
+  /** The calls of procedure tools, in their sequence. */
+  readonly toolCalls: LlmToolCall[];
+  /** The number of requests to the model. */
+  readonly rounds: number;
+  /** True when `maxRounds` stopped the loop before the model stopped. */
+  readonly roundLimit: boolean;
+}
+
+/** The block types that the next turn drops before the final `fallback` block. */
+const DROPPED_BEFORE_FALLBACK = new Set(["thinking", "redacted_thinking", "connector_text", "tool_use"]);
+
+/**
+ * The content of an answer as the next request sends it back. After a server-side fallback in
+ * the middle of an answer, the blocks before the final `fallback` block came from the model that
+ * declined. The next turn drops their thinking, their `connector_text` and their client
+ * `tool_use` blocks, and each server tool use without its result. The `fallback` block, the
+ * text and each block after the final `fallback` block stay where they are. Without a `fallback`
+ * block, the content stays as it came. The rules are in the API docs, "Refusals and fallback",
+ * section "Continuing the conversation". Before, the loop sent the content back unchanged, and
+ * the next request could fail (tracker O34).
+ */
+export function continuationOf(content: readonly ContentBlock[]): ContentBlock[] {
+  let last = -1;
+  for (let index = content.length - 1; index >= 0; index--) {
+    if (content[index]!.type === "fallback") {
+      last = index;
+      break;
+    }
+  }
+  if (last < 0) return [...content];
+  // A server tool result names its tool use. The results can come after the fallback too.
+  const results = new Set(content.map((block) => block["tool_use_id"]).filter((id): id is string => typeof id === "string"));
+  const kept = content.slice(0, last).filter((block) => {
+    if (DROPPED_BEFORE_FALLBACK.has(block.type)) return false;
+    if (block.type.endsWith("_tool_use")) return typeof block["id"] === "string" && results.has(block["id"]);
+    return true;
+  });
+  return [...kept, ...content.slice(last)];
+}
+
+/** A forced `tool_choice` (`any` or `tool`) applies to the first request only. After it, the model can stop. */
+function releaseToolChoice(request: MessagesRequest): void {
+  const choice = request.tool_choice;
+  if (choice === undefined || (choice.type !== "any" && choice.type !== "tool")) return;
+  const { disable_parallel_tool_use } = choice;
+  request.tool_choice = { type: "auto", ...(disable_parallel_tool_use !== undefined ? { disable_parallel_tool_use } : {}) };
+}
+
+/**
+ * This function does the tool loop of one call. It gives the items of each answer and an item at
+ * the start and at the end of each procedure tool. The return value is the end of the loop. It
+ * throws before the first request for a tool that the host refuses or that is not valid.
+ */
+export async function* toolLoop<T>(
+  input: LlmRequestInput & LlmToolInput,
+  ctx: ToolContext | undefined,
+  settings: LlmSettings,
+  path: ProcedurePath,
+  send: SendTurn<T>,
+): AsyncGenerator<T | ToolEvent, LoopEnd, undefined> {
+  const request = baseRequest(input, settings, input.system);
+  const procedures = procedureTools(input.procedures ?? [], ctx, path);
+  addTools(request, [...(input.tools ?? []).map(toolParam), ...procedures.map((tool) => tool.param)], input.toolChoice);
+  const byName = new Map(procedures.map((tool) => [tool.param.name, tool]));
+  const limits: CallLimits = { path, ctx, timeoutMs: timeoutOf(input) };
+  const retries = retriesOf(input.retries, settings);
+  const maxRounds = input.maxRounds ?? DEFAULT_MAX_ROUNDS;
+
+  let usage: LlmUsage = NO_USAGE;
+  let text = "";
+  const toolCalls: LlmToolCall[] = [];
+  for (let round = 1; ; round++) {
+    const answer = yield* send(request, settings, limits, retries);
+    usage = addUsage(usage, answer.usage);
+    text += answer.text;
+    // Only the tool uses that the next turn sends back can get a result: none before a fallback
+    const content = continuationOf(answer.content);
+    const sent = new Set(content.filter((block) => block.type === "tool_use").map((block) => block["id"]));
+    const uses = answer.toolUses.filter((use) => sent.has(use.id));
+    // The procedure runs the tools only when each tool of the answer is a procedure tool
+    const runnable = answer.stopReason === "tool_use" && uses.length > 0 && uses.every((use) => byName.has(use.name));
+    if (!runnable || round >= maxRounds) {
+      return { answer, model: answer.model || request.model, usage, text, toolCalls, rounds: round, roundLimit: runnable };
+    }
+    // One at a time, in the sequence of the answer: a procedure can change data that the next one reads
+    const results: ContentBlock[] = [];
+    for (const use of uses) {
+      const tool = byName.get(use.name)!;
+      yield { type: "tool_call", round, id: use.id, name: use.name, procedure: tool.key, input: use.input };
+      const { call, result } = await runToolUse(use, tool, ctx!);
+      toolCalls.push(call);
+      results.push(result);
+      yield { type: "tool_result", round, ...call };
+    }
+    // The next request continues the conversation: the answer (without the blocks that a fallback
+    // replaced), then the results
+    request.messages = [...request.messages, { role: "assistant", content }, { role: "user", content: results }];
+    releaseToolChoice(request);
+  }
+}
+
+/** This function reads a generator to its end and gives its return value. It ignores the items. */
+export async function returnOf<R>(generator: AsyncGenerator<unknown, R, undefined>): Promise<R> {
+  for (;;) {
+    const step = await generator.next();
+    if (step.done) return step.value;
+  }
+}

package/src/module.ts

@@ -110,16 +110,17 @@ const toolCall = z.object({
   error: z.string().optional().describe("The error of the procedure"),
 });
 
-const LlmCompleteInputSchema = z.object({
-  ...requestFields,
-  ...toolFields,
+/** The fields of procedures as tools, for `llm.complete` and `llm.stream`. */
+const procedureFields = {
   procedures: z
     .array(z.string().regex(/^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/, "A procedure key, for example weather.current"))
     .max(64)
     .optional()
-    .describe("The procedures that the model can use as tools: llm.complete calls them and sends the results back"),
+    .describe("The procedures that the model can use as tools: the procedure calls them and sends the results back"),
   maxRounds: z.number().int().min(1).max(64).optional().describe("The most requests to the model in one call, 1 to 64 (default 8)"),
-});
+};
+
+const LlmCompleteInputSchema = z.object({ ...requestFields, ...toolFields, ...procedureFields });
 
 const LlmCompleteOutputSchema = z.object({
   text: z.string().describe("The text of the last answer"),
@@ -148,10 +149,12 @@ const LlmStructuredOutputSchema = z.object({
   unchecked: z.array(z.string()).describe("The keywords of the schema that the check does not examine fully"),
 });
 
-const LlmStreamInputSchema = z.object({ ...requestFields, ...toolFields });
+const LlmStreamInputSchema = z.object({ ...requestFields, ...toolFields, ...procedureFields });
 
 const index = z.number().int().describe("The index of the content block");
 
+const round = z.number().int().describe("The number of the request whose answer asked for the tool");
+
 const LlmStreamOutputSchema = z
   .discriminatedUnion("type", [
     z.object({ type: z.literal("start"), id: z.string().describe("The ID of the message"), model: z.string().describe("The model that answers"), usage }),
@@ -164,16 +167,28 @@ const LlmStreamOutputSchema = z
       input: z.unknown().describe("The input of the call, or null when its JSON is not complete"),
       partialJson: z.string().optional().describe("The JSON text of an input that does not parse"),
     }),
+    z.object({
+      type: z.literal("tool_call"),
+      round,
+      id: z.string().describe("The ID of the tool_use block"),
+      name: z.string().describe("The name of the tool"),
+      procedure: z.string().describe("The key of the procedure"),
+      input: z.unknown().describe("The input that the model gave"),
+    }),
+    toolCall.extend({ type: z.literal("tool_result"), round }),
     z.object({
       type: z.literal("stop"),
-      model: z.string().describe("The model that answered"),
-      stopReason: z.string().describe("Why the model stopped"),
-      usage,
-      text: z.string().describe("The text of the answer: all the text deltas, joined"),
-      toolUses: z.array(toolUse).describe("The tool_use blocks of the answer"),
+      model: z.string().describe("The model of the last answer"),
+      stopReason: z.string().describe("Why the model stopped in the last answer"),
+      usage: usage.describe("The tokens of all the requests, added together"),
+      text: z.string().describe("The text of all the answers: all the text deltas, joined"),
+      toolUses: z.array(toolUse).describe("The tool_use blocks of the last answer that no procedure ran"),
+      toolCalls: z.array(toolCall).optional().describe("The calls of the procedure tools, in their sequence. Only with procedures"),
+      rounds: z.number().int().optional().describe("The number of requests to the model. Only with procedures"),
+      roundLimit: z.boolean().optional().describe("True when maxRounds stopped the loop before the model stopped. Only with procedures"),
     }),
   ])
-  .describe("One item of the stream: start, text, tool_use or stop");
+  .describe("One item of the stream: start, text, tool_use, tool_call, tool_result or stop");
 
 // =============================================================================
 // Procedures
@@ -182,8 +197,8 @@ const LlmStreamOutputSchema = z
 /**
  * The effects of each procedure: it sends data to a system outside the host and gets an answer.
  * A call costs money and can change the state of an account, so it declares a write. It deletes
- * nothing and a second call is a new call. The procedure tools of `llm.complete` are nested calls:
- * the admission of the host examines each one.
+ * nothing and a second call is a new call. The procedure tools of `llm.complete` and `llm.stream`
+ * are nested calls: the admission of the host examines each one.
  */
 const SENDS_TO_MODEL: Effects = Object.freeze({ reads: Object.freeze(["network"]), writes: Object.freeze(["network"]), destructive: false, idempotent: false });
 
@@ -228,8 +243,8 @@ function llmProcedures(settings: LlmSettings): AnyProcedure[] {
     .input(zodAdapter<LlmStreamInput>(LlmStreamInputSchema))
     .output(zodAdapter<LlmStreamOutput>(LlmStreamOutputSchema))
     .meta({
-      description: "Send a prompt to a language model and stream the events of the answer",
-      tags: ["llm", "anthropic", "stream"],
+      description: "Send a prompt to a language model and stream the events of the answer. The model can use tools and procedures",
+      tags: ["llm", "anthropic", "stream", "tools"],
       args: ["prompt"],
       shorts: { model: "m", system: "s" },
       output: "json",
@@ -240,6 +255,8 @@ function llmProcedures(settings: LlmSettings): AnyProcedure[] {
     })
     .handler(((input: LlmStreamInput, ctx: ProcedureContext) => llmStream(input, ctx, settings)) as ProcedureHandler<LlmStreamInput, LlmStreamOutput>)
     .build();
+  // The same rule as for llm.complete: the input names the procedures that the model can call
+  markDataDriven(llmStreamProcedure);
 
   return [llmCompleteProcedure, llmStructuredProcedure, llmStreamProcedure];
 }

package/src/procedures/llm/complete.ts

@@ -7,61 +7,43 @@
  * With `tools`, the output gives each `tool_use` block of the answer, and the caller starts them.
  * With `procedures`, the procedure starts the tools. It calls each procedure that the model asks
  * for through `ctx.client` and sends the results back. Then it asks again, until the model stops
- * or `maxRounds` requests went out.
+ * or `maxRounds` requests went out. `toolLoop` of `loop.ts` has the loop, and `llm.stream` uses
+ * the same loop.
  */
 
-import { addUsage, DEFAULT_LLM_SETTINGS, NO_USAGE, sendWithRetries, textOf, type CallLimits, type ContentBlock, type LlmSettings } from "../../anthropic.js";
-import { addTools, baseRequest, retriesOf, timeoutOf, toolParam } from "../../request.js";
-import { DEFAULT_MAX_ROUNDS, procedureTools, runToolUse, toolUsesOf, type ToolContext } from "../../tools.js";
-import type { LlmCompleteInput, LlmCompleteOutput, LlmToolCall, LlmUsage } from "../../types.js";
+import { DEFAULT_LLM_SETTINGS, sendWithRetries, textOf, type CallLimits, type LlmSettings, type MessagesRequest } from "../../anthropic.js";
+import { returnOf, toolLoop, type TurnAnswer } from "../../loop.js";
+import { toolUsesOf, type ToolContext } from "../../tools.js";
+import type { LlmCompleteInput, LlmCompleteOutput } from "../../types.js";
 
 const PATH = ["llm", "complete"];
 
+/** One request of `llm.complete`: it gives no items. Its return value is the JSON answer. */
+async function* sendTurn(request: MessagesRequest, settings: LlmSettings, limits: CallLimits, retries: number): AsyncGenerator<never, TurnAnswer, undefined> {
+  const answer = await sendWithRetries(request, settings, limits, retries);
+  return {
+    model: answer.model,
+    content: answer.content,
+    stopReason: answer.stopReason,
+    usage: answer.usage,
+    text: textOf(answer.content),
+    toolUses: toolUsesOf(answer.content),
+  };
+}
+
 /**
  * Send a prompt to a language model and give the text and the usage
  */
 export async function llmComplete(input: LlmCompleteInput, ctx?: ToolContext, settings: LlmSettings = DEFAULT_LLM_SETTINGS): Promise<LlmCompleteOutput> {
-  const request = baseRequest(input, settings, input.system);
-  const procedures = procedureTools(input.procedures ?? [], ctx, PATH);
-  addTools(request, [...(input.tools ?? []).map(toolParam), ...procedures.map((tool) => tool.param)], input.toolChoice);
-  const byName = new Map(procedures.map((tool) => [tool.param.name, tool]));
-  const limits: CallLimits = { path: PATH, ctx, timeoutMs: timeoutOf(input) };
-  const retries = retriesOf(input.retries, settings);
-  const maxRounds = input.maxRounds ?? DEFAULT_MAX_ROUNDS;
-
-  let usage: LlmUsage = NO_USAGE;
-  const toolCalls: LlmToolCall[] = [];
-  for (let round = 1; ; round++) {
-    const answer = await sendWithRetries(request, settings, limits, retries);
-    usage = addUsage(usage, answer.usage);
-    const uses = toolUsesOf(answer.content);
-    // The procedure runs the tools only when each tool of the answer is a procedure tool
-    const runnable = answer.stopReason === "tool_use" && uses.length > 0 && uses.every((use) => byName.has(use.name));
-    if (!runnable || round >= maxRounds) {
-      return {
-        text: textOf(answer.content),
-        model: answer.model || request.model,
-        stopReason: answer.stopReason,
-        usage,
-        toolUses: uses,
-        toolCalls,
-        rounds: round,
-        roundLimit: runnable,
-      };
-    }
-    // One at a time, in the sequence of the answer: a procedure can change data that the next one reads
-    const results: ContentBlock[] = [];
-    for (const use of uses) {
-      const { call, result } = await runToolUse(use, byName.get(use.name)!, ctx!);
-      toolCalls.push(call);
-      results.push(result);
-    }
-    // The next request continues the conversation: the answer as it came, then the results
-    request.messages = [...request.messages, { role: "assistant", content: answer.content }, { role: "user", content: results }];
-    // A forced tool_choice applies to the first request only: then the model can stop
-    if (request.tool_choice !== undefined && (request.tool_choice.type === "any" || request.tool_choice.type === "tool")) {
-      const { disable_parallel_tool_use } = request.tool_choice;
-      request.tool_choice = { type: "auto", ...(disable_parallel_tool_use !== undefined ? { disable_parallel_tool_use } : {}) };
-    }
-  }
+  const end = await returnOf(toolLoop(input, ctx, settings, PATH, sendTurn));
+  return {
+    text: end.answer.text,
+    model: end.model,
+    stopReason: end.answer.stopReason,
+    usage: end.usage,
+    toolUses: [...end.answer.toolUses],
+    toolCalls: end.toolCalls,
+    rounds: end.rounds,
+    roundLimit: end.roundLimit,
+  };
 }

package/src/procedures/llm/stream.ts

@@ -4,20 +4,39 @@
  * Send a prompt to a language model and stream the events of the answer. The request has
  * `"stream": true`. The items are the start of the message, the text deltas and each complete
  * `tool_use` block. The last item has the stop reason, the usage and the full text.
+ *
+ * With `procedures`, the procedure starts the tools, as `llm.complete` does: `toolLoop` of
+ * `loop.ts` has the loop. When an answer stops for its procedure tools, the stream gives a
+ * `tool_call` item and a `tool_result` item for each tool. Then the next answer streams in the
+ * same output.
  */
 
-import { DEFAULT_LLM_SETTINGS, type CallLimits, type LlmSettings } from "../../anthropic.js";
+import { DEFAULT_LLM_SETTINGS, type LlmSettings } from "../../anthropic.js";
 import { streamMessages } from "../../events.js";
-import { addTools, baseRequest, retriesOf, timeoutOf, toolParam } from "../../request.js";
-import type { LlmStreamInput, LlmStreamOutput } from "../../types.js";
+import { toolLoop } from "../../loop.js";
+import type { ToolContext } from "../../tools.js";
+import type { LlmStreamInput, LlmStreamOutput, LlmStreamStop } from "../../types.js";
 
 const PATH = ["llm", "stream"];
 
 /**
  * Send a prompt to a language model and stream the events of the answer
  */
-export async function* llmStream(input: LlmStreamInput, ctx?: CallLimits["ctx"], settings: LlmSettings = DEFAULT_LLM_SETTINGS): AsyncGenerator<LlmStreamOutput, void, undefined> {
-  const request = baseRequest(input, settings, input.system);
-  addTools(request, (input.tools ?? []).map(toolParam), input.toolChoice);
-  yield* streamMessages(request, settings, { path: PATH, ctx, timeoutMs: timeoutOf(input) }, retriesOf(input.retries, settings));
+export async function* llmStream(input: LlmStreamInput, ctx?: ToolContext, settings: LlmSettings = DEFAULT_LLM_SETTINGS): AsyncGenerator<LlmStreamOutput, void, undefined> {
+  const end = yield* toolLoop(input, ctx, settings, PATH, streamMessages);
+  const stop: LlmStreamStop = {
+    type: "stop",
+    model: end.model,
+    stopReason: end.answer.stopReason,
+    usage: end.usage,
+    text: end.text,
+    toolUses: [...end.answer.toolUses],
+  };
+  // Without procedures, the stop item keeps the fields that it had before the tool loop
+  if ((input.procedures ?? []).length > 0) {
+    stop.toolCalls = end.toolCalls;
+    stop.rounds = end.rounds;
+    stop.roundLimit = end.roundLimit;
+  }
+  yield stop;
 }