@codilore/llm 1.15.13

package/test/lib/tool-runtime.ts

@@ -0,0 +1,146 @@
+import { Effect, Stream } from "effect"
+import { LLMClient } from "../../src/route"
+import {
+  LLMEvent,
+  LLMRequest,
+  Message,
+  type ContentPart,
+  type ProviderMetadata,
+  type ToolCallPart,
+  ToolResultPart,
+  type ToolResultValue,
+  type Usage,
+} from "../../src/schema"
+import { type Tools, toDefinitions } from "../../src/tool"
+import { ToolRuntime } from "../../src/tool-runtime"
+
+interface RunOptions<T extends Tools> {
+  readonly request: LLMRequest
+  readonly tools: T
+  readonly maxSteps?: number
+}
+
+/** Test-owned continuation loop. Production callers must own durable history. */
+export const runTools = <T extends Tools>(options: RunOptions<T>) =>
+  Stream.unwrap(
+    Effect.gen(function* () {
+      const names = new Set(Object.keys(options.tools))
+      let request = LLMRequest.update(options.request, {
+        tools: [...options.request.tools.filter((tool) => !names.has(tool.name)), ...toDefinitions(options.tools)],
+      })
+      let usage: Usage | undefined
+      const events: LLMEvent[] = []
+
+      for (let step = 0; step < (options.maxSteps ?? 10); step++) {
+        const streamed = Array.from(yield* LLMClient.stream(request).pipe(Stream.runCollect))
+        const state = stepState(streamed)
+        usage = addUsage(usage, state.usage)
+        events.push(...streamed.filter((event) => event.type !== "finish").map((event) => indexStep(event, step)))
+
+        if (state.toolCalls.length === 0) {
+          events.push(LLMEvent.finish({ reason: state.reason, usage, providerMetadata: state.providerMetadata }))
+          return Stream.fromIterable(events)
+        }
+
+        const dispatched = yield* Effect.forEach(
+          state.toolCalls,
+          (call) => ToolRuntime.dispatch(options.tools, call).pipe(Effect.map((result) => [call, result] as const)),
+          { concurrency: 10 },
+        )
+        events.push(...dispatched.flatMap(([, result]) => result.events))
+
+        if (step + 1 >= (options.maxSteps ?? 10)) {
+          events.push(LLMEvent.finish({ reason: state.reason, usage, providerMetadata: state.providerMetadata }))
+          return Stream.fromIterable(events)
+        }
+
+        request = LLMRequest.update(request, {
+          messages: [
+            ...request.messages,
+            Message.assistant(state.assistantContent),
+            ...dispatched.map(([call, dispatched]) =>
+              Message.tool({ id: call.id, name: call.name, result: dispatched.result }),
+            ),
+          ],
+        })
+      }
+
+      return Stream.fromIterable(events)
+    }),
+  )
+
+const indexStep = (event: LLMEvent, index: number): LLMEvent => {
+  if (event.type === "step-start") return LLMEvent.stepStart({ index })
+  if (event.type === "step-finish") return LLMEvent.stepFinish({ ...event, index })
+  return event
+}
+
+const stepState = (events: ReadonlyArray<LLMEvent>) => {
+  const assistantContent: ContentPart[] = []
+  const toolCalls: ToolCallPart[] = []
+  let reason: Extract<LLMEvent, { type: "finish" }>["reason"] = "unknown"
+  let usage: Usage | undefined
+  let providerMetadata: ProviderMetadata | undefined
+
+  for (const event of events) {
+    if (event.type === "text-delta" || event.type === "reasoning-delta") {
+      appendText(assistantContent, event.type === "text-delta" ? "text" : "reasoning", event.text)
+    } else if (event.type === "text-end" || event.type === "reasoning-end") {
+      appendText(assistantContent, event.type === "text-end" ? "text" : "reasoning", "", event.providerMetadata)
+    } else if (event.type === "tool-call") {
+      assistantContent.push(event)
+      if (!event.providerExecuted) toolCalls.push(event)
+    } else if (event.type === "tool-result" && event.providerExecuted && event.result !== undefined) {
+      assistantContent.push(
+        ToolResultPart.make({
+          id: event.id,
+          name: event.name,
+          result: event.result,
+          providerExecuted: true,
+          providerMetadata: event.providerMetadata,
+        }),
+      )
+    } else if (event.type === "finish") {
+      reason = event.reason
+      usage = event.usage
+      providerMetadata = event.providerMetadata
+    }
+  }
+  return { assistantContent, toolCalls, reason, usage, providerMetadata }
+}
+
+const appendText = (
+  content: ContentPart[],
+  type: "text" | "reasoning",
+  text: string,
+  providerMetadata?: ProviderMetadata,
+) => {
+  const last = content.at(-1)
+  if (last?.type === type) {
+    content[content.length - 1] = {
+      ...last,
+      text: `${last.text}${text}`,
+      providerMetadata: providerMetadata ?? last.providerMetadata,
+    }
+    return
+  }
+  content.push({ type, text, providerMetadata })
+}
+
+const addUsage = (left: Usage | undefined, right: Usage | undefined): Usage | undefined => {
+  if (!left) return right
+  if (!right) return left
+  const sum = (key: keyof Usage) =>
+    typeof left[key] !== "number" && typeof right[key] !== "number"
+      ? undefined
+      : ((left[key] as number | undefined) ?? 0) + ((right[key] as number | undefined) ?? 0)
+  return {
+    inputTokens: sum("inputTokens"),
+    outputTokens: sum("outputTokens"),
+    nonCachedInputTokens: sum("nonCachedInputTokens"),
+    cacheReadInputTokens: sum("cacheReadInputTokens"),
+    cacheWriteInputTokens: sum("cacheWriteInputTokens"),
+    reasoningTokens: sum("reasoningTokens"),
+    totalTokens: sum("totalTokens"),
+  } as Usage
+}

package/test/llm.test.ts

@@ -0,0 +1,167 @@
+import { describe, expect, test } from "bun:test"
+import { CacheHint, LLM, LLMResponse } from "../src"
+import * as OpenAIChat from "../src/protocols/openai-chat"
+import * as OpenAIResponses from "../src/protocols/openai-responses"
+import { LLMRequest, Message, Model, ToolCallPart, ToolChoice, ToolDefinition, ToolResultPart } from "../src/schema"
+
+const chatRoute = OpenAIChat.route
+const responsesRoute = OpenAIResponses.route
+
+describe("llm constructors", () => {
+  test("builds canonical schema classes from ergonomic input", () => {
+    const request = LLM.request({
+      id: "req_1",
+      model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
+      system: "You are concise.",
+      prompt: "Say hello.",
+    })
+
+    expect(request).toBeInstanceOf(LLMRequest)
+    expect(request.model).toBeInstanceOf(Model)
+    expect(request.messages[0]).toBeInstanceOf(Message)
+    expect(request.system).toEqual([{ type: "text", text: "You are concise." }])
+    expect(request.messages[0]?.content).toEqual([{ type: "text", text: "Say hello." }])
+    expect(request.generation).toBeUndefined()
+    expect(request.tools).toEqual([])
+  })
+
+  test("updates requests without spreading schema class instances", () => {
+    const base = LLM.request({
+      id: "req_1",
+      model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
+      prompt: "Say hello.",
+    })
+    const updated = LLM.updateRequest(base, {
+      generation: { maxTokens: 20 },
+      messages: [...base.messages, Message.assistant("Hi.")],
+    })
+
+    expect(updated).toBeInstanceOf(LLMRequest)
+    expect(updated.id).toBe("req_1")
+    expect(updated.model).toEqual(base.model)
+    expect(updated.generation).toEqual({ maxTokens: 20 })
+    expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
+  })
+
+  test("keeps request options separate from route defaults", () => {
+    const request = LLM.request({
+      model: Model.make({
+        id: "fake-model",
+        provider: "fake",
+        route: chatRoute.with({
+          generation: { maxTokens: 100, temperature: 1 },
+          providerOptions: { openai: { store: false, metadata: { model: true } } },
+          http: { body: { metadata: { model: true } }, headers: { "x-shared": "model" }, query: { model: "1" } },
+        }),
+      }),
+      prompt: "Say hello.",
+      generation: { temperature: 0 },
+      providerOptions: { openai: { store: true, metadata: { request: true } } },
+      http: { body: { metadata: { request: true } }, headers: { "x-shared": "request" }, query: { request: "1" } },
+    })
+
+    expect(request.generation).toEqual({ temperature: 0 })
+    expect(request.providerOptions).toEqual({ openai: { store: true, metadata: { request: true } } })
+    expect(request.http).toEqual({
+      body: { metadata: { request: true } },
+      headers: { "x-shared": "request" },
+      query: { request: "1" },
+    })
+  })
+
+  test("updates canonical requests from the request datatype", () => {
+    const base = LLM.request({
+      id: "req_1",
+      model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
+      prompt: "Say hello.",
+    })
+    const updated = LLMRequest.update(base, { messages: [...base.messages, Message.assistant("Hi.")] })
+
+    expect(updated).toBeInstanceOf(LLMRequest)
+    expect(updated.id).toBe("req_1")
+    expect(LLMRequest.input(updated).id).toBe("req_1")
+    expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
+    expect(LLMRequest.update(updated, {})).toBe(updated)
+  })
+
+  test("updates canonical models from the model datatype", () => {
+    const base = Model.make({
+      id: "fake-model",
+      provider: "fake",
+      route: chatRoute,
+    })
+    const updated = Model.update(base, { route: responsesRoute })
+
+    expect(updated).toBeInstanceOf(Model)
+    expect(String(updated.id)).toBe("fake-model")
+    expect(updated.route).toBe(responsesRoute)
+    expect(String(Model.input(updated).provider)).toBe("fake")
+    expect(Model.update(updated, {})).toBe(updated)
+  })
+
+  test("builds tool choices from names and tools", () => {
+    const tool = ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })
+
+    expect(tool).toBeInstanceOf(ToolDefinition)
+    expect(ToolChoice.make("lookup")).toEqual(new ToolChoice({ type: "tool", name: "lookup" }))
+    expect(ToolChoice.named("required")).toEqual(new ToolChoice({ type: "tool", name: "required" }))
+    expect(ToolChoice.make(tool)).toEqual(new ToolChoice({ type: "tool", name: "lookup" }))
+  })
+
+  test("builds tool choice modes from reserved strings", () => {
+    expect(ToolChoice.make("auto")).toEqual(new ToolChoice({ type: "auto" }))
+    expect(ToolChoice.make("none")).toEqual(new ToolChoice({ type: "none" }))
+    expect(ToolChoice.make("required")).toEqual(new ToolChoice({ type: "required" }))
+    expect(
+      LLM.request({
+        model: Model.make({
+          id: "fake-model",
+          provider: "fake",
+          route: chatRoute,
+        }),
+        prompt: "Use tools if needed.",
+        toolChoice: "required",
+      }).toolChoice,
+    ).toEqual(new ToolChoice({ type: "required" }))
+  })
+
+  test("builds assistant tool calls and tool result messages", () => {
+    const call = ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })
+    const result = ToolResultPart.make({ id: "call_1", name: "lookup", result: { temperature: 72 } })
+
+    expect(Message.assistant([call]).content).toEqual([call])
+    expect(Message.tool(result).content).toEqual([
+      { type: "tool-result", id: "call_1", name: "lookup", result: { type: "json", value: { temperature: 72 } } },
+    ])
+  })
+
+  test("builds chronological text-only system updates separately from the initial system prompt", () => {
+    const update = Message.system([
+      { type: "text", text: "Use parameterized SQL.", cache: new CacheHint({ type: "ephemeral" }) },
+    ])
+    const request = LLM.request({
+      model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
+      system: "Initial operator prompt.",
+      messages: [Message.user("Review this."), update],
+    })
+
+    expect(update).toBeInstanceOf(Message)
+    expect(update).toEqual({
+      role: "system",
+      content: [{ type: "text", text: "Use parameterized SQL.", cache: { type: "ephemeral" } }],
+    })
+    expect(request.system).toEqual([{ type: "text", text: "Initial operator prompt." }])
+    expect(request.messages.map((message) => message.role)).toEqual(["user", "system"])
+  })
+
+  test("extracts output text from response events", () => {
+    expect(
+      LLMResponse.text({
+        events: [
+          { type: "text-delta", id: "text-0", text: "hi" },
+          { type: "finish", reason: "stop" },
+        ],
+      }),
+    ).toBe("hi")
+  })
+})

package/test/provider/anthropic-messages-cache.recorded.test.ts

@@ -0,0 +1,54 @@
+import { Redactor } from "@codilore/http-recorder"
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { CacheHint, LLM } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as Anthropic from "../../src/providers/anthropic"
+import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const model = Anthropic.configure({
+  apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
+}).model("claude-haiku-4-5-20251001")
+
+// Two identical generations in a row. The first call writes the prefix into
+// Anthropic's cache; the second should report a cache read against the same
+// prefix. Cassette captures both interactions in order.
+const cacheRequest = LLM.request({
+  id: "recorded_anthropic_cache",
+  model,
+  system: [{ type: "text", text: LARGE_CACHEABLE_SYSTEM, cache: new CacheHint({ type: "ephemeral" }) }],
+  prompt: "Say hi.",
+  // Manual hint on the system part is the only marker we want here — skip the
+  // auto-policy's latest-user-message breakpoint so the cassette body matches.
+  cache: "none",
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
+const recorded = recordedTests({
+  prefix: "anthropic-messages-cache",
+  provider: "anthropic",
+  protocol: "anthropic-messages",
+  requires: ["ANTHROPIC_API_KEY"],
+  // Two identical requests in one cassette — replay walks the cassette in
+  // recording order so the second call replays the cached-hit interaction.
+  options: {
+    redactor: Redactor.defaults({ requestHeaders: { allow: ["content-type", "anthropic-version"] } }),
+  },
+})
+
+describe("Anthropic Messages cache recorded", () => {
+  recorded.effect.with("writes then reads cache_control on identical second call", { tags: ["cache"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(cacheRequest)
+      // The first call may write the cache (cacheWriteInputTokens > 0) or it
+      // may be a fresh miss (both fields 0) depending on whether the prefix is
+      // already warm on Anthropic's side. The assertion that matters is that
+      // the SECOND call reports a non-zero cache read.
+      expect(first.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+
+      const second = yield* LLMClient.generate(cacheRequest)
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
+    }),
+  )
+})

package/test/provider/anthropic-messages.recorded.test.ts

@@ -0,0 +1,46 @@
+import { Redactor } from "@codilore/http-recorder"
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { LLM, LLMError, Message, ToolCallPart } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as Anthropic from "../../src/providers/anthropic"
+import { weatherToolName } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const model = Anthropic.configure({
+  apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
+}).model("claude-haiku-4-5-20251001")
+
+const malformedToolOrderRequest = LLM.request({
+  id: "recorded_anthropic_malformed_tool_order",
+  model,
+  messages: [
+    Message.assistant([
+      ToolCallPart.make({ id: "call_1", name: weatherToolName, input: { city: "Paris" } }),
+      { type: "text", text: "I will check the weather." },
+    ]),
+    Message.tool({ id: "call_1", name: weatherToolName, result: { temperature: "72F" } }),
+    Message.user("Use that result to answer briefly."),
+  ],
+  tools: [{ name: weatherToolName, description: "Get weather", inputSchema: { type: "object", properties: {} } }],
+})
+
+const recorded = recordedTests({
+  prefix: "anthropic-messages",
+  provider: "anthropic",
+  protocol: "anthropic-messages",
+  requires: ["ANTHROPIC_API_KEY"],
+  options: { redactor: Redactor.defaults({ requestHeaders: { allow: ["content-type", "anthropic-version"] } }) },
+})
+
+describe("Anthropic Messages sad-path recorded", () => {
+  recorded.effect.with("rejects malformed assistant tool order", { tags: ["tool", "sad-path"] }, () =>
+    Effect.gen(function* () {
+      const error = yield* LLMClient.generate(malformedToolOrderRequest).pipe(Effect.flip)
+
+      expect(error).toBeInstanceOf(LLMError)
+      expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
+      expect(error.message).toContain("HTTP 400")
+    }),
+  )
+})