@codilore/llm 1.15.13

package/test/auth.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect } from "bun:test"
+import { ConfigProvider, Effect } from "effect"
+import { Headers } from "effect/unstable/http"
+import { LLM } from "../src"
+import { Auth } from "../src/route/auth"
+import * as OpenAIChat from "../src/protocols/openai-chat"
+import { Model } from "../src/schema"
+import { it } from "./lib/effect"
+
+const request = LLM.request({
+  id: "req_auth",
+  model: Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }),
+  prompt: "hello",
+})
+
+const input = {
+  request,
+  method: "POST" as const,
+  url: "https://example.test/v1/chat",
+  body: "{}",
+  headers: Headers.fromInput({ "x-existing": "yes" }),
+}
+
+const withEnv = (env: Record<string, string>) => Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))
+
+describe("Auth", () => {
+  it.effect("renders a config credential as bearer auth", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.config("OPENAI_API_KEY")
+        .bearer()
+        .apply(input)
+        .pipe(withEnv({ OPENAI_API_KEY: "sk-test" }))
+
+      expect(headers.authorization).toBe("Bearer sk-test")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("falls back between credential sources before rendering", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.config("PRIMARY_KEY")
+        .orElse(Auth.value("fallback-key"))
+        .pipe(Auth.header("x-api-key"))
+        .apply(input)
+        .pipe(withEnv({}))
+
+      expect(headers["x-api-key"]).toBe("fallback-key")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("composes header auth in sequence", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.headers({ "x-tenant-id": "tenant-1" })
+        .andThen(Auth.bearer("gateway-token"))
+        .apply(input)
+
+      expect(headers["x-tenant-id"]).toBe("tenant-1")
+      expect(headers.authorization).toBe("Bearer gateway-token")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("renders a direct secret as a custom header", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.header("api-key", "direct-key").apply(input)
+
+      expect(headers["api-key"]).toBe("direct-key")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("renders bearer auth into a custom header", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.bearerHeader("cf-aig-authorization", "gateway-token").apply(input)
+
+      expect(headers["cf-aig-authorization"]).toBe("Bearer gateway-token")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("falls back between full auth values", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.config("OPENAI_API_KEY")
+        .bearer()
+        .orElse(Auth.headers({ authorization: "Bearer supplied" }))
+        .apply(input)
+        .pipe(withEnv({}))
+
+      expect(headers.authorization).toBe("Bearer supplied")
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+
+  it.effect("can intentionally leave auth untouched", () =>
+    Effect.gen(function* () {
+      const headers = yield* Auth.none.apply(input)
+
+      expect(headers.authorization).toBeUndefined()
+      expect(headers["x-existing"]).toBe("yes")
+    }),
+  )
+})

package/test/cache-policy.test.ts

@@ -0,0 +1,262 @@
+import { describe, expect, test } from "bun:test"
+import { Effect } from "effect"
+import { CacheHint, LLM, Message } from "../src"
+import { Auth, LLMClient } from "../src/route"
+import { AmazonBedrock } from "../src/providers"
+import * as AnthropicMessages from "../src/protocols/anthropic-messages"
+import * as Gemini from "../src/protocols/gemini"
+import * as OpenAIChat from "../src/protocols/openai-chat"
+import { applyCachePolicy } from "../src/cache-policy"
+import { it } from "./lib/effect"
+
+const anthropicModel = AnthropicMessages.route
+  .with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
+  .model({ id: "claude-sonnet-4-5" })
+
+const bedrockModel = AmazonBedrock.configure({
+  credentials: { region: "us-east-1", accessKeyId: "fixture", secretAccessKey: "fixture" },
+}).model("anthropic.claude-3-5-sonnet-20241022-v2:0")
+
+const openaiModel = OpenAIChat.route
+  .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
+  .model({ id: "gpt-4o-mini" })
+
+const geminiModel = Gemini.route
+  .with({
+    endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
+    auth: Auth.header("x-goog-api-key", "test"),
+  })
+  .model({ id: "gemini-2.5-flash" })
+
+describe("applyCachePolicy", () => {
+  it.effect("undefined cache resolves to 'auto' (the recommended default)", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: "You are concise.",
+          prompt: "hi",
+        }),
+      )
+
+      // No explicit cache field → auto policy fires → last system part + latest
+      // user message both get cache_control markers.
+      expect(prepared.body).toMatchObject({
+        system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
+        messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
+      })
+    }),
+  )
+
+  it.effect("'auto' marks the last tool, last system part, and latest user message on Anthropic", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: "Sys A",
+          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+          messages: [
+            Message.user("first user"),
+            Message.assistant("assistant reply"),
+            Message.user("latest user message"),
+          ],
+          cache: "auto",
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
+        system: [{ type: "text", text: "Sys A", cache_control: { type: "ephemeral" } }],
+        messages: [
+          { role: "user", content: [{ type: "text", text: "first user" }] },
+          { role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
+          {
+            role: "user",
+            content: [{ type: "text", text: "latest user message", cache_control: { type: "ephemeral" } }],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("'auto' is a no-op on OpenAI (implicit caching protocol)", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: openaiModel,
+          system: "Sys",
+          prompt: "hi",
+          cache: "auto",
+        }),
+      )
+
+      const body = prepared.body as { messages: Array<{ content: unknown }> }
+      // OpenAI doesn't accept cache_control on messages — policy must skip.
+      const flat = JSON.stringify(body)
+      expect(flat).not.toContain("cache_control")
+      expect(flat).not.toContain("cachePoint")
+    }),
+  )
+
+  it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: geminiModel,
+          system: "Sys",
+          prompt: "hi",
+          cache: "auto",
+        }),
+      )
+
+      const flat = JSON.stringify(prepared.body)
+      expect(flat).not.toContain("cache_control")
+      expect(flat).not.toContain("cachePoint")
+    }),
+  )
+
+  it.effect("'auto' on Bedrock emits cachePoint markers in the right places", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: bedrockModel,
+          system: "Sys",
+          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+          messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
+          cache: "auto",
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        toolConfig: {
+          tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
+        },
+        system: [{ text: "Sys" }, { cachePoint: { type: "default" } }],
+        messages: [
+          { role: "user", content: [{ text: "first user" }] },
+          { role: "assistant", content: [{ text: "reply" }] },
+          { role: "user", content: [{ text: "latest user" }, { cachePoint: { type: "default" } }] },
+        ],
+      })
+    }),
+  )
+
+  it.effect("'none' disables auto placement even when manual hints exist", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: "Sys",
+          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+          prompt: "hi",
+          cache: "none",
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        tools: [{ name: "t1", cache_control: undefined }],
+        system: [{ type: "text", text: "Sys", cache_control: undefined }],
+      })
+    }),
+  )
+
+  it.effect("granular object form: tools-only marks just tools", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: "Sys",
+          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+          prompt: "hi",
+          cache: { tools: true },
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
+        system: [{ type: "text", text: "Sys", cache_control: undefined }],
+      })
+    }),
+  )
+
+  it.effect("auto policy preserves manual CacheHints on other parts", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: [
+            { type: "text", text: "first system", cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }) },
+            { type: "text", text: "last system" },
+          ],
+          prompt: "hi",
+          cache: "auto",
+        }),
+      )
+
+      const body = prepared.body as { system: Array<{ text: string; cache_control?: unknown }> }
+      expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
+      expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
+    }),
+  )
+
+  it.effect("ttlSeconds in the policy flows through to wire markers", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          system: "Sys",
+          prompt: "hi",
+          cache: { system: true, ttlSeconds: 3600 },
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        system: [{ type: "text", text: "Sys", cache_control: { type: "ephemeral", ttl: "1h" } }],
+      })
+    }),
+  )
+
+  it.effect("messages: { tail: 2 } marks the last 2 message boundaries", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2"), Message.assistant("a2")],
+          cache: { messages: { tail: 2 } },
+        }),
+      )
+
+      const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
+      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
+      expect(body.messages[1]?.content[0]?.cache_control).toBeUndefined()
+      expect(body.messages[2]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
+      expect(body.messages[3]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
+    }),
+  )
+
+  it.effect("'latest-assistant' marks the last assistant message", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model: anthropicModel,
+          messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2")],
+          cache: { messages: "latest-assistant" },
+        }),
+      )
+
+      const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
+      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
+      expect(body.messages[1]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
+      expect(body.messages[2]?.content[0]?.cache_control).toBeUndefined()
+    }),
+  )
+
+  test("returns the same request reference when policy is a no-op (pure function)", () => {
+    const request = LLM.request({
+      model: anthropicModel,
+      prompt: "hi",
+      cache: "none",
+    })
+    expect(applyCachePolicy(request)).toBe(request)
+  })
+})

package/test/continuation-scenarios.ts

@@ -0,0 +1,104 @@
+import { LLM, Message, ToolCallPart, ToolDefinition, ToolResultPart, type ContentPart, type Model } from "../src"
+
+export const basicContinuation = ["system", "user-text", "assistant-text", "user-follow-up"] as const
+export const toolContinuation = ["tool-call", "tool-result"] as const
+export const reasoningContinuation = ["assistant-reasoning", "encrypted-reasoning"] as const
+export const mediaContinuation = ["user-image"] as const
+export const maximalContinuation = [
+  ...basicContinuation,
+  ...toolContinuation,
+  ...reasoningContinuation,
+  ...mediaContinuation,
+] as const
+
+export type ContinuationFeature = (typeof maximalContinuation)[number]
+
+export const nativeOpenAIResponsesContinuation = [
+  ...basicContinuation,
+  ...toolContinuation,
+  "encrypted-reasoning",
+  ...mediaContinuation,
+] as const satisfies ReadonlyArray<ContinuationFeature>
+
+export const nativeAnthropicMessagesContinuation = [
+  ...basicContinuation,
+  ...toolContinuation,
+  "assistant-reasoning",
+  ...mediaContinuation,
+] as const satisfies ReadonlyArray<ContinuationFeature>
+
+export const continuationTool = ToolDefinition.make({
+  name: "get_weather",
+  description: "Get current weather for a city.",
+  inputSchema: {
+    type: "object",
+    properties: { city: { type: "string" } },
+    required: ["city"],
+    additionalProperties: false,
+  },
+})
+
+export function continuationRequest(input: {
+  readonly id: string
+  readonly model: Model
+  readonly features: ReadonlyArray<ContinuationFeature>
+  readonly image?: string
+}) {
+  const features = new Set(input.features)
+  const messages = []
+  const firstUser: ContentPart[] = []
+  const firstAssistant: ContentPart[] = []
+
+  if (features.has("user-text")) firstUser.push({ type: "text", text: "What is shown here?" })
+  if (features.has("user-image"))
+    firstUser.push({ type: "media", mediaType: "image/png", data: input.image ?? "AAECAw==" })
+  if (firstUser.length > 0) messages.push(Message.user(firstUser))
+
+  if (features.has("assistant-reasoning"))
+    firstAssistant.push({
+      type: "reasoning",
+      text: "I inspected the previous turn.",
+      providerMetadata: { anthropic: { signature: "sig_continuation_1" } },
+    })
+  if (features.has("encrypted-reasoning"))
+    firstAssistant.push({
+      type: "reasoning",
+      text: "I inspected the previous turn.",
+      providerMetadata: {
+        openai: {
+          itemId: "rs_continuation_1",
+          reasoningEncryptedContent: "encrypted-continuation-state",
+        },
+      },
+    })
+  if (features.has("assistant-text")) firstAssistant.push({ type: "text", text: "It shows a small test image." })
+  if (firstAssistant.length > 0) messages.push(Message.assistant(firstAssistant))
+
+  if (features.has("tool-call")) {
+    messages.push(Message.user("Check the weather in Paris before continuing."))
+    messages.push(
+      Message.assistant([ToolCallPart.make({ id: "call_weather_1", name: "get_weather", input: { city: "Paris" } })]),
+    )
+  }
+  if (features.has("tool-result")) {
+    messages.push(
+      Message.tool(ToolResultPart.make({ id: "call_weather_1", name: "get_weather", result: { temperature: 22 } })),
+    )
+    if (features.has("assistant-text")) messages.push(Message.assistant("Paris is 22 degrees."))
+  }
+  if (features.has("user-follow-up"))
+    messages.push(Message.user("Continue from this conversation in one short sentence."))
+
+  return LLM.request({
+    id: input.id,
+    model: input.model,
+    system: features.has("system") ? "You are concise. Continue from the provided history." : undefined,
+    messages,
+    tools: features.has("tool-call") ? [continuationTool] : [],
+    cache: "none",
+    providerOptions: features.has("encrypted-reasoning")
+      ? { openai: { store: false, include: ["reasoning.encrypted_content"], reasoningSummary: "auto" } }
+      : undefined,
+    generation: { maxTokens: 80, temperature: 0 },
+  })
+}

package/test/endpoint.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from "bun:test"
+import { LLM } from "../src"
+import * as OpenAIChat from "../src/protocols/openai-chat"
+import { Endpoint } from "../src/route"
+import { Model } from "../src/schema"
+
+const request = () =>
+  LLM.request({
+    model: Model.make({
+      id: "model-1",
+      provider: "test",
+      route: OpenAIChat.route,
+    }),
+    prompt: "hello",
+  })
+
+describe("Endpoint", () => {
+  test("appends a static path to the model's baseURL", () => {
+    const url = Endpoint.render(Endpoint.path("/chat", { baseURL: "https://api.example.test/v1/" }), {
+      request: request(),
+      body: {},
+    })
+
+    expect(url.toString()).toBe("https://api.example.test/v1/chat")
+  })
+
+  test("endpoint query params are appended to the rendered URL", () => {
+    const url = Endpoint.render(
+      Endpoint.path("/chat?alt=sse", {
+        baseURL: "https://custom.example.test/root/",
+        query: { "api-version": "2026-01-01", alt: "json" },
+      }),
+      {
+        request: request(),
+        body: {},
+      },
+    )
+
+    expect(url.toString()).toBe("https://custom.example.test/root/chat?alt=json&api-version=2026-01-01")
+  })
+
+  test("path may be a function of the validated body", () => {
+    const url = Endpoint.render(
+      Endpoint.path<{ readonly modelId: string }>(
+        ({ body }) => `/model/${encodeURIComponent(body.modelId)}/converse-stream`,
+        { baseURL: "https://bedrock-runtime.us-east-1.amazonaws.com" },
+      ),
+      {
+        request: request(),
+        body: { modelId: "us.amazon.nova-micro-v1:0" },
+      },
+    )
+
+    expect(url.toString()).toBe(
+      "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-micro-v1%3A0/converse-stream",
+    )
+  })
+})