@codilore/llm 1.15.13

package/example/tutorial.ts

@@ -0,0 +1,255 @@
+import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
+import { LLM, LLMClient, Message, ProviderID, Tool, ToolRuntime } from "@codilore/llm"
+import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor, WebSocketExecutor } from "@codilore/llm/route"
+import { OpenAI } from "@codilore/llm/providers"
+
+/**
+ * A runnable walkthrough of the LLM package use-site API.
+ *
+ * Run from `packages/llm` with an OpenAI key in the environment:
+ *
+ *   OPENAI_API_KEY=... bun example/tutorial.ts
+ *
+ * The file is intentionally written as a normal TypeScript program. You can
+ * hover imports and local values to see how the public API is typed.
+ */
+
+const apiKey = Config.redacted("OPENAI_API_KEY")
+
+// 1. Pick a model. The provider helper records provider identity, protocol
+// choice, capabilities, deployment options, authentication, and defaults.
+const model = OpenAI.configure({
+  apiKey,
+  generation: { maxTokens: 160 },
+  providerOptions: {
+    openai: { store: false },
+  },
+}).model("gpt-4o-mini")
+
+// 2. Build a provider-neutral request. This is useful when reusing one request
+// across generate and stream examples.
+//
+// Options can live on both the configured route/provider facade and the request:
+//
+//   - `generation`: common controls such as max tokens, temperature, topP/topK,
+//     penalties, seed, and stop sequences.
+//   - `providerOptions`: namespaced provider-native behavior. For example,
+//     OpenAI cache keys and store behavior, Anthropic thinking, Gemini thinking
+//     config, or OpenRouter routing/reasoning.
+//   - `http`: last-resort serializable overlays for final request body, headers,
+//     and query params. Prefer typed `providerOptions` when a field is stable.
+//
+// Route/provider options are defaults. Request options override them for this call.
+const request = LLM.request({
+  model,
+  system: "You are concise and practical.",
+  prompt: "Tell me a joke",
+  generation: { maxTokens: 80, temperature: 0.7 },
+  providerOptions: {
+    openai: { promptCacheKey: "tutorial-joke" },
+  },
+})
+
+// `http` is intentionally not needed for normal calls. This shows the shape for
+// newly released provider fields before they deserve a typed provider option.
+const rawOverlayExample = LLM.request({
+  model,
+  prompt: "Show the final HTTP overlay shape.",
+  http: {
+    body: { metadata: { example: "tutorial" } },
+    headers: { "x-Codilore-tutorial": "1" },
+    query: { debug: "1" },
+  },
+})
+
+// 3. `generate` sends the request and collects the event stream into one
+// response object. `response.text` is the collected text output.
+const generateOnce = Effect.gen(function* () {
+  const response = yield* LLM.generate(request)
+
+  console.log("\n== generate ==")
+  console.log("generated text:", response.text)
+  console.log("usage", Formatter.formatJson(response.usage, { space: 2 }))
+})
+
+// 4. `stream` exposes provider output as common `LLMEvent`s for UIs that want
+// incremental text, reasoning, tool input, usage, or finish events.
+const streamText = LLM.stream(request).pipe(
+  Stream.tap((event) =>
+    Effect.sync(() => {
+      if (event.type === "text-delta") process.stdout.write(`\ntext: ${event.text}`)
+      if (event.type === "finish") process.stdout.write(`\nfinish: ${event.reason}\n`)
+    }),
+  ),
+  Stream.runDrain,
+)
+
+// 5. Tools are typed with Effect Schema. Provider turns remain explicit:
+// advertise definitions on the request, stream one turn, dispatch local calls,
+// then persist/build follow-up history in the enclosing product flow.
+const tools = {
+  get_weather: Tool.make({
+    description: "Get current weather for a city.",
+    parameters: Schema.Struct({ city: Schema.String }),
+    success: Schema.Struct({ forecast: Schema.String }),
+    execute: (input) => Effect.succeed({ forecast: `${input.city}: sunny, 72F` }),
+  }),
+}
+
+const streamWithTools = Effect.gen(function* () {
+  const request = LLM.request({
+    model,
+    prompt: "Use get_weather for San Francisco, then answer in one sentence.",
+    generation: { maxTokens: 80, temperature: 0 },
+    tools: Tool.toDefinitions(tools),
+  })
+  const events = Array.from(yield* LLM.stream(request).pipe(Stream.runCollect))
+  for (const event of events) {
+    if (event.type === "tool-call") console.log("tool call", event.name, event.input)
+    if (event.type === "text-delta") process.stdout.write(event.text)
+    if (event.type !== "tool-call" || event.providerExecuted) continue
+    const dispatched = yield* ToolRuntime.dispatch(tools, event)
+    console.log("tool result", event.name, dispatched.result)
+
+    // A durable agent would persist these messages before starting another
+    // raw model turn. This tutorial keeps the boundary visible instead.
+    const followUp = LLM.updateRequest(request, {
+      messages: [
+        ...request.messages,
+        Message.assistant([event]),
+        Message.tool({ ...event, result: dispatched.result }),
+      ],
+    })
+    console.log("follow-up history messages:", followUp.messages.length)
+  }
+})
+
+// 6. `generateObject` is the structured-output helper. It forces a synthetic
+// tool call internally, so the same call site works across providers instead of
+// depending on provider-specific JSON mode flags.
+const WeatherReport = Schema.Struct({
+  city: Schema.String,
+  forecast: Schema.String,
+  highFahrenheit: Schema.Number,
+})
+
+const generateStructuredObject = Effect.gen(function* () {
+  const response = yield* LLM.generateObject({
+    model,
+    system: "Return only structured weather data.",
+    prompt: "Give me today's weather for San Francisco.",
+    schema: WeatherReport,
+    generation: { maxTokens: 120, temperature: 0 },
+  })
+
+  console.log("\n== generateObject ==")
+  console.log(Formatter.formatJson(response.object, { space: 2 }))
+})
+
+// If the shape is only known at runtime, pass raw JSON Schema instead. The
+// `.object` type is `unknown`; callers that need static types should validate it.
+const generateDynamicObject = LLM.generateObject({
+  model,
+  prompt: "Extract the city and forecast from: San Francisco is sunny.",
+  jsonSchema: {
+    type: "object",
+    properties: {
+      city: { type: "string" },
+      forecast: { type: "string" },
+    },
+    required: ["city", "forecast"],
+  },
+})
+
+// -----------------------------------------------------------------------------
+// Part 2: provider composition with a fake provider
+// -----------------------------------------------------------------------------
+
+// A protocol is the provider-native API shape: common request -> body, response
+// frames -> common events. This fake one turns text prompts into a JSON body
+// and treats every SSE frame as output text.
+const FakeBody = Schema.Struct({
+  model: Schema.String,
+  input: Schema.String,
+})
+type FakeBody = Schema.Schema.Type<typeof FakeBody>
+
+const FakeProtocol = Protocol.make<FakeBody, string, string, void>({
+  // Protocol ids are open strings, so external packages can define their own
+  // protocols without changing this package.
+  id: "fake-echo",
+  body: {
+    schema: FakeBody,
+    from: (request) =>
+      Effect.succeed({
+        model: request.model.id,
+        input: request.messages
+          .flatMap((message) => message.content)
+          .filter((part) => part.type === "text")
+          .map((part) => part.text)
+          .join("\n"),
+      }),
+  },
+  stream: {
+    event: Schema.String,
+    initial: () => undefined,
+    step: (_, frame) => Effect.succeed([undefined, [{ type: "text-delta", id: "text-0", text: frame }]] as const),
+    onHalt: () => [{ type: "finish", reason: "stop" }],
+  },
+})
+
+// An route is the runnable binding for that protocol. It adds the deployment
+// axes that the protocol deliberately does not know: URL, auth, and framing.
+const FakeAdapter = Route.make({
+  id: "fake-echo",
+  provider: "fake-echo",
+  protocol: FakeProtocol,
+  endpoint: Endpoint.path("/v1/echo", { baseURL: "https://fake.local" }),
+  auth: Auth.passthrough,
+  framing: Framing.sse,
+})
+
+// A provider module exports a configured facade. Configuration happens before
+// model selection; model selectors accept ids only.
+const FakeEcho = {
+  id: ProviderID.make("fake-echo"),
+  configure: () => ({
+    id: ProviderID.make("fake-echo"),
+    model: (id: string) => FakeAdapter.model({ id }),
+  }),
+}
+
+// `LLMClient.prepare` is the lower-level inspection hook: it compiles through
+// body conversion, validation, endpoint, auth, and HTTP construction without
+// sending anything over the network.
+const inspectFakeProvider = Effect.gen(function* () {
+  const prepared = yield* LLMClient.prepare(
+    LLM.request({
+      model: FakeEcho.configure().model("tiny-echo"),
+      prompt: "Show me the provider pipeline.",
+    }),
+  )
+
+  console.log("\n== fake provider prepare ==")
+  console.log("route:", prepared.route)
+  console.log("body:", Formatter.formatJson(prepared.body, { space: 2 }))
+})
+
+// Provide the LLM runtime and the HTTP request executor once. Keep one path
+// enabled at a time so the tutorial can demonstrate generate, prepare, stream,
+// or tool-loop behavior without spending tokens on every example.
+const requestExecutorLayer = RequestExecutor.defaultLayer
+const llmDeps = Layer.mergeAll(requestExecutorLayer, WebSocketExecutor.layer)
+const llmClientLayer = LLMClient.layer.pipe(Layer.provide(llmDeps))
+
+const program = Effect.gen(function* () {
+  // yield* generateOnce
+  // yield* inspectFakeProvider
+  // yield* LLMClient.prepare(rawOverlayExample).pipe(Effect.andThen((prepared) => Effect.sync(() => console.log(prepared.body))))
+  // yield* streamText
+  // yield* generateStructuredObject
+  // yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
+  yield* streamWithTools
+}).pipe(Effect.provide(Layer.mergeAll(llmDeps, llmClientLayer)))
+
+Effect.runPromise(program)

package/package.json

@@ -0,0 +1,50 @@
+{
+  "$schema": "https://json.schemastore.org/package.json",
+  "version": "1.15.13",
+  "name": "@codilore/llm",
+  "type": "module",
+  "license": "MIT",
+  "scripts": {
+    "setup:recording-env": "bun run script/setup-recording-env.ts",
+    "test": "bun test --timeout 30000",
+    "typecheck": "tsgo --noEmit"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./route": "./src/route/index.ts",
+    "./provider": "./src/provider.ts",
+    "./providers": "./src/providers/index.ts",
+    "./providers/amazon-bedrock": "./src/providers/amazon-bedrock.ts",
+    "./providers/anthropic": "./src/providers/anthropic.ts",
+    "./providers/azure": "./src/providers/azure.ts",
+    "./providers/cloudflare": "./src/providers/cloudflare.ts",
+    "./providers/github-copilot": "./src/providers/github-copilot.ts",
+    "./providers/google": "./src/providers/google.ts",
+    "./providers/openai": "./src/providers/openai.ts",
+    "./providers/openai-compatible": "./src/providers/openai-compatible.ts",
+    "./providers/openai-compatible-profile": "./src/providers/openai-compatible-profile.ts",
+    "./providers/openrouter": "./src/providers/openrouter.ts",
+    "./providers/xai": "./src/providers/xai.ts",
+    "./protocols": "./src/protocols/index.ts",
+    "./protocols/anthropic-messages": "./src/protocols/anthropic-messages.ts",
+    "./protocols/bedrock-converse": "./src/protocols/bedrock-converse.ts",
+    "./protocols/gemini": "./src/protocols/gemini.ts",
+    "./protocols/openai-chat": "./src/protocols/openai-chat.ts",
+    "./protocols/openai-compatible-chat": "./src/protocols/openai-compatible-chat.ts",
+    "./protocols/openai-responses": "./src/protocols/openai-responses.ts"
+  },
+  "devDependencies": {
+    "@clack/prompts": "1.0.0-alpha.1",
+    "@effect/platform-node": "^1.15.13",
+    "@codilore/http-recorder": "1.15.13",
+    "@tsconfig/bun": "^1.15.13",
+    "@types/bun": "^1.15.13",
+    "@typescript/native-preview": "^1.15.13"
+  },
+  "dependencies": {
+    "@smithy/eventstream-codec": "4.2.14",
+    "@smithy/util-utf8": "4.2.2",
+    "aws4fetch": "1.0.20",
+    "effect": "^1.15.13"
+  }
+}

package/script/recording-cost-report.ts

@@ -0,0 +1,250 @@
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+
+const RECORDINGS_DIR = path.resolve(import.meta.dir, "..", "test", "fixtures", "recordings")
+const MODELS_DEV_URL = "https://models.dev/api.json"
+
+type JsonRecord = Record<string, unknown>
+
+type Pricing = {
+  readonly input?: number
+  readonly output?: number
+  readonly cache_read?: number
+  readonly cache_write?: number
+  readonly reasoning?: number
+}
+
+type Usage = {
+  readonly inputTokens: number
+  readonly outputTokens: number
+  readonly cacheReadTokens: number
+  readonly cacheWriteTokens: number
+  readonly reasoningTokens: number
+  readonly reportedCost: number
+}
+
+type Row = Usage & {
+  readonly cassette: string
+  readonly provider: string
+  readonly model: string
+  readonly estimatedCost: number
+  readonly pricingSource: string
+}
+
+const isRecord = (value: unknown): value is JsonRecord =>
+  value !== null && typeof value === "object" && !Array.isArray(value)
+
+const asNumber = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : 0)
+
+const asString = (value: unknown) => (typeof value === "string" ? value : undefined)
+
+const readJson = async (file: string) => JSON.parse(await Bun.file(file).text()) as unknown
+
+const walk = async (dir: string): Promise<ReadonlyArray<string>> =>
+  (await fs.readdir(dir, { withFileTypes: true }))
+    .flatMap((entry) => {
+      const file = path.join(dir, entry.name)
+      return entry.isDirectory() ? [] : [file]
+    })
+    .concat(
+      ...(await Promise.all(
+        (await fs.readdir(dir, { withFileTypes: true }))
+          .filter((entry) => entry.isDirectory())
+          .map((entry) => walk(path.join(dir, entry.name))),
+      )),
+    )
+
+const providerFromUrl = (url: string) => {
+  if (url.includes("api.openai.com")) return "openai"
+  if (url.includes("api.anthropic.com")) return "anthropic"
+  if (url.includes("generativelanguage.googleapis.com")) return "google"
+  if (url.includes("bedrock")) return "amazon-bedrock"
+  if (url.includes("openrouter.ai")) return "openrouter"
+  if (url.includes("api.x.ai")) return "xai"
+  if (url.includes("api.groq.com")) return "groq"
+  if (url.includes("api.deepseek.com")) return "deepseek"
+  if (url.includes("api.together.xyz")) return "togetherai"
+  return "unknown"
+}
+
+const providerAliases: Record<string, ReadonlyArray<string>> = {
+  openai: ["openai"],
+  anthropic: ["anthropic"],
+  google: ["google"],
+  "amazon-bedrock": ["amazon-bedrock"],
+  openrouter: ["openrouter", "openai", "anthropic", "google"],
+  xai: ["xai"],
+  groq: ["groq"],
+  deepseek: ["deepseek"],
+  togetherai: ["togetherai"],
+}
+
+const modelAliases = (model: string) => [
+  model,
+  model.replace(/^models\//, ""),
+  model.replace(/-\d{8}$/, ""),
+  model.replace(/-\d{4}-\d{2}-\d{2}$/, ""),
+  model.replace(/-\d{4}-\d{2}-\d{2}$/, "").replace(/-\d{8}$/, ""),
+  model.replace(/^openai\//, ""),
+  model.replace(/^anthropic\//, ""),
+  model.replace(/^google\//, ""),
+]
+
+const pricingFor = (models: JsonRecord, provider: string, model: string) => {
+  for (const providerID of providerAliases[provider] ?? [provider]) {
+    const providerEntry = models[providerID]
+    if (!isRecord(providerEntry) || !isRecord(providerEntry.models)) continue
+    for (const modelID of modelAliases(model)) {
+      const modelEntry = providerEntry.models[modelID]
+      if (isRecord(modelEntry) && isRecord(modelEntry.cost))
+        return { pricing: modelEntry.cost as Pricing, source: `${providerID}/${modelID}` }
+    }
+  }
+  return { pricing: undefined, source: "missing" }
+}
+
+const estimateCost = (usage: Usage, pricing: Pricing | undefined) => {
+  if (!pricing) return 0
+  return (
+    (usage.inputTokens * (pricing.input ?? 0) +
+      usage.outputTokens * (pricing.output ?? 0) +
+      usage.cacheReadTokens * (pricing.cache_read ?? 0) +
+      usage.cacheWriteTokens * (pricing.cache_write ?? 0) +
+      usage.reasoningTokens * (pricing.reasoning ?? 0)) /
+    1_000_000
+  )
+}
+
+const emptyUsage = (): Usage => ({
+  inputTokens: 0,
+  outputTokens: 0,
+  cacheReadTokens: 0,
+  cacheWriteTokens: 0,
+  reasoningTokens: 0,
+  reportedCost: 0,
+})
+
+const addUsage = (a: Usage, b: Usage): Usage => ({
+  inputTokens: a.inputTokens + b.inputTokens,
+  outputTokens: a.outputTokens + b.outputTokens,
+  cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
+  cacheWriteTokens: a.cacheWriteTokens + b.cacheWriteTokens,
+  reasoningTokens: a.reasoningTokens + b.reasoningTokens,
+  reportedCost: a.reportedCost + b.reportedCost,
+})
+
+const usageFromObject = (usage: unknown): Usage => {
+  if (!isRecord(usage)) return emptyUsage()
+  const promptDetails = isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : {}
+  const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}
+  const inputDetails = isRecord(usage.input_tokens_details) ? usage.input_tokens_details : {}
+  const outputDetails = isRecord(usage.output_tokens_details) ? usage.output_tokens_details : {}
+  const cacheWriteTokens = asNumber(promptDetails.cache_write_tokens) + asNumber(inputDetails.cache_write_tokens)
+  return {
+    inputTokens: asNumber(usage.prompt_tokens) + asNumber(usage.input_tokens),
+    outputTokens: asNumber(usage.completion_tokens) + asNumber(usage.output_tokens),
+    cacheReadTokens: asNumber(promptDetails.cached_tokens) + asNumber(inputDetails.cached_tokens),
+    cacheWriteTokens,
+    reasoningTokens: asNumber(completionDetails.reasoning_tokens) + asNumber(outputDetails.reasoning_tokens),
+    reportedCost: asNumber(usage.cost),
+  }
+}
+
+const jsonPayloads = (body: string) =>
+  body
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith("data:"))
+    .map((line) => line.slice("data:".length).trim())
+    .filter((line) => line !== "" && line !== "[DONE]")
+    .flatMap((line) => {
+      try {
+        return [JSON.parse(line) as unknown]
+      } catch {
+        return []
+      }
+    })
+
+const usageFromResponseBody = (body: string) =>
+  jsonPayloads(body).reduce<Usage>((usage, payload) => {
+    if (!isRecord(payload)) return usage
+    return addUsage(
+      usage,
+      addUsage(
+        usageFromObject(payload.usage),
+        usageFromObject(isRecord(payload.response) ? payload.response.usage : undefined),
+      ),
+    )
+  }, emptyUsage())
+
+const modelFromRequest = (request: unknown) => {
+  if (!isRecord(request)) return "unknown"
+  const requestBody = asString(request.body)
+  if (!requestBody) return "unknown"
+  try {
+    const body = JSON.parse(requestBody) as unknown
+    if (!isRecord(body)) return "unknown"
+    return asString(body.model) ?? "unknown"
+  } catch {
+    return "unknown"
+  }
+}
+
+const rowFor = (models: JsonRecord, file: string, cassette: unknown): Row | undefined => {
+  if (!isRecord(cassette) || !Array.isArray(cassette.interactions)) return undefined
+  const first = cassette.interactions.find(isRecord)
+  if (!first || !isRecord(first.request)) return undefined
+  const provider = providerFromUrl(asString(first.request.url) ?? "")
+  const model = modelFromRequest(first.request)
+  const usage = cassette.interactions.filter(isRecord).reduce<Usage>((total, interaction) => {
+    if (!isRecord(interaction.response)) return total
+    const responseBody = asString(interaction.response.body)
+    if (!responseBody) return total
+    return addUsage(total, usageFromResponseBody(responseBody))
+  }, emptyUsage())
+  const priced = pricingFor(models, provider, model)
+  return {
+    cassette: path.relative(RECORDINGS_DIR, file),
+    provider,
+    model,
+    ...usage,
+    estimatedCost: estimateCost(usage, priced.pricing),
+    pricingSource: priced.source,
+  }
+}
+
+const money = (value: number) => (value === 0 ? "$0.000000" : `$${value.toFixed(6)}`)
+const tokens = (value: number) => value.toLocaleString("en-US")
+
+const models = (await (await fetch(MODELS_DEV_URL)).json()) as JsonRecord
+const rows = (
+  await Promise.all(
+    (await walk(RECORDINGS_DIR))
+      .filter((file) => file.endsWith(".json"))
+      .map(async (file) => rowFor(models, file, await readJson(file))),
+  )
+).filter((row): row is Row => row !== undefined)
+
+const totals = rows.reduce(
+  (total, row) => ({
+    ...addUsage(total, row),
+    estimatedCost: total.estimatedCost + row.estimatedCost,
+  }),
+  { ...emptyUsage(), estimatedCost: 0 },
+)
+
+console.log("# Recording Cost Report")
+console.log("")
+console.log(`Pricing: ${MODELS_DEV_URL}`)
+console.log(`Cassettes: ${rows.length}`)
+console.log(`Reported cost: ${money(totals.reportedCost)}`)
+console.log(`Estimated cost: ${money(totals.estimatedCost)}`)
+console.log("")
+console.log("| Provider | Model | Input | Output | Reasoning | Reported | Estimated | Pricing | Cassette |")
+console.log("|---|---:|---:|---:|---:|---:|---:|---|---|")
+for (const row of rows.toSorted((a, b) => b.reportedCost + b.estimatedCost - (a.reportedCost + a.estimatedCost))) {
+  if (row.inputTokens + row.outputTokens + row.reasoningTokens + row.reportedCost + row.estimatedCost === 0) continue
+  console.log(
+    `| ${row.provider} | ${row.model} | ${tokens(row.inputTokens)} | ${tokens(row.outputTokens)} | ${tokens(row.reasoningTokens)} | ${money(row.reportedCost)} | ${money(row.estimatedCost)} | ${row.pricingSource} | ${row.cassette} |`,
+  )
+}