@effect-uai/anthropic 0.11.0 → 0.12.1

package/dist/usage.test.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/usage.test.mjs

@@ -0,0 +1,46 @@
+import { emptyAccumulator, mergeUsage } from "./codec.mjs";
+import { applyEvent } from "./streamEvents.mjs";
+import { n as describe, r as it, t as globalExpect } from "./test.DNmyFkvJ-N3QqMRwZ.mjs";
+//#region src/usage.test.ts
+const wire = {
+	input_tokens: 94,
+	output_tokens: 210,
+	cache_read_input_tokens: 31851,
+	cache_creation_input_tokens: 604
+};
+describe("mergeUsage", () => {
+	it("keeps all three input buckets, not just cache reads", () => {
+		const usage = mergeUsage(emptyAccumulator, wire).usage;
+		globalExpect(usage.input_tokens).toBe(94);
+		globalExpect(usage.input_tokens_details?.cached_tokens).toBe(31851);
+		globalExpect(usage.input_tokens_details?.cache_write_tokens).toBe(604);
+	});
+	it("totals every input bucket plus output", () => {
+		const usage = mergeUsage(emptyAccumulator, wire).usage;
+		globalExpect(usage.total_tokens).toBe(32759);
+	});
+});
+describe("streamed usage", () => {
+	it("computes a non-stale total across message_start and message_delta", () => {
+		const usage = applyEvent(applyEvent(emptyAccumulator, {
+			type: "message_start",
+			message: { usage: {
+				input_tokens: 94,
+				output_tokens: 1,
+				cache_read_input_tokens: 31851,
+				cache_creation_input_tokens: 604
+			} }
+		}), {
+			type: "message_delta",
+			delta: { stop_reason: "end_turn" },
+			usage: { output_tokens: 210 }
+		}).usage;
+		globalExpect(usage.output_tokens).toBe(210);
+		globalExpect(usage.input_tokens_details?.cache_write_tokens).toBe(604);
+		globalExpect(usage.total_tokens).toBe(32759);
+	});
+});
+//#endregion
+export {};
+
+//# sourceMappingURL=usage.test.mjs.map

package/dist/usage.test.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.test.mjs","names":[],"sources":["../src/usage.test.ts"],"sourcesContent":["import { describe, expect, it } from \"vitest\"\nimport { type WireUsage, emptyAccumulator, mergeUsage } from \"./codec.js\"\nimport { type ProviderEvent, applyEvent } from \"./streamEvents.js\"\n\n// A completed cached turn: input split across post-breakpoint tokens, cache\n// reads, and cache writes, all billed separately.\nconst wire: WireUsage = {\n  input_tokens: 94,\n  output_tokens: 210,\n  cache_read_input_tokens: 31851,\n  cache_creation_input_tokens: 604,\n}\n\ndescribe(\"mergeUsage\", () => {\n  it(\"keeps all three input buckets, not just cache reads\", () => {\n    const usage = mergeUsage(emptyAccumulator, wire).usage\n    expect(usage.input_tokens).toBe(94)\n    expect(usage.input_tokens_details?.cached_tokens).toBe(31851)\n    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)\n  })\n\n  it(\"totals every input bucket plus output\", () => {\n    const usage = mergeUsage(emptyAccumulator, wire).usage\n    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)\n  })\n})\n\ndescribe(\"streamed usage\", () => {\n  // Anthropic sends input + cache on message_start and the final output_tokens\n  // on message_delta. total_tokens must reflect the accumulated buckets, not\n  // the (input + initial output) figure frozen at message_start.\n  it(\"computes a non-stale total across message_start and message_delta\", () => {\n    const start: ProviderEvent = {\n      type: \"message_start\",\n      message: {\n        usage: {\n          input_tokens: 94,\n          output_tokens: 1,\n          cache_read_input_tokens: 31851,\n          cache_creation_input_tokens: 604,\n        },\n      },\n    }\n    const delta: ProviderEvent = {\n      type: \"message_delta\",\n      delta: { stop_reason: \"end_turn\" },\n      usage: { output_tokens: 210 },\n    }\n    const usage = applyEvent(applyEvent(emptyAccumulator, start), delta).usage\n    expect(usage.output_tokens).toBe(210)\n    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)\n    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)\n  })\n})\n"],"mappings":";;;;AAMA,MAAM,OAAkB;CACtB,cAAc;CACd,eAAe;CACf,yBAAyB;CACzB,6BAA6B;AAC/B;AAEA,SAAS,oBAAoB;CAC3B,GAAG,6DAA6D;EAC9D,MAAM,QAAQ,WAAW,kBAAkB,IAAI,CAAC,CAAC;EACjD,aAAO,MAAM,YAAY,CAAC,CAAC,KAAK,EAAE;EAClC,aAAO,MAAM,sBAAsB,aAAa,CAAC,CAAC,KAAK,KAAK;EAC5D,aAAO,MAAM,sBAAsB,kBAAkB,CAAC,CAAC,KAAK,GAAG;CACjE,CAAC;CAED,GAAG,+CAA+C;EAChD,MAAM,QAAQ,WAAW,kBAAkB,IAAI,CAAC,CAAC;EACjD,aAAO,MAAM,YAAY,CAAC,CAAC,KAAK,KAAsB;CACxD,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB;CAI/B,GAAG,2EAA2E;EAiB5E,MAAM,QAAQ,WAAW,WAAW,kBAAkB;GAfpD,MAAM;GACN,SAAS,EACP,OAAO;IACL,cAAc;IACd,eAAe;IACf,yBAAyB;IACzB,6BAA6B;GAC/B,EACF;EAOwD,CAAC,GAAG;GAJ5D,MAAM;GACN,OAAO,EAAE,aAAa,WAAW;GACjC,OAAO,EAAE,eAAe,IAAI;EAEoC,CAAC,CAAC,CAAC;EACrE,aAAO,MAAM,aAAa,CAAC,CAAC,KAAK,GAAG;EACpC,aAAO,MAAM,sBAAsB,kBAAkB,CAAC,CAAC,KAAK,GAAG;EAC/D,aAAO,MAAM,YAAY,CAAC,CAAC,KAAK,KAAsB;CACxD,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/anthropic",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "Anthropic Messages API provider for @effect-uai/core.",
   "keywords": [
     "ai",
@@ -43,13 +43,13 @@
     "access": "public"
   },
   "devDependencies": {
-    "effect": "4.0.0-beta.94",
-    "typescript": "^6.0.3",
-    "@effect-uai/core": "0.11.0"
+    "effect": "4.0.0-rc.111",
+    "typescript": "^7.0.2",
+    "@effect-uai/core": "0.12.1"
   },
   "peerDependencies": {
     "@effect-uai/core": ">=0.2.0 <1",
-    "effect": ">=4.0.0-beta.94 <5.0.0"
+    "effect": ">=4.0.0-rc.111 <5.0.0"
   },
   "scripts": {
     "build": "tsdown",

package/src/Anthropic.ts

@@ -24,6 +24,8 @@ import { type Turn, TurnEvent } from "@effect-uai/core/Turn"
 import { renderProviderTools } from "./AnthropicTools.js"
 import {
   type Accumulator,
+  type CacheControl,
+  type CacheTtl,
   type ThinkingConfig,
   accumulatorToTurn,
   buildRequestBody,
@@ -119,6 +121,21 @@ export type Config = {
    * 4096 if neither is set.
    */
   readonly defaultMaxTokens?: number
+  /**
+   * Prompt caching. Off by default, so the library stays unopinionated about
+   * a setting that changes billing. `true` is `{}`: cache with the 5-minute
+   * default TTL.
+   *
+   * Anthropic caches the `tools` → `system` → `messages` prefix and moves the
+   * breakpoint forward as the conversation grows, so a long system prompt and
+   * tool definitions are paid for once rather than on every turn. Read
+   * `usage.input_tokens_details.cached_tokens` to see it working.
+   *
+   * Mutating earlier history (rewriting an old message, compaction) breaks
+   * the prefix from that point on. Anthropic falls back to the longest
+   * unchanged prefix, so this costs savings but never correctness.
+   */
+  readonly promptCaching?: boolean | { readonly ttl?: CacheTtl }
 }
 
 const ANTHROPIC_VERSION = "2023-06-01"
@@ -139,6 +156,24 @@ const outputConfig = (request: AnthropicRequest): Option.Option<Record<string, u
 const resolvedMaxTokens = (cfg: Config, request: AnthropicRequest): number =>
   request.maxOutputTokens ?? cfg.defaultMaxTokens ?? FALLBACK_MAX_TOKENS
 
+/**
+ * One request-level breakpoint when caching is on. Anthropic places it on the
+ * last cacheable block itself, so nothing has to be marked per-block and the
+ * breakpoint stays correct as history grows.
+ */
+const cacheControlOf = (cfg: Config): Option.Option<CacheControl> =>
+  pipe(
+    Option.fromUndefinedOr(cfg.promptCaching),
+    Option.flatMap((opt) =>
+      opt === false
+        ? Option.none<CacheControl>()
+        : Option.some<CacheControl>({
+            type: "ephemeral",
+            ...(opt !== true && opt.ttl !== undefined && { ttl: opt.ttl }),
+          }),
+    ),
+  )
+
 const functionToolDescriptors = (
   request: AnthropicRequest,
 ): ReadonlyArray<Record<string, unknown>> =>
@@ -302,6 +337,7 @@ const buildNativeStream = (cfg: Config) => {
           toolChoice: toolChoiceWire(request),
           userId: Option.fromUndefinedOr(request.user),
           outputConfig: structured,
+          cacheControl: cacheControlOf(cfg),
         })
 
         const body = yield* Result.match(bodyResult, {
@@ -409,13 +445,10 @@ export const layer = (
   const typed = Layer.effect(Anthropic, make(cfg))
   const generic = Layer.effect(
     LanguageModel,
-    Effect.map(
-      make(cfg),
-      (s): LanguageModelService => ({
-        streamTurn: (request) => s.streamTurn(request as AnthropicRequest),
-        turn: (request) => s.turn(request as AnthropicRequest),
-      }),
-    ),
+    Effect.map(make(cfg), (s): LanguageModelService => ({
+      streamTurn: (request) => s.streamTurn(request as AnthropicRequest),
+      turn: (request) => s.turn(request as AnthropicRequest),
+    })),
   )
   return Layer.merge(typed, generic)
 }

package/src/AnthropicTools.ts

@@ -109,12 +109,10 @@ const renderOne = (
       })
     : decodeConfig(tool.config).pipe(
         Result.map(configToWire),
-        Result.mapError(
-          (): ToolRenderError => ({
-            capability: "providerTool",
-            reason: `unrecognized Anthropic provider tool config on "${tool.name}"`,
-          }),
-        ),
+        Result.mapError((): ToolRenderError => ({
+          capability: "providerTool",
+          reason: `unrecognized Anthropic provider tool config on "${tool.name}"`,
+        })),
       )
 
 /**

package/src/anthropicTools.test.ts

@@ -42,6 +42,7 @@ const buildWith = (
       toolChoice,
       userId: Option.none(),
       outputConfig: Option.none(),
+      cacheControl: Option.none(),
     }),
   )
 }

package/src/codec.ts

@@ -195,11 +195,9 @@ const itemToAssistantBlocks = (
       function_call: (f) =>
         pipe(
           parseJson(f.arguments),
-          Result.map(
-            (input): ReadonlyArray<RequestAssistantContentBlock> => [
-              { type: "tool_use", id: f.call_id, name: f.name, input },
-            ],
-          ),
+          Result.map((input): ReadonlyArray<RequestAssistantContentBlock> => [
+            { type: "tool_use", id: f.call_id, name: f.name, input },
+          ]),
         ),
       function_call_output: (): Result.Result<
         ReadonlyArray<RequestAssistantContentBlock>,
@@ -324,6 +322,20 @@ export type ThinkingConfig = {
   readonly budget_tokens: number
 }
 
+/** Cache lifetime. `1h` writes cost 2x base input; `5m` costs 1.25x. */
+export type CacheTtl = "5m" | "1h"
+
+/**
+ * Request-level cache breakpoint. Anthropic applies it to the last cacheable
+ * block and moves it forward as the conversation grows, so the cached prefix
+ * (`tools` → `system` → `messages`) follows the conversation without the
+ * caller re-placing anything.
+ */
+export type CacheControl = {
+  readonly type: "ephemeral"
+  readonly ttl?: CacheTtl
+}
+
 export type RequestBody = {
   readonly model: string
   readonly messages: ReadonlyArray<RequestMessage>
@@ -338,6 +350,7 @@ export type RequestBody = {
   readonly tool_choice?: Record<string, unknown>
   readonly metadata?: { readonly user_id: string }
   readonly output_config?: Record<string, unknown>
+  readonly cache_control?: CacheControl
   readonly stream: true
 }
 
@@ -354,57 +367,60 @@ export const buildRequestBody = (params: {
   readonly toolChoice: Option.Option<Record<string, unknown>>
   readonly userId: Option.Option<string>
   readonly outputConfig: Option.Option<Record<string, unknown>>
+  readonly cacheControl: Option.Option<CacheControl>
 }): Result.Result<RequestBody, JsonParseError> =>
   pipe(
     groupedMessages(params.history),
-    Result.map(
-      (messages): RequestBody => ({
-        model: params.model,
-        messages,
-        max_tokens: params.maxTokens,
-        ...Option.match(systemFromHistory(params.history), {
-          onNone: () => ({}),
-          onSome: (system) => ({ system }),
-        }),
-        ...Option.match(params.temperature, {
-          onNone: () => ({}),
-          onSome: (temperature) => ({ temperature }),
-        }),
-        ...Option.match(params.topP, {
-          onNone: () => ({}),
-          onSome: (top_p) => ({ top_p }),
-        }),
-        ...Option.match(params.topK, {
-          onNone: () => ({}),
-          onSome: (top_k) => ({ top_k }),
-        }),
-        ...Option.match(params.stopSequences, {
-          onNone: () => ({}),
-          onSome: (stop_sequences) => ({ stop_sequences }),
-        }),
-        ...Option.match(params.thinking, {
-          onNone: () => ({}),
-          onSome: (thinking) => ({ thinking }),
-        }),
-        ...Option.match(params.tools, {
-          onNone: () => ({}),
-          onSome: (tools) => ({ tools }),
-        }),
-        ...Option.match(params.toolChoice, {
-          onNone: () => ({}),
-          onSome: (tool_choice) => ({ tool_choice }),
-        }),
-        ...Option.match(params.userId, {
-          onNone: () => ({}),
-          onSome: (user_id) => ({ metadata: { user_id } }),
-        }),
-        ...Option.match(params.outputConfig, {
-          onNone: () => ({}),
-          onSome: (output_config) => ({ output_config }),
-        }),
-        stream: true,
+    Result.map((messages): RequestBody => ({
+      model: params.model,
+      messages,
+      max_tokens: params.maxTokens,
+      ...Option.match(systemFromHistory(params.history), {
+        onNone: () => ({}),
+        onSome: (system) => ({ system }),
       }),
-    ),
+      ...Option.match(params.temperature, {
+        onNone: () => ({}),
+        onSome: (temperature) => ({ temperature }),
+      }),
+      ...Option.match(params.topP, {
+        onNone: () => ({}),
+        onSome: (top_p) => ({ top_p }),
+      }),
+      ...Option.match(params.topK, {
+        onNone: () => ({}),
+        onSome: (top_k) => ({ top_k }),
+      }),
+      ...Option.match(params.stopSequences, {
+        onNone: () => ({}),
+        onSome: (stop_sequences) => ({ stop_sequences }),
+      }),
+      ...Option.match(params.thinking, {
+        onNone: () => ({}),
+        onSome: (thinking) => ({ thinking }),
+      }),
+      ...Option.match(params.tools, {
+        onNone: () => ({}),
+        onSome: (tools) => ({ tools }),
+      }),
+      ...Option.match(params.toolChoice, {
+        onNone: () => ({}),
+        onSome: (tool_choice) => ({ tool_choice }),
+      }),
+      ...Option.match(params.userId, {
+        onNone: () => ({}),
+        onSome: (user_id) => ({ metadata: { user_id } }),
+      }),
+      ...Option.match(params.outputConfig, {
+        onNone: () => ({}),
+        onSome: (output_config) => ({ output_config }),
+      }),
+      ...Option.match(params.cacheControl, {
+        onNone: () => ({}),
+        onSome: (cache_control) => ({ cache_control }),
+      }),
+      stream: true,
+    })),
   )
 
 // ---------------------------------------------------------------------------
@@ -503,23 +519,34 @@ export const setStopReason = (acc: Accumulator, reason: string): Accumulator =>
   stopReason: Option.some(reason),
 })
 
-const cachedFromWire = (wire: WireUsage): Option.Option<number> =>
-  Option.fromNullishOr(wire.cache_read_input_tokens)
-
+// Anthropic splits input across three billed buckets: `input_tokens`
+// (post-breakpoint), cache reads, and cache writes. All three are kept;
+// `input_tokens` is not the input total. Each field carries over from the
+// prior value since a streamed turn spreads usage across events (input +
+// cache on message_start, output_tokens on message_delta), so total_tokens is
+// summed from the accumulated buckets, not one event.
 export const mergeUsage = (acc: Accumulator, wire: WireUsage): Accumulator => {
-  const cached = cachedFromWire(wire)
+  const prev = acc.usage
+  const input_tokens = wire.input_tokens ?? prev.input_tokens
+  const output_tokens = wire.output_tokens ?? prev.output_tokens
+  const cached_tokens = wire.cache_read_input_tokens ?? prev.input_tokens_details?.cached_tokens
+  const cache_write_tokens =
+    wire.cache_creation_input_tokens ?? prev.input_tokens_details?.cache_write_tokens
+
+  const buckets = [input_tokens, cached_tokens, cache_write_tokens, output_tokens]
+  const total_tokens = buckets.every((v) => v === undefined)
+    ? undefined
+    : buckets.reduce<number>((sum, v) => sum + (v ?? 0), 0)
+
+  const details: Items.InputTokensDetails = {
+    ...(cached_tokens !== undefined && { cached_tokens }),
+    ...(cache_write_tokens !== undefined && { cache_write_tokens }),
+  }
   const usage: Items.Usage = {
-    ...acc.usage,
-    ...(wire.input_tokens !== undefined && { input_tokens: wire.input_tokens }),
-    ...(wire.output_tokens !== undefined && { output_tokens: wire.output_tokens }),
-    ...(wire.input_tokens !== undefined &&
-      wire.output_tokens !== undefined && {
-        total_tokens: wire.input_tokens + wire.output_tokens,
-      }),
-    ...Option.match(cached, {
-      onNone: () => ({}),
-      onSome: (cached_tokens) => ({ input_tokens_details: { cached_tokens } }),
-    }),
+    ...(input_tokens !== undefined && { input_tokens }),
+    ...(output_tokens !== undefined && { output_tokens }),
+    ...(total_tokens !== undefined && { total_tokens }),
+    ...(Object.keys(details).length > 0 && { input_tokens_details: details }),
   }
   return { ...acc, usage }
 }
@@ -545,52 +572,41 @@ const blocksByIndex = (acc: Accumulator): ReadonlyArray<BlockBuffer> =>
 
 const blockToItems = (block: BlockBuffer): ReadonlyArray<Items.HistoryItem> =>
   Match.value(block.type).pipe(
-    Match.when(
-      "text",
-      (): ReadonlyArray<Items.HistoryItem> =>
-        block.text.length === 0
-          ? []
-          : [
-              {
-                type: "message",
-                role: "assistant",
-                content: [{ type: "output_text", text: block.text }],
-              },
-            ],
-    ),
-    Match.when(
-      "tool_use",
-      (): ReadonlyArray<Items.HistoryItem> => [
-        {
-          type: "function_call",
-          call_id: Option.getOrElse(block.id, () => ""),
-          name: Option.getOrElse(block.name, () => ""),
-          arguments: block.inputJson,
-        },
-      ],
-    ),
-    Match.when(
-      "thinking",
-      (): ReadonlyArray<Items.HistoryItem> => [
-        {
-          type: "reasoning",
-          ...(block.thinking.length > 0 && { summary: block.thinking }),
-          ...(block.signature.length > 0 && { signature: block.signature }),
-        },
-      ],
-    ),
-    Match.when(
-      "redacted_thinking",
-      (): ReadonlyArray<Items.HistoryItem> => [
-        {
-          type: "reasoning",
-          ...Option.match(block.redactedData, {
-            onNone: () => ({}),
-            onSome: (signature) => ({ signature }),
-          }),
-        },
-      ],
+    Match.when("text", (): ReadonlyArray<Items.HistoryItem> =>
+      block.text.length === 0
+        ? []
+        : [
+            {
+              type: "message",
+              role: "assistant",
+              content: [{ type: "output_text", text: block.text }],
+            },
+          ],
     ),
+    Match.when("tool_use", (): ReadonlyArray<Items.HistoryItem> => [
+      {
+        type: "function_call",
+        call_id: Option.getOrElse(block.id, () => ""),
+        name: Option.getOrElse(block.name, () => ""),
+        arguments: block.inputJson,
+      },
+    ]),
+    Match.when("thinking", (): ReadonlyArray<Items.HistoryItem> => [
+      {
+        type: "reasoning",
+        ...(block.thinking.length > 0 && { summary: block.thinking }),
+        ...(block.signature.length > 0 && { signature: block.signature }),
+      },
+    ]),
+    Match.when("redacted_thinking", (): ReadonlyArray<Items.HistoryItem> => [
+      {
+        type: "reasoning",
+        ...Option.match(block.redactedData, {
+          onNone: () => ({}),
+          onSome: (signature) => ({ signature }),
+        }),
+      },
+    ]),
     Match.exhaustive,
   )
 

package/src/usage.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest"
+import { type WireUsage, emptyAccumulator, mergeUsage } from "./codec.js"
+import { type ProviderEvent, applyEvent } from "./streamEvents.js"
+
+// A completed cached turn: input split across post-breakpoint tokens, cache
+// reads, and cache writes, all billed separately.
+const wire: WireUsage = {
+  input_tokens: 94,
+  output_tokens: 210,
+  cache_read_input_tokens: 31851,
+  cache_creation_input_tokens: 604,
+}
+
+describe("mergeUsage", () => {
+  it("keeps all three input buckets, not just cache reads", () => {
+    const usage = mergeUsage(emptyAccumulator, wire).usage
+    expect(usage.input_tokens).toBe(94)
+    expect(usage.input_tokens_details?.cached_tokens).toBe(31851)
+    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)
+  })
+
+  it("totals every input bucket plus output", () => {
+    const usage = mergeUsage(emptyAccumulator, wire).usage
+    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)
+  })
+})
+
+describe("streamed usage", () => {
+  // Anthropic sends input + cache on message_start and the final output_tokens
+  // on message_delta. total_tokens must reflect the accumulated buckets, not
+  // the (input + initial output) figure frozen at message_start.
+  it("computes a non-stale total across message_start and message_delta", () => {
+    const start: ProviderEvent = {
+      type: "message_start",
+      message: {
+        usage: {
+          input_tokens: 94,
+          output_tokens: 1,
+          cache_read_input_tokens: 31851,
+          cache_creation_input_tokens: 604,
+        },
+      },
+    }
+    const delta: ProviderEvent = {
+      type: "message_delta",
+      delta: { stop_reason: "end_turn" },
+      usage: { output_tokens: 210 },
+    }
+    const usage = applyEvent(applyEvent(emptyAccumulator, start), delta).usage
+    expect(usage.output_tokens).toBe(210)
+    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)
+    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)
+  })
+})