@effect-uai/anthropic 0.11.0 → 0.12.0

package/dist/usage.test.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/usage.test.mjs

@@ -0,0 +1,46 @@
+import { emptyAccumulator, mergeUsage } from "./codec.mjs";
+import { applyEvent } from "./streamEvents.mjs";
+import { n as describe, r as it, t as globalExpect } from "./test.DNmyFkvJ-CPKukVWS.mjs";
+//#region src/usage.test.ts
+const wire = {
+	input_tokens: 94,
+	output_tokens: 210,
+	cache_read_input_tokens: 31851,
+	cache_creation_input_tokens: 604
+};
+describe("mergeUsage", () => {
+	it("keeps all three input buckets, not just cache reads", () => {
+		const usage = mergeUsage(emptyAccumulator, wire).usage;
+		globalExpect(usage.input_tokens).toBe(94);
+		globalExpect(usage.input_tokens_details?.cached_tokens).toBe(31851);
+		globalExpect(usage.input_tokens_details?.cache_write_tokens).toBe(604);
+	});
+	it("totals every input bucket plus output", () => {
+		const usage = mergeUsage(emptyAccumulator, wire).usage;
+		globalExpect(usage.total_tokens).toBe(32759);
+	});
+});
+describe("streamed usage", () => {
+	it("computes a non-stale total across message_start and message_delta", () => {
+		const usage = applyEvent(applyEvent(emptyAccumulator, {
+			type: "message_start",
+			message: { usage: {
+				input_tokens: 94,
+				output_tokens: 1,
+				cache_read_input_tokens: 31851,
+				cache_creation_input_tokens: 604
+			} }
+		}), {
+			type: "message_delta",
+			delta: { stop_reason: "end_turn" },
+			usage: { output_tokens: 210 }
+		}).usage;
+		globalExpect(usage.output_tokens).toBe(210);
+		globalExpect(usage.input_tokens_details?.cache_write_tokens).toBe(604);
+		globalExpect(usage.total_tokens).toBe(32759);
+	});
+});
+//#endregion
+export {};
+
+//# sourceMappingURL=usage.test.mjs.map

package/dist/usage.test.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.test.mjs","names":[],"sources":["../src/usage.test.ts"],"sourcesContent":["import { describe, expect, it } from \"vitest\"\nimport { type WireUsage, emptyAccumulator, mergeUsage } from \"./codec.js\"\nimport { type ProviderEvent, applyEvent } from \"./streamEvents.js\"\n\n// A completed cached turn: input split across post-breakpoint tokens, cache\n// reads, and cache writes, all billed separately.\nconst wire: WireUsage = {\n  input_tokens: 94,\n  output_tokens: 210,\n  cache_read_input_tokens: 31851,\n  cache_creation_input_tokens: 604,\n}\n\ndescribe(\"mergeUsage\", () => {\n  it(\"keeps all three input buckets, not just cache reads\", () => {\n    const usage = mergeUsage(emptyAccumulator, wire).usage\n    expect(usage.input_tokens).toBe(94)\n    expect(usage.input_tokens_details?.cached_tokens).toBe(31851)\n    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)\n  })\n\n  it(\"totals every input bucket plus output\", () => {\n    const usage = mergeUsage(emptyAccumulator, wire).usage\n    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)\n  })\n})\n\ndescribe(\"streamed usage\", () => {\n  // Anthropic sends input + cache on message_start and the final output_tokens\n  // on message_delta. total_tokens must reflect the accumulated buckets, not\n  // the (input + initial output) figure frozen at message_start.\n  it(\"computes a non-stale total across message_start and message_delta\", () => {\n    const start: ProviderEvent = {\n      type: \"message_start\",\n      message: {\n        usage: {\n          input_tokens: 94,\n          output_tokens: 1,\n          cache_read_input_tokens: 31851,\n          cache_creation_input_tokens: 604,\n        },\n      },\n    }\n    const delta: ProviderEvent = {\n      type: \"message_delta\",\n      delta: { stop_reason: \"end_turn\" },\n      usage: { output_tokens: 210 },\n    }\n    const usage = applyEvent(applyEvent(emptyAccumulator, start), delta).usage\n    expect(usage.output_tokens).toBe(210)\n    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)\n    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)\n  })\n})\n"],"mappings":";;;;AAMA,MAAM,OAAkB;CACtB,cAAc;CACd,eAAe;CACf,yBAAyB;CACzB,6BAA6B;CAC9B;AAED,SAAS,oBAAoB;AAC3B,IAAG,6DAA6D;EAC9D,MAAM,QAAQ,WAAW,kBAAkB,KAAK,CAAC;AACjD,eAAO,MAAM,aAAa,CAAC,KAAK,GAAG;AACnC,eAAO,MAAM,sBAAsB,cAAc,CAAC,KAAK,MAAM;AAC7D,eAAO,MAAM,sBAAsB,mBAAmB,CAAC,KAAK,IAAI;GAChE;AAEF,IAAG,+CAA+C;EAChD,MAAM,QAAQ,WAAW,kBAAkB,KAAK,CAAC;AACjD,eAAO,MAAM,aAAa,CAAC,KAAK,MAAuB;GACvD;EACF;AAEF,SAAS,wBAAwB;AAI/B,IAAG,2EAA2E;EAiB5E,MAAM,QAAQ,WAAW,WAAW,kBAAkB;GAfpD,MAAM;GACN,SAAS,EACP,OAAO;IACL,cAAc;IACd,eAAe;IACf,yBAAyB;IACzB,6BAA6B;IAC9B,EACF;GAOwD,CAAC,EAAE;GAJ5D,MAAM;GACN,OAAO,EAAE,aAAa,YAAY;GAClC,OAAO,EAAE,eAAe,KAAK;GAEoC,CAAC,CAAC;AACrE,eAAO,MAAM,cAAc,CAAC,KAAK,IAAI;AACrC,eAAO,MAAM,sBAAsB,mBAAmB,CAAC,KAAK,IAAI;AAChE,eAAO,MAAM,aAAa,CAAC,KAAK,MAAuB;GACvD;EACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/anthropic",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Anthropic Messages API provider for @effect-uai/core.",
   "keywords": [
     "ai",
@@ -45,7 +45,7 @@
   "devDependencies": {
     "effect": "4.0.0-beta.94",
     "typescript": "^6.0.3",
-    "@effect-uai/core": "0.11.0"
+    "@effect-uai/core": "0.12.0"
   },
   "peerDependencies": {
     "@effect-uai/core": ">=0.2.0 <1",

package/src/Anthropic.ts

@@ -24,6 +24,8 @@ import { type Turn, TurnEvent } from "@effect-uai/core/Turn"
 import { renderProviderTools } from "./AnthropicTools.js"
 import {
   type Accumulator,
+  type CacheControl,
+  type CacheTtl,
   type ThinkingConfig,
   accumulatorToTurn,
   buildRequestBody,
@@ -119,6 +121,21 @@ export type Config = {
    * 4096 if neither is set.
    */
   readonly defaultMaxTokens?: number
+  /**
+   * Prompt caching. Off by default, so the library stays unopinionated about
+   * a setting that changes billing. `true` is `{}`: cache with the 5-minute
+   * default TTL.
+   *
+   * Anthropic caches the `tools` → `system` → `messages` prefix and moves the
+   * breakpoint forward as the conversation grows, so a long system prompt and
+   * tool definitions are paid for once rather than on every turn. Read
+   * `usage.input_tokens_details.cached_tokens` to see it working.
+   *
+   * Mutating earlier history (rewriting an old message, compaction) breaks
+   * the prefix from that point on. Anthropic falls back to the longest
+   * unchanged prefix, so this costs savings but never correctness.
+   */
+  readonly promptCaching?: boolean | { readonly ttl?: CacheTtl }
 }
 
 const ANTHROPIC_VERSION = "2023-06-01"
@@ -139,6 +156,24 @@ const outputConfig = (request: AnthropicRequest): Option.Option<Record<string, u
 const resolvedMaxTokens = (cfg: Config, request: AnthropicRequest): number =>
   request.maxOutputTokens ?? cfg.defaultMaxTokens ?? FALLBACK_MAX_TOKENS
 
+/**
+ * One request-level breakpoint when caching is on. Anthropic places it on the
+ * last cacheable block itself, so nothing has to be marked per-block and the
+ * breakpoint stays correct as history grows.
+ */
+const cacheControlOf = (cfg: Config): Option.Option<CacheControl> =>
+  pipe(
+    Option.fromUndefinedOr(cfg.promptCaching),
+    Option.flatMap((opt) =>
+      opt === false
+        ? Option.none<CacheControl>()
+        : Option.some<CacheControl>({
+            type: "ephemeral",
+            ...(opt !== true && opt.ttl !== undefined && { ttl: opt.ttl }),
+          }),
+    ),
+  )
+
 const functionToolDescriptors = (
   request: AnthropicRequest,
 ): ReadonlyArray<Record<string, unknown>> =>
@@ -302,6 +337,7 @@ const buildNativeStream = (cfg: Config) => {
           toolChoice: toolChoiceWire(request),
           userId: Option.fromUndefinedOr(request.user),
           outputConfig: structured,
+          cacheControl: cacheControlOf(cfg),
         })
 
         const body = yield* Result.match(bodyResult, {

package/src/anthropicTools.test.ts

@@ -42,6 +42,7 @@ const buildWith = (
       toolChoice,
       userId: Option.none(),
       outputConfig: Option.none(),
+      cacheControl: Option.none(),
     }),
   )
 }

package/src/codec.ts

@@ -324,6 +324,20 @@ export type ThinkingConfig = {
   readonly budget_tokens: number
 }
 
+/** Cache lifetime. `1h` writes cost 2x base input; `5m` costs 1.25x. */
+export type CacheTtl = "5m" | "1h"
+
+/**
+ * Request-level cache breakpoint. Anthropic applies it to the last cacheable
+ * block and moves it forward as the conversation grows, so the cached prefix
+ * (`tools` → `system` → `messages`) follows the conversation without the
+ * caller re-placing anything.
+ */
+export type CacheControl = {
+  readonly type: "ephemeral"
+  readonly ttl?: CacheTtl
+}
+
 export type RequestBody = {
   readonly model: string
   readonly messages: ReadonlyArray<RequestMessage>
@@ -338,6 +352,7 @@ export type RequestBody = {
   readonly tool_choice?: Record<string, unknown>
   readonly metadata?: { readonly user_id: string }
   readonly output_config?: Record<string, unknown>
+  readonly cache_control?: CacheControl
   readonly stream: true
 }
 
@@ -354,6 +369,7 @@ export const buildRequestBody = (params: {
   readonly toolChoice: Option.Option<Record<string, unknown>>
   readonly userId: Option.Option<string>
   readonly outputConfig: Option.Option<Record<string, unknown>>
+  readonly cacheControl: Option.Option<CacheControl>
 }): Result.Result<RequestBody, JsonParseError> =>
   pipe(
     groupedMessages(params.history),
@@ -402,6 +418,10 @@ export const buildRequestBody = (params: {
           onNone: () => ({}),
           onSome: (output_config) => ({ output_config }),
         }),
+        ...Option.match(params.cacheControl, {
+          onNone: () => ({}),
+          onSome: (cache_control) => ({ cache_control }),
+        }),
         stream: true,
       }),
     ),
@@ -503,23 +523,34 @@ export const setStopReason = (acc: Accumulator, reason: string): Accumulator =>
   stopReason: Option.some(reason),
 })
 
-const cachedFromWire = (wire: WireUsage): Option.Option<number> =>
-  Option.fromNullishOr(wire.cache_read_input_tokens)
-
+// Anthropic splits input across three billed buckets: `input_tokens`
+// (post-breakpoint), cache reads, and cache writes. All three are kept;
+// `input_tokens` is not the input total. Each field carries over from the
+// prior value since a streamed turn spreads usage across events (input +
+// cache on message_start, output_tokens on message_delta), so total_tokens is
+// summed from the accumulated buckets, not one event.
 export const mergeUsage = (acc: Accumulator, wire: WireUsage): Accumulator => {
-  const cached = cachedFromWire(wire)
+  const prev = acc.usage
+  const input_tokens = wire.input_tokens ?? prev.input_tokens
+  const output_tokens = wire.output_tokens ?? prev.output_tokens
+  const cached_tokens = wire.cache_read_input_tokens ?? prev.input_tokens_details?.cached_tokens
+  const cache_write_tokens =
+    wire.cache_creation_input_tokens ?? prev.input_tokens_details?.cache_write_tokens
+
+  const buckets = [input_tokens, cached_tokens, cache_write_tokens, output_tokens]
+  const total_tokens = buckets.every((v) => v === undefined)
+    ? undefined
+    : buckets.reduce<number>((sum, v) => sum + (v ?? 0), 0)
+
+  const details: Items.InputTokensDetails = {
+    ...(cached_tokens !== undefined && { cached_tokens }),
+    ...(cache_write_tokens !== undefined && { cache_write_tokens }),
+  }
   const usage: Items.Usage = {
-    ...acc.usage,
-    ...(wire.input_tokens !== undefined && { input_tokens: wire.input_tokens }),
-    ...(wire.output_tokens !== undefined && { output_tokens: wire.output_tokens }),
-    ...(wire.input_tokens !== undefined &&
-      wire.output_tokens !== undefined && {
-        total_tokens: wire.input_tokens + wire.output_tokens,
-      }),
-    ...Option.match(cached, {
-      onNone: () => ({}),
-      onSome: (cached_tokens) => ({ input_tokens_details: { cached_tokens } }),
-    }),
+    ...(input_tokens !== undefined && { input_tokens }),
+    ...(output_tokens !== undefined && { output_tokens }),
+    ...(total_tokens !== undefined && { total_tokens }),
+    ...(Object.keys(details).length > 0 && { input_tokens_details: details }),
   }
   return { ...acc, usage }
 }

package/src/usage.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest"
+import { type WireUsage, emptyAccumulator, mergeUsage } from "./codec.js"
+import { type ProviderEvent, applyEvent } from "./streamEvents.js"
+
+// A completed cached turn: input split across post-breakpoint tokens, cache
+// reads, and cache writes, all billed separately.
+const wire: WireUsage = {
+  input_tokens: 94,
+  output_tokens: 210,
+  cache_read_input_tokens: 31851,
+  cache_creation_input_tokens: 604,
+}
+
+describe("mergeUsage", () => {
+  it("keeps all three input buckets, not just cache reads", () => {
+    const usage = mergeUsage(emptyAccumulator, wire).usage
+    expect(usage.input_tokens).toBe(94)
+    expect(usage.input_tokens_details?.cached_tokens).toBe(31851)
+    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)
+  })
+
+  it("totals every input bucket plus output", () => {
+    const usage = mergeUsage(emptyAccumulator, wire).usage
+    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)
+  })
+})
+
+describe("streamed usage", () => {
+  // Anthropic sends input + cache on message_start and the final output_tokens
+  // on message_delta. total_tokens must reflect the accumulated buckets, not
+  // the (input + initial output) figure frozen at message_start.
+  it("computes a non-stale total across message_start and message_delta", () => {
+    const start: ProviderEvent = {
+      type: "message_start",
+      message: {
+        usage: {
+          input_tokens: 94,
+          output_tokens: 1,
+          cache_read_input_tokens: 31851,
+          cache_creation_input_tokens: 604,
+        },
+      },
+    }
+    const delta: ProviderEvent = {
+      type: "message_delta",
+      delta: { stop_reason: "end_turn" },
+      usage: { output_tokens: 210 },
+    }
+    const usage = applyEvent(applyEvent(emptyAccumulator, start), delta).usage
+    expect(usage.output_tokens).toBe(210)
+    expect(usage.input_tokens_details?.cache_write_tokens).toBe(604)
+    expect(usage.total_tokens).toBe(94 + 31851 + 604 + 210)
+  })
+})