@opencode-ai/ai 0.0.0-next-16167 → 0.0.0-next-16169

package/README.md

@@ -207,7 +207,9 @@ Prompt caching is **on by default**. Every `LLMRequest` resolves to `cache: "aut
 
 ### Auto placement
 
-`"auto"` places three breakpoints — last tool definition, last system part, latest user message. The last-user-message boundary is the load-bearing detail: in a tool-use loop, a single user turn expands into many assistant/tool round-trips, all sharing that prefix. Caching at that boundary lets every intra-turn API call hit.
+`"auto"` places up to four breakpoints — the last tool definition, the first system part, the last system part when distinct, and the final message boundary. These expose successively larger reusable prefixes for tools, the base agent, project instructions, and the active conversation. The rolling final-message boundary is the load-bearing detail in tool loops: it advances on every request so the previous cache entry stays within Anthropic's 20-block lookback.
+
+Tools precede every system and conversation block in the provider prefix, so tool definitions must remain byte-stable and deterministically ordered for downstream breakpoints to remain reusable.
 
 The math justifies the default: Anthropic's 5-minute cache write is 1.25× base, read is 0.1×, so a single reuse within 5 minutes already wins. One-shot completions below the per-model minimum-cacheable-token threshold silently no-op on the wire, so the worst case is harmless.
 
@@ -235,7 +237,7 @@ cache: {
 
 ### Manual hints
 
-Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints; it only fills gaps.
+Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints, counts them against Anthropic and Bedrock's four-breakpoint limit, and only fills the remaining slots.
 
 ```ts
 LLM.request({
@@ -251,8 +253,8 @@ LLM.request({
 
 | Protocol                | `cache: "auto"`                                                           |
 | ----------------------- | ------------------------------------------------------------------------- |
-| Anthropic Messages      | emits up to 3 `cache_control` markers (4-breakpoint cap enforced)         |
-| Bedrock Converse        | emits up to 3 `cachePoint` blocks (4-breakpoint cap enforced)             |
+| Anthropic Messages      | emits up to 4 `cache_control` markers (4-breakpoint cap enforced)         |
+| Bedrock Converse        | emits up to 4 `cachePoint` blocks (4-breakpoint cap enforced)             |
 | OpenAI Chat / Responses | no-op (implicit caching above 1024 tokens)                                |
 | Gemini                  | no-op (implicit caching on 2.5+; explicit `CachedContent` is out-of-band) |
 

package/dist/cache-policy.js

@@ -2,29 +2,28 @@
 // the policy designates. Runs once at compile time, before the per-protocol
 // body builder, so the existing inline-hint lowering path handles the rest.
 //
-// The default `"auto"` shape places one breakpoint at the last tool definition,
-// one at the last system part, and one at the latest user message. This
-// matches what production agent harnesses (LangChain's caching middleware,
-// kern-ai's 10x cost-reduction playbook) converge on for tool-use loops: the
-// latest user message stays put while a single turn explodes into many
-// assistant/tool round-trips, so caching at that boundary lets every
-// intra-turn API call hit the prefix.
+// The default `"auto"` shape places breakpoints at the last tool definition,
+// the first and last distinct system parts, and the conversation tail. This
+// exposes reusable tool, base-agent, project, and session prefixes while
+// advancing the tail after each tool result keeps the previous cache entry
+// within Anthropic's 20-block lookback during long agent turns.
 //
-// Manual `cache: CacheHint` placements on individual parts are preserved —
-// this function only fills gaps the caller left empty.
+// Manual `cache: CacheHint` placements on individual parts are preserved and
+// count against the four-breakpoint budget; auto only fills remaining slots.
 import { CacheHint } from "./schema/options";
 import { LLMRequest, Message, ToolDefinition } from "./schema/messages";
 const AUTO = {
     tools: true,
     system: true,
-    messages: "latest-user-message",
+    messages: { tail: 1 },
 };
 const NONE = {};
+const BREAKPOINT_CAP = 4;
 // Resolution rules:
 //   - undefined   → "auto" — caching is on by default. The math favors it:
 //                   Anthropic 5m-cache write is 1.25x base, read is 0.1x,
 //                   so a single reuse within 5 minutes already wins.
-//   - "auto"      → tools + system + latest user msg.
+//   - "auto"      → tools + first/last system + final message boundary.
 //   - "none"      → no auto placement; manual `CacheHint`s still flow.
 //   - object form → exactly what the caller asked for.
 const resolve = (policy) => {
@@ -39,27 +38,33 @@ const resolve = (policy) => {
 // whole policy pass for these — emitting hints would be harmless but pointless.
 const RESPECTS_INLINE_HINTS = new Set(["anthropic-messages", "bedrock-converse"]);
 const makeHint = (ttlSeconds) => ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" });
-const markLastTool = (tools, hint) => {
+const markLastTool = (tools, hint, budget) => {
     if (tools.length === 0)
         return tools;
     const last = tools.length - 1;
-    if (tools[last].cache)
+    if (tools[last].cache || budget.remaining === 0)
         return tools;
+    budget.remaining -= 1;
     return tools.map((tool, i) => (i === last ? new ToolDefinition({ ...tool, cache: hint }) : tool));
 };
-const markLastSystem = (system, hint) => {
+const markSystemBoundaries = (system, hint, budget) => {
     if (system.length === 0)
         return system;
-    const last = system.length - 1;
-    if (system[last].cache)
-        return system;
-    return system.map((part, i) => (i === last ? { ...part, cache: hint } : part));
+    let changed = false;
+    const next = system.map((part, index) => {
+        if ((index !== 0 && index !== system.length - 1) || part.cache || budget.remaining === 0)
+            return part;
+        budget.remaining -= 1;
+        changed = true;
+        return { ...part, cache: hint };
+    });
+    return changed ? next : system;
 };
 const lastIndexOfRole = (messages, role) => messages.findLastIndex((m) => m.role === role);
 // Mark the last text part of `messages[index]`. If no text part exists, mark
 // the last content part regardless of type — that's the breakpoint position
 // in tool-result-only messages too.
-const markMessageAt = (messages, index, hint) => {
+const markMessageAt = (messages, index, hint, budget) => {
     if (index < 0 || index >= messages.length)
         return messages;
     const target = messages[index];
@@ -68,8 +73,9 @@ const markMessageAt = (messages, index, hint) => {
     const lastTextIndex = target.content.findLastIndex((part) => part.type === "text");
     const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1;
     const existing = target.content[markAt];
-    if ("cache" in existing && existing.cache)
+    if (("cache" in existing && existing.cache) || budget.remaining === 0)
         return messages;
+    budget.remaining -= 1;
     const nextContent = target.content.map((part, i) => (i === markAt ? { ...part, cache: hint } : part));
     const next = new Message({ ...target, content: nextContent });
     // Single pass over `messages`, substituting the one updated entry. Long
@@ -79,19 +85,23 @@ const markMessageAt = (messages, index, hint) => {
     result[index] = next;
     return result;
 };
-const markMessages = (messages, strategy, hint) => {
+const markMessages = (messages, strategy, hint, budget) => {
     if (messages.length === 0)
         return messages;
     if (strategy === "latest-user-message")
-        return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint);
+        return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget);
     if (strategy === "latest-assistant")
-        return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint);
+        return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget);
     const start = Math.max(0, messages.length - strategy.tail);
     let next = messages;
     for (let i = start; i < messages.length; i++)
-        next = markMessageAt(next, i, hint);
+        next = markMessageAt(next, i, hint, budget);
     return next;
 };
+const countHints = (request) => request.tools.reduce((count, tool) => count + (tool.cache === undefined ? 0 : 1), 0) +
+    request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) +
+    request.messages.reduce((count, message) => count +
+        message.content.reduce((contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0), 0), 0);
 export const applyCachePolicy = (request) => {
     if (!RESPECTS_INLINE_HINTS.has(request.model.route.id))
         return request;
@@ -99,9 +109,10 @@ export const applyCachePolicy = (request) => {
     if (!policy.tools && !policy.system && !policy.messages)
         return request;
     const hint = makeHint(policy.ttlSeconds);
-    const tools = policy.tools ? markLastTool(request.tools, hint) : request.tools;
-    const system = policy.system ? markLastSystem(request.system, hint) : request.system;
-    const messages = policy.messages ? markMessages(request.messages, policy.messages, hint) : request.messages;
+    const budget = { remaining: Math.max(0, BREAKPOINT_CAP - countHints(request)) };
+    const tools = policy.tools ? markLastTool(request.tools, hint, budget) : request.tools;
+    const system = policy.system ? markSystemBoundaries(request.system, hint, budget) : request.system;
+    const messages = policy.messages ? markMessages(request.messages, policy.messages, hint, budget) : request.messages;
     if (tools === request.tools && system === request.system && messages === request.messages)
         return request;
     return LLMRequest.update(request, { tools, system, messages });

package/dist/schema/options.js

@@ -175,11 +175,11 @@ export class CacheHint extends Schema.Class("LLM.CacheHint")({
 // Auto-placement policy for prompt caching. The protocol-neutral lowering step
 // reads this and injects `CacheHint`s at the configured boundaries; the
 // per-protocol body builders then translate those hints into wire markers as
-// usual. `"auto"` is the recommended default for agent loops — it places one
-// breakpoint at the last tool definition, one at the last system part, and one
-// at the latest user message. The combination of provider invalidation
-// hierarchy (tools → system → messages) and Anthropic/Bedrock's 20-block
-// lookback means three trailing breakpoints reliably cover the static prefix.
+// usual. `"auto"` is the recommended default for agent loops — it places
+// breakpoints at the last tool definition, the first and last distinct system
+// parts, and the conversation tail. The rolling message breakpoint keeps a
+// prior cache entry within Anthropic/Bedrock's 20-block lookback during long
+// tool loops.
 //
 // Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular
 // object form to override individual choices.

package/package.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
-  "version": "0.0.0-next-16167",
+  "version": "0.0.0-next-16169",
   "name": "@opencode-ai/ai",
   "type": "module",
   "license": "MIT",
@@ -26,7 +26,7 @@
   "devDependencies": {
     "@clack/prompts": "1.0.0-alpha.1",
     "@effect/platform-node": "4.0.0-beta.98",
-    "@opencode-ai/http-recorder": "0.0.0-next-16167",
+    "@opencode-ai/http-recorder": "0.0.0-next-16169",
     "@tsconfig/bun": "1.0.9",
     "@types/bun": "1.3.13",
     "@typescript/native-preview": "7.0.0-dev.20251207.1",
@@ -35,7 +35,7 @@
   "dependencies": {
     "@smithy/eventstream-codec": "4.2.14",
     "@smithy/util-utf8": "4.2.2",
-    "@opencode-ai/schema": "0.0.0-next-16167",
+    "@opencode-ai/schema": "0.0.0-next-16169",
     "aws4fetch": "1.0.20",
     "effect": "4.0.0-beta.98",
     "google-auth-library": "10.5.0"