@owlmeans/llm 0.1.15 → 0.1.16

package/src/execution/service.ts

@@ -9,7 +9,8 @@ import type {
   HelperExecution, TaskExecution, WithExecutionService,
 } from './types.js'
 import {
-  composeExecState, composeTaskState, effortPatch, freeze, mergeOverride, mergePolicy, resolveRole,
+  composeExecState, composeTaskState, effortPatch, freeze, mergeOverride, mergePolicy,
+  mergePrompt, resolveRole,
 } from './utils.js'
 
 /**
@@ -50,18 +51,21 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
       level: ExecutionLevel.Project,
       purpose: { ...input.purpose },
       policy: { ...input.policy },
+      ...(input.prompt != null ? { prompt: { ...input.prompt } } : {}),
     }) as S['project'],
 
     forTask: (parent, input) => {
-      const { effort, phase, data, ...extras } = input
+      const { effort, phase, data, prompt, ...extras } = input
       const policy = effort != null
         ? mergePolicy(parent.policy, { effort })
         : { ...parent.policy }
+      const merged = mergePrompt(parent.prompt, prompt)
 
       // Spreading the parent carries every collaborator and domain field forward; the
       // task's own state is composed afterwards, from the seeded resumable fields.
       const taskExec = {
         ...parent, ...extras, level: ExecutionLevel.Task, purpose: { ...parent.purpose }, policy,
+        ...(merged != null ? { prompt: merged } : {}),
       } as unknown as TaskExecution
       ;(taskExec as { state: TaskExecutionState }).state = composeTaskState({
         ...taskExec,
@@ -78,9 +82,10 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
     },
 
     forHelper: (parent, input) => {
-      const { role, effort, dedication, ...extras } = input
+      const { role, effort, dedication, prompt, ...extras } = input
       const localPolicy = effort != null ? mergePolicy(parent.policy, { effort }) : parent.policy
       const scoped = { ...parent, policy: localPolicy } as S['exec']
+      const merged = mergePrompt(parent.prompt, prompt)
 
       const helperExec = {
         ...parent,
@@ -90,6 +95,7 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
           ? { ...parent.purpose, dedication }
           : { ...parent.purpose },
         policy: localPolicy,
+        ...(merged != null ? { prompt: merged } : {}),
         role: resolveRole(localPolicy, role),
         model: self().model(scoped, role),
         temperatureFactory: self().temperatureFactory(scoped, role),

package/src/execution/types.ts

@@ -1,10 +1,11 @@
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import type { InitializedService } from '@owlmeans/context'
 import type {
-  ExecutionEffort, ExecutionLevel, ExecutionState, LlmPurpose, ModelConfigOverride,
-  ModelPolicy, ModelRole, TaskExecutionState,
+  ExecutionEffort, ExecutionLevel, ExecutionState, FileProviderRef, LlmPurpose,
+  ModelConfigOverride, ModelPolicy, ModelRole, PromptPolicy, TaskExecutionState,
 } from '@owlmeans/llm-common'
 import type { LlmService, TemperatureFactory } from '../types.js'
+import type { PromptService } from '../prompt/types.js'
 
 /**
  * Runtime execution = serializable {@link ExecutionState} + attached collaborators.
@@ -18,6 +19,10 @@ import type { LlmService, TemperatureFactory } from '../types.js'
 export interface Execution extends ExecutionState {
   /** Resolver for the model factory — a function so the service can be swapped/cloned. */
   models: () => LlmService
+  /** Resolver for the skill registry / prompt composer. Same late-binding rationale. */
+  prompts?: () => PromptService
+  /** File access offered to prompt plugins. Declared a collaborator, never snapshotted. */
+  files?: FileProviderRef
   outputErrors?: boolean
   captureNull?: boolean
 }
@@ -42,8 +47,12 @@ export interface HelperExecution extends Execution {
 
 export interface ProjectExecutionInput {
   models: () => LlmService
+  prompts?: () => PromptService
+  files?: FileProviderRef
   policy: ModelPolicy
   purpose: LlmPurpose
+  /** Baseline role and skills for the whole run. */
+  prompt?: PromptPolicy
   outputErrors?: boolean
   captureNull?: boolean
 }
@@ -51,15 +60,23 @@ export interface ProjectExecutionInput {
 export interface TaskExecutionInput {
   /** Raise (or lower) the effort tier for this task and everything derived from it. */
   effort?: ExecutionEffort
+  /** Skills (and optionally a role) layered on top of the project's. Skills accumulate. */
+  prompt?: PromptPolicy
   /** Optional seeds for the resumable task state. */
   phase?: string
   data?: Record<string, unknown>
 }
 
 export interface HelperExecutionInput {
+  /**
+   * Which MODEL to use. Distinct from `prompt.role`, which is the system-prompt text
+   * defining the persona — one selects hardware, the other writes the job description.
+   */
   role: ModelRole
   /** Local effort bump without escalating the whole branch. */
   effort?: ExecutionEffort
+  /** The helper's persona and its own skills, layered on top of the task's. */
+  prompt?: PromptPolicy
   /** Refines `purpose.dedication`. */
   dedication?: string
 }
@@ -67,6 +84,8 @@ export interface HelperExecutionInput {
 /** Collaborators re-attached to a state that was restored from storage. */
 export interface RestoreCollaborators {
   models?: () => LlmService
+  prompts?: () => PromptService
+  files?: FileProviderRef
 }
 
 /**

package/src/execution/utils.ts

@@ -1,7 +1,7 @@
 import { ExecutionLevel } from '@owlmeans/llm-common'
 import type {
   ExecutionEffort, ExecutionState, ModelConfigOverride, ModelConfigPatch,
-  ModelPolicy, ModelRole, TaskExecutionState,
+  ModelPolicy, ModelRole, PromptPolicy, TaskExecutionState,
 } from '@owlmeans/llm-common'
 import { EFFORT_TABLE } from '../consts.js'
 import type { Execution, TaskExecution } from './types.js'
@@ -19,6 +19,33 @@ export const mergePolicy = (base: ModelPolicy, patch: Partial<ModelPolicy>): Mod
     : undefined,
 })
 
+/**
+ * Overlay a prompt policy onto the one inherited from the parent level.
+ *
+ * Skills ACCUMULATE — a task adds to what the project declared, a helper adds to the
+ * task — because that is how a capability set is built up as work narrows. The role is
+ * replaced instead: the deepest level that names one owns the persona.
+ *
+ * The union preserves first-seen order and de-duplicates, so the composed prompt is
+ * byte-identical no matter how many levels contributed the same skill.
+ */
+export const mergePrompt = (
+  base: PromptPolicy | undefined,
+  patch: PromptPolicy | undefined,
+): PromptPolicy | undefined => {
+  if (base == null && patch == null) {
+    return undefined
+  }
+  const skills = [...new Set([...(base?.skills ?? []), ...(patch?.skills ?? [])])]
+
+  return {
+    ...base,
+    ...patch,
+    ...(base?.role != null || patch?.role != null ? { role: patch?.role ?? base?.role } : {}),
+    ...(skills.length > 0 ? { skills } : {}),
+  }
+}
+
 /** Apply the policy's role→role remap. */
 export const resolveRole = (policy: ModelPolicy, role: ModelRole): ModelRole =>
   (policy.roleOverrides?.[role] as ModelRole | undefined) ?? role

package/src/helpers/cache.ts

@@ -0,0 +1,33 @@
+import type { UsageMetadata } from '@langchain/core/messages'
+import type { CacheUsage } from '@owlmeans/llm-common'
+
+/** LangChain normalizes every provider's cache accounting into these two fields. */
+interface InputTokenDetails {
+  cache_read?: number
+  cache_creation?: number
+}
+
+/**
+ * Prompt-cache accounting for one completion.
+ *
+ * This is the only honest answer to "is caching actually working". A composed prefix can
+ * look perfectly stable and still miss on every call — a stray timestamp, a set iterated
+ * in a different order, a tool list rebuilt per request. If `read` stays at zero across
+ * repeated calls that share a prefix, something is invalidating it; diff the rendered
+ * blocks between two calls to find out what.
+ */
+export const readCacheUsage = (message: { usage_metadata?: UsageMetadata }): CacheUsage => {
+  const usage = message.usage_metadata
+  const details = usage?.input_token_details as InputTokenDetails | undefined
+
+  return {
+    read: details?.cache_read ?? 0,
+    creation: details?.cache_creation ?? 0,
+    input: usage?.input_tokens ?? 0,
+    output: usage?.output_tokens ?? 0,
+  }
+}
+
+/** `true` when the provider reported any cache activity at all. */
+export const hasCacheActivity = (usage: CacheUsage): boolean =>
+  usage.read > 0 || usage.creation > 0

package/src/helpers/index.ts

@@ -3,3 +3,4 @@ export * from './retry.js'
 export * from './json.js'
 export * from './messages.js'
 export * from './spectate.js'
+export * from './cache.js'

package/src/helpers/spectate.ts

@@ -3,6 +3,7 @@ import type { UsageMetadata } from '@langchain/core/messages'
 import { SpectatorContentType } from '@owlmeans/llm-common'
 import type { SpectatorEntryMessage } from '@owlmeans/llm-common'
 import type { LlmSpectator, ModelInputItem } from '../types.js'
+import { hasCacheActivity, readCacheUsage } from './cache.js'
 
 /** Normalize one prompt message into the spectator's storage shape. */
 const describeInput = (msg: ModelInputItem, callType: string): SpectatorEntryMessage => {
@@ -63,5 +64,14 @@ export const spectate = (spectator: LlmSpectator, callType: string) =>
       completion.content = message.content as unknown as string
     }
 
+    // Silent unless the provider reported cache activity, so it costs nothing when
+    // caching is off — and is the one signal that tells a stable prefix from a broken one.
+    const cache = readCacheUsage(message)
+    if (hasCacheActivity(cache)) {
+      console.log(
+        `Prompt cache [${action}]: read ${cache.read}, written ${cache.creation}, uncached ${cache.input}`
+      )
+    }
+
     return spectator.log({ action, retries, startedAt, messages: [...messages, completion] })
   }

package/src/index.ts

@@ -6,6 +6,7 @@ export * from './model.js'
 export * from './service.js'
 export * from './helpers/index.js'
 export * from './execution/index.js'
+export * from './prompt/index.js'
 export type * from './plugins/types.js'
 export { plugins, registerLlmPlugin, pluginOf, pluginFor, resolvePlugin } from './plugins/index.js'
 export { anthropicPlugin, ANTHROPIC_FAMILY } from './plugins/anthropic.js'

package/src/model.ts

@@ -1,12 +1,12 @@
 import { Ajv } from 'ajv'
 import type { JSONSchemaType } from 'ajv'
-import { AIMessage } from '@langchain/core/messages'
-import type { AIMessageChunk, MessageFieldWithRole } from '@langchain/core/messages'
+import { AIMessage, BaseMessage } from '@langchain/core/messages'
+import type { AIMessageChunk, MessageContent, MessageFieldWithRole } from '@langchain/core/messages'
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { StructuredMode } from '@owlmeans/llm-common'
 import type { NullKind } from '@owlmeans/llm-common'
 import {
-  DEFAULT_MAX_OUTPUT_CAP, DEFAULT_MODEL_RETRIES, FALLBACK_AFTER_ATTEMPTS,
+  DEFAULT_MAX_OUTPUT_CAP, DEFAULT_MODEL_RETRIES, FALLBACK_AFTER_ATTEMPTS, MAX_CACHE_BREAKPOINTS,
 } from './consts.js'
 import { LlmModelError } from './errors.js'
 import { pluginFor, pluginOf } from './plugins/index.js'
@@ -18,7 +18,7 @@ import { spectate } from './helpers/spectate.js'
 import { idleTimeout, readConfig } from './utils/config.js'
 import { reportNull } from './utils/null-report.js'
 import type { NullReportParams } from './utils/null-report.js'
-import { applyNoThink, ensureJsonMention } from './utils/prompt.js'
+import { applyNoThink, ensureJsonMention, stripCacheMarkers } from './utils/prompt.js'
 import { resolveSchemaValidator, toToolName, unwrapNamed } from './utils/schema.js'
 import { streamWithDeadline } from './utils/stream.js'
 import type {
@@ -28,6 +28,48 @@ import type {
 
 type StreamOptions = Parameters<BaseChatModel['stream']>[1]
 
+const isSystem = (msg: MessageFieldWithRole): boolean =>
+  msg instanceof BaseMessage ? msg.getType() === 'system' : `${msg.role}` === 'system'
+
+const textOf = (content: MessageContent | undefined): string => {
+  if (typeof content === 'string') {
+    return content.trim()
+  }
+  if (Array.isArray(content)) {
+    return content
+      .map(part => {
+        const text = (part as unknown as { text?: unknown }).text
+        return typeof text === 'string' ? text : ''
+      })
+      .filter(text => text !== '')
+      .join('\n\n')
+      .trim()
+  }
+  return ''
+}
+
+/**
+ * Detach the caller's LEADING system messages and return their text.
+ *
+ * They are re-emitted as the `Context` block of the composed prompt, which is what keeps
+ * a caller that still builds its own `SystemMessage` working unchanged — the text simply
+ * travels a different route and lands in the same place. Only the leading run is taken:
+ * a system message deliberately placed mid-conversation is an operator instruction whose
+ * position carries meaning, and moving it would change what the model sees.
+ */
+const takeLeadingSystem = (msgs: MessageFieldWithRole[]): string[] => {
+  const carried: string[] = []
+  while (msgs.length > 0 && isSystem(msgs[0])) {
+    const [msg] = msgs.splice(0, 1)
+    const text = textOf(msg.content)
+    if (text !== '') {
+      carried.push(text)
+    }
+  }
+
+  return carried
+}
+
 /**
  * Build the four-method model API on top of a LangChain chat model.
  *
@@ -43,6 +85,9 @@ export const makeLlmModel = ({
   captureNull = false,
   retries = DEFAULT_MODEL_RETRIES,
   purpose,
+  prompt,
+  prompts,
+  files,
 }: LlmModelOptions, spectator: LlmSpectator): LlmModel => {
 
   const ajv = new Ajv({ strict: false })
@@ -53,15 +98,67 @@ export const makeLlmModel = ({
   const plugin: LlmPlugin | undefined = pluginOf(config.provider) ?? pluginFor(model)
   const timeout = idleTimeout(config)
 
-  /** Normalize, then apply every in-place prompt adaptation, in dependency order. */
-  const prepare = (input: ModelInput, useCache: boolean, cacheMax: number, json: boolean): MessageFieldWithRole[] => {
+  /**
+   * Normalize, compose the system prompt, then apply every in-place prompt adaptation, in
+   * dependency order.
+   *
+   * With no prompt service wired this is exactly what it always was — the caller's
+   * messages, a JSON nudge, `/no_think`, cache markers. With one, the caller's leading
+   * system text is folded into a composed prompt whose stable sections come first, which
+   * is the whole point: a prompt cache is a PREFIX match, so the bytes every call shares
+   * have to be physically ahead of the bytes that differ.
+   */
+  const prepare = async (
+    input: ModelInput,
+    action: string,
+    useCache: boolean,
+    cacheMax: number,
+    json: boolean,
+    callSkills?: string[],
+  ): Promise<MessageFieldWithRole[]> => {
     const msgs = normalizeInput(input)
+    // The caller may hand back messages this pipeline marked on a PREVIOUS call — the
+    // markers live on its own objects. The budget is per request, so clear them and
+    // re-place our own below; otherwise they accumulate until the provider 400s.
+    stripCacheMarkers(msgs)
+    let reserved = 0
+
+    if (prompts != null) {
+      const carried = takeLeadingSystem(msgs)
+      const composed = await prompts().compose(
+        {
+          ...prompt,
+          context: [...(prompt?.context ?? []), ...carried],
+          callSkills: callSkills ?? prompt?.callSkills,
+        },
+        msgs,
+        { model, provider: plugin, purpose, action, cacheMax, files },
+      )
+      if (composed.system != null) {
+        msgs.unshift(composed.system)
+        reserved = composed.breakpoints
+      } else {
+        // Defensive: nothing was contributed, so hand the caller's own text straight back.
+        for (let i = carried.length - 1; i >= 0; i--) {
+          msgs.unshift({ role: 'system', content: carried[i] })
+        }
+      }
+    }
+
     if (json) ensureJsonMention(msgs)
     applyNoThink(msgs, config.disableThinking)
     // Cache markers replace string content with content blocks, so they must go last.
-    if (plugin?.patchCache?.(msgs, { model, useCache, cacheMax }) === true) {
-      console.log(`Prompt caching enabled for ${plugin.type} (up to ${cacheMax} breakpoints)`)
+    const ttl = prompt?.cacheTtl
+    const marked = plugin?.patchCache?.(msgs, {
+      model, useCache, cacheMax, reserved, ...(ttl != null ? { ttl } : {}),
+    })
+    if (reserved > 0 || marked === true) {
+      console.log(
+        `Prompt caching for ${plugin?.type}: ${reserved} system breakpoint(s)`
+        + `${marked === true ? ', 1 message breakpoint' : ''}`
+      )
     }
+
     return msgs
   }
 
@@ -197,8 +294,11 @@ export const makeLlmModel = ({
   }
 
   const helper: LlmModel = {
-    ask: async (input, { ref, filter, action, useCache = false, cacheMax = 4 }: LlmAskOptions) => {
-      const msgs = prepare(input, useCache, cacheMax, false)
+    ask: async (
+      input,
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmAskOptions
+    ) => {
+      const msgs = await prepare(input, action, useCache, cacheMax, false, skills)
       return withRetry({ retries, outputErrors }, async i => {
         const refined = refineModel(i)
         console.log('Use model to ask: ', refined.getName(), refined.lc_kwargs.model)
@@ -242,8 +342,11 @@ export const makeLlmModel = ({
       })
     },
 
-    talk: async (input, { ref, filter, action, useCache = false, cacheMax = 4 }: LlmTalkOptions) => {
-      const msgs = prepare(input, useCache, cacheMax, false)
+    talk: async (
+      input,
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmTalkOptions
+    ) => {
+      const msgs = await prepare(input, action, useCache, cacheMax, false, skills)
       return withRetry({ retries, outputErrors }, async i => {
         const refined = refineModel(i)
         console.log('Use model to talk: ', refined.getName(), refined.lc_kwargs.model)
@@ -277,9 +380,9 @@ export const makeLlmModel = ({
     invoke: async <T>(
       input: ModelInput,
       schema: JSONSchemaType<T>,
-      { temperature, ref, filter, action, useCache = false, cacheMax = 4 }: LlmInvokeOptions<T>
+      { temperature, ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmInvokeOptions<T>
     ) => {
-      const msgs = prepare(input, useCache, cacheMax, true)
+      const msgs = await prepare(input, action, useCache, cacheMax, true, skills)
       const { name, innerSchema, validate } = resolveSchemaValidator<T>(ajv, schema)
       const toolName = toToolName((innerSchema as { title?: string }).title ?? name)
 
@@ -325,9 +428,9 @@ export const makeLlmModel = ({
     request: async <T>(
       input: ModelInput,
       schema: JSONSchemaType<T>,
-      { ref, filter, action, useCache = false, cacheMax = 4 }: LlmRequestOptions
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmRequestOptions
     ) => {
-      const msgs = prepare(input, useCache, cacheMax, true)
+      const msgs = await prepare(input, action, useCache, cacheMax, true, skills)
       const { name, innerSchema, validate } = resolveSchemaValidator<T>(ajv, schema)
       const toolName = toToolName((innerSchema as { title?: string }).title ?? name)
 

package/src/plugins/anthropic.ts

@@ -1,9 +1,12 @@
 import { ChatAnthropic } from '@langchain/anthropic'
 import { BadRequestError } from '@anthropic-ai/sdk'
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
-import { ModelProvider, StructuredMode } from '@owlmeans/llm-common'
+import type { MessageContent, MessageFieldWithRole } from '@langchain/core/messages'
+import { ModelProvider, PromptBlock, StructuredMode } from '@owlmeans/llm-common'
+import type { CacheTtl } from '@owlmeans/llm-common'
 import type { LlmPlugin } from './types.js'
-import { MAX_CACHE_BREAKPOINTS } from '../consts.js'
+import { CHARS_PER_TOKEN, MAX_CACHE_BREAKPOINTS, MIN_CACHEABLE_TOKENS } from '../consts.js'
+import { readConfig } from '../utils/config.js'
 import { escalateMaxTokens, makeClientOptions } from './utils.js'
 
 /** Model-name prefix that supports prompt caching through `cache_control` markers. */
@@ -11,6 +14,64 @@ const CACHEABLE_PREFIX = 'claude-'
 
 export const ANTHROPIC_FAMILY = 'anthropic'
 
+type ContentBlock = Record<string, unknown>
+
+const supportsCache = (model: BaseChatModel): boolean =>
+  (model as ChatAnthropic).modelName?.startsWith(CACHEABLE_PREFIX) === true
+
+/**
+ * Shortest prefix worth a breakpoint, in characters. Anthropic silently declines to
+ * create an entry below its own per-model minimum, so a marker there wastes one of the
+ * four breakpoints and reports a cache that was never written.
+ */
+const minCacheableChars = (model: BaseChatModel): number =>
+  (readConfig(model).cacheMinTokens ?? MIN_CACHEABLE_TOKENS) * CHARS_PER_TOKEN
+
+/**
+ * The marker itself. `ttl` is omitted for the 5-minute default so the emitted bytes stay
+ * the classic shape — a request that differs only in an explicit `"ttl": "5m"` would not
+ * match a prefix cached without it.
+ */
+const marker = (ttl: CacheTtl): ContentBlock =>
+  ttl === '1h' ? { type: 'ephemeral', ttl: '1h' } : { type: 'ephemeral' }
+
+const contentLength = (content: MessageContent | undefined): number => {
+  if (typeof content === 'string') {
+    return content.length
+  }
+  if (Array.isArray(content)) {
+    return content.reduce<number>((sum, part) => {
+      const text = (part as unknown as { text?: unknown }).text
+      return sum + (typeof text === 'string' ? text.length : 0)
+    }, 0)
+  }
+  return 0
+}
+
+/**
+ * Put a breakpoint on a message's LAST content block, lifting string content into a block
+ * so the marker has somewhere to live. Idempotent, and it never mutates a block the
+ * caller owns — the array is rebuilt around a fresh copy of the final entry.
+ */
+const markMessage = (msg: MessageFieldWithRole, mark: ContentBlock): boolean => {
+  if (typeof msg.content === 'string') {
+    msg.content = [{ type: 'text', text: msg.content, cache_control: mark }] as unknown as MessageContent
+    return true
+  }
+  if (Array.isArray(msg.content) && msg.content.length > 0) {
+    const blocks = [...msg.content] as ContentBlock[]
+    const last = blocks[blocks.length - 1]
+    if (last.cache_control != null) {
+      return true
+    }
+    blocks[blocks.length - 1] = { ...last, cache_control: mark }
+    msg.content = blocks as unknown as MessageContent
+    return true
+  }
+
+  return false
+}
+
 export const anthropicPlugin: LlmPlugin = {
   type: ModelProvider.Anthropic,
 
@@ -70,28 +131,106 @@ export const anthropicPlugin: LlmPlugin = {
   },
 
   /**
-   * Mark the leading messages with an ephemeral `cache_control` breakpoint. Anthropic
-   * allows at most {@link MAX_CACHE_BREAKPOINTS}; string content is lifted into a
-   * single text block so the marker has somewhere to live.
+   * Render the composed system prompt as Anthropic content blocks, one per section, with
+   * a breakpoint on each stability boundary:
+   *
+   * - after `Role` + `Skills` — the region every call of this role shares;
+   * - after `Packages`        — varies with what the request mentions, so it gets its own
+   *                             entry and can never invalidate the block above it;
+   * - after the last block    — so the whole system prompt is cached, which is the
+   *                             default this layer promises.
+   *
+   * Boundaries that coincide collapse into one. Marking stops as soon as the budget is
+   * spent, earliest boundary first — the earliest prefix is the one most calls share.
    */
-  patchCache: (msgs, { model, useCache, cacheMax }) => {
-    if (!useCache || !(model as ChatAnthropic).modelName?.startsWith(CACHEABLE_PREFIX)) return false
-    const max = Math.min(cacheMax, MAX_CACHE_BREAKPOINTS)
-    let i = 0
-    for (const msg of msgs) {
-      msg.content = typeof msg.content === 'string' ? [{
-        type: 'text',
-        text: msg.content,
-        cache_control: { type: 'ephemeral' },
-      }] : msg.content
-      if (++i > max - 1) break
+  patchSystem: (blocks, { model, cacheMax, ttl }) => {
+    const content: ContentBlock[] = blocks.map(block => ({ type: 'text', text: block.text }))
+    if (!supportsCache(model) || cacheMax < 1) {
+      return { content: content as unknown as MessageContent, breakpoints: 0 }
     }
-    return true
+
+    const lastOf = (...wanted: PromptBlock[]): number =>
+      blocks.reduce((found, block, i) => wanted.includes(block.block) ? i : found, -1)
+
+    // Closing the prompt is worth a breakpoint only when the last block is STABLE. A
+    // trailing `Context` changes every call, so marking it would pay a cache write every
+    // time and never read one back — it burns a breakpoint to buy nothing. A prompt that
+    // is ONLY context (a caller that has not adopted role/skills) is still worth marking,
+    // because there it IS the stable part.
+    const last = blocks.length - 1
+    const closing = blocks[last].block === PromptBlock.Context && blocks.length > 1 ? -1 : last
+
+    const boundaries = [...new Set([
+      lastOf(PromptBlock.Role, PromptBlock.Skills),
+      lastOf(PromptBlock.Packages),
+      closing,
+    ].filter(index => index >= 0))].sort((a, b) => a - b)
+
+    const minChars = minCacheableChars(model)
+    let consumed = 0
+    let chars = 0
+    let next = 0
+    for (let i = 0; i < content.length; i++) {
+      chars += blocks[i].text.length
+      if (i !== boundaries[next]) {
+        continue
+      }
+      next++
+      if (chars < minChars || consumed >= cacheMax) {
+        continue
+      }
+      content[i].cache_control = marker(ttl)
+      consumed++
+    }
+
+    return { content: content as unknown as MessageContent, breakpoints: consumed }
+  },
+
+  /**
+   * One breakpoint, at the end of the stable message prefix (`cacheMax` messages).
+   *
+   * Not one marker per message: the request budget is {@link MAX_CACHE_BREAKPOINTS} in
+   * total across tools, system and messages, and the system prompt — the part that is
+   * genuinely identical between calls — has first claim on it. `reserved` is what the
+   * system prompt already spent.
+   */
+  patchCache: (msgs, { model, useCache, cacheMax, reserved = 0, ttl = '5m' }) => {
+    if (!useCache || !supportsCache(model) || msgs.length === 0) {
+      return false
+    }
+    if (Math.min(cacheMax, MAX_CACHE_BREAKPOINTS - reserved) < 1) {
+      return false
+    }
+
+    // The final message is the per-call payload, and `ensureJsonMention` / `applyNoThink`
+    // append to it — including it in the prefix would write a fresh entry every call and
+    // read none. The stable prefix therefore stops one short of the end.
+    const index = Math.min(cacheMax, msgs.length - 1) - 1
+    const target = index >= 0 ? msgs[index] : null
+    if (target == null) {
+      return false
+    }
+    const chars = msgs.slice(0, index + 1)
+      .reduce((sum, msg) => sum + contentLength(msg.content), 0)
+    if (chars < minCacheableChars(model)) {
+      return false
+    }
+
+    return markMessage(target, marker(ttl))
   },
 
   /**
-   * A malformed request (bad schema, unsupported parameter, oversized `max_tokens`)
-   * cannot be fixed by retrying — surface it immediately instead of burning the budget.
+   * A malformed request (bad schema, unsupported parameter, oversized `max_tokens`, too
+   * many cache breakpoints) cannot be fixed by retrying — surface it immediately instead
+   * of burning the budget.
+   *
+   * The `status` check is not redundant with the `instanceof`: `@langchain/anthropic`
+   * carries its OWN nested copy of `@anthropic-ai/sdk`, so the error it throws is an
+   * instance of a DIFFERENT `BadRequestError` class than the one imported here and the
+   * `instanceof` silently fails. That turned every fatal 400 into eight full retries —
+   * a single malformed request became minutes of thrash with the real cause buried.
    */
-  isFatal: e => e instanceof BadRequestError ? e : null,
+  isFatal: e => e instanceof BadRequestError || (e as { status?: unknown })?.status === 400
+    ? e as Error
+    : null,
 }