@owlmeans/llm 0.1.15 → 0.1.16-rc.0

package/src/prompt/types.ts

@@ -0,0 +1,114 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import type { MessageFieldWithRole } from '@langchain/core/messages'
+import type { InitializedService } from '@owlmeans/context'
+import type {
+  CacheTtl, FileProviderRef, LlmPurpose, PromptBlock, PromptPolicy, SkillDefinition,
+} from '@owlmeans/llm-common'
+import type { LlmPlugin, LlmSystemBlock } from '../plugins/types.js'
+
+/**
+ * Everything that shapes one composed system prompt. The serializable part
+ * ({@link PromptPolicy}) travels on the execution state; the rest is per-call.
+ */
+export interface PromptInput extends PromptPolicy {
+  /** Skills supplied verbatim, without registering them first. */
+  inline?: SkillDefinition[]
+  /**
+   * Raw text for the volatile `Context` block — a caller's own system message, carried
+   * through unchanged. Never part of the cached prefix.
+   */
+  context?: string[]
+  /**
+   * Skill aliases requested for THIS call only. Rendered into `Context`, not `Skills`:
+   * a set that changes per call must not sit inside the region other calls cache.
+   */
+  callSkills?: string[]
+}
+
+export interface PromptComposeParams {
+  /** The ORIGINAL (unrefined) model the composed prompt will be sent to. */
+  model: BaseChatModel
+  /** Provider plugin governing the call — supplies the cache-marker strategy. */
+  provider?: LlmPlugin
+  purpose?: LlmPurpose
+  action?: string
+  /** Total breakpoint budget for the request. */
+  cacheMax?: number
+  /** File access a plugin may use to resolve knowledge from disk. */
+  files?: FileProviderRef
+}
+
+/** What a prompt plugin sees and may contribute to. */
+export interface PromptContext extends PromptComposeParams {
+  input: PromptInput
+  /** The call's messages. Read them for detection; do NOT mutate. */
+  messages: readonly MessageFieldWithRole[]
+  /** Append a rendered chunk to a block. Empty text is ignored. */
+  add: (block: PromptBlock, text: string) => void
+  /** Resolve skill aliases through the registry, following `requires`. */
+  resolve: (aliases: readonly string[]) => SkillDefinition[]
+}
+
+/**
+ * A unit of system-prompt composition, registered on the {@link PromptService} at context
+ * composition time. This is the seam that makes the LLM layer pluggable: the package
+ * ships the role and skills plugins, an application adds its own.
+ *
+ * A plugin MUST be deterministic — same input, same bytes. Anything it contributes to a
+ * cached block and cannot reproduce exactly invalidates the prefix for every call that
+ * shares it.
+ */
+export interface LlmPromptPlugin {
+  alias: string
+  /** Lower runs first. The built-ins occupy 0 (role), 10 (skills) and 90 (context). */
+  order?: number
+  /** Contribute static content, before anything has looked at the messages. */
+  compose?: (ctx: PromptContext) => void | Promise<void>
+  /** Contribute content derived from the messages (detection, lookup, fetch). */
+  inspect?: (ctx: PromptContext) => void | Promise<void>
+}
+
+export interface PromptResult {
+  /** The composed system message, or `null` when nothing was contributed. */
+  system: MessageFieldWithRole | null
+  /** Breakpoints the system prompt consumed; subtract from the message budget. */
+  breakpoints: number
+  /** The rendered blocks in emission order — exposed for tests and diagnostics. */
+  blocks: LlmSystemBlock[]
+}
+
+export interface PromptServiceOptions {
+  /** Skills registered up front; equivalent to calling `register` after construction. */
+  skills?: SkillDefinition[]
+  /** Plugins registered up front, in addition to the built-ins. */
+  plugins?: LlmPromptPlugin[]
+  /** Default for `PromptPolicy.cacheSystem`. Defaults to `true`. */
+  cacheSystem?: boolean
+  /** Default for `PromptPolicy.cacheTtl`. */
+  cacheTtl?: CacheTtl
+}
+
+/**
+ * Registry of reusable skills plus the plugin chain that turns a role and a set of skill
+ * aliases into a cache-friendly system prompt.
+ */
+export interface PromptService extends InitializedService {
+  /** Register a composition plugin. Runs in `order` then registration order. */
+  use: (plugin: LlmPromptPlugin) => void
+  /** Register (or replace, by alias) reusable skills. */
+  register: (...skills: SkillDefinition[]) => void
+  has: (alias: string) => boolean
+  /** Resolve aliases to definitions, pulling in `requires` transitively. Unknown aliases are skipped. */
+  resolve: (aliases: readonly string[]) => SkillDefinition[]
+  /** Every registered skill, in deterministic order. */
+  skills: () => SkillDefinition[]
+  compose: (
+    input: PromptInput,
+    messages: readonly MessageFieldWithRole[],
+    params: PromptComposeParams,
+  ) => Promise<PromptResult>
+}
+
+export interface WithPromptService {
+  prompts: () => PromptService
+}

package/src/service.ts

@@ -51,6 +51,9 @@ export const llmServiceApi = (options: LlmServiceOptions, self: () => LlmService
       throw new LlmMissconfiguredError(alias)
     }
     const config: ModelConfig = { ...baseConfig, ...override }
+    // The service-wide idle deadline is a floor, not an override: a preset that states its
+    // own `streamTimeout` knows something specific about that model and keeps it.
+    config.streamTimeout ??= options.streamTimeout
     const preset: Partial<ModelConfig> = config.preset != null
       ? { ...(models.find(m => m.alias === config.preset) ?? {}) }
       : {}

package/src/types.ts

@@ -4,8 +4,10 @@ import type { BaseCallbackHandler, CallbackHandlerMethods } from '@langchain/cor
 import type { JSONSchemaType } from 'ajv'
 import type { InitializedService } from '@owlmeans/context'
 import type {
-  LlmPurpose, ModelProvider, NullCapture, SpectatorArgument, SpectatorEntryLogged,
+  FileProviderRef, LlmPurpose, ModelProvider, NullCapture, SpectatorArgument,
+  SpectatorEntryLogged,
 } from '@owlmeans/llm-common'
+import type { PromptInput, PromptService } from './prompt/types.js'
 
 export type MaybeArray<T> = T | T[]
 
@@ -47,6 +49,20 @@ export interface LlmLogging {
 export interface LlmModelOptions extends LlmLogging {
   model: BaseChatModel
   retries?: number
+  /**
+   * Role and skills for every call this model makes. Composed into a cacheable system
+   * prompt by {@link PromptService}; ignored when no `prompts` resolver is supplied.
+   * Usually just `exec.prompt` from the helper execution.
+   */
+  prompt?: PromptInput
+  /**
+   * Late-bound resolver for the prompt service — a function, like `Execution.models`, so
+   * the service can be swapped or cloned. Without it the model behaves exactly as before
+   * this layer existed: the caller's messages are sent untouched.
+   */
+  prompts?: () => PromptService
+  /** File access offered to prompt plugins that resolve knowledge from disk. */
+  files?: FileProviderRef
 }
 
 export type ModelMessage = BaseMessage | MessageFieldWithRole
@@ -56,10 +72,23 @@ export type ModelInput = MaybeArray<ModelInputItem>
 export interface LlmCallOptions {
   /** Short name of the operation — used as the LangChain run name and in spectator entries. */
   action: string
-  /** Ask the provider to cache the prompt prefix (no-op for providers without prompt caching). */
+  /**
+   * Cache the leading MESSAGES too. The composed system prompt is cached by default and
+   * independently of this flag — this one is about the conversation prefix, which is only
+   * worth caching when the same leading messages recur across calls.
+   */
   useCache?: boolean
-  /** How many leading messages to mark as cacheable (capped by the provider's own limit). */
+  /**
+   * How many leading messages form the stable prefix. One breakpoint is placed at its
+   * end (not one per message), capped by whatever the system prompt left unspent.
+   */
   cacheMax?: number
+  /**
+   * Skill aliases for THIS call only. Rendered into the volatile `Context` block, so they
+   * never disturb the cached region — declare a skill on the execution instead when it
+   * should be part of the shared prefix.
+   */
+  skills?: string[]
 }
 
 export interface LlmAskOptions extends LlmCallOptions {
@@ -183,11 +212,33 @@ export interface ModelConfig {
    * rotating providers mid-call would flip the structured-output format.
    */
   fallback?: Partial<ModelConfig>
+  /**
+   * Per-model override of {@link MIN_CACHEABLE_TOKENS} — the shortest prefix worth a
+   * cache breakpoint. Anthropic's own minimum is model-dependent and NOT monotonic
+   * across generations (512 on the newest, 1024 on most, 4096 on a few older ones), so a
+   * preset that pins an old model should raise this rather than pay for markers that
+   * silently never cache.
+   */
+  cacheMinTokens?: number
+  /**
+   * Cache-routing key for providers whose prompt cache is automatic (OpenAI's
+   * `prompt_cache_key`): requests sharing a key are routed to the same backend, which
+   * raises the hit rate for a shared prefix. Must be STABLE and low-cardinality — one
+   * value per role, never per user or per request. Defaults to the config alias.
+   */
+  cacheKey?: string
 }
 
 export interface LlmServiceOptions {
   /** The full config list; resolved by `alias` on every `getModel` call. */
   models: () => ModelConfig[]
+  /**
+   * Idle deadline (ms) applied to every model this service builds, unless the model's own
+   * config overrides it. This is the knob an application sets where it composes its
+   * context — one place to tune how long the whole deployment waits on a silent provider,
+   * without touching a preset. Falls back to {@link MODEL_STREAM_TIMEOUT_MS}.
+   */
+  streamTimeout?: number
 }
 
 /**

package/src/utils/prompt.ts

@@ -23,6 +23,39 @@ const appendDirective = (msgs: MessageFieldWithRole[], text: string): void => {
   }
 }
 
+/**
+ * Drop every `cache_control` marker from the messages.
+ *
+ * Markers are placed in-place, on the caller's own message objects — and a caller that
+ * carries its message array across calls (the coder's growing conversation, a fix loop
+ * re-sending the same sources) hands them back still marked. The provider counts markers
+ * per REQUEST, not per message: Anthropic rejects the fifth outright with
+ * `400 A maximum of 4 blocks with cache_control may be provided. Found 5.`, and since a
+ * 400 is fatal it burns the entire retry budget before surfacing.
+ *
+ * So the pipeline always starts from a clean slate and re-places its own markers, which
+ * makes the per-request count a function of THIS call alone.
+ */
+export const stripCacheMarkers = (msgs: MessageFieldWithRole[]): void => {
+  for (const msg of msgs) {
+    if (!Array.isArray(msg.content)) {
+      continue
+    }
+    let found = false
+    const blocks = msg.content.map(part => {
+      if (typeof part === 'object' && part !== null && 'cache_control' in part) {
+        found = true
+        const { cache_control: _dropped, ...rest } = part as Record<string, unknown>
+        return rest
+      }
+      return part
+    })
+    if (found) {
+      msg.content = blocks as unknown as typeof msg.content
+    }
+  }
+}
+
 /**
  * Ensure the word "json" appears somewhere in the prompt. Several providers refuse or
  * silently ignore a JSON mode unless it does; when it is missing the JSON instruction is

package/tests/execution.spec.ts

@@ -180,6 +180,48 @@ describe('@owlmeans/llm — snapshot and restore', () => {
   })
 })
 
+describe('@owlmeans/llm — prompt policy accumulation', () => {
+  const withPrompt = () => service.root({
+    models: makeService(),
+    policy: { effort: DEFAULT_EFFORT },
+    purpose: { type: 'spec' },
+    prompt: { role: 'project role', skills: ['base'] },
+  })
+
+  // Skills accumulate as work narrows — that is how a helper ends up knowing everything
+  // the project, the task and its own role declared, without any of them repeating it.
+  test('skills accumulate down the chain while the deepest role wins', () => {
+    const task = service.forTask(withPrompt(), { prompt: { skills: ['task'] } })
+    expect(task.prompt?.skills).toEqual(['base', 'task'])
+    expect(task.prompt?.role).toBe('project role')
+
+    const helper = service.forHelper(task, {
+      role: Role.Analyst, prompt: { role: 'helper role', skills: ['helper'] },
+    })
+    expect(helper.prompt?.skills).toEqual(['base', 'task', 'helper'])
+    expect(helper.prompt?.role).toBe('helper role')
+  })
+
+  // A skill declared twice must not render twice, or the composed prefix differs from the
+  // one a single declaration would have produced.
+  test('a repeated skill is unioned, not duplicated', () => {
+    const task = service.forTask(withPrompt(), { prompt: { skills: ['base', 'task'] } })
+    expect(task.prompt?.skills).toEqual(['base', 'task'])
+  })
+
+  test('levels that declare nothing inherit the policy untouched', () => {
+    const helper = service.forHelper(service.forTask(withPrompt(), {}), { role: Role.Analyst })
+    expect(helper.prompt).toEqual({ role: 'project role', skills: ['base'] })
+  })
+
+  // It is part of ExecutionState, so a resumed run rebuilds the same system prompt.
+  test('the policy survives a snapshot/restore round trip', () => {
+    const helper = service.forHelper(withPrompt(), { role: Role.Analyst })
+    const restored = service.restore(service.snapshot(helper), { models: makeService() })
+    expect(restored.prompt).toEqual({ role: 'project role', skills: ['base'] })
+  })
+})
+
 describe('@owlmeans/llm — resilience plugin seam', () => {
   test('checkpoint is a no-op until a plugin is registered', async () => {
     await expect(service.checkpoint(root, 'key')).resolves.toBeUndefined()

package/tests/plugins.spec.ts

@@ -2,13 +2,14 @@ import { describe, expect, test } from 'bun:test'
 import { ChatAnthropic } from '@langchain/anthropic'
 import { ChatOpenAI } from '@langchain/openai'
 import { BadRequestError } from '@anthropic-ai/sdk'
-import { ModelProvider, StructuredMode } from '@owlmeans/llm-common'
+import { ModelProvider, PromptBlock, StructuredMode } from '@owlmeans/llm-common'
 import {
   anthropicPlugin, compatiblePlugin, makeLlmService, openAiPlugin, pluginFor, pluginOf,
   registerLlmPlugin, resolvePlugin,
 } from '@owlmeans/llm'
 import type { LlmPlugin, ModelConfig } from '@owlmeans/llm'
 import { offlineConfigs, Role } from './context.js'
+import { stripCacheMarkers } from '../src/utils/prompt.js'
 
 const build = (plugin: LlmPlugin, config: Partial<ModelConfig> = {}) =>
   plugin.build({
@@ -164,26 +165,251 @@ describe('@owlmeans/llm — retry escalation behaviour', () => {
   })
 })
 
-describe('@owlmeans/llm — prompt caching', () => {
-  test('anthropic marks up to the requested number of leading messages', () => {
+describe('@owlmeans/llm — message prompt caching', () => {
+  /** `cacheMinTokens: 1` puts the minimum at 4 characters so fixtures stay readable. */
+  const cheap = () => build(anthropicPlugin, { model: 'claude-haiku-4-5-20251001', cacheMinTokens: 1 })
+
+  // The budget is four breakpoints for the WHOLE request, and the composed system prompt
+  // has first claim on it — so the messages get one marker at the end of their stable
+  // prefix, not one marker each.
+  test('anthropic places a single breakpoint at the end of the stable prefix', () => {
+    const msgs = [
+      { role: 'system' as const, content: 'aaaa' },
+      { role: 'user' as const, content: 'bbbb' },
+      { role: 'user' as const, content: 'cccc' },
+    ]
+    expect(anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 2 })).toBe(true)
+    expect(msgs[0]!.content).toBe('aaaa')
+    expect(msgs[1]!.content).toEqual([{ type: 'text', text: 'bbbb', cache_control: { type: 'ephemeral' } }])
+    expect(msgs[2]!.content).toBe('cccc')
+  })
+
+  // The last message is the per-call payload (and `ensureJsonMention` / `applyNoThink`
+  // append to it) — caching it would write a fresh entry every call and read none.
+  test('the final message is never part of the cached prefix', () => {
+    const msgs = [
+      { role: 'system' as const, content: 'aaaa' },
+      { role: 'user' as const, content: 'bbbb' },
+    ]
+    expect(anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 9 })).toBe(true)
+    expect(msgs[0]!.content).toEqual([{ type: 'text', text: 'aaaa', cache_control: { type: 'ephemeral' } }])
+    expect(msgs[1]!.content).toBe('bbbb')
+
+    const single = [{ role: 'user' as const, content: 'aaaa' }]
+    expect(anthropicPlugin.patchCache?.(single, { model: cheap(), useCache: true, cacheMax: 4 })).toBe(false)
+    expect(single[0]!.content).toBe('aaaa')
+  })
+
+  test('a marker is appended to block content rather than replacing it, and is idempotent', () => {
+    const msgs = [
+      { role: 'system' as const, content: [{ type: 'text', text: 'aa' }, { type: 'text', text: 'bb' }] },
+      { role: 'user' as const, content: 'cccc' },
+    ]
+    expect(anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 1 })).toBe(true)
+    expect(msgs[0]!.content).toEqual([
+      { type: 'text', text: 'aa' },
+      { type: 'text', text: 'bb', cache_control: { type: 'ephemeral' } },
+    ])
+
+    const before = JSON.stringify(msgs[0]!.content)
+    expect(anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 1 })).toBe(true)
+    expect(JSON.stringify(msgs[0]!.content)).toBe(before)
+  })
+
+  // A prefix under the provider's own minimum is silently never cached, so a marker there
+  // buys nothing and costs one of the four breakpoints.
+  test('a prefix below the cacheable minimum is left unmarked', () => {
     const model = build(anthropicPlugin, { model: 'claude-haiku-4-5-20251001' })
     const msgs = [
-      { role: 'system' as const, content: 'a' },
-      { role: 'user' as const, content: 'b' },
-      { role: 'user' as const, content: 'c' },
+      { role: 'system' as const, content: 'short' },
+      { role: 'user' as const, content: 'also short' },
     ]
-    expect(anthropicPlugin.patchCache?.(msgs, { model, useCache: true, cacheMax: 2 })).toBe(true)
-    expect(msgs[0]!.content).toEqual([{ type: 'text', text: 'a', cache_control: { type: 'ephemeral' } }])
-    expect(msgs[1]!.content).toEqual([{ type: 'text', text: 'b', cache_control: { type: 'ephemeral' } }])
-    expect(msgs[2]!.content).toBe('c')
+    expect(anthropicPlugin.patchCache?.(msgs, { model, useCache: true, cacheMax: 4 })).toBe(false)
+    expect(msgs[0]!.content).toBe('short')
+  })
+
+  test('the message budget yields to whatever the system prompt already spent', () => {
+    const msgs = [
+      { role: 'system' as const, content: 'aaaa' },
+      { role: 'user' as const, content: 'bbbb' },
+    ]
+    expect(anthropicPlugin.patchCache?.(
+      msgs, { model: cheap(), useCache: true, cacheMax: 4, reserved: 4 }
+    )).toBe(false)
+    expect(msgs[0]!.content).toBe('aaaa')
   })
 
   test('caching is a no-op when not requested, and for providers without it', () => {
-    const model = build(anthropicPlugin, { model: 'claude-haiku-4-5-20251001' })
-    const msgs = [{ role: 'user' as const, content: 'a' }]
-    expect(anthropicPlugin.patchCache?.(msgs, { model, useCache: false, cacheMax: 4 })).toBe(false)
-    expect(msgs[0]!.content).toBe('a')
+    const msgs = [{ role: 'user' as const, content: 'aaaa' }, { role: 'user' as const, content: 'bbbb' }]
+    expect(anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: false, cacheMax: 4 })).toBe(false)
+    expect(msgs[0]!.content).toBe('aaaa')
     expect(openAiPlugin.patchCache).toBeUndefined()
+    expect(openAiPlugin.patchSystem).toBeUndefined()
+  })
+})
+
+describe('@owlmeans/llm — system prompt caching', () => {
+  const model = () => build(anthropicPlugin, { model: 'claude-haiku-4-5-20251001', cacheMinTokens: 1 })
+  const blocks = (...pairs: Array<[PromptBlock, string]>) =>
+    pairs.map(([block, text]) => ({ block, text }))
+
+  const marks = (content: unknown): boolean[] =>
+    (content as Array<Record<string, unknown>>).map(part => part.cache_control != null)
+
+  // Role + skills is the region every call of this role shares; packages vary with the
+  // request. Two boundaries, so a changing package block can never invalidate the skills.
+  test('breakpoints land on the stability boundaries, not on every block', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks(
+        [PromptBlock.Role, 'role text'],
+        [PromptBlock.Skills, 'skill text'],
+        [PromptBlock.Packages, 'package text'],
+      ),
+      { model: model(), cacheMax: 3, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(2)
+    expect(marks(render?.content)).toEqual([false, true, true])
+  })
+
+  // "Fully cached by default" also has to hold for a caller that has not migrated and
+  // still hands over a single system message of its own.
+  test('a lone context block is still cached', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks([PromptBlock.Context, 'legacy system message']),
+      { model: model(), cacheMax: 3, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(1)
+    expect(marks(render?.content)).toEqual([true])
+  })
+
+  test('marking stops at the budget, earliest boundary first', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks(
+        [PromptBlock.Skills, 'skill text'],
+        [PromptBlock.Packages, 'package text'],
+        [PromptBlock.Context, 'context text'],
+      ),
+      { model: model(), cacheMax: 1, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(1)
+    expect(marks(render?.content)).toEqual([true, false, false])
+  })
+
+  // An explicit `"ttl": "5m"` is different BYTES from no ttl at all, and a prefix cached
+  // one way would not match the other.
+  test('the default ttl is omitted from the marker, and 1h is spelled out', () => {
+    const short = anthropicPlugin.patchSystem?.(
+      blocks([PromptBlock.Skills, 'skill text']), { model: model(), cacheMax: 3, ttl: '5m' }
+    )
+    expect((short?.content as Array<Record<string, unknown>>)[0]!.cache_control)
+      .toEqual({ type: 'ephemeral' })
+
+    const long = anthropicPlugin.patchSystem?.(
+      blocks([PromptBlock.Skills, 'skill text']), { model: model(), cacheMax: 3, ttl: '1h' }
+    )
+    expect((long?.content as Array<Record<string, unknown>>)[0]!.cache_control)
+      .toEqual({ type: 'ephemeral', ttl: '1h' })
+  })
+
+  // A trailing context block changes every call. Marking it would pay a cache WRITE per
+  // call and never read one back — a breakpoint spent to buy nothing.
+  test('a trailing context block is never marked when stable blocks precede it', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks(
+        [PromptBlock.Role, 'role text'],
+        [PromptBlock.Skills, 'skill text'],
+        [PromptBlock.Context, 'per-call text'],
+      ),
+      { model: model(), cacheMax: 2, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(1)
+    expect(marks(render?.content)).toEqual([false, true, false])
+  })
+
+  test('the packages boundary is still marked with context trailing behind it', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks(
+        [PromptBlock.Role, 'role text'],
+        [PromptBlock.Skills, 'skill text'],
+        [PromptBlock.Packages, 'package text'],
+        [PromptBlock.Context, 'per-call text'],
+      ),
+      { model: model(), cacheMax: 2, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(2)
+    expect(marks(render?.content)).toEqual([false, true, true, false])
+  })
+
+  test('a non-claude model still renders the blocks, just without markers', () => {
+    const render = anthropicPlugin.patchSystem?.(
+      blocks([PromptBlock.Role, 'role text']),
+      { model: build(anthropicPlugin, { model: 'some-other-model' }), cacheMax: 3, ttl: '5m' },
+    )
+    expect(render?.breakpoints).toBe(0)
+    expect(render?.content).toEqual([{ type: 'text', text: 'role text' }])
+  })
+})
+
+describe('@owlmeans/llm — the four-breakpoint request budget', () => {
+  const cheap = () => build(anthropicPlugin, { model: 'claude-haiku-4-5-20251001', cacheMinTokens: 1 })
+
+  const markers = (msgs: Array<{ content: unknown }>): number =>
+    msgs.reduce((sum, msg) => sum + (Array.isArray(msg.content)
+      ? (msg.content as Array<Record<string, unknown>>).filter(b => b.cache_control != null).length
+      : 0), 0)
+
+  // The live failure: a caller that carries its message array across calls hands back
+  // messages this pipeline already marked. They accumulate until Anthropic rejects the
+  // request with `400 A maximum of 4 blocks with cache_control may be provided. Found 5.`
+  test('markers left over from a previous call are cleared before new ones are placed', () => {
+    const msgs = [
+      { role: 'system' as const, content: 'aaaa' },
+      { role: 'user' as const, content: 'bbbb' },
+      { role: 'user' as const, content: 'cccc' },
+    ]
+    // Call one marks the stable prefix.
+    anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 2 })
+    expect(markers(msgs)).toBe(1)
+
+    // Call two: the caller appends a turn and re-sends the SAME objects.
+    msgs.push({ role: 'user' as const, content: 'dddd' })
+    stripCacheMarkers(msgs)
+    expect(markers(msgs)).toBe(0)
+
+    anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 3 })
+    expect(markers(msgs)).toBe(1)
+  })
+
+  test('the system prompt plus the message prefix never exceed the provider limit', () => {
+    // Worst case: every stability boundary distinct, so the system claims its full share.
+    const system = anthropicPlugin.patchSystem?.(
+      [
+        { block: PromptBlock.Role, text: 'role text' },
+        { block: PromptBlock.Skills, text: 'skill text' },
+        { block: PromptBlock.Packages, text: 'package text' },
+        { block: PromptBlock.Context, text: 'context text' },
+      ],
+      { model: cheap(), cacheMax: 3, ttl: '5m' },
+    )
+    const reserved = system?.breakpoints ?? 0
+    expect(reserved).toBeLessThanOrEqual(3)
+
+    const msgs = [
+      { role: 'system' as const, content: system?.content as never },
+      { role: 'user' as const, content: 'bbbb' },
+      { role: 'user' as const, content: 'cccc' },
+    ]
+    anthropicPlugin.patchCache?.(msgs, { model: cheap(), useCache: true, cacheMax: 4, reserved })
+    expect(markers(msgs)).toBeLessThanOrEqual(4)
+  })
+
+  test('stripping leaves the rest of a content block untouched', () => {
+    const msgs = [{
+      role: 'user' as const,
+      content: [{ type: 'text', text: 'keep me', cache_control: { type: 'ephemeral' } }] as never,
+    }]
+    stripCacheMarkers(msgs)
+    expect(msgs[0]!.content).toEqual([{ type: 'text', text: 'keep me' }] as never)
   })
 })
 
@@ -193,6 +419,22 @@ describe('@owlmeans/llm — fatal error classification', () => {
     expect(anthropicPlugin.isFatal?.(bad)).toBe(bad)
     expect(anthropicPlugin.isFatal?.(new Error('transient'))).toBeNull()
   })
+
+  // `@langchain/anthropic` bundles its OWN nested copy of `@anthropic-ai/sdk`, so the
+  // error it throws is an instance of a different class than the one imported here and
+  // `instanceof` silently misses. That turned every fatal 400 into eight full retries.
+  test('a 400 from a foreign SDK copy is still fatal', () => {
+    const foreign = Object.assign(new Error('400 too many cache_control blocks'), { status: 400 })
+    expect(anthropicPlugin.isFatal?.(foreign)).toBe(foreign)
+    expect(openAiPlugin.isFatal?.(foreign)).toBe(foreign)
+    expect(compatiblePlugin.isFatal?.(foreign)).toBe(foreign)
+  })
+
+  test('a retryable status is not treated as fatal', () => {
+    const overloaded = Object.assign(new Error('529'), { status: 529 })
+    expect(anthropicPlugin.isFatal?.(overloaded)).toBeNull()
+    expect(openAiPlugin.isFatal?.(overloaded)).toBeNull()
+  })
 })
 
 describe('@owlmeans/llm — service', () => {
@@ -203,6 +445,29 @@ describe('@owlmeans/llm — service', () => {
     expect(service.getModel(Role.Analyst, {}, true)).not.toBe(first)
   })
 
+  // The knob an application sets where it composes its context: one place to bound how
+  // long the whole deployment waits on a silent provider.
+  test('a service-wide idle deadline reaches every model it builds', () => {
+    const service = makeLlmService(
+      { models: offlineConfigs, streamTimeout: 90_000 }, 'spec-llm-timeout'
+    )
+    const config = (service.getModel(Role.Analyst) as unknown as {
+      metadata: { config: ModelConfig }
+    }).metadata.config
+    expect(config.streamTimeout).toBe(90_000)
+  })
+
+  // A preset that names its own deadline knows something specific about that model.
+  test('a model config keeps its own deadline over the service default', () => {
+    const configs = (): ModelConfig[] =>
+      offlineConfigs().map(c => c.alias === Role.Analyst ? { ...c, streamTimeout: 12_000 } : c)
+    const service = makeLlmService({ models: configs, streamTimeout: 90_000 }, 'spec-llm-timeout-2')
+    const config = (service.getModel(Role.Analyst) as unknown as {
+      metadata: { config: ModelConfig }
+    }).metadata.config
+    expect(config.streamTimeout).toBe(12_000)
+  })
+
   test('an override participates in the cache key', () => {
     const service = makeLlmService({ models: offlineConfigs }, 'spec-llm-override')
     const plain = service.getModel(Role.Analyst)