@owlmeans/llm 0.1.15 → 0.1.16

package/src/plugins/openai.ts

@@ -33,6 +33,17 @@ export const openAiFamily = {
     json_schema: { name: toolName, schema, strict: false },
   }),
 
+  /**
+   * A 400 means the request itself is malformed — a schema the endpoint rejects, an
+   * unsupported parameter, a `max_tokens` above the model's per-request limit. Retrying
+   * re-sends the same shape (and the escalator raises `max_tokens`, making the last case
+   * strictly worse), so eight attempts only bury the real message. Matched on `status`
+   * rather than an SDK class: aggregators and nested SDK copies throw their own error
+   * types, and an `instanceof` against one of them silently never matches.
+   */
+  isFatal: (e: unknown): Error | null =>
+    (e as { status?: unknown })?.status === 400 ? e as Error : null,
+
   refine: ({ base, attempt, temperature, maxOutputCap }: LlmRefineParams): BaseChatModel => {
     const model = base as ChatOpenAI
     const currentTemperature = temperature ?? model.temperature ?? 0
@@ -75,10 +86,22 @@ export const openAiPlugin: LlmPlugin = {
   structuredMode: (config: ModelConfig): StructuredMode =>
     config.structuredOutput === false ? StructuredMode.Tool : StructuredMode.Native,
 
-  build: ({ config, secret, callbacks }) => {
+  build: ({ alias, config, secret, callbacks }) => {
     const model = config.model ??= 'gpt-5.4-mini'
     const configuration = makeConfiguration({ baseURL: undefined, headers: config.headers })
 
+    // OpenAI's prompt cache is automatic and prefix-based — there is nothing to mark. The
+    // one lever a client has is routing: requests are dispatched by a hash of the prompt's
+    // opening tokens, and `prompt_cache_key` is mixed into that hash, so requests sharing
+    // a key land on the same backend and can actually hit each other's entries. The key
+    // must be stable and low-cardinality; the config alias IS the role, which is exactly
+    // the granularity at which a system prefix is shared.
+    //
+    // Deliberately NOT done for the `compatible` plugin: aggregators there run with
+    // `provider.require_parameters`, and an unknown top-level field can exclude every
+    // serving provider from the route.
+    const modelKwargs = { prompt_cache_key: config.cacheKey ?? alias }
+
     // The Responses API models reject `temperature`/`topP`.
     if (RESPONSES_API_PREFIXES.some(prefix => model.startsWith(prefix))) {
       return new ChatOpenAI({
@@ -89,6 +112,7 @@ export const openAiPlugin: LlmPlugin = {
         useResponsesApi: true,
         metadata: { config },
         callbacks,
+        modelKwargs,
         ...configuration,
       })
     }
@@ -102,6 +126,7 @@ export const openAiPlugin: LlmPlugin = {
       maxRetries: 5,
       metadata: { config },
       callbacks,
+      modelKwargs,
       ...configuration,
     })
   },

package/src/plugins/types.ts

@@ -1,7 +1,7 @@
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import type { BaseCallbackHandler, CallbackHandlerMethods } from '@langchain/core/callbacks/base'
-import type { MessageFieldWithRole } from '@langchain/core/messages'
-import type { StructuredMode } from '@owlmeans/llm-common'
+import type { MessageContent, MessageFieldWithRole } from '@langchain/core/messages'
+import type { CacheTtl, PromptBlock, StructuredMode } from '@owlmeans/llm-common'
 import type { ModelConfig } from '../types.js'
 
 export interface LlmBuildParams {
@@ -32,7 +32,32 @@ export interface LlmCacheParams {
   /** The ORIGINAL (unrefined) model — refined instances do not always keep the name. */
   model: BaseChatModel
   useCache: boolean
+  /** How many leading messages form the stable prefix worth caching. */
   cacheMax: number
+  /** Breakpoints the composed system prompt already spent on this request. */
+  reserved?: number
+  ttl?: CacheTtl
+}
+
+/** One rendered section of the composed system prompt, in emission order. */
+export interface LlmSystemBlock {
+  block: PromptBlock
+  text: string
+}
+
+export interface LlmSystemCacheParams {
+  /** The ORIGINAL (unrefined) model. */
+  model: BaseChatModel
+  /** Breakpoint budget the system prompt may spend. */
+  cacheMax: number
+  ttl: CacheTtl
+}
+
+/** Provider-native rendering of the composed system prompt. */
+export interface LlmSystemRender {
+  content: MessageContent
+  /** Breakpoints consumed — subtracted from the message-level budget. */
+  breakpoints: number
 }
 
 /**
@@ -78,12 +103,24 @@ export interface LlmPlugin {
   responseFormat?: (toolName: string, schema: unknown) => Record<string, unknown>
 
   /**
-   * Mark leading messages as cacheable in-place. Returns `true` when caching was
-   * actually applied (the model may not support it). Omit for providers with no
-   * explicit prompt-cache markers.
+   * Mark the stable message prefix as cacheable, in-place. ONE breakpoint at the end of
+   * that prefix — not one per message: the budget is four per request and the system
+   * prompt has first claim on it. Returns `true` when a marker was actually placed.
+   * Omit for providers with no explicit prompt-cache markers.
    */
   patchCache?: (msgs: MessageFieldWithRole[], params: LlmCacheParams) => boolean
 
+  /**
+   * Render the composed system blocks into provider-native content, placing cache
+   * breakpoints at the stability boundaries between blocks (see `PromptBlock`).
+   *
+   * Return `null` — or omit the method entirely — for providers whose prompt cache is
+   * automatic and prefix-based (OpenAI and friends): the service then joins the blocks
+   * into a plain string, which is all those providers need, since the block ORDER is
+   * what makes their prefix stable.
+   */
+  patchSystem?: (blocks: LlmSystemBlock[], params: LlmSystemCacheParams) => LlmSystemRender | null
+
   /**
    * Classify a thrown error as fatal for the retry loop. Return the error to throw
    * immediately, or `null` to let it be retried.

package/src/prompt/index.ts

@@ -0,0 +1,5 @@
+
+export type * from './types.js'
+export * from './render.js'
+export * from './plugins.js'
+export * from './service.js'

package/src/prompt/plugins.ts

@@ -0,0 +1,69 @@
+import { PromptBlock } from '@owlmeans/llm-common'
+import type { SkillDefinition } from '@owlmeans/llm-common'
+import { joinChunks, renderSkill, sortSkills } from './render.js'
+import type { LlmPromptPlugin } from './types.js'
+
+/**
+ * Block 0 — the base system prompt that tells the model who it is.
+ *
+ * The most stable thing in the whole request, so it goes first and every cache boundary
+ * sits behind it.
+ */
+export const rolePlugin: LlmPromptPlugin = {
+  alias: 'role',
+  order: 0,
+  compose: ctx => {
+    if (ctx.input.role != null && ctx.input.role.trim() !== '') {
+      ctx.add(PromptBlock.Role, ctx.input.role)
+    }
+  },
+}
+
+/**
+ * Block 1 — the declared capabilities, rendered in a deterministic order.
+ *
+ * Registry skills and inline skills are merged by alias with inline winning, so a caller
+ * can override one registered entry without forking the catalogue.
+ */
+export const skillsPlugin: LlmPromptPlugin = {
+  alias: 'skills',
+  order: 10,
+  compose: ctx => {
+    const declared = ctx.resolve(ctx.input.skills ?? [])
+    const merged = new Map<string, SkillDefinition>()
+    for (const skill of [...declared, ...(ctx.input.inline ?? [])]) {
+      merged.set(skill.alias, skill)
+    }
+    for (const skill of sortSkills([...merged.values()])) {
+      ctx.add(skill.block ?? PromptBlock.Skills, renderSkill(skill))
+    }
+  },
+}
+
+/**
+ * Block 3 — whatever the caller handed over verbatim, plus any skills requested for this
+ * one call. Emitted last and never marked cacheable: its content varies per request by
+ * definition, and a varying tail must not sit inside a prefix other calls depend on.
+ */
+export const contextPlugin: LlmPromptPlugin = {
+  alias: 'context',
+  order: 90,
+  compose: ctx => {
+    // Everything volatile is merged into ONE chunk rather than added piece by piece: the
+    // block carries no cache breakpoint, so there is nothing to gain from keeping the
+    // parts separable, and a single contiguous section reads as one instruction to the
+    // model instead of a pile of loose fragments.
+    const parts = [
+      ...sortSkills(ctx.resolve(ctx.input.callSkills ?? [])).map(renderSkill),
+      ...(ctx.input.context ?? []),
+    ]
+    if (parts.length > 0) {
+      ctx.add(PromptBlock.Context, joinChunks(parts))
+    }
+  },
+}
+
+/** The plugins every {@link PromptService} starts with, in run order. */
+export const BUILT_IN_PROMPT_PLUGINS: readonly LlmPromptPlugin[] = [
+  rolePlugin, skillsPlugin, contextPlugin,
+] as const

package/src/prompt/render.ts

@@ -0,0 +1,48 @@
+import { DEFAULT_SKILL_ORDER } from '@owlmeans/llm-common'
+import type { SkillDefinition } from '@owlmeans/llm-common'
+
+/**
+ * Separator between rendered chunks. Every join in this file goes through it: the
+ * composed prompt must be byte-identical between calls, so there is exactly one way to
+ * glue things together.
+ */
+export const CHUNK_SEPARATOR = '\n\n'
+
+/**
+ * Code-unit comparison, NOT `localeCompare`.
+ *
+ * `localeCompare` orders differently depending on the host's ICU data and locale, so two
+ * processes could render the same skill set in different orders and never share a cache
+ * entry. Skill aliases are ASCII slugs; a plain comparison is both correct and stable.
+ */
+export const compareAlias = (a: string, b: string): number => a < b ? -1 : a > b ? 1 : 0
+
+/** Deterministic skill order: declared weight first, alias as the tiebreaker. */
+export const sortSkills = (skills: readonly SkillDefinition[]): SkillDefinition[] =>
+  [...skills].sort((a, b) => {
+    const left = a.order ?? DEFAULT_SKILL_ORDER
+    const right = b.order ?? DEFAULT_SKILL_ORDER
+    return left !== right ? left - right : compareAlias(a.alias, b.alias)
+  })
+
+/** Fixed rendering of one skill. Changing this shape invalidates every cached prefix. */
+export const renderSkill = (skill: SkillDefinition): string =>
+  `## ${skill.title ?? skill.alias}\n\n${skill.body.trim()}`
+
+/** Join rendered chunks into one block, dropping empties. */
+export const joinChunks = (parts: readonly string[]): string =>
+  parts.map(part => part.trim()).filter(part => part !== '').join(CHUNK_SEPARATOR)
+
+/**
+ * Stable digest of a cache prefix — FNV-1a, so there is no crypto dependency and the
+ * result is identical on every runtime. Used as a provider cache-routing key (OpenAI's
+ * `prompt_cache_key`), never for security.
+ */
+export const prefixHash = (text: string): string => {
+  let hash = 0x811c9dc5
+  for (let i = 0; i < text.length; i++) {
+    hash ^= text.charCodeAt(i)
+    hash = Math.imul(hash, 0x01000193) >>> 0
+  }
+  return hash.toString(36)
+}

package/src/prompt/service.ts

@@ -0,0 +1,194 @@
+import { createService } from '@owlmeans/context'
+import type { BasicConfig, BasicContext } from '@owlmeans/context'
+import { PROMPT_BLOCK_ORDER, PromptBlock } from '@owlmeans/llm-common'
+import type { SkillDefinition } from '@owlmeans/llm-common'
+import {
+  DEFAULT_CACHE_TTL, MAX_CACHE_BREAKPOINTS, MAX_SYSTEM_BREAKPOINTS, PROMPT_SERVICE,
+} from '../consts.js'
+import type { LlmSystemBlock } from '../plugins/types.js'
+import { BUILT_IN_PROMPT_PLUGINS } from './plugins.js'
+import { CHUNK_SEPARATOR, compareAlias, joinChunks } from './render.js'
+import type {
+  LlmPromptPlugin, PromptContext, PromptResult, PromptService, PromptServiceOptions,
+  WithPromptService,
+} from './types.js'
+
+/** Sort weight of a plugin that declares none — between the built-in skills and context. */
+const DEFAULT_PLUGIN_ORDER = 50
+
+/** The part of {@link PromptService} this package implements — see {@link promptServiceApi}. */
+export type PromptServiceApi =
+  Pick<PromptService, 'use' | 'register' | 'has' | 'resolve' | 'skills' | 'compose'>
+
+/**
+ * Build the skill registry and composition chain WITHOUT registering a context service,
+ * so a consumer can spread it into its own `createService` and publish extra methods
+ * alongside it — the same pattern as `llmServiceApi` / `executionServiceApi`.
+ *
+ * `self` is late-bound because plugins resolve skills through the finished service, which
+ * a consumer may have extended.
+ */
+export const promptServiceApi = (
+  options: PromptServiceOptions,
+  self: () => PromptService,
+): PromptServiceApi => {
+  const registry = new Map<string, SkillDefinition>()
+  /** Registration index per plugin alias — the stable tiebreaker for equal `order`. */
+  const seats = new Map<string, { plugin: LlmPromptPlugin; index: number }>()
+  let seq = 0
+
+  const seat = (plugin: LlmPromptPlugin): void => {
+    const existing = seats.get(plugin.alias)
+    // Re-registering under the same alias REPLACES rather than appends, so wiring the
+    // same plugin twice (a shared context builder plus an app) cannot double-emit.
+    seats.set(plugin.alias, { plugin, index: existing?.index ?? seq++ })
+  }
+
+  for (const plugin of [...BUILT_IN_PROMPT_PLUGINS, ...(options.plugins ?? [])]) {
+    seat(plugin)
+  }
+
+  const ordered = (): LlmPromptPlugin[] =>
+    [...seats.values()]
+      .sort((a, b) => {
+        const left = a.plugin.order ?? DEFAULT_PLUGIN_ORDER
+        const right = b.plugin.order ?? DEFAULT_PLUGIN_ORDER
+        return left !== right ? left - right : a.index - b.index
+      })
+      .map(entry => entry.plugin)
+
+  const api: PromptServiceApi = {
+
+    use: plugin => {
+      seat(plugin)
+    },
+
+    register: (...skills) => {
+      for (const skill of skills) {
+        registry.set(skill.alias, skill)
+      }
+    },
+
+    has: alias => registry.has(alias),
+
+    skills: () => [...registry.values()].sort((a, b) => compareAlias(a.alias, b.alias)),
+
+    /**
+     * Depth-first over `requires` so a dependency is emitted before the skill that pulled
+     * it in. Unknown aliases are skipped rather than thrown: a skill catalogue is often
+     * assembled from several packages and a missing optional one should degrade the
+     * prompt, not break the call.
+     */
+    resolve: aliases => {
+      const seen = new Set<string>()
+      const out: SkillDefinition[] = []
+      const walk = (alias: string): void => {
+        if (seen.has(alias)) {
+          return
+        }
+        seen.add(alias)
+        const skill = registry.get(alias)
+        if (skill == null) {
+          return
+        }
+        for (const required of skill.requires ?? []) {
+          walk(required)
+        }
+        out.push(skill)
+      }
+      for (const alias of aliases) {
+        walk(alias)
+      }
+
+      return out
+    },
+
+    compose: async (input, messages, params): Promise<PromptResult> => {
+      const sections = new Map<PromptBlock, string[]>()
+      const ctx: PromptContext = {
+        ...params,
+        input,
+        messages,
+        add: (block, text) => {
+          const trimmed = text.trim()
+          if (trimmed === '') {
+            return
+          }
+          const chunks = sections.get(block)
+          if (chunks == null) {
+            sections.set(block, [trimmed])
+          } else {
+            chunks.push(trimmed)
+          }
+        },
+        resolve: aliases => self().resolve(aliases),
+      }
+
+      // Two passes, not one: every static contribution must be in place before a plugin
+      // that reacts to the messages runs, so detection can see what is already covered.
+      const chain = ordered()
+      for (const plugin of chain) {
+        await plugin.compose?.(ctx)
+      }
+      for (const plugin of chain) {
+        await plugin.inspect?.(ctx)
+      }
+
+      const blocks: LlmSystemBlock[] = []
+      for (const block of PROMPT_BLOCK_ORDER) {
+        const text = joinChunks(sections.get(block) ?? [])
+        if (text !== '') {
+          blocks.push({ block, text })
+        }
+      }
+      if (blocks.length === 0) {
+        return { system: null, breakpoints: 0, blocks }
+      }
+
+      const cacheSystem = input.cacheSystem ?? options.cacheSystem ?? true
+      const ttl = input.cacheTtl ?? options.cacheTtl ?? DEFAULT_CACHE_TTL
+      const budget = Math.min(params.cacheMax ?? MAX_CACHE_BREAKPOINTS, MAX_SYSTEM_BREAKPOINTS)
+      const render = cacheSystem && budget > 0
+        ? params.provider?.patchSystem?.(blocks, { model: params.model, cacheMax: budget, ttl })
+        : null
+
+      return {
+        system: {
+          role: 'system',
+          content: render?.content ?? blocks.map(block => block.text).join(CHUNK_SEPARATOR),
+        },
+        breakpoints: render?.breakpoints ?? 0,
+        blocks,
+      }
+    },
+  }
+
+  api.register(...(options.skills ?? []))
+
+  return api
+}
+
+export const makePromptService = (
+  options: PromptServiceOptions = {},
+  alias: string = PROMPT_SERVICE,
+): PromptService => {
+  const service: PromptService = createService<PromptService>(
+    alias, promptServiceApi(options, () => service) as PromptService
+  )
+
+  return service
+}
+
+export const appendPromptService = <C extends BasicConfig, T extends BasicContext<C>>(
+  ctx: T,
+  options: PromptServiceOptions = {},
+  alias: string = PROMPT_SERVICE,
+): T & WithPromptService => {
+  const context = ctx as T & WithPromptService
+
+  context.registerService(makePromptService(options, alias))
+
+  context.prompts = () => context.service<PromptService>(alias)
+
+  return context
+}

package/src/prompt/types.ts

@@ -0,0 +1,114 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import type { MessageFieldWithRole } from '@langchain/core/messages'
+import type { InitializedService } from '@owlmeans/context'
+import type {
+  CacheTtl, FileProviderRef, LlmPurpose, PromptBlock, PromptPolicy, SkillDefinition,
+} from '@owlmeans/llm-common'
+import type { LlmPlugin, LlmSystemBlock } from '../plugins/types.js'
+
+/**
+ * Everything that shapes one composed system prompt. The serializable part
+ * ({@link PromptPolicy}) travels on the execution state; the rest is per-call.
+ */
+export interface PromptInput extends PromptPolicy {
+  /** Skills supplied verbatim, without registering them first. */
+  inline?: SkillDefinition[]
+  /**
+   * Raw text for the volatile `Context` block — a caller's own system message, carried
+   * through unchanged. Never part of the cached prefix.
+   */
+  context?: string[]
+  /**
+   * Skill aliases requested for THIS call only. Rendered into `Context`, not `Skills`:
+   * a set that changes per call must not sit inside the region other calls cache.
+   */
+  callSkills?: string[]
+}
+
+export interface PromptComposeParams {
+  /** The ORIGINAL (unrefined) model the composed prompt will be sent to. */
+  model: BaseChatModel
+  /** Provider plugin governing the call — supplies the cache-marker strategy. */
+  provider?: LlmPlugin
+  purpose?: LlmPurpose
+  action?: string
+  /** Total breakpoint budget for the request. */
+  cacheMax?: number
+  /** File access a plugin may use to resolve knowledge from disk. */
+  files?: FileProviderRef
+}
+
+/** What a prompt plugin sees and may contribute to. */
+export interface PromptContext extends PromptComposeParams {
+  input: PromptInput
+  /** The call's messages. Read them for detection; do NOT mutate. */
+  messages: readonly MessageFieldWithRole[]
+  /** Append a rendered chunk to a block. Empty text is ignored. */
+  add: (block: PromptBlock, text: string) => void
+  /** Resolve skill aliases through the registry, following `requires`. */
+  resolve: (aliases: readonly string[]) => SkillDefinition[]
+}
+
+/**
+ * A unit of system-prompt composition, registered on the {@link PromptService} at context
+ * composition time. This is the seam that makes the LLM layer pluggable: the package
+ * ships the role and skills plugins, an application adds its own.
+ *
+ * A plugin MUST be deterministic — same input, same bytes. Anything it contributes to a
+ * cached block and cannot reproduce exactly invalidates the prefix for every call that
+ * shares it.
+ */
+export interface LlmPromptPlugin {
+  alias: string
+  /** Lower runs first. The built-ins occupy 0 (role), 10 (skills) and 90 (context). */
+  order?: number
+  /** Contribute static content, before anything has looked at the messages. */
+  compose?: (ctx: PromptContext) => void | Promise<void>
+  /** Contribute content derived from the messages (detection, lookup, fetch). */
+  inspect?: (ctx: PromptContext) => void | Promise<void>
+}
+
+export interface PromptResult {
+  /** The composed system message, or `null` when nothing was contributed. */
+  system: MessageFieldWithRole | null
+  /** Breakpoints the system prompt consumed; subtract from the message budget. */
+  breakpoints: number
+  /** The rendered blocks in emission order — exposed for tests and diagnostics. */
+  blocks: LlmSystemBlock[]
+}
+
+export interface PromptServiceOptions {
+  /** Skills registered up front; equivalent to calling `register` after construction. */
+  skills?: SkillDefinition[]
+  /** Plugins registered up front, in addition to the built-ins. */
+  plugins?: LlmPromptPlugin[]
+  /** Default for `PromptPolicy.cacheSystem`. Defaults to `true`. */
+  cacheSystem?: boolean
+  /** Default for `PromptPolicy.cacheTtl`. */
+  cacheTtl?: CacheTtl
+}
+
+/**
+ * Registry of reusable skills plus the plugin chain that turns a role and a set of skill
+ * aliases into a cache-friendly system prompt.
+ */
+export interface PromptService extends InitializedService {
+  /** Register a composition plugin. Runs in `order` then registration order. */
+  use: (plugin: LlmPromptPlugin) => void
+  /** Register (or replace, by alias) reusable skills. */
+  register: (...skills: SkillDefinition[]) => void
+  has: (alias: string) => boolean
+  /** Resolve aliases to definitions, pulling in `requires` transitively. Unknown aliases are skipped. */
+  resolve: (aliases: readonly string[]) => SkillDefinition[]
+  /** Every registered skill, in deterministic order. */
+  skills: () => SkillDefinition[]
+  compose: (
+    input: PromptInput,
+    messages: readonly MessageFieldWithRole[],
+    params: PromptComposeParams,
+  ) => Promise<PromptResult>
+}
+
+export interface WithPromptService {
+  prompts: () => PromptService
+}

package/src/service.ts

@@ -51,6 +51,9 @@ export const llmServiceApi = (options: LlmServiceOptions, self: () => LlmService
       throw new LlmMissconfiguredError(alias)
     }
     const config: ModelConfig = { ...baseConfig, ...override }
+    // The service-wide idle deadline is a floor, not an override: a preset that states its
+    // own `streamTimeout` knows something specific about that model and keeps it.
+    config.streamTimeout ??= options.streamTimeout
     const preset: Partial<ModelConfig> = config.preset != null
       ? { ...(models.find(m => m.alias === config.preset) ?? {}) }
       : {}

package/src/types.ts

@@ -4,8 +4,10 @@ import type { BaseCallbackHandler, CallbackHandlerMethods } from '@langchain/cor
 import type { JSONSchemaType } from 'ajv'
 import type { InitializedService } from '@owlmeans/context'
 import type {
-  LlmPurpose, ModelProvider, NullCapture, SpectatorArgument, SpectatorEntryLogged,
+  FileProviderRef, LlmPurpose, ModelProvider, NullCapture, SpectatorArgument,
+  SpectatorEntryLogged,
 } from '@owlmeans/llm-common'
+import type { PromptInput, PromptService } from './prompt/types.js'
 
 export type MaybeArray<T> = T | T[]
 
@@ -47,6 +49,20 @@ export interface LlmLogging {
 export interface LlmModelOptions extends LlmLogging {
   model: BaseChatModel
   retries?: number
+  /**
+   * Role and skills for every call this model makes. Composed into a cacheable system
+   * prompt by {@link PromptService}; ignored when no `prompts` resolver is supplied.
+   * Usually just `exec.prompt` from the helper execution.
+   */
+  prompt?: PromptInput
+  /**
+   * Late-bound resolver for the prompt service — a function, like `Execution.models`, so
+   * the service can be swapped or cloned. Without it the model behaves exactly as before
+   * this layer existed: the caller's messages are sent untouched.
+   */
+  prompts?: () => PromptService
+  /** File access offered to prompt plugins that resolve knowledge from disk. */
+  files?: FileProviderRef
 }
 
 export type ModelMessage = BaseMessage | MessageFieldWithRole
@@ -56,10 +72,23 @@ export type ModelInput = MaybeArray<ModelInputItem>
 export interface LlmCallOptions {
   /** Short name of the operation — used as the LangChain run name and in spectator entries. */
   action: string
-  /** Ask the provider to cache the prompt prefix (no-op for providers without prompt caching). */
+  /**
+   * Cache the leading MESSAGES too. The composed system prompt is cached by default and
+   * independently of this flag — this one is about the conversation prefix, which is only
+   * worth caching when the same leading messages recur across calls.
+   */
   useCache?: boolean
-  /** How many leading messages to mark as cacheable (capped by the provider's own limit). */
+  /**
+   * How many leading messages form the stable prefix. One breakpoint is placed at its
+   * end (not one per message), capped by whatever the system prompt left unspent.
+   */
   cacheMax?: number
+  /**
+   * Skill aliases for THIS call only. Rendered into the volatile `Context` block, so they
+   * never disturb the cached region — declare a skill on the execution instead when it
+   * should be part of the shared prefix.
+   */
+  skills?: string[]
 }
 
 export interface LlmAskOptions extends LlmCallOptions {
@@ -183,11 +212,33 @@ export interface ModelConfig {
    * rotating providers mid-call would flip the structured-output format.
    */
   fallback?: Partial<ModelConfig>
+  /**
+   * Per-model override of {@link MIN_CACHEABLE_TOKENS} — the shortest prefix worth a
+   * cache breakpoint. Anthropic's own minimum is model-dependent and NOT monotonic
+   * across generations (512 on the newest, 1024 on most, 4096 on a few older ones), so a
+   * preset that pins an old model should raise this rather than pay for markers that
+   * silently never cache.
+   */
+  cacheMinTokens?: number
+  /**
+   * Cache-routing key for providers whose prompt cache is automatic (OpenAI's
+   * `prompt_cache_key`): requests sharing a key are routed to the same backend, which
+   * raises the hit rate for a shared prefix. Must be STABLE and low-cardinality — one
+   * value per role, never per user or per request. Defaults to the config alias.
+   */
+  cacheKey?: string
 }
 
 export interface LlmServiceOptions {
   /** The full config list; resolved by `alias` on every `getModel` call. */
   models: () => ModelConfig[]
+  /**
+   * Idle deadline (ms) applied to every model this service builds, unless the model's own
+   * config overrides it. This is the knob an application sets where it composes its
+   * context — one place to tune how long the whole deployment waits on a silent provider,
+   * without touching a preset. Falls back to {@link MODEL_STREAM_TIMEOUT_MS}.
+   */
+  streamTimeout?: number
 }
 
 /**