@owlmeans/llm 0.1.14 → 0.1.16-rc.0

package/src/consts.ts

@@ -1,5 +1,5 @@
 import { ExecutionEffort } from '@owlmeans/llm-common'
-import type { ModelConfigPatch } from '@owlmeans/llm-common'
+import type { CacheTtl, ModelConfigPatch } from '@owlmeans/llm-common'
 
 /** Context-service alias for the {@link LlmService} (model factory / registry). */
 export const LLM_SERVICE = 'owlmeans-llm-service'
@@ -7,6 +7,9 @@ export const LLM_SERVICE = 'owlmeans-llm-service'
 /** Context-service alias for the {@link ExecutionService}. */
 export const EXECUTION_SERVICE = 'owlmeans-llm-execution-service'
 
+/** Context-service alias for the {@link PromptService} (skill registry + composition). */
+export const PROMPT_SERVICE = 'owlmeans-llm-prompt-service'
+
 /** Default number of attempts a single model call makes before giving up. */
 export const DEFAULT_MODEL_RETRIES = 8
 
@@ -17,9 +20,15 @@ export const DEFAULT_MODEL_RETRIES = 8
  * against a provider that accepts the request and then never streams anything (observed
  * with throughput-sorted OpenRouter routing), which would otherwise block forever —
  * `maxRetries` never helps there because the request never errors, it just hangs.
- * Overridable per model via `ModelConfig.streamTimeout`.
+ * Overridable per model via `ModelConfig.streamTimeout`, or for every model at once via
+ * `LlmServiceOptions.streamTimeout` where the application composes its context.
+ *
+ * Three minutes, not longer: the timer resets on every token, so a legitimately long
+ * generation is never at risk — this only bounds SILENCE. The cost of a high value is
+ * paid entirely by hung calls, and a hang that takes five minutes to notice is a hang
+ * that can stall an agent run for half an hour once retries multiply it.
  */
-export const MODEL_STREAM_TIMEOUT_MS = 5 * 60 * 1000
+export const MODEL_STREAM_TIMEOUT_MS = 3 * 60 * 1000
 
 /**
  * Number of failed attempts after which the retry escalator switches from a role's
@@ -37,9 +46,38 @@ export const FALLBACK_AFTER_ATTEMPTS = 3
  */
 export const DEFAULT_MAX_OUTPUT_CAP = 3 * 64000
 
-/** Provider hard limit on prompt-cache breakpoints (Anthropic). */
+/** Provider hard limit on prompt-cache breakpoints per REQUEST (Anthropic). */
 export const MAX_CACHE_BREAKPOINTS = 4
 
+/**
+ * Share of {@link MAX_CACHE_BREAKPOINTS} the composed system prompt may spend.
+ *
+ * Two is all it can use: the only STABLE boundaries are the end of role+skills and the end
+ * of the packages block. The trailing context block is volatile and deliberately never
+ * marked, so the remaining two breakpoints always stay available to the message prefix.
+ */
+export const MAX_SYSTEM_BREAKPOINTS = 2
+
+/** Cache lifetime used when neither the call nor the service asks for another. */
+export const DEFAULT_CACHE_TTL: CacheTtl = '5m'
+
+/**
+ * Smallest prefix worth marking as cacheable, in tokens. Providers silently refuse to
+ * create an entry below their own threshold (1024 tokens on most Claude models, 512 on
+ * the newest, 4096 on a few older ones — it is NOT monotonic across generations), so a
+ * marker on a short prefix costs nothing but wastes a breakpoint and produces a
+ * "caching enabled" log for an entry that was never written. Override per model with
+ * `ModelConfig.cacheMinTokens`.
+ */
+export const MIN_CACHEABLE_TOKENS = 1024
+
+/**
+ * Characters per token used to size a prefix against {@link MIN_CACHEABLE_TOKENS}. A rough
+ * average for English prose and TypeScript; only ever used to decide whether marking is
+ * worth a breakpoint, never for billing or budgeting.
+ */
+export const CHARS_PER_TOKEN = 4
+
 /**
  * Appended to the prompt of `invoke`/`request` when no message already mentions JSON.
  * Some providers refuse or ignore JSON modes unless the word appears in the prompt;
@@ -85,5 +123,5 @@ export const EFFORT_TABLE: Record<ExecutionEffort, ModelConfigPatch> = {
  * without excluding it every `derive`/`escalate`/`withPurpose` would nest another copy.
  */
 export const COLLABORATOR_KEYS: string[] = [
-  'state', 'models', 'model', 'temperatureFactory', 'outputErrors',
+  'state', 'models', 'model', 'temperatureFactory', 'outputErrors', 'files', 'prompts',
 ]

package/src/execution/service.ts

@@ -9,7 +9,8 @@ import type {
   HelperExecution, TaskExecution, WithExecutionService,
 } from './types.js'
 import {
-  composeExecState, composeTaskState, effortPatch, freeze, mergeOverride, mergePolicy, resolveRole,
+  composeExecState, composeTaskState, effortPatch, freeze, mergeOverride, mergePolicy,
+  mergePrompt, resolveRole,
 } from './utils.js'
 
 /**
@@ -50,18 +51,21 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
       level: ExecutionLevel.Project,
       purpose: { ...input.purpose },
       policy: { ...input.policy },
+      ...(input.prompt != null ? { prompt: { ...input.prompt } } : {}),
     }) as S['project'],
 
     forTask: (parent, input) => {
-      const { effort, phase, data, ...extras } = input
+      const { effort, phase, data, prompt, ...extras } = input
       const policy = effort != null
         ? mergePolicy(parent.policy, { effort })
         : { ...parent.policy }
+      const merged = mergePrompt(parent.prompt, prompt)
 
       // Spreading the parent carries every collaborator and domain field forward; the
       // task's own state is composed afterwards, from the seeded resumable fields.
       const taskExec = {
         ...parent, ...extras, level: ExecutionLevel.Task, purpose: { ...parent.purpose }, policy,
+        ...(merged != null ? { prompt: merged } : {}),
       } as unknown as TaskExecution
       ;(taskExec as { state: TaskExecutionState }).state = composeTaskState({
         ...taskExec,
@@ -78,9 +82,10 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
     },
 
     forHelper: (parent, input) => {
-      const { role, effort, dedication, ...extras } = input
+      const { role, effort, dedication, prompt, ...extras } = input
       const localPolicy = effort != null ? mergePolicy(parent.policy, { effort }) : parent.policy
       const scoped = { ...parent, policy: localPolicy } as S['exec']
+      const merged = mergePrompt(parent.prompt, prompt)
 
       const helperExec = {
         ...parent,
@@ -90,6 +95,7 @@ export const executionServiceApi = <S extends ExecutionShape = ExecutionShape>(
           ? { ...parent.purpose, dedication }
           : { ...parent.purpose },
         policy: localPolicy,
+        ...(merged != null ? { prompt: merged } : {}),
         role: resolveRole(localPolicy, role),
         model: self().model(scoped, role),
         temperatureFactory: self().temperatureFactory(scoped, role),

package/src/execution/types.ts

@@ -1,10 +1,11 @@
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import type { InitializedService } from '@owlmeans/context'
 import type {
-  ExecutionEffort, ExecutionLevel, ExecutionState, LlmPurpose, ModelConfigOverride,
-  ModelPolicy, ModelRole, TaskExecutionState,
+  ExecutionEffort, ExecutionLevel, ExecutionState, FileProviderRef, LlmPurpose,
+  ModelConfigOverride, ModelPolicy, ModelRole, PromptPolicy, TaskExecutionState,
 } from '@owlmeans/llm-common'
 import type { LlmService, TemperatureFactory } from '../types.js'
+import type { PromptService } from '../prompt/types.js'
 
 /**
  * Runtime execution = serializable {@link ExecutionState} + attached collaborators.
@@ -18,6 +19,10 @@ import type { LlmService, TemperatureFactory } from '../types.js'
 export interface Execution extends ExecutionState {
   /** Resolver for the model factory — a function so the service can be swapped/cloned. */
   models: () => LlmService
+  /** Resolver for the skill registry / prompt composer. Same late-binding rationale. */
+  prompts?: () => PromptService
+  /** File access offered to prompt plugins. Declared a collaborator, never snapshotted. */
+  files?: FileProviderRef
   outputErrors?: boolean
   captureNull?: boolean
 }
@@ -42,8 +47,12 @@ export interface HelperExecution extends Execution {
 
 export interface ProjectExecutionInput {
   models: () => LlmService
+  prompts?: () => PromptService
+  files?: FileProviderRef
   policy: ModelPolicy
   purpose: LlmPurpose
+  /** Baseline role and skills for the whole run. */
+  prompt?: PromptPolicy
   outputErrors?: boolean
   captureNull?: boolean
 }
@@ -51,15 +60,23 @@ export interface ProjectExecutionInput {
 export interface TaskExecutionInput {
   /** Raise (or lower) the effort tier for this task and everything derived from it. */
   effort?: ExecutionEffort
+  /** Skills (and optionally a role) layered on top of the project's. Skills accumulate. */
+  prompt?: PromptPolicy
   /** Optional seeds for the resumable task state. */
   phase?: string
   data?: Record<string, unknown>
 }
 
 export interface HelperExecutionInput {
+  /**
+   * Which MODEL to use. Distinct from `prompt.role`, which is the system-prompt text
+   * defining the persona — one selects hardware, the other writes the job description.
+   */
   role: ModelRole
   /** Local effort bump without escalating the whole branch. */
   effort?: ExecutionEffort
+  /** The helper's persona and its own skills, layered on top of the task's. */
+  prompt?: PromptPolicy
   /** Refines `purpose.dedication`. */
   dedication?: string
 }
@@ -67,6 +84,8 @@ export interface HelperExecutionInput {
 /** Collaborators re-attached to a state that was restored from storage. */
 export interface RestoreCollaborators {
   models?: () => LlmService
+  prompts?: () => PromptService
+  files?: FileProviderRef
 }
 
 /**

package/src/execution/utils.ts

@@ -1,7 +1,7 @@
 import { ExecutionLevel } from '@owlmeans/llm-common'
 import type {
   ExecutionEffort, ExecutionState, ModelConfigOverride, ModelConfigPatch,
-  ModelPolicy, ModelRole, TaskExecutionState,
+  ModelPolicy, ModelRole, PromptPolicy, TaskExecutionState,
 } from '@owlmeans/llm-common'
 import { EFFORT_TABLE } from '../consts.js'
 import type { Execution, TaskExecution } from './types.js'
@@ -19,6 +19,33 @@ export const mergePolicy = (base: ModelPolicy, patch: Partial<ModelPolicy>): Mod
     : undefined,
 })
 
+/**
+ * Overlay a prompt policy onto the one inherited from the parent level.
+ *
+ * Skills ACCUMULATE — a task adds to what the project declared, a helper adds to the
+ * task — because that is how a capability set is built up as work narrows. The role is
+ * replaced instead: the deepest level that names one owns the persona.
+ *
+ * The union preserves first-seen order and de-duplicates, so the composed prompt is
+ * byte-identical no matter how many levels contributed the same skill.
+ */
+export const mergePrompt = (
+  base: PromptPolicy | undefined,
+  patch: PromptPolicy | undefined,
+): PromptPolicy | undefined => {
+  if (base == null && patch == null) {
+    return undefined
+  }
+  const skills = [...new Set([...(base?.skills ?? []), ...(patch?.skills ?? [])])]
+
+  return {
+    ...base,
+    ...patch,
+    ...(base?.role != null || patch?.role != null ? { role: patch?.role ?? base?.role } : {}),
+    ...(skills.length > 0 ? { skills } : {}),
+  }
+}
+
 /** Apply the policy's role→role remap. */
 export const resolveRole = (policy: ModelPolicy, role: ModelRole): ModelRole =>
   (policy.roleOverrides?.[role] as ModelRole | undefined) ?? role

package/src/helpers/cache.ts

@@ -0,0 +1,33 @@
+import type { UsageMetadata } from '@langchain/core/messages'
+import type { CacheUsage } from '@owlmeans/llm-common'
+
+/** LangChain normalizes every provider's cache accounting into these two fields. */
+interface InputTokenDetails {
+  cache_read?: number
+  cache_creation?: number
+}
+
+/**
+ * Prompt-cache accounting for one completion.
+ *
+ * This is the only honest answer to "is caching actually working". A composed prefix can
+ * look perfectly stable and still miss on every call — a stray timestamp, a set iterated
+ * in a different order, a tool list rebuilt per request. If `read` stays at zero across
+ * repeated calls that share a prefix, something is invalidating it; diff the rendered
+ * blocks between two calls to find out what.
+ */
+export const readCacheUsage = (message: { usage_metadata?: UsageMetadata }): CacheUsage => {
+  const usage = message.usage_metadata
+  const details = usage?.input_token_details as InputTokenDetails | undefined
+
+  return {
+    read: details?.cache_read ?? 0,
+    creation: details?.cache_creation ?? 0,
+    input: usage?.input_tokens ?? 0,
+    output: usage?.output_tokens ?? 0,
+  }
+}
+
+/** `true` when the provider reported any cache activity at all. */
+export const hasCacheActivity = (usage: CacheUsage): boolean =>
+  usage.read > 0 || usage.creation > 0

package/src/helpers/index.ts

@@ -3,3 +3,4 @@ export * from './retry.js'
 export * from './json.js'
 export * from './messages.js'
 export * from './spectate.js'
+export * from './cache.js'

package/src/helpers/spectate.ts

@@ -3,6 +3,7 @@ import type { UsageMetadata } from '@langchain/core/messages'
 import { SpectatorContentType } from '@owlmeans/llm-common'
 import type { SpectatorEntryMessage } from '@owlmeans/llm-common'
 import type { LlmSpectator, ModelInputItem } from '../types.js'
+import { hasCacheActivity, readCacheUsage } from './cache.js'
 
 /** Normalize one prompt message into the spectator's storage shape. */
 const describeInput = (msg: ModelInputItem, callType: string): SpectatorEntryMessage => {
@@ -63,5 +64,14 @@ export const spectate = (spectator: LlmSpectator, callType: string) =>
       completion.content = message.content as unknown as string
     }
 
+    // Silent unless the provider reported cache activity, so it costs nothing when
+    // caching is off — and is the one signal that tells a stable prefix from a broken one.
+    const cache = readCacheUsage(message)
+    if (hasCacheActivity(cache)) {
+      console.log(
+        `Prompt cache [${action}]: read ${cache.read}, written ${cache.creation}, uncached ${cache.input}`
+      )
+    }
+
     return spectator.log({ action, retries, startedAt, messages: [...messages, completion] })
   }

package/src/index.ts

@@ -6,6 +6,7 @@ export * from './model.js'
 export * from './service.js'
 export * from './helpers/index.js'
 export * from './execution/index.js'
+export * from './prompt/index.js'
 export type * from './plugins/types.js'
 export { plugins, registerLlmPlugin, pluginOf, pluginFor, resolvePlugin } from './plugins/index.js'
 export { anthropicPlugin, ANTHROPIC_FAMILY } from './plugins/anthropic.js'

package/src/model.ts

@@ -1,12 +1,12 @@
 import { Ajv } from 'ajv'
 import type { JSONSchemaType } from 'ajv'
-import { AIMessage } from '@langchain/core/messages'
-import type { AIMessageChunk, MessageFieldWithRole } from '@langchain/core/messages'
+import { AIMessage, BaseMessage } from '@langchain/core/messages'
+import type { AIMessageChunk, MessageContent, MessageFieldWithRole } from '@langchain/core/messages'
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { StructuredMode } from '@owlmeans/llm-common'
 import type { NullKind } from '@owlmeans/llm-common'
 import {
-  DEFAULT_MAX_OUTPUT_CAP, DEFAULT_MODEL_RETRIES, FALLBACK_AFTER_ATTEMPTS,
+  DEFAULT_MAX_OUTPUT_CAP, DEFAULT_MODEL_RETRIES, FALLBACK_AFTER_ATTEMPTS, MAX_CACHE_BREAKPOINTS,
 } from './consts.js'
 import { LlmModelError } from './errors.js'
 import { pluginFor, pluginOf } from './plugins/index.js'
@@ -18,7 +18,7 @@ import { spectate } from './helpers/spectate.js'
 import { idleTimeout, readConfig } from './utils/config.js'
 import { reportNull } from './utils/null-report.js'
 import type { NullReportParams } from './utils/null-report.js'
-import { applyNoThink, ensureJsonMention } from './utils/prompt.js'
+import { applyNoThink, ensureJsonMention, stripCacheMarkers } from './utils/prompt.js'
 import { resolveSchemaValidator, toToolName, unwrapNamed } from './utils/schema.js'
 import { streamWithDeadline } from './utils/stream.js'
 import type {
@@ -28,6 +28,48 @@ import type {
 
 type StreamOptions = Parameters<BaseChatModel['stream']>[1]
 
+const isSystem = (msg: MessageFieldWithRole): boolean =>
+  msg instanceof BaseMessage ? msg.getType() === 'system' : `${msg.role}` === 'system'
+
+const textOf = (content: MessageContent | undefined): string => {
+  if (typeof content === 'string') {
+    return content.trim()
+  }
+  if (Array.isArray(content)) {
+    return content
+      .map(part => {
+        const text = (part as unknown as { text?: unknown }).text
+        return typeof text === 'string' ? text : ''
+      })
+      .filter(text => text !== '')
+      .join('\n\n')
+      .trim()
+  }
+  return ''
+}
+
+/**
+ * Detach the caller's LEADING system messages and return their text.
+ *
+ * They are re-emitted as the `Context` block of the composed prompt, which is what keeps
+ * a caller that still builds its own `SystemMessage` working unchanged — the text simply
+ * travels a different route and lands in the same place. Only the leading run is taken:
+ * a system message deliberately placed mid-conversation is an operator instruction whose
+ * position carries meaning, and moving it would change what the model sees.
+ */
+const takeLeadingSystem = (msgs: MessageFieldWithRole[]): string[] => {
+  const carried: string[] = []
+  while (msgs.length > 0 && isSystem(msgs[0])) {
+    const [msg] = msgs.splice(0, 1)
+    const text = textOf(msg.content)
+    if (text !== '') {
+      carried.push(text)
+    }
+  }
+
+  return carried
+}
+
 /**
  * Build the four-method model API on top of a LangChain chat model.
  *
@@ -43,6 +85,9 @@ export const makeLlmModel = ({
   captureNull = false,
   retries = DEFAULT_MODEL_RETRIES,
   purpose,
+  prompt,
+  prompts,
+  files,
 }: LlmModelOptions, spectator: LlmSpectator): LlmModel => {
 
   const ajv = new Ajv({ strict: false })
@@ -53,15 +98,67 @@ export const makeLlmModel = ({
   const plugin: LlmPlugin | undefined = pluginOf(config.provider) ?? pluginFor(model)
   const timeout = idleTimeout(config)
 
-  /** Normalize, then apply every in-place prompt adaptation, in dependency order. */
-  const prepare = (input: ModelInput, useCache: boolean, cacheMax: number, json: boolean): MessageFieldWithRole[] => {
+  /**
+   * Normalize, compose the system prompt, then apply every in-place prompt adaptation, in
+   * dependency order.
+   *
+   * With no prompt service wired this is exactly what it always was — the caller's
+   * messages, a JSON nudge, `/no_think`, cache markers. With one, the caller's leading
+   * system text is folded into a composed prompt whose stable sections come first, which
+   * is the whole point: a prompt cache is a PREFIX match, so the bytes every call shares
+   * have to be physically ahead of the bytes that differ.
+   */
+  const prepare = async (
+    input: ModelInput,
+    action: string,
+    useCache: boolean,
+    cacheMax: number,
+    json: boolean,
+    callSkills?: string[],
+  ): Promise<MessageFieldWithRole[]> => {
     const msgs = normalizeInput(input)
+    // The caller may hand back messages this pipeline marked on a PREVIOUS call — the
+    // markers live on its own objects. The budget is per request, so clear them and
+    // re-place our own below; otherwise they accumulate until the provider 400s.
+    stripCacheMarkers(msgs)
+    let reserved = 0
+
+    if (prompts != null) {
+      const carried = takeLeadingSystem(msgs)
+      const composed = await prompts().compose(
+        {
+          ...prompt,
+          context: [...(prompt?.context ?? []), ...carried],
+          callSkills: callSkills ?? prompt?.callSkills,
+        },
+        msgs,
+        { model, provider: plugin, purpose, action, cacheMax, files },
+      )
+      if (composed.system != null) {
+        msgs.unshift(composed.system)
+        reserved = composed.breakpoints
+      } else {
+        // Defensive: nothing was contributed, so hand the caller's own text straight back.
+        for (let i = carried.length - 1; i >= 0; i--) {
+          msgs.unshift({ role: 'system', content: carried[i] })
+        }
+      }
+    }
+
     if (json) ensureJsonMention(msgs)
     applyNoThink(msgs, config.disableThinking)
     // Cache markers replace string content with content blocks, so they must go last.
-    if (plugin?.patchCache?.(msgs, { model, useCache, cacheMax }) === true) {
-      console.log(`Prompt caching enabled for ${plugin.type} (up to ${cacheMax} breakpoints)`)
+    const ttl = prompt?.cacheTtl
+    const marked = plugin?.patchCache?.(msgs, {
+      model, useCache, cacheMax, reserved, ...(ttl != null ? { ttl } : {}),
+    })
+    if (reserved > 0 || marked === true) {
+      console.log(
+        `Prompt caching for ${plugin?.type}: ${reserved} system breakpoint(s)`
+        + `${marked === true ? ', 1 message breakpoint' : ''}`
+      )
     }
+
     return msgs
   }
 
@@ -197,8 +294,11 @@ export const makeLlmModel = ({
   }
 
   const helper: LlmModel = {
-    ask: async (input, { ref, filter, action, useCache = false, cacheMax = 4 }: LlmAskOptions) => {
-      const msgs = prepare(input, useCache, cacheMax, false)
+    ask: async (
+      input,
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmAskOptions
+    ) => {
+      const msgs = await prepare(input, action, useCache, cacheMax, false, skills)
       return withRetry({ retries, outputErrors }, async i => {
         const refined = refineModel(i)
         console.log('Use model to ask: ', refined.getName(), refined.lc_kwargs.model)
@@ -242,8 +342,11 @@ export const makeLlmModel = ({
       })
     },
 
-    talk: async (input, { ref, filter, action, useCache = false, cacheMax = 4 }: LlmTalkOptions) => {
-      const msgs = prepare(input, useCache, cacheMax, false)
+    talk: async (
+      input,
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmTalkOptions
+    ) => {
+      const msgs = await prepare(input, action, useCache, cacheMax, false, skills)
       return withRetry({ retries, outputErrors }, async i => {
         const refined = refineModel(i)
         console.log('Use model to talk: ', refined.getName(), refined.lc_kwargs.model)
@@ -277,9 +380,9 @@ export const makeLlmModel = ({
     invoke: async <T>(
       input: ModelInput,
       schema: JSONSchemaType<T>,
-      { temperature, ref, filter, action, useCache = false, cacheMax = 4 }: LlmInvokeOptions<T>
+      { temperature, ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmInvokeOptions<T>
     ) => {
-      const msgs = prepare(input, useCache, cacheMax, true)
+      const msgs = await prepare(input, action, useCache, cacheMax, true, skills)
       const { name, innerSchema, validate } = resolveSchemaValidator<T>(ajv, schema)
       const toolName = toToolName((innerSchema as { title?: string }).title ?? name)
 
@@ -325,9 +428,9 @@ export const makeLlmModel = ({
     request: async <T>(
       input: ModelInput,
       schema: JSONSchemaType<T>,
-      { ref, filter, action, useCache = false, cacheMax = 4 }: LlmRequestOptions
+      { ref, filter, action, useCache = false, cacheMax = MAX_CACHE_BREAKPOINTS, skills }: LlmRequestOptions
     ) => {
-      const msgs = prepare(input, useCache, cacheMax, true)
+      const msgs = await prepare(input, action, useCache, cacheMax, true, skills)
       const { name, innerSchema, validate } = resolveSchemaValidator<T>(ajv, schema)
       const toolName = toToolName((innerSchema as { title?: string }).title ?? name)