@owlmeans/llm 0.1.14

package/src/plugins/anthropic.ts

@@ -0,0 +1,97 @@
+import { ChatAnthropic } from '@langchain/anthropic'
+import { BadRequestError } from '@anthropic-ai/sdk'
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { ModelProvider, StructuredMode } from '@owlmeans/llm-common'
+import type { LlmPlugin } from './types.js'
+import { MAX_CACHE_BREAKPOINTS } from '../consts.js'
+import { escalateMaxTokens, makeClientOptions } from './utils.js'
+
+/** Model-name prefix that supports prompt caching through `cache_control` markers. */
+const CACHEABLE_PREFIX = 'claude-'
+
+export const ANTHROPIC_FAMILY = 'anthropic'
+
+export const anthropicPlugin: LlmPlugin = {
+  type: ModelProvider.Anthropic,
+
+  family: ANTHROPIC_FAMILY,
+
+  owns: model => model instanceof ChatAnthropic,
+
+  /**
+   * Anthropic has no `response_format: json_schema` mode, so structured output is
+   * always the forced-tool-call hack. `ModelConfig.structuredOutput` is ignored.
+   */
+  structuredMode: () => StructuredMode.Tool,
+
+  /**
+   * Anthropic 400s on the OpenAI spelling: "tool_choice: Input tag 'function' … does not
+   * match any of the expected tags: 'auto','any','tool','none'".
+   */
+  toolChoice: (toolName: string): unknown => ({ type: 'tool', name: toolName }),
+
+  build: ({ config, secret, callbacks }) => {
+    const model = config.model ??= 'claude-haiku-4-5-20251001'
+    const cfg = {
+      model,
+      apiKey: secret,
+      // Neither knob set → pin temperature to 0 for determinism.
+      ...(config.temperature == null && config.topP == null ? { temperature: 0 } : {}),
+      maxTokens: config.maxTokens ?? 4096,
+      maxRetries: 5,
+      metadata: { config },
+      callbacks,
+      ...(config.temperature != null ? { temperature: config.temperature } : {}),
+      ...(config.topP != null && config.temperature == null ? { topP: config.topP } : {}),
+      ...makeClientOptions({ headers: config.headers }),
+    }
+    // Anthropic rejects temperature and top_p together.
+    if (cfg.temperature != null && cfg.topP != null) {
+      delete cfg.topP
+    }
+
+    return new ChatAnthropic(cfg)
+  },
+
+  refine: ({ base, attempt, temperature, maxOutputCap }): BaseChatModel => {
+    const model = base as ChatAnthropic
+    const currentTemperature = temperature ?? model.temperature ?? 0
+    const maxTokens = escalateMaxTokens(model.maxTokens, attempt, maxOutputCap)
+    const cfg: Partial<ChatAnthropic> = {
+      ...(model.lc_kwargs as Partial<ChatAnthropic>), temperature: currentTemperature, maxTokens,
+    }
+    if (cfg.temperature != null && cfg.temperature > 0 && cfg.topP != null) {
+      delete cfg.topP
+    } else if (cfg.temperature != null && cfg.temperature <= 0 && cfg.topP != null) {
+      delete cfg.temperature
+    }
+
+    return new ChatAnthropic(cfg)
+  },
+
+  /**
+   * Mark the leading messages with an ephemeral `cache_control` breakpoint. Anthropic
+   * allows at most {@link MAX_CACHE_BREAKPOINTS}; string content is lifted into a
+   * single text block so the marker has somewhere to live.
+   */
+  patchCache: (msgs, { model, useCache, cacheMax }) => {
+    if (!useCache || !(model as ChatAnthropic).modelName?.startsWith(CACHEABLE_PREFIX)) return false
+    const max = Math.min(cacheMax, MAX_CACHE_BREAKPOINTS)
+    let i = 0
+    for (const msg of msgs) {
+      msg.content = typeof msg.content === 'string' ? [{
+        type: 'text',
+        text: msg.content,
+        cache_control: { type: 'ephemeral' },
+      }] : msg.content
+      if (++i > max - 1) break
+    }
+    return true
+  },
+
+  /**
+   * A malformed request (bad schema, unsupported parameter, oversized `max_tokens`)
+   * cannot be fixed by retrying — surface it immediately instead of burning the budget.
+   */
+  isFatal: e => e instanceof BadRequestError ? e : null,
+}

package/src/plugins/compatible.ts

@@ -0,0 +1,62 @@
+import { ChatOpenAI } from '@langchain/openai'
+import { ModelProvider, StructuredMode } from '@owlmeans/llm-common'
+import type { LlmPlugin } from './types.js'
+import type { ModelConfig } from '../types.js'
+import { makeConfiguration } from './utils.js'
+import { openAiFamily } from './openai.js'
+
+/** Aggregators that encode the serving provider as a `model:provider` suffix. */
+const HUGGINGFACE_MARKER = 'huggingface'
+
+/**
+ * Any OpenAI-compatible endpoint that is not OpenAI itself — OpenRouter, the
+ * HuggingFace router, Together, a self-hosted vLLM, …
+ *
+ * Differences from the proprietary `openai` plugin:
+ * - Structured output defaults to the forced-tool-call hack. Native `response_format`
+ *   support is inconsistent across the long tail of servers behind these endpoints, and
+ *   a server that silently ignores it returns prose instead of JSON.
+ * - `modelKwargs` forwards aggregator-specific top-level request fields verbatim:
+ *   `reasoning` (hard-caps/disables thinking per the config) and
+ *   `provider.require_parameters`, which tells the aggregator to exclude servers that do
+ *   not honour every parameter in the request — without it a request can be routed to a
+ *   server that ignores `tool_choice` and answers `finish_reason='stop'` with no tool
+ *   call, or that sends the final SSE chunk twice.
+ */
+export const compatiblePlugin: LlmPlugin = {
+  ...openAiFamily,
+
+  type: ModelProvider.Compatible,
+
+  structuredMode: (config: ModelConfig): StructuredMode =>
+    config.structuredOutput === true ? StructuredMode.Native : StructuredMode.Tool,
+
+  build: ({ config, secret, callbacks }) => {
+    const baseModel = config.model ??= 'Qwen/Qwen3-235B-A22B-Instruct-2507'
+    const baseURL = config.baseUrl
+    const isHuggingFace = baseURL != null && baseURL.includes(HUGGINGFACE_MARKER)
+    const model = config.inferenceProvider != null && isHuggingFace
+      ? `${baseModel}:${config.inferenceProvider}`
+      : baseModel
+    const headers: Record<string, string> = {
+      ...(isHuggingFace && config.organization != null ? { 'X-HF-Bill-To': config.organization } : {}),
+      ...config.headers,
+    }
+
+    return new ChatOpenAI({
+      model,
+      apiKey: secret,
+      temperature: config.temperature ?? 0,
+      maxTokens: config.maxTokens ?? 4096,
+      topP: config.topP ?? 0.8,
+      maxRetries: 5,
+      metadata: { config },
+      callbacks,
+      modelKwargs: {
+        ...(config.reasoning != null ? { reasoning: config.reasoning } : {}),
+        provider: { require_parameters: true },
+      },
+      ...makeConfiguration({ baseURL, headers }),
+    })
+  },
+}

package/src/plugins/export.ts

@@ -0,0 +1,6 @@
+
+export type * from './types.js'
+export * from './anthropic.js'
+export * from './compatible.js'
+export * from './openai.js'
+export { plugins, registerLlmPlugin, pluginOf, pluginFor, resolvePlugin } from './index.js'

package/src/plugins/index.ts

@@ -0,0 +1,53 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { LlmPluginError } from '../errors.js'
+import type { LlmPlugin } from './types.js'
+import { anthropicPlugin } from './anthropic.js'
+import { compatiblePlugin } from './compatible.js'
+import { openAiPlugin } from './openai.js'
+
+export const plugins: Record<string, LlmPlugin> = {}
+
+/**
+ * Lookup order for INSTANCE-based resolution (a model whose config metadata is not
+ * reachable). The first plugin whose `owns` matches wins, so the conservative member of
+ * a client family must come first: `compatible` precedes `openai` because both build a
+ * `ChatOpenAI`, and assuming the tool-calling hack for an unlabelled model is safe
+ * everywhere, while assuming native JSON-schema support is not.
+ */
+const order: string[] = []
+
+/** Register (or replace) a provider plugin. Later registrations go last in the lookup order. */
+export const registerLlmPlugin = (plugin: LlmPlugin): void => {
+  if (plugins[plugin.type] == null) {
+    order.push(plugin.type)
+  }
+  plugins[plugin.type] = plugin
+}
+
+registerLlmPlugin(anthropicPlugin)
+registerLlmPlugin(compatiblePlugin)
+registerLlmPlugin(openAiPlugin)
+
+/** The plugin registered for `provider`, or `undefined`. */
+export const pluginOf = (provider: string | undefined): LlmPlugin | undefined =>
+  provider != null ? plugins[provider] : undefined
+
+/** The first registered plugin that recognises this model instance, or `undefined`. */
+export const pluginFor = (model: BaseChatModel): LlmPlugin | undefined =>
+  order.map(type => plugins[type]).find(plugin => plugin?.owns(model) === true)
+
+/**
+ * Resolve the plugin governing a call. The config's `provider` is authoritative; when it
+ * is unavailable (a refined instance whose metadata did not survive) the model instance
+ * is matched against the registration order.
+ */
+export const resolvePlugin = (
+  config: { provider?: string } | undefined,
+  model?: BaseChatModel,
+): LlmPlugin => {
+  const byType = pluginOf(config?.provider)
+  if (byType != null) return byType
+  const byModel = model != null ? pluginFor(model) : undefined
+  if (byModel != null) return byModel
+  throw new LlmPluginError(`${LlmPluginError.NO_PLUGIN}:${config?.provider ?? 'unknown'}`)
+}

package/src/plugins/openai.ts

@@ -0,0 +1,108 @@
+import { ChatOpenAI } from '@langchain/openai'
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { ModelProvider, StructuredMode } from '@owlmeans/llm-common'
+import type { LlmPlugin, LlmRefineParams } from './types.js'
+import type { ModelConfig } from '../types.js'
+import { escalateMaxTokens, makeConfiguration } from './utils.js'
+
+/** Model families served through OpenAI's Responses API rather than chat completions. */
+const RESPONSES_API_PREFIXES = ['gpt-5', 'codex-']
+
+export const OPENAI_FAMILY = 'openai'
+
+/** Every plugin that constructs a `ChatOpenAI` shares these instance-level behaviours. */
+export const openAiFamily = {
+  family: OPENAI_FAMILY,
+
+  owns: (model: BaseChatModel): boolean => model instanceof ChatOpenAI,
+
+  /**
+   * langchain converts the OpenAI-shaped tool DEFINITION for either provider, but the
+   * `tool_choice` shape is NOT converted — this is the OpenAI spelling.
+   */
+  toolChoice: (toolName: string): unknown => ({ type: 'function', function: { name: toolName } }),
+
+  /**
+   * `strict: false` keeps schemas that do not satisfy OpenAI strict-mode rules
+   * acceptable; the model's own ajv validation still enforces conformance afterwards.
+   * With `provider.require_parameters` already set, sending `response_format` also makes
+   * an aggregator route only to providers that actually support structured outputs.
+   */
+  responseFormat: (toolName: string, schema: unknown): Record<string, unknown> => ({
+    type: 'json_schema',
+    json_schema: { name: toolName, schema, strict: false },
+  }),
+
+  refine: ({ base, attempt, temperature, maxOutputCap }: LlmRefineParams): BaseChatModel => {
+    const model = base as ChatOpenAI
+    const currentTemperature = temperature ?? model.temperature ?? 0
+    const maxTokens = escalateMaxTokens(model.maxTokens, attempt, maxOutputCap)
+    const baseKwargs = model.lc_kwargs as ConstructorParameters<typeof ChatOpenAI>[0] & {
+      modelKwargs?: { reasoning?: { max_tokens?: number } } & Record<string, unknown>
+    }
+    // The dominant cause of an empty response is a reasoning model spending the whole
+    // budget on hidden thinking (finish_reason=length, empty content). The retry already
+    // raises maxTokens; ALSO shrink the absolute reasoning cap so the extra budget becomes
+    // visible output instead of more reasoning. Only touches `{ max_tokens: N }` reasoning
+    // configs — effort/enabled/exclude shapes are left untouched.
+    const reasoning = baseKwargs.modelKwargs?.reasoning
+    const modelKwargs = attempt > 0 && typeof reasoning?.max_tokens === 'number'
+      ? {
+        ...baseKwargs.modelKwargs,
+        reasoning: { ...reasoning, max_tokens: Math.max(256, Math.floor(reasoning.max_tokens / Math.pow(2, attempt))) },
+      }
+      : baseKwargs.modelKwargs
+
+    return new ChatOpenAI({
+      ...baseKwargs,
+      temperature: currentTemperature,
+      maxTokens,
+      ...(modelKwargs != null ? { modelKwargs } : {}),
+    })
+  },
+}
+
+/**
+ * Proprietary OpenAI endpoint. Defaults to the provider's NATIVE JSON-schema mode for
+ * structured output — it is reliable there, unlike on the long tail of
+ * OpenAI-compatible endpoints (see the `compatible` plugin).
+ */
+export const openAiPlugin: LlmPlugin = {
+  ...openAiFamily,
+
+  type: ModelProvider.OpenAI,
+
+  structuredMode: (config: ModelConfig): StructuredMode =>
+    config.structuredOutput === false ? StructuredMode.Tool : StructuredMode.Native,
+
+  build: ({ config, secret, callbacks }) => {
+    const model = config.model ??= 'gpt-5.4-mini'
+    const configuration = makeConfiguration({ baseURL: undefined, headers: config.headers })
+
+    // The Responses API models reject `temperature`/`topP`.
+    if (RESPONSES_API_PREFIXES.some(prefix => model.startsWith(prefix))) {
+      return new ChatOpenAI({
+        model,
+        apiKey: secret,
+        maxTokens: config.maxTokens ?? 4096,
+        maxRetries: 5,
+        useResponsesApi: true,
+        metadata: { config },
+        callbacks,
+        ...configuration,
+      })
+    }
+
+    return new ChatOpenAI({
+      model,
+      apiKey: secret,
+      temperature: config.temperature ?? 0,
+      maxTokens: config.maxTokens ?? 4096,
+      topP: config.topP ?? 0.8,
+      maxRetries: 5,
+      metadata: { config },
+      callbacks,
+      ...configuration,
+    })
+  },
+}

package/src/plugins/types.ts

@@ -0,0 +1,92 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import type { BaseCallbackHandler, CallbackHandlerMethods } from '@langchain/core/callbacks/base'
+import type { MessageFieldWithRole } from '@langchain/core/messages'
+import type { StructuredMode } from '@owlmeans/llm-common'
+import type { ModelConfig } from '../types.js'
+
+export interface LlmBuildParams {
+  /** Alias the config was registered under — used only for error reporting. */
+  alias: string
+  /**
+   * Fully resolved config with the secret already stripped. Whatever the plugin puts
+   * on the client as `metadata.config` is what `readConfig` later reads back, so the
+   * plugin should apply its model default to this object before building.
+   */
+  config: ModelConfig
+  secret: string
+  callbacks: (BaseCallbackHandler | CallbackHandlerMethods)[]
+}
+
+export interface LlmRefineParams {
+  /** The model to rebuild — the primary, or the fallback once escalation kicked in. */
+  base: BaseChatModel
+  /** 0-based retry attempt; the output budget doubles with it. */
+  attempt: number
+  /** Call-site temperature override (`invoke`). */
+  temperature?: number | undefined
+  /** Hard ceiling the doubled output budget is clamped to. */
+  maxOutputCap: number
+}
+
+export interface LlmCacheParams {
+  /** The ORIGINAL (unrefined) model — refined instances do not always keep the name. */
+  model: BaseChatModel
+  useCache: boolean
+  cacheMax: number
+}
+
+/**
+ * Everything that differs between inference providers, in one replaceable unit.
+ *
+ * Registration order matters for instance-based lookup (`resolvePlugin` with no
+ * `provider` in the config): the FIRST plugin whose `owns` matches wins, so the
+ * conservative member of a family must be registered before the permissive one.
+ * See `plugins/index.ts`.
+ */
+export interface LlmPlugin {
+  /** Provider identifier — a `ModelProvider` value, or a custom string. */
+  type: string
+
+  /**
+   * Instance behaviour group. Two plugins that construct the same client class share a
+   * family (e.g. `openai` and `compatible` both produce `ChatOpenAI`). The retry
+   * escalator refuses to switch to a fallback from a different family, because the
+   * structured-output call shape would change mid-call.
+   */
+  family: string
+
+  /** Construct the chat client from a resolved config. */
+  build: (params: LlmBuildParams) => BaseChatModel
+
+  /** Does this model instance belong to this plugin's client family? */
+  owns: (model: BaseChatModel) => boolean
+
+  /**
+   * Rebuild the model for retry attempt N: raise the output budget toward
+   * `maxOutputCap` and apply any provider-specific escalation (e.g. shrinking a
+   * reasoning budget so the extra tokens become visible output).
+   */
+  refine: (params: LlmRefineParams) => BaseChatModel
+
+  /** How this provider should be asked for schema-conforming output. */
+  structuredMode: (config: ModelConfig) => StructuredMode
+
+  /** Provider-specific `tool_choice` shape that pins the model to `toolName`. */
+  toolChoice: (toolName: string) => unknown
+
+  /** Provider-specific `response_format` for {@link StructuredMode.Native}. */
+  responseFormat?: (toolName: string, schema: unknown) => Record<string, unknown>
+
+  /**
+   * Mark leading messages as cacheable in-place. Returns `true` when caching was
+   * actually applied (the model may not support it). Omit for providers with no
+   * explicit prompt-cache markers.
+   */
+  patchCache?: (msgs: MessageFieldWithRole[], params: LlmCacheParams) => boolean
+
+  /**
+   * Classify a thrown error as fatal for the retry loop. Return the error to throw
+   * immediately, or `null` to let it be retried.
+   */
+  isFatal?: (e: unknown) => Error | null
+}

package/src/plugins/utils.ts

@@ -0,0 +1,38 @@
+/**
+ * Shared construction bits for the built-in plugins. Internal to `plugins/` — not
+ * part of the package's public surface.
+ */
+
+/**
+ * `{ configuration: { baseURL?, defaultHeaders? } }` for the OpenAI-compatible client,
+ * or an empty object when there is nothing to set (the client rejects an empty
+ * `configuration` in some versions).
+ */
+export const makeConfiguration = (
+  { baseURL, headers }: { baseURL: string | undefined, headers: Record<string, string> | undefined }
+): { configuration: Record<string, unknown> } | Record<string, never> => {
+  const hasBaseURL = baseURL != null
+  const hasHeaders = headers != null && Object.keys(headers).length > 0
+  if (!hasBaseURL && !hasHeaders) return {}
+  return {
+    configuration: {
+      ...(hasBaseURL ? { baseURL } : {}),
+      ...(hasHeaders ? { defaultHeaders: headers } : {}),
+    }
+  }
+}
+
+/** `{ clientOptions: { defaultHeaders } }` for the Anthropic client, or an empty object. */
+export const makeClientOptions = (
+  { headers }: { headers: Record<string, string> | undefined }
+): { clientOptions: Record<string, unknown> } | Record<string, never> => {
+  if (headers == null || Object.keys(headers).length === 0) return {}
+  return { clientOptions: { defaultHeaders: headers } }
+}
+
+/**
+ * Output budget for retry attempt N: double the base budget per attempt, clamped to
+ * the model's hard ceiling.
+ */
+export const escalateMaxTokens = (base: number | undefined, attempt: number, cap: number): number =>
+  Math.min((base ?? 2048) * Math.pow(2, attempt), cap)

package/src/service.ts

@@ -0,0 +1,125 @@
+import { createService } from '@owlmeans/context'
+import type { BasicConfig, BasicContext } from '@owlmeans/context'
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { LLM_SERVICE } from './consts.js'
+import { LlmMissconfiguredError } from './errors.js'
+import { resolvePlugin } from './plugins/index.js'
+import type { LlmService, LlmServiceOptions, ModelConfig, WithLlmService } from './types.js'
+
+/** The part of {@link LlmService} this package implements — see {@link llmServiceApi}. */
+export type LlmServiceApi = Pick<LlmService, 'models' | 'callbacks' | 'addCallbacks' | 'getModel' | 'configs'>
+
+/**
+ * Build the model-factory half of an LLM service, WITHOUT registering it as a context
+ * service. Spread it into your own `createService` implementation to publish extra
+ * methods (role-named accessors, domain helpers) alongside it:
+ *
+ * ```ts
+ * const service = createService<MyService>(alias, {
+ *   ...llmServiceApi(options, () => service),
+ *   getPicker: override => service.getModel(MyRole.Picker, override),
+ * } as MyService)
+ * ```
+ *
+ * `self` is a late-bound reference to the finished service, because the methods read
+ * mutable state (`models`, `callbacks`) off the registered instance rather than the
+ * literal.
+ */
+export const llmServiceApi = (options: LlmServiceOptions, self: () => LlmService): LlmServiceApi => {
+
+  /** Build one client from a fully-resolved config — no preset/fallback logic here. */
+  const buildModel = (alias: string, config: ModelConfig): BaseChatModel => {
+    if (config.provider == null || config.secret == null) {
+      throw new LlmMissconfiguredError(alias)
+    }
+    const { secret, ...rest } = config
+    return resolvePlugin(rest).build({
+      alias, config: rest as ModelConfig, secret, callbacks: self().callbacks,
+    })
+  }
+
+  /**
+   * Resolve `alias` → config (inheriting a `preset`, applying `override`) and build it.
+   * A declared `fallback` is built as well and attached to the primary as a
+   * non-enumerable `__fallbackModel`, which the model's retry escalator reads. The
+   * fallback spec is merged OVER the primary config, so it inherits secret/headers.
+   */
+  const createModel = (alias: string, override: Partial<ModelConfig> = {}): BaseChatModel => {
+    const models = options.models()
+    const baseConfig = models.find(m => m.alias === alias)
+    if (baseConfig == null) {
+      throw new LlmMissconfiguredError(alias)
+    }
+    const config: ModelConfig = { ...baseConfig, ...override }
+    const preset: Partial<ModelConfig> = config.preset != null
+      ? { ...(models.find(m => m.alias === config.preset) ?? {}) }
+      : {}
+    if (preset.alias != null) {
+      delete preset.alias
+    }
+    Object.assign(config, preset)
+
+    const { fallback, ...primaryConfig } = config
+    const primary = buildModel(alias, primaryConfig)
+    if (fallback != null) {
+      const { fallback: _nested, ...fallbackConfig } = { ...primaryConfig, ...fallback }
+      Object.defineProperty(primary, '__fallbackModel', {
+        value: buildModel(alias, fallbackConfig),
+        enumerable: false,
+        configurable: true,
+      })
+    }
+
+    return primary
+  }
+
+  const cacheKey = (alias: string, override: Partial<ModelConfig> = {}): string =>
+    `${alias}:${JSON.stringify(override)}`
+
+  return {
+    models: new Map(),
+
+    callbacks: [],
+
+    addCallbacks: callbacks => {
+      self().callbacks.push(...callbacks)
+    },
+
+    configs: () => options.models(),
+
+    getModel: (alias, override = {}, createNew = false) => {
+      const service = self()
+      const key = cacheKey(alias, override)
+      const model = createNew || !service.models.has(key)
+        ? createModel(alias, override)
+        : service.models.get(key)!
+      if (!createNew) {
+        service.models.set(key, model)
+      }
+
+      return model
+    },
+  }
+}
+
+export const makeLlmService = (options: LlmServiceOptions, alias: string = LLM_SERVICE): LlmService => {
+  const service: LlmService = createService<LlmService>(
+    alias, llmServiceApi(options, () => service) as LlmService
+  )
+
+  return service
+}
+
+export const appendLlmService = <C extends BasicConfig, T extends BasicContext<C>>(
+  ctx: T,
+  options: LlmServiceOptions,
+  alias: string = LLM_SERVICE
+): T & WithLlmService => {
+  const context = ctx as T & WithLlmService
+
+  context.registerService(makeLlmService(options, alias))
+
+  context.llm = () => ctx.service(alias)
+
+  return context
+}