@owlmeans/llm 0.1.14

package/src/types.ts

@@ -0,0 +1,214 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import type { AIMessage, BaseMessage, MessageFieldWithRole } from '@langchain/core/messages'
+import type { BaseCallbackHandler, CallbackHandlerMethods } from '@langchain/core/callbacks/base'
+import type { JSONSchemaType } from 'ajv'
+import type { InitializedService } from '@owlmeans/context'
+import type {
+  LlmPurpose, ModelProvider, NullCapture, SpectatorArgument, SpectatorEntryLogged,
+} from '@owlmeans/llm-common'
+
+export type MaybeArray<T> = T | T[]
+
+/**
+ * Out-of-band result channel for a model call: the caller passes a `ref` and receives
+ * the raw message, the spectator entry it was logged under, and an optional callback
+ * fired as soon as the value is known (before the caller's own await resolves).
+ */
+export interface RefferedResult<T> {
+  spectatorEntry?: SpectatorEntryLogged
+  value?: T
+  callback?: (arg: T) => Promise<void>
+}
+
+export interface RetryOptions {
+  retries: number
+  outputErrors?: boolean
+  /**
+   * Abort the retry loop for this call. Return the error to throw, or `null` to keep
+   * retrying. Consulted in addition to the globally registered resolvers
+   * (`registerFatalError`) and the provider plugins' `isFatal`.
+   */
+  fatal?: (e: unknown) => Error | null
+}
+
+/** Decides whether an error must abort a retry loop instead of being retried. */
+export interface FatalErrorResolver {
+  (e: unknown): Error | null
+}
+
+export interface LlmLogging {
+  /** Print every swallowed retry error to the console. */
+  outputErrors?: boolean
+  /** Hand a full {@link NullCapture} to the spectator when a call returns nothing usable. */
+  captureNull?: boolean
+  purpose: LlmPurpose
+}
+
+export interface LlmModelOptions extends LlmLogging {
+  model: BaseChatModel
+  retries?: number
+}
+
+export type ModelMessage = BaseMessage | MessageFieldWithRole
+export type ModelInputItem = ModelMessage | string
+export type ModelInput = MaybeArray<ModelInputItem>
+
+export interface LlmCallOptions {
+  /** Short name of the operation — used as the LangChain run name and in spectator entries. */
+  action: string
+  /** Ask the provider to cache the prompt prefix (no-op for providers without prompt caching). */
+  useCache?: boolean
+  /** How many leading messages to mark as cacheable (capped by the provider's own limit). */
+  cacheMax?: number
+}
+
+export interface LlmAskOptions extends LlmCallOptions {
+  ref?: RefferedResult<AIMessage>
+  filter?: (output: string, result: AIMessage) => (string | null) | Promise<string | null>
+}
+
+export interface LlmTalkOptions extends LlmCallOptions {
+  ref?: RefferedResult<AIMessage>
+  filter?: (result: AIMessage) => Promise<AIMessage | null>
+}
+
+export interface LlmInvokeOptions<T> extends LlmCallOptions {
+  temperature?: number | undefined
+  ref?: RefferedResult<AIMessage>
+  filter?: (output: T, result: AIMessage) => (T | null) | Promise<T | null>
+}
+
+export interface LlmRequestOptions extends LlmCallOptions {
+  ref?: RefferedResult<AIMessage>
+  filter?: (result: AIMessage) => Promise<AIMessage | null>
+}
+
+/**
+ * The four ways to talk to a model. Every method streams under an idle deadline,
+ * retries with escalating output budget, logs to the spectator and validates the result.
+ *
+ * - `ask` → plain text.
+ * - `talk` → the raw `AIMessage`.
+ * - `invoke` → a schema-validated object.
+ * - `request` → an `AIMessage` whose `content` is the schema-validated JSON.
+ */
+export interface LlmModel {
+  ask: (input: ModelInput, options: LlmAskOptions) => Promise<string>
+  talk: (input: ModelInput, options: LlmTalkOptions) => Promise<AIMessage>
+  invoke: <T>(input: ModelInput, schema: JSONSchemaType<T>, options: LlmInvokeOptions<T>) => Promise<T>
+  request: <T>(input: ModelInput, schema: JSONSchemaType<T>, options: LlmRequestOptions) => Promise<AIMessage>
+}
+
+/**
+ * Observability sink the model writes every call to. Deliberately minimal — a consumer's
+ * richer spectator (with `derive`, `update`, storage backends) satisfies it structurally.
+ */
+export interface LlmSpectator {
+  log: (arg: SpectatorArgument) => Promise<SpectatorEntryLogged>
+  /** Optional sink for full diagnostics of a call that returned nothing usable. */
+  captureNull?: (capture: NullCapture) => Promise<void>
+}
+
+/** Resolves a model of the same role at a different temperature. */
+export interface TemperatureFactory {
+  (temperature?: number | undefined): BaseChatModel
+}
+
+/**
+ * Full runtime configuration of one model alias. The JSON-safe subset that may travel
+ * inside an execution state is `ModelConfigPatch` (`@owlmeans/llm-common`) — this type
+ * additionally carries credentials and provider wiring and must never be serialized.
+ */
+export interface ModelConfig {
+  provider?: ModelProvider | string
+  secret?: string
+  alias: string
+  /** Inherit every field of another alias in the same config list. */
+  preset?: string
+  model?: string
+  temperature?: number
+  /** Initial output-token budget per request (`max_tokens`). */
+  maxTokens?: number
+  /**
+   * Hard ceiling for output tokens used by the retry escalator. Each retry doubles
+   * `maxTokens` toward this cap; without it the escalator uses
+   * {@link DEFAULT_MAX_OUTPUT_CAP}, which exceeds many models' real per-request output
+   * limit and turns retries into 400 "max_tokens exceeds model limit" errors.
+   */
+  maxTokensCap?: number
+  topP?: number
+  baseUrl?: string
+  organization?: string
+  /** Provider-routing hint for aggregators that expose one (e.g. HuggingFace). */
+  inferenceProvider?: string
+  headers?: Record<string, string>
+  /**
+   * Suppress the model's chain-of-thought / "thinking" tokens. For the Qwen3 family
+   * this injects the `/no_think` soft switch into the prompt; without it those models
+   * routinely spend the entire output budget on hidden reasoning and return empty
+   * content with `finish_reason="length"`.
+   */
+  disableThinking?: boolean
+  /**
+   * OpenAI-compatible reasoning control, forwarded verbatim as the top-level `reasoning`
+   * request-body field. Use `{ max_tokens: N }` to hard-cap the thinking budget —
+   * universal, and it works even for always-reasoning models (the request `max_tokens`
+   * must stay strictly higher) — or `{ effort: 'none' }` to disable thinking on hybrid
+   * models that enumerate effort levels. `{ exclude: true }` keeps reasoning internal but
+   * omits it from the response.
+   */
+  reasoning?: {
+    effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'
+    max_tokens?: number
+    exclude?: boolean
+    enabled?: boolean
+  }
+  /**
+   * Force how `invoke`/`request` obtain structured output, overriding the provider
+   * plugin's default: `true` → the provider's NATIVE JSON-schema mode, `false` → the
+   * forced-`tool_choice` tool-calling hack. Plugins that support only one mode
+   * (Anthropic — tool calling) ignore this flag.
+   */
+  structuredOutput?: boolean
+  /**
+   * Idle/inactivity timeout (ms) for a streamed response. NOT a total cap — the timer
+   * resets on every chunk. Falls back to {@link MODEL_STREAM_TIMEOUT_MS}.
+   */
+  streamTimeout?: number
+  /**
+   * Stronger model the retry escalator switches to once the primary has failed
+   * {@link FALLBACK_AFTER_ATTEMPTS} times. Specified inline as a partial config merged
+   * over the base config, so it inherits `secret`, `headers`, etc. The escalation only
+   * happens when the fallback belongs to the SAME plugin family as the primary —
+   * rotating providers mid-call would flip the structured-output format.
+   */
+  fallback?: Partial<ModelConfig>
+}
+
+export interface LlmServiceOptions {
+  /** The full config list; resolved by `alias` on every `getModel` call. */
+  models: () => ModelConfig[]
+}
+
+/**
+ * Model factory and registry. Builds a `BaseChatModel` from a `ModelConfig` through the
+ * registered provider plugins, memoizing per alias+override so repeated resolution of a
+ * role does not rebuild the client.
+ */
+export interface LlmService extends InitializedService {
+  models: Map<string, BaseChatModel>
+
+  callbacks: (BaseCallbackHandler | CallbackHandlerMethods)[]
+
+  addCallbacks: (callbacks: (BaseCallbackHandler | CallbackHandlerMethods)[]) => void
+
+  /** Resolve (and cache) the model registered under `alias`, with an optional patch. */
+  getModel: (alias: string, override?: Partial<ModelConfig>, createNew?: boolean) => BaseChatModel
+
+  /** The configured model list, as supplied by {@link LlmServiceOptions.models}. */
+  configs: () => ModelConfig[]
+}
+
+export interface WithLlmService {
+  llm: () => LlmService
+}

package/src/utils/config.ts

@@ -0,0 +1,19 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { MODEL_STREAM_TIMEOUT_MS } from '../consts.js'
+import type { ModelConfig } from '../types.js'
+
+/**
+ * Read the original `ModelConfig` back off a model instance. Every plugin's `build`
+ * passes `metadata: { config }` on the client constructor, so the config is reachable
+ * for any provider. Returns an empty object when unavailable — notably for a REFINED
+ * instance, which is rebuilt from `lc_kwargs` and does not reliably carry metadata.
+ * That is why callers read the config from the ORIGINAL model.
+ */
+export const readConfig = (model: BaseChatModel): Partial<ModelConfig> => {
+  const meta = (model as unknown as { metadata?: { config?: Partial<ModelConfig> } }).metadata
+  return meta?.config ?? {}
+}
+
+/** Per-model idle stream timeout (ms), falling back to the package default. */
+export const idleTimeout = (config: Partial<ModelConfig>): number =>
+  config.streamTimeout ?? MODEL_STREAM_TIMEOUT_MS

package/src/utils/null-report.ts

@@ -0,0 +1,126 @@
+import util from 'util'
+import type { AIMessageChunk, MessageFieldWithRole } from '@langchain/core/messages'
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { createIdOfLength } from '@owlmeans/basic-ids'
+import type { LlmPurpose, NullCapture, NullKind } from '@owlmeans/llm-common'
+import type { LlmSpectator, ModelConfig } from '../types.js'
+
+export interface NullReportParams {
+  kind: NullKind
+  action: string
+  purpose?: LlmPurpose
+  attempt: number
+  startedAt: number
+  /** The instance that actually ran — its `lc_kwargs` carry the effective request shape. */
+  refined: BaseChatModel
+  /** The ORIGINAL model config (refined instances do not reliably keep metadata). */
+  config: Partial<ModelConfig>
+  msgs: MessageFieldWithRole[]
+  raw: AIMessageChunk | null
+  schema?: { toolName: string; innerSchema: unknown }
+  useCache: boolean
+}
+
+/** How many characters of each prompt message the console preview keeps. */
+const PREVIEW_CHARS = 300
+
+/**
+ * Assemble a complete, replayable record of a model call that returned nothing usable:
+ * the effective request, the raw response with its metadata, and the diagnostics that
+ * distinguish the common causes (budget spent on hidden reasoning vs. a refused tool
+ * call vs. an empty content array).
+ */
+export const buildNullReport = (p: NullReportParams): NullCapture => {
+  // Read the request shape from lc_kwargs — the refined instance is rebuilt from those
+  // and does not always preserve `metadata`.
+  type Kwargs = {
+    model?: string
+    configuration?: { baseURL?: string }
+    modelKwargs?: { reasoning?: unknown }
+    topP?: number
+  }
+  const kwargs = p.refined.lc_kwargs as Kwargs
+  const raw = p.raw
+  const responseMeta = raw?.response_metadata as {
+    finish_reason?: string
+    usage?: { prompt_tokens?: number; completion_tokens?: number; reasoning_tokens?: number }
+  } | undefined
+  const usageMeta = raw?.usage_metadata as { input_tokens?: number; output_tokens?: number } | undefined
+  const toolCalls = (raw as unknown as { tool_calls?: unknown[] } | null)?.tool_calls
+
+  return {
+    meta: {
+      kind: p.kind,
+      action: p.action,
+      purpose: p.purpose,
+      attempt: p.attempt,
+      id: createIdOfLength(12),
+      timestamp: Date.now(),
+      elapsedMs: Date.now() - p.startedAt,
+    },
+    model: {
+      // The provider-side model slug (needed for replay); falls back to the config alias.
+      id: kwargs.model ?? p.config.model,
+      provider: p.config.provider,
+      baseUrl: kwargs.configuration?.baseURL,
+      maxTokens: (p.refined as unknown as { maxTokens?: number }).maxTokens,
+      reasoning: kwargs.modelKwargs?.reasoning,
+      temperature: (p.refined as unknown as { temperature?: number }).temperature,
+      topP: kwargs.topP,
+    },
+    request: {
+      messages: p.msgs as unknown[],
+      schema: p.schema,
+      useCache: p.useCache,
+    },
+    response: raw != null ? {
+      content: raw.content,
+      additional_kwargs: raw.additional_kwargs,
+      response_metadata: raw.response_metadata,
+      usage_metadata: raw.usage_metadata,
+      tool_calls: toolCalls,
+    } : null,
+    diagnostics: {
+      finishReason: responseMeta?.finish_reason,
+      inputTokens: usageMeta?.input_tokens ?? responseMeta?.usage?.prompt_tokens,
+      outputTokens: usageMeta?.output_tokens ?? responseMeta?.usage?.completion_tokens,
+      reasoningTokens: responseMeta?.usage?.reasoning_tokens,
+      contentEmpty: raw == null || raw.content === '' || raw.content == null
+        || (Array.isArray(raw.content) && raw.content.length === 0),
+      hadToolCall: Array.isArray(toolCalls) && toolCalls.length > 0,
+    },
+  }
+}
+
+/**
+ * Print the diagnostics of a null result, and hand the full capture to the spectator
+ * sink when the caller opted into capturing. A failing sink must never mask the model
+ * error the caller is about to throw.
+ */
+export const reportNull = async (
+  spectator: LlmSpectator,
+  captureNull: boolean,
+  p: NullReportParams,
+): Promise<void> => {
+  const capture = buildNullReport(p)
+  const requestPreview = p.msgs.map(msg => {
+    const text = typeof msg.content === 'string' ? msg.content.substring(0, PREVIEW_CHARS) : '[complex content]'
+    return `[${(msg as { role?: string }).role ?? 'unknown'}] ${text}`
+  }).join('\n---\n')
+
+  console.error('[MODEL-NULL]', util.inspect(
+    {
+      meta: capture.meta, model: capture.model, diagnostics: capture.diagnostics,
+      response: capture.response, requestPreview,
+    },
+    { depth: null, maxStringLength: 2000, breakLength: 120 }
+  ))
+
+  if (captureNull) {
+    try {
+      await spectator.captureNull?.(capture)
+    } catch (e) {
+      console.warn('[MODEL-NULL] capture write failed', e)
+    }
+  }
+}

package/src/utils/prompt.ts

@@ -0,0 +1,46 @@
+import type { MessageFieldWithRole } from '@langchain/core/messages'
+import { JSON_INSTRUCTION, NO_THINK_DIRECTIVE } from '../consts.js'
+
+const messageMentions = (msg: MessageFieldWithRole, needle: string): boolean => {
+  if (typeof msg.content === 'string') return msg.content.toLowerCase().includes(needle)
+  if (Array.isArray(msg.content)) {
+    return msg.content.some(
+      (part: unknown) => typeof part === 'object' && part !== null && 'text' in (part as Record<string, unknown>)
+        && typeof (part as Record<string, string>).text === 'string'
+        && (part as Record<string, string>).text.toLowerCase().includes(needle)
+    )
+  }
+  return false
+}
+
+/** Append `text` to the last message, or push it as a new user message when that is not possible. */
+const appendDirective = (msgs: MessageFieldWithRole[], text: string): void => {
+  const last = msgs[msgs.length - 1]
+  if (last != null && typeof last.content === 'string') {
+    last.content = `${last.content}\n${text}`
+  } else {
+    msgs.push({ role: 'user', content: text })
+  }
+}
+
+/**
+ * Ensure the word "json" appears somewhere in the prompt. Several providers refuse or
+ * silently ignore a JSON mode unless it does; when it is missing the JSON instruction is
+ * appended in place.
+ */
+export const ensureJsonMention = (msgs: MessageFieldWithRole[]): void => {
+  if (msgs.some(msg => messageMentions(msg, 'json'))) return
+  appendDirective(msgs, JSON_INSTRUCTION)
+}
+
+/**
+ * Append the `/no_think` soft switch when the config asks for it. Without it,
+ * thinking-mode models frequently spend their entire output budget on hidden reasoning
+ * and return empty content with `finish_reason="length"`, which then fails every filter
+ * and exhausts the retries.
+ */
+export const applyNoThink = (msgs: MessageFieldWithRole[], disableThinking: boolean | undefined): void => {
+  if (disableThinking !== true) return
+  if (msgs.some(msg => typeof msg.content === 'string' && msg.content.includes(NO_THINK_DIRECTIVE))) return
+  appendDirective(msgs, NO_THINK_DIRECTIVE)
+}

package/src/utils/schema.ts

@@ -0,0 +1,35 @@
+import type { Ajv, JSONSchemaType, ValidateFunction } from 'ajv'
+import { DEFAULT_TOOL_NAME } from '../consts.js'
+
+/**
+ * Split the caller's schema into the optional wrapper `name` and the schema proper, and
+ * compile a validator for the latter. A `name` means the model is expected to answer
+ * `{ [name]: <object> }`; see {@link unwrapNamed}.
+ */
+export const resolveSchemaValidator = <T>(ajv: Ajv, schema: JSONSchemaType<T>): {
+  name: string | undefined
+  innerSchema: JSONSchemaType<T>
+  validate: ValidateFunction<T>
+} => {
+  const { name, ...innerSchema } = schema as JSONSchemaType<T> & { name?: string }
+  const validate = ajv.compile<T>(innerSchema as JSONSchemaType<T>)
+  return { name, innerSchema: innerSchema as JSONSchemaType<T>, validate }
+}
+
+/**
+ * Derive a function/tool name for tool-calling structured output. Provider tool names
+ * must match `^[A-Za-z0-9_-]+$`, so the schema title/name is sanitised; falls back to
+ * {@link DEFAULT_TOOL_NAME} when nothing usable is present.
+ */
+export const toToolName = (raw: string | undefined): string => {
+  const cleaned = (raw ?? '').replace(/[^A-Za-z0-9_-]+/g, '_').replace(/^_+|_+$/g, '')
+  return cleaned.length > 0 ? cleaned : DEFAULT_TOOL_NAME
+}
+
+/** Unwrap `{ [name]: value }` when the schema declared a wrapper name. */
+export const unwrapNamed = <T>(result: T, name: string | undefined): T => {
+  if (name != null && result != null && typeof result === 'object' && name in (result as Record<string, unknown>)) {
+    return (result as Record<string, unknown>)[name] as T
+  }
+  return result
+}

package/src/utils/stream.ts

@@ -0,0 +1,58 @@
+import { LlmModelError } from '../errors.js'
+import { MODEL_STREAM_TIMEOUT_MS } from '../consts.js'
+
+/**
+ * Extract `finish_reason` from a stream chunk regardless of whether it is a plain
+ * `AIMessageChunk` (ask/talk) or a `{ raw, parsed }` combined chunk (structured output).
+ */
+export const getChunkFinishReason = (chunk: unknown): string | undefined => {
+  const c = chunk as {
+    response_metadata?: { finish_reason?: string }
+    raw?: { response_metadata?: { finish_reason?: string } }
+  }
+  return c.response_metadata?.finish_reason ?? c.raw?.response_metadata?.finish_reason
+}
+
+/**
+ * Iterate a model stream under an IDLE (inactivity) deadline. `start` receives an
+ * `AbortSignal` to forward to `model.stream(..., { signal })`; the timer is re-armed on
+ * every received chunk, so it only fires after `timeoutMs` of SILENCE — a provider that
+ * accepted the request but stalls and never streams another token. On fire the call is
+ * aborted and surfaced as a retryable {@link LlmModelError} so the retry escalator moves
+ * on instead of hanging forever. Because the timer resets per token, long but actively
+ * streaming generations are never aborted.
+ *
+ * The loop also breaks after the first chunk carrying a non-empty `finish_reason`. Some
+ * providers send the final SSE data event twice, which makes `AIMessageChunk.concat()`
+ * double-append every string field (`finish_reason` becomes `'stopstop'`, the model name
+ * doubles) and corrupts accumulated tool-call argument strings, breaking structured-output
+ * parsing. Nothing meaningful arrives after `finish_reason`, so breaking there is safe.
+ */
+export async function* streamWithDeadline<T>(
+  start: (signal: AbortSignal) => Promise<AsyncIterable<T>>,
+  timeoutMs: number = MODEL_STREAM_TIMEOUT_MS,
+): AsyncGenerator<T> {
+  const controller = new AbortController()
+  let timer: ReturnType<typeof setTimeout>
+  const arm = () => {
+    clearTimeout(timer)
+    timer = setTimeout(() => controller.abort(), timeoutMs)
+  }
+  arm()
+  try {
+    const stream = await start(controller.signal)
+    for await (const chunk of stream) {
+      arm() // reset the idle timer on each received token
+      yield chunk
+      const reason = getChunkFinishReason(chunk)
+      if (reason != null && reason !== '') break
+    }
+  } catch (e) {
+    if (controller.signal.aborted) {
+      throw new LlmModelError(`stream-stalled:no token for ${timeoutMs}ms (idle deadline)`)
+    }
+    throw e
+  } finally {
+    clearTimeout(timer!)
+  }
+}

package/tests/context.ts

@@ -0,0 +1,110 @@
+import { makeGates } from '@owlmeans/test'
+import { ModelProvider } from '@owlmeans/llm-common'
+import type { SpectatorArgument, SpectatorEntryLogged } from '@owlmeans/llm-common'
+import type { LlmSpectator, ModelConfig } from '@owlmeans/llm'
+
+/**
+ * Live-provider gates. An empty variable means the corresponding integration spec
+ * self-skips with a printed reason — never a failure. The offline specs never gate.
+ */
+export const gates = makeGates({
+  openrouter: ['OPENROUTER_SECRET'],
+  anthropic: ['ANTHROPIC_SECRET'],
+})
+
+/** Role aliases the fixtures register — mirrors how a consumer names its own roles. */
+export const Role = {
+  Analyst: 'analyst',
+  Picker: 'picker',
+} as const
+
+/**
+ * The OpenRouter preset the live specs resolve models through — the same shape a
+ * consumer builds (`Compatible` provider + baseUrl + a per-role reasoning cap), so the
+ * integration run exercises the real configuration path and not a special test one.
+ */
+export const openRouterConfigs = (): ModelConfig[] => [
+  {
+    alias: Role.Analyst,
+    provider: ModelProvider.Compatible,
+    model: 'z-ai/glm-5.1:nitro',
+    secret: process.env.OPENROUTER_SECRET!,
+    baseUrl: process.env.OPENROUTER_URL ?? 'https://openrouter.ai/api/v1',
+    maxTokens: 8192,
+    maxTokensCap: 32000,
+    reasoning: { max_tokens: 1024 },
+  },
+  {
+    alias: Role.Picker,
+    preset: Role.Analyst,
+    provider: ModelProvider.Compatible,
+    secret: process.env.OPENROUTER_SECRET!,
+  },
+]
+
+export const anthropicConfigs = (): ModelConfig[] => [
+  {
+    alias: Role.Analyst,
+    provider: ModelProvider.Anthropic,
+    model: 'claude-haiku-4-5-20251001',
+    secret: process.env.ANTHROPIC_SECRET!,
+    maxTokens: 4096,
+    maxTokensCap: 16000,
+  },
+]
+
+/**
+ * A config list that needs no credentials — for offline construction/plugin specs.
+ * The analyst is deliberately a chat-completions model: the `gpt-5*` family goes through
+ * the Responses API, which rejects sampling parameters, and specs here assert on them.
+ */
+export const offlineConfigs = (): ModelConfig[] => [
+  { alias: Role.Analyst, provider: ModelProvider.OpenAI, model: 'gpt-4.1-mini', secret: 'sk-test' },
+  {
+    alias: Role.Picker, provider: ModelProvider.Compatible, model: 'some/model', secret: 'sk-test',
+    baseUrl: 'https://openrouter.ai/api/v1',
+    fallback: { model: 'some/stronger-model' },
+  },
+]
+
+export interface RecordingSpectator extends LlmSpectator {
+  entries: SpectatorEntryLogged[]
+  nulls: unknown[]
+}
+
+/** In-memory spectator: records what the model logged, so specs can assert on it. */
+export const recordingSpectator = (): RecordingSpectator => {
+  const entries: SpectatorEntryLogged[] = []
+  const nulls: unknown[] = []
+  return {
+    entries,
+    nulls,
+    log: async (arg: SpectatorArgument) => {
+      const entry: SpectatorEntryLogged = {
+        id: `entry-${entries.length}`,
+        kind: 'general',
+        model: 'test-model',
+        purpose: { type: 'test' },
+        timestamp: entries.length,
+        ...arg,
+      }
+      entries.push(entry)
+      return entry
+    },
+    captureNull: async capture => {
+      nulls.push(capture)
+    },
+  }
+}
+
+/** Minimal object schema used by the structured-output specs. */
+export const SpecificationSchema = {
+  type: 'object',
+  title: 'Specification',
+  properties: {
+    title: { type: 'string' },
+    summary: { type: 'string' },
+  },
+  required: ['title', 'summary'],
+  additionalProperties: true,
+} as const