@owlmeans/llm 0.1.14

package/tests/execution.spec.ts

@@ -0,0 +1,200 @@
+import { beforeEach, describe, expect, test } from 'bun:test'
+import { ExecutionEffort, ExecutionLevel } from '@owlmeans/llm-common'
+import type { ExecutionState, ModelConfigPatch, TaskExecutionState } from '@owlmeans/llm-common'
+import { DEFAULT_EFFORT, EFFORT_TABLE, makeExecutionService, makeLlmService } from '@owlmeans/llm'
+import type { ExecutionService, ProjectExecution, TaskExecution } from '@owlmeans/llm'
+import { offlineConfigs, Role } from './context.js'
+
+let service: ExecutionService
+let root: ProjectExecution
+/** Every `getModel` call the executions made, in order — asserts policy resolution. */
+let resolved: Array<{ alias: string, override?: Partial<ModelConfigPatch> }>
+
+const makeService = () => {
+  const llm = makeLlmService({ models: offlineConfigs }, `spec-exec-llm-${resolved.length}`)
+  const recording = {
+    ...llm,
+    getModel: (alias: string, override?: Partial<ModelConfigPatch>) => {
+      resolved.push({ alias, override })
+      return llm.getModel(alias, override)
+    },
+  }
+  return () => recording as unknown as ReturnType<typeof makeLlmService>
+}
+
+beforeEach(() => {
+  resolved = []
+  service = makeExecutionService(`spec-exec-${Math.trunc(performance.now() * 1000)}`)
+  root = service.root({
+    models: makeService(),
+    policy: { effort: DEFAULT_EFFORT },
+    purpose: { type: 'spec' },
+  })
+})
+
+describe('@owlmeans/llm — execution immutability', () => {
+  test('every construction and refinement returns a frozen object', () => {
+    expect(Object.isFrozen(root)).toBe(true)
+    expect(Object.isFrozen(service.forTask(root, {}))).toBe(true)
+    expect(Object.isFrozen(service.forHelper(root, { role: Role.Analyst }))).toBe(true)
+    expect(Object.isFrozen(service.derive(root, {}))).toBe(true)
+  })
+
+  test('refining never mutates the parent', () => {
+    const escalated = service.escalate(root, { effort: ExecutionEffort.Max })
+    expect(escalated).not.toBe(root)
+    expect(root.policy.effort).toBe(DEFAULT_EFFORT)
+    expect(escalated.policy.effort).toBe(ExecutionEffort.Max)
+  })
+
+  test('purpose refinement is additive and local', () => {
+    const dedicated = service.withPurpose(root, { dedication: 'inner' })
+    expect(dedicated.purpose).toEqual({ type: 'spec', dedication: 'inner' })
+    expect(root.purpose.dedication).toBeUndefined()
+  })
+})
+
+describe('@owlmeans/llm — execution levels', () => {
+  test('a task inherits the parent context and composes its own state', () => {
+    const task = service.forTask(root, { phase: 'draft', data: { step: 1 } })
+    expect(task.level).toBe(ExecutionLevel.Task)
+    expect(task.state.phase).toBe('draft')
+    expect(task.state.data).toEqual({ step: 1 })
+    expect(task.models).toBe(root.models)
+  })
+
+  test('a task effort applies to the task and everything derived from it', () => {
+    const task = service.forTask(root, { effort: ExecutionEffort.High })
+    expect(task.policy.effort).toBe(ExecutionEffort.High)
+    expect(service.forHelper(task, { role: Role.Analyst }).policy.effort).toBe(ExecutionEffort.High)
+    expect(root.policy.effort).toBe(DEFAULT_EFFORT)
+  })
+
+  test('a helper resolves a model and a temperature factory bound to its role', () => {
+    const helper = service.forHelper(root, { role: Role.Analyst, dedication: 'summarise' })
+    expect(helper.level).toBe(ExecutionLevel.Helper)
+    expect(helper.role).toBe(Role.Analyst)
+    expect(helper.model).toBeDefined()
+    expect(helper.purpose.dedication).toBe('summarise')
+    expect(resolved[0]?.alias).toBe(Role.Analyst)
+
+    helper.temperatureFactory(0.9)
+    expect(resolved.at(-1)).toEqual({ alias: Role.Analyst, override: { temperature: 0.9, topP: 0.8 } })
+  })
+
+  test('a helper effort bump does not escalate the branch it came from', () => {
+    const helper = service.forHelper(root, { role: Role.Analyst, effort: ExecutionEffort.Max })
+    expect(helper.policy.effort).toBe(ExecutionEffort.Max)
+    expect(root.policy.effort).toBe(DEFAULT_EFFORT)
+  })
+
+  test('a helper derived from a task carries no resumable state of its own', () => {
+    const task = service.forTask(root, { phase: 'draft' })
+    const helper = service.forHelper(task, { role: Role.Analyst })
+    expect((helper as unknown as { state?: unknown }).state).toBeUndefined()
+  })
+})
+
+describe('@owlmeans/llm — model policy resolution', () => {
+  test('the effort tier is merged into the resolution override', () => {
+    const high = service.escalate(root, { effort: ExecutionEffort.High })
+    service.model(high, Role.Analyst)
+    expect(resolved.at(-1)?.override).toEqual(EFFORT_TABLE[ExecutionEffort.High])
+  })
+
+  test('a role override remaps which role is actually resolved', () => {
+    const remapped = service.escalate(root, { roleOverrides: { [Role.Analyst]: Role.Picker } })
+    service.model(remapped, Role.Analyst)
+    expect(resolved.at(-1)?.alias).toBe(Role.Picker)
+  })
+
+  test('an explicit model override wins over the effort tier', () => {
+    const pinned = service.escalate(root, {
+      effort: ExecutionEffort.High,
+      modelOverrides: { [Role.Analyst]: { maxTokens: 999 } },
+    })
+    service.model(pinned, Role.Analyst)
+    expect(resolved.at(-1)?.override).toEqual({ ...EFFORT_TABLE[ExecutionEffort.High], maxTokens: 999 })
+  })
+
+  test('a call-site override wins over the policy', () => {
+    const pinned = service.escalate(root, { modelOverrides: { [Role.Analyst]: { maxTokens: 999 } } })
+    service.model(pinned, Role.Analyst, { maxTokens: 111 })
+    expect(resolved.at(-1)?.override).toEqual({ maxTokens: 111 })
+  })
+
+  test('a string override is read as a preset alias', () => {
+    const pinned = service.escalate(root, { modelOverrides: { [Role.Analyst]: Role.Picker } })
+    service.model(pinned, Role.Analyst)
+    expect(resolved.at(-1)?.override).toEqual({ preset: Role.Picker })
+  })
+
+  test('undefined keys never reach the factory', () => {
+    service.model(root, Role.Analyst, { maxTokens: undefined })
+    expect(resolved.at(-1)?.override).toEqual({})
+  })
+})
+
+describe('@owlmeans/llm — snapshot and restore', () => {
+  test('a snapshot carries the state and none of the collaborators', () => {
+    const state = service.snapshot(root) as ExecutionState & Record<string, unknown>
+    expect(state.level).toBe(ExecutionLevel.Project)
+    expect(state.purpose).toEqual({ type: 'spec' })
+    expect(state.models).toBeUndefined()
+    expect(JSON.stringify(state)).toBeString()
+  })
+
+  // Regression: `composeExecState` used to copy `state` itself, so every refinement of a
+  // task nested another copy of the previous state (state.state.state…), growing the
+  // snapshot without bound.
+  test('repeatedly refining a task never nests its state', () => {
+    let task: TaskExecution = service.forTask(root, { phase: 'draft' })
+    for (let i = 0; i < 5; i++) {
+      task = service.escalate(task, { effort: ExecutionEffort.High })
+      task = service.withPurpose(task, { dedication: `round-${i}` })
+      task = service.derive(task, {})
+    }
+    expect((task.state as unknown as { state?: unknown }).state).toBeUndefined()
+    expect((service.snapshot(task) as unknown as { state?: unknown }).state).toBeUndefined()
+    expect(task.state.phase).toBe('draft')
+    expect(task.purpose.dedication).toBe('round-4')
+  })
+
+  test('a task snapshot keeps the resumable fields across refinement', () => {
+    const task = service.forTask(root, { phase: 'draft', data: { step: 1 } })
+    const advanced = service.escalate(task, { effort: ExecutionEffort.Max })
+    const state = service.snapshot(advanced) as TaskExecutionState
+    expect(state.phase).toBe('draft')
+    expect(state.data).toEqual({ step: 1 })
+    expect(state.policy.effort).toBe(ExecutionEffort.Max)
+  })
+
+  test('restore re-attaches the collaborators to a persisted state', () => {
+    const task = service.forTask(root, { phase: 'draft' })
+    const state = JSON.parse(JSON.stringify(service.snapshot(task))) as ExecutionState
+    const restored = service.restore(state, { models: root.models })
+    expect(restored.level).toBe(ExecutionLevel.Task)
+    expect(restored.models).toBe(root.models)
+    expect((restored as TaskExecution).state.phase).toBe('draft')
+    expect(Object.isFrozen(restored)).toBe(true)
+  })
+})
+
+describe('@owlmeans/llm — resilience plugin seam', () => {
+  test('checkpoint is a no-op until a plugin is registered', async () => {
+    await expect(service.checkpoint(root, 'key')).resolves.toBeUndefined()
+  })
+
+  test('a registered plugin receives the JSON-safe state and the execution', async () => {
+    const seen: Array<{ state: ExecutionState, key?: string }> = []
+    service.use({ onCheckpoint: async (state, _exec, key) => { seen.push({ state, key }) } })
+
+    const task = service.forTask(root, { phase: 'draft' })
+    await service.checkpoint(task, 'project-1')
+
+    expect(seen).toHaveLength(1)
+    expect(seen[0]!.key).toBe('project-1')
+    expect((seen[0]!.state as TaskExecutionState).phase).toBe('draft')
+    expect((seen[0]!.state as unknown as { models?: unknown }).models).toBeUndefined()
+  })
+})

package/tests/helpers.spec.ts

@@ -0,0 +1,192 @@
+import { describe, expect, test } from 'bun:test'
+import { AIMessage, HumanMessage } from '@langchain/core/messages'
+import {
+  coerceToSchema, LlmRetryExceededError, normalizeInput, parseJsonContent, registerFatalError,
+  spectate, withRetry,
+} from '@owlmeans/llm'
+import { recordingSpectator } from './context.js'
+
+describe('helpers/messages — input normalization', () => {
+  test('lifts a bare string into a user message', () => {
+    expect(normalizeInput('hello')).toEqual([{ role: 'user', content: 'hello' }])
+  })
+
+  test('wraps a single message and passes an array through', () => {
+    const msg = { role: 'system' as const, content: 'be terse' }
+    expect(normalizeInput(msg)).toEqual([msg])
+    expect(normalizeInput([msg, 'hi'])).toEqual([msg, { role: 'user', content: 'hi' }])
+  })
+
+  test('produces a fresh array the model may mutate without touching the caller', () => {
+    const input = [{ role: 'user' as const, content: 'hi' }]
+    const normalized = normalizeInput(input)
+    normalized.push({ role: 'user', content: 'extra' })
+    expect(input).toHaveLength(1)
+  })
+})
+
+describe('helpers/json — recovering JSON a model emitted as content', () => {
+  test('parses plain JSON', () => {
+    expect(parseJsonContent('{"a":1}')).toEqual({ a: 1 })
+  })
+
+  test('unwraps a markdown fence', () => {
+    expect(parseJsonContent('```json\n{"a":1}\n```')).toEqual({ a: 1 })
+    expect(parseJsonContent('```\n[1,2]\n```')).toEqual([1, 2])
+  })
+
+  test('salvages an object from surrounding prose', () => {
+    expect(parseJsonContent('Sure! Here it is: {"a":1} — hope that helps')).toEqual({ a: 1 })
+  })
+
+  test('salvages an array from surrounding prose', () => {
+    expect(parseJsonContent('Result: ["x","y"] done')).toEqual(['x', 'y'])
+  })
+
+  test('reads structured content blocks', () => {
+    expect(parseJsonContent([{ type: 'text', text: '{"a":' }, { type: 'text', text: '1}' }])).toEqual({ a: 1 })
+  })
+
+  test('returns null when there is nothing parseable', () => {
+    expect(parseJsonContent('no json here')).toBeNull()
+    expect(parseJsonContent('')).toBeNull()
+    expect(parseJsonContent(null)).toBeNull()
+    expect(parseJsonContent(42)).toBeNull()
+  })
+})
+
+describe('helpers/json — reconciling a model answer with its schema', () => {
+  // Observed in the wild: a model fills an `array` field with the STRING "[]".
+  test('parses stringified arrays and objects back', () => {
+    const schema = {
+      type: 'object',
+      properties: { files: { type: 'array', items: { type: 'string' } } },
+    }
+    expect(coerceToSchema({ files: '["a","b"]' }, schema)).toEqual({ files: ['a', 'b'] })
+    expect(coerceToSchema({ files: '[]' }, schema)).toEqual({ files: [] })
+  })
+
+  test('coerces stringified scalars to their declared type', () => {
+    const schema = {
+      type: 'object',
+      properties: { count: { type: 'integer' }, ok: { type: 'boolean' }, ratio: { type: 'number' } },
+    }
+    expect(coerceToSchema({ count: '7', ok: 'true', ratio: '0.5' }, schema))
+      .toEqual({ count: 7, ok: true, ratio: 0.5 })
+  })
+
+  // A value that cannot be coerced is kept, so validation reports the real problem.
+  test('keeps an uncoercible value untouched', () => {
+    const schema = { type: 'object', properties: { count: { type: 'integer' } } }
+    expect(coerceToSchema({ count: 'seven' }, schema)).toEqual({ count: 'seven' })
+  })
+
+  // Also observed: a `string[]` field filled with `[{ path: '…' }]`.
+  test('unwraps a scalar the model over-wrapped in an object', () => {
+    const schema = { type: 'array', items: { type: 'string' } }
+    expect(coerceToSchema([{ path: 'src/a.ts' }, { value: 'src/b.ts' }], schema))
+      .toEqual(['src/a.ts', 'src/b.ts'])
+    expect(coerceToSchema([{ onlyOne: 'src/c.ts' }], schema)).toEqual(['src/c.ts'])
+  })
+
+  test('walks nested structures', () => {
+    const schema = {
+      type: 'object',
+      properties: {
+        items: {
+          type: 'array',
+          items: { type: 'object', properties: { n: { type: 'integer' } } },
+        },
+      },
+    }
+    expect(coerceToSchema({ items: [{ n: '1' }, { n: '2' }] }, schema))
+      .toEqual({ items: [{ n: 1 }, { n: 2 }] })
+  })
+
+  test('leaves conforming values and unknown schemas alone', () => {
+    expect(coerceToSchema({ a: 1 }, { type: 'object', properties: { a: { type: 'integer' } } })).toEqual({ a: 1 })
+    expect(coerceToSchema('x', undefined)).toBe('x')
+  })
+})
+
+describe('helpers/retry', () => {
+  test('returns the first success and reports the attempt index', async () => {
+    const attempts: number[] = []
+    const result = await withRetry({ retries: 5 }, async i => {
+      attempts.push(i)
+      if (i < 2) throw new Error('transient')
+      return 'ok'
+    })
+    expect(result).toBe('ok')
+    expect(attempts).toEqual([0, 1, 2])
+  })
+
+  test('exhausting the budget throws, carrying the last error and attempt', async () => {
+    const failure = new Error('always')
+    let thrown: unknown
+    try {
+      await withRetry({ retries: 3 }, async () => { throw failure })
+    } catch (e) {
+      thrown = e
+    }
+    expect(thrown).toBeInstanceOf(LlmRetryExceededError)
+    expect((thrown as LlmRetryExceededError).cause).toBe(failure)
+    expect((thrown as LlmRetryExceededError).attempt).toBe(2)
+  })
+
+  test('a per-call fatal predicate aborts immediately', async () => {
+    let calls = 0
+    const boom = new Error('unrecoverable')
+    await expect(withRetry(
+      { retries: 5, fatal: e => e === boom ? boom : null },
+      async () => { calls += 1; throw boom }
+    )).rejects.toThrow('unrecoverable')
+    expect(calls).toBe(1)
+  })
+
+  // This is how a host registers "budget exhausted" so a model call stops retrying.
+  test('a globally registered resolver aborts every retry loop', async () => {
+    class SpecBudgetError extends Error { }
+    registerFatalError(e => e instanceof SpecBudgetError ? e : null)
+
+    let calls = 0
+    await expect(withRetry({ retries: 5 }, async () => {
+      calls += 1
+      throw new SpecBudgetError('out of budget')
+    })).rejects.toThrow('out of budget')
+    expect(calls).toBe(1)
+  })
+})
+
+describe('helpers/spectate', () => {
+  test('logs every prompt message plus the completion, in order', async () => {
+    const spectator = recordingSpectator()
+    await spectate(spectator, 'ask')(
+      [new HumanMessage('question'), { role: 'system', content: 'be terse' }, 'plain'],
+      new AIMessage('answer'), 'test-action', 2, 1000,
+    )
+
+    expect(spectator.entries).toHaveLength(1)
+    const entry = spectator.entries[0]!
+    expect(entry.action).toBe('test-action')
+    expect(entry.retries).toBe(2)
+    expect(entry.startedAt).toBe(1000)
+    expect(entry.messages).toHaveLength(4)
+    expect(entry.messages.map(m => m.callType)).toEqual(['ask', 'ask', 'ask', 'ask'])
+    expect(entry.messages[1]!.type).toBe('system')
+    expect(entry.messages[3]!.content).toBe('answer')
+  })
+
+  test('records tool-call arguments rather than the empty content of a tool completion', async () => {
+    const spectator = recordingSpectator()
+    const completion = new AIMessage({
+      content: '',
+      tool_calls: [{ id: '1', name: 'extract', args: { a: 1 } }],
+    })
+    await spectate(spectator, 'invoke')(['q'], completion, 'structured', 0)
+
+    const last = spectator.entries[0]!.messages.at(-1)!
+    expect(last.contentType).toBe('tool_call')
+    expect(last.content).toEqual([{ id: '1', name: 'extract', args: { a: 1 } }] as unknown as string)
+  })
+})

package/tests/internals.spec.ts

@@ -0,0 +1,141 @@
+import { describe, expect, test } from 'bun:test'
+import type { MessageFieldWithRole } from '@langchain/core/messages'
+import { JSON_INSTRUCTION, NO_THINK_DIRECTIVE } from '../src/consts.js'
+import { applyNoThink, ensureJsonMention } from '../src/utils/prompt.js'
+import { toToolName, unwrapNamed } from '../src/utils/schema.js'
+import { getChunkFinishReason, streamWithDeadline } from '../src/utils/stream.js'
+
+/**
+ * Internal utilities — deliberately not part of the package surface (`utils/` is
+ * library-private; `helpers/` is what a consumer may use alongside a model), so they are
+ * imported from source.
+ */
+
+describe('utils/prompt — JSON mention', () => {
+  test('appends the instruction when nothing mentions JSON', () => {
+    const msgs: MessageFieldWithRole[] = [{ role: 'user', content: 'Describe the project' }]
+    ensureJsonMention(msgs)
+    expect(msgs).toHaveLength(1)
+    expect(msgs[0]!.content).toBe(`Describe the project\n${JSON_INSTRUCTION}`)
+  })
+
+  test('leaves the prompt alone when JSON is already mentioned, in any casing', () => {
+    const msgs: MessageFieldWithRole[] = [{ role: 'user', content: 'Answer as JSON please' }]
+    ensureJsonMention(msgs)
+    expect(msgs[0]!.content).toBe('Answer as JSON please')
+  })
+
+  test('finds the mention inside structured content blocks', () => {
+    const msgs = [{ role: 'user', content: [{ type: 'text', text: 'reply in json' }] }] as unknown as MessageFieldWithRole[]
+    ensureJsonMention(msgs)
+    expect(msgs).toHaveLength(1)
+  })
+
+  test('pushes a new message when the last one cannot be appended to', () => {
+    const msgs = [{ role: 'user', content: [{ type: 'text', text: 'look at this' }] }] as unknown as MessageFieldWithRole[]
+    ensureJsonMention(msgs)
+    expect(msgs).toHaveLength(2)
+    expect(msgs[1]!.content).toBe(JSON_INSTRUCTION)
+  })
+})
+
+describe('utils/prompt — thinking suppression', () => {
+  test('injects the soft switch only when the config asks for it', () => {
+    const off: MessageFieldWithRole[] = [{ role: 'user', content: 'hi' }]
+    applyNoThink(off, undefined)
+    expect(off[0]!.content).toBe('hi')
+
+    const on: MessageFieldWithRole[] = [{ role: 'user', content: 'hi' }]
+    applyNoThink(on, true)
+    expect(on[0]!.content).toBe(`hi\n${NO_THINK_DIRECTIVE}`)
+  })
+
+  test('is idempotent', () => {
+    const msgs: MessageFieldWithRole[] = [{ role: 'user', content: 'hi' }]
+    applyNoThink(msgs, true)
+    applyNoThink(msgs, true)
+    expect(msgs[0]!.content).toBe(`hi\n${NO_THINK_DIRECTIVE}`)
+  })
+})
+
+describe('utils/schema — tool naming and unwrapping', () => {
+  test('sanitises a schema title into a provider-acceptable tool name', () => {
+    expect(toToolName('User Story')).toBe('User_Story')
+    expect(toToolName('spec.v2/final')).toBe('spec_v2_final')
+    expect(toToolName('__weird__')).toBe('weird')
+  })
+
+  test('falls back to the default name when nothing usable is present', () => {
+    expect(toToolName(undefined)).toBe('extract')
+    expect(toToolName('!!!')).toBe('extract')
+  })
+
+  test('unwraps a named envelope, and passes anything else through', () => {
+    expect(unwrapNamed({ spec: { a: 1 } }, 'spec')).toEqual({ a: 1 })
+    expect(unwrapNamed({ a: 1 }, 'spec')).toEqual({ a: 1 })
+    expect(unwrapNamed({ a: 1 }, undefined)).toEqual({ a: 1 })
+    expect(unwrapNamed('plain' as unknown as object, 'spec')).toBe('plain' as unknown as object)
+  })
+})
+
+describe('utils/stream — idle deadline and duplicate-final-chunk dedup', () => {
+  const chunks = (...items: unknown[]): AsyncIterable<unknown> => ({
+    async *[Symbol.asyncIterator]() {
+      for (const item of items) yield item
+    },
+  })
+
+  const collect = async (stream: AsyncGenerator<unknown>): Promise<unknown[]> => {
+    const out: unknown[] = []
+    for await (const chunk of stream) out.push(chunk)
+    return out
+  }
+
+  test('yields every chunk of a well-behaved stream', async () => {
+    const out = await collect(streamWithDeadline(async () => chunks({ a: 1 }, { a: 2 }), 1000))
+    expect(out).toEqual([{ a: 1 }, { a: 2 }])
+  })
+
+  // Some providers send the final SSE data event twice; concatenating it doubles every
+  // string field and corrupts accumulated tool-call arguments.
+  test('stops at the first chunk carrying a finish_reason', async () => {
+    const final = { response_metadata: { finish_reason: 'stop' } }
+    const out = await collect(streamWithDeadline(async () => chunks({ a: 1 }, final, final), 1000))
+    expect(out).toEqual([{ a: 1 }, final])
+  })
+
+  test('reads the finish reason out of a combined structured chunk too', () => {
+    expect(getChunkFinishReason({ raw: { response_metadata: { finish_reason: 'length' } } })).toBe('length')
+    expect(getChunkFinishReason({})).toBeUndefined()
+  })
+
+  test('an empty finish_reason does not end the stream early', async () => {
+    const out = await collect(streamWithDeadline(
+      async () => chunks({ response_metadata: { finish_reason: '' } }, { a: 2 }), 1000
+    ))
+    expect(out).toHaveLength(2)
+  })
+
+  test('a provider that accepts the request and then goes silent is aborted, not awaited forever', async () => {
+    const stalled = (signal: AbortSignal): Promise<AsyncIterable<unknown>> => Promise.resolve({
+      async *[Symbol.asyncIterator]() {
+        yield { a: 1 }
+        await new Promise<void>((resolve, reject) => {
+          signal.addEventListener('abort', () => reject(new Error('aborted')))
+        })
+      },
+    })
+
+    const started = Date.now()
+    await expect(collect(streamWithDeadline(stalled, 60))).rejects.toThrow(/stream-stalled/)
+    // The deadline is per-token, so the first chunk re-arms it: expect roughly one window.
+    expect(Date.now() - started).toBeLessThan(2000)
+  })
+
+  test('an error from the provider itself is surfaced unchanged', async () => {
+    const failing = async (): Promise<AsyncIterable<unknown>> => {
+      throw new Error('401 unauthorized')
+    }
+    await expect(collect(streamWithDeadline(failing, 1000))).rejects.toThrow('401 unauthorized')
+  })
+})

package/tests/model.spec.ts

@@ -0,0 +1,116 @@
+import { describe, expect, test } from 'bun:test'
+import { Ajv } from 'ajv'
+import type { JSONSchemaType } from 'ajv'
+import { DEFAULT_EFFORT, makeExecutionService, makeLlmModel, makeLlmService } from '@owlmeans/llm'
+import type { HelperExecution, LlmModel } from '@owlmeans/llm'
+import {
+  anthropicConfigs, gates, openRouterConfigs, recordingSpectator, Role, SpecificationSchema,
+} from './context.js'
+import type { ModelConfig } from '@owlmeans/llm'
+
+/**
+ * Integration specs against a real provider. They reproduce the LIVE call path a consumer
+ * uses — resolve a model through the execution service by role, wrap it with a spectator,
+ * then ask/talk/invoke/request — rather than a special test-only arrangement.
+ *
+ * Gated on credentials: with none configured the suite reports the reason and skips.
+ */
+
+interface Specification {
+  title: string
+  summary: string
+}
+
+const validate = new Ajv({ strict: false }).compile(SpecificationSchema as unknown as JSONSchemaType<Specification>)
+
+const PROMPT = 'Produce a 50 word specification for a to-do list web application.'
+
+/** Build the same three-layer arrangement a consumer builds, and return a model helper. */
+const liveModel = (configs: () => ModelConfig[], alias: string): {
+  model: LlmModel, spectator: ReturnType<typeof recordingSpectator>, helper: HelperExecution
+} => {
+  const llm = makeLlmService({ models: configs }, `${alias}-llm`)
+  const executions = makeExecutionService(`${alias}-exec`)
+  const root = executions.root({
+    models: () => llm,
+    policy: { effort: DEFAULT_EFFORT },
+    purpose: { type: 'integration-spec' },
+    outputErrors: true,
+  })
+  const helper = executions.forHelper(root, { role: Role.Analyst, dedication: 'specification' })
+  const spectator = recordingSpectator()
+
+  return {
+    helper,
+    spectator,
+    model: makeLlmModel({
+      model: helper.model,
+      purpose: helper.purpose,
+      outputErrors: true,
+      retries: 3,
+    }, spectator),
+  }
+}
+
+const suite = (name: string, configs: () => ModelConfig[], skip: string | null) => {
+  describe(`@owlmeans/llm — live model (${name})`, () => {
+    test.skipIf(skip != null)(`ask returns text${skip != null ? ` — SKIPPED: ${skip}` : ''}`, async () => {
+      const { model, spectator } = liveModel(configs, `${name}-ask`)
+      const result = await model.ask(PROMPT, { action: 'spec-ask' })
+      expect(typeof result).toBe('string')
+      expect(result.trim().length).toBeGreaterThan(0)
+      // Every call is observable: the prompt and the completion reach the spectator.
+      expect(spectator.entries).toHaveLength(1)
+      expect(spectator.entries[0]!.action).toBe('spec-ask')
+      expect(spectator.entries[0]!.messages.length).toBeGreaterThanOrEqual(2)
+    }, 480000)
+
+    test.skipIf(skip != null)('talk returns the raw message', async () => {
+      const { model } = liveModel(configs, `${name}-talk`)
+      const result = await model.talk(PROMPT, { action: 'spec-talk' })
+      expect(typeof result).toBe('object')
+      expect(typeof result.content).toBe('string')
+    }, 480000)
+
+    test.skipIf(skip != null)('invoke returns a schema-validated object', async () => {
+      const { model } = liveModel(configs, `${name}-invoke`)
+      const result = await model.invoke<Specification>(
+        PROMPT, SpecificationSchema as unknown as JSONSchemaType<Specification>,
+        { action: 'spec-invoke' }
+      )
+      expect(validate(result)).toBe(true)
+      expect(result.title).toBeString()
+    }, 480000)
+
+    test.skipIf(skip != null)('request returns a message whose content is the validated JSON', async () => {
+      const { model } = liveModel(configs, `${name}-request`)
+      const result = await model.request<Specification>(
+        PROMPT, SpecificationSchema as unknown as JSONSchemaType<Specification>,
+        { action: 'spec-request' }
+      )
+      expect(typeof result.content).toBe('string')
+      expect(validate(JSON.parse(result.content as string))).toBe(true)
+    }, 480000)
+
+    test.skipIf(skip != null)('a filter that rejects the output is retried, then surfaced', async () => {
+      const { model } = liveModel(configs, `${name}-filter`)
+      let seen = 0
+      await expect(model.ask(PROMPT, {
+        action: 'spec-filter',
+        filter: async () => { seen += 1; return null },
+      })).rejects.toThrow()
+      expect(seen).toBeGreaterThan(1)
+    }, 480000)
+
+    test.skipIf(skip != null)('a ref receives the message and the spectator entry', async () => {
+      const { model } = liveModel(configs, `${name}-ref`)
+      const ref: Parameters<LlmModel['ask']>[1]['ref'] = {}
+      await model.ask(PROMPT, { action: 'spec-ref', ref })
+      expect(ref.value).toBeDefined()
+      expect(ref.spectatorEntry?.action).toBe('spec-ref')
+    }, 480000)
+  })
+}
+
+suite('openrouter', openRouterConfigs, 'skip' in gates.openrouter ? gates.openrouter.reason : null)
+suite('anthropic', anthropicConfigs, 'skip' in gates.anthropic ? gates.anthropic.reason : null)