@ultimat3/ai 29.4.0 → 30.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CLAUDE.md +63 -90
  2. package/README.md +125 -8
  3. package/package.json +23 -11
  4. package/src/agent-checkpoint.ts +44 -0
  5. package/src/agent-compact-def.ts +11 -0
  6. package/src/agent-compact-measure.ts +38 -0
  7. package/src/agent-compact-request.ts +100 -0
  8. package/src/agent-compact-text.ts +95 -0
  9. package/src/agent-compact.ts +75 -50
  10. package/src/agent-continue.ts +15 -3
  11. package/src/agent-def.ts +9 -0
  12. package/src/agent-errors.ts +12 -2
  13. package/src/agent-facts.ts +8 -0
  14. package/src/agent-send.ts +82 -0
  15. package/src/agent-stream.ts +76 -0
  16. package/src/agent.ts +71 -21
  17. package/src/answer-errors.ts +1 -1
  18. package/src/anthropic-provider.ts +291 -0
  19. package/src/content-errors.ts +1 -1
  20. package/src/context-overflow.ts +86 -0
  21. package/src/error-codes.ts +95 -0
  22. package/src/errors-retry.ts +8 -4
  23. package/src/errors.ts +6 -87
  24. package/src/eval-agent.ts +100 -0
  25. package/src/eval-errors.ts +31 -5
  26. package/src/evals.ts +125 -45
  27. package/src/gateway-providers.ts +215 -0
  28. package/src/gateway.ts +63 -153
  29. package/src/hive-errors.ts +1 -1
  30. package/src/in-process-tool.ts +20 -11
  31. package/src/index.ts +31 -7
  32. package/src/llm.ts +39 -8
  33. package/src/model-resolve.ts +7 -1
  34. package/src/openai-body.ts +12 -5
  35. package/src/openai-provider.ts +28 -8
  36. package/src/openai-wire.ts +11 -7
  37. package/src/prompt-cache-rows.ts +82 -0
  38. package/src/prompt-cache.ts +24 -2
  39. package/src/provider.ts +15 -274
  40. package/src/remote-embedder.ts +38 -11
  41. package/src/run-meter.ts +40 -2
  42. package/src/runtime.ts +38 -3
  43. package/src/scorers.ts +35 -1
  44. package/src/scripted-provider.ts +146 -0
  45. package/src/tools-gate.ts +46 -0
  46. package/src/tools.ts +101 -27
  47. package/src/wire.ts +12 -4
package/CLAUDE.md CHANGED
@@ -9,13 +9,9 @@ Shipped: `action` (`llm()`), `cache`, `core`, `db` (pgvector), `http` (`statusFo
9
9
  Keep `policy` dev-only: tests and `*-fixture.ts` harnesses import `allow`. The
10
10
  `!src/**/*-fixture.ts` exclusion keeps harnesses unpublished; their imports still need a declaration.
11
11
 
12
- `mcp` is the same tier, so the LLM-tool projection is restated structurally in `tools.ts`, never
13
- imported: same contract, same *decision* (`isMcpExposed` from `@ultimat3/core`, never a local
14
- `=== true`), same NAME (`action.name` verbatim, never derived) and the same TOOL —
15
- `asProjectableAction` and mcp's `primitiveFromAction` are two copies the tier table forces, held
16
- equal by `scripts/ai-mcp-tool-parity.test.ts` (name, description, exposure, the whole input schema
17
- incl. `tool-idempotency.ts`'s reserved `idempotencyKey`, the policy verdict, the replay). A field
18
- one adds, the other adds.
12
+ `mcp` is the same tier, so the LLM-tool projection is restated in `tools.ts`, never imported —
13
+ `asProjectableAction` and mcp's `primitiveFromAction` held equal by
14
+ `scripts/ai-mcp-tool-parity.test.ts`; a field one adds, the other adds. Full rule: `docs/history/ai.md`.
19
15
 
20
16
  ## Owns
21
17
 
@@ -24,10 +20,12 @@ one adds, the other adds.
24
20
  | `models.ts` | the model REGISTRY: `registerModel`, limits, prices, the reasoning controls each one accepts — and NO row of its own |
25
21
  | `model-resolve.ts` | `resolveModel` / `modelForCall` — declaration → prompt → gateway `defaultModel`, none is `X_AI_MODEL_UNRESOLVED` naming all three; `describedModel` for a published fact |
26
22
  | `model-fixture.ts` | the rows this package's own suites register (`useFixtureModels()`). Not shipped |
27
- | `provider.ts` | `Provider` interface, the request half, `anthropicProvider()` |
28
- | `echo-provider.ts` | `echoProvider()`, the deterministic double — split from `provider.ts` at its line ceiling |
23
+ | `provider.ts` | `Provider` interface, the request half; re-exports the Anthropic names |
24
+ | `anthropic-provider.ts` | `anthropicProvider()`, its stream-only rule (`requiresStreaming`) and message parse |
25
+ | `echo-provider.ts` | `echoProvider()`, the deterministic double — split from `provider.ts` at its line ceiling. A fixture that needs a tool call uses `scriptedProvider`, never a hand-rolled `Provider` |
26
+ | `scripted-provider.ts` | `scriptedProvider()`, the scripted double that calls tools, and `X_AI_SCRIPT_EXHAUSTED`: one call past the script is refused, never answered with the last turn again |
29
27
  | `content-blocks.ts` | the `image` / `document` block shapes, the screen every request runs them through, their token estimate |
30
- | `content-errors.ts` | the `X_AI_CONTENT_UNSUPPORTED` class; its code and title belong in `errors.ts` |
28
+ | `content-errors.ts` | the `X_AI_CONTENT_UNSUPPORTED` class; its code and title belong in `error-codes.ts` |
31
29
  | `wire.ts` | the response half: `usage` / `stop_reason` shapes, and the SSE `MessageStream` |
32
30
  | `error-body.ts` | what a failure body SAYS (`detailOf`) and what must never survive into it (`withoutKey`) — one copy, both transports |
33
31
  | `sse.ts` | Server-Sent Events framing — protocol only, knows nothing about Anthropic |
@@ -35,13 +33,16 @@ one adds, the other adds.
35
33
  | `openai-messages.ts` | the format mapping's request half: `AiMessage` blocks → OpenAI messages, `LlmTool` → functions, `tool_choice` |
36
34
  | `openai-body.ts` | one chat-completions body, and the per-model reasoning mapping |
37
35
  | `openai-wire.ts` | its response half: one completion, and `ChatCompletionStream` (fragmented tool calls, trailing usage, `[DONE]`) |
38
- | `gateway.ts` | routing, retries, cache, budget wiring |
36
+ | `gateway.ts` | routing, cache, budget wiring |
37
+ | `gateway-providers.ts` | the retry loop across providers, `backoffMs`, `isRetryable`, the breakers |
39
38
  | `budget.ts` | token ledgers per request/actor/org, ALS carrier |
40
39
  | `prompt.ts` | `definePrompt`, `promptHash`, version registry |
41
40
  | `prompt-fence.ts` | a template's data fences: which tag pairs enclose each slot (`promptFences`), and the closer-breaking `render` applies inside one |
42
41
  | `embeddings.ts` | `Embedder`, `hashEmbedder()`, cosine helpers |
43
42
  | `remote-embedder.ts` | `remoteEmbedder()` — the production `/v1/embeddings` client |
44
43
  | `evals.ts` | `defineEval`, the run, the baseline gate, prompt coverage |
44
+ | `eval-agent.ts` | the agent arm of `defineEval`: one case is one `agent.run()` in the eval scope; `agentPrompt` reads the prompt off `agentFactsOf` |
45
+ | `eval-agent-fixture.ts` | the agent-arm suites' harness. Not shipped |
45
46
  | `eval-baseline.ts` | the recorded scores: path, read/write, what counts as a regression |
46
47
  | `scorers.ts` | what a `Scorer` is, the built-in ones, and `llmJudge` |
47
48
  | `vector.ts` | `VectorStore`, in-memory cosine + BM25, RRF hybrid |
@@ -51,9 +52,10 @@ one adds, the other adds.
51
52
  | `pg-vector.ts` | `postgresVectorStore()` — the production store |
52
53
  | `rag.ts` | chunker, retriever, reranker, budgeted context assembler |
53
54
  | `tools.ts` | action → LLM tool definition; the `AgentTool` union and `asProjectableAction`; `runLlmToolCall` |
55
+ | `tools-gate.ts` | a tool's way to `run`: actor half, the name's gate, the tool |
54
56
  | `tool-idempotency.ts` | an idempotent action's reserved `idempotencyKey` tool argument — MCP's, restated (same tier), held equal by the parity test |
55
57
  | `tool-arguments.ts` / `answer-errors.ts` | one argument-string reader for both wires and what an unreadable one means (`assertToolArguments`); `X_LLM_TRUNCATED` + `X_LLM_TOOL_ARGUMENTS_INVALID` |
56
- | `billed.ts` / `run-meter.ts` | `meta.billed` + `billedOf`; the meter every `llm()`/`agent()` provider call goes through |
58
+ | `billed.ts` / `run-meter.ts` | `meta.billed` + `billedOf`; the meter every `llm()`/`agent()` provider call goes through, and `meteredRun`, its one public total (`RunMeter` stays internal) |
57
59
  | `failure-disclosure.ts` | `discloseFailure` — the ONE rule for what of a throw a remote reader sees (tool result, hive member); the withheld half is logged, never its message or cause |
58
60
  | `llm.ts` | `llm()` — the model call, declared as an `action`; and what a streamed answer must satisfy |
59
61
  | `respond.ts` | the `respond` tool for one `output` (a non-object one wrapped in `{ value }`), its reader, and `parseJsonish` |
@@ -62,22 +64,24 @@ one adds, the other adds.
62
64
  | `agent-def.ts` / `agent-declared.ts` / `agent-system.ts` / `agent-usage.ts` | the declaration's types; its defaults, bound screen, limits and fact row; the system one run sends; its running usage |
63
65
  | `llm-def.ts` / `cost.ts` | `llm()`'s declaration types and `LlmCall`; the money arithmetic (`costOf`, `exactCostOf`) |
64
66
  | `agent-guards.ts` / `agent-errors.ts` | the repeated-turn detector and the run deadline; their two classes |
65
- | `prompt-cache.ts` | PROVIDER prompt caching: `SystemParts`, `PromptCacheHint`, `PROMPT_CACHE_RULES` (sources in its header) |
67
+ | `prompt-cache.ts` / `-rows.ts` | PROVIDER prompt caching, `PROMPT_CACHE_RULES` (sources in its header); the provider-rows screen |
66
68
  | `prompt-cache-blocks.ts` | the Anthropic wire's marks (chat-completions': `openai-messages.ts`) |
67
69
  | `agent-per-run.ts` / `agent-run-limits.ts` | one run's `offerTools()`/`model()`/`limits()`, narrow only; `costPerRun` |
68
- | `agent-compact-def.ts` / `agent-compact.ts` | `compact`'s declaration and event; the trigger, the cut, the summary call |
70
+ | `agent-compact{,-def,-measure,-request,-text}.ts` | trigger, cut; `compact`'s option and event; the token ratio; the summary request, as text when too long |
71
+ | `agent-send.ts` / `context-overflow.ts` / `agent-checkpoint.ts` | a turn sent, an overflow compacted once; `X_AI_CONTEXT_OVERFLOW`'s phrases; `onCheckpoint` |
69
72
  | `agent-continue.ts` / `agent-turn.ts` | `.run()` / `.continue()`, the transcript handle and its screen; one turn's report and refusals |
70
73
  | `agent-transcript.ts` | what one turn leaves in the transcript: the assistant replay, the tool results, the correction |
71
74
  | `agent-facts.ts` | `describeAgents()` — the agent registry, and the row a manifest WOULD publish (its consumer has to be `cli`, tier 5) |
72
75
  | `agent-job.ts` | `agentJob()` — an agent as a real `JobHandle`, composed from `job()` |
73
76
  | `hive.ts` | `hive()` — one action fanned out over many inputs, declared as an `action` |
74
77
  | `hive-result.ts` | `HiveMember` / `HiveResult`, and the SCHEMA built from the member's own `output` |
75
- | `hive-errors.ts` | the `X_HIVE_*` class; its code and title stay in `errors.ts` |
78
+ | `hive-errors.ts` | the `X_HIVE_*` class; its code and title stay in `error-codes.ts` |
76
79
  | `redaction.ts` | the one gate between `vars()` and the provider: a `Secret` never reaches a prompt |
77
- | `eval-errors.ts` | the five `X_EVAL_*` classes; their codes and titles stay in `errors.ts` |
80
+ | `eval-errors.ts` | the six `X_EVAL_*` classes; their codes and titles stay in `error-codes.ts` |
81
+ | `error-codes.ts` | the codes this package owns, their titles and the one `registerErrorCodes` call (`new-error-code --package ai` writes here); `errors.ts` holds the core classes |
78
82
  | `llm-cache.ts` | the semantic cache half of `llm()`: what a declaration may partition on, and the store it reaches |
79
83
  | `llm-fixture.ts` / `bounds-fixture.ts` | the `llm()` harness; the one `refusal`/`asyncRefusal` of every bound suite. Not shipped (`!src/**/*-fixture.ts`) |
80
- | `runtime.ts` | the ambient gateway / embedder / semantic caches an `llm()` reaches |
84
+ | `runtime.ts` | the ambient gateway / embedder / semantic caches an `llm()` reaches, and the private agent-eval scope (`withEvalScope`): `aiGateway`/`installedGateway` read its gateway first; `runLlmToolCall` resolves its doubles |
81
85
  | `fix-line*` | the package's dogfood eval — proof the baseline convention fails a build |
82
86
 
83
87
  ## Invariants — bounds and budget
@@ -124,6 +128,9 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
124
128
  - **`llm()` forwards `ctx.signal`** onto `GenerateRequest`; `.stream()` inherits it.
125
129
  - **`.stream()` is the SAME action**, marked with an ambient sink: UNVALIDATED text, then one `done`
126
130
  with the validated value; no repair turn (`X_LLM_STREAM_INVALID`), no `respond` tool. LAZY.
131
+ - **`llm()` streams over its own route** (`stream: LlmTextChunk`, exported): `text/event-stream`
132
+ gets `{ type: 'text' }` frames, then `done`; `thinking` NEVER crosses the wire. Drained
133
+ (`chunksRead()` false: JSON, MCP, `.job()`) it is the ordinary call, repair turn included.
127
134
  - **`configureAi({ redact })` is the one seam between `vars()` and the provider**; a `Secret` in
128
135
  `vars()` is `X_AI_PROMPT_SECRET` regardless.
129
136
  - **Fallback is across PROVIDERS serving one model, never across models**; `GenerateResult.provider`
@@ -147,14 +154,16 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
147
154
  - **The gateway's reads of a provider's throw are total**: `isRetryable` fails closed; the line goes
148
155
  through `renderThrowable`.
149
156
  - **`X_AI_PROVIDER_UNAVAILABLE` is the one code classified `retryable`** (`AI_ERROR_RETRY`);
150
- `AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Six
157
+ `AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Nine
151
158
  deterministic failures are `terminal` (`errors-retry.ts`); the rest stay UNREGISTERED on purpose.
159
+ - **One breaker per provider** (`gateway-providers.ts`, `DEFAULT_GATEWAY_BREAKER` `{ failures: 5,
160
+ cooldownMs: 30_000 }`, `breaker: 'off'` declines). Every settle carries the `admit()`
161
+ ticket. Full rule: `docs/history/ai.md`.
152
162
  - **The retry SCHEDULE is core's (`backoffDelay`, `isRetryableStatus`); the LOOP is this package's** —
153
163
  core's `retry()` retries unclassified throws, so it would retry a 400 (`gateway-backoff.test.ts`).
154
- - Every non-2xx and every in-band `error` frame is `AiTransportError` with a real `status`.
155
- - **Both wire formats answer the same question the same way** (`provider-parity.test.ts`): an in-band
156
- `error` in a 200 throws (`throwInBandError`); a `refusal` detail forces `stopReason: 'refusal'`; a
157
- tool call's `input` is parsed (`parseToolArguments`); the credential is scrubbed from `detail`.
164
+ - Every non-2xx and in-band `error` frame is `AiTransportError` with a real `status` — but a
165
+ window overflow (`providerFailure`).
166
+ - **Both wire formats answer the same question the same way** (`provider-parity.test.ts`). Full rule: `docs/history/ai.md`.
158
167
  - **Arguments that are not JSON are never thrown by a wire reader**: recorded as `malformed`, and
159
168
  the PROVIDER decides once usage is known — dropped on a truncated turn, else
160
169
  `X_LLM_TOOL_ARGUMENTS_INVALID` with `meta.billed`, which the gateway RECORDS (`settleFailure`).
@@ -165,46 +174,25 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
165
174
  - A tool call is emitted whole. Thinking chunks are never appended to `text`.
166
175
  - Anthropic body: no `top_p`/`top_k`, no `budget_tokens`, `effort` inside `output_config`. Model
167
176
  IDs are exact aliases, never date-suffixed.
168
- - **`temperature`/`seed` (`sampling.ts`) are sent ONLY to a row with `sampling: { … : true }`** —
169
- else left out (never refused; `llm.sampling.ignored`); no `seed` on the Anthropic wire; in
170
- `promptCacheKey` only when set.
171
- - **Apps bring their own models and providers (M12, 25.0.0): no vendor choice, no vendor data.** The
172
- registry starts EMPTY; a model resolves declaration → prompt → `Gateway.defaultModel`
173
- (`resolveModel` / `modelForCall`), none is `X_AI_MODEL_UNRESOLVED` (no gateway at all:
174
- `X_AI_GATEWAY_MISSING`). `vendor-neutral.test.ts` proves it; no fix line names a vendor id.
177
+ - **`temperature`/`seed` (`sampling.ts`) are sent ONLY to a row with `sampling: { … : true }`**, else
178
+ left out, never refused. Full rule: `docs/history/ai.md`.
179
+ - **Apps bring their own models and providers: no vendor choice, no vendor data.** The registry
180
+ starts EMPTY (`vendor-neutral.test.ts`); resolution is `model-resolve.ts`'s. Full rule: `docs/history/ai.md`.
175
181
  - **`ModelId` is `string`; the catalogue is an open registry** — `modelSpec(id)` refuses an
176
- unregistered id (`X_AI_MODEL_UNKNOWN`) before a socket or a reservation. Re-registering REPLACES
177
- the spec and keeps its rung; registration order is the ladder within a `family`
178
- (`moreCapableThan`); `X_LLM_REFUSED`'s fix names a rung ABOVE or nothing.
182
+ unregistered id (`X_AI_MODEL_UNKNOWN`) before a socket or a reservation. Full rule: `docs/history/ai.md`.
179
183
  - **Every provider serves the app's list** (`{ models }` required, non-empty); no provider has a
180
184
  model of its own.
181
185
  - **The reasoning controls are PER MODEL (`models.ts`)**: an unasked control is omitted; an asked
182
- control the model lacks is `X_AI_REQUEST_INVALID`, never dropped. Adding a model is a row the app
183
- registers. `disableThinkingUpTo: 'never'` refuses `thinking: 'disabled'` at every effort;
184
- `disabledThinking: 'between_tools'` is the wire spelling of the same declaration
185
- (`model-rules.test.ts`, over the fixture rows).
186
- - **Media blocks are screened once, by `assertMediaContent`, at the top of BOTH body builders** —
187
- malformed is `X_AI_REQUEST_INVALID`, untakeable (role, the row's `input`, the wire) is
188
- `X_AI_CONTENT_UNSUPPORTED`; never dropped, never approximated (`provider-parity-content.test.ts`).
186
+ control the model lacks is `X_AI_REQUEST_INVALID`, never dropped. Full rule: `docs/history/ai.md`.
187
+ - **Media blocks are screened once, by `assertMediaContent`, at the top of BOTH body builders**;
188
+ never dropped, never approximated. Full rule: `docs/history/ai.md`.
189
189
  - `generate()` above `STREAM_ONLY_MAX_TOKENS` runs the streaming transport.
190
190
  - **Every provider and embedder is built by its factory** (`anthropicProvider()`, `echoProvider()`,
191
191
  `hashEmbedder()`, `remoteEmbedder()`); the classes are type exports only (`bun run factory-names`).
192
192
  - **`openAiProvider()` is a FORMAT, not a vendor** — `baseUrl`, `auth`, `headers` are the
193
- differences; never a second class per vendor:
194
- - structured output is the `respond` tool, never `response_format`; `tool_choice` names the tool
195
- only when the request offers exactly one and the row is not `forcedToolChoice: false`;
196
- - `strict: true` only when `satisfiesStrictMode` says the schema qualifies;
197
- - `max_completion_tokens`; `role: 'system'`; `stream_options: { include_usage: true }` and an
198
- ESTIMATE when usage never arrives; subtract `cached_tokens` from `prompt_tokens`;
199
- - tool-call deltas merge by `tool_calls[].index`, a differing `id` starting a new call; calls are
200
- read at `end()` (`[DONE]` or socket close), never at the finish reason; `isComplete()` accepts
201
- `[DONE]` or a finish reason, except `[DONE]` with pending tool fragments, which is refused;
202
- - no code of its own (a media refusal is the shared `X_AI_CONTENT_UNSUPPORTED`);
203
- `AiTransportError` takes the provider's `envVar`; the key is revealed late,
204
- never stored, scrubbed from `detail`.
193
+ differences; never a second class per vendor. Wire rules: [history](../../docs/history/ai.md#openaiprovider-wire-rules).
205
194
  - **`embedBatched` enforces the `Embedder` arity per batch** (`X_AI_EMBEDDER_INVALID` with both
206
- counts). One `remoteEmbedder()` for every vendor; vectors L2-normalised; a wrong width is
207
- `X_VECTOR_DIM_MISMATCH`.
195
+ counts). Full rule: `docs/history/ai.md`.
208
196
  - **A caught value is read with `renderThrowable`** (`remote-embedder.ts`, `wire.ts`,
209
197
  `openai-wire.ts`) — `scripts/error-render.ts` cannot see `catch` bindings.
210
198
  - **A caller's string is never an object KEY**: the wire tables are `Map`s; `prompt.render` and
@@ -225,60 +213,43 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
225
213
  - `ctx.signal` is read at the top of every turn, before every tool batch, and forwarded to
226
214
  `fetch`; `X_ABORTED` is core's.
227
215
  - One turn's tools run concurrently, results paired by index; no tool STARTS after an abort.
216
+ - **`mcpConfirmations({ tools })` holds a tool here**, one pass each however it arrives
217
+ (`tools-gate.ts`). Proof: `scripts/ai-tool-confirmation.test.ts`.
228
218
  - `onTurn` reports facts only (also an `agent.turn` span event); a throw from it fails the run.
229
- `.stream()` on `agent()` is deliberately not shipped. No semantic cache.
219
+ Streams per turn on its route (`stream: AgentChunk`). No semantic cache.
230
220
  - Every replayed `tool_use` is answered by a `tool_result` in the very next message
231
221
  (`agent-transcript.ts`); an unaccepted `respond` comes back as an `is_error` `tool_result`
232
222
  ("superseded").
233
223
  - A tool result is rendered totally (`'null'` for `undefined`; a non-JSON result reported as such,
234
224
  never as a failure); the throw is read with `stringField`.
235
- - **A 5xx code's cause reaches the model only when core's `hasPublicCause` allows it** (status from
236
- `@ultimat3/http`'s `statusFor`) — else `CODE: HIDDEN_TOOL_CAUSE`, plus a branded `callerFix`. The
237
- same verdict a problem document and `@ultimat3/mcp` give, from `failure-disclosure.ts` — the
238
- rule a hive member's `reason` follows too. Never a second copy. One widening, for an
239
- `inProcess` tool only (`discloseOwnFailure`): a branded error under a code with NO declared
240
- status is shown as `{ error: { code, cause, fix } }`; `discloseErrors: false` opts out.
225
+ - **A 5xx code's cause reaches the model only when core's `hasPublicCause` allows it**, else
226
+ `CODE: HIDDEN_TOOL_CAUSE` plus a branded `callerFix` (`failure-disclosure.ts`, never a second
227
+ copy). Full rule: `docs/history/ai.md`.
241
228
  - **A run only NARROWS its declaration**: `offerTools()` ⊆ declared names (`X_AGENT_TOOL_UNDECLARED`),
242
229
  `model()` via `modelSpec`, `limits()` screened and ≤ declared (`X_AI_REQUEST_INVALID`); all
243
230
  before `vars()`. The handle binds the run's model and OFFERED subset; `costPerRun` is checked per turn.
244
231
  - **`.run()`/`.continue()` are the SAME action**, marked by an ambient session only the declaring
245
- agent may claim, once (a nested agent starts fresh). The handle is SIGNED (`sealTranscript`: the
246
- cursor secret, own label) and bound to agent, model, tool names and `orgId`; screened before
247
- anything is loaded. `maxTurns`/`tokensPerRun` are cumulative — the remainder is the ceiling,
248
- NEVER a ledger debit. A failure seals an updated handle onto the error (`transcriptOf`).
249
- - **`compact` (default `'auto'`) acts BEFORE a turn, never after a 400**: the summary call is the
250
- run's own request up to a cut BETWEEN turns (the cached prefix, same session), billed, not a turn;
251
- system, tools, kept turns byte-identical; failed → whole, or `X_AGENT_CONTEXT_EXHAUSTED`, never
252
- truncated ([why](../../docs/history/ai.md#auto-compaction-2026-10)).
232
+ agent may claim, once (a nested agent starts fresh). The handle is SIGNED (`sealTranscript`)
233
+ and bound to agent, model, tool names and `orgId`. Full rule: `docs/history/ai.md`.
234
+ - **`compact` (default `'auto'`) acts BEFORE a turn on the MEASURED estimate; an overflow gets one
235
+ forced compaction and a re-send**;
236
+ never truncated (`X_AGENT_CONTEXT_EXHAUSTED`). Full rule: `docs/history/ai.md`.
237
+ - **A summary request too long for its writer goes as TEXT** (`agent-compact-request.ts`,
238
+ `agent-compact-text.ts`): oldest tool results clipped to `SUMMARY_CLIP_CHARS` (2,000, head +
239
+ tail), then left out; a turn is never dropped. Reported on `AgentCompaction`: `summaryInput`
240
+ (`'messages' | 'text'`), `clipped`, `omitted`.
253
241
  - **`hive()` is a fan-out action** (`PRIMITIVE_FACTORIES`): `HiveResult` is a SCHEMA from the
254
- member's `output`; three arms (`ok`/`failed`/`skipped`) and three counters; `members` in SPLIT
255
- order with `index`; the hive never names an actor; no hive budget code; a member's throw goes
256
- through `discloseFailure` (`HIDDEN_MEMBER_CAUSE`);
257
- `skipped` reasons `SKIPPED_ABORTED` / `SKIPPED_NO_INPUT`; `onMemberError` is required;
258
- `concurrency` 4, `minMembers` 2 (a below-floor split runs serially); an empty split is
259
- `X_HIVE_EMPTY`; an aborted `ctx` is `X_ABORTED`.
242
+ member's `output`; a member's throw goes through `discloseFailure`; the hive never names an actor.
243
+ Full rule: `docs/history/ai.md`.
260
244
  - **`describeAgents()` / `registeredModels()` are offered, not published** — their only legal consumer
261
245
  is `@ultimat3/cli`. The facts are a THUNK; an unnamed agent has no row.
262
246
  - **`agentJob()` composes `job()`** (never an imitation). `name`, `tenant`, `retry` are
263
- REQUIRED; both reads of `target.job()` are LAZY. No result is kept: wrap an ACTION that runs the
264
- agent and writes, target bound from its input — never a model-aimed write tool; every write
265
- idempotent (unenforceable). **`actor`** re-resolves who it acts FOR each attempt (a served worker:
266
- ANONYMOUS) via `impersonate`; off-tenant is TERMINAL `X_JOB_TENANT_MISMATCH`.
247
+ REQUIRED; both reads of `target.job()` are LAZY. No result is kept; `actor` re-resolves
248
+ each attempt via `impersonate` (off-tenant: `X_JOB_TENANT_MISMATCH`). Full rule: `docs/history/ai.md`.
267
249
 
268
250
  ## Invariants — provider prompt caching
269
251
 
270
- - **`GenerateRequest.promptCache` is the one switch; what it sends is a ROW in `PROMPT_CACHE_RULES`**
271
- (wire, host, model prefix → marks, session carrier), never an `if`. An unnamed endpoint gets
272
- NOTHING. At most 3 `cache_control`, a caller's own counted first (cap 4), never on a blank text
273
- block; never a `provider` routing object.
274
- - **`agent()` defaults `'auto'`, `llm()` `'off'`.** The stable system part (`prompt.system` +
275
- `system().stable`) is sent FIRST; the mark follows the tail. The hint is not in `promptCacheKey`.
276
- - **Usage**: chat-completions `prompt_tokens` INCLUDES read and written tokens; both are subtracted,
277
- clamped. A budget debits all four counters; the estimate prices a WRITE only where a provider
278
- would mark (`marksPromptCache`), at the dearer rate.
279
- - **`maxRepeatedToolCalls` compares calls AND results over cycles of 1–3 turns**; `deadlineMs` has no
280
- default and rides the request's AND each tool's signal; both stops are `terminal`
281
- (`errors-retry.ts`).
252
+ One switch (`GenerateRequest.promptCache`), `promptCacheRule`: provider rows, then `PROMPT_CACHE_RULES`, then nothing. An app's own endpoint is a row ON ITS PROVIDER (`promptCache: { rules }`, `ProviderCacheRow`, first match wins), screened at construction against the wire (`prompt-cache-rows.ts`, `X_AI_REQUEST_INVALID`); `agent()` defaults `'auto'`, `llm()` `'off'`. In full: [`docs/history/ai.md`](../../docs/history/ai.md#moved-out-2026-11-03-plan-101-wave-3-the-claudemd-ceiling).
282
253
 
283
254
  ## Invariants — evals, retrieval, fix lines
284
255
 
@@ -287,6 +258,8 @@ In brief ([full text](../../docs/history/ai.md#evals-retrieval-fix-lines)): no f
287
258
  retrieval is hybrid only; an unbound store read under an org actor is `X_VECTOR_UNSCOPED`;
288
259
  `postgresVectorStore()` filters tenant and policy in SQL on every statement; `scoped()` TIGHTENS.
289
260
 
261
+ - **An eval never runs a real tool**: no double is `X_EVAL_TOOL_UNDOUBLED` (history, 2026-11-06).
262
+
290
263
  ## Commands
291
264
 
292
265
  ```
package/README.md CHANGED
@@ -3,6 +3,8 @@
3
3
  The LLM gateway primitive. Every model call in an Ultimate app goes through it, so budgets
4
4
  and cost accounting cannot be bypassed by a stray `fetch`.
5
5
 
6
+ `bun add @ultimat3/ai` · guide: [MCP And AI](https://github.com/developerz-ai/ultimate/wiki/MCP-And-AI)
7
+
6
8
  **Register your models, pick your provider.** The framework ships the mechanism — the `Provider`
7
9
  contract, two wire-format adapters, the gateway, budgets, `registerModel` — and no vendor choice
8
10
  and no vendor data: no default model, no catalogue row, no price. Which models an app runs, at what
@@ -71,6 +73,7 @@ provider whose `models` lists it.
71
73
  | `X_LLM_REFUSED`'s "a more capable model" | a rung above, in the same `family`, among the rows the APP registered — or no suggestion at all |
72
74
  | a direct provider call naming no model | `X_AI_MODEL_UNRESOLVED` from every shipped provider — `anthropicProvider()`, `openAiProvider()` and `echoProvider()` alike; a provider's `models` list is what it serves, never a fallback |
73
75
  | `echoProvider()` | serves whatever the app registered, and has no model of its own |
76
+ | `scriptedProvider({ turns, usage?, name? })` | the test double that calls tools: each provider call answers the next turn — `{ text }`, `{ call, input? }`, `{ calls: [...] }` or `{ answer }` (a `respond` call), each with an optional `stop` — and `.seen` keeps every request. Serves whatever the app registered. One call past the last turn is `X_AI_SCRIPT_EXHAUSTED`, never the last turn again: a loop must not look like a pass |
74
77
 
75
78
  ## Budgets — and which of the three is fleet-wide
76
79
 
@@ -150,7 +153,9 @@ default store at `replicas: 6` is six ledgers of twenty million, which is a budg
150
153
  | `reserve()` **debits, then checks** | the in-memory scopes check and debit with no `await` between, the store's through its atomic `take`; a read-then-write let concurrent calls — one request per scope racing one org key — all read the same `spent()` and all pass. `record` reconciles and `release` gives it back |
151
154
  | The cache key is every tool **whole**, through core's `fingerprint` | every `llm()` tool is named `respond`, so a key over names served an answer shaped for an old `output` schema; a cache `set` that throws is logged, never turned into a failure of a call already paid for |
152
155
  | A refusal is never cached | a cached one keeps serving a classifier decision after the prompt was fixed |
156
+ | A cache hit reads zero `usage`, `cost` and `costExact`, in the same currency | it is not debited, so a meter summing the stored figures read more than the budget beside it |
153
157
  | Retries use **full jitter**, from core's one curve | synchronised retries from N workers reproduce the rate limit. `backoffMs` is `@ultimat3/core`'s `backoffDelay` with the gateway's field names mapped onto it, and the roll is `providerGateway({ random })` — injectable, so the schedule is a unit test rather than a range |
158
+ | **One breaker per provider**, on by default: `{ failures: 5, cooldownMs: 30_000 }` (`DEFAULT_GATEWAY_BREAKER`) | a provider that ran out its retries five passes in a row is skipped, no call and no sleep, until one probe after the cooldown answers; without it every call re-spent every attempt on a dead provider. `providerGateway({ breaker })` takes `GatewayBreakerOption`, `((provider) => Breaker) \| 'off'`: your own core `breaker({ … })` per provider, or `'off'` for the old behaviour. A stream goes to the first provider whose breaker admits it |
154
159
  | A 4xx is never retried **except 408, 409 and 425** | the same body gets the same rejection and burns the budget — but a request the server stopped reading (408), a round a concurrent writer won (409) and a handshake that had not finished (425) are transient by construction, and core's `isRetryableStatus` is the one table that says so |
155
160
 
156
161
  ## Streaming
@@ -401,6 +406,16 @@ invocation is an ordinary one, marked so the model half streams. Two consequence
401
406
  | no `respond` tool is offered | a tool call is emitted whole, so forcing one leaves nothing to stream — the answer is prose, and its JSON parse is what a non-string `output` validates |
402
407
  | lazy | nothing is authorised, budgeted or sent until the first pull |
403
408
 
409
+ ### Streaming on the wire
410
+
411
+ `llm()` declares `stream: LlmTextChunk` (`{ type: 'text', text }`), so its own route answers
412
+ `text/event-stream` to the typed client's `summarize.stream(input)`: each text increment as the
413
+ provider sends it, then `done` with the value that satisfied `output`. Same policy, budget, cache
414
+ and span; the streamed turn is the one-attempt turn server-side `.stream()` takes. `thinking` never
415
+ crosses the wire. Every other caller (JSON, MCP, `.job()`) gets the non-streamed call, repair turn
416
+ included. `agent()` declares `stream: AgentChunk` the same way: per turn, `text`, `tool_call`,
417
+ `tool_result` and `turn` facts, then `done`.
418
+
404
419
  ## `agent()` — the tool loop, also an action
405
420
 
406
421
  The second half of "no ninth primitive": a tool-using run is still one server-authoritative
@@ -436,6 +451,7 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug
436
451
  | the tools of **one turn** run concurrently, results paired by `tool_use` id | a turn asking for five tools cost 5x wall clock and nothing said so. Order is positional, never by completion; the batch is bounded by what one turn asked for, and each tool is an action with its own `policy` and `rateLimit`, so a second ceiling here would be a throttle competing with those |
437
452
  | `onTurn` reports each completed turn as it happens (and an `agent.turn` span event, always) | a 90-second run emitted nothing until it returned. Observation only — it cannot steer the loop, see the transcript or reach the actor — and a throw from it fails the run rather than being swallowed |
438
453
  | a tool is an `mcp: { expose: true }` action or an `inProcessTool()`; anything else is `X_AGENT_TOOL_UNEXPOSED` **at declaration** | a silently dropped tool reads as offered and is not; `isMcpExposed` is the one predicate for ACTIONS, so an in-app agent and an external MCP client see the same catalogue of them — and a tool no MCP client may see is declared as one that is this process's alone |
454
+ | a tool in `mcpConfirmations({ tools })` answers `X_MCP_CONFIRMATION_PENDING` until a person approves | one list for both surfaces, so no option an author can forget. The decision action declares no `mcp.expose`, so listing it is `X_AGENT_TOOL_UNEXPOSED`: **exposing it hands an in-app agent, whose actor is the user, its own approval** |
439
455
  | running out of turns is `X_AGENT_MAX_TURNS`, never a partial answer | a half-finished transcript returned as a result is working notes presented as a decision |
440
456
  | a cycle of 1 to 3 tool turns — same calls, arguments, results and failures — gone round `maxRepeatedToolCalls` (default 3) times is `X_AGENT_TOOL_LOOP` | such a turn can only be followed by another like it; a changed result is progress; a repair turn neither counts nor resets |
441
457
  | `deadlineMs` (no default) is the run's wall-clock ceiling: `X_AGENT_DEADLINE` | the in-flight model call is aborted on the socket and every tool's `ctx.signal` fires; a tool that ignores it still finishes late. A caller that leaves first is still `X_ABORTED` |
@@ -452,10 +468,48 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug
452
468
 
453
469
  `cost` is the currency's minor units, rounded up once per call; `costExact` beside it is the same figure exactly (`Money` at a finer `scale`, from `exactCostOf`). `agent()` reports both per turn and for the run in `onTurn`; `llm()` per provider call in `onCall`; a ledger sums exact costs and rounds once when read. `repairOf` on either event is what a repair turn was told to fix.
454
470
 
471
+ One total across several calls — a job that runs two `llm()` calls and an `agent()` — is
472
+ `meteredRun(fn)` → `{ value, billed }`: every provider call `fn` makes, however deeply nested, summed
473
+ once (`Billed`, `undefined` when none started). A throw is rethrown with that total on `meta.billed`,
474
+ so `billedOf(error)` reads it. A meter, not a ledger: it refuses nothing.
475
+
455
476
  ### Continuing a run
456
477
 
457
478
  `agent()` returns an `AgentAction`: the action, plus `.run(input, opts)` → `{ output, transcript, turns, usage }` and `.continue(input, { transcript, user }, opts)`, which appends one user turn to a stored transcript and resumes the loop. The transcript is re-sent byte for byte (the provider's prompt cache reads it back; no tool re-runs); `maxTurns` and `budget.tokensPerRun` are cumulative; the handle is signed, bound to the agent, model, tools and tenant, and screened (`X_AGENT_TRANSCRIPT_INVALID`); a failed call leaves an updated one on the error (`transcriptOf(error)`), which is the one a job stores. Full contract: [Agents → Continuing a run](https://github.com/developerz-ai/ultimate/wiki/Agents#continuing-a-run-run--continue).
458
479
 
480
+ `onCheckpoint(event, { input, ctx })` hands the app the same signed handle **between** turns — after a
481
+ turn's tool results (or its correction) land, before the next turn is bought, never after the answer
482
+ (`.run()` returns that one). Store `event.transcript` on the run's own row; the next attempt calls
483
+ `.continue(input, { transcript, user })` and pays only for the turns left. Awaited; a throw fails
484
+ the run; sealed only when declared. A worker dying DURING a tool batch re-asks that turn, so its
485
+ tools may run again — at-least-once still binds.
486
+
487
+ ```ts
488
+ import type { AgentAction, AgentTranscript } from '@ultimat3/ai';
489
+ import type { Ctx } from '@ultimat3/core';
490
+ import { stringField } from '@ultimat3/core';
491
+ import type { t } from '@ultimat3/schema';
492
+
493
+ type Review = ReturnType<typeof t.object<{ verdict: typeof t.string }>>;
494
+ type In = ReturnType<typeof t.object<{ postId: typeof t.string }>>;
495
+ // Declared with onCheckpoint: ({ transcript }, { input }) => saveProgress(input.postId, transcript)
496
+ declare const reviewer: AgentAction<In, Review>;
497
+ declare function progress(postId: string): Promise<AgentTranscript | undefined>;
498
+
499
+ // The action an agentJob() runs: resume from the stored handle, else start.
500
+ export async function review(ctx: Ctx, input: { postId: string }) {
501
+ const stored = await progress(input.postId);
502
+ try {
503
+ if (stored === undefined) return await reviewer.run(input, { ctx });
504
+ return await reviewer.continue(input, { transcript: stored, user: 'Carry on.' }, { ctx });
505
+ } catch (error) {
506
+ // A prompt bump or a model change invalidates a stored handle by design: start fresh.
507
+ if (stringField(error, 'code') !== 'X_AGENT_TRANSCRIPT_INVALID') throw error;
508
+ return reviewer.run(input, { ctx });
509
+ }
510
+ }
511
+ ```
512
+
459
513
  ### Auto-compaction
460
514
 
461
515
  ```ts
@@ -480,15 +534,17 @@ export const researcher = agent({
480
534
  });
481
535
  ```
482
536
 
483
- Before each turn, the gateway's estimate of that turn's request is compared with the model row's
484
- `contextWindow` less `maxTokens`. Over `at` of it, ONE model call summarises every turn older than
537
+ Before each turn, the gateway's estimate of that turn's request — scaled by what the provider
538
+ reported for the last one — is compared with the model row's `contextWindow` less `maxTokens`. Over `at` of it, ONE model call summarises every turn older than
485
539
  the last `keepTurns`; the summary joins the first user message (after the prompt, verbatim, marked
486
540
  `[compacted: …]`), the kept turns follow byte for byte, and the turn is sent.
487
541
 
488
542
  | Rule | Why |
489
543
  |---|---|
490
544
  | default **`'auto'`**; `'off'` never compacts and never checks | a summary changes what the model sees, but only above `at` of the window, where the alternative is a run lost to a 400 after every turn was paid for. Below it a run is byte-identical to `'off'` — requests and handle — and it is never silent: `onCompact`, the `agent.compact` span event, the handle's `compacted` |
491
- | triggered on the **estimate of the next request** — the gateway's prompt estimate plus the tool definitions — over `at` × (`contextWindow` − `maxTokens`), before the call | after a 400 there is no request left to fix; `maxTokens` is reserved because the completion shares the window |
545
+ | triggered on the **estimate of the next request** — the gateway's prompt estimate plus the tool definitions — over `at` × (`contextWindow` − `maxTokens`), before the call | `maxTokens` is reserved because the completion shares the window |
546
+ | the estimate is **scaled by the measurement**: after each turn, `max(1, (inputTokens + cacheReadTokens + cacheWriteTokens) / estimate)` of that turn's request; `before`/`after` report the scaled figure | four characters a token is wrong by 3x on dense JSON and non-Latin text, and a run walked past the window it was meant to compact before. A turn reporting zero keeps the last ratio (a provider that never reports compacts where it always did); below 1 is ignored; per run, never signed — a continuation's first turn estimates plainly |
547
+ | a provider that refuses the turn anyway (**`X_AI_CONTEXT_OVERFLOW`**) gets **one forced compaction** and the same turn re-sent; a second refusal, a compaction that cannot be had, or `compact: 'off'` is `X_AGENT_CONTEXT_EXHAUSTED` with the provider's text on `meta.refused` | the loop still holds the transcript after that 400. The refusal billed nothing (`meta.billed` at zero), is not a turn, and is never retried by the gateway: the same request overflows on every provider |
492
548
  | the system prompt, its stable block and the tools are **never touched** | a provider cache is a prefix match over tools → system → messages: they are the part every turn and every run shares, so they are still read at the cache rate after a compaction. Only the messages are rewritten, once |
493
549
  | the summary call is the run's **own request** up to the cut, plus one instruction — same gateway, tools, system and `promptCache` hint | it reads the prefix the turn that ended at the cut already cached instead of paying for it again; on OpenRouter the same `x-session-id` routes it to the provider holding that cache (`compact.model` trades that read for a cheaper writer) |
494
550
  | the cut falls **between turns**: the prompt and the last `keepTurns` (4, at least 1) turns are verbatim | a turn is an assistant message and the user message answering every `tool_use` in it, so no call is ever separated from its result (a 400 on every provider); the model keeps its task as written and the results it just asked for |
@@ -501,7 +557,8 @@ the last `keepTurns`; the summary joins the first user message (after the prompt
501
557
 
502
558
  | Limit | Detail |
503
559
  |---|---|
504
- | the estimate | ~4 characters a token, which is why `at` defaults below 1 |
560
+ | the estimate | ~4 characters a token until the first turn reports, which is why `at` defaults below 1 |
561
+ | the forced compaction | its summary call is the refused request less the kept turns: with a tiny `keepTurns` it may not fit either — `compact.model` with a larger window is the way out |
505
562
  | `tools: []` on the chat-completions wire | only `respond` is offered, so it is forced, the summary call answers no text and the run falls back to whole |
506
563
  | per run | `compact` is the declaration's; `limits()` does not take it, `describeAgents()` does not publish it |
507
564
 
@@ -555,6 +612,15 @@ export const kbAgent = agent({
555
612
  | chat completions | `api.openai.com` | any | none (automatic) | `prompt_cache_key` |
556
613
  | chat completions | anything else | any | none | — |
557
614
 
615
+ An endpoint the table does not know is the app's to declare, on its provider:
616
+ `openAiProvider({ baseUrl, models, promptCache: { rules: [{ model: 'anthropic/', marks: 'parts', session: 'x-session-id' }] } })`
617
+ marks for a company gateway; `anthropicProvider({ baseUrl, models, promptCache: { rules: [{ model: '', marks: 'none', session: 'none' }] } })`
618
+ stops marks to a proxy that rejects `cache_control`. A row (`ProviderCacheRow`) is `{ model, marks,
619
+ session }`, read before the table, first match by model prefix; the pre-flight estimate follows it.
620
+ The Messages wire speaks `blocks` or `none` and no session; chat-completions `parts` or `none`, and
621
+ `x-session-id`, `prompt_cache_key` or `none`. Anything else is `X_AI_REQUEST_INVALID` when the
622
+ provider is built.
623
+
558
624
  Usage: the chat-completions reader takes `prompt_tokens_details.cached_tokens` and
559
625
  `.cache_write_tokens` out of `prompt_tokens` (which includes both) and reports them as
560
626
  `cacheReadTokens` / `cacheWriteTokens`; a missing field is zero. `costOf` prices them at the model
@@ -567,7 +633,7 @@ ever sent.
567
633
  | Limit | Detail |
568
634
  |---|---|
569
635
  | `run.cost` and the ledger | one contract: cost is computed exactly and rounded up once. `AgentTurn.run.cost` is the run's summed usage priced once; `gateway.spent()` / `BudgetLedger.report().cost` is the exact cost of every recorded call summed, rounded once when read (`costExact` beside it). They agree for a run on built-in providers; a `Provider` that states no `costExact` is summed from its rounded `cost`. Budgets are enforced against the ledger |
570
- | the Anthropic row matches any host | an `anthropicProvider({ baseUrl })` behind a proxy still sends marks |
636
+ | the Anthropic row matches any host | an `anthropicProvider({ baseUrl })` behind a proxy still sends marks, unless it declares a `marks: 'none'` row |
571
637
  | more than about 9 tool calls in one turn, on OpenRouter | only the tail is marked there and the provider looks back 20 blocks, so that turn's transcript is rewritten, not read |
572
638
  | a mark never lands on an empty or whitespace-only text block, and never makes a fifth | it moves back a block or is dropped; a raw caller's own marks are counted first |
573
639
  | `session` | OpenRouter header: 1 to 256 printable ASCII, else `X_AI_REQUEST_INVALID`. OpenAI `prompt_cache_key`: over 64 characters it is sent as `h:` + 62 hex of its SHA-256 |
@@ -854,7 +920,46 @@ X_EVAL_THRESHOLD: an eval scored below its tolerance
854
920
 
855
921
  Built-in scorers: `exact`, `contains`, `jsonValid`, `jsonSchemaValid(keys)`,
856
922
  `numericTolerance(t)`, `llmJudge({ judge })` — the judge prompt is itself versioned, so a
857
- judge that drifts is a measuring instrument that lies, and its hash is in the scorer name.
923
+ judge that drifts is a measuring instrument that lies, and its hash is in the scorer name —
924
+ and `calledTool(name)`, 1 when an agent run called that tool.
925
+
926
+ ### The agent arm
927
+
928
+ `DefineEvalInput` is a union: `PromptEvalInput` (`prompt` + `cases[].vars`) or `AgentEvalInput`
929
+ (`agent` + `cases[].input` + `tools`). Narrow with `'prompt' in input`. The agent arm runs
930
+ `agent.run(input, { actor })` per case — default actor `serviceActor({ id: 'eval' })` — with the
931
+ gateway handed to `run` / `assert` and every tool answered by its double in `tools`:
932
+
933
+ ```ts
934
+ import {
935
+ calledTool,
936
+ defineEval,
937
+ type EvaluatedAgent,
938
+ providerGateway,
939
+ type ScriptedTurn,
940
+ scriptedProvider,
941
+ } from '@ultimat3/ai';
942
+
943
+ declare const supportAgent: EvaluatedAgent<{ question: string }>; // your registered agent()
944
+ declare const turns: ScriptedTurn[];
945
+
946
+ const supportEval = defineEval({
947
+ name: 'support.answer',
948
+ agent: supportAgent,
949
+ cases: [{ name: 'where-is-o-1', input: { question: 'Where is o-1?' } }],
950
+ tools: { lookupOrder: () => ({ status: 'shipped' }) },
951
+ scorers: [calledTool('lookupOrder')],
952
+ baseline: import.meta.resolve('./support.baseline.json'),
953
+ tolerance: 0.05,
954
+ });
955
+ await supportEval.assert(providerGateway({ providers: [scriptedProvider({ turns })] }));
956
+ ```
957
+
958
+ An undoubled call is `X_EVAL_TOOL_UNDOUBLED` (`EvalToolUndoubledError`) and ends the eval: an eval
959
+ never runs a real tool. Any other failed run scores 0 and records its code in `cases[].failed`.
960
+ Scorers get `run: { turns, toolCalls, usage }` (`EvalRun`, `EvalToolCall`, `ScorerInput`); the
961
+ baseline is the same file, keyed by the agent's prompt hash. Exported types: `AgentEvalCase`,
962
+ `AgentEvalInput`, `EvaluatedAgent`, `PromptEvalInput`, `ToolDouble`.
858
963
 
859
964
  ## Prompts are versioned artifacts
860
965
 
@@ -987,9 +1092,19 @@ An in-app agent and an external MCP agent both end at the same `invoke` — `run
987
1092
  carries it, and an action facade has no `.run` of its own. So they authorize identically. The
988
1093
  actor comes from the request context, never from the model.
989
1094
 
1095
+ A tool in `mcpConfirmations({ tools })` waits for a person here too. `run` asks the policy's actor
1096
+ half, then the gate registered for that name (read per call), then `invoke`; `asProjectableAction`
1097
+ puts a hand-built tool behind the same gate, and `runLlmToolCall` projects whatever it is handed, so
1098
+ it is one pass whichever way a tool arrives. The model reads `X_MCP_CONFIRMATION_PENDING`, reports
1099
+ through `output`, and a later user turn repeats the call. No new run outcome. The gate binds a tool's
1100
+ **name**, never what it reaches: `inProcessTool({ name: 'refundNow', execute: () => invoke(refundOrder, …) })`
1101
+ is not gated by `refundOrder`'s row, so list `refundNow` in `tools`.
1102
+ [Agents → A tool that waits for a person](https://github.com/developerz-ai/ultimate/wiki/Agents#a-tool-that-waits-for-a-person).
1103
+
990
1104
  `inProcessTool({ name, description, input, execute })` is a tool that is this process's alone: no `mcp`, no policy of its own (the agent's decides), in no catalogue but the agent's. Its own typed refusal reads back as `{"error":{"code","cause","fix"}}`. [Agents → In-process tools](https://github.com/developerz-ai/ultimate/wiki/Agents#in-process-tools).
991
1105
 
992
- A failed tool reads back as `CODE: cause (fix: …)` — except a 5xx code whose cause core's
1106
+ A failed tool reads back as `CODE: cause (fix: …)`, the fix being the error's `callerFix` when it
1107
+ declares one — except a 5xx code whose cause core's
993
1108
  `hasPublicCause` does not declare public, which reads `CODE: ` plus a fixed sentence (and a
994
1109
  `callerFix`, when the `UltimateError` declared one): the rule a production problem document follows,
995
1110
  and the one `@ultimat3/mcp` applies. The model's provider keeps every request it is sent, so a
@@ -1021,7 +1136,8 @@ fail-closed `terminal` default.
1021
1136
  | `X_AGENT_TOOL_UNDECLARED` | an `agent()` run's `offerTools()` named a tool its declaration does not list |
1022
1137
  | `X_AGENT_TOOL_LOOP` | an `agent()` made the same tool calls and read the same results `maxRepeatedToolCalls` turns in a row |
1023
1138
  | `X_AGENT_DEADLINE` | an `agent()` outlived its declared `deadlineMs` |
1024
- | `X_AGENT_CONTEXT_EXHAUSTED` | an `agent()`'s next request would not fit its model's context window, and compaction could not bring it under; refused before it is sent |
1139
+ | `X_AGENT_CONTEXT_EXHAUSTED` | an `agent()`'s next request would not fit its model's context window, and compaction could not bring it under — refused before it is sent, or after the provider refused it twice (`meta.refused`) |
1140
+ | `X_AI_CONTEXT_OVERFLOW` | a provider answered 400/413 saying the prompt outgrew the model's window; never retried, `terminal` for a job. `llm()` and `generate()` raise it as itself; `agent()` compacts once first |
1025
1141
  | `X_EVAL_THRESHOLD` | an eval scored below its bar |
1026
1142
  | `X_VECTOR_DIM_MISMATCH` | a vector's length disagrees with the store |
1027
1143
  | `X_VECTOR_SCOPE_WIDENED` | a derived vector scope tried to leave the tenant it was bound to |
@@ -1061,6 +1177,7 @@ a job boundary the class is gone and the `code` is what survives — match on th
1061
1177
  | `EvalMissingError` | `X_EVAL_MISSING` | `src/eval-errors.ts` |
1062
1178
  | `EvalRecordingError` | `X_EVAL_RECORDING` | `src/eval-errors.ts` |
1063
1179
  | `EvalThresholdError` | `X_EVAL_THRESHOLD` | `src/eval-errors.ts` |
1180
+ | `EvalToolUndoubledError` | `X_EVAL_TOOL_UNDOUBLED` | `src/eval-errors.ts` |
1064
1181
  | `HiveEmptyError` | `X_HIVE_EMPTY` | `src/hive-errors.ts` |
1065
1182
  | `LlmOutputInvalidError` | `X_LLM_OUTPUT_INVALID` | `src/errors.ts` |
1066
1183
  | `LlmRefusedError` | `X_LLM_REFUSED` | `src/errors.ts` |
package/package.json CHANGED
@@ -1,7 +1,19 @@
1
1
  {
2
2
  "name": "@ultimat3/ai",
3
- "version": "29.4.0",
3
+ "version": "30.1.0",
4
4
  "description": "LLM gateway, versioned prompts, evals as tests, embeddings, hybrid vector search, RAG",
5
+ "keywords": [
6
+ "ultimate",
7
+ "bun",
8
+ "framework",
9
+ "ai",
10
+ "llm",
11
+ "agents",
12
+ "evals",
13
+ "rag",
14
+ "embeddings"
15
+ ],
16
+ "homepage": "https://github.com/developerz-ai/ultimate/wiki/MCP-And-AI",
5
17
  "license": "MIT",
6
18
  "type": "module",
7
19
  "repository": {
@@ -32,17 +44,17 @@
32
44
  "test": "bun test"
33
45
  },
34
46
  "dependencies": {
35
- "@ultimat3/action": "29.4.0",
36
- "@ultimat3/cache": "29.4.0",
37
- "@ultimat3/core": "29.4.0",
38
- "@ultimat3/db": "29.4.0",
39
- "@ultimat3/http": "29.4.0",
40
- "@ultimat3/jobs": "29.4.0",
41
- "@ultimat3/money": "29.4.0",
42
- "@ultimat3/schema": "29.4.0",
43
- "@ultimat3/time": "29.4.0"
47
+ "@ultimat3/action": "30.1.0",
48
+ "@ultimat3/cache": "30.1.0",
49
+ "@ultimat3/core": "30.1.0",
50
+ "@ultimat3/db": "30.1.0",
51
+ "@ultimat3/http": "30.1.0",
52
+ "@ultimat3/jobs": "30.1.0",
53
+ "@ultimat3/money": "30.1.0",
54
+ "@ultimat3/schema": "30.1.0",
55
+ "@ultimat3/time": "30.1.0"
44
56
  },
45
57
  "devDependencies": {
46
- "@ultimat3/policy": "29.4.0"
58
+ "@ultimat3/policy": "30.1.0"
47
59
  }
48
60
  }