@ultimat3/ai 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +63 -90
- package/README.md +125 -8
- package/package.json +23 -11
- package/src/agent-checkpoint.ts +44 -0
- package/src/agent-compact-def.ts +11 -0
- package/src/agent-compact-measure.ts +38 -0
- package/src/agent-compact-request.ts +100 -0
- package/src/agent-compact-text.ts +95 -0
- package/src/agent-compact.ts +75 -50
- package/src/agent-continue.ts +15 -3
- package/src/agent-def.ts +9 -0
- package/src/agent-errors.ts +12 -2
- package/src/agent-facts.ts +8 -0
- package/src/agent-send.ts +82 -0
- package/src/agent-stream.ts +76 -0
- package/src/agent.ts +71 -21
- package/src/answer-errors.ts +1 -1
- package/src/anthropic-provider.ts +291 -0
- package/src/content-errors.ts +1 -1
- package/src/context-overflow.ts +86 -0
- package/src/error-codes.ts +95 -0
- package/src/errors-retry.ts +8 -4
- package/src/errors.ts +6 -87
- package/src/eval-agent.ts +100 -0
- package/src/eval-errors.ts +31 -5
- package/src/evals.ts +125 -45
- package/src/gateway-providers.ts +215 -0
- package/src/gateway.ts +63 -153
- package/src/hive-errors.ts +1 -1
- package/src/in-process-tool.ts +20 -11
- package/src/index.ts +31 -7
- package/src/llm.ts +39 -8
- package/src/model-resolve.ts +7 -1
- package/src/openai-body.ts +12 -5
- package/src/openai-provider.ts +28 -8
- package/src/openai-wire.ts +11 -7
- package/src/prompt-cache-rows.ts +82 -0
- package/src/prompt-cache.ts +24 -2
- package/src/provider.ts +15 -274
- package/src/remote-embedder.ts +38 -11
- package/src/run-meter.ts +40 -2
- package/src/runtime.ts +38 -3
- package/src/scorers.ts +35 -1
- package/src/scripted-provider.ts +146 -0
- package/src/tools-gate.ts +46 -0
- package/src/tools.ts +101 -27
- package/src/wire.ts +12 -4
package/CLAUDE.md
CHANGED
|
@@ -9,13 +9,9 @@ Shipped: `action` (`llm()`), `cache`, `core`, `db` (pgvector), `http` (`statusFo
|
|
|
9
9
|
Keep `policy` dev-only: tests and `*-fixture.ts` harnesses import `allow`. The
|
|
10
10
|
`!src/**/*-fixture.ts` exclusion keeps harnesses unpublished; their imports still need a declaration.
|
|
11
11
|
|
|
12
|
-
`mcp` is the same tier, so the LLM-tool projection is restated
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
`asProjectableAction` and mcp's `primitiveFromAction` are two copies the tier table forces, held
|
|
16
|
-
equal by `scripts/ai-mcp-tool-parity.test.ts` (name, description, exposure, the whole input schema
|
|
17
|
-
incl. `tool-idempotency.ts`'s reserved `idempotencyKey`, the policy verdict, the replay). A field
|
|
18
|
-
one adds, the other adds.
|
|
12
|
+
`mcp` is the same tier, so the LLM-tool projection is restated in `tools.ts`, never imported —
|
|
13
|
+
`asProjectableAction` and mcp's `primitiveFromAction` held equal by
|
|
14
|
+
`scripts/ai-mcp-tool-parity.test.ts`; a field one adds, the other adds. Full rule: `docs/history/ai.md`.
|
|
19
15
|
|
|
20
16
|
## Owns
|
|
21
17
|
|
|
@@ -24,10 +20,12 @@ one adds, the other adds.
|
|
|
24
20
|
| `models.ts` | the model REGISTRY: `registerModel`, limits, prices, the reasoning controls each one accepts — and NO row of its own |
|
|
25
21
|
| `model-resolve.ts` | `resolveModel` / `modelForCall` — declaration → prompt → gateway `defaultModel`, none is `X_AI_MODEL_UNRESOLVED` naming all three; `describedModel` for a published fact |
|
|
26
22
|
| `model-fixture.ts` | the rows this package's own suites register (`useFixtureModels()`). Not shipped |
|
|
27
|
-
| `provider.ts` | `Provider` interface, the request half
|
|
28
|
-
| `
|
|
23
|
+
| `provider.ts` | `Provider` interface, the request half; re-exports the Anthropic names |
|
|
24
|
+
| `anthropic-provider.ts` | `anthropicProvider()`, its stream-only rule (`requiresStreaming`) and message parse |
|
|
25
|
+
| `echo-provider.ts` | `echoProvider()`, the deterministic double — split from `provider.ts` at its line ceiling. A fixture that needs a tool call uses `scriptedProvider`, never a hand-rolled `Provider` |
|
|
26
|
+
| `scripted-provider.ts` | `scriptedProvider()`, the scripted double that calls tools, and `X_AI_SCRIPT_EXHAUSTED`: one call past the script is refused, never answered with the last turn again |
|
|
29
27
|
| `content-blocks.ts` | the `image` / `document` block shapes, the screen every request runs them through, their token estimate |
|
|
30
|
-
| `content-errors.ts` | the `X_AI_CONTENT_UNSUPPORTED` class; its code and title belong in `
|
|
28
|
+
| `content-errors.ts` | the `X_AI_CONTENT_UNSUPPORTED` class; its code and title belong in `error-codes.ts` |
|
|
31
29
|
| `wire.ts` | the response half: `usage` / `stop_reason` shapes, and the SSE `MessageStream` |
|
|
32
30
|
| `error-body.ts` | what a failure body SAYS (`detailOf`) and what must never survive into it (`withoutKey`) — one copy, both transports |
|
|
33
31
|
| `sse.ts` | Server-Sent Events framing — protocol only, knows nothing about Anthropic |
|
|
@@ -35,13 +33,16 @@ one adds, the other adds.
|
|
|
35
33
|
| `openai-messages.ts` | the format mapping's request half: `AiMessage` blocks → OpenAI messages, `LlmTool` → functions, `tool_choice` |
|
|
36
34
|
| `openai-body.ts` | one chat-completions body, and the per-model reasoning mapping |
|
|
37
35
|
| `openai-wire.ts` | its response half: one completion, and `ChatCompletionStream` (fragmented tool calls, trailing usage, `[DONE]`) |
|
|
38
|
-
| `gateway.ts` | routing,
|
|
36
|
+
| `gateway.ts` | routing, cache, budget wiring |
|
|
37
|
+
| `gateway-providers.ts` | the retry loop across providers, `backoffMs`, `isRetryable`, the breakers |
|
|
39
38
|
| `budget.ts` | token ledgers per request/actor/org, ALS carrier |
|
|
40
39
|
| `prompt.ts` | `definePrompt`, `promptHash`, version registry |
|
|
41
40
|
| `prompt-fence.ts` | a template's data fences: which tag pairs enclose each slot (`promptFences`), and the closer-breaking `render` applies inside one |
|
|
42
41
|
| `embeddings.ts` | `Embedder`, `hashEmbedder()`, cosine helpers |
|
|
43
42
|
| `remote-embedder.ts` | `remoteEmbedder()` — the production `/v1/embeddings` client |
|
|
44
43
|
| `evals.ts` | `defineEval`, the run, the baseline gate, prompt coverage |
|
|
44
|
+
| `eval-agent.ts` | the agent arm of `defineEval`: one case is one `agent.run()` in the eval scope; `agentPrompt` reads the prompt off `agentFactsOf` |
|
|
45
|
+
| `eval-agent-fixture.ts` | the agent-arm suites' harness. Not shipped |
|
|
45
46
|
| `eval-baseline.ts` | the recorded scores: path, read/write, what counts as a regression |
|
|
46
47
|
| `scorers.ts` | what a `Scorer` is, the built-in ones, and `llmJudge` |
|
|
47
48
|
| `vector.ts` | `VectorStore`, in-memory cosine + BM25, RRF hybrid |
|
|
@@ -51,9 +52,10 @@ one adds, the other adds.
|
|
|
51
52
|
| `pg-vector.ts` | `postgresVectorStore()` — the production store |
|
|
52
53
|
| `rag.ts` | chunker, retriever, reranker, budgeted context assembler |
|
|
53
54
|
| `tools.ts` | action → LLM tool definition; the `AgentTool` union and `asProjectableAction`; `runLlmToolCall` |
|
|
55
|
+
| `tools-gate.ts` | a tool's way to `run`: actor half, the name's gate, the tool |
|
|
54
56
|
| `tool-idempotency.ts` | an idempotent action's reserved `idempotencyKey` tool argument — MCP's, restated (same tier), held equal by the parity test |
|
|
55
57
|
| `tool-arguments.ts` / `answer-errors.ts` | one argument-string reader for both wires and what an unreadable one means (`assertToolArguments`); `X_LLM_TRUNCATED` + `X_LLM_TOOL_ARGUMENTS_INVALID` |
|
|
56
|
-
| `billed.ts` / `run-meter.ts` | `meta.billed` + `billedOf`; the meter every `llm()`/`agent()` provider call goes through |
|
|
58
|
+
| `billed.ts` / `run-meter.ts` | `meta.billed` + `billedOf`; the meter every `llm()`/`agent()` provider call goes through, and `meteredRun`, its one public total (`RunMeter` stays internal) |
|
|
57
59
|
| `failure-disclosure.ts` | `discloseFailure` — the ONE rule for what of a throw a remote reader sees (tool result, hive member); the withheld half is logged, never its message or cause |
|
|
58
60
|
| `llm.ts` | `llm()` — the model call, declared as an `action`; and what a streamed answer must satisfy |
|
|
59
61
|
| `respond.ts` | the `respond` tool for one `output` (a non-object one wrapped in `{ value }`), its reader, and `parseJsonish` |
|
|
@@ -62,22 +64,24 @@ one adds, the other adds.
|
|
|
62
64
|
| `agent-def.ts` / `agent-declared.ts` / `agent-system.ts` / `agent-usage.ts` | the declaration's types; its defaults, bound screen, limits and fact row; the system one run sends; its running usage |
|
|
63
65
|
| `llm-def.ts` / `cost.ts` | `llm()`'s declaration types and `LlmCall`; the money arithmetic (`costOf`, `exactCostOf`) |
|
|
64
66
|
| `agent-guards.ts` / `agent-errors.ts` | the repeated-turn detector and the run deadline; their two classes |
|
|
65
|
-
| `prompt-cache.ts` | PROVIDER prompt caching
|
|
67
|
+
| `prompt-cache.ts` / `-rows.ts` | PROVIDER prompt caching, `PROMPT_CACHE_RULES` (sources in its header); the provider-rows screen |
|
|
66
68
|
| `prompt-cache-blocks.ts` | the Anthropic wire's marks (chat-completions': `openai-messages.ts`) |
|
|
67
69
|
| `agent-per-run.ts` / `agent-run-limits.ts` | one run's `offerTools()`/`model()`/`limits()`, narrow only; `costPerRun` |
|
|
68
|
-
| `agent-compact
|
|
70
|
+
| `agent-compact{,-def,-measure,-request,-text}.ts` | trigger, cut; `compact`'s option and event; the token ratio; the summary request, as text when too long |
|
|
71
|
+
| `agent-send.ts` / `context-overflow.ts` / `agent-checkpoint.ts` | a turn sent, an overflow compacted once; `X_AI_CONTEXT_OVERFLOW`'s phrases; `onCheckpoint` |
|
|
69
72
|
| `agent-continue.ts` / `agent-turn.ts` | `.run()` / `.continue()`, the transcript handle and its screen; one turn's report and refusals |
|
|
70
73
|
| `agent-transcript.ts` | what one turn leaves in the transcript: the assistant replay, the tool results, the correction |
|
|
71
74
|
| `agent-facts.ts` | `describeAgents()` — the agent registry, and the row a manifest WOULD publish (its consumer has to be `cli`, tier 5) |
|
|
72
75
|
| `agent-job.ts` | `agentJob()` — an agent as a real `JobHandle`, composed from `job()` |
|
|
73
76
|
| `hive.ts` | `hive()` — one action fanned out over many inputs, declared as an `action` |
|
|
74
77
|
| `hive-result.ts` | `HiveMember` / `HiveResult`, and the SCHEMA built from the member's own `output` |
|
|
75
|
-
| `hive-errors.ts` | the `X_HIVE_*` class; its code and title stay in `
|
|
78
|
+
| `hive-errors.ts` | the `X_HIVE_*` class; its code and title stay in `error-codes.ts` |
|
|
76
79
|
| `redaction.ts` | the one gate between `vars()` and the provider: a `Secret` never reaches a prompt |
|
|
77
|
-
| `eval-errors.ts` | the
|
|
80
|
+
| `eval-errors.ts` | the six `X_EVAL_*` classes; their codes and titles stay in `error-codes.ts` |
|
|
81
|
+
| `error-codes.ts` | the codes this package owns, their titles and the one `registerErrorCodes` call (`new-error-code --package ai` writes here); `errors.ts` holds the core classes |
|
|
78
82
|
| `llm-cache.ts` | the semantic cache half of `llm()`: what a declaration may partition on, and the store it reaches |
|
|
79
83
|
| `llm-fixture.ts` / `bounds-fixture.ts` | the `llm()` harness; the one `refusal`/`asyncRefusal` of every bound suite. Not shipped (`!src/**/*-fixture.ts`) |
|
|
80
|
-
| `runtime.ts` | the ambient gateway / embedder / semantic caches an `llm()` reaches |
|
|
84
|
+
| `runtime.ts` | the ambient gateway / embedder / semantic caches an `llm()` reaches, and the private agent-eval scope (`withEvalScope`): `aiGateway`/`installedGateway` read its gateway first; `runLlmToolCall` resolves its doubles |
|
|
81
85
|
| `fix-line*` | the package's dogfood eval — proof the baseline convention fails a build |
|
|
82
86
|
|
|
83
87
|
## Invariants — bounds and budget
|
|
@@ -124,6 +128,9 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
|
|
|
124
128
|
- **`llm()` forwards `ctx.signal`** onto `GenerateRequest`; `.stream()` inherits it.
|
|
125
129
|
- **`.stream()` is the SAME action**, marked with an ambient sink: UNVALIDATED text, then one `done`
|
|
126
130
|
with the validated value; no repair turn (`X_LLM_STREAM_INVALID`), no `respond` tool. LAZY.
|
|
131
|
+
- **`llm()` streams over its own route** (`stream: LlmTextChunk`, exported): `text/event-stream`
|
|
132
|
+
gets `{ type: 'text' }` frames, then `done`; `thinking` NEVER crosses the wire. Drained
|
|
133
|
+
(`chunksRead()` false: JSON, MCP, `.job()`) it is the ordinary call, repair turn included.
|
|
127
134
|
- **`configureAi({ redact })` is the one seam between `vars()` and the provider**; a `Secret` in
|
|
128
135
|
`vars()` is `X_AI_PROMPT_SECRET` regardless.
|
|
129
136
|
- **Fallback is across PROVIDERS serving one model, never across models**; `GenerateResult.provider`
|
|
@@ -147,14 +154,16 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
|
|
|
147
154
|
- **The gateway's reads of a provider's throw are total**: `isRetryable` fails closed; the line goes
|
|
148
155
|
through `renderThrowable`.
|
|
149
156
|
- **`X_AI_PROVIDER_UNAVAILABLE` is the one code classified `retryable`** (`AI_ERROR_RETRY`);
|
|
150
|
-
`AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`.
|
|
157
|
+
`AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Nine
|
|
151
158
|
deterministic failures are `terminal` (`errors-retry.ts`); the rest stay UNREGISTERED on purpose.
|
|
159
|
+
- **One breaker per provider** (`gateway-providers.ts`, `DEFAULT_GATEWAY_BREAKER` `{ failures: 5,
|
|
160
|
+
cooldownMs: 30_000 }`, `breaker: 'off'` declines). Every settle carries the `admit()`
|
|
161
|
+
ticket. Full rule: `docs/history/ai.md`.
|
|
152
162
|
- **The retry SCHEDULE is core's (`backoffDelay`, `isRetryableStatus`); the LOOP is this package's** —
|
|
153
163
|
core's `retry()` retries unclassified throws, so it would retry a 400 (`gateway-backoff.test.ts`).
|
|
154
|
-
- Every non-2xx and
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
tool call's `input` is parsed (`parseToolArguments`); the credential is scrubbed from `detail`.
|
|
164
|
+
- Every non-2xx and in-band `error` frame is `AiTransportError` with a real `status` — but a
|
|
165
|
+
window overflow (`providerFailure`).
|
|
166
|
+
- **Both wire formats answer the same question the same way** (`provider-parity.test.ts`). Full rule: `docs/history/ai.md`.
|
|
158
167
|
- **Arguments that are not JSON are never thrown by a wire reader**: recorded as `malformed`, and
|
|
159
168
|
the PROVIDER decides once usage is known — dropped on a truncated turn, else
|
|
160
169
|
`X_LLM_TOOL_ARGUMENTS_INVALID` with `meta.billed`, which the gateway RECORDS (`settleFailure`).
|
|
@@ -165,46 +174,25 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
|
|
|
165
174
|
- A tool call is emitted whole. Thinking chunks are never appended to `text`.
|
|
166
175
|
- Anthropic body: no `top_p`/`top_k`, no `budget_tokens`, `effort` inside `output_config`. Model
|
|
167
176
|
IDs are exact aliases, never date-suffixed.
|
|
168
|
-
- **`temperature`/`seed` (`sampling.ts`) are sent ONLY to a row with `sampling: { … : true }
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
registry starts EMPTY; a model resolves declaration → prompt → `Gateway.defaultModel`
|
|
173
|
-
(`resolveModel` / `modelForCall`), none is `X_AI_MODEL_UNRESOLVED` (no gateway at all:
|
|
174
|
-
`X_AI_GATEWAY_MISSING`). `vendor-neutral.test.ts` proves it; no fix line names a vendor id.
|
|
177
|
+
- **`temperature`/`seed` (`sampling.ts`) are sent ONLY to a row with `sampling: { … : true }`**, else
|
|
178
|
+
left out, never refused. Full rule: `docs/history/ai.md`.
|
|
179
|
+
- **Apps bring their own models and providers: no vendor choice, no vendor data.** The registry
|
|
180
|
+
starts EMPTY (`vendor-neutral.test.ts`); resolution is `model-resolve.ts`'s. Full rule: `docs/history/ai.md`.
|
|
175
181
|
- **`ModelId` is `string`; the catalogue is an open registry** — `modelSpec(id)` refuses an
|
|
176
|
-
unregistered id (`X_AI_MODEL_UNKNOWN`) before a socket or a reservation.
|
|
177
|
-
the spec and keeps its rung; registration order is the ladder within a `family`
|
|
178
|
-
(`moreCapableThan`); `X_LLM_REFUSED`'s fix names a rung ABOVE or nothing.
|
|
182
|
+
unregistered id (`X_AI_MODEL_UNKNOWN`) before a socket or a reservation. Full rule: `docs/history/ai.md`.
|
|
179
183
|
- **Every provider serves the app's list** (`{ models }` required, non-empty); no provider has a
|
|
180
184
|
model of its own.
|
|
181
185
|
- **The reasoning controls are PER MODEL (`models.ts`)**: an unasked control is omitted; an asked
|
|
182
|
-
control the model lacks is `X_AI_REQUEST_INVALID`, never dropped.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
(`model-rules.test.ts`, over the fixture rows).
|
|
186
|
-
- **Media blocks are screened once, by `assertMediaContent`, at the top of BOTH body builders** —
|
|
187
|
-
malformed is `X_AI_REQUEST_INVALID`, untakeable (role, the row's `input`, the wire) is
|
|
188
|
-
`X_AI_CONTENT_UNSUPPORTED`; never dropped, never approximated (`provider-parity-content.test.ts`).
|
|
186
|
+
control the model lacks is `X_AI_REQUEST_INVALID`, never dropped. Full rule: `docs/history/ai.md`.
|
|
187
|
+
- **Media blocks are screened once, by `assertMediaContent`, at the top of BOTH body builders**;
|
|
188
|
+
never dropped, never approximated. Full rule: `docs/history/ai.md`.
|
|
189
189
|
- `generate()` above `STREAM_ONLY_MAX_TOKENS` runs the streaming transport.
|
|
190
190
|
- **Every provider and embedder is built by its factory** (`anthropicProvider()`, `echoProvider()`,
|
|
191
191
|
`hashEmbedder()`, `remoteEmbedder()`); the classes are type exports only (`bun run factory-names`).
|
|
192
192
|
- **`openAiProvider()` is a FORMAT, not a vendor** — `baseUrl`, `auth`, `headers` are the
|
|
193
|
-
differences; never a second class per vendor:
|
|
194
|
-
- structured output is the `respond` tool, never `response_format`; `tool_choice` names the tool
|
|
195
|
-
only when the request offers exactly one and the row is not `forcedToolChoice: false`;
|
|
196
|
-
- `strict: true` only when `satisfiesStrictMode` says the schema qualifies;
|
|
197
|
-
- `max_completion_tokens`; `role: 'system'`; `stream_options: { include_usage: true }` and an
|
|
198
|
-
ESTIMATE when usage never arrives; subtract `cached_tokens` from `prompt_tokens`;
|
|
199
|
-
- tool-call deltas merge by `tool_calls[].index`, a differing `id` starting a new call; calls are
|
|
200
|
-
read at `end()` (`[DONE]` or socket close), never at the finish reason; `isComplete()` accepts
|
|
201
|
-
`[DONE]` or a finish reason, except `[DONE]` with pending tool fragments, which is refused;
|
|
202
|
-
- no code of its own (a media refusal is the shared `X_AI_CONTENT_UNSUPPORTED`);
|
|
203
|
-
`AiTransportError` takes the provider's `envVar`; the key is revealed late,
|
|
204
|
-
never stored, scrubbed from `detail`.
|
|
193
|
+
differences; never a second class per vendor. Wire rules: [history](../../docs/history/ai.md#openaiprovider-wire-rules).
|
|
205
194
|
- **`embedBatched` enforces the `Embedder` arity per batch** (`X_AI_EMBEDDER_INVALID` with both
|
|
206
|
-
counts).
|
|
207
|
-
`X_VECTOR_DIM_MISMATCH`.
|
|
195
|
+
counts). Full rule: `docs/history/ai.md`.
|
|
208
196
|
- **A caught value is read with `renderThrowable`** (`remote-embedder.ts`, `wire.ts`,
|
|
209
197
|
`openai-wire.ts`) — `scripts/error-render.ts` cannot see `catch` bindings.
|
|
210
198
|
- **A caller's string is never an object KEY**: the wire tables are `Map`s; `prompt.render` and
|
|
@@ -225,60 +213,43 @@ Why each is shaped this way: [`docs/history/ai.md`](../../docs/history/ai.md#llm
|
|
|
225
213
|
- `ctx.signal` is read at the top of every turn, before every tool batch, and forwarded to
|
|
226
214
|
`fetch`; `X_ABORTED` is core's.
|
|
227
215
|
- One turn's tools run concurrently, results paired by index; no tool STARTS after an abort.
|
|
216
|
+
- **`mcpConfirmations({ tools })` holds a tool here**, one pass each however it arrives
|
|
217
|
+
(`tools-gate.ts`). Proof: `scripts/ai-tool-confirmation.test.ts`.
|
|
228
218
|
- `onTurn` reports facts only (also an `agent.turn` span event); a throw from it fails the run.
|
|
229
|
-
|
|
219
|
+
Streams per turn on its route (`stream: AgentChunk`). No semantic cache.
|
|
230
220
|
- Every replayed `tool_use` is answered by a `tool_result` in the very next message
|
|
231
221
|
(`agent-transcript.ts`); an unaccepted `respond` comes back as an `is_error` `tool_result`
|
|
232
222
|
("superseded").
|
|
233
223
|
- A tool result is rendered totally (`'null'` for `undefined`; a non-JSON result reported as such,
|
|
234
224
|
never as a failure); the throw is read with `stringField`.
|
|
235
|
-
- **A 5xx code's cause reaches the model only when core's `hasPublicCause` allows it
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
rule a hive member's `reason` follows too. Never a second copy. One widening, for an
|
|
239
|
-
`inProcess` tool only (`discloseOwnFailure`): a branded error under a code with NO declared
|
|
240
|
-
status is shown as `{ error: { code, cause, fix } }`; `discloseErrors: false` opts out.
|
|
225
|
+
- **A 5xx code's cause reaches the model only when core's `hasPublicCause` allows it**, else
|
|
226
|
+
`CODE: HIDDEN_TOOL_CAUSE` plus a branded `callerFix` (`failure-disclosure.ts`, never a second
|
|
227
|
+
copy). Full rule: `docs/history/ai.md`.
|
|
241
228
|
- **A run only NARROWS its declaration**: `offerTools()` ⊆ declared names (`X_AGENT_TOOL_UNDECLARED`),
|
|
242
229
|
`model()` via `modelSpec`, `limits()` screened and ≤ declared (`X_AI_REQUEST_INVALID`); all
|
|
243
230
|
before `vars()`. The handle binds the run's model and OFFERED subset; `costPerRun` is checked per turn.
|
|
244
231
|
- **`.run()`/`.continue()` are the SAME action**, marked by an ambient session only the declaring
|
|
245
|
-
agent may claim, once (a nested agent starts fresh). The handle is SIGNED (`sealTranscript
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
232
|
+
agent may claim, once (a nested agent starts fresh). The handle is SIGNED (`sealTranscript`)
|
|
233
|
+
and bound to agent, model, tool names and `orgId`. Full rule: `docs/history/ai.md`.
|
|
234
|
+
- **`compact` (default `'auto'`) acts BEFORE a turn on the MEASURED estimate; an overflow gets one
|
|
235
|
+
forced compaction and a re-send**;
|
|
236
|
+
never truncated (`X_AGENT_CONTEXT_EXHAUSTED`). Full rule: `docs/history/ai.md`.
|
|
237
|
+
- **A summary request too long for its writer goes as TEXT** (`agent-compact-request.ts`,
|
|
238
|
+
`agent-compact-text.ts`): oldest tool results clipped to `SUMMARY_CLIP_CHARS` (2,000, head +
|
|
239
|
+
tail), then left out; a turn is never dropped. Reported on `AgentCompaction`: `summaryInput`
|
|
240
|
+
(`'messages' | 'text'`), `clipped`, `omitted`.
|
|
253
241
|
- **`hive()` is a fan-out action** (`PRIMITIVE_FACTORIES`): `HiveResult` is a SCHEMA from the
|
|
254
|
-
member's `output`;
|
|
255
|
-
|
|
256
|
-
through `discloseFailure` (`HIDDEN_MEMBER_CAUSE`);
|
|
257
|
-
`skipped` reasons `SKIPPED_ABORTED` / `SKIPPED_NO_INPUT`; `onMemberError` is required;
|
|
258
|
-
`concurrency` 4, `minMembers` 2 (a below-floor split runs serially); an empty split is
|
|
259
|
-
`X_HIVE_EMPTY`; an aborted `ctx` is `X_ABORTED`.
|
|
242
|
+
member's `output`; a member's throw goes through `discloseFailure`; the hive never names an actor.
|
|
243
|
+
Full rule: `docs/history/ai.md`.
|
|
260
244
|
- **`describeAgents()` / `registeredModels()` are offered, not published** — their only legal consumer
|
|
261
245
|
is `@ultimat3/cli`. The facts are a THUNK; an unnamed agent has no row.
|
|
262
246
|
- **`agentJob()` composes `job()`** (never an imitation). `name`, `tenant`, `retry` are
|
|
263
|
-
REQUIRED; both reads of `target.job()` are LAZY. No result is kept
|
|
264
|
-
|
|
265
|
-
idempotent (unenforceable). **`actor`** re-resolves who it acts FOR each attempt (a served worker:
|
|
266
|
-
ANONYMOUS) via `impersonate`; off-tenant is TERMINAL `X_JOB_TENANT_MISMATCH`.
|
|
247
|
+
REQUIRED; both reads of `target.job()` are LAZY. No result is kept; `actor` re-resolves
|
|
248
|
+
each attempt via `impersonate` (off-tenant: `X_JOB_TENANT_MISMATCH`). Full rule: `docs/history/ai.md`.
|
|
267
249
|
|
|
268
250
|
## Invariants — provider prompt caching
|
|
269
251
|
|
|
270
|
-
|
|
271
|
-
(wire, host, model prefix → marks, session carrier), never an `if`. An unnamed endpoint gets
|
|
272
|
-
NOTHING. At most 3 `cache_control`, a caller's own counted first (cap 4), never on a blank text
|
|
273
|
-
block; never a `provider` routing object.
|
|
274
|
-
- **`agent()` defaults `'auto'`, `llm()` `'off'`.** The stable system part (`prompt.system` +
|
|
275
|
-
`system().stable`) is sent FIRST; the mark follows the tail. The hint is not in `promptCacheKey`.
|
|
276
|
-
- **Usage**: chat-completions `prompt_tokens` INCLUDES read and written tokens; both are subtracted,
|
|
277
|
-
clamped. A budget debits all four counters; the estimate prices a WRITE only where a provider
|
|
278
|
-
would mark (`marksPromptCache`), at the dearer rate.
|
|
279
|
-
- **`maxRepeatedToolCalls` compares calls AND results over cycles of 1–3 turns**; `deadlineMs` has no
|
|
280
|
-
default and rides the request's AND each tool's signal; both stops are `terminal`
|
|
281
|
-
(`errors-retry.ts`).
|
|
252
|
+
One switch (`GenerateRequest.promptCache`), `promptCacheRule`: provider rows, then `PROMPT_CACHE_RULES`, then nothing. An app's own endpoint is a row ON ITS PROVIDER (`promptCache: { rules }`, `ProviderCacheRow`, first match wins), screened at construction against the wire (`prompt-cache-rows.ts`, `X_AI_REQUEST_INVALID`); `agent()` defaults `'auto'`, `llm()` `'off'`. In full: [`docs/history/ai.md`](../../docs/history/ai.md#moved-out-2026-11-03-plan-101-wave-3-the-claudemd-ceiling).
|
|
282
253
|
|
|
283
254
|
## Invariants — evals, retrieval, fix lines
|
|
284
255
|
|
|
@@ -287,6 +258,8 @@ In brief ([full text](../../docs/history/ai.md#evals-retrieval-fix-lines)): no f
|
|
|
287
258
|
retrieval is hybrid only; an unbound store read under an org actor is `X_VECTOR_UNSCOPED`;
|
|
288
259
|
`postgresVectorStore()` filters tenant and policy in SQL on every statement; `scoped()` TIGHTENS.
|
|
289
260
|
|
|
261
|
+
- **An eval never runs a real tool**: no double is `X_EVAL_TOOL_UNDOUBLED` (history, 2026-11-06).
|
|
262
|
+
|
|
290
263
|
## Commands
|
|
291
264
|
|
|
292
265
|
```
|
package/README.md
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
The LLM gateway primitive. Every model call in an Ultimate app goes through it, so budgets
|
|
4
4
|
and cost accounting cannot be bypassed by a stray `fetch`.
|
|
5
5
|
|
|
6
|
+
`bun add @ultimat3/ai` · guide: [MCP And AI](https://github.com/developerz-ai/ultimate/wiki/MCP-And-AI)
|
|
7
|
+
|
|
6
8
|
**Register your models, pick your provider.** The framework ships the mechanism — the `Provider`
|
|
7
9
|
contract, two wire-format adapters, the gateway, budgets, `registerModel` — and no vendor choice
|
|
8
10
|
and no vendor data: no default model, no catalogue row, no price. Which models an app runs, at what
|
|
@@ -71,6 +73,7 @@ provider whose `models` lists it.
|
|
|
71
73
|
| `X_LLM_REFUSED`'s "a more capable model" | a rung above, in the same `family`, among the rows the APP registered — or no suggestion at all |
|
|
72
74
|
| a direct provider call naming no model | `X_AI_MODEL_UNRESOLVED` from every shipped provider — `anthropicProvider()`, `openAiProvider()` and `echoProvider()` alike; a provider's `models` list is what it serves, never a fallback |
|
|
73
75
|
| `echoProvider()` | serves whatever the app registered, and has no model of its own |
|
|
76
|
+
| `scriptedProvider({ turns, usage?, name? })` | the test double that calls tools: each provider call answers the next turn — `{ text }`, `{ call, input? }`, `{ calls: [...] }` or `{ answer }` (a `respond` call), each with an optional `stop` — and `.seen` keeps every request. Serves whatever the app registered. One call past the last turn is `X_AI_SCRIPT_EXHAUSTED`, never the last turn again: a loop must not look like a pass |
|
|
74
77
|
|
|
75
78
|
## Budgets — and which of the three is fleet-wide
|
|
76
79
|
|
|
@@ -150,7 +153,9 @@ default store at `replicas: 6` is six ledgers of twenty million, which is a budg
|
|
|
150
153
|
| `reserve()` **debits, then checks** | the in-memory scopes check and debit with no `await` between, the store's through its atomic `take`; a read-then-write let concurrent calls — one request per scope racing one org key — all read the same `spent()` and all pass. `record` reconciles and `release` gives it back |
|
|
151
154
|
| The cache key is every tool **whole**, through core's `fingerprint` | every `llm()` tool is named `respond`, so a key over names served an answer shaped for an old `output` schema; a cache `set` that throws is logged, never turned into a failure of a call already paid for |
|
|
152
155
|
| A refusal is never cached | a cached one keeps serving a classifier decision after the prompt was fixed |
|
|
156
|
+
| A cache hit reads zero `usage`, `cost` and `costExact`, in the same currency | it is not debited, so a meter summing the stored figures read more than the budget beside it |
|
|
153
157
|
| Retries use **full jitter**, from core's one curve | synchronised retries from N workers reproduce the rate limit. `backoffMs` is `@ultimat3/core`'s `backoffDelay` with the gateway's field names mapped onto it, and the roll is `providerGateway({ random })` — injectable, so the schedule is a unit test rather than a range |
|
|
158
|
+
| **One breaker per provider**, on by default: `{ failures: 5, cooldownMs: 30_000 }` (`DEFAULT_GATEWAY_BREAKER`) | a provider that ran out its retries five passes in a row is skipped, no call and no sleep, until one probe after the cooldown answers; without it every call re-spent every attempt on a dead provider. `providerGateway({ breaker })` takes `GatewayBreakerOption`, `((provider) => Breaker) \| 'off'`: your own core `breaker({ … })` per provider, or `'off'` for the old behaviour. A stream goes to the first provider whose breaker admits it |
|
|
154
159
|
| A 4xx is never retried **except 408, 409 and 425** | the same body gets the same rejection and burns the budget — but a request the server stopped reading (408), a round a concurrent writer won (409) and a handshake that had not finished (425) are transient by construction, and core's `isRetryableStatus` is the one table that says so |
|
|
155
160
|
|
|
156
161
|
## Streaming
|
|
@@ -401,6 +406,16 @@ invocation is an ordinary one, marked so the model half streams. Two consequence
|
|
|
401
406
|
| no `respond` tool is offered | a tool call is emitted whole, so forcing one leaves nothing to stream — the answer is prose, and its JSON parse is what a non-string `output` validates |
|
|
402
407
|
| lazy | nothing is authorised, budgeted or sent until the first pull |
|
|
403
408
|
|
|
409
|
+
### Streaming on the wire
|
|
410
|
+
|
|
411
|
+
`llm()` declares `stream: LlmTextChunk` (`{ type: 'text', text }`), so its own route answers
|
|
412
|
+
`text/event-stream` to the typed client's `summarize.stream(input)`: each text increment as the
|
|
413
|
+
provider sends it, then `done` with the value that satisfied `output`. Same policy, budget, cache
|
|
414
|
+
and span; the streamed turn is the one-attempt turn server-side `.stream()` takes. `thinking` never
|
|
415
|
+
crosses the wire. Every other caller (JSON, MCP, `.job()`) gets the non-streamed call, repair turn
|
|
416
|
+
included. `agent()` declares `stream: AgentChunk` the same way: per turn, `text`, `tool_call`,
|
|
417
|
+
`tool_result` and `turn` facts, then `done`.
|
|
418
|
+
|
|
404
419
|
## `agent()` — the tool loop, also an action
|
|
405
420
|
|
|
406
421
|
The second half of "no ninth primitive": a tool-using run is still one server-authoritative
|
|
@@ -436,6 +451,7 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug
|
|
|
436
451
|
| the tools of **one turn** run concurrently, results paired by `tool_use` id | a turn asking for five tools cost 5x wall clock and nothing said so. Order is positional, never by completion; the batch is bounded by what one turn asked for, and each tool is an action with its own `policy` and `rateLimit`, so a second ceiling here would be a throttle competing with those |
|
|
437
452
|
| `onTurn` reports each completed turn as it happens (and an `agent.turn` span event, always) | a 90-second run emitted nothing until it returned. Observation only — it cannot steer the loop, see the transcript or reach the actor — and a throw from it fails the run rather than being swallowed |
|
|
438
453
|
| a tool is an `mcp: { expose: true }` action or an `inProcessTool()`; anything else is `X_AGENT_TOOL_UNEXPOSED` **at declaration** | a silently dropped tool reads as offered and is not; `isMcpExposed` is the one predicate for ACTIONS, so an in-app agent and an external MCP client see the same catalogue of them — and a tool no MCP client may see is declared as one that is this process's alone |
|
|
454
|
+
| a tool in `mcpConfirmations({ tools })` answers `X_MCP_CONFIRMATION_PENDING` until a person approves | one list for both surfaces, so no option an author can forget. The decision action declares no `mcp.expose`, so listing it is `X_AGENT_TOOL_UNEXPOSED`: **exposing it hands an in-app agent, whose actor is the user, its own approval** |
|
|
439
455
|
| running out of turns is `X_AGENT_MAX_TURNS`, never a partial answer | a half-finished transcript returned as a result is working notes presented as a decision |
|
|
440
456
|
| a cycle of 1 to 3 tool turns — same calls, arguments, results and failures — gone round `maxRepeatedToolCalls` (default 3) times is `X_AGENT_TOOL_LOOP` | such a turn can only be followed by another like it; a changed result is progress; a repair turn neither counts nor resets |
|
|
441
457
|
| `deadlineMs` (no default) is the run's wall-clock ceiling: `X_AGENT_DEADLINE` | the in-flight model call is aborted on the socket and every tool's `ctx.signal` fires; a tool that ignores it still finishes late. A caller that leaves first is still `X_ABORTED` |
|
|
@@ -452,10 +468,48 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug
|
|
|
452
468
|
|
|
453
469
|
`cost` is the currency's minor units, rounded up once per call; `costExact` beside it is the same figure exactly (`Money` at a finer `scale`, from `exactCostOf`). `agent()` reports both per turn and for the run in `onTurn`; `llm()` per provider call in `onCall`; a ledger sums exact costs and rounds once when read. `repairOf` on either event is what a repair turn was told to fix.
|
|
454
470
|
|
|
471
|
+
One total across several calls — a job that runs two `llm()` calls and an `agent()` — is
|
|
472
|
+
`meteredRun(fn)` → `{ value, billed }`: every provider call `fn` makes, however deeply nested, summed
|
|
473
|
+
once (`Billed`, `undefined` when none started). A throw is rethrown with that total on `meta.billed`,
|
|
474
|
+
so `billedOf(error)` reads it. A meter, not a ledger: it refuses nothing.
|
|
475
|
+
|
|
455
476
|
### Continuing a run
|
|
456
477
|
|
|
457
478
|
`agent()` returns an `AgentAction`: the action, plus `.run(input, opts)` → `{ output, transcript, turns, usage }` and `.continue(input, { transcript, user }, opts)`, which appends one user turn to a stored transcript and resumes the loop. The transcript is re-sent byte for byte (the provider's prompt cache reads it back; no tool re-runs); `maxTurns` and `budget.tokensPerRun` are cumulative; the handle is signed, bound to the agent, model, tools and tenant, and screened (`X_AGENT_TRANSCRIPT_INVALID`); a failed call leaves an updated one on the error (`transcriptOf(error)`), which is the one a job stores. Full contract: [Agents → Continuing a run](https://github.com/developerz-ai/ultimate/wiki/Agents#continuing-a-run-run--continue).
|
|
458
479
|
|
|
480
|
+
`onCheckpoint(event, { input, ctx })` hands the app the same signed handle **between** turns — after a
|
|
481
|
+
turn's tool results (or its correction) land, before the next turn is bought, never after the answer
|
|
482
|
+
(`.run()` returns that one). Store `event.transcript` on the run's own row; the next attempt calls
|
|
483
|
+
`.continue(input, { transcript, user })` and pays only for the turns left. Awaited; a throw fails
|
|
484
|
+
the run; sealed only when declared. A worker dying DURING a tool batch re-asks that turn, so its
|
|
485
|
+
tools may run again — at-least-once still binds.
|
|
486
|
+
|
|
487
|
+
```ts
|
|
488
|
+
import type { AgentAction, AgentTranscript } from '@ultimat3/ai';
|
|
489
|
+
import type { Ctx } from '@ultimat3/core';
|
|
490
|
+
import { stringField } from '@ultimat3/core';
|
|
491
|
+
import type { t } from '@ultimat3/schema';
|
|
492
|
+
|
|
493
|
+
type Review = ReturnType<typeof t.object<{ verdict: typeof t.string }>>;
|
|
494
|
+
type In = ReturnType<typeof t.object<{ postId: typeof t.string }>>;
|
|
495
|
+
// Declared with onCheckpoint: ({ transcript }, { input }) => saveProgress(input.postId, transcript)
|
|
496
|
+
declare const reviewer: AgentAction<In, Review>;
|
|
497
|
+
declare function progress(postId: string): Promise<AgentTranscript | undefined>;
|
|
498
|
+
|
|
499
|
+
// The action an agentJob() runs: resume from the stored handle, else start.
|
|
500
|
+
export async function review(ctx: Ctx, input: { postId: string }) {
|
|
501
|
+
const stored = await progress(input.postId);
|
|
502
|
+
try {
|
|
503
|
+
if (stored === undefined) return await reviewer.run(input, { ctx });
|
|
504
|
+
return await reviewer.continue(input, { transcript: stored, user: 'Carry on.' }, { ctx });
|
|
505
|
+
} catch (error) {
|
|
506
|
+
// A prompt bump or a model change invalidates a stored handle by design: start fresh.
|
|
507
|
+
if (stringField(error, 'code') !== 'X_AGENT_TRANSCRIPT_INVALID') throw error;
|
|
508
|
+
return reviewer.run(input, { ctx });
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
```
|
|
512
|
+
|
|
459
513
|
### Auto-compaction
|
|
460
514
|
|
|
461
515
|
```ts
|
|
@@ -480,15 +534,17 @@ export const researcher = agent({
|
|
|
480
534
|
});
|
|
481
535
|
```
|
|
482
536
|
|
|
483
|
-
Before each turn, the gateway's estimate of that turn's request
|
|
484
|
-
`contextWindow` less `maxTokens`. Over `at` of it, ONE model call summarises every turn older than
|
|
537
|
+
Before each turn, the gateway's estimate of that turn's request — scaled by what the provider
|
|
538
|
+
reported for the last one — is compared with the model row's `contextWindow` less `maxTokens`. Over `at` of it, ONE model call summarises every turn older than
|
|
485
539
|
the last `keepTurns`; the summary joins the first user message (after the prompt, verbatim, marked
|
|
486
540
|
`[compacted: …]`), the kept turns follow byte for byte, and the turn is sent.
|
|
487
541
|
|
|
488
542
|
| Rule | Why |
|
|
489
543
|
|---|---|
|
|
490
544
|
| default **`'auto'`**; `'off'` never compacts and never checks | a summary changes what the model sees, but only above `at` of the window, where the alternative is a run lost to a 400 after every turn was paid for. Below it a run is byte-identical to `'off'` — requests and handle — and it is never silent: `onCompact`, the `agent.compact` span event, the handle's `compacted` |
|
|
491
|
-
| triggered on the **estimate of the next request** — the gateway's prompt estimate plus the tool definitions — over `at` × (`contextWindow` − `maxTokens`), before the call |
|
|
545
|
+
| triggered on the **estimate of the next request** — the gateway's prompt estimate plus the tool definitions — over `at` × (`contextWindow` − `maxTokens`), before the call | `maxTokens` is reserved because the completion shares the window |
|
|
546
|
+
| the estimate is **scaled by the measurement**: after each turn, `max(1, (inputTokens + cacheReadTokens + cacheWriteTokens) / estimate)` of that turn's request; `before`/`after` report the scaled figure | four characters a token is wrong by 3x on dense JSON and non-Latin text, and a run walked past the window it was meant to compact before. A turn reporting zero keeps the last ratio (a provider that never reports compacts where it always did); below 1 is ignored; per run, never signed — a continuation's first turn estimates plainly |
|
|
547
|
+
| a provider that refuses the turn anyway (**`X_AI_CONTEXT_OVERFLOW`**) gets **one forced compaction** and the same turn re-sent; a second refusal, a compaction that cannot be had, or `compact: 'off'` is `X_AGENT_CONTEXT_EXHAUSTED` with the provider's text on `meta.refused` | the loop still holds the transcript after that 400. The refusal billed nothing (`meta.billed` at zero), is not a turn, and is never retried by the gateway: the same request overflows on every provider |
|
|
492
548
|
| the system prompt, its stable block and the tools are **never touched** | a provider cache is a prefix match over tools → system → messages: they are the part every turn and every run shares, so they are still read at the cache rate after a compaction. Only the messages are rewritten, once |
|
|
493
549
|
| the summary call is the run's **own request** up to the cut, plus one instruction — same gateway, tools, system and `promptCache` hint | it reads the prefix the turn that ended at the cut already cached instead of paying for it again; on OpenRouter the same `x-session-id` routes it to the provider holding that cache (`compact.model` trades that read for a cheaper writer) |
|
|
494
550
|
| the cut falls **between turns**: the prompt and the last `keepTurns` (4, at least 1) turns are verbatim | a turn is an assistant message and the user message answering every `tool_use` in it, so no call is ever separated from its result (a 400 on every provider); the model keeps its task as written and the results it just asked for |
|
|
@@ -501,7 +557,8 @@ the last `keepTurns`; the summary joins the first user message (after the prompt
|
|
|
501
557
|
|
|
502
558
|
| Limit | Detail |
|
|
503
559
|
|---|---|
|
|
504
|
-
| the estimate | ~4 characters a token, which is why `at` defaults below 1 |
|
|
560
|
+
| the estimate | ~4 characters a token until the first turn reports, which is why `at` defaults below 1 |
|
|
561
|
+
| the forced compaction | its summary call is the refused request less the kept turns: with a tiny `keepTurns` it may not fit either — `compact.model` with a larger window is the way out |
|
|
505
562
|
| `tools: []` on the chat-completions wire | only `respond` is offered, so it is forced, the summary call answers no text and the run falls back to whole |
|
|
506
563
|
| per run | `compact` is the declaration's; `limits()` does not take it, `describeAgents()` does not publish it |
|
|
507
564
|
|
|
@@ -555,6 +612,15 @@ export const kbAgent = agent({
|
|
|
555
612
|
| chat completions | `api.openai.com` | any | none (automatic) | `prompt_cache_key` |
|
|
556
613
|
| chat completions | anything else | any | none | — |
|
|
557
614
|
|
|
615
|
+
An endpoint the table does not know is the app's to declare, on its provider:
|
|
616
|
+
`openAiProvider({ baseUrl, models, promptCache: { rules: [{ model: 'anthropic/', marks: 'parts', session: 'x-session-id' }] } })`
|
|
617
|
+
marks for a company gateway; `anthropicProvider({ baseUrl, models, promptCache: { rules: [{ model: '', marks: 'none', session: 'none' }] } })`
|
|
618
|
+
stops marks to a proxy that rejects `cache_control`. A row (`ProviderCacheRow`) is `{ model, marks,
|
|
619
|
+
session }`, read before the table, first match by model prefix; the pre-flight estimate follows it.
|
|
620
|
+
The Messages wire speaks `blocks` or `none` and no session; chat-completions `parts` or `none`, and
|
|
621
|
+
`x-session-id`, `prompt_cache_key` or `none`. Anything else is `X_AI_REQUEST_INVALID` when the
|
|
622
|
+
provider is built.
|
|
623
|
+
|
|
558
624
|
Usage: the chat-completions reader takes `prompt_tokens_details.cached_tokens` and
|
|
559
625
|
`.cache_write_tokens` out of `prompt_tokens` (which includes both) and reports them as
|
|
560
626
|
`cacheReadTokens` / `cacheWriteTokens`; a missing field is zero. `costOf` prices them at the model
|
|
@@ -567,7 +633,7 @@ ever sent.
|
|
|
567
633
|
| Limit | Detail |
|
|
568
634
|
|---|---|
|
|
569
635
|
| `run.cost` and the ledger | one contract: cost is computed exactly and rounded up once. `AgentTurn.run.cost` is the run's summed usage priced once; `gateway.spent()` / `BudgetLedger.report().cost` is the exact cost of every recorded call summed, rounded once when read (`costExact` beside it). They agree for a run on built-in providers; a `Provider` that states no `costExact` is summed from its rounded `cost`. Budgets are enforced against the ledger |
|
|
570
|
-
| the Anthropic row matches any host | an `anthropicProvider({ baseUrl })` behind a proxy still sends marks |
|
|
636
|
+
| the Anthropic row matches any host | an `anthropicProvider({ baseUrl })` behind a proxy still sends marks, unless it declares a `marks: 'none'` row |
|
|
571
637
|
| more than about 9 tool calls in one turn, on OpenRouter | only the tail is marked there and the provider looks back 20 blocks, so that turn's transcript is rewritten, not read |
|
|
572
638
|
| a mark never lands on an empty or whitespace-only text block, and never makes a fifth | it moves back a block or is dropped; a raw caller's own marks are counted first |
|
|
573
639
|
| `session` | OpenRouter header: 1 to 256 printable ASCII, else `X_AI_REQUEST_INVALID`. OpenAI `prompt_cache_key`: over 64 characters it is sent as `h:` + 62 hex of its SHA-256 |
|
|
@@ -854,7 +920,46 @@ X_EVAL_THRESHOLD: an eval scored below its tolerance
|
|
|
854
920
|
|
|
855
921
|
Built-in scorers: `exact`, `contains`, `jsonValid`, `jsonSchemaValid(keys)`,
|
|
856
922
|
`numericTolerance(t)`, `llmJudge({ judge })` — the judge prompt is itself versioned, so a
|
|
857
|
-
judge that drifts is a measuring instrument that lies, and its hash is in the scorer name
|
|
923
|
+
judge that drifts is a measuring instrument that lies, and its hash is in the scorer name —
|
|
924
|
+
and `calledTool(name)`, 1 when an agent run called that tool.
|
|
925
|
+
|
|
926
|
+
### The agent arm
|
|
927
|
+
|
|
928
|
+
`DefineEvalInput` is a union: `PromptEvalInput` (`prompt` + `cases[].vars`) or `AgentEvalInput`
|
|
929
|
+
(`agent` + `cases[].input` + `tools`). Narrow with `'prompt' in input`. The agent arm runs
|
|
930
|
+
`agent.run(input, { actor })` per case — default actor `serviceActor({ id: 'eval' })` — with the
|
|
931
|
+
gateway handed to `run` / `assert` and every tool answered by its double in `tools`:
|
|
932
|
+
|
|
933
|
+
```ts
|
|
934
|
+
import {
|
|
935
|
+
calledTool,
|
|
936
|
+
defineEval,
|
|
937
|
+
type EvaluatedAgent,
|
|
938
|
+
providerGateway,
|
|
939
|
+
type ScriptedTurn,
|
|
940
|
+
scriptedProvider,
|
|
941
|
+
} from '@ultimat3/ai';
|
|
942
|
+
|
|
943
|
+
declare const supportAgent: EvaluatedAgent<{ question: string }>; // your registered agent()
|
|
944
|
+
declare const turns: ScriptedTurn[];
|
|
945
|
+
|
|
946
|
+
const supportEval = defineEval({
|
|
947
|
+
name: 'support.answer',
|
|
948
|
+
agent: supportAgent,
|
|
949
|
+
cases: [{ name: 'where-is-o-1', input: { question: 'Where is o-1?' } }],
|
|
950
|
+
tools: { lookupOrder: () => ({ status: 'shipped' }) },
|
|
951
|
+
scorers: [calledTool('lookupOrder')],
|
|
952
|
+
baseline: import.meta.resolve('./support.baseline.json'),
|
|
953
|
+
tolerance: 0.05,
|
|
954
|
+
});
|
|
955
|
+
await supportEval.assert(providerGateway({ providers: [scriptedProvider({ turns })] }));
|
|
956
|
+
```
|
|
957
|
+
|
|
958
|
+
An undoubled call is `X_EVAL_TOOL_UNDOUBLED` (`EvalToolUndoubledError`) and ends the eval: an eval
|
|
959
|
+
never runs a real tool. Any other failed run scores 0 and records its code in `cases[].failed`.
|
|
960
|
+
Scorers get `run: { turns, toolCalls, usage }` (`EvalRun`, `EvalToolCall`, `ScorerInput`); the
|
|
961
|
+
baseline is the same file, keyed by the agent's prompt hash. Exported types: `AgentEvalCase`,
|
|
962
|
+
`AgentEvalInput`, `EvaluatedAgent`, `PromptEvalInput`, `ToolDouble`.
|
|
858
963
|
|
|
859
964
|
## Prompts are versioned artifacts
|
|
860
965
|
|
|
@@ -987,9 +1092,19 @@ An in-app agent and an external MCP agent both end at the same `invoke` — `run
|
|
|
987
1092
|
carries it, and an action facade has no `.run` of its own. So they authorize identically. The
|
|
988
1093
|
actor comes from the request context, never from the model.
|
|
989
1094
|
|
|
1095
|
+
A tool in `mcpConfirmations({ tools })` waits for a person here too. `run` asks the policy's actor
|
|
1096
|
+
half, then the gate registered for that name (read per call), then `invoke`; `asProjectableAction`
|
|
1097
|
+
puts a hand-built tool behind the same gate, and `runLlmToolCall` projects whatever it is handed, so
|
|
1098
|
+
it is one pass whichever way a tool arrives. The model reads `X_MCP_CONFIRMATION_PENDING`, reports
|
|
1099
|
+
through `output`, and a later user turn repeats the call. No new run outcome. The gate binds a tool's
|
|
1100
|
+
**name**, never what it reaches: `inProcessTool({ name: 'refundNow', execute: () => invoke(refundOrder, …) })`
|
|
1101
|
+
is not gated by `refundOrder`'s row, so list `refundNow` in `tools`.
|
|
1102
|
+
[Agents → A tool that waits for a person](https://github.com/developerz-ai/ultimate/wiki/Agents#a-tool-that-waits-for-a-person).
|
|
1103
|
+
|
|
990
1104
|
`inProcessTool({ name, description, input, execute })` is a tool that is this process's alone: no `mcp`, no policy of its own (the agent's decides), in no catalogue but the agent's. Its own typed refusal reads back as `{"error":{"code","cause","fix"}}`. [Agents → In-process tools](https://github.com/developerz-ai/ultimate/wiki/Agents#in-process-tools).
|
|
991
1105
|
|
|
992
|
-
A failed tool reads back as `CODE: cause (fix: …)
|
|
1106
|
+
A failed tool reads back as `CODE: cause (fix: …)`, the fix being the error's `callerFix` when it
|
|
1107
|
+
declares one — except a 5xx code whose cause core's
|
|
993
1108
|
`hasPublicCause` does not declare public, which reads `CODE: ` plus a fixed sentence (and a
|
|
994
1109
|
`callerFix`, when the `UltimateError` declared one): the rule a production problem document follows,
|
|
995
1110
|
and the one `@ultimat3/mcp` applies. The model's provider keeps every request it is sent, so a
|
|
@@ -1021,7 +1136,8 @@ fail-closed `terminal` default.
|
|
|
1021
1136
|
| `X_AGENT_TOOL_UNDECLARED` | an `agent()` run's `offerTools()` named a tool its declaration does not list |
|
|
1022
1137
|
| `X_AGENT_TOOL_LOOP` | an `agent()` made the same tool calls and read the same results `maxRepeatedToolCalls` turns in a row |
|
|
1023
1138
|
| `X_AGENT_DEADLINE` | an `agent()` outlived its declared `deadlineMs` |
|
|
1024
|
-
| `X_AGENT_CONTEXT_EXHAUSTED` | an `agent()`'s next request would not fit its model's context window, and compaction could not bring it under
|
|
1139
|
+
| `X_AGENT_CONTEXT_EXHAUSTED` | an `agent()`'s next request would not fit its model's context window, and compaction could not bring it under — refused before it is sent, or after the provider refused it twice (`meta.refused`) |
|
|
1140
|
+
| `X_AI_CONTEXT_OVERFLOW` | a provider answered 400/413 saying the prompt outgrew the model's window; never retried, `terminal` for a job. `llm()` and `generate()` raise it as itself; `agent()` compacts once first |
|
|
1025
1141
|
| `X_EVAL_THRESHOLD` | an eval scored below its bar |
|
|
1026
1142
|
| `X_VECTOR_DIM_MISMATCH` | a vector's length disagrees with the store |
|
|
1027
1143
|
| `X_VECTOR_SCOPE_WIDENED` | a derived vector scope tried to leave the tenant it was bound to |
|
|
@@ -1061,6 +1177,7 @@ a job boundary the class is gone and the `code` is what survives — match on th
|
|
|
1061
1177
|
| `EvalMissingError` | `X_EVAL_MISSING` | `src/eval-errors.ts` |
|
|
1062
1178
|
| `EvalRecordingError` | `X_EVAL_RECORDING` | `src/eval-errors.ts` |
|
|
1063
1179
|
| `EvalThresholdError` | `X_EVAL_THRESHOLD` | `src/eval-errors.ts` |
|
|
1180
|
+
| `EvalToolUndoubledError` | `X_EVAL_TOOL_UNDOUBLED` | `src/eval-errors.ts` |
|
|
1064
1181
|
| `HiveEmptyError` | `X_HIVE_EMPTY` | `src/hive-errors.ts` |
|
|
1065
1182
|
| `LlmOutputInvalidError` | `X_LLM_OUTPUT_INVALID` | `src/errors.ts` |
|
|
1066
1183
|
| `LlmRefusedError` | `X_LLM_REFUSED` | `src/errors.ts` |
|
package/package.json
CHANGED
|
@@ -1,7 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/ai",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "30.0.0",
|
|
4
4
|
"description": "LLM gateway, versioned prompts, evals as tests, embeddings, hybrid vector search, RAG",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ultimate",
|
|
7
|
+
"bun",
|
|
8
|
+
"framework",
|
|
9
|
+
"ai",
|
|
10
|
+
"llm",
|
|
11
|
+
"agents",
|
|
12
|
+
"evals",
|
|
13
|
+
"rag",
|
|
14
|
+
"embeddings"
|
|
15
|
+
],
|
|
16
|
+
"homepage": "https://github.com/developerz-ai/ultimate/wiki/MCP-And-AI",
|
|
5
17
|
"license": "MIT",
|
|
6
18
|
"type": "module",
|
|
7
19
|
"repository": {
|
|
@@ -32,17 +44,17 @@
|
|
|
32
44
|
"test": "bun test"
|
|
33
45
|
},
|
|
34
46
|
"dependencies": {
|
|
35
|
-
"@ultimat3/action": "
|
|
36
|
-
"@ultimat3/cache": "
|
|
37
|
-
"@ultimat3/core": "
|
|
38
|
-
"@ultimat3/db": "
|
|
39
|
-
"@ultimat3/http": "
|
|
40
|
-
"@ultimat3/jobs": "
|
|
41
|
-
"@ultimat3/money": "
|
|
42
|
-
"@ultimat3/schema": "
|
|
43
|
-
"@ultimat3/time": "
|
|
47
|
+
"@ultimat3/action": "30.0.0",
|
|
48
|
+
"@ultimat3/cache": "30.0.0",
|
|
49
|
+
"@ultimat3/core": "30.0.0",
|
|
50
|
+
"@ultimat3/db": "30.0.0",
|
|
51
|
+
"@ultimat3/http": "30.0.0",
|
|
52
|
+
"@ultimat3/jobs": "30.0.0",
|
|
53
|
+
"@ultimat3/money": "30.0.0",
|
|
54
|
+
"@ultimat3/schema": "30.0.0",
|
|
55
|
+
"@ultimat3/time": "30.0.0"
|
|
44
56
|
},
|
|
45
57
|
"devDependencies": {
|
|
46
|
-
"@ultimat3/policy": "
|
|
58
|
+
"@ultimat3/policy": "30.0.0"
|
|
47
59
|
}
|
|
48
60
|
}
|