@ultimat3/ai 29.4.0 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CLAUDE.md +63 -90
  2. package/README.md +125 -8
  3. package/package.json +23 -11
  4. package/src/agent-checkpoint.ts +44 -0
  5. package/src/agent-compact-def.ts +11 -0
  6. package/src/agent-compact-measure.ts +38 -0
  7. package/src/agent-compact-request.ts +100 -0
  8. package/src/agent-compact-text.ts +95 -0
  9. package/src/agent-compact.ts +75 -50
  10. package/src/agent-continue.ts +15 -3
  11. package/src/agent-def.ts +9 -0
  12. package/src/agent-errors.ts +12 -2
  13. package/src/agent-facts.ts +8 -0
  14. package/src/agent-send.ts +82 -0
  15. package/src/agent-stream.ts +76 -0
  16. package/src/agent.ts +71 -21
  17. package/src/answer-errors.ts +1 -1
  18. package/src/anthropic-provider.ts +291 -0
  19. package/src/content-errors.ts +1 -1
  20. package/src/context-overflow.ts +86 -0
  21. package/src/error-codes.ts +95 -0
  22. package/src/errors-retry.ts +8 -4
  23. package/src/errors.ts +6 -87
  24. package/src/eval-agent.ts +100 -0
  25. package/src/eval-errors.ts +31 -5
  26. package/src/evals.ts +125 -45
  27. package/src/gateway-providers.ts +215 -0
  28. package/src/gateway.ts +63 -153
  29. package/src/hive-errors.ts +1 -1
  30. package/src/in-process-tool.ts +20 -11
  31. package/src/index.ts +31 -7
  32. package/src/llm.ts +39 -8
  33. package/src/model-resolve.ts +7 -1
  34. package/src/openai-body.ts +12 -5
  35. package/src/openai-provider.ts +28 -8
  36. package/src/openai-wire.ts +11 -7
  37. package/src/prompt-cache-rows.ts +82 -0
  38. package/src/prompt-cache.ts +24 -2
  39. package/src/provider.ts +15 -274
  40. package/src/remote-embedder.ts +38 -11
  41. package/src/run-meter.ts +40 -2
  42. package/src/runtime.ts +38 -3
  43. package/src/scorers.ts +35 -1
  44. package/src/scripted-provider.ts +146 -0
  45. package/src/tools-gate.ts +46 -0
  46. package/src/tools.ts +101 -27
  47. package/src/wire.ts +12 -4
package/src/run-meter.ts CHANGED
@@ -104,6 +104,10 @@ export function runMeter(): RunMeter {
104
104
  unsettled -= 1;
105
105
  absorb(error);
106
106
  parent?.report(bill);
107
+ } else {
108
+ // Never answered: its bill cannot be known, so it travels up as a count. Without this a
109
+ // caught failure, or one beside a sibling that settled, vanished from the parent's total.
110
+ parent?.report(UNSETTLED_CALL);
107
111
  }
108
112
  throw error;
109
113
  }
@@ -131,7 +135,36 @@ export function runMeter(): RunMeter {
131
135
  };
132
136
  }
133
137
 
134
- /** Two bills as one. A cost in another currency cannot be added and is left out of the sum. */
138
+ /**
139
+ * One bill across every `llm()` and `agent()` call `fn` makes, however deep they nest — what a
140
+ * job reads to say what its run cost. A throw is rethrown carrying the total on `meta.billed`
141
+ * (`billedOf(error)`); `billed` is `undefined` when no provider call was ever started. A meter,
142
+ * not a ledger: it refuses nothing.
143
+ */
144
+ export async function meteredRun<T>(
145
+ fn: () => T | Promise<T>,
146
+ ): Promise<{ readonly value: T; readonly billed: Billed | undefined }> {
147
+ const meter = runMeter();
148
+ try {
149
+ const value = await meter.within(fn);
150
+ return { value, billed: meter.billed() };
151
+ } catch (error) {
152
+ throw meter.fail(error);
153
+ }
154
+ }
155
+
156
+ /** One started call that never reported usage: counted, never priced. */
157
+ const UNSETTLED_CALL: Billed = {
158
+ usage: ZERO_USAGE,
159
+ cost: { minor: 0, currency: 'USD' },
160
+ calls: 0,
161
+ unsettled: 1,
162
+ };
163
+
164
+ /**
165
+ * Two bills as one. A cost in another currency cannot be added and is left out of the sum — except
166
+ * a zero, which carries no currency worth keeping: an unsettled call's placeholder never decides it.
167
+ */
135
168
  function sum(a: Billed, b: Billed): Billed {
136
169
  return {
137
170
  usage: {
@@ -140,7 +173,12 @@ function sum(a: Billed, b: Billed): Billed {
140
173
  cacheReadTokens: a.usage.cacheReadTokens + b.usage.cacheReadTokens,
141
174
  cacheWriteTokens: a.usage.cacheWriteTokens + b.usage.cacheWriteTokens,
142
175
  },
143
- cost: a.cost.currency === b.cost.currency ? addMoney(a.cost, b.cost) : a.cost,
176
+ cost:
177
+ a.cost.currency === b.cost.currency
178
+ ? addMoney(a.cost, b.cost)
179
+ : a.cost.minor === 0
180
+ ? b.cost
181
+ : a.cost,
144
182
  calls: a.calls + b.calls,
145
183
  unsettled: a.unsettled + b.unsettled,
146
184
  };
package/src/runtime.ts CHANGED
@@ -9,7 +9,7 @@
9
9
 
10
10
  import type { SemanticCache } from '@ultimat3/cache';
11
11
  import { memorySemanticCache } from '@ultimat3/cache';
12
- import { cachedFormatter } from '@ultimat3/core';
12
+ import { asyncContext, cachedFormatter } from '@ultimat3/core';
13
13
  import type { Embedder } from './embeddings';
14
14
  import { hashEmbedder } from './embeddings';
15
15
  import { AiGatewayMissingError } from './errors';
@@ -71,15 +71,50 @@ export function configureAi(input: AiRuntimeInput): void {
71
71
  caches.clear();
72
72
  }
73
73
 
74
- /** The installed gateway. `prompt` names the caller so the miss says what was about to run. */
74
+ /** A tool double an agent eval declares: the model's arguments in, the tool's result out. */
75
+ export type ToolDouble = (input: Readonly<Record<string, unknown>>) => unknown;
76
+
77
+ /**
78
+ * An agent eval in flight (`defineEval({ agent })`): the gateway `run` / `assert` was handed, and
79
+ * the doubles every tool call resolves to. PRIVATE — not exported from the package — because it is
80
+ * a measurement seam, never a way to swap an agent's tools in production: `agent({ tools })` stays
81
+ * the one declaration of what an agent may call.
82
+ */
83
+ export interface EvalScope {
84
+ readonly eval: string;
85
+ readonly gateway: Gateway;
86
+ readonly tools: Readonly<Record<string, ToolDouble>>;
87
+ /** Every doubled call the run made, in the order the model asked for them. */
88
+ readonly calls: { name: string; input: Readonly<Record<string, unknown>> }[];
89
+ }
90
+
91
+ const evalScopes = asyncContext<EvalScope>('an agent eval');
92
+
93
+ /** Run `fn` as one agent eval case: its model calls reach `scope.gateway`, its tools the doubles. */
94
+ export function withEvalScope<T>(scope: EvalScope, fn: () => T): T {
95
+ return evalScopes.run(scope, fn);
96
+ }
97
+
98
+ /** The agent eval in flight, or `undefined` everywhere else. */
99
+ export function currentEvalScope(): EvalScope | undefined {
100
+ return evalScopes.get();
101
+ }
102
+
103
+ /**
104
+ * The installed gateway. `prompt` names the caller so the miss says what was about to run.
105
+ * Inside an agent eval it is the eval's: the run measures the gateway the test handed over, never
106
+ * whatever an earlier test file left installed.
107
+ */
75
108
  export function aiGateway(prompt: string): Gateway {
109
+ const scoped = evalScopes.get()?.gateway;
110
+ if (scoped !== undefined) return scoped;
76
111
  if (runtime === undefined) throw new AiGatewayMissingError({ prompt });
77
112
  return runtime.gateway;
78
113
  }
79
114
 
80
115
  /** The installed gateway, or `undefined` — for a caller that may run without one (`hive()`). */
81
116
  export function installedGateway(): Gateway | undefined {
82
- return runtime?.gateway;
117
+ return evalScopes.get()?.gateway ?? runtime?.gateway;
83
118
  }
84
119
 
85
120
  export function aiEmbedder(): Embedder {
package/src/scorers.ts CHANGED
@@ -5,13 +5,36 @@
5
5
  // prompt is a versioned artifact and its hash is part of the scorer's name.
6
6
 
7
7
  import { assert, finiteCount, finiteOption } from '@ultimat3/core';
8
+ import type { AgentRunUsage } from './agent-usage';
8
9
  import type { Gateway } from './gateway';
9
10
  import type { Prompt } from './prompt';
10
11
 
12
+ /** One tool call an agent eval's run made, answered by the eval's double. */
13
+ export interface EvalToolCall {
14
+ readonly name: string;
15
+ readonly input: Readonly<Record<string, unknown>>;
16
+ }
17
+
18
+ /** What an agent eval's run did beside answering — handed to every scorer of the agent arm. */
19
+ export interface EvalRun {
20
+ readonly turns: number;
21
+ /** In the order the model asked for them. */
22
+ readonly toolCalls: readonly EvalToolCall[];
23
+ readonly usage: AgentRunUsage;
24
+ }
25
+
26
+ /** What a scorer reads. `run` is present on the agent arm (`defineEval({ agent })`) only. */
27
+ export interface ScorerInput {
28
+ /** The answer: the model's text, or on the agent arm the validated output as JSON. */
29
+ readonly output: string;
30
+ readonly expected?: string;
31
+ readonly run?: EvalRun;
32
+ }
33
+
11
34
  /** A scorer returns 0..1. Deterministic scorers are preferred; judges are a last resort. */
12
35
  export interface Scorer {
13
36
  readonly name: string;
14
- score(input: { output: string; expected?: string }): Promise<number> | number;
37
+ score(input: ScorerInput): Promise<number> | number;
15
38
  }
16
39
 
17
40
  /**
@@ -44,6 +67,17 @@ export const contains: Scorer = {
44
67
  expected !== undefined && output.toLowerCase().includes(expected.toLowerCase()) ? 1 : 0,
45
68
  };
46
69
 
70
+ /**
71
+ * 1 when the agent's run called `tool` at least once, else 0 — and 0 on the prompt arm, which has
72
+ * no run. Named per tool, so two of them on one eval are two columns rather than one overwritten.
73
+ */
74
+ export function calledTool(tool: string): Scorer {
75
+ return {
76
+ name: `called-tool:${tool}`,
77
+ score: ({ run }) => (run?.toolCalls.some((call) => call.name === tool) === true ? 1 : 0),
78
+ };
79
+ }
80
+
47
81
  /** Output parses as JSON. Pairs with a prompt that declares an `output` schema. */
48
82
  export const jsonValid: Scorer = {
49
83
  name: 'json-valid',
@@ -0,0 +1,146 @@
1
+ // `scriptedProvider()`: the test double that answers with turns written in advance — text, one
2
+ // tool call, several, or the `respond` answer — and records every request. Apart from
3
+ // ./echo-provider because echo cannot call a tool, and a loop under test has to be driven by one.
4
+
5
+ import { UltimateError } from '@ultimat3/core';
6
+ import { costOf, exactCostOf } from './cost';
7
+ import { resolveModel } from './model-resolve';
8
+ import type { ModelId } from './models';
9
+ import { modelIds } from './models';
10
+ import type {
11
+ GenerateRequest,
12
+ GenerateResult,
13
+ Provider,
14
+ StopReason,
15
+ StreamChunk,
16
+ TokenUsage,
17
+ } from './provider';
18
+ import { estimateTextTokens, estimateTokens } from './provider';
19
+ import { RESPOND } from './respond';
20
+ import type { LlmToolCall } from './tools';
21
+
22
+ /** One tool call a turn makes. `input` defaults to `{}`. */
23
+ export interface ScriptedCall {
24
+ readonly call: string;
25
+ readonly input?: Record<string, unknown>;
26
+ }
27
+
28
+ /**
29
+ * One model turn. `stop` overrides the stop reason the form implies (`end_turn` for text,
30
+ * `tool_use` otherwise) — `'max_tokens'` scripts a cut-off turn.
31
+ */
32
+ export type ScriptedTurn = (
33
+ | { readonly text: string }
34
+ | ScriptedCall
35
+ | { readonly calls: readonly ScriptedCall[] }
36
+ | { readonly answer: Record<string, unknown> }
37
+ ) & { readonly stop?: StopReason };
38
+
39
+ export interface ScriptedProviderInput {
40
+ /** Answered in order, one per provider call. One call past the end is `X_AI_SCRIPT_EXHAUSTED`. */
41
+ readonly turns: readonly ScriptedTurn[];
42
+ /** Every turn's usage. Absent: estimated from the request and the turn's text, as echo does. */
43
+ readonly usage?: TokenUsage;
44
+ /** What the gateway reports as `provider`, and keys its breaker by. Defaults to `scripted`. */
45
+ readonly name?: string;
46
+ }
47
+
48
+ export interface ScriptedProvider extends Provider {
49
+ /** Every request, in the order it arrived — a streamed one included. */
50
+ readonly seen: readonly GenerateRequest[];
51
+ }
52
+
53
+ /**
54
+ * More provider calls than scripted turns. Never answered by repeating the last turn: a loop that
55
+ * never calls `respond` would then spin to its ceiling on a script that looks like it passed.
56
+ */
57
+ export class AiScriptExhaustedError extends UltimateError {
58
+ constructor(input: { turns: number; model: string }) {
59
+ super({
60
+ code: 'X_AI_SCRIPT_EXHAUSTED',
61
+ cause:
62
+ `scriptedProvider holds ${input.turns} turn${input.turns === 1 ? '' : 's'} and was ` +
63
+ `asked for turn ${input.turns + 1} (model "${input.model}")`,
64
+ fix: 'scriptedProvider({ turns: [...turns, { answer: { /* the expected output */ } }] }) # one more turn, or fix the loop that keeps calling the model',
65
+ meta: { turns: input.turns },
66
+ });
67
+ }
68
+ }
69
+
70
+ /** The scripted test provider. Serves every registered model, read at call time. */
71
+ export function scriptedProvider(config: ScriptedProviderInput): ScriptedProvider {
72
+ const seen: GenerateRequest[] = [];
73
+ // `async`, so every refusal is a rejection — never a synchronous throw out of `generate()`.
74
+ const generate = async (request: GenerateRequest): Promise<GenerateResult> => {
75
+ // No model of its own, as echo: the gateway resolves before it routes.
76
+ const model = resolveModel('scripted-provider', request.model);
77
+ const turn = config.turns[seen.length];
78
+ seen.push(request);
79
+ if (turn === undefined) throw new AiScriptExhaustedError({ turns: config.turns.length, model });
80
+ return resultOf(model, request, turn, seen.length, config.usage);
81
+ };
82
+ return {
83
+ name: config.name ?? 'scripted',
84
+ // A getter over the whole registry, not a snapshot: a model registered after construction.
85
+ get models(): readonly ModelId[] {
86
+ return modelIds();
87
+ },
88
+ seen,
89
+ generate,
90
+ // Not a generator: the turn is taken when `stream()` is CALLED, in call order with `generate()`.
91
+ // A lazy one took it at the first read, so two calls in flight could swap their turns.
92
+ stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
93
+ const pending = generate(request);
94
+ // Read or not, a refusal is not an unhandled rejection; a reader still gets the throw.
95
+ pending.catch(() => undefined);
96
+ return chunksOf(pending);
97
+ },
98
+ };
99
+ }
100
+
101
+ async function* chunksOf(pending: Promise<GenerateResult>): AsyncIterable<StreamChunk> {
102
+ const result = await pending;
103
+ if (result.text !== '') yield { type: 'text', text: result.text };
104
+ for (const call of result.toolCalls) yield { type: 'tool-call', call };
105
+ yield { type: 'done', result };
106
+ }
107
+
108
+ function callsOf(turn: ScriptedTurn, n: number): readonly LlmToolCall[] {
109
+ if ('answer' in turn) return [{ id: `call-${n}`, name: RESPOND, input: turn.answer }];
110
+ if ('call' in turn) return [{ id: `call-${n}`, name: turn.call, input: turn.input ?? {} }];
111
+ if ('calls' in turn) {
112
+ return turn.calls.map((one, i) => ({
113
+ id: `call-${n}-${i + 1}`,
114
+ name: one.call,
115
+ input: one.input ?? {},
116
+ }));
117
+ }
118
+ return [];
119
+ }
120
+
121
+ function resultOf(
122
+ model: ModelId,
123
+ request: GenerateRequest,
124
+ turn: ScriptedTurn,
125
+ n: number,
126
+ fixed: TokenUsage | undefined,
127
+ ): GenerateResult {
128
+ const text = 'text' in turn ? turn.text : '';
129
+ const toolCalls = callsOf(turn, n);
130
+ const usage: TokenUsage = fixed ?? {
131
+ inputTokens: estimateTokens({ ...request, model }),
132
+ outputTokens: estimateTextTokens(text === '' ? JSON.stringify(toolCalls) : text),
133
+ cacheReadTokens: 0,
134
+ cacheWriteTokens: 0,
135
+ };
136
+ return {
137
+ model,
138
+ text,
139
+ toolCalls,
140
+ stopReason: turn.stop ?? (toolCalls.length > 0 ? 'tool_use' : 'end_turn'),
141
+ stopDetails: undefined,
142
+ usage,
143
+ cost: costOf(model, usage),
144
+ costExact: exactCostOf(model, usage),
145
+ };
146
+ }
@@ -0,0 +1,46 @@
1
+ // The way a projected tool reaches its `run`: the actor half of the policy, then the tool gate
2
+ // `mcpConfirmations` registered for that name, then the tool. Split from `tools.ts`, which adapts
3
+ // and renders. MCP's order, restated because `mcp` is this package's tier, and for the same reason:
4
+ // a caller the tool refuses outright opens no row for a person to look at.
5
+
6
+ import { toolGateFor } from '@ultimat3/action';
7
+ import type { Actor } from '@ultimat3/core';
8
+ import type { ProjectableAction } from './tools';
9
+
10
+ /**
11
+ * What already passes the gate, by its `run`: projecting one twice would ask twice and spend the
12
+ * approval on the outer pass. Keyed on the function because `{ ...projected, description }` is a
13
+ * new object carrying the same `run`, and a second gate on it would never let the tool run.
14
+ */
15
+ const throughGate = new WeakSet<ProjectableAction['run']>();
16
+
17
+ export const passesToolGate = (tool: ProjectableAction): boolean => throughGate.has(tool.run);
18
+
19
+ /**
20
+ * `tool`, with its `run` behind the gate. The gate is looked up per call, because the module declaring
21
+ * it may evaluate after the agent listing the tool. `admit` throws for a caller the tool's policy
22
+ * refuses before reading the input. An ungated tool skips it: `invoke` opens with the same question.
23
+ */
24
+ export function behindToolGate(
25
+ tool: ProjectableAction,
26
+ admit?: (actor: Actor) => void,
27
+ ): ProjectableAction {
28
+ const run: ProjectableAction['run'] = async (args) => {
29
+ const gate = toolGateFor(tool.name);
30
+ if (gate === undefined) return tool.run(args);
31
+ admit?.(args.actor);
32
+ return gate(
33
+ {
34
+ tool: tool.name,
35
+ args: args.input,
36
+ inputSchema: tool.inputJsonSchema,
37
+ actor: args.actor,
38
+ },
39
+ async () => tool.run(args),
40
+ );
41
+ };
42
+ throughGate.add(run);
43
+ // Inherits from `tool`, so what a class declares on its prototype still reads; the own fields are
44
+ // copied as well, so a spread of this copy keeps them and is not a nameless tool.
45
+ return Object.assign(Object.create(tool), tool, { run });
46
+ }
package/src/tools.ts CHANGED
@@ -19,13 +19,28 @@
19
19
  // boundary error. Both packages describe the same structural contract.
20
20
 
21
21
  import type { AnyAction } from '@ultimat3/action';
22
- import { actionName, invoke, isAction } from '@ultimat3/action';
22
+ import {
23
+ actionName,
24
+ admitActionCaller,
25
+ invoke,
26
+ isAction,
27
+ parseActionInput,
28
+ } from '@ultimat3/action';
23
29
  import type { Actor } from '@ultimat3/core';
24
- import { isMcpExposed, tryUseContext } from '@ultimat3/core';
30
+ import {
31
+ isMcpExposed,
32
+ isUltimateError,
33
+ stringField,
34
+ toolDescription,
35
+ tryUseContext,
36
+ } from '@ultimat3/core';
25
37
  import type { WireJsonSchema } from '@ultimat3/schema';
26
38
  import { toWireSchema } from '@ultimat3/schema';
39
+ import { EvalToolUndoubledError } from './eval-errors';
27
40
  import { discloseFailure, discloseOwnFailure } from './failure-disclosure';
41
+ import { currentEvalScope } from './runtime';
28
42
  import { takeToolIdempotencyKey, withToolIdempotencyKey } from './tool-idempotency';
43
+ import { behindToolGate, passesToolGate } from './tools-gate';
29
44
 
30
45
  /**
31
46
  * The JSON Schema subset the framework emits for tool arguments — `@ultimat3/schema`'s
@@ -77,6 +92,12 @@ export interface ProjectableAction {
77
92
  readonly inProcess?: boolean;
78
93
  /** In-process only: `false` withholds its refusals as a projected tool's are. Default `true`. */
79
94
  readonly discloseErrors?: boolean;
95
+ /**
96
+ * The input parse `run` makes before anything executes, alone: the parsed value, or the same
97
+ * refusal `run` would throw. What an agent eval holds its tool double to, so arguments production
98
+ * refuses are refused there too. Absent on a hand-built tool: there is no schema to hold it to.
99
+ */
100
+ readonly parseInput?: (raw: unknown) => Promise<unknown>;
80
101
  run(args: { input: unknown; actor: Actor }): Promise<unknown>;
81
102
  }
82
103
 
@@ -104,33 +125,51 @@ export type AgentTool = AnyAction | ProjectableAction;
104
125
  * projectable object it claims to be.
105
126
  */
106
127
  export function asProjectableAction(listed: AgentTool): ProjectableAction {
107
- if (!isAction(listed)) return listed;
108
- const mcp = listed.mcp;
128
+ if (!isAction(listed)) return passesToolGate(listed) ? listed : behindToolGate(listed);
109
129
  // Throws `X_ACTION_UNREGISTERED` on an unnamed action rather than offering a tool called `''`:
110
130
  // a nameless tool is unaddressable by the model, by `runLlmToolCall` and by the author.
111
131
  const name = actionName(listed);
132
+ const mcp = withDeclaredRefusals(listed.mcp, name, listed.errors);
112
133
  const wire = toWireSchema(listed.input);
113
134
  // `idempotent: true` is honoured here as over MCP and HTTP: the model's key reaches `invoke`,
114
135
  // which files it under the action, the caller and the key. Until 25.0.0 this adapter dropped it,
115
136
  // so an action an external agent could retry safely was one an in-app agent could not.
116
137
  const keyed = listed.describe().idempotent;
117
- return {
118
- name,
119
- ...(mcp === undefined ? {} : { mcp }),
120
- ...(mcp?.description === undefined ? {} : { description: mcp.description }),
121
- inputJsonSchema: keyed ? withToolIdempotencyKey(name, wire) : wire,
122
- // The actor rides in on the options and `invoke` swaps it inside the one execution path —
123
- // the action's own `policy` still decides, and its `input:` still parses what the model sent,
124
- // which is what drops a `{ actor: 'admin' }` the model invented before any handler sees it.
125
- // No `admit`: MCP's is its early gate ahead of ITS argument validation, and this surface has
126
- // none — `invoke` asks the same policy before it reads the input, which is the same order.
127
- run: keyed
128
- ? ({ input: args, actor }) => {
129
- const { input, idempotencyKey } = takeToolIdempotencyKey(args);
130
- return invoke(listed, input, { surface: 'mcp', actor, idempotencyKey });
131
- }
132
- : ({ input, actor }) => invoke(listed, input, { surface: 'mcp', actor }),
133
- };
138
+ return behindToolGate(
139
+ {
140
+ name,
141
+ ...(mcp === undefined ? {} : { mcp }),
142
+ ...(mcp?.description === undefined ? {} : { description: mcp.description }),
143
+ inputJsonSchema: keyed ? withToolIdempotencyKey(name, wire) : wire,
144
+ // `invoke`'s own parse and refusal (`parseActionInput`), after the reserved key is taken off.
145
+ parseInput: (raw) =>
146
+ parseActionInput(listed.input, keyed ? takeToolIdempotencyKey(raw).input : raw, name),
147
+ // The actor rides in on the options and `invoke` swaps it inside the one execution path —
148
+ // the action's own `policy` still decides, and its `input:` still parses what the model sent,
149
+ // which is what drops a `{ actor: 'admin' }` the model invented before any handler sees it.
150
+ run: keyed
151
+ ? ({ input: args, actor }) => {
152
+ const { input, idempotencyKey } = takeToolIdempotencyKey(args);
153
+ return invoke(listed, input, { surface: 'mcp', actor, idempotencyKey });
154
+ }
155
+ : ({ input, actor }) => invoke(listed, input, { surface: 'mcp', actor }),
156
+ },
157
+ admitActionCaller(listed, name),
158
+ );
159
+ }
160
+
161
+ /**
162
+ * A declared `errors:` ends the description — core's `toolDescription`, the one `@ultimat3/mcp`'s
163
+ * `exposureOf` calls (held equal by `scripts/ai-mcp-tool-parity.test.ts`). With none written,
164
+ * its default lead is spelled out before the list.
165
+ */
166
+ function withDeclaredRefusals<M extends { readonly description?: string }>(
167
+ mcp: M | undefined,
168
+ name: string,
169
+ errors: readonly string[] | undefined,
170
+ ): M | undefined {
171
+ if (mcp === undefined || errors === undefined || errors.length === 0) return mcp;
172
+ return { ...mcp, description: toolDescription({ name, mcp }, errors) };
134
173
  }
135
174
 
136
175
  /**
@@ -148,8 +187,7 @@ const EMPTY_SCHEMA: JsonSchema = { type: 'object', properties: {}, additionalPro
148
187
  export function toLlmTool(action: ProjectableAction): LlmTool {
149
188
  return {
150
189
  name: action.name,
151
- description:
152
- action.mcp?.description ?? action.description ?? `Run the "${action.name}" action.`,
190
+ description: toolDescription(action),
153
191
  input_schema: action.inputJsonSchema ?? EMPTY_SCHEMA,
154
192
  strict: true,
155
193
  };
@@ -188,12 +226,18 @@ export async function runLlmToolCall(
188
226
  /** `discloseErrors: false` withholds every in-process tool's refusal (`agent({ discloseToolErrors })`). */
189
227
  options: { readonly discloseErrors?: boolean } = {},
190
228
  ): Promise<LlmToolResult> {
191
- const action = actions.find((a) => a.name === call.name && isOfferedTool(a));
192
- if (action === undefined) {
229
+ const found = actions.find((a) => a.name === call.name && isOfferedTool(a));
230
+ if (found === undefined) {
193
231
  return { toolUseId: call.id, content: `unknown tool: ${call.name}`, isError: true };
194
232
  }
233
+ // This function is exported and documented, so a hand-built tool can arrive unprojected. Its one
234
+ // pass through the gate is decided here, where the model's call actually runs.
235
+ const action = asProjectableAction(found);
236
+ // Outside the `try`: an undoubled call inside an agent eval ends the run, never reaches the model
237
+ // as a tool result it could route around.
238
+ const run = evalDoubleFor(action, call);
195
239
  try {
196
- const output = await action.run({ input: call.input, actor });
240
+ const output = await run({ input: call.input, actor });
197
241
  return resultOf(call.id, output);
198
242
  } catch (error) {
199
243
  // A policy denial is an outcome the model should read and react to, not a crash.
@@ -202,6 +246,31 @@ export async function runLlmToolCall(
202
246
  }
203
247
  }
204
248
 
249
+ /**
250
+ * Inside an agent eval (`defineEval({ agent })`) a call resolves to the eval's double and the real
251
+ * tool never runs: an eval in CI cannot write. A tool with no double is `X_EVAL_TOOL_UNDOUBLED`.
252
+ * Everywhere else, the tool itself.
253
+ */
254
+ function evalDoubleFor(tool: ProjectableAction, call: LlmToolCall): ProjectableAction['run'] {
255
+ const scope = currentEvalScope();
256
+ if (scope === undefined) return (args) => tool.run(args);
257
+ const double = Object.hasOwn(scope.tools, call.name) ? scope.tools[call.name] : undefined;
258
+ if (double === undefined) {
259
+ throw new EvalToolUndoubledError({
260
+ eval: scope.eval,
261
+ tool: call.name,
262
+ doubled: Object.keys(scope.tools),
263
+ });
264
+ }
265
+ return async ({ input }) => {
266
+ // Inside the caller's `try`: a refused input is answered to the model exactly as production
267
+ // answers it, and the double — like the tool's handler — never sees it.
268
+ const parsed = tool.parseInput === undefined ? input : await tool.parseInput(input);
269
+ scope.calls.push({ name: call.name, input: call.input });
270
+ return double(parsed as Readonly<Record<string, unknown>>);
271
+ };
272
+ }
273
+
205
274
  /**
206
275
  * One tool's return value as the `tool_result` text. `content` is typed `string` and the agent
207
276
  * loop TRUNCATES it, so anything else ends the run in a `TypeError` two frames away — and the
@@ -266,5 +335,10 @@ function describeFailure(error: unknown, tool: ProjectableAction, disclose: bool
266
335
  });
267
336
  }
268
337
  const line = `${shown.code}: ${shown.cause ?? HIDDEN_TOOL_CAUSE}`;
269
- return shown.fix === undefined ? line : `${line} (fix: ${shown.fix})`;
338
+ // The model is a caller: an error we built that says what a caller does says it here, as an app's
339
+ // MCP server does (`errorAudience: 'caller'`). `x policy explain` is not a command a model can run,
340
+ // and a pending confirmation's developer fix tells it to call a decision action it is not offered.
341
+ const callerFix = isUltimateError(error) ? stringField(error, 'callerFix') : undefined;
342
+ const fix = callerFix ?? shown.fix;
343
+ return fix === undefined ? line : `${line} (fix: ${fix})`;
270
344
  }
package/src/wire.ts CHANGED
@@ -215,17 +215,25 @@ export class MessageStream {
215
215
  }
216
216
 
217
217
  private payloadOf(frame: SseFrame): Record<string, unknown> {
218
+ let parsed: unknown;
218
219
  try {
219
- const parsed: unknown = JSON.parse(frame.data);
220
- const record = asRecord(parsed);
221
- if (record === undefined) throw new SyntaxError('frame data is not an object');
222
- return record;
220
+ parsed = JSON.parse(frame.data);
223
221
  } catch (error) {
224
222
  throw new AiTransportError({
225
223
  provider: 'anthropic',
226
224
  detail: `unreadable "${frame.event}" frame: ${renderThrowable(error)}`,
227
225
  });
228
226
  }
227
+ const record = asRecord(parsed);
228
+ if (record === undefined) {
229
+ // The same coded refusal the catch above produces, thrown where the frame is known: a bare
230
+ // `SyntaxError` in shipped source carries no code, no cause and no fix.
231
+ throw new AiTransportError({
232
+ provider: 'anthropic',
233
+ detail: `unreadable "${frame.event}" frame: frame data is not an object`,
234
+ });
235
+ }
236
+ return record;
229
237
  }
230
238
 
231
239
  private onMessageStart(payload: Record<string, unknown>): readonly StreamChunk[] {