eve 0.89.0 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -9
- package/dist/src/channel/adapter.d.ts +12 -3
- package/dist/src/channel/adapter.js +1 -1
- package/dist/src/channel/auth/definitions.d.ts +7 -20
- package/dist/src/channel/auth/definitions.js +1 -1
- package/dist/src/channel/auth/http-basic.js +1 -1
- package/dist/src/channel/auth/jwt-ecdsa.js +1 -1
- package/dist/src/channel/auth/jwt-hmac.js +1 -1
- package/dist/src/channel/auth/oidc.js +1 -1
- package/dist/src/channel/auth/result.d.ts +2 -3
- package/dist/src/channel/auth/result.js +1 -1
- package/dist/src/channel/channel-address.d.ts +1 -1
- package/dist/src/channel/channel-address.js +1 -1
- package/dist/src/channel/channel-operations.d.ts +1 -1
- package/dist/src/channel/schedule.js +1 -1
- package/dist/src/channel/types.d.ts +2 -0
- package/dist/src/chunks/{use-eve-agent-Bbl4H5aA.js → use-eve-agent-Bwloorf8.js} +77 -63
- package/dist/src/chunks/{use-eve-agent-CNpxgKEH.js → use-eve-agent-XCB0-tLX.js} +77 -63
- package/dist/src/cli/dev/environment.js +1 -1
- package/dist/src/cli/dev/tui/prompt-commands.d.ts +1 -1
- package/dist/src/cli/dev/tui/remote-connection-probe.js +1 -1
- package/dist/src/cli/dev/tui/remote-connection-types.d.ts +1 -0
- package/dist/src/cli/dev/tui/runner.js +2 -2
- package/dist/src/cli/dev/tui/status-line.d.ts +2 -2
- package/dist/src/cli/dev/tui/status-line.js +1 -1
- package/dist/src/client/agent-info-schema.d.ts +341 -326
- package/dist/src/client/agent-info-schema.js +1 -1
- package/dist/src/client/agent-info-types.d.ts +13 -10
- package/dist/src/client/client-error.d.ts +1 -1
- package/dist/src/client/client.d.ts +5 -3
- package/dist/src/client/client.js +1 -1
- package/dist/src/client/session.d.ts +1 -1
- package/dist/src/compiled/.vendor-stamp.json +1 -1
- package/dist/src/evals/target.js +1 -1
- package/dist/src/eve-channel/auth.d.ts +19 -0
- package/dist/src/eve-channel/auth.js +1 -0
- package/dist/src/eve-channel/index.d.ts +1 -0
- package/dist/src/eve-channel/index.js +1 -1
- package/dist/src/eve-channel/types.d.ts +23 -4
- package/dist/src/execution/catalog/connection-entry.js +1 -1
- package/dist/src/execution/coordination-dispatch-shared.js +1 -1
- package/dist/src/execution/dynamic-workflow/tool.js +1 -1
- package/dist/src/execution/proxied-deliver-step.js +1 -1
- package/dist/src/execution/sandbox/bash.js +2 -2
- package/dist/src/execution/session/active-turn.js +1 -1
- package/dist/src/execution/session/admission.d.ts +6 -0
- package/dist/src/execution/session/admission.js +1 -1
- package/dist/src/execution/session/delivery-facts.js +1 -1
- package/dist/src/execution/session/next-input.d.ts +2 -2
- package/dist/src/execution/session/next-input.js +1 -1
- package/dist/src/execution/session/program.js +1 -1
- package/dist/src/execution/session-inbox/inbox.d.ts +2 -2
- package/dist/src/execution/session-inbox/inbox.js +1 -1
- package/dist/src/execution/session-inbox/protocol.js +1 -1
- package/dist/src/execution/tools/agent-router-workflow.js +1 -1
- package/dist/src/harness/compaction/engine.js +2 -2
- package/dist/src/harness/current-messages.js +1 -1
- package/dist/src/harness/emission.d.ts +3 -0
- package/dist/src/harness/emission.js +1 -1
- package/dist/src/harness/facts.d.ts +5 -2
- package/dist/src/harness/facts.js +1 -1
- package/dist/src/harness/model-call/call.d.ts +7 -1
- package/dist/src/harness/model-call/call.js +3 -1
- package/dist/src/harness/provider-tool-history.d.ts +8 -0
- package/dist/src/harness/provider-tool-history.js +1 -1
- package/dist/src/harness/session-machine/transitions.d.ts +3 -2
- package/dist/src/harness/step/after-model.js +1 -1
- package/dist/src/harness/step/context.js +1 -1
- package/dist/src/harness/step/intake.js +1 -1
- package/dist/src/harness/step/prompt.js +1 -1
- package/dist/src/harness/tool-history-repair.d.ts +2 -0
- package/dist/src/harness/tool-history-repair.js +1 -1
- package/dist/src/harness/tool-loop.js +1 -1
- package/dist/src/internal/application/stamped-package-version.js +1 -1
- package/dist/src/internal/authored-module-graph.js +1 -1
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response.d.ts +2 -2
- package/dist/src/protocol/routes.d.ts +7 -1
- package/dist/src/protocol/routes.js +1 -1
- package/dist/src/protocol/session-events/families/delivery.d.ts +5 -3
- package/dist/src/protocol/session-events/families/delivery.js +1 -1
- package/dist/src/protocol/session-projection/fold.js +1 -1
- package/dist/src/public/channels/discord/hitl.js +1 -1
- package/dist/src/public/channels/github/githubChannel.js +1 -1
- package/dist/src/public/channels/github/state.d.ts +7 -0
- package/dist/src/public/channels/github/state.js +1 -1
- package/dist/src/public/channels/mcp.js +1 -1
- package/dist/src/public/channels/slack/hitl.d.ts +2 -1
- package/dist/src/public/channels/slack/hitl.js +3 -3
- package/dist/src/public/channels/slack/interaction-cards.js +1 -1
- package/dist/src/public/channels/slack/request-card-render.js +1 -1
- package/dist/src/public/channels/teams/hitl.js +1 -1
- package/dist/src/public/channels/teams/verify.js +1 -1
- package/dist/src/public/channels/telegram/telegramChannel.js +1 -1
- package/dist/src/public/connections/index.d.ts +1 -1
- package/dist/src/public/definitions/channel.js +1 -1
- package/dist/src/public/definitions/connections/tool-call.d.ts +2 -2
- package/dist/src/public/definitions/hook.d.ts +90 -7
- package/dist/src/public/definitions/hook.js +1 -1
- package/dist/src/public/hooks/index.d.ts +3 -2
- package/dist/src/reactions/continuation.d.ts +19 -0
- package/dist/src/reactions/continuation.js +1 -0
- package/dist/src/reactions/frame.js +1 -1
- package/dist/src/reactions/intents.d.ts +10 -4
- package/dist/src/reactions/intents.js +1 -1
- package/dist/src/reactions/kinds/hook.js +1 -1
- package/dist/src/reactions/runner.js +1 -1
- package/dist/src/reactions/state.d.ts +15 -2
- package/dist/src/reactions/state.js +1 -1
- package/dist/src/runtime/channels/registry.d.ts +2 -2
- package/dist/src/runtime/channels/registry.js +1 -1
- package/dist/src/runtime/connections/approval-scope.d.ts +9 -0
- package/dist/src/runtime/connections/approval-scope.js +1 -0
- package/dist/src/runtime/connections/mcp-client.d.ts +1 -1
- package/dist/src/runtime/connections/mcp-client.js +1 -1
- package/dist/src/runtime/connections/provided-arguments.js +1 -1
- package/dist/src/runtime/resolve-connection.js +1 -1
- package/dist/src/runtime/subagents/workflow.js +1 -1
- package/dist/src/setup/primitives/open-url.js +1 -1
- package/dist/src/setup/scaffold/create/project.js +1 -1
- package/dist/src/shared/agent-info-access.d.ts +5 -0
- package/dist/src/shared/agent-info-access.js +1 -0
- package/dist/src/shared/connection-types.d.ts +21 -4
- package/dist/src/shared/value-delta.d.ts +5 -4
- package/dist/src/shared/value-delta.js +1 -1
- package/dist/src/svelte/index.js +1 -1
- package/dist/src/svelte/use-eve-agent.js +1 -1
- package/dist/src/tools/provided/ask-question-workflow.js +1 -1
- package/dist/src/tools/provided/sleep-workflow.js +1 -1
- package/dist/src/tools/provided/web-fetch.js +1 -1
- package/dist/src/vue/index.js +1 -1
- package/dist/src/vue/use-eve-agent.js +1 -1
- package/docs/agent-config.md +199 -222
- package/docs/channels/chat-sdk.mdx +60 -64
- package/docs/channels/custom.mdx +265 -280
- package/docs/channels/discord.mdx +176 -123
- package/docs/channels/eve.mdx +136 -204
- package/docs/channels/github.mdx +24 -22
- package/docs/channels/linear.mdx +30 -30
- package/docs/channels/linq.mdx +25 -13
- package/docs/channels/mcp.mdx +154 -116
- package/docs/channels/overview.mdx +31 -32
- package/docs/channels/photon.mdx +32 -56
- package/docs/channels/slack.mdx +538 -375
- package/docs/channels/teams.mdx +40 -26
- package/docs/channels/telegram.mdx +13 -13
- package/docs/channels/twilio.mdx +11 -11
- package/docs/code-extension.mdx +40 -42
- package/docs/concepts/built-in-tools.md +89 -57
- package/docs/concepts/context-control.md +20 -20
- package/docs/concepts/default-harness.md +20 -36
- package/docs/concepts/execution-model-and-durability.mdx +96 -160
- package/docs/concepts/project-structure.mdx +21 -14
- package/docs/concepts/security-model.md +27 -27
- package/docs/concepts/sessions-runs-and-streaming.md +184 -194
- package/docs/concepts/state.md +7 -15
- package/docs/connections/mcp.mdx +109 -93
- package/docs/connections/openapi.mdx +57 -59
- package/docs/connections/overview.mdx +205 -170
- package/docs/evals/assertions.mdx +36 -29
- package/docs/evals/cases.mdx +38 -42
- package/docs/evals/judge.mdx +21 -29
- package/docs/evals/overview.mdx +42 -38
- package/docs/evals/reporters.mdx +54 -27
- package/docs/evals/running.mdx +15 -8
- package/docs/evals/targets.mdx +17 -18
- package/docs/evals/tool-stubs.mdx +111 -97
- package/docs/experimental/dynamic-schedules.mdx +118 -63
- package/docs/extensions.md +58 -65
- package/docs/getting-started.mdx +4 -6
- package/docs/guides/auth-and-route-protection.md +100 -70
- package/docs/guides/client/continuations.mdx +23 -26
- package/docs/guides/client/messages.mdx +47 -66
- package/docs/guides/client/output-schema.mdx +8 -8
- package/docs/guides/client/overview.mdx +47 -41
- package/docs/guides/client/streaming.mdx +62 -71
- package/docs/guides/decide.md +24 -79
- package/docs/guides/deployment/meta.json +1 -1
- package/docs/guides/deployment/overview.md +27 -18
- package/docs/guides/deployment/self-hosted-operations.md +99 -0
- package/docs/guides/deployment/self-hosting.md +63 -80
- package/docs/guides/deployment/vercel.mdx +31 -27
- package/docs/guides/deployment/workflow-worlds.md +100 -0
- package/docs/guides/dev-tui.md +48 -41
- package/docs/guides/dynamic-capabilities.md +49 -50
- package/docs/guides/frontend/nextjs.mdx +2 -2
- package/docs/guides/frontend/nuxt.mdx +11 -11
- package/docs/guides/frontend/overview.mdx +1 -1
- package/docs/guides/frontend/sveltekit.mdx +11 -11
- package/docs/guides/frontend/tanstack.mdx +16 -16
- package/docs/guides/frontend/use-eve-agent-svelte.mdx +21 -21
- package/docs/guides/frontend/use-eve-agent-vue.mdx +22 -22
- package/docs/guides/frontend/use-eve-agent.mdx +25 -25
- package/docs/guides/hooks.md +152 -106
- package/docs/guides/instrumentation/agent-runs.mdx +26 -30
- package/docs/guides/instrumentation/instrumentation.mdx +49 -76
- package/docs/guides/instrumentation/meta.json +1 -1
- package/docs/guides/instrumentation/otel.mdx +113 -179
- package/docs/guides/prepare-for-1-0.md +23 -1
- package/docs/guides/remote-agents.md +125 -187
- package/docs/guides/self-modification.md +18 -18
- package/docs/guides/session-context.md +16 -16
- package/docs/install-integrations.mdx +26 -24
- package/docs/instructions.mdx +37 -42
- package/docs/memory/custom-provider.md +35 -41
- package/docs/memory/file.md +93 -76
- package/docs/memory/overview.mdx +145 -156
- package/docs/patterns/durable-cross-channel-notifications.md +15 -19
- package/docs/patterns/dynamic-scheduling.md +19 -23
- package/docs/patterns/multi-tenant-approvals.md +17 -22
- package/docs/patterns/multi-tenant-auth.md +43 -42
- package/docs/patterns/multi-tenant-memory.md +39 -36
- package/docs/patterns/voice-notes.md +3 -3
- package/docs/protocols/acp.md +13 -13
- package/docs/protocols/ucp.mdx +7 -7
- package/docs/reference/agent-files.md +6 -6
- package/docs/reference/cli.md +99 -98
- package/docs/reference/telemetry.md +1 -1
- package/docs/reference/typescript-api.md +52 -53
- package/docs/responsible-use.md +2 -2
- package/docs/sandbox/custom-providers.mdx +60 -56
- package/docs/sandbox/default.mdx +23 -24
- package/docs/sandbox/docker.mdx +16 -10
- package/docs/sandbox/index.mdx +93 -61
- package/docs/sandbox/just-bash.mdx +17 -11
- package/docs/sandbox/troubleshooting.mdx +29 -23
- package/docs/sandbox/vercel.mdx +45 -52
- package/docs/schedules.mdx +65 -54
- package/docs/skills.mdx +74 -49
- package/docs/subagents/index.mdx +97 -136
- package/docs/tools/human-in-the-loop.md +160 -90
- package/docs/tools/meta.json +1 -1
- package/docs/tools/overview.mdx +229 -237
- package/docs/tools/tasks.md +118 -117
- package/docs/tools/workflows.mdx +447 -543
- package/docs/tutorial/connect-a-warehouse.mdx +11 -13
- package/docs/tutorial/first-agent.mdx +14 -16
- package/docs/tutorial/guard-the-spend.mdx +6 -8
- package/docs/tutorial/how-it-runs.mdx +8 -8
- package/docs/tutorial/query-sample-data.mdx +8 -8
- package/docs/tutorial/remember-definitions.mdx +4 -4
- package/docs/tutorial/run-analysis.mdx +20 -22
- package/docs/tutorial/ship-it.mdx +19 -19
- package/docs/tutorial/team-playbooks.mdx +9 -9
- package/docs/use-cases.mdx +13 -11
- package/package.json +1 -1
- package/docs/guides/instrumentation/migration.md +0 -96
- package/docs/tools/tasks-upgrade.md +0 -218
|
@@ -3,9 +3,9 @@ title: "Tool Stubs"
|
|
|
3
3
|
description: "Define tool outcomes and outcome sequences in your evals."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
A tool stub makes a tool return fixed JSON, or throw an error, for one eval session. Pass `stubs` to `t.session()`. Stubs work against local and deployed agents, with no mock code to deploy.
|
|
7
7
|
|
|
8
|
-
Stubs
|
|
8
|
+
Stubs set what tools return; [assertions](./assertions) check what the agent does with it. The target must [permit tool replacement](#permit-tool-replacement) for the eval's caller.
|
|
9
9
|
|
|
10
10
|
## Author an eval
|
|
11
11
|
|
|
@@ -65,21 +65,50 @@ export default defineEval({
|
|
|
65
65
|
|
|
66
66
|
Use your agent's actual tool names and response shapes. The client accepts the same option on `client.sessions.create({ stubs })` or `client.sessions.create({ message, stubs })`.
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
Responses are a fixed sequence, not a simulated database: the second `list_tasks` call returns two tasks even if `complete_task` never ran. The assertions check only that the agent asked for the write. To test a real update, use the real tool.
|
|
69
69
|
|
|
70
|
-
##
|
|
70
|
+
## Permit tool replacement
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
72
|
+
eve rejects stubs by default, including in development. With [Vercel OIDC](../guides/auth-and-route-protection#verceloidc), use `eveVercelOidc` and grant the permission to the eval runner's verified subject:
|
|
73
|
+
|
|
74
|
+
```ts title="agent/channels/eve.ts"
|
|
75
|
+
import { vercelSubject } from "eve/channels/auth";
|
|
76
|
+
import { eveChannel, eveVercelOidc } from "eve/channels/eve";
|
|
77
|
+
|
|
78
|
+
export default eveChannel({
|
|
79
|
+
auth: [
|
|
80
|
+
eveVercelOidc({
|
|
81
|
+
subjects: [
|
|
82
|
+
vercelSubject({ teamSlug: "acme", projectName: "app" }),
|
|
83
|
+
{
|
|
84
|
+
subject: vercelSubject({ teamSlug: "acme", projectName: "eval-runner" }),
|
|
85
|
+
allowToolStubs: true,
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
}),
|
|
89
|
+
],
|
|
90
|
+
});
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Both projects can authenticate, but only the eval runner can replace tools. `vercelSubject` matches production only; pass `environment: "*"` to include all environments. Only the target's server configuration grants the permission, after token verification. A request can't grant it, and users don't inherit a project's grant.
|
|
94
|
+
|
|
95
|
+
Custom authenticators can return the same `allowToolStubs: true` after verifying the caller. See [authentication and route protection](../guides/auth-and-route-protection#tool-replacement-permission) for the grant rules and a local development example.
|
|
96
|
+
|
|
97
|
+
eve checks the permission only when it creates the session. Later requests use normal channel authentication, so anyone your channel admits to the session can use its stubs. Isolate sessions with your session-access policy, or use a separate eval deployment of the same agent code.
|
|
98
|
+
|
|
99
|
+
Checking the stubs after the eval also requires `allowToolStubs`.
|
|
100
|
+
|
|
101
|
+
## Stub fields
|
|
79
102
|
|
|
80
|
-
|
|
103
|
+
| Field | Meaning |
|
|
104
|
+
| ---------- | --------------------------------------------------------------------------------------------------------------- |
|
|
105
|
+
| `id` | A rule name, unique in the session, that tracks the rule's place in its sequence. Traces don't show it. |
|
|
106
|
+
| `tool` | The existing tool's name, optionally prefixed with a local subagent path. |
|
|
107
|
+
| `match` | Optional constraints on input properties, written as JSON Schema. Omit it or use `{}` to match any valid input. |
|
|
108
|
+
| `outcome` | One outcome returned on every matching call, such as `{ response: { tasks: [] } }`. |
|
|
109
|
+
| `outcomes` | A nonempty array of outcomes returned in order. Use exactly one of `outcome` or `outcomes`. |
|
|
81
110
|
|
|
82
|
-
|
|
111
|
+
Each outcome contains exactly one of `response`, the JSON the tool returns, or `throw`, an error [raised instead](#inject-tool-errors). The tool returns only the `response` value, not the outcome object.
|
|
83
112
|
|
|
84
113
|
Stubs are plain JSON and can be shared between evals:
|
|
85
114
|
|
|
@@ -91,11 +120,13 @@ export const taskStubs = [
|
|
|
91
120
|
] satisfies readonly ToolStub[];
|
|
92
121
|
```
|
|
93
122
|
|
|
94
|
-
Pass the fixture to `t.session({ stubs: taskStubs })
|
|
123
|
+
Pass the fixture to `t.session({ stubs: taskStubs })`; each session keeps its own place in each sequence. Both `eve/evals` and `eve/client` export `ToolStub` and `ToolStubOutcome`. These types check the JSON shape but don't infer the response type from your tool definition.
|
|
124
|
+
|
|
125
|
+
Configuration is limited to 100 rules, 20,000 JSON nodes, nesting depth 32, and about one million characters. The `__proto__` property name is reserved.
|
|
95
126
|
|
|
96
127
|
## Match tool arguments
|
|
97
128
|
|
|
98
|
-
**
|
|
129
|
+
**eve checks rules from top to bottom. The first matching stub wins.** Put specific rules before an unconditional fallback:
|
|
99
130
|
|
|
100
131
|
```ts
|
|
101
132
|
const stubs = [
|
|
@@ -109,9 +140,7 @@ const stubs = [
|
|
|
109
140
|
];
|
|
110
141
|
```
|
|
111
142
|
|
|
112
|
-
A call for milk receives success; other valid calls receive the fallback.
|
|
113
|
-
|
|
114
|
-
If eve cannot record an output-processing failure, it cancels the workflow serving stubs. The original tool error or fallback is preserved, but verification rejects the eval. Failed or cancelled stub sessions cannot pass verification.
|
|
143
|
+
A call for milk receives success; other valid calls receive the fallback. Reversed, the unconditional rule wins every time. If no rule matches, eve runs the real tool.
|
|
115
144
|
|
|
116
145
|
Every property listed in `match` must exist and satisfy its constraint. Extra top-level properties are allowed:
|
|
117
146
|
|
|
@@ -132,70 +161,84 @@ Every property listed in `match` must exist and satisfy its constraint. Extra to
|
|
|
132
161
|
}
|
|
133
162
|
```
|
|
134
163
|
|
|
135
|
-
Nested properties are optional unless listed in `required`. Matching
|
|
164
|
+
Nested properties are optional unless listed in `required`. Matching doesn't coerce values, apply defaults, or change arguments. The real tool's input schema still validates the input first, and normal approvals and output processing still apply.
|
|
136
165
|
|
|
137
|
-
Constraints use a
|
|
166
|
+
Constraints use a subset of JSON Schema 2020-12 with no references:
|
|
138
167
|
|
|
139
|
-
|
|
168
|
+
- Types, and scalar `const` and `enum`
|
|
169
|
+
- String patterns and bounds, and numeric bounds
|
|
170
|
+
- Array `items` and `contains`
|
|
171
|
+
- Object `properties`, `required`, and `additionalProperties`
|
|
172
|
+
- Logical and conditional schemas
|
|
140
173
|
|
|
141
|
-
|
|
174
|
+
`pattern` uses JavaScript regular expressions. eve validates schemas and patterns before it creates the session.
|
|
142
175
|
|
|
143
|
-
|
|
176
|
+
Unsupported constraints fail setup: `patternProperties`, references, formats, `multipleOf`, `minContains`, `maxContains`, `uniqueItems`, dependencies, and object/array values in `const` or `enum`. Nested `properties` and `required` can't use inherited property names such as `constructor` or `toString`. Use nested properties or array items to match structured values.
|
|
144
177
|
|
|
145
|
-
|
|
178
|
+
Unknown static tool names, invalid local subagent paths, and paths into remote agents fail session creation. eve checks dynamic tool names and connection operations where it can and allows the rest. After the eval, eve warns about rules that matched no calls but doesn't fail the eval. Use tool assertions when a call must happen.
|
|
146
179
|
|
|
147
|
-
|
|
180
|
+
A broken stub, or a failure while processing its response, fails the eval even if the agent recovers. An intentional `throw` outcome is a normal tool failure the eval can assert on. Neither falls back to the real tool. If eve can't record such a failure, it cancels the session's run. A failed or cancelled stubbed session always fails the eval.
|
|
148
181
|
|
|
149
|
-
|
|
182
|
+
## Sequences and session scope
|
|
150
183
|
|
|
151
|
-
|
|
184
|
+
`outcome` repeats on every matching call. `outcomes` moves to the next entry on each matching call, then repeats its last entry. For example, `outcomes: [{ response: "pending" }, { response: "completed" }]` returns `"pending"` once and `"completed"` after that. Only the selected rule moves forward; running out of entries never selects a later rule.
|
|
152
185
|
|
|
153
|
-
|
|
186
|
+
Two calls in one turn use two entries, and later turns continue the same sequence. Calls waiting for approval, or denied, don't use an entry. A replayed workflow step returns its recorded outcome without using another entry. Each session supports up to 10,000 tool calls.
|
|
154
187
|
|
|
155
|
-
|
|
188
|
+
Rules are fixed when the session is created. Local subagents use them through qualified tool paths, and sequence positions survive a handoff to a new deployment. A separate session starts each sequence over.
|
|
156
189
|
|
|
157
|
-
|
|
190
|
+
Parallel calls use responses in the order they arrive, which can vary. Use separate argument matchers when particular arguments must receive particular responses.
|
|
158
191
|
|
|
159
|
-
|
|
192
|
+
## Inject tool errors
|
|
160
193
|
|
|
161
|
-
|
|
162
|
-
import { vercelOidc, vercelSubject } from "eve/channels/auth";
|
|
163
|
-
import { eveChannel } from "eve/channels/eve";
|
|
194
|
+
Use `throw` to test how the agent handles a failed tool call. Mix errors and responses to test recovery:
|
|
164
195
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
],
|
|
177
|
-
});
|
|
196
|
+
```ts
|
|
197
|
+
const stubs = [
|
|
198
|
+
{
|
|
199
|
+
id: "tasks",
|
|
200
|
+
tool: "list_tasks",
|
|
201
|
+
outcomes: [
|
|
202
|
+
{ throw: { name: "TimeoutError", message: "Task service timed out" } },
|
|
203
|
+
{ response: { tasks: [] } },
|
|
204
|
+
],
|
|
205
|
+
},
|
|
206
|
+
];
|
|
178
207
|
```
|
|
179
208
|
|
|
180
|
-
|
|
209
|
+
`throw.message` is required. Omit `throw.name` to use `"Error"`. eve throws a plain JavaScript `Error` with that name and message. A `TimeoutError` fails the call right away instead of waiting for a deadline.
|
|
181
210
|
|
|
182
|
-
|
|
211
|
+
Ordinary tools and connection operations show the model the message without the name; workflow task errors keep the name. Put details the model needs in `message`.
|
|
183
212
|
|
|
184
|
-
|
|
213
|
+
Use `outcome: { throw: { message: "Service unavailable" } }` to fail every matching call. In a sequence, a throwing entry is used up like a response.
|
|
185
214
|
|
|
186
|
-
|
|
215
|
+
An injected error follows the tool's normal error handling. Ordinary and blocking workflow calls are recorded as failed; background tasks report failure when they finish. The model can report the failure or retry. Assert the failed call explicitly:
|
|
216
|
+
|
|
217
|
+
```ts
|
|
218
|
+
const turn = await session.send(
|
|
219
|
+
"List my tasks once. If the service fails, explain the failure and wait for me to retry.",
|
|
220
|
+
);
|
|
221
|
+
turn.expectOk();
|
|
222
|
+
turn.calledTool("list_tasks", { status: "failed", count: 1 });
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
`noFailedCalls()` fails on the injected error. Stub setup and response-processing failures still fail the eval.
|
|
226
|
+
|
|
227
|
+
`throw` works for ordinary tools, workflow tools, whole-subagent replacements, and qualified connection operations. An error in a persistent workflow tool ends that task, so retry with a new call instead of the failed task ID. Normal approvals apply before eve uses an outcome.
|
|
228
|
+
|
|
229
|
+
`{ response: { error: "timeout" } }` is ordinary JSON; only `throw` raises an error.
|
|
187
230
|
|
|
188
231
|
## Supported tools and subagents
|
|
189
232
|
|
|
190
|
-
Stubs replace execution of existing ordinary, dynamic, and workflow tools.
|
|
233
|
+
Stubs replace the execution of existing ordinary, dynamic, and workflow tools. The real tool's input schema, approvals, and output processing still apply. A stub doesn't add a tool or show one to the model.
|
|
191
234
|
|
|
192
|
-
|
|
235
|
+
An ordinary tool with stub rules can emit an extra `call.progress` event containing the final response, even when no rule matches. Assert final results on `call.settled`, which stays correct.
|
|
193
236
|
|
|
194
237
|
### Subagents
|
|
195
238
|
|
|
196
|
-
Stubbing a
|
|
239
|
+
Stubbing a subagent's tool returns the configured result without starting the subagent. Persistent tools implemented with `serve`, including the built-in `agent`, require an unconditional stub: omit `match` or use `{}`. You can't mix real and stubbed calls to the same persistent tool.
|
|
197
240
|
|
|
198
|
-
To let a local subagent run but replace its tools,
|
|
241
|
+
To let a local subagent run but replace its tools, prefix the tool name with the subagent path and `/`:
|
|
199
242
|
|
|
200
243
|
```ts
|
|
201
244
|
const stubs = [
|
|
@@ -205,55 +248,26 @@ const stubs = [
|
|
|
205
248
|
];
|
|
206
249
|
```
|
|
207
250
|
|
|
208
|
-
`researcher` replaces the whole
|
|
251
|
+
`researcher` replaces the whole subagent; `researcher/list_tasks` replaces only that subagent's list tool. The built-in `agent` uses `agent/list_tasks`. The model still sees the usual tool names. A root rule never matches a subagent's tool with the same name.
|
|
209
252
|
|
|
210
|
-
Repeated subagent
|
|
253
|
+
Repeated sessions of a subagent at the same path share that rule's sequence. Remote agents don't receive stub rules, so replace the whole remote agent tool.
|
|
211
254
|
|
|
212
255
|
### Connections and provider tools
|
|
213
256
|
|
|
214
257
|
Use the qualified operation name, such as `linear__list_issues` or `researcher/linear__list_issues`. Put operation arguments in `match` and its JSON result in `outcome.response`, or in each sequence entry's `response`.
|
|
215
258
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
A rule names the tool itself, whether the model calls it directly or through [`eve__tool`](/docs/concepts/built-in-tools#eve__search-eve__tool-and-eve__skill), as it does for deferred tools and connection operations. A call through `eve__tool` matches by the name and `input` of the tool it reaches. `eve__search`, `eve__tool`, and `eve__skill` cannot themselves be stubbed.
|
|
219
|
-
|
|
220
|
-
Tools executed by the model provider, such as provider-hosted web search, have no eve executor to replace. A search tool implemented in your own agent can be stubbed.
|
|
221
|
-
|
|
222
|
-
## Inject tool errors
|
|
223
|
-
|
|
224
|
-
Use `throw` to test how the agent handles a failed tool call. Mix errors and responses to test recovery:
|
|
259
|
+
A stubbed connection must still be reachable, because eve lists the connection's tools to validate the operation and its arguments.
|
|
225
260
|
|
|
226
|
-
|
|
227
|
-
const stubs = [
|
|
228
|
-
{
|
|
229
|
-
id: "tasks",
|
|
230
|
-
tool: "list_tasks",
|
|
231
|
-
outcomes: [
|
|
232
|
-
{ throw: { name: "TimeoutError", message: "Task service timed out" } },
|
|
233
|
-
{ response: { tasks: [] } },
|
|
234
|
-
],
|
|
235
|
-
},
|
|
236
|
-
];
|
|
237
|
-
```
|
|
238
|
-
|
|
239
|
-
`throw.message` is required. `throw.name` is optional and defaults to `"Error"`. eve creates an ordinary JavaScript `Error` with that name and message; it does not instantiate custom error classes. `TimeoutError` reports a timeout immediately, without delaying the call or testing an actual deadline.
|
|
240
|
-
|
|
241
|
-
Ordinary tools and connection operations expose the error message to the model, without the error name. Workflow task errors also retain the name. Put any details the model needs to act on in `message`.
|
|
242
|
-
|
|
243
|
-
Use `outcome: { throw: { message: "Service unavailable" } }` to fail every matching call. A sequence consumes one entry per new call, whether it returns or throws, and repeats its final entry. Workflow replay reuses the same recorded outcome.
|
|
244
|
-
|
|
245
|
-
An injected exception follows the tool's normal error handling. Ordinary and blocking workflow calls are recorded as failed; background tasks report failure when they settle. The model can report the failure or try again. Assert the failed call explicitly:
|
|
246
|
-
|
|
247
|
-
```ts
|
|
248
|
-
const turn = await session.send(
|
|
249
|
-
"List my tasks once. If the service fails, explain the failure and wait for me to retry.",
|
|
250
|
-
);
|
|
251
|
-
turn.expectOk();
|
|
252
|
-
turn.calledTool("list_tasks", { status: "failed", count: 1 });
|
|
253
|
-
```
|
|
261
|
+
A rule names the tool itself, whether the model calls it directly or through [`eve__tool`](/docs/concepts/built-in-tools#eve__search-eve__tool-and-eve__skill). The model uses `eve__tool` for deferred tools and connection operations. A call through `eve__tool` matches by the name and `input` of the tool it reaches. `eve__search`, `eve__tool`, and `eve__skill` can't be stubbed.
|
|
254
262
|
|
|
255
|
-
|
|
263
|
+
Tools the model provider runs, such as provider-hosted web search, can't be stubbed. A search tool in your own agent can.
|
|
256
264
|
|
|
257
|
-
|
|
265
|
+
## Troubleshooting
|
|
258
266
|
|
|
259
|
-
|
|
267
|
+
| What you see | What to do |
|
|
268
|
+
| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
|
269
|
+
| `403` with `Tool stubbing is not permitted for this caller.` | Grant `allowToolStubs: true` to the eval runner's subject; see [Permit tool replacement](#permit-tool-replacement). |
|
|
270
|
+
| `Tool stub "<id>" names an unknown or unsupported tool path "<tool>".` | Fix the tool name or subagent path. For a remote agent, stub the whole remote agent tool. |
|
|
271
|
+
| `Persistent tool "<tool>" requires an unconditional stub.` | Remove `match` from every rule for that tool, or set it to `{}`. |
|
|
272
|
+
| Warning `Tool stubs did not match any calls: <ids>.` | Check the rule order, the `match` constraints, and the tool name. eve ran the real tool for calls no rule matched. |
|
|
273
|
+
| `Tool stub session exceeded 10,000 calls. Start a new eval session.` | Split the eval into several sessions. |
|