eve 0.13.8 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/src/channel/compiled-channel.js +2 -1
- package/dist/src/channel/routes.d.ts +5 -0
- package/dist/src/channel/send.js +1 -1
- package/dist/src/channel/types.d.ts +5 -0
- package/dist/src/cli/dev/tui/runner.js +1 -1
- package/dist/src/cli/dev/tui/setup-panel.d.ts +3 -4
- package/dist/src/cli/dev/tui/setup-panel.js +2 -2
- package/dist/src/cli/dev/tui/terminal-renderer.js +1 -1
- package/dist/src/compiled/.vendor-stamp.json +12 -8
- package/dist/src/compiled/@ai-sdk/anthropic/index.d.ts +2 -2
- package/dist/src/compiled/@ai-sdk/anthropic/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/google/index.d.ts +25 -3
- package/dist/src/compiled/@ai-sdk/google/index.js +6 -6
- package/dist/src/compiled/@ai-sdk/mcp/index.d.ts +647 -30
- package/dist/src/compiled/@ai-sdk/mcp/index.js +1 -1
- package/dist/src/compiled/@ai-sdk/openai/index.d.ts +8 -2
- package/dist/src/compiled/@ai-sdk/openai/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/otel/index.d.ts +165 -28
- package/dist/src/compiled/@ai-sdk/otel/index.js +3 -3
- package/dist/src/compiled/@ai-sdk/provider/index.d.ts +10 -2
- package/dist/src/compiled/@ai-sdk/provider-utils/index.d.ts +2362 -0
- package/dist/src/compiled/@ai-sdk/provider-utils/index.js +1 -0
- package/dist/src/compiled/@workflow/serde/LICENSE.md +3 -0
- package/dist/src/compiled/@workflow/serde/index.d.ts +30 -0
- package/dist/src/compiled/@workflow/serde/index.js +1 -0
- package/dist/src/compiled/_chunks/workflow/{dist-C9PV_vnE.js → dist-D7CzPkf8.js} +1 -1
- package/dist/src/compiled/eventsource-parser/stream/LICENSE +21 -0
- package/dist/src/compiled/eventsource-parser/stream/index.d.ts +121 -0
- package/dist/src/compiled/eventsource-parser/stream/index.js +1 -0
- package/dist/src/compiled/experimental-ai-sdk-code-mode/approval-continuation.d.ts +7 -7
- package/dist/src/compiled/experimental-ai-sdk-code-mode/continuation-capability.d.ts +15 -4
- package/dist/src/compiled/experimental-ai-sdk-code-mode/index.d.ts +1 -1
- package/dist/src/compiled/experimental-ai-sdk-code-mode/index.js +15 -15
- package/dist/src/compiled/experimental-ai-sdk-code-mode/interrupt-continuation.d.ts +5 -5
- package/dist/src/compiled/experimental-ai-sdk-code-mode/runtime/protocol.d.ts +13 -2
- package/dist/src/compiled/experimental-ai-sdk-code-mode/types.d.ts +26 -0
- package/dist/src/compiled/json-schema/LICENSE +21 -0
- package/dist/src/compiled/json-schema/index.d.ts +749 -0
- package/dist/src/compiled/json-schema/index.js +1 -0
- package/dist/src/compiler/manifest.js +1 -1
- package/dist/src/compiler/normalize-agent-config.js +1 -1
- package/dist/src/context/build-dynamic-tools.js +1 -1
- package/dist/src/context/dynamic-tool-lifecycle.js +1 -1
- package/dist/src/context/keys.d.ts +1 -1
- package/dist/src/evals/assertions/collector.d.ts +19 -6
- package/dist/src/evals/assertions/collector.js +1 -1
- package/dist/src/evals/assertions/run.d.ts +24 -17
- package/dist/src/evals/assertions/run.js +2 -2
- package/dist/src/evals/assertions/scoped.d.ts +17 -0
- package/dist/src/evals/assertions/scoped.js +1 -0
- package/dist/src/evals/context.d.ts +1 -0
- package/dist/src/evals/context.js +1 -1
- package/dist/src/evals/control-flow.d.ts +9 -0
- package/dist/src/evals/control-flow.js +1 -0
- package/dist/src/evals/define-eval.d.ts +1 -1
- package/dist/src/evals/define-eval.js +1 -1
- package/dist/src/evals/expect/index.d.ts +5 -3
- package/dist/src/evals/expect/index.js +1 -1
- package/dist/src/evals/index.d.ts +2 -2
- package/dist/src/evals/match.d.ts +50 -13
- package/dist/src/evals/match.js +1 -1
- package/dist/src/evals/runner/artifacts.js +1 -1
- package/dist/src/evals/runner/derive-run-facts.js +1 -1
- package/dist/src/evals/runner/execute-eval.js +1 -1
- package/dist/src/evals/runner/execute-task.d.ts +1 -0
- package/dist/src/evals/runner/execute-task.js +1 -1
- package/dist/src/evals/runner/reporters/braintrust.js +1 -1
- package/dist/src/evals/runner/reporters/console.js +1 -1
- package/dist/src/evals/runner/reporters/junit.js +3 -2
- package/dist/src/evals/runner/run-evals.js +1 -1
- package/dist/src/evals/runner/verdict.d.ts +1 -0
- package/dist/src/evals/runner/verdict.js +1 -1
- package/dist/src/evals/session.d.ts +9 -5
- package/dist/src/evals/session.js +1 -1
- package/dist/src/evals/types.d.ts +69 -47
- package/dist/src/execution/dispatch-runtime-actions-step.js +1 -1
- package/dist/src/execution/dispatch-workflow-runtime-actions-step.d.ts +12 -0
- package/dist/src/execution/dispatch-workflow-runtime-actions-step.js +1 -0
- package/dist/src/execution/next-driver-action.d.ts +1 -1
- package/dist/src/execution/node-step.js +1 -1
- package/dist/src/execution/session.js +3 -3
- package/dist/src/execution/turn-workflow.js +1 -1
- package/dist/src/execution/workflow-entry.js +1 -1
- package/dist/src/execution/workflow-runtime.d.ts +2 -11
- package/dist/src/execution/workflow-runtime.js +1 -1
- package/dist/src/execution/workflow-steps.d.ts +1 -1
- package/dist/src/execution/workflow-steps.js +1 -1
- package/dist/src/harness/action-result-helpers.d.ts +2 -2
- package/dist/src/harness/emission.d.ts +13 -27
- package/dist/src/harness/emission.js +1 -1
- package/dist/src/harness/execute-tool.d.ts +2 -2
- package/dist/src/harness/input-extraction.js +1 -1
- package/dist/src/harness/step-hooks.d.ts +3 -3
- package/dist/src/harness/step-hooks.js +1 -1
- package/dist/src/harness/tool-loop.js +1 -1
- package/dist/src/harness/tools.d.ts +4 -6
- package/dist/src/harness/tools.js +1 -1
- package/dist/src/harness/types.d.ts +4 -10
- package/dist/src/harness/workflow-continuation-security.d.ts +4 -0
- package/dist/src/harness/workflow-continuation-security.js +1 -0
- package/dist/src/harness/workflow-interrupt-state.d.ts +14 -0
- package/dist/src/harness/workflow-interrupt-state.js +1 -0
- package/dist/src/harness/workflow-lifecycle.d.ts +13 -0
- package/dist/src/harness/workflow-lifecycle.js +1 -0
- package/dist/src/harness/workflow-runtime-action-state.d.ts +9 -0
- package/dist/src/harness/workflow-runtime-action-state.js +1 -0
- package/dist/src/harness/workflow-sandbox.d.ts +22 -0
- package/dist/src/harness/workflow-sandbox.js +2 -0
- package/dist/src/harness/workflow-tool-description.d.ts +2 -2
- package/dist/src/harness/workflow-tool-description.js +16 -4
- package/dist/src/internal/application/compiled-artifacts.js +1 -1
- package/dist/src/internal/application/package.d.ts +12 -0
- package/dist/src/internal/application/package.js +1 -1
- package/dist/src/internal/application/paths.d.ts +6 -0
- package/dist/src/internal/application/paths.js +1 -1
- package/dist/src/internal/authored-definition/connection.js +1 -1
- package/dist/src/internal/authored-definition/core.js +1 -1
- package/dist/src/internal/authored-definition/schema-backed.js +1 -1
- package/dist/src/internal/nitro/host/configure-nitro-routes.js +2 -2
- package/dist/src/internal/nitro/host/create-application-nitro.js +1 -1
- package/dist/src/internal/nitro/host/workflow-sandbox-runtime-plugin.d.ts +1 -0
- package/dist/src/internal/nitro/host/workflow-sandbox-runtime-plugin.js +1 -0
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response-from-manifest.js +1 -1
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response.js +1 -1
- package/dist/src/internal/workflow/configure-world.d.ts +6 -0
- package/dist/src/internal/workflow/configure-world.js +1 -1
- package/dist/src/internal/workflow/runtime.d.ts +1 -0
- package/dist/src/internal/workflow/world-compatibility.d.ts +32 -0
- package/dist/src/internal/workflow/world-compatibility.js +1 -0
- package/dist/src/protocol/message.d.ts +7 -5
- package/dist/src/public/channels/slack/attachments.js +1 -1
- package/dist/src/public/channels/slack/auth.d.ts +2 -0
- package/dist/src/public/channels/slack/auth.js +1 -1
- package/dist/src/public/channels/slack/defaults.js +2 -2
- package/dist/src/public/channels/slack/inbound.d.ts +4 -11
- package/dist/src/public/channels/slack/inbound.js +1 -2
- package/dist/src/public/channels/slack/model-context.d.ts +28 -0
- package/dist/src/public/channels/slack/model-context.js +3 -0
- package/dist/src/public/channels/slack/slackChannel.d.ts +8 -0
- package/dist/src/public/channels/slack/slackChannel.js +1 -1
- package/dist/src/public/connections/index.d.ts +1 -1
- package/dist/src/public/definitions/agent.d.ts +1 -1
- package/dist/src/public/definitions/approval.d.ts +40 -0
- package/dist/src/public/definitions/approval.js +1 -0
- package/dist/src/public/definitions/connections/mcp.d.ts +11 -9
- package/dist/src/public/definitions/connections/mcp.js +1 -1
- package/dist/src/public/definitions/connections/openapi.d.ts +9 -7
- package/dist/src/public/definitions/connections/openapi.js +1 -1
- package/dist/src/public/definitions/tool.d.ts +7 -20
- package/dist/src/public/index.d.ts +1 -1
- package/dist/src/public/tools/approval/approval-helpers.d.ts +7 -7
- package/dist/src/public/tools/approval/approval-helpers.js +1 -1
- package/dist/src/public/tools/approval/index.d.ts +1 -1
- package/dist/src/public/tools/index.d.ts +2 -1
- package/dist/src/public/tools/internal.js +1 -1
- package/dist/src/runtime/actions/types.d.ts +10 -11
- package/dist/src/runtime/agent/bootstrap.d.ts +1 -0
- package/dist/src/runtime/agent/bootstrap.js +1 -1
- package/dist/src/runtime/agent/mock-model-adapter.js +2 -3
- package/dist/src/runtime/connections/mcp-client.d.ts +3 -3
- package/dist/src/runtime/connections/mcp-client.js +1 -1
- package/dist/src/runtime/connections/registry.d.ts +2 -2
- package/dist/src/runtime/connections/resolve-authorization.d.ts +11 -0
- package/dist/src/runtime/connections/resolve-authorization.js +1 -0
- package/dist/src/runtime/connections/types.d.ts +27 -11
- package/dist/src/runtime/framework-tools/connection-search-dynamic.js +1 -1
- package/dist/src/runtime/resolve-agent-graph.js +1 -1
- package/dist/src/runtime/resolve-agent.js +1 -1
- package/dist/src/runtime/resolve-connection.js +1 -1
- package/dist/src/runtime/resolve-tool.d.ts +1 -1
- package/dist/src/runtime/resolve-tool.js +1 -1
- package/dist/src/runtime/types.d.ts +8 -7
- package/dist/src/setup/boxes/resolve-provisioning.js +1 -1
- package/dist/src/setup/primitives/pm/pnpm.js +6 -5
- package/dist/src/setup/scaffold/create/add-to-project.js +1 -1
- package/dist/src/setup/scaffold/create/project.js +2 -2
- package/dist/src/setup/scaffold/update/channels.js +1 -1
- package/dist/src/shared/agent-definition.d.ts +11 -13
- package/dist/src/shared/dynamic-tool-definition.d.ts +3 -3
- package/dist/src/shared/tool-definition.d.ts +1 -2
- package/dist/src/shared/workflow-sandbox.d.ts +37 -0
- package/dist/src/shared/workflow-sandbox.js +1 -0
- package/docs/agent-config.md +43 -13
- package/docs/channels/custom.mdx +2 -2
- package/docs/channels/slack.mdx +7 -13
- package/docs/concepts/execution-model-and-durability.md +1 -1
- package/docs/concepts/sessions-runs-and-streaming.md +1 -1
- package/docs/connections/mcp.mdx +2 -2
- package/docs/connections/openapi.mdx +37 -2
- package/docs/connections/overview.mdx +72 -2
- package/docs/evals/assertions.mdx +97 -39
- package/docs/evals/cases.mdx +24 -15
- package/docs/evals/judge.mdx +2 -2
- package/docs/evals/overview.mdx +7 -5
- package/docs/evals/reporters.mdx +2 -2
- package/docs/evals/running.mdx +3 -1
- package/docs/evals/targets.mdx +2 -2
- package/docs/guides/auth-and-route-protection.md +48 -1
- package/docs/guides/client/streaming.mdx +1 -1
- package/docs/guides/deployment.md +5 -4
- package/docs/guides/dynamic-workflows.md +1 -7
- package/docs/guides/frontend/overview.mdx +1 -1
- package/docs/guides/frontend/use-eve-agent-svelte.mdx +1 -1
- package/docs/guides/frontend/use-eve-agent-vue.mdx +1 -1
- package/docs/guides/session-context.md +2 -1
- package/docs/reference/typescript-api.md +2 -2
- package/docs/subagents.mdx +2 -2
- package/docs/tools/human-in-the-loop.md +43 -6
- package/docs/tools/overview.mdx +3 -3
- package/docs/tutorial/connect-a-warehouse.mdx +2 -0
- package/docs/tutorial/first-agent.mdx +1 -1
- package/docs/tutorial/guard-the-spend.mdx +4 -3
- package/docs/tutorial/ship-it.mdx +1 -1
- package/package.json +14 -11
- package/dist/src/compiled/@ai-sdk/anthropic/_provider-utils.d.ts +0 -15
- package/dist/src/compiled/@ai-sdk/google/_provider-utils.d.ts +0 -11
- package/dist/src/compiled/@ai-sdk/openai/_provider-utils.d.ts +0 -15
- package/dist/src/compiled/@ai-sdk/provider/_json-schema.d.ts +0 -5
- package/dist/src/execution/dispatch-code-mode-runtime-actions-step.d.ts +0 -21
- package/dist/src/execution/dispatch-code-mode-runtime-actions-step.js +0 -1
- package/dist/src/harness/code-mode-interrupt-state.d.ts +0 -26
- package/dist/src/harness/code-mode-interrupt-state.js +0 -1
- package/dist/src/harness/code-mode-lifecycle.d.ts +0 -16
- package/dist/src/harness/code-mode-lifecycle.js +0 -1
- package/dist/src/harness/code-mode-runtime-action-state.d.ts +0 -6
- package/dist/src/harness/code-mode-runtime-action-state.js +0 -1
- package/dist/src/harness/code-mode.d.ts +0 -26
- package/dist/src/harness/code-mode.js +0 -1
- package/dist/src/harness/sandbox-surface.d.ts +0 -68
- package/dist/src/harness/sandbox-surface.js +0 -1
- package/dist/src/internal/nitro/host/code-mode-runtime-dependency-plugin.d.ts +0 -1
- package/dist/src/internal/nitro/host/code-mode-runtime-dependency-plugin.js +0 -1
- package/dist/src/runtime/framework-tools/code-mode-connection-auth.d.ts +0 -29
- package/dist/src/runtime/framework-tools/code-mode-connection-auth.js +0 -1
- package/dist/src/shared/code-mode.d.ts +0 -29
- package/dist/src/shared/code-mode.js +0 -1
package/docs/evals/cases.mdx
CHANGED
|
@@ -5,6 +5,14 @@ description: "Author single-turn and multi-turn evals with test(t), and fan one
|
|
|
5
5
|
|
|
6
6
|
Each eval file is one graded case by default, and a single file can fan out over a dataset by default-exporting an array (covered below). The runner executes each `test(t)` function against the target, captures every event, and computes a verdict from the [assertions](./assertions) you recorded. Every eval shares one shape, whether single-turn, multi-turn, human-in-the-loop (HITL), or dataset-driven: one `async test(t)` function that drives the agent and asserts inline.
|
|
7
7
|
|
|
8
|
+
Before adding a case, create the required config at the root of `evals/`. An empty config is enough when you do not need shared judge, reporter, concurrency, or timeout settings:
|
|
9
|
+
|
|
10
|
+
```ts title="evals/evals.config.ts"
|
|
11
|
+
import { defineEvalConfig } from "eve/evals";
|
|
12
|
+
|
|
13
|
+
export default defineEvalConfig({});
|
|
14
|
+
```
|
|
15
|
+
|
|
8
16
|
## Single-turn evals
|
|
9
17
|
|
|
10
18
|
The common case sends one turn and asserts on the reply. `t.send(input)` resolves once the turn settles, and `t.reply` is the last assistant message:
|
|
@@ -16,7 +24,7 @@ import { includes } from "eve/evals/expect";
|
|
|
16
24
|
export default defineEval({
|
|
17
25
|
async test(t) {
|
|
18
26
|
await t.send("What is the weather in Brooklyn?");
|
|
19
|
-
t.
|
|
27
|
+
t.succeeded();
|
|
20
28
|
t.check(t.reply, includes("Sunny"));
|
|
21
29
|
},
|
|
22
30
|
});
|
|
@@ -30,7 +38,7 @@ import { defineEval } from "eve/evals";
|
|
|
30
38
|
export default defineEval({
|
|
31
39
|
async test(t) {
|
|
32
40
|
await t.send("Hello!");
|
|
33
|
-
t.
|
|
41
|
+
t.succeeded();
|
|
34
42
|
t.notCalledTool("get_weather");
|
|
35
43
|
},
|
|
36
44
|
});
|
|
@@ -42,6 +50,7 @@ Identity is the file path, so directories are the grouping mechanism. `evals/wea
|
|
|
42
50
|
|
|
43
51
|
```text
|
|
44
52
|
evals/
|
|
53
|
+
├── evals.config.ts
|
|
45
54
|
├── weather/
|
|
46
55
|
│ ├── shared.ts # helpers, not an eval
|
|
47
56
|
│ ├── brooklyn-forecast.eval.ts
|
|
@@ -69,25 +78,21 @@ export default defineEval({
|
|
|
69
78
|
});
|
|
70
79
|
```
|
|
71
80
|
|
|
72
|
-
|
|
81
|
+
Use scoped assertions for intermediate turns. When later control flow depends on a value-level check, `t.require` records a gate and stops the script if it fails:
|
|
73
82
|
|
|
74
83
|
```ts title="evals/session-continuity.eval.ts"
|
|
75
84
|
import { defineEval } from "eve/evals";
|
|
76
|
-
import {
|
|
85
|
+
import { equals } from "eve/evals/expect";
|
|
77
86
|
|
|
78
87
|
export default defineEval({
|
|
79
88
|
async test(t) {
|
|
80
|
-
await t.send("My favorite word is marigold.");
|
|
81
|
-
const firstSessionId = t.sessionId;
|
|
89
|
+
const first = await t.send("My favorite word is marigold.");
|
|
82
90
|
|
|
83
91
|
const second = await t.send("Thanks for remembering.");
|
|
84
|
-
second.
|
|
85
|
-
if (t.sessionId !== firstSessionId) {
|
|
86
|
-
throw new Error(`Expected one session; got ${firstSessionId} then ${t.sessionId}.`);
|
|
87
|
-
}
|
|
92
|
+
await t.require(second.sessionId, equals(first.sessionId));
|
|
88
93
|
|
|
89
|
-
t.
|
|
90
|
-
|
|
94
|
+
t.succeeded();
|
|
95
|
+
second.messageIncludes("Thanks for remembering.");
|
|
91
96
|
},
|
|
92
97
|
});
|
|
93
98
|
```
|
|
@@ -98,12 +103,16 @@ export default defineEval({
|
|
|
98
103
|
|
|
99
104
|
- `t.send(input)` sends a turn and waits for it to settle. It accepts the same input as `ClientSession.send()` (a string or a structured message) and resolves to a turn carrying `.message` and `.expectOk()`.
|
|
100
105
|
- `t.sendFile(text, path, mediaType?)` attaches a local file as a data URL.
|
|
101
|
-
- `t.
|
|
106
|
+
- `t.requireInputRequest(filter?)` records a gate, requires exactly one pending request, and returns it. Filters match tool name, action input, prompt, display, and option ids.
|
|
102
107
|
- `t.respond(...responses)` answers specific pending input requests and sends them as the next turn.
|
|
103
108
|
- `t.respondAll(optionId)` answers every pending input request with the same option and sends the responses as the next turn.
|
|
104
109
|
- `t.reply` is the last assistant message (or `null`); `t.sessionId` is the current session id; `t.events` is the full typed event stream captured so far.
|
|
105
110
|
|
|
106
|
-
Each `send` (and `respond`/`respondAll`) resolves to
|
|
111
|
+
Each `send` (and `respond`/`respondAll`) resolves to an immutable turn with `.message`, `.data`, `.events`, `.inputRequests`, `.toolCalls`, `.sessionId`, `.status`, and `.expectOk()`. Use `.sessionId` to relate turns or attach follow-up work to the session that produced a specific turn. `expectOk()` throws only when the turn ended failed; a session left open for a next message is the normal end state of a successful turn.
|
|
112
|
+
|
|
113
|
+
Use `expectOk()` only when the next operation depends on that intermediate turn succeeding. A final `t.succeeded()` already records a complete-run gate.
|
|
114
|
+
|
|
115
|
+
To intentionally omit an eval for the current target, call `t.skip(reason)` before sending messages or recording assertions. Skipped evals are reported separately and do not affect the exit code.
|
|
107
116
|
|
|
108
117
|
Events from every session are captured in the result and artifacts. `t.log(message)` records debug lines into the eval artifact; `--verbose` also streams them to stdout as evals run. `t.signal` is an `AbortSignal` that fires on timeout.
|
|
109
118
|
|
|
@@ -126,7 +135,7 @@ export default rows.map((row) =>
|
|
|
126
135
|
description: row.task,
|
|
127
136
|
async test(t) {
|
|
128
137
|
await t.send(row.prompt);
|
|
129
|
-
t.
|
|
138
|
+
t.succeeded();
|
|
130
139
|
t.check(t.reply, equals(row.sql));
|
|
131
140
|
},
|
|
132
141
|
}),
|
package/docs/evals/judge.mdx
CHANGED
|
@@ -11,7 +11,7 @@ import { defineEval } from "eve/evals";
|
|
|
11
11
|
export default defineEval({
|
|
12
12
|
async test(t) {
|
|
13
13
|
await t.send("Explain quantum tunneling to a 10-year-old.");
|
|
14
|
-
t.
|
|
14
|
+
t.succeeded();
|
|
15
15
|
t.judge.autoevals.closedQA("uses no math beyond arithmetic").atLeast(0.8);
|
|
16
16
|
},
|
|
17
17
|
});
|
|
@@ -52,7 +52,7 @@ t.judge.autoevals.closedQA("cites a source").atLeast(0.6); // soft, fails under
|
|
|
52
52
|
t.judge.autoevals.factuality(reference).gate(0.8); // hard gate at 0.8
|
|
53
53
|
```
|
|
54
54
|
|
|
55
|
-
A judge runs once per assertion and burns tokens, so reach for one only when nothing deterministic will do.
|
|
55
|
+
A judge runs once per assertion and burns tokens, so reach for one only when nothing deterministic will do. Judge calls start when recorded, and the runner waits for all of them during finalization; assertion handles themselves are intentionally not awaitable.
|
|
56
56
|
|
|
57
57
|
## Configuring the judge model
|
|
58
58
|
|
package/docs/evals/overview.mdx
CHANGED
|
@@ -33,7 +33,7 @@ export default defineEval({
|
|
|
33
33
|
description: "Basic message and tool-usage coverage for the weather agent.",
|
|
34
34
|
async test(t) {
|
|
35
35
|
await t.send("What is the weather in Brooklyn?");
|
|
36
|
-
t.
|
|
36
|
+
t.succeeded();
|
|
37
37
|
t.calledTool("get_weather");
|
|
38
38
|
t.check(t.reply, includes("Sunny"));
|
|
39
39
|
},
|
|
@@ -62,14 +62,14 @@ Everything is optional. `judge` sets the default model for [LLM-as-judge](./judg
|
|
|
62
62
|
|
|
63
63
|
`t` is both the driver and the assertion surface. There are no separate `input`, `run`, `checks`, or `scores` fields. You write ordinary control flow, sending turns and asserting inline.
|
|
64
64
|
|
|
65
|
-
- **Drive** the agent: `t.send(...)`, `t.respond(...)`, `t.respondAll(...)`, `t.sendFile(...)`, `t.
|
|
65
|
+
- **Drive** the agent: `t.send(...)`, `t.respond(...)`, `t.respondAll(...)`, `t.sendFile(...)`, `t.requireInputRequest(...)`, `t.newSession()`. Read what came back with `t.reply` (the last assistant message), `t.sessionId`, and `t.events`. See [Cases](./cases).
|
|
66
66
|
- **Assert** with three surfaces, covered next.
|
|
67
67
|
|
|
68
68
|
## Three assertion surfaces
|
|
69
69
|
|
|
70
70
|
Each surface matches a genuinely different kind of judgment:
|
|
71
71
|
|
|
72
|
-
- **
|
|
72
|
+
- **Scoped methods** read the final whole run on `t`, snapshot one independent session when invoked there, or inspect one immutable `EveEvalTurn`. See [Assertions](./assertions).
|
|
73
73
|
- **`t.check(value, assertion)`** grades an explicit value with a deterministic builder from `eve/evals/expect`, such as `t.check(t.reply, includes("sunny"))`. Grade `t.reply`, an intermediate draft, parsed JSON, or anything else. See [Assertions](./assertions).
|
|
74
74
|
- **`t.judge.autoevals.*`** is the LLM-as-judge surface, like `t.judge.autoevals.closedQA("cites a source")`. It grades `t.reply` by default and uses the configured judge model, never the agent under test. See [Judge](./judge).
|
|
75
75
|
|
|
@@ -83,12 +83,14 @@ Every assertion returns a chainable handle, so severity rides on the assertion i
|
|
|
83
83
|
Override per assertion: `.gate(threshold?)` promotes to a hard gate, `.soft(threshold?)` demotes to tracked, and `.atLeast(threshold)` is a soft assertion with a bar.
|
|
84
84
|
|
|
85
85
|
```ts
|
|
86
|
-
t.
|
|
86
|
+
t.succeeded(); // gate
|
|
87
87
|
t.calledTool("get_weather").soft(); // record as a metric, don't gate
|
|
88
88
|
t.judge.autoevals.closedQA("cites a source"); // soft, tracked (no threshold)
|
|
89
89
|
t.judge.autoevals.factuality(reference).atLeast(0.7); // soft, gated under --strict at 0.7
|
|
90
90
|
```
|
|
91
91
|
|
|
92
|
+
Use `await t.require(value, assertion)` for a gate that must pass before the script can safely continue. Use `t.skip(reason)` as the first operation for an intentionally unsupported target capability.
|
|
93
|
+
|
|
92
94
|
## Run evals with eve eval
|
|
93
95
|
|
|
94
96
|
```bash
|
|
@@ -101,7 +103,7 @@ Exit code `0` means every eval passed its gates. See [Running evals](./running)
|
|
|
101
103
|
|
|
102
104
|
## A good baseline
|
|
103
105
|
|
|
104
|
-
Most apps do fine with a few small smoke evals. Assert behavior with `t.
|
|
106
|
+
Most apps do fine with a few small smoke evals. Assert behavior with `t.succeeded()` plus one or two content checks, keep dataset fixtures in `evals/data/`, and reach for a judge or Braintrust only when you need fuzzy grading or shared result review. In CI, run `eve eval --strict` so soft threshold misses fail the build too.
|
|
105
107
|
|
|
106
108
|
## What to read next
|
|
107
109
|
|
package/docs/evals/reporters.mdx
CHANGED
|
@@ -33,7 +33,7 @@ export default defineEval({
|
|
|
33
33
|
reporters: [Braintrust({ projectName: "weather-agent" })],
|
|
34
34
|
async test(t) {
|
|
35
35
|
await t.send("What is the weather in Brooklyn?");
|
|
36
|
-
t.
|
|
36
|
+
t.succeeded();
|
|
37
37
|
},
|
|
38
38
|
});
|
|
39
39
|
```
|
|
@@ -52,7 +52,7 @@ Braintrust needs its SDK installed in the app and credentials in the environment
|
|
|
52
52
|
eve eval --strict --junit .eve/junit.xml
|
|
53
53
|
```
|
|
54
54
|
|
|
55
|
-
Each eval becomes one `<testcase>` named by its path-derived id; failed gates and execution errors
|
|
55
|
+
Each eval becomes one `<testcase>` named by its path-derived id; failed gates and execution errors become failures, while `t.skip(reason)` produces a JUnit `<skipped>` result.
|
|
56
56
|
|
|
57
57
|
## Custom reporters
|
|
58
58
|
|
package/docs/evals/running.mdx
CHANGED
|
@@ -26,10 +26,12 @@ Positional ids match exactly or by directory prefix: `eve eval weather` runs `ev
|
|
|
26
26
|
|
|
27
27
|
| Code | Means |
|
|
28
28
|
| ---- | ------------------------------------------------------------------------------- |
|
|
29
|
-
| `0` | Every eval passed its gates (and soft thresholds, under `--strict`)
|
|
29
|
+
| `0` | Every non-skipped eval passed its gates (and soft thresholds, under `--strict`) |
|
|
30
30
|
| `1` | Any eval failed (a failed gate, an execution error, or a strict threshold miss) |
|
|
31
31
|
| `2` | Configuration error |
|
|
32
32
|
|
|
33
|
+
An eval that calls `t.skip(reason)` is reported as skipped, does not count as passed or failed, and never changes the exit code.
|
|
34
|
+
|
|
33
35
|
## Artifacts
|
|
34
36
|
|
|
35
37
|
Each run drops artifacts under `.eve/evals/<timestamp>/`: a run `summary.json`, a `results.jsonl` index, and per-eval assertion results, verdicts, captured event streams, and `t.log` lines under `evals/`. The console output stays tight on purpose; when an eval fails, the artifact has the full story.
|
package/docs/evals/targets.mdx
CHANGED
|
@@ -16,7 +16,7 @@ export default defineEval({
|
|
|
16
16
|
async test(t) {
|
|
17
17
|
const { sessionIds } = await t.target.dispatchSchedule("heartbeat");
|
|
18
18
|
await t.target.attachSession(sessionIds[0]!);
|
|
19
|
-
t.
|
|
19
|
+
t.succeeded();
|
|
20
20
|
t.calledTool("send_report");
|
|
21
21
|
},
|
|
22
22
|
});
|
|
@@ -26,7 +26,7 @@ export default defineEval({
|
|
|
26
26
|
- `t.target.dispatchSchedule(id)` triggers a [schedule](../schedules) through the dev-only schedule route and returns the session ids it created. It works only against a target with dev routes enabled (the local `eve eval` dev server, or a deployment running in development mode), and throws otherwise.
|
|
27
27
|
- `t.target.attachSession(sessionId, { startIndex? })` consumes one turn from a session created outside the eval, by a channel or a schedule, so its events feed the run-level assertions. `startIndex` skips events before that position, so a session already partway through its stream resumes from where you left off rather than replaying from the start.
|
|
28
28
|
|
|
29
|
-
Sessions attached this way are full `EveEvalSession`s: you can keep driving them
|
|
29
|
+
Sessions attached this way are full `EveEvalSession`s: you can keep driving them and assert directly on that session (`session.succeeded()`, `session.calledTool(...)`). Aggregate assertions on `t` continue to read the whole run, including every attached session.
|
|
30
30
|
|
|
31
31
|
## Authentication
|
|
32
32
|
|
|
@@ -224,6 +224,53 @@ Route auth does not enforce session ownership. If multiple users or tenants can
|
|
|
224
224
|
|
|
225
225
|
Tool and connection auth is how your agent reaches an external service that wants an interactive sign-in, like an OAuth MCP server. Connections declare `auth` on the connection definition. Tools should resolve providers inline with `ctx.getToken(provider)` and call `ctx.requireAuth(provider)` only when a downstream service rejects a token; eve drives the sign-in, caches the token per step, and re-runs the call once the caller authorizes.
|
|
226
226
|
|
|
227
|
+
The principal for user-scoped tool and connection auth comes from route auth. `connect("...")` from `@vercel/connect/eve` defaults to `principalType: "user"`, so the active session must have `ctx.session.auth.current.principalType === "user"` before the first token lookup can start OAuth. If the session is anonymous, local-dev-only, runtime-scoped, or service-scoped, eve fails fast with `reason: "principal_required"` because there is no end-user identity to bind the OAuth grant to.
|
|
228
|
+
|
|
229
|
+
Use app-scoped auth when the external service should act as the agent itself:
|
|
230
|
+
|
|
231
|
+
```ts
|
|
232
|
+
auth: connect({ connector: "linear/myagent", principalType: "app" });
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Use user-scoped auth when the external service should act as the signed-in person:
|
|
236
|
+
|
|
237
|
+
```ts
|
|
238
|
+
auth: connect("linear/myagent");
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
For user-scoped auth in a browser app, the route-auth entry for the eve channel should verify your app session and return a user principal:
|
|
242
|
+
|
|
243
|
+
```ts title="agent/channels/eve.ts"
|
|
244
|
+
import { eveChannel } from "eve/channels/eve";
|
|
245
|
+
import { localDev, type AuthFn } from "eve/channels/auth";
|
|
246
|
+
import { getSession } from "@/lib/auth";
|
|
247
|
+
|
|
248
|
+
function appSession(): AuthFn<Request> {
|
|
249
|
+
return async (request) => {
|
|
250
|
+
const session = await getSession(request);
|
|
251
|
+
if (!session) return null;
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
authenticator: "app",
|
|
255
|
+
principalId: session.userId,
|
|
256
|
+
principalType: "user",
|
|
257
|
+
attributes: {
|
|
258
|
+
email: session.email,
|
|
259
|
+
teamId: session.teamId,
|
|
260
|
+
},
|
|
261
|
+
};
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export default eveChannel({
|
|
266
|
+
auth: [appSession(), localDev()],
|
|
267
|
+
});
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Keep `principalId` stable for the same person, and include an `issuer` when the same app may accept users from multiple identity providers. The connection token cache keys user credentials by issuer and principal id so two providers cannot accidentally share a grant.
|
|
271
|
+
|
|
272
|
+
Built-in platform channels that identify a human sender, such as Slack, Discord, Teams, Telegram, Twilio, Linear, and GitHub, attach a user principal for that sender by default. A Slack mention, DM, or button click can therefore authorize a user-scoped connection for the Slack user who sent it without adding a separate browser-session auth function.
|
|
273
|
+
|
|
227
274
|
### On a connection
|
|
228
275
|
|
|
229
276
|
Attach `connect()` from `@vercel/connect/eve` to the connection:
|
|
@@ -241,7 +288,7 @@ export default defineMcpClientConnection({
|
|
|
241
288
|
});
|
|
242
289
|
```
|
|
243
290
|
|
|
244
|
-
The first call that needs
|
|
291
|
+
The first call that needs a user-scoped connection kicks off an OAuth sign-in, surfaced as an authorization challenge (a URL the caller visits). [Vercel Connect](https://vercel.com/docs/connect) brokers the flow and holds the credentials, which are resolved and cached per workflow step, never serialized into history, and never shown to the model. For non-interactive connections, pass a static token or `connect({ connector, principalType: "app" })` in place of user-scoped `connect()`. [Connections](../connections) covers both shapes.
|
|
245
292
|
|
|
246
293
|
### On a single tool
|
|
247
294
|
|
|
@@ -66,7 +66,7 @@ The most common UI events are:
|
|
|
66
66
|
| `message.received` | Confirm the user message landed. |
|
|
67
67
|
| `reasoning.appended` | Render reasoning deltas when the model provides them. |
|
|
68
68
|
| `message.appended` | Render assistant text deltas. |
|
|
69
|
-
| `actions.requested` | Show tool calls
|
|
69
|
+
| `actions.requested` | Show tool calls as the model requests them, before execution. |
|
|
70
70
|
| `action.result` | Show tool call results. |
|
|
71
71
|
| `input.requested` | Pause the UI for approval or a question answer. |
|
|
72
72
|
| `result.completed` | Read structured output from an [output schema](./output-schema). |
|
|
@@ -74,11 +74,11 @@ import { anthropic } from "@ai-sdk/anthropic";
|
|
|
74
74
|
import { defineAgent } from "eve";
|
|
75
75
|
|
|
76
76
|
export default defineAgent({
|
|
77
|
-
model: anthropic("claude-opus-4
|
|
77
|
+
model: anthropic("claude-opus-4-8"),
|
|
78
78
|
});
|
|
79
79
|
```
|
|
80
80
|
|
|
81
|
-
With that shape, the model call goes directly to Anthropic and the runtime reads `ANTHROPIC_API_KEY`. The same pattern works for OpenAI after installing `@ai-sdk/openai`, using `openai("...")`, and setting `OPENAI_API_KEY`. This is the usual choice when self-deploying without any Vercel-managed services.
|
|
81
|
+
With that shape, the model call goes directly to Anthropic and the runtime reads `ANTHROPIC_API_KEY`. Direct Anthropic model ids use hyphens (`claude-opus-4-8`), unlike the dotted Gateway id (`anthropic/claude-opus-4.8`). The same pattern works for OpenAI after installing `@ai-sdk/openai`, using `openai("...")`, and setting `OPENAI_API_KEY`. This is the usual choice when self-deploying without any Vercel-managed services.
|
|
82
82
|
|
|
83
83
|
## 4. Sandbox backend
|
|
84
84
|
|
|
@@ -140,7 +140,8 @@ Eve writes the standard Nitro output under `.output/` instead of Vercel Build Ou
|
|
|
140
140
|
|
|
141
141
|
Self-deployed agents should make the Vercel-specific choices explicit:
|
|
142
142
|
|
|
143
|
-
- Let the Workflow SDK use its default local world, which stores workflow state under `.workflow-data`, configure your host so that directory is on persistent storage, or select another world with `experimental.workflow.world` in the root `agent.ts`.
|
|
143
|
+
- Let the Workflow SDK use its default local world, which stores workflow state under `.workflow-data`, configure your host so that directory is on persistent storage, or select another world with `experimental.workflow.world` in the root `agent.ts`. When you select a custom world, install a world package built against the same `@workflow/*` line as your eve release (currently the `5.0.0-beta` line). The npm `latest` tag may lag, so pin the version explicitly, for example `pnpm add @workflow/world-postgres@5.0.0-beta.x`. A mismatched world (such as a `4.x` package against a `5.x` core) fails with a `ZodError: invalid_union` during run replay.
|
|
144
|
+
- If you put a reverse proxy or ingress in front of eve, forward **both** `/eve/` and `/.well-known/workflow/`. The workflow world delivers run callbacks to `/.well-known/workflow/v1/flow`; a proxy restricted to `/eve/` lets sessions start but silently stalls runs forever, because the callbacks never reach eve.
|
|
144
145
|
- Install the AI SDK package for your provider, then use a direct provider model object and `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` when you want no Gateway dependency.
|
|
145
146
|
- Use `AI_GATEWAY_API_KEY` if you still want Gateway routing from a non-Vercel host.
|
|
146
147
|
- Replace `vercelOidc()` with auth that your host can verify.
|
|
@@ -148,7 +149,7 @@ Self-deployed agents should make the Vercel-specific choices explicit:
|
|
|
148
149
|
- If the agent defines schedules, the default `eve build && eve start` path starts Nitro's schedule runner, and Vercel wires schedules to Vercel Cron automatically. If you adapt the output to a custom HTTP-only host or preset, make sure it also runs Nitro scheduled tasks, or trigger the same work from your own scheduler.
|
|
149
150
|
- Treat Vercel Cron, Vercel Sandbox prewarm, Vercel Deployment Protection bypass, and the Agent Runs dashboard as Vercel-only conveniences.
|
|
150
151
|
|
|
151
|
-
The HTTP contract is unchanged: health, session creation, streaming, channels, tools, and subagents use the same routes. Any client that can reach and authenticate to those routes can talk to the agent.
|
|
152
|
+
The HTTP contract is unchanged: health, session creation, streaming, channels, tools, and subagents use the same routes under `/eve/`, and the workflow dispatch route lives under `/.well-known/workflow/`. A reverse proxy must preserve both prefixes. Any client that can reach and authenticate to those routes can talk to the agent.
|
|
152
153
|
|
|
153
154
|
## 9. Verify the deployment
|
|
154
155
|
|
|
@@ -3,7 +3,7 @@ title: "Dynamic Workflows"
|
|
|
3
3
|
description: "The experimental Workflow tool: let the model orchestrate its own subagents from model-authored JavaScript as one durable step."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
The experimental `Workflow` tool lets the model write JavaScript that coordinates the agent's own subagents as a single durable step. The program can run them in sequence, feed one result into the next, fan out over a list, and combine the results. You enable the capability and the model decides and runs the orchestration.
|
|
6
|
+
The experimental `Workflow` tool lets the model write JavaScript that coordinates the agent's own subagents as a single durable step. The program can run them in sequence, feed one result into the next, fan out over a list, and combine the results. You enable the capability and the model decides and runs the orchestration.
|
|
7
7
|
|
|
8
8
|
A single turn can already call several subagents, and parallel tool calls dispatch concurrently. What a workflow adds is _programmatic_ coordination. The program decides how many subagents to run based on an earlier result, which output feeds which call, and how to combine everything. That is logic the model cannot express as a few one-off calls.
|
|
9
9
|
|
|
@@ -63,12 +63,6 @@ That is an allowlist, not a denylist. The sandbox cannot read files, open a sock
|
|
|
63
63
|
- **Approval-safe.** A subagent that needs human approval (HITL, human-in-the-loop) mid-run surfaces its request to the user, and the workflow picks back up once that is answered, same as direct delegation.
|
|
64
64
|
- **Observable.** Every orchestrated subagent emits the usual `subagent.called` / `subagent.completed` events on the parent stream and gets its own child session and stream. The telemetry matches direct delegation, so existing dashboards and cost attribution keep working.
|
|
65
65
|
|
|
66
|
-
## Relationship to code mode
|
|
67
|
-
|
|
68
|
-
[Code mode](../agent-config#other-defineagent-fields) is the broader version, where the model drives _all_ of an agent's tools (files, shell, web, and agents) from JavaScript. A workflow covers only the subagents. The two do not interfere. Enabling the `Workflow` tool leaves code mode untouched, and an agent can run both at once.
|
|
69
|
-
|
|
70
|
-
`codeMode` is experimental and may change or be removed.
|
|
71
|
-
|
|
72
66
|
## What to read next
|
|
73
67
|
|
|
74
68
|
- Declare the subagents a workflow orchestrates → [Subagents](../subagents)
|
|
@@ -100,7 +100,7 @@ Assistant text, reasoning, tool calls, and tool results stream into `data` as th
|
|
|
100
100
|
|
|
101
101
|
## Human-in-the-loop prompts
|
|
102
102
|
|
|
103
|
-
Tools opt into approval with `
|
|
103
|
+
Tools opt into approval with `approval`, and the model can also ask a question with `ask_question` — see [Human-in-the-loop](/docs/human-in-the-loop) for the server-side model. Either way the stream emits an `input.requested` event, and the pending request rides on a `dynamic-tool` part of the latest message at `part.toolMetadata?.eve?.inputRequest`. Read it, then answer through the same session with `send()`:
|
|
104
104
|
|
|
105
105
|
```tsx
|
|
106
106
|
const request = agent.data.messages
|
|
@@ -78,7 +78,7 @@ await agent.send({
|
|
|
78
78
|
|
|
79
79
|
## Human-in-the-loop prompts
|
|
80
80
|
|
|
81
|
-
A tool opts into approval with `
|
|
81
|
+
A tool opts into approval with `approval` ([Tools](../../tools)). When one fires, the pending request rides along on a `dynamic-tool` part of the latest message at `part.toolMetadata?.eve?.inputRequest`. Read it, then answer through the same session with `agent.send({ inputResponses })`:
|
|
82
82
|
|
|
83
83
|
```ts
|
|
84
84
|
import type { EveDynamicToolPart, EveMessagePart } from "eve/svelte";
|
|
@@ -89,7 +89,7 @@ async function onFileChange(event: Event) {
|
|
|
89
89
|
|
|
90
90
|
## Human-in-the-loop prompts
|
|
91
91
|
|
|
92
|
-
A tool opts into approval with `
|
|
92
|
+
A tool opts into approval with `approval` ([Tools](../../tools)). When it triggers, the pending request shows up as a `dynamic-tool` part on the latest message at `part.toolMetadata?.eve?.inputRequest`. Read it, then answer through the same session with `send({ inputResponses })`:
|
|
93
93
|
|
|
94
94
|
```ts
|
|
95
95
|
import type { EveDynamicToolPart, EveMessagePart } from "eve/vue";
|
|
@@ -3,7 +3,7 @@ title: "Session Context"
|
|
|
3
3
|
description: "Runtime helpers: ctx.session, ctx.getSandbox, ctx.getSkill, and defineState."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
eve exposes runtime state through the `ctx` parameter passed to tool `execute`, hook handlers,
|
|
6
|
+
eve exposes runtime state through the `ctx` parameter passed to tool `execute`, hook handlers, channel event handlers, and connection auth/header resolvers:
|
|
7
7
|
|
|
8
8
|
- `ctx.session`: session metadata, turn, auth, and parent lineage
|
|
9
9
|
- `ctx.getSandbox()`: live sandbox handle for the current agent
|
|
@@ -119,6 +119,7 @@ export const budget = defineState<BudgetState>("myapp.budget", () => ({
|
|
|
119
119
|
Safe places:
|
|
120
120
|
|
|
121
121
|
- inside `defineTool(...).execute(input, ctx)`
|
|
122
|
+
- inside connection `auth: (ctx) => provider` and `headers: (ctx) => values` resolvers
|
|
122
123
|
- inside authored callbacks eve runs inside the runtime
|
|
123
124
|
- after asynchronous boundaries inside the same authored execution chain
|
|
124
125
|
|
|
@@ -51,11 +51,11 @@ export default defineTool({
|
|
|
51
51
|
| `defineEvalConfig` | `eve/evals` | `evals/evals.config.ts` | [Evals](../evals/overview) |
|
|
52
52
|
| `useEveAgent` | `eve/react`, `eve/vue`, `eve/svelte` | frontend | [Frontend](../guides/frontend/overview) |
|
|
53
53
|
|
|
54
|
-
A few non-`define*` helpers round out the set: `disableTool` and `ExperimentalWorkflow` from `eve/tools` (see [Default harness](../concepts/default-harness)), the route verbs `GET`/`POST`/`PUT`/`PATCH`/`DELETE`/`WS` from `eve/channels`, the approval
|
|
54
|
+
A few non-`define*` helpers round out the set: `disableTool` and `ExperimentalWorkflow` from `eve/tools` (see [Default harness](../concepts/default-harness)), the route verbs `GET`/`POST`/`PUT`/`PATCH`/`DELETE`/`WS` from `eve/channels`, the approval policies `always`/`once`/`never` from `eve/tools/approval`, and the channel auth helpers `localDev`/`vercelOidc`/`placeholderAuth` from `eve/channels/auth`. To wrap a built-in tool, import its default value from `eve/tools/defaults` (`bash`, `readFile`, `writeFile`, `glob`, `grep`, `webFetch`, `webSearch`, `todo`, `loadSkill`). `AgentReasoningDefinition` is exported from `eve` for the top-level `defineAgent({ reasoning })` setting. `AgentWorkflowDefinition` and `AgentWorkflowWorldDefinition` are exported from `eve` for the `defineAgent({ experimental: { workflow } })` config shape.
|
|
55
55
|
|
|
56
56
|
## Runtime context (`ctx`)
|
|
57
57
|
|
|
58
|
-
`ctx` is passed to your tool `execute`, hook handlers,
|
|
58
|
+
`ctx` is passed to your tool `execute`, hook handlers, channel event handlers, and connection auth/header resolvers. It is live only while authored code is running, so reaching for it at module top level throws. See [Session context](../guides/session-context) for the full model.
|
|
59
59
|
|
|
60
60
|
| Member | Use |
|
|
61
61
|
| --------------------------- | ---------------------------------------------------------------------------- |
|
package/docs/subagents.mdx
CHANGED
|
@@ -16,7 +16,7 @@ Every agent gets an `agent` tool by default. The model calls it to delegate a su
|
|
|
16
16
|
}
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
The copy shares the parent's sandbox and tools, and a child's file writes are immediately visible to the parent. That is what makes parallel calls natural: fan out a
|
|
19
|
+
The copy shares the parent's sandbox and tools, and a child's file writes are immediately visible to the parent. That is what makes parallel calls natural: emit multiple `agent` calls in one response to fan out a small, fixed set of independent subtasks. eve runs that batch concurrently and returns every result before the parent continues. Give children non-overlapping write scopes when they work in the shared sandbox. The copy inherits auth and connections, but starts with fresh conversation history and fresh state. If a declared subagent calls `agent`, the child is a copy of _that_ subagent, not the root.
|
|
20
20
|
|
|
21
21
|
The parent transfers data to the child through the `message` input it gives the subagent. Do not include sensitive data in a subagent request unless that child and its inherited tools, connections, sandbox, and telemetry path are appropriate for that data.
|
|
22
22
|
|
|
@@ -88,7 +88,7 @@ A declared subagent's tool name is the bare path-derived name, with no prefix. `
|
|
|
88
88
|
|
|
89
89
|
Because the name lives in the same runtime tool namespace as authored tools, a subagent named `researcher` collides with a tool named `researcher`. eve rejects the build rather than picking a winner, so keep subagent directory names distinct from tool names.
|
|
90
90
|
|
|
91
|
-
Do not rely on subagent delegation by itself as an approval boundary. Put sensitive tools behind `
|
|
91
|
+
Do not rely on subagent delegation by itself as an approval boundary. Put sensitive tools behind `approval`, connection approval, route/session authorization, or other controls wherever those tools can be called.
|
|
92
92
|
|
|
93
93
|
Each delegated subagent spins up its own child session and stream. The parent stream carries only the control-plane events `subagent.called` and `subagent.completed`. To follow the child's full progress, read `subagent.called.data.childSessionId` and subscribe at `GET /eve/v1/session/:childSessionId/stream`.
|
|
94
94
|
|
|
@@ -13,7 +13,7 @@ Either way the run parks at `session.waiting`, durably, for as long as it takes
|
|
|
13
13
|
|
|
14
14
|
## Approvals
|
|
15
15
|
|
|
16
|
-
Approval is a property of a [tool](/docs/tools) that pauses for a person before it runs. Gate a tool with `
|
|
16
|
+
Approval is a property of a [tool](/docs/tools) that pauses for a person before it runs. Gate a tool with `approval` and the helpers from `eve/tools/approval`:
|
|
17
17
|
|
|
18
18
|
```ts title="agent/tools/refund_charge.ts"
|
|
19
19
|
import { defineTool } from "eve/tools";
|
|
@@ -22,8 +22,8 @@ import { z } from "zod";
|
|
|
22
22
|
|
|
23
23
|
export default defineTool({
|
|
24
24
|
description: "Refund a charge.",
|
|
25
|
-
inputSchema: z.object({ chargeId: z.string(), amount: z.number() }),
|
|
26
|
-
|
|
25
|
+
inputSchema: z.object({ tenantId: z.string(), chargeId: z.string(), amount: z.number() }),
|
|
26
|
+
approval: always(), // or once() / never() / a policy
|
|
27
27
|
async execute(input) {
|
|
28
28
|
return refund(input);
|
|
29
29
|
},
|
|
@@ -36,16 +36,53 @@ export default defineTool({
|
|
|
36
36
|
| `once()` | Require approval only the first time the tool runs in a session; auto-allow after. |
|
|
37
37
|
| `always()` | Require approval before every call. |
|
|
38
38
|
|
|
39
|
-
By default, omitted `
|
|
39
|
+
By default, omitted `approval` behaves like `never()`, so tool calls may execute without human approval. Require human approval or other safeguards for sensitive, irreversible, regulated, financial, healthcare, employment, housing, legal, safety-impacting, user-impacting, or external side-effecting actions.
|
|
40
40
|
|
|
41
|
-
When the decision depends on the input, pass your own
|
|
41
|
+
When the decision depends on the input, pass your own policy instead of a helper. It receives the same session context as tool execution, plus `{ toolName, toolInput, approvedTools }`, and returns an AI SDK 7 approval status synchronously or as a promise. Use `ctx.session.auth.current` to guard by the caller of the current turn and `ctx.session.auth.initiator` to guard by the caller that created the session. Return `"user-approval"` to pause for a person or `"not-applicable"` to continue without a prompt. `toolInput` can be undefined, so guard the access. This policy denies cross-tenant calls, then requires approval only when an amount crosses a threshold:
|
|
42
42
|
|
|
43
43
|
```ts
|
|
44
|
-
|
|
44
|
+
approval: ({ session, toolInput }) => {
|
|
45
|
+
const callerTenant = session.auth.current?.attributes.tenantId;
|
|
46
|
+
if (callerTenant === undefined || callerTenant !== toolInput?.tenantId) {
|
|
47
|
+
return { type: "denied", reason: "Caller cannot access this tenant." };
|
|
48
|
+
}
|
|
49
|
+
return (toolInput?.amount ?? 0) > 1000 ? "user-approval" : "not-applicable";
|
|
50
|
+
},
|
|
45
51
|
```
|
|
46
52
|
|
|
53
|
+
For compatibility with the previous predicate shape, policies may return booleans: `true` is treated as `"user-approval"` and `false` as `"not-applicable"`. Boolean promises are supported too.
|
|
54
|
+
|
|
55
|
+
Policies can also return `"approved"` or `"denied"` to decide automatically. Use `{ type: "approved" | "denied", reason }` when the model should receive a reason. The `Approval`, `ApprovalContext`, and `ApprovalStatus` types are exported from both `eve/tools` and `eve/tools/approval`.
|
|
56
|
+
|
|
47
57
|
Gating a side effect on approval is also how you make non-idempotent work safe across replays: a charge or email that sits behind `always()` can't fire from a re-run step without a fresh human decision.
|
|
48
58
|
|
|
59
|
+
### Skipping approval for schedule-dispatched turns
|
|
60
|
+
|
|
61
|
+
`session.auth.current` identifies the caller of this turn. Markdown schedules use the app principal (`authenticator: "app"`, `principalId: "eve:app"`, `principalType: "runtime"`) automatically. A `run` schedule must pass its `appAuth` to `receive(...)` for the child session to use that principal. Match all three fields to skip approval for automated turns while still prompting when a person calls the same tool:
|
|
62
|
+
|
|
63
|
+
```ts title="agent/tools/refund_charge.ts"
|
|
64
|
+
import { defineTool } from "eve/tools";
|
|
65
|
+
import { z } from "zod";
|
|
66
|
+
|
|
67
|
+
export default defineTool({
|
|
68
|
+
description: "Refund a charge.",
|
|
69
|
+
inputSchema: z.object({ chargeId: z.string(), amount: z.number() }),
|
|
70
|
+
approval: ({ session }) => {
|
|
71
|
+
const auth = session.auth.current;
|
|
72
|
+
return auth?.authenticator === "app" &&
|
|
73
|
+
auth.principalId === "eve:app" &&
|
|
74
|
+
auth.principalType === "runtime"
|
|
75
|
+
? "not-applicable"
|
|
76
|
+
: "user-approval";
|
|
77
|
+
},
|
|
78
|
+
async execute(input) {
|
|
79
|
+
return refund(input);
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`session` in `approval` has the same shape as `ctx.session` in `execute`: `id`, `auth`, `turn`, and an optional `parent`. If a person later resumes a schedule-started session, `session.auth.current` becomes that person while `session.auth.initiator` remains the app principal. Inspect `initiator` only when the policy should apply to the whole session. Skipping approval on scheduled turns means any non-idempotent side effect will re-fire if a step replays, so pair this pattern with idempotency keys or `once()` where needed.
|
|
85
|
+
|
|
49
86
|
## Questions
|
|
50
87
|
|
|
51
88
|
The built-in `ask_question` tool lets the model pause and ask the user, rather than guessing. It has no `execute` — the model calls it with `{ prompt, options?, allowFreeform? }`:
|
package/docs/tools/overview.mdx
CHANGED
|
@@ -46,7 +46,7 @@ eve never runs authored tools during discovery. The model sees descriptors first
|
|
|
46
46
|
|
|
47
47
|
## Gate a tool on human approval
|
|
48
48
|
|
|
49
|
-
A tool can require a person to sign off before it runs. Set `
|
|
49
|
+
A tool can require a person to sign off before it runs. Set `approval` with the helpers from `eve/tools/approval`:
|
|
50
50
|
|
|
51
51
|
```ts title="agent/tools/refund_charge.ts"
|
|
52
52
|
import { defineTool } from "eve/tools";
|
|
@@ -56,14 +56,14 @@ import { z } from "zod";
|
|
|
56
56
|
export default defineTool({
|
|
57
57
|
description: "Refund a charge.",
|
|
58
58
|
inputSchema: z.object({ chargeId: z.string(), amount: z.number() }),
|
|
59
|
-
|
|
59
|
+
approval: always(), // or once() / never() / a policy
|
|
60
60
|
async execute(input) {
|
|
61
61
|
return refund(input);
|
|
62
62
|
},
|
|
63
63
|
});
|
|
64
64
|
```
|
|
65
65
|
|
|
66
|
-
Approval is one half of eve's [human-in-the-loop](./human-in-the-loop) model — the page covers the `always/once/never` helpers, input-dependent
|
|
66
|
+
Approval is one half of eve's [human-in-the-loop](./human-in-the-loop) model — the page covers the `always/once/never` helpers, input-dependent policies, and how a gated call pauses and resumes durably.
|
|
67
67
|
|
|
68
68
|
## Shape what the model sees with `toModelOutput`
|
|
69
69
|
|
|
@@ -26,6 +26,8 @@ export default defineMcpClientConnection({
|
|
|
26
26
|
|
|
27
27
|
`"warehouse"` is the UID you chose when registering the Connect client. By default this OAuth is user-scoped. Each end-user authorizes in their own browser, and eve resolves that user's token before every tool call.
|
|
28
28
|
|
|
29
|
+
Before testing the warehouse from a web app, make sure the eve channel route auth maps your signed-in app user to `principalType: "user"`. A Connect-backed connection can only start per-user OAuth when the active session already has an authenticated user principal. If route auth still only accepts `localDev()`, a runtime token, or a placeholder guard, the first warehouse tool call fails with `reason: "principal_required"` instead of showing the sign-in challenge.
|
|
30
|
+
|
|
29
31
|
Once Connect is enabled on your account, wire it up:
|
|
30
32
|
|
|
31
33
|
1. Install the package: `npm install @vercel/connect`.
|
|
@@ -10,7 +10,7 @@ Step 1 gets it talking. The scaffold bundles a small sample dataset, so your fir
|
|
|
10
10
|
## Prerequisites
|
|
11
11
|
|
|
12
12
|
- Node 24 or newer and npm.
|
|
13
|
-
- A model credential. The scaffold's default model goes through the [Vercel AI Gateway](../getting-started), so you need `AI_GATEWAY_API_KEY` (or `VERCEL_OIDC_TOKEN` pulled via `vercel link`). A direct provider model like `anthropic("claude-opus-4
|
|
13
|
+
- A model credential. The scaffold's default model goes through the [Vercel AI Gateway](../getting-started), so you need `AI_GATEWAY_API_KEY` (or `VERCEL_OIDC_TOKEN` pulled via `vercel link`). A direct provider model like `anthropic("claude-opus-4-8")` instead needs that provider's AI SDK package and key, here `@ai-sdk/anthropic` and `ANTHROPIC_API_KEY`.
|
|
14
14
|
|
|
15
15
|
If you have not run eve before, complete [Getting Started](../getting-started) first. Without a credential, "Run the agent" below fails when the runtime tries to reach the model; the dev TUI's `/model` flow walks you through pasting a key or linking a project.
|
|
16
16
|
|
|
@@ -5,7 +5,7 @@ description: "Part 8 of the Build an Agent tutorial. Gate expensive queries with
|
|
|
5
5
|
|
|
6
6
|
A single warehouse query can scan terabytes and run up the bill. So before the analytics assistant fires off an expensive scan, make it stop and check with you. The agent pauses, asks you, and resumes with your answer. That's human-in-the-loop, and you wire it up with one field on the tool.
|
|
7
7
|
|
|
8
|
-
`
|
|
8
|
+
`approval` runs before `execute`. Return `"user-approval"` and the turn parks on an approval request; you answer, and the run picks up from that exact step. The function gets the tool input, so you can make the decision cost-based.
|
|
9
9
|
|
|
10
10
|
## Estimate, then gate
|
|
11
11
|
|
|
@@ -32,7 +32,8 @@ export default defineTool({
|
|
|
32
32
|
description: "Run a read-only SQL query against the analytics tables.",
|
|
33
33
|
inputSchema: z.object({ sql: z.string() }),
|
|
34
34
|
// Cost-based gate: only the expensive queries need a human yes.
|
|
35
|
-
|
|
35
|
+
approval: ({ toolInput }) =>
|
|
36
|
+
estimateScanGb(toolInput?.sql ?? "") > THRESHOLD_GB ? "user-approval" : "not-applicable",
|
|
36
37
|
async execute({ sql }) {
|
|
37
38
|
const { columns, rows } = await runReadOnlySql(sql);
|
|
38
39
|
return { columns, rows: rows.slice(0, 500), truncated: rows.length > 500 };
|
|
@@ -50,7 +51,7 @@ Ask for something that forces a large unfiltered scan:
|
|
|
50
51
|
Total revenue across all customers, all time, broken out by day.
|
|
51
52
|
```
|
|
52
53
|
|
|
53
|
-
The model proposes the query, `
|
|
54
|
+
The model proposes the query, `approval` returns `"user-approval"`, and the turn parks. The stream emits `input.requested`, then `session.waiting`. How the prompt looks depends on the channel, whether buttons in the TUI, Block Kit in Slack, or a UI control on the web. Approve it and the run resumes from exactly that step, then the query runs. Deny it and the tool is skipped, with the model told why.
|
|
54
55
|
|
|
55
56
|
Each session has exactly one active continuation. Answer an approval against a stale handle and it's rejected, so there's no way to double-resume the same parked turn.
|
|
56
57
|
|
|
@@ -142,7 +142,7 @@ Across the nine steps you built and shipped one agent, and along the way you use
|
|
|
142
142
|
- **The sandbox** to compute and chart beyond SQL in an isolated `/workspace`.
|
|
143
143
|
- **State** (`defineState`) to remember the team's glossary across turns.
|
|
144
144
|
- **Dynamic skills** (`defineDynamic`) to load the right team playbook per caller.
|
|
145
|
-
- **Human-in-the-loop** approval (`
|
|
145
|
+
- **Human-in-the-loop** approval (`approval`) to gate expensive queries.
|
|
146
146
|
- **Channel auth** to turn a request into an authenticated principal.
|
|
147
147
|
- **Deployment** to Vercel, with the runtime behind your web app.
|
|
148
148
|
|