eve 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +1 -0
- package/dist/src/channel/session.d.ts +2 -0
- package/dist/src/channel/session.js +1 -1
- package/dist/src/channel/types.d.ts +8 -1
- package/dist/src/chunks/{use-eve-agent-X0asmJg-.js → use-eve-agent-Dmc6sX4d.js} +558 -233
- package/dist/src/chunks/{use-eve-agent-CxWcZCsU.js → use-eve-agent-jSR8P4nK.js} +565 -235
- package/dist/src/cli/dev/tui/runner.d.ts +4 -0
- package/dist/src/cli/dev/tui/runner.js +1 -1
- package/dist/src/cli/dev/tui/setup-issues.js +1 -1
- package/dist/src/cli/dev/tui/terminal-renderer.js +12 -12
- package/dist/src/client/eve-agent-projection.d.ts +10 -0
- package/dist/src/client/eve-agent-projection.js +1 -0
- package/dist/src/client/eve-agent-store-helpers.d.ts +6 -2
- package/dist/src/client/eve-agent-store-helpers.js +2 -2
- package/dist/src/client/eve-agent-store-state.d.ts +54 -1
- package/dist/src/client/eve-agent-store.d.ts +12 -82
- package/dist/src/client/eve-agent-store.js +1 -1
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +1 -1
- package/dist/src/client/message-response.d.ts +5 -1
- package/dist/src/client/message-response.js +1 -1
- package/dist/src/client/ndjson.d.ts +1 -0
- package/dist/src/client/ndjson.js +2 -2
- package/dist/src/client/open-stream.d.ts +4 -1
- package/dist/src/client/open-stream.js +1 -1
- package/dist/src/client/session-event-stream.d.ts +30 -0
- package/dist/src/client/session-event-stream.js +1 -0
- package/dist/src/client/session-utils.d.ts +2 -1
- package/dist/src/client/session-utils.js +1 -1
- package/dist/src/client/session.d.ts +14 -1
- package/dist/src/client/session.js +1 -1
- package/dist/src/client/sessions.d.ts +8 -2
- package/dist/src/client/sessions.js +1 -1
- package/dist/src/client/types.d.ts +7 -0
- package/dist/src/compiled/.vendor-stamp.json +2 -3
- package/dist/src/compiled/@vercel/sandbox/api-client/api-client.d.ts +88 -1
- package/dist/src/compiled/@vercel/sandbox/api-client/index.d.ts +1 -1
- package/dist/src/compiled/@vercel/sandbox/api-client/validators.d.ts +41 -1
- package/dist/src/compiled/@vercel/sandbox/constants.d.ts +3 -3
- package/dist/src/compiled/@vercel/{sandbox-drives → sandbox}/drive.d.ts +13 -2
- package/dist/src/compiled/@vercel/sandbox/index.d.ts +3 -2
- package/dist/src/compiled/@vercel/sandbox/index.js +15 -9
- package/dist/src/compiled/@vercel/sandbox/sandbox.d.ts +40 -5
- package/dist/src/compiled/_chunks/node/auth-DDrvYko5.js +2 -0
- package/dist/src/compiled/_chunks/node/version-C__MqbLM.js +1 -0
- package/dist/src/compiler/extension-compatibility.d.ts +14 -14
- package/dist/src/compiler/extension-compatibility.js +1 -1
- package/dist/src/context/build-dynamic-tools.js +1 -1
- package/dist/src/context/dynamic-tool-lifecycle.js +1 -1
- package/dist/src/context/memory-tools.js +1 -1
- package/dist/src/evals/context.d.ts +1 -1
- package/dist/src/evals/context.js +1 -1
- package/dist/src/evals/define-eval.d.ts +1 -1
- package/dist/src/evals/define-eval.js +1 -1
- package/dist/src/evals/index.js +1 -1
- package/dist/src/evals/judge.d.ts +1 -1
- package/dist/src/evals/runner/execute-task.d.ts +1 -1
- package/dist/src/evals/runner/execute-task.js +1 -1
- package/dist/src/evals/session-manager.d.ts +27 -0
- package/dist/src/evals/session-manager.js +1 -0
- package/dist/src/evals/session.d.ts +8 -28
- package/dist/src/evals/session.js +2 -1
- package/dist/src/evals/target.d.ts +1 -1
- package/dist/src/evals/target.js +1 -1
- package/dist/src/evals/types.d.ts +19 -15
- package/dist/src/eve-channel/create-request.d.ts +24 -0
- package/dist/src/eve-channel/create-request.js +1 -0
- package/dist/src/eve-channel/index.js +1 -1
- package/dist/src/eve-channel/request.d.ts +3 -12
- package/dist/src/eve-channel/request.js +2 -2
- package/dist/src/eve-channel/types.d.ts +4 -2
- package/dist/src/execution/dynamic-workflow/tool.js +1 -1
- package/dist/src/execution/legacy-session/turn-workflow.js +1 -1
- package/dist/src/execution/runtime-context.js +1 -1
- package/dist/src/execution/sandbox/bindings/vercel-lifecycle.d.ts +3 -3
- package/dist/src/execution/sandbox/bindings/vercel-lifecycle.js +1 -1
- package/dist/src/execution/sandbox/bindings/vercel-sdk-types.d.ts +1 -4
- package/dist/src/execution/sandbox/bindings/vercel.d.ts +1 -2
- package/dist/src/execution/sandbox/bindings/vercel.js +1 -1
- package/dist/src/execution/session/entry.js +1 -1
- package/dist/src/execution/session/handoff-steps.js +1 -1
- package/dist/src/execution/session/program.d.ts +2 -0
- package/dist/src/execution/session/program.js +1 -1
- package/dist/src/execution/session/turn-step.js +1 -1
- package/dist/src/execution/session/turn.js +1 -1
- package/dist/src/execution/session-inbox/protocol.js +1 -1
- package/dist/src/execution/tools/connection-search.js +1 -1
- package/dist/src/execution/tools/sleep-workflow.js +1 -1
- package/dist/src/execution/workflow-runtime.js +1 -1
- package/dist/src/experimental/evaluate/auto-model.js +2 -2
- package/dist/src/experimental/evaluate/evaluate.d.ts +7 -0
- package/dist/src/experimental/evaluate/evaluate.js +1 -0
- package/dist/src/harness/compaction.d.ts +3 -3
- package/dist/src/harness/compaction.js +1 -1
- package/dist/src/harness/request-envelope.d.ts +11 -0
- package/dist/src/harness/request-envelope.js +1 -0
- package/dist/src/harness/tool-loop.js +2 -2
- package/dist/src/instrumentation/prepare-trace-context.d.ts +2 -1
- package/dist/src/instrumentation/prepare-trace-context.js +1 -1
- package/dist/src/internal/application/package.js +1 -1
- package/dist/src/internal/authored-module-loader.d.ts +4 -1
- package/dist/src/internal/authored-module-loader.js +1 -1
- package/dist/src/internal/bundler/nitro-rolldown.d.ts +2 -0
- package/dist/src/internal/model-auth/transport.d.ts +3 -1
- package/dist/src/internal/model-auth/transport.js +1 -1
- package/dist/src/internal/nitro/host/build-extension.js +1 -1
- package/dist/src/internal/nitro/host/extension-capability-requirements.d.ts +1 -0
- package/dist/src/internal/nitro/host/extension-capability-requirements.js +1 -1
- package/dist/src/internal/nitro/host/extension-distribution.d.ts +1 -1
- package/dist/src/internal/nitro/host/extension-distribution.js +1 -1
- package/dist/src/internal/workflow-bundle/dynamic-tool-transform.js +1 -1
- package/dist/src/public/experimental/evaluate/index.d.ts +1 -0
- package/dist/src/public/experimental/evaluate/index.js +1 -1
- package/dist/src/public/models/anthropic/index.js +1 -1
- package/dist/src/public/sandbox/vercel-sandbox.d.ts +1 -1
- package/dist/src/public/sandbox/vercel.d.ts +1 -1
- package/dist/src/public/sandbox/vercel.js +1 -1
- package/dist/src/public/tools/index.d.ts +1 -0
- package/dist/src/public/tools/index.js +1 -1
- package/dist/src/react/use-eve-agent.d.ts +4 -0
- package/dist/src/react/use-eve-agent.js +1 -1
- package/dist/src/runtime/agent/resolve-model.js +1 -1
- package/dist/src/setup/flows/model-login-connection.js +1 -1
- package/dist/src/setup/scaffold/create/project.js +1 -1
- package/dist/src/subagents/handle-dispatch.js +1 -1
- package/dist/src/svelte/index.js +1 -1
- package/dist/src/svelte/use-eve-agent.d.ts +4 -0
- package/dist/src/svelte/use-eve-agent.js +1 -1
- package/dist/src/tools/definition.d.ts +2 -0
- package/dist/src/tools/definition.js +1 -1
- package/dist/src/tools/durable-callbacks.d.ts +8 -2
- package/dist/src/tools/durable-callbacks.js +1 -1
- package/dist/src/tools/durable-schema.d.ts +16 -0
- package/dist/src/tools/durable-schema.js +1 -0
- package/dist/src/vue/index.js +1 -1
- package/dist/src/vue/use-eve-agent.d.ts +4 -0
- package/dist/src/vue/use-eve-agent.js +1 -1
- package/docs/README.md +1 -0
- package/docs/channels/eve.mdx +20 -10
- package/docs/concepts/context-control.md +4 -0
- package/docs/concepts/sessions-runs-and-streaming.md +18 -3
- package/docs/evals/assertions.mdx +6 -6
- package/docs/evals/cases.mdx +59 -23
- package/docs/evals/judge.mdx +10 -8
- package/docs/evals/overview.mdx +6 -6
- package/docs/evals/reporters.mdx +1 -1
- package/docs/evals/targets.mdx +5 -0
- package/docs/guides/client/messages.mdx +5 -5
- package/docs/guides/client/overview.mdx +15 -1
- package/docs/guides/dynamic-capabilities.md +22 -0
- package/docs/guides/evaluate.md +58 -4
- package/docs/guides/frontend/overview.mdx +54 -5
- package/docs/guides/frontend/use-eve-agent-svelte.mdx +13 -8
- package/docs/guides/frontend/use-eve-agent-vue.mdx +13 -8
- package/docs/reference/typescript-api.md +38 -36
- package/package.json +2 -3
- package/dist/src/compiled/@vercel/sandbox-drives/LICENSE +0 -201
- package/dist/src/compiled/@vercel/sandbox-drives/_async-retry.d.ts +0 -8
- package/dist/src/compiled/@vercel/sandbox-drives/_workflow-serde.d.ts +0 -5
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/api-client.d.ts +0 -1815
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/api-error.d.ts +0 -29
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/base-client.d.ts +0 -39
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/file-writer.d.ts +0 -66
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/index.d.ts +0 -2
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/validators.d.ts +0 -932
- package/dist/src/compiled/@vercel/sandbox-drives/api-client/with-retry.d.ts +0 -10
- package/dist/src/compiled/@vercel/sandbox-drives/auth/error.d.ts +0 -15
- package/dist/src/compiled/@vercel/sandbox-drives/auth/file.d.ts +0 -18
- package/dist/src/compiled/@vercel/sandbox-drives/auth/index.d.ts +0 -6
- package/dist/src/compiled/@vercel/sandbox-drives/auth/oauth.d.ts +0 -111
- package/dist/src/compiled/@vercel/sandbox-drives/auth/poll-for-token.d.ts +0 -28
- package/dist/src/compiled/@vercel/sandbox-drives/auth/project.d.ts +0 -40
- package/dist/src/compiled/@vercel/sandbox-drives/command.d.ts +0 -292
- package/dist/src/compiled/@vercel/sandbox-drives/constants.d.ts +0 -15
- package/dist/src/compiled/@vercel/sandbox-drives/execution-context.d.ts +0 -116
- package/dist/src/compiled/@vercel/sandbox-drives/filesystem.d.ts +0 -258
- package/dist/src/compiled/@vercel/sandbox-drives/index.d.ts +0 -14
- package/dist/src/compiled/@vercel/sandbox-drives/index.js +0 -9
- package/dist/src/compiled/@vercel/sandbox-drives/network-policy.d.ts +0 -169
- package/dist/src/compiled/@vercel/sandbox-drives/proxy.d.ts +0 -55
- package/dist/src/compiled/@vercel/sandbox-drives/sandbox-user.d.ts +0 -211
- package/dist/src/compiled/@vercel/sandbox-drives/sandbox.d.ts +0 -1287
- package/dist/src/compiled/@vercel/sandbox-drives/session.d.ts +0 -435
- package/dist/src/compiled/@vercel/sandbox-drives/snapshot.d.ts +0 -238
- package/dist/src/compiled/@vercel/sandbox-drives/utils/get-credentials.d.ts +0 -21
- package/dist/src/compiled/@vercel/sandbox-drives/utils/paginator.d.ts +0 -16
- package/dist/src/compiled/@vercel/sandbox-drives/utils/resolveSignal.d.ts +0 -15
- package/dist/src/compiled/@vercel/sandbox-drives/utils/sandbox-snapshot.d.ts +0 -11
- package/dist/src/compiled/@vercel/sandbox-drives/utils/types.d.ts +0 -11
- package/dist/src/compiled/_chunks/node/auth-8aQozzfi.js +0 -2
- package/dist/src/compiled/_chunks/node/auth-Bvv0xGTY.js +0 -2
- package/dist/src/compiled/_chunks/node/esm-Fqlolk7e.js +0 -7
- package/dist/src/compiled/_chunks/node/version-BC6t2Txc.js +0 -1
- package/dist/src/compiled/_chunks/node/version-DSQqVZ5T.js +0 -1
- package/dist/src/compiled/_chunks/node/xdg-app-paths-F-QHrdza.js +0 -1
|
@@ -35,6 +35,8 @@ export interface UseEveAgentReturn<TData> {
|
|
|
35
35
|
readonly events: ComputedRef<readonly MessageStreamEvent[]>;
|
|
36
36
|
/** Replay the attached durable session and follow its in-flight turn, if any. */
|
|
37
37
|
readonly resume: () => Promise<void>;
|
|
38
|
+
/** Create the session without starting its first turn. */
|
|
39
|
+
readonly prewarm: () => Promise<void>;
|
|
38
40
|
/** Clear all state and start a new session. */
|
|
39
41
|
readonly reset: () => void;
|
|
40
42
|
/** Send a message with optional turn settings. */
|
|
@@ -98,6 +100,8 @@ export interface UseEveAgentOptions<TData> extends EveAgentStoreCallbacks<TData>
|
|
|
98
100
|
* @default true
|
|
99
101
|
*/
|
|
100
102
|
readonly optimistic?: boolean;
|
|
103
|
+
/** Prewarm an owned session on mount and after reset. @default false */
|
|
104
|
+
readonly prewarm?: boolean;
|
|
101
105
|
/**
|
|
102
106
|
* Projects stream events into `TData`.
|
|
103
107
|
*
|
package/docs/README.md
CHANGED
|
@@ -18,6 +18,7 @@ Important naming note:
|
|
|
18
18
|
| Create a project and run your first agent | [Getting Started](./getting-started.mdx) |
|
|
19
19
|
| Look up agent files and discovery rules | [Agent Files](./reference/agent-files.md) |
|
|
20
20
|
| Set the model, reasoning, or other agent-wide config | [Agents](./agent-config.md) |
|
|
21
|
+
| Choose an agent model automatically for each request | [Automatic Model Selection](./guides/evaluate.md) |
|
|
21
22
|
| Change what the agent does and how it behaves | [Instructions](./instructions.mdx) |
|
|
22
23
|
| Give the agent a typed capability it can call | [Tools](./tools/overview.mdx) |
|
|
23
24
|
| Require approval, or ask the user something mid-turn | [Human in the Loop](./tools/human-in-the-loop.md) |
|
package/docs/channels/eve.mdx
CHANGED
|
@@ -24,7 +24,7 @@ The default eve channel inspects the agent, creates sessions, accepts callbacks
|
|
|
24
24
|
|
|
25
25
|
- `GET /eve/v1/health` (check whether the application is reachable)
|
|
26
26
|
- `GET /eve/v1/info` (inspect the agent)
|
|
27
|
-
- `POST /eve/v1/session` (
|
|
27
|
+
- `POST /eve/v1/session` (create a session, optionally sending its first message)
|
|
28
28
|
- `POST /eve/v1/session/:sessionId` (send a follow-up)
|
|
29
29
|
- `POST /eve/v1/session/:sessionId/cancel` (cancel the in-flight turn)
|
|
30
30
|
- `POST /eve/v1/session/:sessionId/clear` (clear the session's model history)
|
|
@@ -38,7 +38,21 @@ The session routes use only durable session IDs. Create a session explicitly, th
|
|
|
38
38
|
|
|
39
39
|
### Start and continue a session
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
Create a conversation session before its first message with a bodyless request:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
curl -X POST https://<deployment>/eve/v1/session
|
|
45
|
+
# {"ok":true,"sessionId":"wrun_A","status":"accepted"}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
See [Start a session](../concepts/sessions-runs-and-streaming#start-a-session) for the
|
|
49
|
+
initialization boundary, first-turn identity, readiness response, and retry behavior.
|
|
50
|
+
|
|
51
|
+
The create request still requires route auth. Session audience and the timeout are selected
|
|
52
|
+
at creation; they do not change when the first message arrives. Task mode and turn-scoped
|
|
53
|
+
fields such as `clientContext` and `outputSchema` require a message on the create request.
|
|
54
|
+
|
|
55
|
+
Include an initial message to create the session and start its first turn in one request:
|
|
42
56
|
|
|
43
57
|
```bash
|
|
44
58
|
curl -X POST https://<deployment>/eve/v1/session \
|
|
@@ -47,11 +61,6 @@ curl -X POST https://<deployment>/eve/v1/session \
|
|
|
47
61
|
# {"ok":true,"sessionId":"wrun_A","status":"accepted"}
|
|
48
62
|
```
|
|
49
63
|
|
|
50
|
-
The `202` response means Workflow accepted the durable run. The route does not wait for command
|
|
51
|
-
inbox or continuation-token ownership. The inbox may still be starting, so an immediate follow-up
|
|
52
|
-
or control request can report that the session is not active. The event-stream client retries the
|
|
53
|
-
brief window before the new run becomes readable.
|
|
54
|
-
|
|
55
64
|
Authenticated callers that may retry a create request can pass their own `operationId` for
|
|
56
65
|
create-once semantics. Once the operation owner is active, the same operation under the same
|
|
57
66
|
authenticated principal returns that session instead of dispatching the input again. The create
|
|
@@ -69,7 +78,7 @@ curl -X POST https://<deployment>/eve/v1/session \
|
|
|
69
78
|
# {"ok":true,"sessionId":"wrun_A","status":"accepted"}
|
|
70
79
|
```
|
|
71
80
|
|
|
72
|
-
|
|
81
|
+
When the create request includes a message, a follow-up request accepts exactly one of
|
|
73
82
|
`message` or `inputResponses`; use the latter to answer a pending HITL request:
|
|
74
83
|
|
|
75
84
|
```bash
|
|
@@ -82,8 +91,9 @@ Follow-up messages use `turnPolicy: "steer"` by default. Before assistant answer
|
|
|
82
91
|
|
|
83
92
|
Set `turnPolicy: "queue"` on `eveChannel(...)` when follow-ups should wait for active turns to finish. `inputResponses` answer their addressed requests.
|
|
84
93
|
|
|
85
|
-
Sending a message to an unknown
|
|
94
|
+
Sending a message to an unknown or terminal session ID returns `409` with
|
|
86
95
|
`{"code":"session_not_active","error":"The session is no longer active.","ok":false}`.
|
|
96
|
+
An existing workflow whose inbox is not yet available returns `409 session_not_ready` instead.
|
|
87
97
|
TypeScript clients expose the stable code as `ClientError.code`. The route never
|
|
88
98
|
creates or follows a replacement session.
|
|
89
99
|
|
|
@@ -225,7 +235,7 @@ export default eveChannel({
|
|
|
225
235
|
});
|
|
226
236
|
```
|
|
227
237
|
|
|
228
|
-
`onMessage` must return an auth result. Return `title` alongside `auth` to set the title when the dispatch
|
|
238
|
+
`onMessage` must return an auth result. Return `title` alongside `auth` to set the title when the dispatch creates a session or sends the first message to a prewarmed session. A successful canonical eve HTTP message always dispatches and therefore always produces or continues a session.
|
|
229
239
|
|
|
230
240
|
## Clients
|
|
231
241
|
|
|
@@ -58,6 +58,10 @@ See [Dynamic capabilities](../guides/dynamic-capabilities) for the resolver API,
|
|
|
58
58
|
|
|
59
59
|
## Compaction and clear
|
|
60
60
|
|
|
61
|
+
Before a model call, eve checks the projected history together with the effective system instructions and advertised tool schemas. The check runs after `step.started` resolves dynamic capabilities. When the provider reports input usage, eve adds estimates for new messages and growth in the instructions and tool catalog to that count; unchanged schemas are not counted again.
|
|
62
|
+
|
|
63
|
+
Compaction reserves space for those instructions and tools while reducing conversation history. It cannot shrink the instructions or tool catalog themselves, so keep them within the selected model's context window.
|
|
64
|
+
|
|
61
65
|
User-role instructions follow the normal history lifecycle. Compaction can summarize them, and clear removes them without rerunning their static definitions or dynamic resolvers. System-role instructions remain outside history and continue to apply after either operation.
|
|
62
66
|
|
|
63
67
|
Recalled memory also uses user-role messages, but eve keeps their attribution
|
|
@@ -33,16 +33,31 @@ React, Vue, and Svelte apps reach for [`useEveAgent()`](../guides/frontend/overv
|
|
|
33
33
|
|
|
34
34
|
## Start a session
|
|
35
35
|
|
|
36
|
+
Create and park a conversation session before its first turn by omitting `message`:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
curl -X POST http://127.0.0.1:2000/eve/v1/session
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
eve starts the durable workflow, establishes its inbox, and waits for the first message before
|
|
43
|
+
running session-scoped initialization or emitting `session.started`. The first message sent to the returned
|
|
44
|
+
`sessionId` remains `turn_0`. Message-free creation supports conversation mode only and does not
|
|
45
|
+
accept turn-scoped `clientContext`, `outputSchema`, callbacks, or activity observers.
|
|
46
|
+
|
|
47
|
+
To create the session and start its first turn in one request, include the message:
|
|
48
|
+
|
|
36
49
|
```bash
|
|
37
50
|
curl -X POST http://127.0.0.1:2000/eve/v1/session \
|
|
38
51
|
-H 'content-type: application/json' \
|
|
39
52
|
-d '{"message":"Summarize the latest forecast."}'
|
|
40
53
|
```
|
|
41
54
|
|
|
42
|
-
eve responds with `202` and the durable `sessionId` in the JSON body and
|
|
55
|
+
In both forms, eve responds with `202` and the durable `sessionId` in the JSON body and
|
|
43
56
|
`x-eve-session-id` header as soon as Workflow accepts the run. The command inbox can still be
|
|
44
|
-
starting at that point. An immediate follow-up can return `409
|
|
45
|
-
|
|
57
|
+
starting at that point. An immediate follow-up can return `409 session_not_ready`; retry that
|
|
58
|
+
code with bounded backoff. The TypeScript client retries sends for up to 20 seconds and respects
|
|
59
|
+
the caller's abort signal. Do not wait
|
|
60
|
+
for `session.waiting` on a prewarmed session: initialization and its first events require a message.
|
|
46
61
|
|
|
47
62
|
## Stream a session
|
|
48
63
|
|
|
@@ -50,15 +50,15 @@ attached.messageIncludes("Sunny");
|
|
|
50
50
|
|
|
51
51
|
## Value assertions with `t.check`
|
|
52
52
|
|
|
53
|
-
`t.check(value, assertion)` grades an explicit value against a builder from `eve/evals/expect`. The value can be
|
|
53
|
+
`t.check(value, assertion)` grades an explicit value against a builder from `eve/evals/expect`. The value can be a turn’s `.message`, parsed JSON, or any local you computed:
|
|
54
54
|
|
|
55
55
|
```ts
|
|
56
56
|
import { includes, equals, matches, satisfies, similarity } from "eve/evals/expect";
|
|
57
57
|
|
|
58
|
-
t.check(
|
|
58
|
+
t.check(turn.message, includes(/sunny/i)); // substring or RegExp (gate)
|
|
59
59
|
t.check(parsed, equals({ city: "Brooklyn" })); // deep structural equality (gate)
|
|
60
60
|
t.check(parsed, matches(WeatherSchema)); // Standard Schema, e.g. Zod (gate)
|
|
61
|
-
t.check(
|
|
61
|
+
t.check(turn.message, similarity("Sunny, 72F")); // fuzzy 0–1 Levenshtein (soft)
|
|
62
62
|
t.check(
|
|
63
63
|
latencyMs,
|
|
64
64
|
satisfies((value) => value < 1_000, "latency under one second"),
|
|
@@ -114,7 +114,7 @@ const request = session.requireInputRequest({
|
|
|
114
114
|
|
|
115
115
|
## Run state and derived facts
|
|
116
116
|
|
|
117
|
-
Beyond the raw `
|
|
117
|
+
Beyond the raw `session.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output, lifecycle status), subagent calls, and HITL input requests. A turn that leaves the session open for a next message is the normal end state of a successful turn; parking on unanswered HITL input is tracked separately.
|
|
118
118
|
|
|
119
119
|
Typed event matching covers presence, absence, numeric or predicate counts, partial event data, and ordering:
|
|
120
120
|
|
|
@@ -166,8 +166,8 @@ The defaults are chosen so you rarely set severity. Run-level methods and `inclu
|
|
|
166
166
|
|
|
167
167
|
```ts
|
|
168
168
|
t.calledTool("get_weather").soft(); // record the tool call as a metric, don't gate
|
|
169
|
-
t.check(
|
|
170
|
-
t.check(
|
|
169
|
+
t.check(turn.message, similarity("Sunny")).label("forecast wording").atLeast(0.8);
|
|
170
|
+
t.check(turn.message, includes("error")).soft(); // track without failing the build
|
|
171
171
|
```
|
|
172
172
|
|
|
173
173
|
## What to read next
|
package/docs/evals/cases.mdx
CHANGED
|
@@ -15,7 +15,7 @@ export default defineEvalConfig({});
|
|
|
15
15
|
|
|
16
16
|
## Single-turn evals
|
|
17
17
|
|
|
18
|
-
The common case sends one turn and asserts on the reply. `t.send(input)` resolves once the turn settles, and `
|
|
18
|
+
The common case sends one turn and asserts on the reply. `t.send(input)` resolves once the turn settles, and the returned turn’s `.message` is the assistant reply:
|
|
19
19
|
|
|
20
20
|
```ts title="evals/weather/brooklyn-forecast.eval.ts"
|
|
21
21
|
import { defineEval } from "eve/evals";
|
|
@@ -23,9 +23,9 @@ import { includes } from "eve/evals/expect";
|
|
|
23
23
|
|
|
24
24
|
export default defineEval({
|
|
25
25
|
async test(t) {
|
|
26
|
-
await t.send("What is the weather in Brooklyn?");
|
|
26
|
+
const turn = await t.send("What is the weather in Brooklyn?");
|
|
27
27
|
t.succeeded();
|
|
28
|
-
t.check(
|
|
28
|
+
t.check(turn.message, includes("Sunny"));
|
|
29
29
|
},
|
|
30
30
|
});
|
|
31
31
|
```
|
|
@@ -60,7 +60,7 @@ evals/
|
|
|
60
60
|
|
|
61
61
|
## Multi-turn evals
|
|
62
62
|
|
|
63
|
-
Drive several turns in sequence for branching, HITL approvals, structured output, attachments, or multiple sessions. Because assertions live in the function, an intermediate value is a local variable.
|
|
63
|
+
Drive several turns in sequence for branching, HITL approvals, structured output, attachments, or multiple sessions. Because assertions live in the function, an intermediate value is a local variable. Keep each turn in a local variable and continue through its `.session`.
|
|
64
64
|
|
|
65
65
|
```ts title="evals/draft-then-send.eval.ts"
|
|
66
66
|
import { defineEval } from "eve/evals";
|
|
@@ -72,7 +72,7 @@ export default defineEval({
|
|
|
72
72
|
t.check(draft.message, includes("Best regards"));
|
|
73
73
|
t.judge.autoevals.closedQA("professional tone", { on: draft.message }).atLeast(0.6);
|
|
74
74
|
|
|
75
|
-
await
|
|
75
|
+
await draft.session.send("Now send it.");
|
|
76
76
|
t.calledTool("send_email");
|
|
77
77
|
},
|
|
78
78
|
});
|
|
@@ -88,7 +88,7 @@ export default defineEval({
|
|
|
88
88
|
async test(t) {
|
|
89
89
|
const first = await t.send("My favorite word is marigold.");
|
|
90
90
|
|
|
91
|
-
const second = await
|
|
91
|
+
const second = await first.session.send("What is my favorite word?");
|
|
92
92
|
await t.require(second.sessionId, equals(first.sessionId));
|
|
93
93
|
|
|
94
94
|
t.succeeded();
|
|
@@ -96,7 +96,7 @@ export default defineEval({
|
|
|
96
96
|
|
|
97
97
|
t.judge.autoevals
|
|
98
98
|
.closedQA("The assistant remembers the user's favorite word across turns", {
|
|
99
|
-
on:
|
|
99
|
+
on: first.session.transcript,
|
|
100
100
|
})
|
|
101
101
|
.atLeast(0.8);
|
|
102
102
|
},
|
|
@@ -105,21 +105,56 @@ export default defineEval({
|
|
|
105
105
|
|
|
106
106
|
## The drive API
|
|
107
107
|
|
|
108
|
-
`t`
|
|
108
|
+
Every `t.session()` or `t.send()` creates a new independent conversation. Continue an existing conversation through its session handle. Events from all sessions feed the same run-level assertions.
|
|
109
109
|
|
|
110
|
-
- `t.
|
|
111
|
-
- `t.
|
|
112
|
-
- `
|
|
113
|
-
- `
|
|
114
|
-
- `
|
|
115
|
-
- `
|
|
116
|
-
- `
|
|
117
|
-
- `
|
|
118
|
-
- `
|
|
110
|
+
- `t.session(options?)` creates a session without starting a turn or consuming its stream. It resolves on `202 Accepted` with an `EveEvalSession` whose `.sessionId` and `.state` are available. Options accept per-request `headers` and `signal`; creation uses the eval timeout signal by default.
|
|
111
|
+
- `t.send(message, options?)` creates a new session with its first message in one request and waits for the turn to settle. The returned turn carries `.session`, `.message`, and `.expectOk()`.
|
|
112
|
+
- `session.send(message, options?)` sends a follow-up on that session and waits for it to settle.
|
|
113
|
+
- `session.start(message, options?)` starts a text turn and returns as soon as the server accepts it. The live turn exposes `.session`, `.sessionId`, `.waitForEvent(...)`, `.cancel()`, and `.result()`.
|
|
114
|
+
- `session.cancel()` requests cooperative cancellation of that session's active turn. Both `accepted` and `no_active_turn` are successful outcomes.
|
|
115
|
+
- `session.sendFile(text, path, mediaType?)` attaches a local file as a data URL.
|
|
116
|
+
- `session.requireInputRequest(filter?)` records a gate, requires exactly one pending request, and returns it. Filters match tool name, action input, prompt, display, and option ids.
|
|
117
|
+
- `session.respond(responses, options?)` answers specific pending input requests and sends them as the next turn. `session.startRespond(...)` returns a live turn for the same operation.
|
|
118
|
+
- `session.respondAll(optionId)` answers every pending input request with the same option and sends the responses as the next turn.
|
|
119
|
+
- `session.events` contains the events captured for that session. `session.transcript` formats its observed user and assistant messages in turn order.
|
|
119
120
|
|
|
120
|
-
The transcript uses `User:` and `Assistant:` labels separated by blank lines. It excludes reasoning, tool calls, tool results, and messages from other sessions. It updates after each turn settles, so read it after `await
|
|
121
|
+
The transcript uses `User:` and `Assistant:` labels separated by blank lines. It excludes reasoning, tool calls, tool results, and messages from other sessions. It updates after each turn settles, so read it after `await session.send(...)`, `await session.respond(...)`, or `await live.result()`.
|
|
121
122
|
|
|
122
|
-
Each `send` (and `respond`/`respondAll`) resolves to
|
|
123
|
+
Each `send` (and `respond`/`respondAll`) resolves to a turn with `.message`, `.data`, `.events`, `.inputRequests`, `.toolCalls`, `.session`, `.sessionId`, `.status`, and `.expectOk()`. Turn results retain their captured output and events; `.session` references the live conversation handle. `expectOk()` throws only when the turn ended failed; a session left open for a next message is the normal end state of a successful turn.
|
|
124
|
+
|
|
125
|
+
## Prewarming a session
|
|
126
|
+
|
|
127
|
+
Use `t.session()` to create a session before its first message:
|
|
128
|
+
|
|
129
|
+
```ts title="evals/prewarm.eval.ts"
|
|
130
|
+
import { defineEval } from "eve/evals";
|
|
131
|
+
import { equals } from "eve/evals/expect";
|
|
132
|
+
|
|
133
|
+
export default defineEval({
|
|
134
|
+
async test(t) {
|
|
135
|
+
const session = await t.session();
|
|
136
|
+
const turn = await session.send("Greet Alice briefly.");
|
|
137
|
+
await t.require(turn.sessionId, equals(session.sessionId));
|
|
138
|
+
turn.event("session.started", { count: 1 });
|
|
139
|
+
turn.event("turn.started", { count: 1, data: { turnId: "turn_0" } });
|
|
140
|
+
t.succeeded();
|
|
141
|
+
},
|
|
142
|
+
});
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Acceptance is not an initialization barrier. The first message runs session initialization;
|
|
146
|
+
the TypeScript client handles bounded retries if the workflow inbox is still starting.
|
|
147
|
+
Creation failures reject `t.session()` without returning a handle. Each call creates an
|
|
148
|
+
independent session, including concurrent calls. Accepted sessions appear in eval results
|
|
149
|
+
and participate in timeout cleanup even when no message was sent.
|
|
150
|
+
|
|
151
|
+
## Migrating existing evals
|
|
152
|
+
|
|
153
|
+
Replace `t.newSession()` and `t.prewarm()` with `const session = await t.session()`.
|
|
154
|
+
Replace consecutive `t.send()` calls with `session.send()` calls, or continue the first
|
|
155
|
+
returned turn through `turn.session.send()`. Read identity, cursor, events, and transcript
|
|
156
|
+
from `session`; read replies from `turn.message`. Session operations such as `respond()`,
|
|
157
|
+
`requireInputRequest()`, and `cancel()` also move from `t` to `session`.
|
|
123
158
|
|
|
124
159
|
## In-flight turns
|
|
125
160
|
|
|
@@ -130,7 +165,8 @@ import { defineEval } from "eve/evals";
|
|
|
130
165
|
|
|
131
166
|
export default defineEval({
|
|
132
167
|
async test(t) {
|
|
133
|
-
const
|
|
168
|
+
const session = await t.session();
|
|
169
|
+
const live = await session.start("Run the long operation.");
|
|
134
170
|
|
|
135
171
|
await live.waitForEvent("actions.requested", {
|
|
136
172
|
data: {
|
|
@@ -152,7 +188,7 @@ export default defineEval({
|
|
|
152
188
|
|
|
153
189
|
Use `expectOk()` only when the next operation depends on that intermediate turn succeeding. A final `t.succeeded()` already records a complete-run gate.
|
|
154
190
|
|
|
155
|
-
To intentionally omit an eval for the current target, call `t.skip(reason)` before sending messages or recording assertions. Skipped evals are reported separately and do not affect the exit code.
|
|
191
|
+
To intentionally omit an eval for the current target, call `t.skip(reason)` before creating sessions, sending messages, or recording assertions. Skipped evals are reported separately and do not affect the exit code.
|
|
156
192
|
|
|
157
193
|
Events from every session are captured in the result and artifacts. `t.log(message)` records debug lines into the eval artifact; `--verbose` also streams them to stdout as evals run. `t.signal` is an `AbortSignal` that fires on timeout.
|
|
158
194
|
|
|
@@ -174,9 +210,9 @@ export default rows.map((row) =>
|
|
|
174
210
|
defineEval({
|
|
175
211
|
description: row.task,
|
|
176
212
|
async test(t) {
|
|
177
|
-
await t.send(row.prompt);
|
|
213
|
+
const turn = await t.send(row.prompt);
|
|
178
214
|
t.succeeded();
|
|
179
|
-
t.check(
|
|
215
|
+
t.check(turn.message, equals(row.sql));
|
|
180
216
|
},
|
|
181
217
|
}),
|
|
182
218
|
);
|
package/docs/evals/judge.mdx
CHANGED
|
@@ -19,7 +19,7 @@ export default defineEval({
|
|
|
19
19
|
|
|
20
20
|
## The graders
|
|
21
21
|
|
|
22
|
-
The judges live under `t.judge.autoevals`. The namespace names the [Braintrust autoevals](https://github.com/braintrustdata/autoevals) grader family, so the factuality and closedQA semantics are autoevals', not eve-invented. Each grader scores
|
|
22
|
+
The judges live under `t.judge.autoevals`. The namespace names the [Braintrust autoevals](https://github.com/braintrustdata/autoevals) grader family, so the factuality and closedQA semantics are autoevals', not eve-invented. Each grader scores the most recently settled turn’s assistant message by default and is soft by default (tracked, no gate):
|
|
23
23
|
|
|
24
24
|
| Grader | Grades |
|
|
25
25
|
| ---------------------------------------- | -------------------------------------------------------------------------------------- |
|
|
@@ -28,6 +28,8 @@ The judges live under `t.judge.autoevals`. The namespace names the [Braintrust a
|
|
|
28
28
|
| `t.judge.autoevals.closedQA(criteria)` | Whether the reply satisfies a free-form yes/no criterion (no expected answer to match) |
|
|
29
29
|
| `t.judge.autoevals.sql(expected)` | Semantic equivalence of two SQL statements |
|
|
30
30
|
|
|
31
|
+
Judges receive the latest message text from the session whose turn most recently settled as the input prompt. Approval responses and stream reads preserve that prompt. For multipart messages, including `session.sendFile()`, the prompt contains the text parts.
|
|
32
|
+
|
|
31
33
|
A failed `closedQA` assertion prints the input and judge explanation:
|
|
32
34
|
|
|
33
35
|
```text
|
|
@@ -40,7 +42,7 @@ A failed `closedQA` assertion prints the input and judge explanation:
|
|
|
40
42
|
|
|
41
43
|
The reference or criteria is the positional argument. An options object follows:
|
|
42
44
|
|
|
43
|
-
- `on` is the value to grade, defaulting to
|
|
45
|
+
- `on` is the value to grade, defaulting to the most recently settled turn’s assistant message. Pass an intermediate draft or parsed value to grade it instead.
|
|
44
46
|
- `model` and `modelOptions` are a per-call judge override (see below).
|
|
45
47
|
|
|
46
48
|
```ts
|
|
@@ -48,20 +50,20 @@ const draft = await t.send("Draft the welcome email.");
|
|
|
48
50
|
t.judge.autoevals.closedQA("professional tone", { on: draft.message }).atLeast(0.6);
|
|
49
51
|
```
|
|
50
52
|
|
|
51
|
-
For a multi-turn eval, pass `
|
|
53
|
+
For a multi-turn eval, pass `session.transcript` to grade a session's complete observed conversation instead of only its final reply:
|
|
52
54
|
|
|
53
55
|
```ts
|
|
54
|
-
await t.send("My favorite word is marigold. Remember it.");
|
|
55
|
-
await
|
|
56
|
+
const { session } = await t.send("My favorite word is marigold. Remember it.");
|
|
57
|
+
await session.send("What is my favorite word?");
|
|
56
58
|
|
|
57
59
|
t.judge.autoevals
|
|
58
60
|
.closedQA("The assistant remembers the user's favorite word across turns", {
|
|
59
|
-
on:
|
|
61
|
+
on: session.transcript,
|
|
60
62
|
})
|
|
61
63
|
.atLeast(0.8);
|
|
62
64
|
```
|
|
63
65
|
|
|
64
|
-
`
|
|
66
|
+
`session.transcript` contains the session's user and assistant messages in turn order. It excludes reasoning, tool calls, and tool results. See [Multi-turn evals](./cases#multi-turn-evals) for the transcript format and independent sessions.
|
|
65
67
|
|
|
66
68
|
## Soft scoring and thresholds
|
|
67
69
|
|
|
@@ -91,7 +93,7 @@ The judge model is resolved once when the runner builds `t`. It is **never** the
|
|
|
91
93
|
import { defineEvalConfig } from "eve/evals";
|
|
92
94
|
|
|
93
95
|
export default defineEvalConfig({
|
|
94
|
-
judge: { model: "openai/gpt-5.
|
|
96
|
+
judge: { model: "openai/gpt-5.6-luna" }, // the default judge for every eval in this tree
|
|
95
97
|
});
|
|
96
98
|
```
|
|
97
99
|
|
package/docs/evals/overview.mdx
CHANGED
|
@@ -32,10 +32,10 @@ import { includes } from "eve/evals/expect";
|
|
|
32
32
|
export default defineEval({
|
|
33
33
|
description: "Basic message and tool-usage coverage for the weather agent.",
|
|
34
34
|
async test(t) {
|
|
35
|
-
await t.send("What is the weather in Brooklyn?");
|
|
35
|
+
const turn = await t.send("What is the weather in Brooklyn?");
|
|
36
36
|
t.succeeded();
|
|
37
37
|
t.calledTool("get_weather");
|
|
38
|
-
t.check(
|
|
38
|
+
t.check(turn.message, includes("Sunny"));
|
|
39
39
|
},
|
|
40
40
|
});
|
|
41
41
|
```
|
|
@@ -51,7 +51,7 @@ import { defineEvalConfig } from "eve/evals";
|
|
|
51
51
|
import { Braintrust } from "eve/evals/reporters";
|
|
52
52
|
|
|
53
53
|
export default defineEvalConfig({
|
|
54
|
-
judge: { model: "openai/gpt-5.
|
|
54
|
+
judge: { model: "openai/gpt-5.6-luna" },
|
|
55
55
|
reporters: [Braintrust({ projectName: "my-agent" })],
|
|
56
56
|
});
|
|
57
57
|
```
|
|
@@ -100,7 +100,7 @@ model: mockModel({
|
|
|
100
100
|
|
|
101
101
|
`t` is both the driver and the assertion surface. There are no separate `input`, `run`, `checks`, or `scores` fields. You write ordinary control flow, sending turns and asserting inline.
|
|
102
102
|
|
|
103
|
-
- **Drive** the agent: `t.
|
|
103
|
+
- **Drive** the agent: `t.session()` creates an empty session; `t.send(...)` creates a session with its first message and returns a turn. Continue through `turn.session.send(...)`. Session handles also expose `start`, `cancel`, `respond`, `respondAll`, `sendFile`, and `requireInputRequest`. Read output from `turn.message`, and conversation state from `turn.session`. See [Cases](./cases).
|
|
104
104
|
- **Assert** with three surfaces, covered next.
|
|
105
105
|
|
|
106
106
|
## Three assertion surfaces
|
|
@@ -108,8 +108,8 @@ model: mockModel({
|
|
|
108
108
|
Each surface matches a genuinely different kind of judgment:
|
|
109
109
|
|
|
110
110
|
- **Scoped methods** read the final whole run on `t`, snapshot one independent session when invoked there, or inspect one immutable `EveEvalTurn`. See [Assertions](./assertions).
|
|
111
|
-
- **`t.check(value, assertion)`** grades an explicit value with a deterministic builder from `eve/evals/expect`, such as `t.check(
|
|
112
|
-
- **`t.judge.autoevals.*`** is the LLM-as-judge surface, like `t.judge.autoevals.closedQA("cites a source")`. It grades
|
|
111
|
+
- **`t.check(value, assertion)`** grades an explicit value with a deterministic builder from `eve/evals/expect`, such as `t.check(turn.message, includes("sunny"))`. Grade `turn.message`, an intermediate draft, parsed JSON, or anything else. See [Assertions](./assertions).
|
|
112
|
+
- **`t.judge.autoevals.*`** is the LLM-as-judge surface, like `t.judge.autoevals.closedQA("cites a source")`. It grades the most recently settled turn’s assistant message by default; pass `{ on: turn.session.transcript }` to grade a multi-turn conversation. The judge uses the configured judge model, never the agent under test. See [Judge](./judge).
|
|
113
113
|
|
|
114
114
|
## Gate vs soft
|
|
115
115
|
|
package/docs/evals/reporters.mdx
CHANGED
|
@@ -18,7 +18,7 @@ import { defineEvalConfig } from "eve/evals";
|
|
|
18
18
|
import { Braintrust } from "eve/evals/reporters";
|
|
19
19
|
|
|
20
20
|
export default defineEvalConfig({
|
|
21
|
-
judge: { model: "openai/gpt-5.
|
|
21
|
+
judge: { model: "openai/gpt-5.6-luna" },
|
|
22
22
|
reporters: [Braintrust({ projectName: "weather-agent" })],
|
|
23
23
|
});
|
|
24
24
|
```
|
package/docs/evals/targets.mdx
CHANGED
|
@@ -29,6 +29,11 @@ export default defineEval({
|
|
|
29
29
|
|
|
30
30
|
Sessions attached this way are full `EveEvalSession`s: you can keep driving them and assert directly on that session (`session.succeeded()`, `session.calledTool(...)`). Aggregate assertions on `t` continue to read the whole run, including every attached session.
|
|
31
31
|
|
|
32
|
+
For session creation and message delivery, use the [drive API](./cases#the-drive-api).
|
|
33
|
+
`t.session()` creates a session without a first message; call `session.send()` or `session.start()`
|
|
34
|
+
on the returned handle to use it. `t.send()` creates a new session with its first message.
|
|
35
|
+
These helpers handle the session protocol, retries, and event collection for the eval.
|
|
36
|
+
|
|
32
37
|
## Authentication
|
|
33
38
|
|
|
34
39
|
Local targets send no auth: `eve eval` owns the dev server it boots. For a remote `--url`, eve gets the expected Vercel owner and project from `VERCEL_ORG_ID` and `VERCEL_PROJECT_ID` when both are set. Otherwise it reads `.vercel/project.json`. Eve then asks Vercel to resolve the exact HTTPS origin and sends ambient credentials only when the project IDs match. An arbitrary URL remains anonymous.
|
|
@@ -36,11 +36,11 @@ console.log(result.status, result.message);
|
|
|
36
36
|
|
|
37
37
|
When the stream includes `session.failed`, the turn returns `status: "failed"` rather than throwing. Transport and route errors throw `ClientError`.
|
|
38
38
|
|
|
39
|
-
`session.send()` retries `409
|
|
40
|
-
but its command inbox is still starting. The
|
|
41
|
-
errors, `session.respond()`, control methods, and raw HTTP
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
`session.send()` retries `409 session_not_ready` with bounded exponential backoff for up to 20
|
|
40
|
+
seconds when a durable run has been accepted but its command inbox is still starting. The caller's
|
|
41
|
+
abort signal cancels the wait. Other errors, `session.respond()`, control methods, and raw HTTP
|
|
42
|
+
requests do not use this retry. An unknown
|
|
43
|
+
or terminal session throws `ClientError` immediately with `session_not_active`. The client never creates a replacement session.
|
|
44
44
|
|
|
45
45
|
## Send a full turn payload
|
|
46
46
|
|
|
@@ -113,7 +113,21 @@ Per-request headers override client-level values with the same name. For example
|
|
|
113
113
|
|
|
114
114
|
## Sessions
|
|
115
115
|
|
|
116
|
-
|
|
116
|
+
Create and park an ID-addressed conversation before its first turn when you can start session work
|
|
117
|
+
while the user is still composing:
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
const { session } = await client.sessions.create();
|
|
121
|
+
|
|
122
|
+
const response = await session.send("Summarize account A.");
|
|
123
|
+
await response.result();
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The create promise resolves when the server accepts the durable workflow. It does not open the
|
|
127
|
+
event stream or wait for initialization; an early `session.send()` uses the
|
|
128
|
+
same bounded startup retries as other follow-ups.
|
|
129
|
+
|
|
130
|
+
To create the session and start its first turn in one request, include the message:
|
|
117
131
|
|
|
118
132
|
```ts
|
|
119
133
|
const { session, response } = await client.sessions.create({ message: "Summarize account A." });
|
|
@@ -265,6 +265,12 @@ Write callback properties as inline function expressions, arrows, method shortha
|
|
|
265
265
|
|
|
266
266
|
Closure values must be JSON-serializable. Plain objects, arrays, strings, finite numbers, booleans, and `null` are supported; `undefined` object properties are omitted. Functions, class instances, `Date`, `Map`, symbols, non-finite numbers, and cyclic values fail resolution with the tool name and callback phase instead of being serialized lossily.
|
|
267
267
|
|
|
268
|
+
Dynamic tools preserve authored input and output validation, including Zod refinements and transformations. JSON Schema describes the tool to the model; the authored validator checks its input and output. eve transforms inline `inputSchema` and `outputSchema` expressions into schema factories and snapshots the JSON-serializable values they capture, just as it does for callbacks.
|
|
269
|
+
|
|
270
|
+
Write schemas inline in `defineTool()`, or reference a stable module-level schema. A schema stored in a resolver-local variable is a non-serializable capture; inline its construction instead. Schema factories must be synchronous and deterministic for their captured values. Keep external reads in the resolver and capture the resulting JSON data. Imported functions and module-level values remain live code, so they must not hide per-session state.
|
|
271
|
+
|
|
272
|
+
Schema factories use the same session, scope, resolver entry, and recovery rules as the tool's callbacks. A recovered factory receives its original captures, even if re-running the resolver produces different values. If a required factory is missing, validation fails explicitly; eve does not substitute its JSON Schema description.
|
|
273
|
+
|
|
268
274
|
Call expressions such as `execute: makeExecutor()` are not transformed. Put the callback body directly in `defineTool()` inside an authored module; eve rejects a dynamic tool if a callback lacks durable metadata.
|
|
269
275
|
|
|
270
276
|
### Create dynamic tools in a package
|
|
@@ -298,6 +304,22 @@ export function createSearchTool(baseUrl: string) {
|
|
|
298
304
|
|
|
299
305
|
Wrap every callback property with the helper. This includes labels, approval policies, `approvalKey`, `execute`, and `toModelOutput`.
|
|
300
306
|
|
|
307
|
+
For live schemas created in a provider package, use `defineDurableSchema` from `eve/tools`. Put the schema's per-tool values in `closure` and construct the schema in `schema`. Plain JSON Schema objects need no helper.
|
|
308
|
+
|
|
309
|
+
```ts
|
|
310
|
+
import { defineDurableSchema } from "eve/tools";
|
|
311
|
+
import { z } from "zod";
|
|
312
|
+
|
|
313
|
+
export function amountSchema(limit: number) {
|
|
314
|
+
return defineDurableSchema({
|
|
315
|
+
closure: { limit },
|
|
316
|
+
schema: ({ limit }) => z.object({ amount: z.number().refine((amount) => amount <= limit) }),
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
Pass the result as `inputSchema` or `outputSchema` in `defineTool()`. Rebuild existing extensions with the current `eve extension build` to generate schema factories. A live dynamic schema without a durable factory is rejected at resolution with instructions to inline its construction or use `defineDurableSchema`.
|
|
322
|
+
|
|
301
323
|
`closure` is the callback's only durable snapshot. Store the identifiers and configuration needed to reproduce the call there. Reconstruct clients or look up live runtime state when the callback runs. The callback may call stable imported functions, but it must not capture runtime objects outside `closure`. Those values disappear on a cold start.
|
|
302
324
|
|
|
303
325
|
eve-provided factories, including [memory provider tools](../memory), use the same durable callback mechanism.
|