eve 0.13.8 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/src/channel/compiled-channel.js +2 -1
- package/dist/src/channel/routes.d.ts +5 -0
- package/dist/src/channel/send.js +1 -1
- package/dist/src/channel/types.d.ts +5 -0
- package/dist/src/cli/dev/tui/runner.js +1 -1
- package/dist/src/cli/dev/tui/setup-panel.d.ts +3 -4
- package/dist/src/cli/dev/tui/setup-panel.js +2 -2
- package/dist/src/cli/dev/tui/terminal-renderer.js +1 -1
- package/dist/src/compiled/.vendor-stamp.json +12 -8
- package/dist/src/compiled/@ai-sdk/anthropic/index.d.ts +2 -2
- package/dist/src/compiled/@ai-sdk/anthropic/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/google/index.d.ts +25 -3
- package/dist/src/compiled/@ai-sdk/google/index.js +6 -6
- package/dist/src/compiled/@ai-sdk/mcp/index.d.ts +647 -30
- package/dist/src/compiled/@ai-sdk/mcp/index.js +1 -1
- package/dist/src/compiled/@ai-sdk/openai/index.d.ts +8 -2
- package/dist/src/compiled/@ai-sdk/openai/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/otel/index.d.ts +165 -28
- package/dist/src/compiled/@ai-sdk/otel/index.js +3 -3
- package/dist/src/compiled/@ai-sdk/provider/index.d.ts +10 -2
- package/dist/src/compiled/@ai-sdk/provider-utils/index.d.ts +2362 -0
- package/dist/src/compiled/@ai-sdk/provider-utils/index.js +1 -0
- package/dist/src/compiled/@workflow/serde/LICENSE.md +3 -0
- package/dist/src/compiled/@workflow/serde/index.d.ts +30 -0
- package/dist/src/compiled/@workflow/serde/index.js +1 -0
- package/dist/src/compiled/_chunks/workflow/{dist-C9PV_vnE.js → dist-D7CzPkf8.js} +1 -1
- package/dist/src/compiled/eventsource-parser/stream/LICENSE +21 -0
- package/dist/src/compiled/eventsource-parser/stream/index.d.ts +121 -0
- package/dist/src/compiled/eventsource-parser/stream/index.js +1 -0
- package/dist/src/compiled/experimental-ai-sdk-code-mode/approval-continuation.d.ts +7 -7
- package/dist/src/compiled/experimental-ai-sdk-code-mode/continuation-capability.d.ts +15 -4
- package/dist/src/compiled/experimental-ai-sdk-code-mode/index.d.ts +1 -1
- package/dist/src/compiled/experimental-ai-sdk-code-mode/index.js +15 -15
- package/dist/src/compiled/experimental-ai-sdk-code-mode/interrupt-continuation.d.ts +5 -5
- package/dist/src/compiled/experimental-ai-sdk-code-mode/runtime/protocol.d.ts +13 -2
- package/dist/src/compiled/experimental-ai-sdk-code-mode/types.d.ts +26 -0
- package/dist/src/compiled/json-schema/LICENSE +21 -0
- package/dist/src/compiled/json-schema/index.d.ts +749 -0
- package/dist/src/compiled/json-schema/index.js +1 -0
- package/dist/src/compiler/manifest.js +1 -1
- package/dist/src/compiler/normalize-agent-config.js +1 -1
- package/dist/src/context/build-dynamic-tools.js +1 -1
- package/dist/src/context/dynamic-tool-lifecycle.js +1 -1
- package/dist/src/context/keys.d.ts +1 -1
- package/dist/src/evals/assertions/collector.d.ts +19 -6
- package/dist/src/evals/assertions/collector.js +1 -1
- package/dist/src/evals/assertions/run.d.ts +24 -17
- package/dist/src/evals/assertions/run.js +2 -2
- package/dist/src/evals/assertions/scoped.d.ts +17 -0
- package/dist/src/evals/assertions/scoped.js +1 -0
- package/dist/src/evals/context.d.ts +1 -0
- package/dist/src/evals/context.js +1 -1
- package/dist/src/evals/control-flow.d.ts +9 -0
- package/dist/src/evals/control-flow.js +1 -0
- package/dist/src/evals/define-eval.d.ts +1 -1
- package/dist/src/evals/define-eval.js +1 -1
- package/dist/src/evals/expect/index.d.ts +5 -3
- package/dist/src/evals/expect/index.js +1 -1
- package/dist/src/evals/index.d.ts +2 -2
- package/dist/src/evals/match.d.ts +50 -13
- package/dist/src/evals/match.js +1 -1
- package/dist/src/evals/runner/artifacts.js +1 -1
- package/dist/src/evals/runner/derive-run-facts.js +1 -1
- package/dist/src/evals/runner/execute-eval.js +1 -1
- package/dist/src/evals/runner/execute-task.d.ts +1 -0
- package/dist/src/evals/runner/execute-task.js +1 -1
- package/dist/src/evals/runner/reporters/braintrust.js +1 -1
- package/dist/src/evals/runner/reporters/console.js +1 -1
- package/dist/src/evals/runner/reporters/junit.js +3 -2
- package/dist/src/evals/runner/run-evals.js +1 -1
- package/dist/src/evals/runner/verdict.d.ts +1 -0
- package/dist/src/evals/runner/verdict.js +1 -1
- package/dist/src/evals/session.d.ts +9 -5
- package/dist/src/evals/session.js +1 -1
- package/dist/src/evals/types.d.ts +69 -47
- package/dist/src/execution/dispatch-runtime-actions-step.js +1 -1
- package/dist/src/execution/dispatch-workflow-runtime-actions-step.d.ts +12 -0
- package/dist/src/execution/dispatch-workflow-runtime-actions-step.js +1 -0
- package/dist/src/execution/next-driver-action.d.ts +1 -1
- package/dist/src/execution/node-step.js +1 -1
- package/dist/src/execution/session.js +3 -3
- package/dist/src/execution/turn-workflow.js +1 -1
- package/dist/src/execution/workflow-entry.js +1 -1
- package/dist/src/execution/workflow-runtime.d.ts +2 -11
- package/dist/src/execution/workflow-runtime.js +1 -1
- package/dist/src/execution/workflow-steps.d.ts +1 -1
- package/dist/src/execution/workflow-steps.js +1 -1
- package/dist/src/harness/action-result-helpers.d.ts +2 -2
- package/dist/src/harness/emission.d.ts +13 -27
- package/dist/src/harness/emission.js +1 -1
- package/dist/src/harness/execute-tool.d.ts +2 -2
- package/dist/src/harness/input-extraction.js +1 -1
- package/dist/src/harness/step-hooks.d.ts +3 -3
- package/dist/src/harness/step-hooks.js +1 -1
- package/dist/src/harness/tool-loop.js +1 -1
- package/dist/src/harness/tools.d.ts +4 -6
- package/dist/src/harness/tools.js +1 -1
- package/dist/src/harness/types.d.ts +4 -10
- package/dist/src/harness/workflow-continuation-security.d.ts +4 -0
- package/dist/src/harness/workflow-continuation-security.js +1 -0
- package/dist/src/harness/workflow-interrupt-state.d.ts +14 -0
- package/dist/src/harness/workflow-interrupt-state.js +1 -0
- package/dist/src/harness/workflow-lifecycle.d.ts +13 -0
- package/dist/src/harness/workflow-lifecycle.js +1 -0
- package/dist/src/harness/workflow-runtime-action-state.d.ts +9 -0
- package/dist/src/harness/workflow-runtime-action-state.js +1 -0
- package/dist/src/harness/workflow-sandbox.d.ts +22 -0
- package/dist/src/harness/workflow-sandbox.js +2 -0
- package/dist/src/harness/workflow-tool-description.d.ts +2 -2
- package/dist/src/harness/workflow-tool-description.js +16 -4
- package/dist/src/internal/application/compiled-artifacts.js +1 -1
- package/dist/src/internal/application/package.d.ts +12 -0
- package/dist/src/internal/application/package.js +1 -1
- package/dist/src/internal/application/paths.d.ts +6 -0
- package/dist/src/internal/application/paths.js +1 -1
- package/dist/src/internal/authored-definition/connection.js +1 -1
- package/dist/src/internal/authored-definition/core.js +1 -1
- package/dist/src/internal/authored-definition/schema-backed.js +1 -1
- package/dist/src/internal/nitro/host/configure-nitro-routes.js +2 -2
- package/dist/src/internal/nitro/host/create-application-nitro.js +1 -1
- package/dist/src/internal/nitro/host/workflow-sandbox-runtime-plugin.d.ts +1 -0
- package/dist/src/internal/nitro/host/workflow-sandbox-runtime-plugin.js +1 -0
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response-from-manifest.js +1 -1
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response.js +1 -1
- package/dist/src/internal/workflow/configure-world.d.ts +6 -0
- package/dist/src/internal/workflow/configure-world.js +1 -1
- package/dist/src/internal/workflow/runtime.d.ts +1 -0
- package/dist/src/internal/workflow/world-compatibility.d.ts +32 -0
- package/dist/src/internal/workflow/world-compatibility.js +1 -0
- package/dist/src/protocol/message.d.ts +7 -5
- package/dist/src/public/channels/slack/attachments.js +1 -1
- package/dist/src/public/channels/slack/auth.d.ts +2 -0
- package/dist/src/public/channels/slack/auth.js +1 -1
- package/dist/src/public/channels/slack/defaults.js +2 -2
- package/dist/src/public/channels/slack/inbound.d.ts +4 -11
- package/dist/src/public/channels/slack/inbound.js +1 -2
- package/dist/src/public/channels/slack/model-context.d.ts +28 -0
- package/dist/src/public/channels/slack/model-context.js +3 -0
- package/dist/src/public/channels/slack/slackChannel.d.ts +8 -0
- package/dist/src/public/channels/slack/slackChannel.js +1 -1
- package/dist/src/public/connections/index.d.ts +1 -1
- package/dist/src/public/definitions/agent.d.ts +1 -1
- package/dist/src/public/definitions/approval.d.ts +40 -0
- package/dist/src/public/definitions/approval.js +1 -0
- package/dist/src/public/definitions/connections/mcp.d.ts +11 -9
- package/dist/src/public/definitions/connections/mcp.js +1 -1
- package/dist/src/public/definitions/connections/openapi.d.ts +9 -7
- package/dist/src/public/definitions/connections/openapi.js +1 -1
- package/dist/src/public/definitions/tool.d.ts +7 -20
- package/dist/src/public/index.d.ts +1 -1
- package/dist/src/public/tools/approval/approval-helpers.d.ts +7 -7
- package/dist/src/public/tools/approval/approval-helpers.js +1 -1
- package/dist/src/public/tools/approval/index.d.ts +1 -1
- package/dist/src/public/tools/index.d.ts +2 -1
- package/dist/src/public/tools/internal.js +1 -1
- package/dist/src/runtime/actions/types.d.ts +10 -11
- package/dist/src/runtime/agent/bootstrap.d.ts +1 -0
- package/dist/src/runtime/agent/bootstrap.js +1 -1
- package/dist/src/runtime/agent/mock-model-adapter.js +2 -3
- package/dist/src/runtime/connections/mcp-client.d.ts +3 -3
- package/dist/src/runtime/connections/mcp-client.js +1 -1
- package/dist/src/runtime/connections/registry.d.ts +2 -2
- package/dist/src/runtime/connections/resolve-authorization.d.ts +11 -0
- package/dist/src/runtime/connections/resolve-authorization.js +1 -0
- package/dist/src/runtime/connections/types.d.ts +27 -11
- package/dist/src/runtime/framework-tools/connection-search-dynamic.js +1 -1
- package/dist/src/runtime/resolve-agent-graph.js +1 -1
- package/dist/src/runtime/resolve-agent.js +1 -1
- package/dist/src/runtime/resolve-connection.js +1 -1
- package/dist/src/runtime/resolve-tool.d.ts +1 -1
- package/dist/src/runtime/resolve-tool.js +1 -1
- package/dist/src/runtime/types.d.ts +8 -7
- package/dist/src/setup/boxes/resolve-provisioning.js +1 -1
- package/dist/src/setup/primitives/pm/pnpm.js +6 -5
- package/dist/src/setup/scaffold/create/add-to-project.js +1 -1
- package/dist/src/setup/scaffold/create/project.js +2 -2
- package/dist/src/setup/scaffold/update/channels.js +1 -1
- package/dist/src/shared/agent-definition.d.ts +11 -13
- package/dist/src/shared/dynamic-tool-definition.d.ts +3 -3
- package/dist/src/shared/tool-definition.d.ts +1 -2
- package/dist/src/shared/workflow-sandbox.d.ts +37 -0
- package/dist/src/shared/workflow-sandbox.js +1 -0
- package/docs/agent-config.md +43 -13
- package/docs/channels/custom.mdx +2 -2
- package/docs/channels/slack.mdx +7 -13
- package/docs/concepts/execution-model-and-durability.md +1 -1
- package/docs/concepts/sessions-runs-and-streaming.md +1 -1
- package/docs/connections/mcp.mdx +2 -2
- package/docs/connections/openapi.mdx +37 -2
- package/docs/connections/overview.mdx +72 -2
- package/docs/evals/assertions.mdx +97 -39
- package/docs/evals/cases.mdx +24 -15
- package/docs/evals/judge.mdx +2 -2
- package/docs/evals/overview.mdx +7 -5
- package/docs/evals/reporters.mdx +2 -2
- package/docs/evals/running.mdx +3 -1
- package/docs/evals/targets.mdx +2 -2
- package/docs/guides/auth-and-route-protection.md +48 -1
- package/docs/guides/client/streaming.mdx +1 -1
- package/docs/guides/deployment.md +5 -4
- package/docs/guides/dynamic-workflows.md +1 -7
- package/docs/guides/frontend/overview.mdx +1 -1
- package/docs/guides/frontend/use-eve-agent-svelte.mdx +1 -1
- package/docs/guides/frontend/use-eve-agent-vue.mdx +1 -1
- package/docs/guides/session-context.md +2 -1
- package/docs/reference/typescript-api.md +2 -2
- package/docs/subagents.mdx +2 -2
- package/docs/tools/human-in-the-loop.md +43 -6
- package/docs/tools/overview.mdx +3 -3
- package/docs/tutorial/connect-a-warehouse.mdx +2 -0
- package/docs/tutorial/first-agent.mdx +1 -1
- package/docs/tutorial/guard-the-spend.mdx +4 -3
- package/docs/tutorial/ship-it.mdx +1 -1
- package/package.json +14 -11
- package/dist/src/compiled/@ai-sdk/anthropic/_provider-utils.d.ts +0 -15
- package/dist/src/compiled/@ai-sdk/google/_provider-utils.d.ts +0 -11
- package/dist/src/compiled/@ai-sdk/openai/_provider-utils.d.ts +0 -15
- package/dist/src/compiled/@ai-sdk/provider/_json-schema.d.ts +0 -5
- package/dist/src/execution/dispatch-code-mode-runtime-actions-step.d.ts +0 -21
- package/dist/src/execution/dispatch-code-mode-runtime-actions-step.js +0 -1
- package/dist/src/harness/code-mode-interrupt-state.d.ts +0 -26
- package/dist/src/harness/code-mode-interrupt-state.js +0 -1
- package/dist/src/harness/code-mode-lifecycle.d.ts +0 -16
- package/dist/src/harness/code-mode-lifecycle.js +0 -1
- package/dist/src/harness/code-mode-runtime-action-state.d.ts +0 -6
- package/dist/src/harness/code-mode-runtime-action-state.js +0 -1
- package/dist/src/harness/code-mode.d.ts +0 -26
- package/dist/src/harness/code-mode.js +0 -1
- package/dist/src/harness/sandbox-surface.d.ts +0 -68
- package/dist/src/harness/sandbox-surface.js +0 -1
- package/dist/src/internal/nitro/host/code-mode-runtime-dependency-plugin.d.ts +0 -1
- package/dist/src/internal/nitro/host/code-mode-runtime-dependency-plugin.js +0 -1
- package/dist/src/runtime/framework-tools/code-mode-connection-auth.d.ts +0 -29
- package/dist/src/runtime/framework-tools/code-mode-connection-auth.js +0 -1
- package/dist/src/shared/code-mode.d.ts +0 -29
- package/dist/src/shared/code-mode.js +0 -1
package/docs/agent-config.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "agent.ts"
|
|
3
|
-
description: "Set the agent's runtime config in agent.ts with defineAgent, including the model and compaction."
|
|
3
|
+
description: "Set the agent's runtime config in agent.ts with defineAgent, including the model, reasoning effort, and compaction."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
An agent's `agent.ts` calls `defineAgent` (from `eve`) to set its runtime config.
|
|
@@ -33,12 +33,33 @@ import { anthropic } from "@ai-sdk/anthropic";
|
|
|
33
33
|
import { defineAgent } from "eve";
|
|
34
34
|
|
|
35
35
|
export default defineAgent({
|
|
36
|
-
model: anthropic("claude-opus-4
|
|
36
|
+
model: anthropic("claude-opus-4-8"),
|
|
37
37
|
});
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
Direct provider model ids use the provider's native format. For Anthropic, the
|
|
41
|
+
version uses hyphens (`claude-opus-4-8`), while the Gateway id above uses a dot
|
|
42
|
+
(`anthropic/claude-opus-4.8`).
|
|
43
|
+
|
|
40
44
|
Model use is subject to the terms, data-processing commitments, retention behavior, and available controls of the selected provider and routing path. Review the [AI Gateway model catalog](https://vercel.com/ai-gateway/models) for gateway-routed models, and review the provider's terms when you configure a direct `LanguageModel`.
|
|
41
45
|
|
|
46
|
+
## Reasoning effort
|
|
47
|
+
|
|
48
|
+
Set `reasoning` to control the model's reasoning effort through AI SDK's
|
|
49
|
+
provider-agnostic option:
|
|
50
|
+
|
|
51
|
+
```ts title="agent/agent.ts"
|
|
52
|
+
export default defineAgent({
|
|
53
|
+
model: "openai/gpt-5.5",
|
|
54
|
+
reasoning: "high",
|
|
55
|
+
});
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Supported values are `"provider-default"`, `"none"`, `"minimal"`, `"low"`,
|
|
59
|
+
`"medium"`, `"high"`, and `"xhigh"`. The selected model and provider determine
|
|
60
|
+
which levels are available and how they map to provider-native settings. Use
|
|
61
|
+
`modelOptions.providerOptions` when you need provider-specific reasoning controls.
|
|
62
|
+
|
|
42
63
|
## Compaction
|
|
43
64
|
|
|
44
65
|
Compaction summarizes older turns as you approach the context window. It's on by default, so you only tune when it kicks in. Lower `thresholdPercent` to compact sooner:
|
|
@@ -75,24 +96,33 @@ export default defineAgent({
|
|
|
75
96
|
```
|
|
76
97
|
|
|
77
98
|
Install that package in your app. It should export a default factory or
|
|
78
|
-
`createWorld()` function.
|
|
99
|
+
`createWorld()` function. Pin a version built against the same `@workflow/*`
|
|
100
|
+
line as your eve release (currently the `5.0.0-beta` line):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
pnpm add @workflow/world-postgres@5.0.0-beta.x
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The npm `latest` tag can lag behind that line, so an unpinned install may pull
|
|
107
|
+
an incompatible major that fails with `ZodError: invalid_union` at run replay.
|
|
79
108
|
|
|
80
109
|
Put credentials and host-specific options in runtime environment variables read
|
|
81
|
-
by the world package, not in `agent.ts`.
|
|
82
|
-
|
|
110
|
+
by the world package, not in `agent.ts`. For the Postgres world, that means
|
|
111
|
+
putting the connection string or credentials in the env vars it reads. If the
|
|
112
|
+
installed package must stay external in hosted output, list it in
|
|
113
|
+
`build.externalDependencies`.
|
|
83
114
|
|
|
84
115
|
## Other defineAgent fields
|
|
85
116
|
|
|
86
117
|
`defineAgent` takes a few more fields, all optional. For the exported types, see the [TypeScript API](./reference/typescript-api).
|
|
87
118
|
|
|
88
|
-
| Field | Type
|
|
89
|
-
| -------------- |
|
|
90
|
-
| `
|
|
91
|
-
| `
|
|
92
|
-
| `
|
|
93
|
-
| `
|
|
94
|
-
|
|
95
|
-
`codeMode` is experimental and may change or be removed.
|
|
119
|
+
| Field | Type | Default | Description |
|
|
120
|
+
| -------------- | --------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
121
|
+
| `reasoning` | `AgentReasoningDefinition` | provider default | Provider-agnostic reasoning effort forwarded to the agent's turn model calls. |
|
|
122
|
+
| `modelOptions` | `AgentModelOptionsDefinition` | none | Provider option overrides forwarded to the model call. |
|
|
123
|
+
| `experimental` | `{ workflow?: { world?: string } }` | unset | Opt-in settings that can change or disappear in any release. Treat them as unstable. `workflow.world` selects the Workflow world package backing session state, queues, hooks, and streams on the root agent. |
|
|
124
|
+
| `outputSchema` | Standard Schema or a JSON Schema object | none | Structured return type for task-mode runs (a subagent, schedule, or remote job). Interactive conversation turns ignore it unless the client supplies a per-message schema. |
|
|
125
|
+
| `build` | `{ externalDependencies?: string[] }` | none | Hosted-build packaging controls. `externalDependencies` keeps listed packages external while eve compiles authored modules such as tools and channels, and traces those packages into the hosted output. |
|
|
96
126
|
|
|
97
127
|
`externalDependencies` is a packaging control only. It keeps selected packages as runtime dependencies in the hosted output; it does not authorize, configure, or review any third-party service those packages may call.
|
|
98
128
|
|
package/docs/channels/custom.mdx
CHANGED
|
@@ -46,7 +46,7 @@ export default defineChannel({
|
|
|
46
46
|
|
|
47
47
|
Declare routes with the `POST()` and `GET()` helpers. Each route handler receives the raw `Request` and a helpers object:
|
|
48
48
|
|
|
49
|
-
- `send(message, { auth, continuationToken, state? })` starts or resumes a session. Returns a `Session`.
|
|
49
|
+
- `send(message, { auth, continuationToken, state?, title? })` starts or resumes a session. `title` overrides the new workflow session's display title without changing the model message. Returns a `Session`.
|
|
50
50
|
- `getSession(sessionId)` looks up an existing session. The returned `Session` exposes `getEventStream({ startIndex? })` for streaming.
|
|
51
51
|
- `receive(channel, ...)` hands inbound work to a different channel for cross-channel hand-off.
|
|
52
52
|
- `params` holds route parameters extracted from the path pattern.
|
|
@@ -182,7 +182,7 @@ When a parent agent dispatches a subagent, the framework forwards the parent's c
|
|
|
182
182
|
|
|
183
183
|
## Continuation tokens
|
|
184
184
|
|
|
185
|
-
Each call to `send(message, { auth, continuationToken, state? })` from a channel route addresses a session by its channel-local raw token. The framework prepends the channel name, derived from the file stem under `agent/channels/`, before handing the token to the runtime.
|
|
185
|
+
Each call to `send(message, { auth, continuationToken, state?, title? })` from a channel route addresses a session by its channel-local raw token. The framework prepends the channel name, derived from the file stem under `agent/channels/`, before handing the token to the runtime.
|
|
186
186
|
|
|
187
187
|
```ts
|
|
188
188
|
import { slackContinuationToken } from "eve/channels/slack";
|
package/docs/channels/slack.mdx
CHANGED
|
@@ -54,28 +54,22 @@ Inbound hooks decide whether to dispatch a turn and with what `auth`. Return `{
|
|
|
54
54
|
- `onDirectMessage(ctx, message)` handles `message.im` events (needs `im:history` scope). Bot-authored messages and edits are filtered out first.
|
|
55
55
|
- `onInteraction(action, ctx)` handles `block_actions` callbacks not consumed by HITL.
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
eve attaches the triggering Slack user id to the same model message as the message text. This keeps speaker attribution intact when several people use one thread without making profile lookup requests.
|
|
58
|
+
|
|
59
|
+
You get the triggering mention by default, but not the earlier replies in the thread. Enable `threadContext` to fetch and inject them with every message attributed by stable Slack user id. Use `since: "last-agent-reply"` so repeated mentions inject only what is new:
|
|
58
60
|
|
|
59
61
|
```ts
|
|
60
|
-
import {
|
|
62
|
+
import { slackChannel } from "eve/channels/slack";
|
|
61
63
|
import { connectSlackCredentials } from "@vercel/connect/eve";
|
|
62
64
|
|
|
63
65
|
export default slackChannel({
|
|
64
66
|
credentials: connectSlackCredentials("slack/my-agent"),
|
|
65
|
-
|
|
66
|
-
const auth = defaultSlackAuth(message, ctx);
|
|
67
|
-
const prior = await loadThreadContextMessages(ctx.thread, message, {
|
|
68
|
-
since: "last-agent-reply",
|
|
69
|
-
});
|
|
70
|
-
if (prior.length === 0) return { auth };
|
|
71
|
-
const transcript = prior
|
|
72
|
-
.map((m) => `${m.isMe ? "you" : (m.user ?? "user")}: ${m.markdown}`)
|
|
73
|
-
.join("\n");
|
|
74
|
-
return { auth, context: [`Recent thread messages since your last reply:\n\n${transcript}`] };
|
|
75
|
-
},
|
|
67
|
+
threadContext: { since: "last-agent-reply" },
|
|
76
68
|
});
|
|
77
69
|
```
|
|
78
70
|
|
|
71
|
+
`threadContext` performs one `conversations.replies` request for each triggering thread reply and requires the matching Slack history scope. Omit it when the agent should see only direct mentions. `loadThreadContextMessages` remains available when you need custom filtering or non-model processing of the raw thread messages.
|
|
72
|
+
|
|
79
73
|
### Delivery
|
|
80
74
|
|
|
81
75
|
The default handlers reply in-thread and show progress. Typing indicators post automatically: `Thinking…` on inbound, `Working…` on `turn.started`, a truncated reasoning snippet on `reasoning.appended`, and tool status on `actions.requested`. Reasoning snippets build progressively: extensions of at least four characters appear immediately, while smaller streamed deltas use the five-second refresh interval to avoid one Slack request per token. Override `events["reasoning.appended"]` if you prefer generic wording. Override `onAppMention` or the `events` handlers to customize.
|
|
@@ -36,7 +36,7 @@ export default defineAgent({
|
|
|
36
36
|
});
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
The world package backs workflow state, queues, hooks, and streams. Keep secrets and deployment-specific options in runtime environment variables read by that package, not in `agent.ts`. See [agent.ts](../agent-config#workflow-world) and [Workflow Worlds](https://workflow-sdk.dev/worlds).
|
|
39
|
+
The world package backs workflow state, queues, hooks, and streams. Keep secrets and deployment-specific options in runtime environment variables read by that package, not in `agent.ts`. The selected world must match eve's bundled `@workflow/*` line (currently the `5.0.0-beta` line); pin it explicitly, since a mismatched world fails with a `ZodError: invalid_union` during run replay. See the [deployment guide](../guides/deployment#8-deploy-without-vercel) for the install command, plus [agent.ts](../agent-config#workflow-world) and [Workflow Worlds](https://workflow-sdk.dev/worlds).
|
|
40
40
|
|
|
41
41
|
## Resuming after a crash
|
|
42
42
|
|
|
@@ -40,7 +40,7 @@ The stream is newline-delimited JSON (NDJSON), one event per line:
|
|
|
40
40
|
| `turn.started` | A new turn began. |
|
|
41
41
|
| `message.received` | An inbound user message was accepted. |
|
|
42
42
|
| `step.started` | A model step began. |
|
|
43
|
-
| `actions.requested` | The model requested tool calls.
|
|
43
|
+
| `actions.requested` | The model requested one or more actions, including tool calls; calls stream before execution. |
|
|
44
44
|
| `action.result` | A tool call returned. |
|
|
45
45
|
| `input.requested` | The run paused for human input ([HITL](/docs/human-in-the-loop) approval or `ask_question`); carries `requests`. |
|
|
46
46
|
| `subagent.called` | A subagent was delegated; carries `childSessionId` to attach to. |
|
package/docs/connections/mcp.mdx
CHANGED
|
@@ -48,8 +48,8 @@ Use `allow` for the smallest safe surface, especially on MCP servers that expose
|
|
|
48
48
|
|
|
49
49
|
MCP connections support the shared connection options:
|
|
50
50
|
|
|
51
|
-
- `auth` for static tokens, Vercel Connect,
|
|
52
|
-
- `headers` for API-key schemes
|
|
51
|
+
- `auth` for static tokens, Vercel Connect, self-hosted interactive OAuth, or a per-caller provider resolver.
|
|
52
|
+
- `headers` for static or per-caller API-key schemes and extra server configuration.
|
|
53
53
|
- `approval` for human-in-the-loop gates before connection tools run.
|
|
54
54
|
|
|
55
55
|
See [Connections](/docs/connections) for the shared auth, headers, and approval shapes.
|
|
@@ -34,6 +34,41 @@ OpenAPI connections use the shared connection fields from [Connections](/docs/co
|
|
|
34
34
|
|
|
35
35
|
Use `baseUrl` when the spec's `servers` list is absent, points at the wrong environment, or needs to be pinned for this agent.
|
|
36
36
|
|
|
37
|
+
## Path parameters
|
|
38
|
+
|
|
39
|
+
When an operation path contains dynamic segments, declare each segment as an OpenAPI path parameter. eve exposes path, query, header, and cookie parameters as top-level tool inputs, then substitutes `in: "path"` values into the matching `{name}` placeholder before making the request.
|
|
40
|
+
|
|
41
|
+
```ts title="agent/connections/cart.ts"
|
|
42
|
+
import { defineOpenAPIConnection } from "eve/connections";
|
|
43
|
+
|
|
44
|
+
export default defineOpenAPIConnection({
|
|
45
|
+
baseUrl: "https://api.example.com",
|
|
46
|
+
description: "Cart and checkout API.",
|
|
47
|
+
spec: {
|
|
48
|
+
openapi: "3.0.3",
|
|
49
|
+
info: { title: "Cart API", version: "1.0.0" },
|
|
50
|
+
paths: {
|
|
51
|
+
"/api/{dynamicsegment}/somethingelse": {
|
|
52
|
+
get: {
|
|
53
|
+
operationId: "getSomethingElse",
|
|
54
|
+
parameters: [
|
|
55
|
+
{
|
|
56
|
+
name: "dynamicsegment",
|
|
57
|
+
in: "path",
|
|
58
|
+
required: true,
|
|
59
|
+
schema: { type: "string" },
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
responses: { "200": { description: "OK" } },
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The parameter `name` must exactly match the placeholder inside the path. If the spec omits the `in: "path"` parameter, the generated tool has no input for that segment and eve cannot fill it in from query parameters.
|
|
71
|
+
|
|
37
72
|
## Operation filters
|
|
38
73
|
|
|
39
74
|
To narrow which generated tools the model sees, set exactly one of `operations.allow` or `operations.block`:
|
|
@@ -55,8 +90,8 @@ Filters match `operationId`. If an operation does not declare one, use the deter
|
|
|
55
90
|
|
|
56
91
|
OpenAPI connections support the shared connection options:
|
|
57
92
|
|
|
58
|
-
- `auth` for static tokens, Vercel Connect,
|
|
59
|
-
- `headers` for API-key schemes
|
|
93
|
+
- `auth` for static tokens, Vercel Connect, self-hosted interactive OAuth, or a per-caller provider resolver.
|
|
94
|
+
- `headers` for static or per-caller API-key schemes and extra server configuration.
|
|
60
95
|
- `approval` for human-in-the-loop gates before generated operation tools run.
|
|
61
96
|
|
|
62
97
|
See [Connections](/docs/connections) for the shared auth, headers, and approval shapes.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "
|
|
2
|
+
title: "Overview"
|
|
3
3
|
description: "Expose external MCP and OpenAPI servers to the model, with connection tokens the model never sees."
|
|
4
4
|
url: /connections
|
|
5
5
|
---
|
|
@@ -28,6 +28,18 @@ When `getToken` is the only auth, `principalType` defaults to `"app"`: one share
|
|
|
28
28
|
|
|
29
29
|
eve resolves and caches connection tokens per step; they never land in conversation history or reach the model.
|
|
30
30
|
|
|
31
|
+
## Choose app vs. user auth
|
|
32
|
+
|
|
33
|
+
A connection credential can belong to the agent or to the person using it. This choice is separate from route auth, but user-scoped connection auth depends on route auth: eve can only resolve a user token when the active session has `ctx.session.auth.current?.principalType === "user"`.
|
|
34
|
+
|
|
35
|
+
| Credential owner | Use when | Auth shape |
|
|
36
|
+
| ---------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
37
|
+
| App | The agent should use one shared service, bot, installation, or app credential. | `auth: { getToken }` defaults to `principalType: "app"`, or use `connect({ connector: "linear/myagent", principalType: "app" })` with Vercel Connect. |
|
|
38
|
+
| User | Each end-user should authorize and use their own third-party account. | `connect("linear/myagent")`, `connect({ connector: "linear/myagent", principalType: "user" })`, or `auth: { principalType: "user", getToken }`. |
|
|
39
|
+
| User from a job | Background work should use the same user's OAuth grant that started the work. | Start or resume the session through a channel whose route auth resolved that user, or pass an explicit user auth context when dispatching through a channel. |
|
|
40
|
+
|
|
41
|
+
`principalType: "user"` does not mean "ask any human later." It means "key this credential to the authenticated user already attached to the eve session." If the run was started by a schedule, a same-project runtime token, `localDev()`, or another internal runtime path without an end-user principal, a user-scoped connection fails with `reason: "principal_required"` instead of starting OAuth. In that case, either authenticate the inbound channel as a user or configure the connection as app-scoped.
|
|
42
|
+
|
|
31
43
|
## No auth
|
|
32
44
|
|
|
33
45
|
Drop `auth` entirely for servers that need no token, such as a localhost server during development or a public one:
|
|
@@ -57,6 +69,38 @@ export default defineMcpClientConnection({
|
|
|
57
69
|
});
|
|
58
70
|
```
|
|
59
71
|
|
|
72
|
+
## Per-caller auth and headers
|
|
73
|
+
|
|
74
|
+
When credentials or routing depend on the caller, make `auth` or `headers` a function. eve calls it inside the active turn and passes the same session context exposed to tools and hooks:
|
|
75
|
+
|
|
76
|
+
```ts title="agent/connections/warehouse.ts"
|
|
77
|
+
import { defineOpenAPIConnection } from "eve/connections";
|
|
78
|
+
|
|
79
|
+
export default defineOpenAPIConnection({
|
|
80
|
+
spec: "https://warehouse.example.com/openapi.json",
|
|
81
|
+
description: "The caller's tenant-scoped warehouse.",
|
|
82
|
+
auth: (ctx) => ({
|
|
83
|
+
principalType: "user",
|
|
84
|
+
getToken: async () => ({
|
|
85
|
+
token: await tenantToken(ctx.session.auth.current),
|
|
86
|
+
}),
|
|
87
|
+
}),
|
|
88
|
+
headers: (ctx) => ({
|
|
89
|
+
"X-Tenant-Id": tenantId(ctx.session.auth.current),
|
|
90
|
+
}),
|
|
91
|
+
});
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
An `auth` resolver returns the same provider object accepted by static `auth`, including a `connect(...)` provider for interactive OAuth. The resolver itself can be async. Header callbacks can return the whole map, as above, or resolve individual values:
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
headers: {
|
|
98
|
+
"X-Tenant-Id": (ctx) => tenantId(ctx.session.auth.current),
|
|
99
|
+
}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Use `principalType: "user"` for per-user tokens so eve rejects unauthenticated callers and keys its step-local token cache by user. The resolver supplements route auth; it does not authenticate the inbound request. Static auth objects and header maps remain available when every caller shares the same configuration.
|
|
103
|
+
|
|
60
104
|
## Per-connection approval
|
|
61
105
|
|
|
62
106
|
To put every tool a connection serves behind a human, use the helpers from `eve/tools/approval`:
|
|
@@ -92,7 +136,33 @@ export default defineMcpClientConnection({
|
|
|
92
136
|
});
|
|
93
137
|
```
|
|
94
138
|
|
|
95
|
-
`"linear/myagent"` is the UID you chose when registering the Connect client.
|
|
139
|
+
`"linear/myagent"` is the UID you chose when registering the Connect client. `connect("linear/myagent")` is shorthand for a user-scoped interactive OAuth connection: eve resolves a token for the active user before each tool call, emits `authorization.required` when that user has not authorized yet, and resumes the parked turn after the callback completes.
|
|
140
|
+
|
|
141
|
+
That means the channel that creates or continues the session must authenticate a real user. For a web app, configure `agent/channels/eve.ts` so your app session maps to `principalType: "user"`; for platform channels, use the built-in channel auth that maps the sender to a user principal. If no authenticated user is attached to the session, the first user-scoped connection call fails with `reason: "principal_required"`.
|
|
142
|
+
|
|
143
|
+
If the remote service should act as the agent itself instead of the end-user, make the Connect connection app-scoped:
|
|
144
|
+
|
|
145
|
+
```ts title="agent/connections/linear.ts"
|
|
146
|
+
import { connect } from "@vercel/connect/eve";
|
|
147
|
+
import { defineMcpClientConnection } from "eve/connections";
|
|
148
|
+
|
|
149
|
+
export default defineMcpClientConnection({
|
|
150
|
+
url: "https://mcp.linear.app/sse",
|
|
151
|
+
description: "Linear workspace: issues, projects, cycles, and comments.",
|
|
152
|
+
auth: connect({ connector: "linear/myagent", principalType: "app" }),
|
|
153
|
+
});
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
App-scoped Connect auth is non-interactive. eve asks Vercel Connect for an app token and does not emit a browser consent challenge; if the connector is not installed or cannot issue an app token, the tool call fails terminally so an operator can fix the connector setup. The full setup (Connect client provisioning, project linking, the runtime consent flow) lives in [Auth & route protection](/docs/guides/auth-and-route-protection).
|
|
157
|
+
|
|
158
|
+
### Troubleshooting Vercel Connect auth
|
|
159
|
+
|
|
160
|
+
| Symptom | What it means | Fix |
|
|
161
|
+
| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
162
|
+
| `reason: "principal_required"` | A user-scoped connection ran without an authenticated user on the active session. | Return `principalType: "user"` from the channel's route auth, or change the connection to `principalType: "app"` if it should be shared. |
|
|
163
|
+
| `authorization.required` appears but no UI | eve parked the turn for OAuth, but the channel or frontend is not rendering the challenge. | Render the challenge from the stream event and continue the same session after the callback. |
|
|
164
|
+
| OAuth works locally but fails after deploy | The project may not be linked to the Connect client, or the deployed runtime may not have the expected Vercel OIDC/project scope. | Run Connect setup from the consuming project directory, link the project, deploy again, and verify the connector UID in `connect("...")`. |
|
|
165
|
+
| A scheduled or internal run needs user OAuth | Schedules and runtime callers do not automatically carry an end-user principal. | Dispatch through a user-authenticated channel when work is user-owned, or use app-scoped auth for agent-owned background work. |
|
|
96
166
|
|
|
97
167
|
## Self-hosted interactive OAuth
|
|
98
168
|
|
|
@@ -1,39 +1,51 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "Assertions"
|
|
3
|
-
description: "
|
|
3
|
+
description: "Scoped methods, value assertions, the matcher mini-language, and gate vs soft severity."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
Assertions are how an eval grades what its `test(t)` function produced. Each one
|
|
6
|
+
Assertions are how an eval grades what its `test(t)` function produced. Each one records a result and returns a chainable handle. The runner reads every recorded result to compute the verdict, so a single run reports every failing assertion rather than dying on the first. There are two deterministic surfaces: scoped methods and `t.check` for grading a specific value. For model-graded assertions, see [Judge](./judge).
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## Scoped assertions
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
Scoped assertions take no explicit value and gate by default. Assertions on `t` inspect the whole run after `test` finishes. Session assertions snapshot that session when called, and turn assertions inspect one immutable response. A scope is **parked** when it paused on unanswered human-in-the-loop (HITL) input.
|
|
11
11
|
|
|
12
|
-
| Assertion
|
|
13
|
-
|
|
|
14
|
-
| `t.
|
|
15
|
-
| `t.
|
|
16
|
-
| `t.
|
|
17
|
-
| `
|
|
18
|
-
| `t.
|
|
19
|
-
| `t.
|
|
20
|
-
| `t.
|
|
21
|
-
| `t.
|
|
22
|
-
| `t.
|
|
23
|
-
| `t.
|
|
24
|
-
| `t.
|
|
25
|
-
| `t.
|
|
26
|
-
| `t.
|
|
27
|
-
| `t.
|
|
12
|
+
| Assertion | Asserts |
|
|
13
|
+
| ----------------------------------------------------- | ----------------------------------------------------------------------------- |
|
|
14
|
+
| `t.succeeded()` | The run did not fail and did not park on unanswered HITL input |
|
|
15
|
+
| `t.parked()` | The run cleanly parked on HITL input |
|
|
16
|
+
| `t.messageIncludes(token)` | Joined assistant text contains `token` (string or RegExp) |
|
|
17
|
+
| `turn.outputEquals(value)` / `.outputMatches(schema)` | Deep equality or Standard Schema validation of turn/session structured output |
|
|
18
|
+
| `t.calledTool(name, opts?)` | A matching tool call completed (`input`, `output`, `status`, `count`) |
|
|
19
|
+
| `t.loadedSkill(skill, opts?)` | Sugar for `t.calledTool("load_skill", { input: { skill }, ...opts })` |
|
|
20
|
+
| `t.notCalledTool(name)` | No request for `name` in any lifecycle state |
|
|
21
|
+
| `t.toolOrder([...names])` | Tool requests appear in order |
|
|
22
|
+
| `t.usedNoTools()` | No tool calls at all |
|
|
23
|
+
| `t.maxToolCalls(n)` | At most `n` tool calls |
|
|
24
|
+
| `t.noFailedActions()` | No tool, subagent, or skill action reported a failure |
|
|
25
|
+
| `t.calledSubagent(name, opts?)` | A subagent delegation happened (`remoteUrl`, `output` constraints) |
|
|
26
|
+
| `t.event(type, opts?)` / `t.notEvent(type, opts?)` | Typed event presence, data, and count matching |
|
|
27
|
+
| `t.eventOrder([...matchers])` | Matching event groups occur in order |
|
|
28
|
+
| `t.eventsSatisfy(label, predicate)` | Escape hatch: any predicate over the typed event stream |
|
|
28
29
|
|
|
29
|
-
`
|
|
30
|
+
`succeeded()` accepts both a closed session and a healthy session left open for the next user message; it rejects protocol failures and unanswered HITL. `parked()` requires a clean HITL park. Structured output assertions live on turns and independent sessions, where the output is unambiguous (see the [output schema guide](../guides/client/output-schema)).
|
|
30
31
|
|
|
31
32
|
```ts
|
|
32
33
|
await t.send("What is the weather in Brooklyn?");
|
|
33
|
-
t.
|
|
34
|
+
t.succeeded();
|
|
34
35
|
t.calledTool("get_weather");
|
|
35
36
|
```
|
|
36
37
|
|
|
38
|
+
The same vocabulary narrows naturally in multi-turn and externally-created session evals:
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
const first = await t.send("Call get_weather for Brooklyn");
|
|
42
|
+
first.calledTool("get_weather", { count: 1 });
|
|
43
|
+
|
|
44
|
+
const attached = await t.target.attachSession(sessionId);
|
|
45
|
+
attached.succeeded();
|
|
46
|
+
attached.messageIncludes("Sunny");
|
|
47
|
+
```
|
|
48
|
+
|
|
37
49
|
`t.calledTool` and `t.usedNoTools` are mutually exclusive; assert one or the other, never both in the same run.
|
|
38
50
|
|
|
39
51
|
## Value assertions with `t.check`
|
|
@@ -41,52 +53,98 @@ t.calledTool("get_weather");
|
|
|
41
53
|
`t.check(value, assertion)` grades an explicit value against a builder from `eve/evals/expect`. The value can be `t.reply`, a turn's `.message`, parsed JSON, or any local you computed:
|
|
42
54
|
|
|
43
55
|
```ts
|
|
44
|
-
import { includes, equals, matches, similarity } from "eve/evals/expect";
|
|
56
|
+
import { includes, equals, matches, satisfies, similarity } from "eve/evals/expect";
|
|
45
57
|
|
|
46
|
-
t.check(t.reply, includes(
|
|
58
|
+
t.check(t.reply, includes(/sunny/i)); // substring or RegExp (gate)
|
|
47
59
|
t.check(parsed, equals({ city: "Brooklyn" })); // deep structural equality (gate)
|
|
48
60
|
t.check(parsed, matches(WeatherSchema)); // Standard Schema, e.g. Zod (gate)
|
|
49
61
|
t.check(t.reply, similarity("Sunny, 72F")); // fuzzy 0–1 Levenshtein (soft)
|
|
62
|
+
t.check(
|
|
63
|
+
latencyMs,
|
|
64
|
+
satisfies((value) => value < 1_000, "latency under one second"),
|
|
65
|
+
);
|
|
50
66
|
```
|
|
51
67
|
|
|
52
|
-
| Builder | Scores
|
|
53
|
-
| ---------------------- |
|
|
54
|
-
| `includes(
|
|
55
|
-
| `equals(value)` | deep structural equality
|
|
56
|
-
| `matches(schema)` | validates against a Standard Schema
|
|
57
|
-
| `similarity(expected)` | normalized Levenshtein similarity, 1 = identical
|
|
68
|
+
| Builder | Scores | Default |
|
|
69
|
+
| ---------------------- | ------------------------------------------------------- | ------- |
|
|
70
|
+
| `includes(value)` | coerced string contains a substring or matches a RegExp | gate |
|
|
71
|
+
| `equals(value)` | deep structural equality | gate |
|
|
72
|
+
| `matches(schema)` | validates against a Standard Schema | gate |
|
|
73
|
+
| `similarity(expected)` | normalized Levenshtein similarity, 1 = identical | soft |
|
|
74
|
+
| `satisfies(fn, label)` | custom boolean predicate | gate |
|
|
58
75
|
|
|
59
76
|
Pick the cheapest builder that captures what "correct" means. When exact match is too strict but a judge model is overkill, `similarity` is the middle ground. For nuanced grading, reach for the [judge](./judge).
|
|
60
77
|
|
|
61
78
|
## The matcher mini-language
|
|
62
79
|
|
|
63
|
-
`t.calledTool` and `t.calledSubagent` take
|
|
80
|
+
`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ remoteUrl, output, status, count }`. Calls match `status: "completed"` by default; use `"pending"`, `"failed"`, or `"rejected"` explicitly for lifecycle checks. `count` is the exact number of calls matching every supplied constraint.
|
|
81
|
+
|
|
82
|
+
Matcher values accept a literal (objects partial-deep-match), a RegExp, or a predicate function that returns a boolean:
|
|
64
83
|
|
|
65
84
|
```ts
|
|
66
|
-
t.calledTool("bash", { input: { command: /^pwd/ },
|
|
85
|
+
t.calledTool("bash", { input: { command: /^pwd/ }, count: 1 });
|
|
67
86
|
|
|
68
87
|
t.calledTool("echo", { output: (value) => String(value).includes(marker) });
|
|
69
88
|
|
|
89
|
+
parked.calledTool("guarded", { status: "pending", count: 1 });
|
|
90
|
+
t.calledTool("guarded", { output: /approved/, count: 1 });
|
|
91
|
+
|
|
70
92
|
t.calledSubagent("weather", {
|
|
71
|
-
remoteUrl: () => process.env.WEATHER_AGENT_URL
|
|
93
|
+
remoteUrl: (value) => value === process.env.WEATHER_AGENT_URL,
|
|
72
94
|
output: /72F/,
|
|
73
95
|
});
|
|
74
96
|
```
|
|
75
97
|
|
|
98
|
+
`requireInputRequest` uses the same matcher language for `input`, `prompt`, and `display`. Its `optionIds` matcher receives option ids in request order; a literal array must match that complete ordered list exactly:
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
const request = session.requireInputRequest({
|
|
102
|
+
toolName: "ask_question",
|
|
103
|
+
optionIds: ["red", "blue"],
|
|
104
|
+
});
|
|
105
|
+
```
|
|
106
|
+
|
|
76
107
|
## Run state and derived facts
|
|
77
108
|
|
|
78
|
-
Beyond the raw `t.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output,
|
|
109
|
+
Beyond the raw `t.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output, lifecycle status), subagent calls, and HITL input requests. A turn that leaves the session open for a next message is the normal end state of a successful turn; parking on unanswered HITL input is tracked separately.
|
|
79
110
|
|
|
80
|
-
|
|
111
|
+
Typed event matching covers presence, absence, exact counts, partial event data, and ordering:
|
|
81
112
|
|
|
82
113
|
```ts
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
"
|
|
114
|
+
turn.notEvent("result.completed");
|
|
115
|
+
turn.eventOrder([
|
|
116
|
+
{ type: "subagent.called", data: { name: "researcher" }, count: 2 },
|
|
117
|
+
{ type: "subagent.completed", data: { subagentName: "researcher" }, count: 2 },
|
|
118
|
+
]);
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
When a protocol invariant needs cross-event correlation, `eventsSatisfy` remains the escape hatch:
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
t.eventsSatisfy("assistant reply includes the marker", (events) =>
|
|
125
|
+
events.some((e) => e.type === "message.completed" && e.data.message?.includes(marker)),
|
|
87
126
|
);
|
|
88
127
|
```
|
|
89
128
|
|
|
129
|
+
## Required preconditions
|
|
130
|
+
|
|
131
|
+
Recorded assertions never throw and are not awaitable. When later control flow depends on a value, use `await t.require(value, assertion)`. It records a gate, returns the original value when it passes, and stops the test body without adding a duplicate execution error when it fails:
|
|
132
|
+
|
|
133
|
+
```ts
|
|
134
|
+
await t.require(
|
|
135
|
+
sessionIds,
|
|
136
|
+
satisfies((ids) => ids.length > 0, "dispatch started a session"),
|
|
137
|
+
);
|
|
138
|
+
await t.target.attachSession(sessionIds[0]!);
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Use the matching `require*` lookups when dependent code needs protocol data:
|
|
142
|
+
|
|
143
|
+
```ts
|
|
144
|
+
const call = turn.requireToolCall("search");
|
|
145
|
+
const request = session.requireInputRequest({ toolName: "guarded" });
|
|
146
|
+
```
|
|
147
|
+
|
|
90
148
|
## Severity
|
|
91
149
|
|
|
92
150
|
Every assertion returns a chainable handle. Severity rides on the assertion, so there is no separate thresholds map to keep in sync.
|