eve 0.6.0-beta.9 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +281 -0
- package/README.md +9 -6
- package/dist/docs/public/README.md +17 -12
- package/dist/docs/public/agent-config.md +10 -10
- package/dist/docs/public/channels/custom.mdx +4 -4
- package/dist/docs/public/channels/discord.mdx +1 -1
- package/dist/docs/public/channels/eve.mdx +10 -10
- package/dist/docs/public/channels/github.mdx +1 -1
- package/dist/docs/public/channels/overview.mdx +21 -15
- package/dist/docs/public/channels/slack.mdx +16 -4
- package/dist/docs/public/channels/teams.mdx +1 -1
- package/dist/docs/public/channels/telegram.mdx +1 -1
- package/dist/docs/public/channels/twilio.mdx +1 -1
- package/dist/docs/public/{advanced → concepts}/context-control.md +3 -3
- package/dist/docs/public/{advanced → concepts}/default-harness.md +5 -5
- package/dist/docs/public/{advanced → concepts}/execution-model-and-durability.md +3 -1
- package/dist/docs/public/concepts/meta.json +10 -0
- package/dist/docs/public/{advanced → concepts}/security-model.md +3 -3
- package/dist/docs/public/{advanced → concepts}/sessions-runs-and-streaming.md +7 -7
- package/dist/docs/public/connections.mdx +6 -4
- package/dist/docs/public/evals/assertions.mdx +108 -0
- package/dist/docs/public/evals/cases.mdx +143 -0
- package/dist/docs/public/evals/judge.mdx +94 -0
- package/dist/docs/public/evals/meta.json +4 -0
- package/dist/docs/public/evals/overview.mdx +118 -0
- package/dist/docs/public/evals/reporters.mdx +62 -0
- package/dist/docs/public/evals/running.mdx +63 -0
- package/dist/docs/public/evals/targets.mdx +54 -0
- package/dist/docs/public/getting-started.mdx +38 -33
- package/dist/docs/public/{advanced → guides}/auth-and-route-protection.md +5 -3
- package/dist/docs/public/{client → guides/client}/continuations.mdx +2 -2
- package/dist/docs/public/{client → guides/client}/messages.mdx +1 -1
- package/dist/docs/public/{client → guides/client}/meta.json +1 -1
- package/dist/docs/public/{client → guides/client}/output-schema.mdx +2 -2
- package/dist/docs/public/{client → guides/client}/overview.mdx +5 -5
- package/dist/docs/public/{client → guides/client}/streaming.mdx +1 -1
- package/dist/docs/public/{advanced → guides}/deployment.md +9 -1
- package/dist/docs/public/guides/dev-tui.md +50 -0
- package/dist/docs/public/{advanced → guides}/dynamic-capabilities.md +1 -1
- package/dist/docs/public/{advanced → guides}/dynamic-workflows.md +1 -1
- package/dist/docs/public/{frontend → guides/frontend}/nextjs.mdx +16 -7
- package/dist/docs/public/{frontend → guides/frontend}/nuxt.mdx +7 -7
- package/dist/docs/public/{frontend → guides/frontend}/overview.mdx +6 -6
- package/dist/docs/public/{frontend → guides/frontend}/sveltekit.mdx +5 -5
- package/dist/docs/public/{frontend → guides/frontend}/use-eve-agent-svelte.mdx +2 -2
- package/dist/docs/public/{frontend → guides/frontend}/use-eve-agent-vue.mdx +2 -2
- package/dist/docs/public/{advanced → guides}/hooks.md +2 -2
- package/dist/docs/public/{advanced → guides}/instrumentation.md +3 -1
- package/dist/docs/public/{advanced → guides}/meta.json +8 -12
- package/dist/docs/public/{advanced → guides}/session-context.md +3 -3
- package/dist/docs/public/{advanced → guides}/state.md +1 -1
- package/dist/docs/public/instructions.mdx +2 -2
- package/dist/docs/public/introduction.md +5 -2
- package/dist/docs/public/meta.json +4 -3
- package/dist/docs/public/reference/cli.md +35 -19
- package/dist/docs/public/reference/meta.json +1 -1
- package/dist/docs/public/reference/project-layout.md +5 -1
- package/dist/docs/public/reference/typescript-api.md +27 -23
- package/dist/docs/public/sandbox.mdx +1 -1
- package/dist/docs/public/schedules.mdx +2 -2
- package/dist/docs/public/skills.mdx +3 -3
- package/dist/docs/public/subagents.mdx +3 -3
- package/dist/docs/public/tools.mdx +4 -8
- package/dist/docs/public/tutorial/connect-a-warehouse.mdx +3 -3
- package/dist/docs/public/tutorial/first-agent.mdx +6 -3
- package/dist/docs/public/tutorial/guard-the-spend.mdx +1 -1
- package/dist/docs/public/tutorial/how-it-runs.mdx +2 -2
- package/dist/docs/public/tutorial/meta.json +1 -1
- package/dist/docs/public/tutorial/query-sample-data.mdx +1 -1
- package/dist/docs/public/tutorial/remember-definitions.mdx +3 -3
- package/dist/docs/public/tutorial/run-analysis.mdx +1 -1
- package/dist/docs/public/tutorial/ship-it.mdx +4 -4
- package/dist/docs/public/tutorial/team-playbooks.mdx +3 -3
- package/dist/src/chunks/{use-eve-agent-DCZbkLG7.js → use-eve-agent-DErQj5hs.js} +125 -37
- package/dist/src/chunks/{use-eve-agent-DoheC4_o.js → use-eve-agent-DoR8C4i6.js} +125 -37
- package/dist/src/cli/banner.d.ts +7 -0
- package/dist/src/cli/banner.js +1 -0
- package/dist/src/cli/commands/channel-add-conflicts.d.ts +1 -1
- package/dist/src/cli/commands/channels.d.ts +9 -6
- package/dist/src/cli/commands/channels.js +1 -1
- package/dist/src/cli/commands/deploy.d.ts +21 -0
- package/dist/src/cli/commands/deploy.js +1 -0
- package/dist/src/cli/commands/init-git.d.ts +15 -0
- package/dist/src/cli/commands/init-git.js +1 -0
- package/dist/src/cli/commands/init.d.ts +29 -0
- package/dist/src/cli/commands/init.js +1 -0
- package/dist/src/cli/commands/link.d.ts +21 -0
- package/dist/src/cli/commands/link.js +1 -0
- package/dist/src/cli/commands/preconditions.d.ts +7 -0
- package/dist/src/cli/commands/preconditions.js +1 -0
- package/dist/src/cli/commands/register-project-commands.d.ts +12 -0
- package/dist/src/cli/commands/register-project-commands.js +1 -0
- package/dist/src/cli/dev/tui/agent-header.d.ts +15 -9
- package/dist/src/cli/dev/tui/agent-header.js +1 -1
- package/dist/src/cli/dev/tui/blocks.d.ts +1 -1
- package/dist/src/cli/dev/tui/blocks.js +3 -2
- package/dist/src/cli/dev/tui/command-typeahead.d.ts +47 -0
- package/dist/src/cli/dev/tui/command-typeahead.js +1 -0
- package/dist/src/cli/dev/tui/dev-rebuild-status.d.ts +21 -0
- package/dist/src/cli/dev/tui/dev-rebuild-status.js +1 -0
- package/dist/src/cli/dev/tui/errors.d.ts +18 -0
- package/dist/src/cli/dev/tui/errors.js +1 -1
- package/dist/src/cli/dev/tui/prompt-command-handler.d.ts +14 -0
- package/dist/src/cli/dev/tui/prompt-command-handler.js +1 -0
- package/dist/src/cli/dev/tui/prompt-commands.d.ts +54 -0
- package/dist/src/cli/dev/tui/prompt-commands.js +2 -0
- package/dist/src/cli/dev/tui/runner.d.ts +64 -7
- package/dist/src/cli/dev/tui/runner.js +1 -1
- package/dist/src/cli/dev/tui/setup-commands.d.ts +48 -0
- package/dist/src/cli/dev/tui/setup-commands.js +2 -0
- package/dist/src/cli/dev/tui/setup-flow.d.ts +35 -0
- package/dist/src/cli/dev/tui/setup-issues.d.ts +40 -0
- package/dist/src/cli/dev/tui/setup-issues.js +1 -0
- package/dist/src/cli/dev/tui/setup-panel.d.ts +103 -0
- package/dist/src/cli/dev/tui/setup-panel.js +1 -0
- package/dist/src/cli/dev/tui/status-line.d.ts +25 -0
- package/dist/src/cli/dev/tui/status-line.js +1 -0
- package/dist/src/cli/dev/tui/stream-format.d.ts +16 -1
- package/dist/src/cli/dev/tui/stream-format.js +1 -1
- package/dist/src/cli/dev/tui/terminal-renderer.d.ts +32 -3
- package/dist/src/cli/dev/tui/terminal-renderer.js +5 -2
- package/dist/src/cli/dev/tui/test/index.d.ts +3 -1
- package/dist/src/cli/dev/tui/test/index.js +1 -1
- package/dist/src/cli/dev/tui/test/mock-terminal.d.ts +1 -0
- package/dist/src/cli/dev/tui/test/mock-terminal.js +1 -1
- package/dist/src/cli/dev/tui/theme.d.ts +10 -0
- package/dist/src/cli/dev/tui/theme.js +1 -1
- package/dist/src/cli/dev/tui/tui-prompter.d.ts +20 -0
- package/dist/src/cli/dev/tui/tui-prompter.js +1 -0
- package/dist/src/cli/dev/tui/tui.d.ts +6 -8
- package/dist/src/cli/dev/tui/tui.js +1 -1
- package/dist/src/cli/dev/tui/types.d.ts +4 -3
- package/dist/src/cli/dev/tui/vercel-status.d.ts +47 -0
- package/dist/src/cli/dev/tui/vercel-status.js +1 -0
- package/dist/src/cli/run.d.ts +9 -18
- package/dist/src/cli/run.js +2 -2
- package/dist/src/client/client.d.ts +8 -0
- package/dist/src/client/client.js +1 -1
- package/dist/src/client/file-parts.d.ts +18 -0
- package/dist/src/client/file-parts.js +1 -0
- package/dist/src/client/index.d.ts +3 -2
- package/dist/src/client/index.js +1 -1
- package/dist/src/client/message-response.js +1 -1
- package/dist/src/client/open-stream.d.ts +6 -0
- package/dist/src/client/open-stream.js +1 -1
- package/dist/src/client/session-utils.d.ts +5 -0
- package/dist/src/client/session-utils.js +1 -1
- package/dist/src/client/session.js +1 -1
- package/dist/src/client/types.d.ts +9 -2
- package/dist/src/compiled/.vendor-stamp.json +8 -8
- package/dist/src/compiled/@ai-sdk/anthropic/index.d.ts +56 -31
- package/dist/src/compiled/@ai-sdk/anthropic/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/google/index.js +1 -1
- package/dist/src/compiled/@ai-sdk/mcp/index.js +1 -1
- package/dist/src/compiled/@ai-sdk/openai/index.d.ts +16 -9
- package/dist/src/compiled/@ai-sdk/openai/index.js +2 -2
- package/dist/src/compiled/@ai-sdk/otel/index.js +2 -2
- package/dist/src/compiled/@vercel/sandbox/index.js +1 -1
- package/dist/src/compiled/@workflow/core/capabilities.d.ts +19 -1
- package/dist/src/compiled/@workflow/core/class-serialization.d.ts +32 -0
- package/dist/src/compiled/@workflow/core/create-hook.d.ts +37 -0
- package/dist/src/compiled/@workflow/core/global.d.ts +11 -1
- package/dist/src/compiled/@workflow/core/index.js +2 -2
- package/dist/src/compiled/@workflow/core/runtime/helpers.d.ts +4 -2
- package/dist/src/compiled/@workflow/core/runtime/start.d.ts +6 -0
- package/dist/src/compiled/@workflow/core/runtime/suspension-handler.d.ts +15 -2
- package/dist/src/compiled/@workflow/core/runtime/wait-continuation.d.ts +84 -0
- package/dist/src/compiled/@workflow/core/runtime/wait-until.d.ts +18 -0
- package/dist/src/compiled/@workflow/core/runtime.d.ts +3 -1
- package/dist/src/compiled/@workflow/core/runtime.js +28 -28
- package/dist/src/compiled/@workflow/core/serialization/types.d.ts +21 -0
- package/dist/src/compiled/@workflow/core/serialization.d.ts +113 -6
- package/dist/src/compiled/@workflow/core/symbols.d.ts +2 -0
- package/dist/src/compiled/@workflow/core/util.d.ts +0 -5
- package/dist/src/compiled/@workflow/core/version.d.ts +1 -1
- package/dist/src/compiled/@workflow/core/workflow/attribute-dispatcher.d.ts +6 -0
- package/dist/src/compiled/@workflow/core/workflow/set-attributes.d.ts +3 -4
- package/dist/src/compiled/@workflow/core/workflow.js +1 -1
- package/dist/src/compiled/@workflow/world/events.d.ts +48 -0
- package/dist/src/compiled/@workflow/world/index.d.ts +3 -3
- package/dist/src/compiled/@workflow/world/queue.d.ts +31 -2
- package/dist/src/compiled/@workflow/world/runs.d.ts +2 -0
- package/dist/src/compiled/@workflow/world/spec-version.d.ts +2 -1
- package/dist/src/compiled/_chunks/workflow/attribute-changes-DGVGRGfw.js +59 -0
- package/dist/src/compiled/_chunks/workflow/{dist-gEXVSMPU.js → dist-CkMRLaRV.js} +1 -1
- package/dist/src/compiled/_chunks/workflow/functions-DuPjIvMH.js +1 -0
- package/dist/src/compiled/_chunks/workflow/resume-hook-DMSadN9o.js +1 -0
- package/dist/src/compiled/_chunks/workflow/run-BRdn7zy_.js +1 -0
- package/dist/src/compiled/_chunks/workflow/sleep-CpXfoXLF.js +1 -0
- package/dist/src/compiled/just-bash/index.d.ts +4 -4
- package/dist/src/compiler/artifacts.js +1 -1
- package/dist/src/compiler/manifest.d.ts +8 -8
- package/dist/src/compiler/normalize-agent-config.js +1 -1
- package/dist/src/compiler/normalize-channel.d.ts +2 -1
- package/dist/src/compiler/normalize-channel.js +1 -1
- package/dist/src/compiler/normalize-connection.d.ts +2 -1
- package/dist/src/compiler/normalize-connection.js +1 -1
- package/dist/src/compiler/normalize-helpers.d.ts +5 -0
- package/dist/src/compiler/normalize-helpers.js +1 -1
- package/dist/src/compiler/normalize-instructions.d.ts +3 -2
- package/dist/src/compiler/normalize-instructions.js +1 -1
- package/dist/src/compiler/normalize-manifest.js +2 -2
- package/dist/src/compiler/normalize-sandbox.d.ts +2 -1
- package/dist/src/compiler/normalize-sandbox.js +1 -1
- package/dist/src/compiler/normalize-schedule.d.ts +2 -1
- package/dist/src/compiler/normalize-schedule.js +1 -1
- package/dist/src/compiler/normalize-skill.d.ts +2 -1
- package/dist/src/compiler/normalize-skill.js +1 -1
- package/dist/src/compiler/normalize-subagent.d.ts +4 -1
- package/dist/src/compiler/normalize-subagent.js +1 -1
- package/dist/src/compiler/normalize-tool.d.ts +2 -1
- package/dist/src/compiler/normalize-tool.js +1 -1
- package/dist/src/compiler/workspace-resources.js +1 -1
- package/dist/src/context/node.d.ts +1 -1
- package/dist/src/evals/assertions/collector.d.ts +43 -0
- package/dist/src/evals/assertions/collector.js +1 -0
- package/dist/src/evals/assertions/run.d.ts +72 -0
- package/dist/src/evals/assertions/run.js +2 -0
- package/dist/src/evals/autoevals-client.js +2 -0
- package/dist/src/evals/cli/eval-client.d.ts +22 -0
- package/dist/src/evals/cli/eval-client.js +1 -0
- package/dist/src/evals/cli/eval.d.ts +8 -5
- package/dist/src/evals/cli/eval.js +1 -1
- package/dist/src/evals/context.d.ts +19 -0
- package/dist/src/evals/context.js +1 -0
- package/dist/src/evals/define-eval-config.d.ts +16 -0
- package/dist/src/evals/define-eval-config.js +1 -0
- package/dist/src/evals/define-eval.d.ts +20 -0
- package/dist/src/evals/define-eval.js +1 -0
- package/dist/src/evals/expect/index.d.ts +25 -0
- package/dist/src/evals/expect/index.js +1 -0
- package/dist/src/evals/index.d.ts +6 -2
- package/dist/src/evals/index.js +1 -1
- package/dist/src/evals/judge.d.ts +20 -0
- package/dist/src/evals/judge.js +1 -0
- package/dist/src/evals/{checks/match.d.ts → match.d.ts} +17 -18
- package/dist/src/evals/match.js +1 -0
- package/dist/src/evals/reporters/index.d.ts +1 -0
- package/dist/src/evals/reporters/index.js +1 -1
- package/dist/src/evals/requirements.d.ts +3 -0
- package/dist/src/evals/requirements.js +1 -0
- package/dist/src/evals/runner/artifacts.d.ts +7 -6
- package/dist/src/evals/runner/artifacts.js +3 -3
- package/dist/src/evals/runner/discover.d.ts +31 -10
- package/dist/src/evals/runner/discover.js +1 -1
- package/dist/src/evals/runner/execute-eval.d.ts +25 -0
- package/dist/src/evals/runner/execute-eval.js +1 -0
- package/dist/src/evals/runner/execute-task.d.ts +31 -0
- package/dist/src/evals/runner/execute-task.js +1 -0
- package/dist/src/evals/runner/reporters/braintrust.d.ts +7 -5
- package/dist/src/evals/runner/reporters/braintrust.js +2 -2
- package/dist/src/evals/runner/reporters/console.d.ts +4 -4
- package/dist/src/evals/runner/reporters/console.js +1 -1
- package/dist/src/evals/runner/reporters/junit.d.ts +10 -0
- package/dist/src/evals/runner/reporters/junit.js +4 -0
- package/dist/src/evals/runner/reporters/types.d.ts +14 -8
- package/dist/src/evals/runner/run-evals.d.ts +38 -0
- package/dist/src/evals/runner/run-evals.js +1 -0
- package/dist/src/evals/runner/verdict.d.ts +10 -15
- package/dist/src/evals/runner/verdict.js +1 -1
- package/dist/src/evals/session.d.ts +52 -0
- package/dist/src/evals/session.js +1 -0
- package/dist/src/evals/target.d.ts +23 -0
- package/dist/src/evals/target.js +1 -0
- package/dist/src/evals/types.d.ts +294 -219
- package/dist/src/execution/compaction.d.ts +14 -0
- package/dist/src/execution/compaction.js +1 -0
- package/dist/src/execution/delegated-parent-notification.js +1 -1
- package/dist/src/execution/dispatch-runtime-actions-step.js +1 -1
- package/dist/src/execution/node-step.js +1 -1
- package/dist/src/execution/sandbox/bash-tool.d.ts +6 -6
- package/dist/src/execution/sandbox/bash-tool.js +1 -1
- package/dist/src/execution/sandbox/bindings/local.js +1 -1
- package/dist/src/execution/sandbox/bindings/vercel.d.ts +2 -6
- package/dist/src/execution/sandbox/bindings/vercel.js +1 -1
- package/dist/src/execution/sandbox/glob-tool.js +3 -3
- package/dist/src/execution/sandbox/grep-tool.js +3 -3
- package/dist/src/execution/sandbox/read-file-tool.js +1 -1
- package/dist/src/execution/subagent-adapter.js +1 -1
- package/dist/src/execution/tool-auth.js +1 -1
- package/dist/src/execution/turn-workflow.js +1 -1
- package/dist/src/execution/workflow-runtime.d.ts +2 -2
- package/dist/src/execution/workflow-runtime.js +1 -1
- package/dist/src/execution/workflow-steps.js +1 -1
- package/dist/src/harness/action-result-helpers.js +1 -1
- package/dist/src/harness/authorization.d.ts +26 -0
- package/dist/src/harness/authorization.js +1 -1
- package/dist/src/harness/code-mode-lifecycle.js +1 -1
- package/dist/src/harness/emission.d.ts +12 -5
- package/dist/src/harness/emission.js +1 -1
- package/dist/src/harness/model-call-error.d.ts +35 -6
- package/dist/src/harness/model-call-error.js +1 -1
- package/dist/src/harness/step-hooks.d.ts +10 -4
- package/dist/src/harness/step-hooks.js +1 -1
- package/dist/src/harness/tool-loop.js +1 -1
- package/dist/src/harness/tools.d.ts +4 -6
- package/dist/src/harness/tools.js +1 -1
- package/dist/src/harness/turn-tag-state.d.ts +4 -0
- package/dist/src/harness/turn-tag-state.js +1 -1
- package/dist/src/harness/types.d.ts +4 -15
- package/dist/src/internal/application/cache-metadata.js +1 -1
- package/dist/src/internal/application/compiled-artifacts.js +1 -1
- package/dist/src/internal/application/package.js +1 -1
- package/dist/src/internal/application/paths.js +1 -1
- package/dist/src/internal/authored-definition/schema-backed.js +1 -1
- package/dist/src/internal/authored-module-loader.d.ts +4 -1
- package/dist/src/internal/authored-module-loader.js +2 -2
- package/dist/src/internal/authored-module-map-loader.js +1 -1
- package/dist/src/internal/nitro/dev-runtime-artifacts.js +1 -1
- package/dist/src/internal/nitro/host/build-application.js +1 -1
- package/dist/src/internal/nitro/host/build-vercel-agent-summary.js +1 -1
- package/dist/src/internal/nitro/host/configure-nitro-routes.js +3 -3
- package/dist/src/internal/nitro/host/create-application-nitro.js +1 -1
- package/dist/src/internal/nitro/host/dev-authored-source-watcher.js +1 -1
- package/dist/src/internal/nitro/host/dev-watcher-log.d.ts +37 -0
- package/dist/src/internal/nitro/host/dev-watcher-log.js +1 -0
- package/dist/src/internal/nitro/host/ports.d.ts +8 -0
- package/dist/src/internal/nitro/host/ports.js +1 -0
- package/dist/src/internal/nitro/host/prepare-application-host.js +1 -1
- package/dist/src/internal/nitro/host/server-external-packages.d.ts +1 -1
- package/dist/src/internal/nitro/host/server-external-packages.js +1 -1
- package/dist/src/internal/nitro/host/start-development-server.js +1 -1
- package/dist/src/internal/nitro/host/start-production-server.js +1 -1
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response-from-manifest.d.ts +5 -0
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response-from-manifest.js +1 -0
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response.d.ts +31 -2
- package/dist/src/internal/nitro/routes/agent-info/build-agent-info-response.js +1 -1
- package/dist/src/internal/nitro/routes/agent-info/load-agent-info-data.d.ts +13 -0
- package/dist/src/internal/nitro/routes/agent-info/load-agent-info-data.js +1 -1
- package/dist/src/internal/nitro/routes/info.d.ts +2 -2
- package/dist/src/internal/nitro/routes/info.js +1 -1
- package/dist/src/internal/workflow/queue-namespace.d.ts +5 -0
- package/dist/src/internal/workflow/queue-namespace.js +1 -0
- package/dist/src/internal/workflow-bundle/builder-support.js +2 -2
- package/dist/src/internal/workflow-bundle/builder.js +3 -5
- package/dist/src/internal/workflow-bundle/vercel-workflow-output.js +1 -1
- package/dist/src/internal/workflow-bundle/workflow-builders.d.ts +1 -1
- package/dist/src/internal/workflow-bundle/workflow-builders.js +1 -1
- package/dist/src/node_modules/.pnpm/@clack_core@1.3.1/node_modules/@clack/core/dist/index.js +4 -4
- package/dist/src/protocol/message.d.ts +15 -0
- package/dist/src/protocol/message.js +2 -2
- package/dist/src/public/channels/slack/api.d.ts +8 -0
- package/dist/src/public/channels/slack/api.js +1 -1
- package/dist/src/public/channels/slack/connections.d.ts +26 -18
- package/dist/src/public/channels/slack/connections.js +1 -1
- package/dist/src/public/channels/slack/defaults.d.ts +5 -2
- package/dist/src/public/channels/slack/defaults.js +1 -1
- package/dist/src/public/channels/slack/index.d.ts +1 -1
- package/dist/src/public/channels/slack/slackChannel.d.ts +65 -5
- package/dist/src/public/channels/slack/slackChannel.js +1 -1
- package/dist/src/public/channels/teams/defaults.js +1 -1
- package/dist/src/public/connections/errors.d.ts +8 -0
- package/dist/src/public/definitions/tool.d.ts +0 -33
- package/dist/src/public/next/index.d.ts +7 -1
- package/dist/src/public/next/index.js +1 -1
- package/dist/src/public/next/server.d.ts +1 -0
- package/dist/src/public/next/server.js +1 -1
- package/dist/src/public/nuxt/dev-server.js +1 -1
- package/dist/src/public/sveltekit/dev-server.js +1 -1
- package/dist/src/public/sveltekit/index.d.ts +1 -1
- package/dist/src/public/tools/defaults.d.ts +2 -4
- package/dist/src/public/tools/defaults.js +1 -1
- package/dist/src/public/tools/define-bash-tool.d.ts +3 -3
- package/dist/src/public/tools/define-bash-tool.js +1 -1
- package/dist/src/public/tools/define-read-file-tool.d.ts +0 -6
- package/dist/src/public/tools/define-read-file-tool.js +1 -1
- package/dist/src/public/tools/index.d.ts +2 -2
- package/dist/src/public/tools/index.js +1 -1
- package/dist/src/public/tools/internal.js +1 -1
- package/dist/src/runtime/actions/types.d.ts +11 -11
- package/dist/src/runtime/agent/mock-model-adapter.js +1 -1
- package/dist/src/runtime/agent/mock-model-fixtures.js +3 -2
- package/dist/src/runtime/agent/mock-model-skill-selection.js +3 -4
- package/dist/src/runtime/connections/callback-route.js +1 -1
- package/dist/src/runtime/connections/mcp-client.js +1 -1
- package/dist/src/runtime/connections/scoped-authorization.d.ts +21 -5
- package/dist/src/runtime/connections/scoped-authorization.js +1 -1
- package/dist/src/runtime/connections/types.d.ts +33 -0
- package/dist/src/runtime/connections/validate-authorization.js +1 -1
- package/dist/src/runtime/framework-tools/bash.d.ts +3 -3
- package/dist/src/runtime/framework-tools/bash.js +1 -1
- package/dist/src/runtime/framework-tools/connection-search-dynamic.d.ts +1 -1
- package/dist/src/runtime/framework-tools/connection-search-dynamic.js +1 -1
- package/dist/src/runtime/framework-tools/file-state.d.ts +3 -3
- package/dist/src/runtime/framework-tools/index.js +1 -1
- package/dist/src/runtime/framework-tools/read-file.js +2 -2
- package/dist/src/runtime/framework-tools/todo.d.ts +7 -0
- package/dist/src/runtime/framework-tools/todo.js +2 -2
- package/dist/src/runtime/governance/auth/http-basic.js +1 -1
- package/dist/src/runtime/input/types.d.ts +1 -1
- package/dist/src/runtime/resolve-tool.d.ts +2 -2
- package/dist/src/runtime/resolve-tool.js +1 -1
- package/dist/src/runtime/sandbox/keys.js +1 -1
- package/dist/src/runtime/session-callback-route.js +1 -1
- package/dist/src/runtime/types.d.ts +1 -7
- package/dist/src/services/dev-client/client-options.d.ts +8 -0
- package/dist/src/services/dev-client/client-options.js +1 -0
- package/dist/src/services/dev-client/runtime-artifacts.d.ts +13 -0
- package/dist/src/services/dev-client/runtime-artifacts.js +1 -0
- package/dist/src/services/dev-client.d.ts +13 -46
- package/dist/src/services/dev-client.js +1 -1
- package/dist/src/setup/ask.d.ts +205 -0
- package/dist/src/setup/ask.js +1 -0
- package/dist/src/setup/boxes/add-channels.d.ts +100 -16
- package/dist/src/setup/boxes/add-channels.js +2 -1
- package/dist/src/setup/boxes/add-connections.d.ts +13 -23
- package/dist/src/setup/boxes/add-connections.js +1 -1
- package/dist/src/setup/boxes/apply-ai-gateway-credential.d.ts +2 -2
- package/dist/src/setup/boxes/apply-ai-gateway-credential.js +1 -1
- package/dist/src/setup/boxes/deploy-project.d.ts +46 -14
- package/dist/src/setup/boxes/deploy-project.js +1 -1
- package/dist/src/setup/boxes/detect-ai-gateway.d.ts +10 -3
- package/dist/src/setup/boxes/detect-ai-gateway.js +1 -1
- package/dist/src/setup/boxes/link-project.d.ts +3 -3
- package/dist/src/setup/boxes/link-project.js +1 -1
- package/dist/src/setup/boxes/one-shot-next-steps.d.ts +18 -0
- package/dist/src/setup/boxes/one-shot-next-steps.js +2 -0
- package/dist/src/setup/boxes/preflight.d.ts +14 -6
- package/dist/src/setup/boxes/preflight.js +1 -1
- package/dist/src/setup/boxes/resolve-provisioning.d.ts +36 -8
- package/dist/src/setup/boxes/resolve-provisioning.js +1 -1
- package/dist/src/setup/boxes/resolve-target.d.ts +25 -8
- package/dist/src/setup/boxes/resolve-target.js +1 -1
- package/dist/src/setup/boxes/scaffold.d.ts +12 -6
- package/dist/src/setup/boxes/scaffold.js +1 -1
- package/dist/src/setup/boxes/select-channels.d.ts +38 -9
- package/dist/src/setup/boxes/select-channels.js +1 -1
- package/dist/src/setup/boxes/select-chat.d.ts +15 -11
- package/dist/src/setup/boxes/select-chat.js +1 -1
- package/dist/src/setup/boxes/select-connections.d.ts +30 -0
- package/dist/src/setup/boxes/select-connections.js +1 -0
- package/dist/src/setup/boxes/select-model.d.ts +18 -14
- package/dist/src/setup/boxes/select-model.js +1 -1
- package/dist/src/setup/boxes/select-setup-mode.d.ts +32 -0
- package/dist/src/setup/boxes/select-setup-mode.js +1 -0
- package/dist/src/setup/channel-add-conflicts.d.ts +28 -0
- package/dist/src/setup/channel-add-conflicts.js +1 -0
- package/dist/src/setup/cli/channel-setup-prompter.d.ts +23 -0
- package/dist/src/setup/cli/channel-setup-prompter.js +1 -0
- package/dist/src/setup/cli/connection-add-prompter.d.ts +8 -0
- package/dist/src/setup/cli/connection-add-prompter.js +1 -0
- package/dist/src/setup/{scaffold/cli → cli}/index.d.ts +4 -3
- package/dist/src/setup/cli/index.js +1 -0
- package/dist/src/setup/{scaffold/cli → cli}/prompt-ui.d.ts +39 -15
- package/dist/src/setup/cli/prompt-ui.js +5 -0
- package/dist/src/setup/{scaffold/cli → cli}/rail-log.d.ts +2 -0
- package/dist/src/setup/{scaffold/cli → cli}/rail-log.js +2 -2
- package/dist/src/setup/{scaffold/cli → cli}/select-component.d.ts +18 -3
- package/dist/src/setup/cli/select-component.js +1 -0
- package/dist/src/setup/cli/select-option-codec.d.ts +12 -0
- package/dist/src/setup/cli/select-option-codec.js +1 -0
- package/dist/src/setup/{scaffold/cli → cli}/select-state.d.ts +13 -1
- package/dist/src/setup/cli/select-state.js +1 -0
- package/dist/src/setup/cli/whimsy.d.ts +16 -0
- package/dist/src/setup/cli/whimsy.js +1 -0
- package/dist/src/setup/{scaffold/steps/setup-connection.d.ts → connection-connector.d.ts} +3 -2
- package/dist/src/setup/connection-connector.js +1 -0
- package/dist/src/setup/flows/channels.d.ts +43 -0
- package/dist/src/setup/flows/channels.js +1 -0
- package/dist/src/setup/flows/deploy.d.ts +40 -0
- package/dist/src/setup/flows/deploy.js +1 -0
- package/dist/src/setup/flows/in-project.d.ts +16 -0
- package/dist/src/setup/flows/in-project.js +1 -0
- package/dist/src/setup/flows/link.d.ts +43 -0
- package/dist/src/setup/flows/link.js +1 -0
- package/dist/src/setup/flows/model.d.ts +112 -0
- package/dist/src/setup/flows/model.js +1 -0
- package/dist/src/setup/flows/vercel.d.ts +31 -0
- package/dist/src/setup/flows/vercel.js +2 -0
- package/dist/src/setup/gateway-models.js +1 -1
- package/dist/src/setup/headless.d.ts +1 -1
- package/dist/src/setup/index.d.ts +10 -4
- package/dist/src/setup/index.js +1 -1
- package/dist/src/setup/onboarding.d.ts +7 -4
- package/dist/src/setup/onboarding.js +1 -1
- package/dist/src/setup/package-manager.d.ts +27 -0
- package/dist/src/setup/package-manager.js +1 -0
- package/dist/src/setup/primitives/index.d.ts +3 -0
- package/dist/src/setup/primitives/index.js +1 -0
- package/dist/src/setup/primitives/pm/bun.d.ts +10 -0
- package/dist/src/setup/primitives/pm/bun.js +1 -0
- package/dist/src/setup/primitives/pm/index.d.ts +11 -0
- package/dist/src/setup/primitives/pm/index.js +1 -0
- package/dist/src/setup/primitives/pm/npm.d.ts +10 -0
- package/dist/src/setup/primitives/pm/npm.js +1 -0
- package/dist/src/setup/primitives/pm/pnpm.d.ts +27 -0
- package/dist/src/setup/primitives/pm/pnpm.js +8 -0
- package/dist/src/setup/primitives/pm/run.d.ts +23 -0
- package/dist/src/setup/primitives/pm/run.js +1 -0
- package/dist/src/setup/primitives/pm/shared.d.ts +8 -0
- package/dist/src/setup/primitives/pm/shared.js +1 -0
- package/dist/src/setup/primitives/pm/types.d.ts +37 -0
- package/dist/src/setup/primitives/pm/types.js +1 -0
- package/dist/src/setup/primitives/pm/yarn.d.ts +10 -0
- package/dist/src/setup/primitives/pm/yarn.js +1 -0
- package/dist/src/setup/primitives/run-pnpm.d.ts +1 -0
- package/dist/src/setup/primitives/run-pnpm.js +1 -0
- package/dist/src/setup/{scaffold/primitives → primitives}/run-vercel.d.ts +7 -0
- package/dist/src/setup/primitives/run-vercel.js +1 -0
- package/dist/src/setup/project-name.d.ts +4 -0
- package/dist/src/setup/project-name.js +1 -0
- package/dist/src/setup/project-resolution.d.ts +54 -0
- package/dist/src/setup/project-resolution.js +1 -0
- package/dist/src/setup/prompter.d.ts +52 -4
- package/dist/src/setup/prompter.js +1 -1
- package/dist/src/setup/quit-guard.d.ts +1 -1
- package/dist/src/setup/run-vercel-link.d.ts +1 -1
- package/dist/src/setup/run-vercel-link.js +1 -1
- package/dist/src/setup/runner.d.ts +5 -4
- package/dist/src/setup/runner.js +1 -1
- package/dist/src/setup/scaffold/channels-catalog.d.ts +3 -3
- package/dist/src/setup/scaffold/channels-catalog.js +1 -1
- package/dist/src/setup/scaffold/create/add-to-project.d.ts +26 -0
- package/dist/src/setup/scaffold/create/add-to-project.js +1 -0
- package/dist/src/setup/scaffold/create/project.d.ts +54 -0
- package/dist/src/setup/scaffold/create/project.js +80 -0
- package/dist/src/setup/scaffold/index.d.ts +4 -4
- package/dist/src/setup/scaffold/index.js +1 -1
- package/dist/src/setup/scaffold/{channels.d.ts → update/channels.d.ts} +11 -0
- package/dist/src/setup/scaffold/update/channels.js +7 -0
- package/dist/src/setup/scaffold/{connections.d.ts → update/connections.d.ts} +1 -1
- package/dist/src/setup/scaffold/update/connections.js +21 -0
- package/dist/src/setup/scaffold/version-tokens.d.ts +11 -0
- package/dist/src/setup/scaffold/version-tokens.js +1 -0
- package/dist/src/setup/{scaffold/steps/setup-slackbot.d.ts → slackbot.d.ts} +24 -20
- package/dist/src/setup/slackbot.js +1 -0
- package/dist/src/setup/state.d.ts +62 -15
- package/dist/src/setup/state.js +1 -1
- package/dist/src/setup/step.d.ts +9 -18
- package/dist/src/setup/vercel-project.d.ts +15 -8
- package/dist/src/setup/vercel-project.js +1 -1
- package/dist/src/shared/agent-definition.d.ts +5 -3
- package/dist/src/shared/default-agent-model.d.ts +5 -0
- package/dist/src/shared/default-agent-model.js +1 -0
- package/dist/src/source-change/apply-model-name.d.ts +25 -0
- package/dist/src/source-change/apply-model-name.js +2 -0
- package/dist/src/source-change/static-source-change.d.ts +36 -0
- package/dist/src/source-change/static-source-change.js +1 -0
- package/dist/src/svelte/index.js +1 -1
- package/dist/src/svelte/use-eve-agent.js +1 -1
- package/dist/src/vue/index.js +1 -1
- package/dist/src/vue/use-eve-agent.js +1 -1
- package/package.json +22 -42
- package/dist/docs/evals-v2-plan.md +0 -939
- package/dist/docs/public/advanced/dev-tui.md +0 -52
- package/dist/docs/public/advanced/evals.md +0 -158
- package/dist/docs/public/reference/faqs.md +0 -48
- package/dist/src/cli/commands/setup.d.ts +0 -55
- package/dist/src/cli/commands/setup.js +0 -1
- package/dist/src/cli/dev/repl/input-requests.d.ts +0 -38
- package/dist/src/cli/dev/repl/input-requests.js +0 -1
- package/dist/src/cli/dev/repl/input.d.ts +0 -19
- package/dist/src/cli/dev/repl/input.js +0 -1
- package/dist/src/cli/dev/repl/repl.d.ts +0 -62
- package/dist/src/cli/dev/repl/repl.js +0 -2
- package/dist/src/cli/dev/repl/terminal.d.ts +0 -21
- package/dist/src/cli/dev/repl/terminal.js +0 -5
- package/dist/src/compiled/_chunks/workflow/resume-hook-0Zk0zSvq.js +0 -12
- package/dist/src/compiled/_chunks/workflow/sleep-DXZr2BgM.js +0 -1
- package/dist/src/compiled/_chunks/workflow/symbols-BWCAoPHE.js +0 -48
- package/dist/src/evals/checks/checks.d.ts +0 -66
- package/dist/src/evals/checks/checks.js +0 -2
- package/dist/src/evals/checks/index.d.ts +0 -21
- package/dist/src/evals/checks/index.js +0 -1
- package/dist/src/evals/checks/match.js +0 -1
- package/dist/src/evals/define-eval-suite.d.ts +0 -18
- package/dist/src/evals/define-eval-suite.js +0 -1
- package/dist/src/evals/runner/execute-case.d.ts +0 -23
- package/dist/src/evals/runner/execute-case.js +0 -1
- package/dist/src/evals/runner/execute-suite.d.ts +0 -24
- package/dist/src/evals/runner/execute-suite.js +0 -1
- package/dist/src/evals/scorers/autoevals-client.js +0 -2
- package/dist/src/evals/scorers/autoevals.d.ts +0 -58
- package/dist/src/evals/scorers/autoevals.js +0 -1
- package/dist/src/evals/scorers/json.d.ts +0 -10
- package/dist/src/evals/scorers/json.js +0 -1
- package/dist/src/evals/scorers/model-marker.d.ts +0 -12
- package/dist/src/evals/scorers/model-marker.js +0 -1
- package/dist/src/evals/scorers/run.d.ts +0 -24
- package/dist/src/evals/scorers/run.js +0 -1
- package/dist/src/evals/scorers/sql.d.ts +0 -9
- package/dist/src/evals/scorers/sql.js +0 -1
- package/dist/src/evals/scorers/text.d.ts +0 -18
- package/dist/src/evals/scorers/text.js +0 -1
- package/dist/src/evals/scores/index.d.ts +0 -72
- package/dist/src/evals/scores/index.js +0 -1
- package/dist/src/execution/tool-compaction.d.ts +0 -9
- package/dist/src/execution/tool-compaction.js +0 -1
- package/dist/src/services/dev-client/stream.d.ts +0 -5
- package/dist/src/services/dev-client/stream.js +0 -1
- package/dist/src/services/dev-client/url.d.ts +0 -11
- package/dist/src/services/dev-client/url.js +0 -1
- package/dist/src/setup/channel-setup-prompter.d.ts +0 -8
- package/dist/src/setup/channel-setup-prompter.js +0 -1
- package/dist/src/setup/scaffold/channels.js +0 -7
- package/dist/src/setup/scaffold/cli/channel-add-prompter.d.ts +0 -12
- package/dist/src/setup/scaffold/cli/channel-add-prompter.js +0 -1
- package/dist/src/setup/scaffold/cli/channel-setup-prompter.d.ts +0 -56
- package/dist/src/setup/scaffold/cli/connection-add-prompter.d.ts +0 -44
- package/dist/src/setup/scaffold/cli/connection-add-prompter.js +0 -1
- package/dist/src/setup/scaffold/cli/index.js +0 -1
- package/dist/src/setup/scaffold/cli/prompt-ui.js +0 -5
- package/dist/src/setup/scaffold/cli/select-component.js +0 -1
- package/dist/src/setup/scaffold/cli/select-state.js +0 -1
- package/dist/src/setup/scaffold/connections.js +0 -21
- package/dist/src/setup/scaffold/pnpm-workspace.d.ts +0 -3
- package/dist/src/setup/scaffold/pnpm-workspace.js +0 -11
- package/dist/src/setup/scaffold/primitives/detect-deployment.d.ts +0 -13
- package/dist/src/setup/scaffold/primitives/detect-deployment.js +0 -1
- package/dist/src/setup/scaffold/primitives/index.d.ts +0 -3
- package/dist/src/setup/scaffold/primitives/index.js +0 -1
- package/dist/src/setup/scaffold/primitives/pnpm-invocation.d.ts +0 -12
- package/dist/src/setup/scaffold/primitives/pnpm-invocation.js +0 -1
- package/dist/src/setup/scaffold/primitives/run-pnpm.d.ts +0 -17
- package/dist/src/setup/scaffold/primitives/run-pnpm.js +0 -1
- package/dist/src/setup/scaffold/primitives/run-vercel.js +0 -1
- package/dist/src/setup/scaffold/project.d.ts +0 -21
- package/dist/src/setup/scaffold/project.js +0 -80
- package/dist/src/setup/scaffold/steps/deploy-to-vercel.d.ts +0 -17
- package/dist/src/setup/scaffold/steps/deploy-to-vercel.js +0 -1
- package/dist/src/setup/scaffold/steps/index.d.ts +0 -4
- package/dist/src/setup/scaffold/steps/index.js +0 -1
- package/dist/src/setup/scaffold/steps/project-resolution.d.ts +0 -19
- package/dist/src/setup/scaffold/steps/project-resolution.js +0 -1
- package/dist/src/setup/scaffold/steps/run-add-connection.d.ts +0 -40
- package/dist/src/setup/scaffold/steps/run-add-connection.js +0 -1
- package/dist/src/setup/scaffold/steps/run-add-to-agent.d.ts +0 -81
- package/dist/src/setup/scaffold/steps/run-add-to-agent.js +0 -2
- package/dist/src/setup/scaffold/steps/setup-connection.js +0 -1
- package/dist/src/setup/scaffold/steps/setup-slackbot.js +0 -1
- /package/dist/docs/public/{frontend → guides/frontend}/meta.json +0 -0
- /package/dist/docs/public/{advanced → guides}/remote-agents.md +0 -0
- /package/dist/src/{setup/scaffold/cli/channel-setup-prompter.js → cli/dev/tui/setup-flow.js} +0 -0
- /package/dist/src/evals/{scorers/autoevals-client.d.ts → autoevals-client.d.ts} +0 -0
- /package/dist/src/setup/{scaffold/cli → cli}/command-output.d.ts +0 -0
- /package/dist/src/setup/{scaffold/cli → cli}/command-output.js +0 -0
- /package/dist/src/setup/{scaffold/human-action.d.ts → human-action.d.ts} +0 -0
- /package/dist/src/setup/{scaffold/human-action.js → human-action.js} +0 -0
- /package/dist/src/setup/{scaffold/primitives → primitives}/process-output.d.ts +0 -0
- /package/dist/src/setup/{scaffold/primitives → primitives}/process-output.js +0 -0
- /package/dist/src/setup/scaffold/{web-template.d.ts → create/web-template.d.ts} +0 -0
- /package/dist/src/setup/scaffold/{web-template.js → create/web-template.js} +0 -0
- /package/dist/src/setup/scaffold/{module-files.d.ts → update/module-files.d.ts} +0 -0
- /package/dist/src/setup/scaffold/{module-files.js → update/module-files.js} +0 -0
- /package/dist/src/setup/scaffold/{package-json.d.ts → update/package-json.d.ts} +0 -0
- /package/dist/src/setup/scaffold/{package-json.js → update/package-json.js} +0 -0
- /package/dist/src/setup/scaffold/{primitives → update}/update-connection-connector.d.ts +0 -0
- /package/dist/src/setup/scaffold/{primitives → update}/update-connection-connector.js +0 -0
- /package/dist/src/setup/scaffold/{primitives → update}/update-slack-channel.d.ts +0 -0
- /package/dist/src/setup/scaffold/{primitives → update}/update-slack-channel.js +0 -0
|
@@ -5,7 +5,7 @@ description: "How users reach your agent: the channel contract, the base Eve HTT
|
|
|
5
5
|
|
|
6
6
|
A channel is the edge adapter between a platform and your agent, and its job is deliberately small. It does exactly three things: normalizes platform input into a user message, owns the `continuationToken` (the resume handle for a conversation on that surface), and decides delivery (how, where, and whether a response goes back).
|
|
7
7
|
|
|
8
|
-
Eve ships a base HTTP channel plus first-class platform channels
|
|
8
|
+
Eve ships a base HTTP channel plus first-class platform channels, and you can author your own. Browse the full set in the [Integrations](/integrations) gallery.
|
|
9
9
|
|
|
10
10
|
## Where channels live
|
|
11
11
|
|
|
@@ -24,30 +24,36 @@ Scaffold a channel file with `eve channels add` (interactive), or pass a kind: `
|
|
|
24
24
|
|
|
25
25
|
## The Eve HTTP channel (default)
|
|
26
26
|
|
|
27
|
-
Eve's canonical HTTP session API: the routes the terminal UI, [`useEveAgent`](../frontend/overview), and `curl` all talk to. It is enabled by default, even with no `agent/channels/eve.ts` file. Add that file only to override the defaults, most often the route auth policy. See [Eve channel](./eve) for the routes, auth, and customization.
|
|
27
|
+
Eve's canonical HTTP session API: the routes the terminal UI, [`useEveAgent`](../guides/frontend/overview), and `curl` all talk to. It is enabled by default, even with no `agent/channels/eve.ts` file. Add that file only to override the defaults, most often the route auth policy. See [Eve channel](./eve) for the routes, auth, and customization.
|
|
28
28
|
|
|
29
29
|
## Custom channels
|
|
30
30
|
|
|
31
31
|
When Eve doesn't ship a channel for your surface, build one with `defineChannel` from `eve/channels`: route handlers (`GET`, `POST`, `PUT`, `PATCH`, `DELETE`, `WS`), an `events` map, and a `send` call inside a handler to start or resume a session. See [Custom channels](./custom) for the full walkthrough, including WebSocket routes, cross-channel hand-off, channel metadata, continuation tokens, and file uploads.
|
|
32
32
|
|
|
33
|
+
## Relationship to the Chat SDK
|
|
34
|
+
|
|
35
|
+
Eve uses the Chat SDK's **card-builder components** (Cards, Buttons, Actions, etc.) for composing rich Slack messages. When you build a card with the [Slack channel](./slack), the underlying primitives come from the Chat SDK and get converted to Slack Block Kit at post time.
|
|
36
|
+
|
|
37
|
+
Eve does **not** use the Chat SDK's runtime. The `Chat`, `Adapter`, and `Thread` primitives are never imported or reachable through Eve's public API. Eve's channel layer (webhook handling, signature verification, event parsing, and thread management) is implemented inside Eve itself. In short: building Slack messages feels just like Chat SDK cards, but wiring a channel means authoring against Eve's `defineChannel(...)` API, not a Chat SDK adapter.
|
|
38
|
+
|
|
33
39
|
## Which channel?
|
|
34
40
|
|
|
35
|
-
| You want… | Use
|
|
36
|
-
| ------------------------------------------- |
|
|
37
|
-
| A web app / browser chat UI | Eve channel + [`useEveAgent`](../frontend/overview) |
|
|
38
|
-
| Local tooling, SDK clients, `curl` | Eve channel (default)
|
|
39
|
-
| Slack mentions, DMs, buttons | [Slack](./slack)
|
|
40
|
-
| Discord slash commands, components | [Discord](./discord)
|
|
41
|
-
| Microsoft Teams messages + Adaptive Cards | [Teams](./teams)
|
|
42
|
-
| Telegram bot messages | [Telegram](./telegram)
|
|
43
|
-
| SMS or speech-transcribed phone calls | [Twilio](./twilio)
|
|
44
|
-
| GitHub @mentions, PR review with checkout | [GitHub](./github)
|
|
45
|
-
| Linear issue delegation and Agent Sessions | [Linear](./linear)
|
|
46
|
-
| Anything else (internal webhook, WebSocket) | Custom channel (`defineChannel`, above)
|
|
41
|
+
| You want… | Use |
|
|
42
|
+
| ------------------------------------------- | ---------------------------------------------------------- |
|
|
43
|
+
| A web app / browser chat UI | Eve channel + [`useEveAgent`](../guides/frontend/overview) |
|
|
44
|
+
| Local tooling, SDK clients, `curl` | Eve channel (default) |
|
|
45
|
+
| Slack mentions, DMs, buttons | [Slack](./slack) |
|
|
46
|
+
| Discord slash commands, components | [Discord](./discord) |
|
|
47
|
+
| Microsoft Teams messages + Adaptive Cards | [Teams](./teams) |
|
|
48
|
+
| Telegram bot messages | [Telegram](./telegram) |
|
|
49
|
+
| SMS or speech-transcribed phone calls | [Twilio](./twilio) |
|
|
50
|
+
| GitHub @mentions, PR review with checkout | [GitHub](./github) |
|
|
51
|
+
| Linear issue delegation and Agent Sessions | [Linear](./linear) |
|
|
52
|
+
| Anything else (internal webhook, WebSocket) | Custom channel (`defineChannel`, above) |
|
|
47
53
|
|
|
48
54
|
## What to read next
|
|
49
55
|
|
|
50
56
|
- [Slack](./slack): the most common platform channel, end to end
|
|
51
57
|
- [Custom channels](./custom): build a channel for any surface with `defineChannel`
|
|
52
|
-
- [Frontend](../frontend/overview): browser chat on the Eve channel with `useEveAgent`
|
|
58
|
+
- [Frontend](../guides/frontend/overview): browser chat on the Eve channel with `useEveAgent`
|
|
53
59
|
- [Integrations](/integrations): browse every built-in channel and connection in one gallery
|
|
@@ -4,14 +4,14 @@ description: "Reach your agent from Slack app mentions and DMs, with thread anch
|
|
|
4
4
|
type: integration
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
The Slack channel puts your agent inside a workspace: it answers `@mentions` and DMs, replies in threads, shows typing indicators, and turns HITL prompts into buttons. Use it when the conversation should happen where your team already works. Credentials run through [Vercel Connect](../
|
|
7
|
+
The Slack channel puts your agent inside a workspace: it answers `@mentions` and DMs, replies in threads, shows typing indicators, and turns HITL prompts into buttons. Use it when the conversation should happen where your team already works. Credentials run through [Vercel Connect](../guides/auth-and-route-protection), which handles both the outbound bot token and inbound webhook verification, so there's no `SLACK_BOT_TOKEN` or `SLACK_SIGNING_SECRET` for you to manage. See [Channels](./overview) for the contract this builds on.
|
|
8
8
|
|
|
9
9
|
## Set up Connect
|
|
10
10
|
|
|
11
11
|
Create a Slack Connect client and copy its UID (e.g. `slack/my-agent`), then attach this project as the trigger destination at Eve's Slack route:
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
|
|
14
|
+
npm install -g vercel@latest && export FF_CONNECT_ENABLED=1
|
|
15
15
|
vercel connect create slack --triggers
|
|
16
16
|
vercel connect detach <uid> --yes
|
|
17
17
|
vercel connect attach <uid> --triggers --trigger-path /eve/v1/slack --yes
|
|
@@ -24,7 +24,7 @@ vercel connect attach <uid> --triggers --trigger-path /eve/v1/slack --yes
|
|
|
24
24
|
Scaffold the channel and its dependency with `eve channels add slack`, or set it up by hand:
|
|
25
25
|
|
|
26
26
|
```bash
|
|
27
|
-
|
|
27
|
+
npm install @vercel/connect
|
|
28
28
|
```
|
|
29
29
|
|
|
30
30
|
```ts title="agent/channels/slack.ts"
|
|
@@ -68,6 +68,18 @@ async onAppMention(ctx, message) {
|
|
|
68
68
|
|
|
69
69
|
**HITL** renders as Slack buttons/selects; submissions resume the parked session.
|
|
70
70
|
|
|
71
|
+
**Authorization prompts are private.** A sign-in challenge (OAuth URL, device code) is a credential. Anyone who completes it binds their identity to the session's connection. The default `authorization.required` handler delivers the challenge ephemerally to the triggering user, device code included, and posts a public link-free status only when it has no user to target. The handler receives a private-delivery context with `postEphemeral`, `postDirectMessage` (needs the `im:write` scope), and `state`. There is, intentionally, no public `post` and no raw API access.
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
events: {
|
|
75
|
+
"authorization.required"(event, ctx) {
|
|
76
|
+
const userId = ctx.state.triggeringUserId;
|
|
77
|
+
if (!userId || !event.authorization?.url) return;
|
|
78
|
+
return ctx.postDirectMessage(userId, `Sign in to continue: ${event.authorization.url}`);
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
```
|
|
82
|
+
|
|
71
83
|
```ts
|
|
72
84
|
import { defaultSlackAuth, slackChannel } from "eve/channels/slack";
|
|
73
85
|
|
|
@@ -89,4 +101,4 @@ Event handlers receive `(eventData, ctx)` with platform handles on `ctx.thread`
|
|
|
89
101
|
## What to read next
|
|
90
102
|
|
|
91
103
|
- [Channels overview](./overview): the channel contract and every built-in channel
|
|
92
|
-
- [Auth & route protection](../
|
|
104
|
+
- [Auth & route protection](../guides/auth-and-route-protection): authenticating inbound traffic
|
|
@@ -52,4 +52,4 @@ export default teamsChannel({
|
|
|
52
52
|
## What to read next
|
|
53
53
|
|
|
54
54
|
- [Channels overview](./overview): the channel contract and every built-in channel
|
|
55
|
-
- [Auth & route protection](../
|
|
55
|
+
- [Auth & route protection](../guides/auth-and-route-protection): authenticating inbound traffic
|
|
@@ -53,4 +53,4 @@ export default telegramChannel({
|
|
|
53
53
|
## What to read next
|
|
54
54
|
|
|
55
55
|
- [Channels overview](./overview): the channel contract and every built-in channel
|
|
56
|
-
- [Auth & route protection](../
|
|
56
|
+
- [Auth & route protection](../guides/auth-and-route-protection): authenticating inbound traffic
|
|
@@ -59,4 +59,4 @@ export default twilioChannel({
|
|
|
59
59
|
## What to read next
|
|
60
60
|
|
|
61
61
|
- [Channels overview](./overview): the channel contract and every built-in channel
|
|
62
|
-
- [Auth & route protection](../
|
|
62
|
+
- [Auth & route protection](../guides/auth-and-route-protection): authenticating inbound traffic
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Context
|
|
2
|
+
title: "Context Control"
|
|
3
3
|
description: "Control what the model sees and when: instructions, skills, the workspace, and subagents."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -80,7 +80,7 @@ See [Subagents](../subagents).
|
|
|
80
80
|
|
|
81
81
|
## Dynamic context with `defineDynamic`
|
|
82
82
|
|
|
83
|
-
The levers above are static: authored once, the same on every session. When the right context depends on who is calling (their team, tenant, plan, or feature flags), resolve it at runtime instead. `defineDynamic` in `agent/instructions/` returns the per-session system prompt, and `defineDynamic` in `agent/skills/` returns the set of skills a caller can load. Both read `ctx.session.auth` or channel metadata, so a caller on the billing team gets the billing instructions and playbook while nobody else sees them. See [Dynamic capabilities](
|
|
83
|
+
The levers above are static: authored once, the same on every session. When the right context depends on who is calling (their team, tenant, plan, or feature flags), resolve it at runtime instead. `defineDynamic` in `agent/instructions/` returns the per-session system prompt, and `defineDynamic` in `agent/skills/` returns the set of skills a caller can load. Both read `ctx.session.auth` or channel metadata, so a caller on the billing team gets the billing instructions and playbook while nobody else sees them. See [Dynamic capabilities](../guides/dynamic-capabilities) for the resolver API and when each event fires.
|
|
84
84
|
|
|
85
85
|
## Choosing the right lever
|
|
86
86
|
|
|
@@ -106,4 +106,4 @@ For most agents:
|
|
|
106
106
|
- [Tools](../tools)
|
|
107
107
|
- [Skills](../skills)
|
|
108
108
|
- [Subagents](../subagents)
|
|
109
|
-
- [Hooks](
|
|
109
|
+
- [Hooks](../guides/hooks)
|
|
@@ -18,7 +18,7 @@ export default defineAgent({
|
|
|
18
18
|
});
|
|
19
19
|
```
|
|
20
20
|
|
|
21
|
-
Compaction
|
|
21
|
+
Compaction also preserves the framework's own tool state automatically. When the harness compacts history, it resets read-before-write tracking (so a write afterward re-reads the file whose read evidence was summarized away) and re-injects the active todo list, so the model keeps its task list across the summary. There is no per-tool hook to configure.
|
|
22
22
|
|
|
23
23
|
## Built-in tools
|
|
24
24
|
|
|
@@ -41,7 +41,7 @@ These ship with every agent, no imports. Discovery never runs them: the harness
|
|
|
41
41
|
|
|
42
42
|
Notes:
|
|
43
43
|
|
|
44
|
-
- **`agent`** runs a copy of the current agent on a focused task. It inherits the same tools, connections, and instructions, but starts with fresh conversation history and fresh [state](
|
|
44
|
+
- **`agent`** runs a copy of the current agent on a focused task. It inherits the same tools, connections, and instructions, but starts with fresh conversation history and fresh [state](../guides/state). The child shares the parent's sandbox filesystem, so anything it writes is visible to the parent. See [Subagents](../subagents).
|
|
45
45
|
- **`load_skill`** only pulls instructions into context. It adds no new execution surface, because behavior still comes from the tools the agent already has.
|
|
46
46
|
- **`connection_search`** is the model-facing `connection__search` tool. A search surfaces a connection's tools by their qualified name (e.g. `connection__linear__list_issues`), and the model can then call them directly. It's registered only when the agent has connections.
|
|
47
47
|
- **`web_search`** has no local executor; the provider runs it. To supply your own implementation, override it with `defineTool()`.
|
|
@@ -85,10 +85,10 @@ There's also an experimental `Workflow` tool, shipped but off by default. To tur
|
|
|
85
85
|
export { ExperimentalWorkflow as default } from "eve/tools";
|
|
86
86
|
```
|
|
87
87
|
|
|
88
|
-
With it on, the model can orchestrate the agent's own subagents from model-authored JavaScript, all as one durable step. See [Dynamic workflows](
|
|
88
|
+
With it on, the model can orchestrate the agent's own subagents from model-authored JavaScript, all as one durable step. See [Dynamic workflows](../guides/dynamic-workflows).
|
|
89
89
|
|
|
90
90
|
## What to read next
|
|
91
91
|
|
|
92
|
-
- [Tools](../tools): define your own tools, gate them on approval, and shape their output
|
|
93
|
-
- [Dynamic capabilities](
|
|
92
|
+
- [Tools](../tools): define your own tools, gate them on approval, and shape their output with `toModelOutput`
|
|
93
|
+
- [Dynamic capabilities](../guides/dynamic-capabilities): generate the tool set per session with `defineDynamic`
|
|
94
94
|
- [Sandbox](../sandbox): the sandbox the shell and file tools run in
|
|
@@ -21,6 +21,8 @@ Crash the process, hit a timeout, or redeploy mid-turn, and the run picks up fro
|
|
|
21
21
|
|
|
22
22
|
There's nothing to configure here. Eve owns the workflow lifecycle, and sessions are durable by default.
|
|
23
23
|
|
|
24
|
+
You don't write workflow code directly. Workflow primitives (`start()`, `resumeHook()`, etc.) are an implementation detail of Eve's runtime layer; channels, tools, and hooks never touch them. When you do need session data from your own code, there are two supported surfaces: tools read the current session's metadata (id, turn, auth, parent lineage) via `ctx.session`, and [`defineState`](../guides/session-context) reads or writes session-scoped durable state. See [State](../guides/state) for the read/write model.
|
|
25
|
+
|
|
24
26
|
## Parked work
|
|
25
27
|
|
|
26
28
|
Some work has to wait: a human approving a [tool](../tools), an interactive OAuth sign-in for a [connection](../connections), or a long-running [subagent](../subagents). At those points the turn parks durably. The workflow suspends and holds no compute until the input it's waiting on shows up (a click, a callback, a child completing), even if that's much later. When it does, the conversation picks up exactly where it left off.
|
|
@@ -45,4 +47,4 @@ Conversation history within a session is append-only. Turns land in order, and t
|
|
|
45
47
|
|
|
46
48
|
- [Sessions, runs & streaming](./sessions-runs-and-streaming): the handles you hold and the event stream you watch.
|
|
47
49
|
- [Security model](./security-model): the trust boundaries the runtime enforces.
|
|
48
|
-
- [State](../
|
|
50
|
+
- [State](../guides/state): durable per-session memory that persists across step boundaries.
|
|
@@ -42,7 +42,7 @@ A [channel](../channels/overview) is your agent's front door, which makes authen
|
|
|
42
42
|
claims. A body field is attacker-controlled; treating it as identity is
|
|
43
43
|
cross-user impersonation.
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
A custom channel that accepts dashboard-style webhooks should follow the same shape: authenticate the raw body with an HMAC, compare signatures in constant time, and trust any body-supplied principal only after the signature verifies.
|
|
46
46
|
|
|
47
47
|
## Authored markdown is data
|
|
48
48
|
|
|
@@ -50,7 +50,7 @@ The `support-fixture` dashboard channel is a concrete custom-channel fixture for
|
|
|
50
50
|
|
|
51
51
|
## Auth fails closed
|
|
52
52
|
|
|
53
|
-
Routes reject unauthenticated traffic by default: if no `AuthFn` in the walk accepts the request, it gets a `401`, and admitting anonymous callers takes an explicit `none()`. The scaffold's `placeholderAuth()` keeps a half-configured app closed in production until you replace it. See [Auth & route protection](../
|
|
53
|
+
Routes reject unauthenticated traffic by default: if no `AuthFn` in the walk accepts the request, it gets a `401`, and admitting anonymous callers takes an explicit `none()`. The scaffold's `placeholderAuth()` keeps a half-configured app closed in production until you replace it. See [Auth & route protection](../guides/auth-and-route-protection) for the full walk and verifiers.
|
|
54
54
|
|
|
55
55
|
## Pre-production checklist
|
|
56
56
|
|
|
@@ -73,7 +73,7 @@ Before exposing an agent to real traffic:
|
|
|
73
73
|
|
|
74
74
|
## What to read next
|
|
75
75
|
|
|
76
|
-
- [Auth & route protection](
|
|
76
|
+
- [Auth & route protection](../guides/auth-and-route-protection): the full auth walk and verifier helpers
|
|
77
77
|
- [Sandbox](../sandbox): backends, network policy, and brokering config
|
|
78
78
|
- [Execution model & durability](./execution-model-and-durability): how durable sessions run
|
|
79
79
|
- [Connections](../connections): static-token and OAuth connections
|
|
@@ -14,7 +14,7 @@ Two handles do two jobs here, and mixing them up is the most common mistake. One
|
|
|
14
14
|
|
|
15
15
|
A session has one active continuation at a time: each follow-up uses the current `continuationToken`, and a stale one is rejected.
|
|
16
16
|
|
|
17
|
-
React, Vue, and Svelte apps reach for [`useEveAgent()`](../frontend/overview) instead of calling these routes by hand. Next.js and Nuxt apps can proxy them to the Eve runtime from the same origin.
|
|
17
|
+
React, Vue, and Svelte apps reach for [`useEveAgent()`](../guides/frontend/overview) instead of calling these routes by hand. Next.js and Nuxt apps can proxy them to the Eve runtime from the same origin.
|
|
18
18
|
|
|
19
19
|
## Start a session
|
|
20
20
|
|
|
@@ -96,7 +96,7 @@ curl "http://127.0.0.1:3000/eve/v1/session/<sessionId>/stream?startIndex=<count>
|
|
|
96
96
|
|
|
97
97
|
For scripts, server-to-server calls, tests, evals, and custom UIs, `eve/client` wraps these routes in a typed client so you don't hand-roll the POST and NDJSON stream loop.
|
|
98
98
|
|
|
99
|
-
Start with the [TypeScript
|
|
99
|
+
Start with the [TypeScript SDK](../guides/client/overview) guide. It covers basic usage, sending messages, continuations, streaming, and per-turn `outputSchema` results.
|
|
100
100
|
|
|
101
101
|
## Inspect the agent over HTTP
|
|
102
102
|
|
|
@@ -106,7 +106,7 @@ Start with the [TypeScript Client](../client/overview) guide. It covers basic us
|
|
|
106
106
|
curl http://127.0.0.1:3000/eve/v1/info
|
|
107
107
|
```
|
|
108
108
|
|
|
109
|
-
The route uses the same default auth chain as the eve channel (`[localDev(), vercelOidc()]`). Locally it answers anonymously; a deployed Vercel target requires a valid OIDC bearer, with a same-project bypass for in-deployment callers. See [auth & route protection](../
|
|
109
|
+
The route uses the same default auth chain as the eve channel (`[localDev(), vercelOidc()]`). Locally it answers anonymously; a deployed Vercel target requires a valid OIDC bearer, with a same-project bypass for in-deployment callers. See [auth & route protection](../guides/auth-and-route-protection).
|
|
110
110
|
|
|
111
111
|
## Dispatch order
|
|
112
112
|
|
|
@@ -114,8 +114,8 @@ Every stream event runs four steps, in this order:
|
|
|
114
114
|
|
|
115
115
|
1. **Channel handler**: the channel's event handler runs and can mutate adapter state.
|
|
116
116
|
2. **Metadata projection**: the framework re-evaluates the channel's `metadata(state)` and stores the result.
|
|
117
|
-
3. **Hooks**: authored [hooks](../
|
|
118
|
-
4. **Dynamic resolvers**: [dynamic](../
|
|
117
|
+
3. **Hooks**: authored [hooks](../guides/hooks) subscribed to the event fire.
|
|
118
|
+
4. **Dynamic resolvers**: [dynamic](../guides/dynamic-capabilities) tool, skill, and instruction resolvers fire, and `ctx.channel.metadata` already holds the freshly projected metadata from step 2.
|
|
119
119
|
|
|
120
120
|
The order isn't incidental, it's structural. By the time a resolver or hook reads channel metadata, the channel has already updated its state and the projection is current.
|
|
121
121
|
|
|
@@ -123,5 +123,5 @@ The order isn't incidental, it's structural. By the time a resolver or hook read
|
|
|
123
123
|
|
|
124
124
|
- [Execution model & durability](./execution-model-and-durability): what makes a session durable and how parked work resumes.
|
|
125
125
|
- [Channels](../channels/overview): what owns the continuation token and delivery.
|
|
126
|
-
- [TypeScript
|
|
127
|
-
- [Frontend](../frontend/overview): `useEveAgent` instead of raw routes.
|
|
126
|
+
- [TypeScript SDK](../guides/client/overview): call these routes from scripts and server-side code.
|
|
127
|
+
- [Frontend](../guides/frontend/overview): `useEveAgent` instead of raw routes.
|
|
@@ -124,7 +124,7 @@ export default defineMcpClientConnection({
|
|
|
124
124
|
});
|
|
125
125
|
```
|
|
126
126
|
|
|
127
|
-
`"linear"` is the UID you chose when registering the Connect client. Connect-managed OAuth is user-scoped by default, so the runtime resolves the per-user token before each tool call. The full setup (Connect client provisioning, project linking, the runtime consent flow) lives in [Auth & route protection](./
|
|
127
|
+
`"linear"` is the UID you chose when registering the Connect client. Connect-managed OAuth is user-scoped by default, so the runtime resolves the per-user token before each tool call. The full setup (Connect client provisioning, project linking, the runtime consent flow) lives in [Auth & route protection](./guides/auth-and-route-protection).
|
|
128
128
|
|
|
129
129
|
## Self-hosted interactive OAuth
|
|
130
130
|
|
|
@@ -168,7 +168,9 @@ export default defineMcpClientConnection({
|
|
|
168
168
|
});
|
|
169
169
|
```
|
|
170
170
|
|
|
171
|
-
`getToken` runs before every tool call. `startAuthorization` and `completeAuthorization` are both-or-neither: provide one without the other and you get a definition error. The `challenge` rides along verbatim on the `authorization.required` event. Set `url` for redirect/device flows, `userCode` for a device code,
|
|
171
|
+
`getToken` runs before every tool call. `startAuthorization` and `completeAuthorization` are both-or-neither: provide one without the other and you get a definition error. The `challenge` rides along verbatim on the `authorization.required` event. Set `url` for redirect/device flows, `userCode` for a device code, `instructions` as the call to action when there's no URL, and `displayName` for the human-readable provider name channels show on the sign-in affordance (e.g. "Salesforce"). Drop `resume` when the provider keeps flow state server-side, so nothing has to cross the step boundary.
|
|
172
|
+
|
|
173
|
+
`displayName` is presentation-only — the connection's path-derived name still keys the authorization scope, token cache, and callback URL. You can also set `displayName` on the `auth` definition itself (e.g. `auth: { ...connect("sfdc"), displayName: "Salesforce" }`); that definition-level value wins over one the strategy stamps on the challenge, and channels fall back to title-casing the connection name when neither is set.
|
|
172
174
|
|
|
173
175
|
### Signaling authorization state
|
|
174
176
|
|
|
@@ -213,5 +215,5 @@ A tool can require both sign-in (`auth`) and a human approval. Today the model's
|
|
|
213
215
|
|
|
214
216
|
- [Integrations](/integrations): browse every channel and connection Eve ships, in one gallery.
|
|
215
217
|
- [Tools](./tools): authored tools live alongside connection-provided tools; the same approval helpers apply.
|
|
216
|
-
- [Auth & route protection](./
|
|
217
|
-
- [Security model](./
|
|
218
|
+
- [Auth & route protection](./guides/auth-and-route-protection): the full interactive-OAuth flow with Vercel Connect.
|
|
219
|
+
- [Security model](./concepts/security-model): how connection credentials stay out of the model's reach.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Assertions"
|
|
3
|
+
description: "Run-level methods, t.check value assertions, the matcher mini-language, and gate vs soft severity."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Assertions are how an eval grades what its `test(t)` function produced. Each one **records** a result onto `t` and returns a chainable handle — the runner reads the recorded results to compute the verdict, so a single run reports every failing assertion rather than dying on the first. There are two deterministic surfaces: run-level methods on `t`, and `t.check` for grading a specific value. For model-graded assertions, see [Judge](./judge).
|
|
7
|
+
|
|
8
|
+
## Run-level assertions
|
|
9
|
+
|
|
10
|
+
Run-level assertions read the whole run, so they take no value. They are methods on `t` and gate by default.
|
|
11
|
+
|
|
12
|
+
| Assertion | Asserts |
|
|
13
|
+
| --------------------------------------------------- | --------------------------------------------------------------------------------- |
|
|
14
|
+
| `t.completed()` | The run did not fail and did not park on unanswered HITL input |
|
|
15
|
+
| `t.didNotFail()` | No terminal failure and no `turn.failed`/`step.failed` events (parked runs pass) |
|
|
16
|
+
| `t.waiting()` | The run parked on HITL input (for approval-shaped evals) |
|
|
17
|
+
| `t.messageIncludes(token)` | Joined assistant text contains `token` (string or RegExp) |
|
|
18
|
+
| `t.outputEquals(value)` / `t.outputMatches(schema)` | Deep equality / Standard Schema (e.g. Zod) validation of the parsed output |
|
|
19
|
+
| `t.calledTool(name, opts?)` | A matching tool call happened (`input`, `output`, `isError`, `times` constraints) |
|
|
20
|
+
| `t.notCalledTool(name)` | No call to `name` |
|
|
21
|
+
| `t.toolOrder([...names])` | Tool names appear in order (other calls may interleave) |
|
|
22
|
+
| `t.usedNoTools()` | No tool calls at all |
|
|
23
|
+
| `t.maxToolCalls(n)` | At most `n` tool calls |
|
|
24
|
+
| `t.noFailedActions()` | No tool, subagent, or skill action reported a failure |
|
|
25
|
+
| `t.calledSubagent(name, opts?)` | A subagent delegation happened (`remoteUrl`, `output` constraints) |
|
|
26
|
+
| `t.event(predicate, label)` | Escape hatch: any predicate over the typed event stream |
|
|
27
|
+
|
|
28
|
+
`t.completed()` subsumes `t.didNotFail()` — reach for `completed` unless you specifically want to allow a parked run.
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
await t.send("What is the weather in Brooklyn?");
|
|
32
|
+
t.completed();
|
|
33
|
+
t.calledTool("get_weather");
|
|
34
|
+
t.usedNoTools(); // mutually exclusive with the line above — pick the one you mean
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Value assertions with `t.check`
|
|
38
|
+
|
|
39
|
+
`t.check(value, assertion)` grades an explicit value against a builder from `eve/evals/expect`. The value can be `t.reply`, a turn's `.message`, parsed JSON, or any local you computed:
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
import { includes, equals, matches, similarity } from "eve/evals/expect";
|
|
43
|
+
|
|
44
|
+
t.check(t.reply, includes("sunny")); // substring (gate)
|
|
45
|
+
t.check(parsed, equals({ city: "Brooklyn" })); // deep structural equality (gate)
|
|
46
|
+
t.check(parsed, matches(WeatherSchema)); // Standard Schema, e.g. Zod (gate)
|
|
47
|
+
t.check(t.reply, similarity("Sunny, 72F")); // fuzzy 0–1 Levenshtein (soft)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
| Builder | Scores | Default |
|
|
51
|
+
| ---------------------- | ------------------------------------------------ | ------- |
|
|
52
|
+
| `includes(substring)` | value (coerced to string) contains `substring` | gate |
|
|
53
|
+
| `equals(value)` | deep structural equality | gate |
|
|
54
|
+
| `matches(schema)` | validates against a Standard Schema | gate |
|
|
55
|
+
| `similarity(expected)` | normalized Levenshtein similarity, 1 = identical | soft |
|
|
56
|
+
|
|
57
|
+
Pick the cheapest builder that captures what "correct" means. When exact match is too strict but a judge model is overkill, `similarity` is the middle ground; for nuanced grading, reach for the [judge](./judge).
|
|
58
|
+
|
|
59
|
+
## The matcher mini-language
|
|
60
|
+
|
|
61
|
+
`t.calledTool` and `t.calledSubagent` take a matcher object — `{ input, output, isError, times }` for tools, `{ remoteUrl, output }` for subagents. Each field accepts a literal (objects partial-deep-match), a RegExp, or a function. A matcher function receives the value and returns either a boolean (acts as a predicate) or an expected value to compare against (handy for runner-assigned values like environment-provided URLs):
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
t.calledTool("bash", { input: { command: /^pwd/ }, isError: false, times: 1 });
|
|
65
|
+
|
|
66
|
+
t.calledTool("echo", { output: (value) => String(value).includes(marker) });
|
|
67
|
+
|
|
68
|
+
t.calledSubagent("weather", {
|
|
69
|
+
remoteUrl: () => process.env.WEATHER_AGENT_URL!,
|
|
70
|
+
output: /72F/,
|
|
71
|
+
});
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Run state and derived facts
|
|
75
|
+
|
|
76
|
+
A turn that leaves the session open for a next message is the normal end state of a successful turn. Parking on unanswered HITL input is tracked separately — that is what `t.completed()` and `t.waiting()` key off.
|
|
77
|
+
|
|
78
|
+
Beyond the raw `t.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output, error state), subagent calls, and HITL input requests. The built-in assertions cover almost everything; when you need to read the stream directly, `t.event(predicate, label)` is the escape hatch:
|
|
79
|
+
|
|
80
|
+
```ts
|
|
81
|
+
t.event(
|
|
82
|
+
(events) =>
|
|
83
|
+
events.some((e) => e.type === "message.completed" && e.data.message?.includes(marker)),
|
|
84
|
+
"assistant reply includes the marker",
|
|
85
|
+
);
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Severity
|
|
89
|
+
|
|
90
|
+
Every assertion returns a chainable handle. Severity rides on the assertion — there is no separate thresholds map to keep in sync.
|
|
91
|
+
|
|
92
|
+
- `.gate(threshold?)` — hard. A miss marks the eval `failed` and `eve eval` exits non-zero.
|
|
93
|
+
- `.soft(threshold?)` — tracked data. A below-threshold miss marks the eval `scored`, fatal only under `--strict`. With no threshold, it is tracked-only and never fails.
|
|
94
|
+
- `.atLeast(threshold)` — soft with a bar (equivalent to `.soft(threshold)`).
|
|
95
|
+
|
|
96
|
+
The defaults are chosen so you rarely set severity. Run-level methods and `includes`/`equals`/`matches` are gates; `similarity` and every `t.judge.*` assertion are soft. Annotate only when you deviate:
|
|
97
|
+
|
|
98
|
+
```ts
|
|
99
|
+
t.calledTool("get_weather").soft(); // record the tool call as a metric, don't gate
|
|
100
|
+
t.check(t.reply, similarity("Sunny")).atLeast(0.8); // gate the fuzzy match under --strict
|
|
101
|
+
t.check(t.reply, includes("error")).soft(); // track without failing the build
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## What to read next
|
|
105
|
+
|
|
106
|
+
- [Judge](./judge): LLM-graded assertions with thresholds
|
|
107
|
+
- [Cases](./cases): where assertions attach
|
|
108
|
+
- [Running evals](./running): how verdicts map to exit codes
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Cases"
|
|
3
|
+
description: "Author single-turn and multi-turn evals with test(t), and fan one file out over a dataset."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Each eval file is one graded case. The runner executes its `test(t)` function against the target, captures every event, and computes a verdict from the [assertions](./assertions) you recorded. Every eval — single-turn, multi-turn, HITL, or dataset-driven — is the same shape: one `async test(t)` function that drives the agent and asserts inline.
|
|
7
|
+
|
|
8
|
+
## Single-turn evals
|
|
9
|
+
|
|
10
|
+
The common case sends one turn and asserts on the reply. `t.send(input)` resolves once the turn settles; `t.reply` is the last assistant message:
|
|
11
|
+
|
|
12
|
+
```ts title="evals/weather/brooklyn-forecast.eval.ts"
|
|
13
|
+
import { defineEval } from "eve/evals";
|
|
14
|
+
import { includes } from "eve/evals/expect";
|
|
15
|
+
|
|
16
|
+
export default defineEval({
|
|
17
|
+
async test(t) {
|
|
18
|
+
await t.send("What is the weather in Brooklyn?");
|
|
19
|
+
t.completed();
|
|
20
|
+
t.check(t.reply, includes("Sunny"));
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Some evals only care about behavior, not text — assert on the run and skip the content check entirely:
|
|
26
|
+
|
|
27
|
+
```ts title="evals/weather/no-tools-for-greetings.eval.ts"
|
|
28
|
+
import { defineEval } from "eve/evals";
|
|
29
|
+
|
|
30
|
+
export default defineEval({
|
|
31
|
+
async test(t) {
|
|
32
|
+
await t.send("Hello!");
|
|
33
|
+
t.completed();
|
|
34
|
+
t.notCalledTool("get_weather");
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Organizing with directories
|
|
40
|
+
|
|
41
|
+
Identity is the file path, so directories are the grouping mechanism. `evals/weather/brooklyn-forecast.eval.ts` gets the id `weather/brooklyn-forecast`, and `eve eval weather` runs everything under `evals/weather/`. Shared constants and helpers live in sibling non-eval files (any name that doesn't end in `.eval.ts`):
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
evals/
|
|
45
|
+
├── weather/
|
|
46
|
+
│ ├── shared.ts # helpers — not an eval
|
|
47
|
+
│ ├── brooklyn-forecast.eval.ts
|
|
48
|
+
│ └── no-tools-for-greetings.eval.ts
|
|
49
|
+
└── smoke.eval.ts
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Multi-turn evals
|
|
53
|
+
|
|
54
|
+
Drive several turns in sequence — branching, HITL approvals, structured output, attachments, multiple sessions. Because assertions live in the function, an intermediate value is just a local variable: judge a draft before the next turn overwrites it, then keep going.
|
|
55
|
+
|
|
56
|
+
```ts title="evals/draft-then-send.eval.ts"
|
|
57
|
+
import { defineEval } from "eve/evals";
|
|
58
|
+
import { includes } from "eve/evals/expect";
|
|
59
|
+
|
|
60
|
+
export default defineEval({
|
|
61
|
+
async test(t) {
|
|
62
|
+
const draft = await t.send("Draft the follow-up email.");
|
|
63
|
+
t.check(draft.message, includes("Best regards"));
|
|
64
|
+
t.judge.autoevals.closedQA("professional tone", { on: draft.message }).atLeast(0.6);
|
|
65
|
+
|
|
66
|
+
await t.send("Now send it.");
|
|
67
|
+
t.calledTool("send_email");
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Bespoke preconditions that no built-in assertion expresses are plain `throw`s — a thrown error marks the eval `failed` with the message in the result:
|
|
73
|
+
|
|
74
|
+
```ts title="evals/session-continuity.eval.ts"
|
|
75
|
+
import { defineEval } from "eve/evals";
|
|
76
|
+
import { includes } from "eve/evals/expect";
|
|
77
|
+
|
|
78
|
+
export default defineEval({
|
|
79
|
+
requires: ["mockModels"],
|
|
80
|
+
async test(t) {
|
|
81
|
+
await t.send("My favorite word is marigold.");
|
|
82
|
+
const firstSessionId = t.sessionId;
|
|
83
|
+
|
|
84
|
+
const second = await t.send("Thanks for remembering.");
|
|
85
|
+
second.expectOk();
|
|
86
|
+
if (t.sessionId !== firstSessionId) {
|
|
87
|
+
throw new Error(`Expected one session; got ${firstSessionId} then ${t.sessionId}.`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
t.completed();
|
|
91
|
+
t.check(second.message, includes("Thanks for remembering."));
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## The drive API
|
|
97
|
+
|
|
98
|
+
`t` drives the primary session; `t.newSession()` returns an independent `EveEvalSession` against the same target, whose events feed the same run-level assertions.
|
|
99
|
+
|
|
100
|
+
- `t.send(input)` sends a turn and waits for it to settle. It accepts the same input as `ClientSession.send()` — a string or a structured message — and resolves to a turn carrying `.message` and `.expectOk()`.
|
|
101
|
+
- `t.sendFile(text, path, mediaType?)` attaches a local file as a data URL.
|
|
102
|
+
- `t.expectInputRequests(filter?)` asserts the previous turn parked on HITL input and returns the pending requests.
|
|
103
|
+
- `t.respond(...responses)` answers specific pending input requests and sends them as the next turn.
|
|
104
|
+
- `t.respondAll(optionId)` answers every pending input request with the same option and sends the responses as the next turn.
|
|
105
|
+
- `t.reply` is the last assistant message (or `null`); `t.sessionId` is the current session id; `t.events` is the full typed event stream captured so far.
|
|
106
|
+
|
|
107
|
+
Each `send` (and `respond`/`respondAll`) resolves to a turn whose `expectOk()` throws only when the turn ended failed — a session left open for a next message is the normal end state of a successful turn.
|
|
108
|
+
|
|
109
|
+
Events from every session are captured in the result and artifacts. `t.log(message)` records debug lines into the eval artifact; `--verbose` also streams them to stdout as evals run. `t.signal` is an `AbortSignal` that fires on timeout.
|
|
110
|
+
|
|
111
|
+
For driving sessions created outside the eval — by a channel webhook or a schedule — see [Targets and requirements](./targets).
|
|
112
|
+
|
|
113
|
+
## Datasets: exporting an array
|
|
114
|
+
|
|
115
|
+
To fan one file out over a dataset, default-export an array of `defineEval(...)` values. Eval modules are ESM, so top-level `await` can load anything. Ids derive from the file name plus a zero-padded index (`sql/0000`, `sql/0001`, …, in array order). The loaders (`loadJson`, `loadYaml` from `eve/evals/loaders`) parse fixture files relative to the app root:
|
|
116
|
+
|
|
117
|
+
```ts title="evals/sql.eval.ts"
|
|
118
|
+
import { defineEval } from "eve/evals";
|
|
119
|
+
import { loadYaml } from "eve/evals/loaders";
|
|
120
|
+
import { equals } from "eve/evals/expect";
|
|
121
|
+
|
|
122
|
+
const doc = await loadYaml("evals/data/cases.yaml");
|
|
123
|
+
const rows = doc.evals as readonly { task: string; prompt: string; sql: string }[];
|
|
124
|
+
|
|
125
|
+
export default rows.map((row) =>
|
|
126
|
+
defineEval({
|
|
127
|
+
description: row.task,
|
|
128
|
+
async test(t) {
|
|
129
|
+
await t.send(row.prompt);
|
|
130
|
+
t.completed();
|
|
131
|
+
t.check(t.reply, equals(row.sql));
|
|
132
|
+
},
|
|
133
|
+
}),
|
|
134
|
+
);
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The loaders are meant for fixtures, not runtime agent code.
|
|
138
|
+
|
|
139
|
+
## What to read next
|
|
140
|
+
|
|
141
|
+
- [Assertions](./assertions): assert on what the eval did
|
|
142
|
+
- [Judge](./judge): grade quality with an LLM judge
|
|
143
|
+
- [TypeScript client](../guides/client/messages): the send/turn protocol eval sessions build on
|