eve 0.77.0 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/src/approval/definition.d.ts +42 -0
- package/dist/src/cli/commands/trace-detail.d.ts +2 -2
- package/dist/src/cli/commands/trace-detail.js +1 -1
- package/dist/src/cli/commands/trace.d.ts +1 -1
- package/dist/src/cli/commands/trace.js +1 -1
- package/dist/src/cli/dev/tui/traces/trace-conversation.d.ts +1 -1
- package/dist/src/cli/dev/tui/traces/trace-conversation.js +1 -1
- package/dist/src/cli/dev/tui/traces/trace-store.d.ts +1 -1
- package/dist/src/cli/dev/tui/traces/trace-store.js +1 -1
- package/dist/src/cli/dev/tui/traces/trace-view.d.ts +1 -1
- package/dist/src/cli/dev/tui/traces/trace-view.js +1 -1
- package/dist/src/cli/dev/tui/traces/trace-viewer-state.d.ts +1 -1
- package/dist/src/compiled/.vendor-stamp.json +3 -1
- package/dist/src/compiled/@eve/tracing/adapters/agent-transport.d.ts +8 -0
- package/dist/src/compiled/@eve/tracing/adapters/ai-sdk-payload.d.ts +15 -0
- package/dist/src/compiled/@eve/tracing/adapters/ai-sdk-telemetry.d.ts +14 -0
- package/dist/src/{tracing/agent-span-id-generator.d.ts → compiled/@eve/tracing/adapters/otel-ids.d.ts} +1 -1
- package/dist/src/compiled/@eve/tracing/adapters/otel.d.ts +33 -0
- package/dist/src/{tracing/agent-otel-content.d.ts → compiled/@eve/tracing/adapters/serialization.d.ts} +9 -3
- package/dist/src/compiled/@eve/tracing/agent-tracing.d.ts +84 -0
- package/dist/src/compiled/@eve/tracing/ai-sdk.d.ts +7 -0
- package/dist/src/compiled/@eve/tracing/ai-sdk.js +1 -0
- package/dist/src/compiled/@eve/tracing/capture.d.ts +5 -0
- package/dist/src/compiled/@eve/tracing/core/activation.d.ts +3 -0
- package/dist/src/compiled/@eve/tracing/core/attributes.d.ts +14 -0
- package/dist/src/{tracing/content-attributes.d.ts → compiled/@eve/tracing/core/content-policy.d.ts} +1 -1
- package/dist/src/compiled/@eve/tracing/core/context.d.ts +3 -0
- package/dist/src/compiled/@eve/tracing/core/delegation.d.ts +13 -0
- package/dist/src/compiled/@eve/tracing/core/mcp.d.ts +24 -0
- package/dist/src/compiled/@eve/tracing/core/scopes.d.ts +59 -0
- package/dist/src/compiled/@eve/tracing/core/snapshot.d.ts +7 -0
- package/dist/src/compiled/@eve/tracing/core/span-kinds.d.ts +12 -0
- package/dist/src/compiled/@eve/tracing/core/types.d.ts +292 -0
- package/dist/src/compiled/@eve/tracing/core/writer.d.ts +15 -0
- package/dist/src/compiled/@eve/tracing/delegation.d.ts +2 -0
- package/dist/src/compiled/@eve/tracing/delegation.js +1 -0
- package/dist/src/compiled/@eve/tracing/index.d.ts +6 -0
- package/dist/src/compiled/@eve/tracing/index.js +1 -0
- package/dist/src/compiled/@eve/tracing/operations.d.ts +96 -0
- package/dist/src/compiled/@eve/tracing/otel.d.ts +12 -0
- package/dist/src/compiled/@eve/tracing/otel.js +1 -0
- package/dist/src/compiled/@opentelemetry/api/index.d.ts +4 -0
- package/dist/src/compiled/@workflow/world-vercel/index.js +7 -7
- package/dist/src/compiled/_chunks/workflow/delegation-BqhZ398D.js +1 -0
- package/dist/src/compiled/_chunks/workflow/mcp-CrGp4EC8.js +5 -0
- package/dist/src/compiler/extension-compatibility.d.ts +6 -6
- package/dist/src/compiler/extension-compatibility.js +1 -1
- package/dist/src/context/build-dynamic-tools.js +1 -1
- package/dist/src/context/dynamic-tool-lifecycle.js +1 -1
- package/dist/src/context/keys.d.ts +16 -0
- package/dist/src/context/keys.js +1 -1
- package/dist/src/evals/assertions/collector.d.ts +4 -1
- package/dist/src/evals/assertions/collector.js +1 -1
- package/dist/src/evals/context.js +1 -1
- package/dist/src/evals/index.d.ts +1 -1
- package/dist/src/evals/runner/artifacts.js +1 -1
- package/dist/src/evals/runner/reporters/assertion-diagnostics.js +2 -2
- package/dist/src/evals/runner/reporters/assertion-scores.d.ts +11 -2
- package/dist/src/evals/runner/reporters/assertion-scores.js +1 -1
- package/dist/src/evals/runner/reporters/braintrust.js +1 -1
- package/dist/src/evals/runner/reporters/console.js +1 -1
- package/dist/src/evals/runner/reporters/datadog.js +2 -2
- package/dist/src/evals/runner/reporters/junit.js +1 -1
- package/dist/src/evals/runner/verdict.d.ts +2 -2
- package/dist/src/evals/runner/verdict.js +1 -1
- package/dist/src/evals/types.d.ts +22 -6
- package/dist/src/eve-channel/index.js +1 -1
- package/dist/src/eve-channel/stream-headers.d.ts +1 -0
- package/dist/src/eve-channel/stream-headers.js +1 -0
- package/dist/src/execution/agent-sessions/context.d.ts +1 -1
- package/dist/src/execution/agent-sessions/context.js +1 -1
- package/dist/src/execution/agent-sessions/remote.js +1 -1
- package/dist/src/execution/agent-sessions/steps.d.ts +2 -0
- package/dist/src/execution/agent-sessions/steps.js +1 -1
- package/dist/src/execution/cancelled-model-call-batch.js +1 -1
- package/dist/src/execution/catalog/connection-entry.js +1 -1
- package/dist/src/execution/catalog/search.js +1 -1
- package/dist/src/execution/catalog/step-catalog.d.ts +12 -3
- package/dist/src/execution/catalog/step-catalog.js +1 -1
- package/dist/src/execution/coordination-dispatch-shared.d.ts +1 -2
- package/dist/src/execution/coordination-dispatch-shared.js +1 -1
- package/dist/src/execution/coordination-dispatch-step.js +1 -1
- package/dist/src/execution/dynamic-workflow/tool.js +1 -1
- package/dist/src/execution/eve-workflow-attributes.js +1 -1
- package/dist/src/execution/invoke-tool.d.ts +1 -1
- package/dist/src/execution/invoke-tool.js +1 -1
- package/dist/src/execution/sandbox/bindings/docker-container.js +1 -1
- package/dist/src/execution/sandbox/bindings/docker.js +1 -1
- package/dist/src/execution/sandbox/bindings/just-bash-runtime.js +1 -1
- package/dist/src/execution/sandbox/bindings/just-bash.js +1 -1
- package/dist/src/execution/sandbox/bindings/microsandbox-lifecycle.js +1 -1
- package/dist/src/execution/sandbox/bindings/microsandbox-runtime.js +1 -1
- package/dist/src/execution/sandbox/bindings/vercel.js +1 -1
- package/dist/src/execution/sandbox/session.d.ts +6 -0
- package/dist/src/execution/sandbox/session.js +2 -2
- package/dist/src/execution/session/active-turn.d.ts +9 -1
- package/dist/src/execution/session/active-turn.js +1 -1
- package/dist/src/execution/session/checkpoint-migrations.js +1 -1
- package/dist/src/execution/session/handoff.d.ts +1 -1
- package/dist/src/execution/session/handoff.js +1 -1
- package/dist/src/execution/session/next-input.js +1 -1
- package/dist/src/execution/session/turn-step.js +1 -1
- package/dist/src/execution/session/turn.d.ts +11 -4
- package/dist/src/execution/session/turn.js +1 -1
- package/dist/src/execution/session-inbox/inbox.d.ts +3 -0
- package/dist/src/execution/session-inbox/inbox.js +1 -1
- package/dist/src/execution/session-workflow-tool-run.d.ts +14 -12
- package/dist/src/execution/session-workflow-tool-run.js +1 -1
- package/dist/src/execution/tasks/start.d.ts +14 -1
- package/dist/src/execution/tasks/start.js +1 -1
- package/dist/src/execution/tasks/steps.d.ts +7 -4
- package/dist/src/execution/tasks/steps.js +1 -1
- package/dist/src/execution/tools/agent-router-workflow.js +1 -1
- package/dist/src/execution/tools/workflow/apply-run-messages-step.d.ts +20 -0
- package/dist/src/execution/tools/workflow/apply-run-messages-step.js +1 -0
- package/dist/src/execution/tools/workflow/ask.d.ts +0 -1
- package/dist/src/execution/tools/workflow/ask.js +1 -1
- package/dist/src/execution/tools/workflow/interrupt.d.ts +5 -2
- package/dist/src/execution/tools/workflow/interrupt.js +1 -1
- package/dist/src/execution/tools/workflow/messages.d.ts +14 -0
- package/dist/src/execution/tools/workflow/messages.js +1 -1
- package/dist/src/execution/tools/workflow/owner.d.ts +18 -6
- package/dist/src/execution/tools/workflow/owner.js +1 -1
- package/dist/src/execution/tools/workflow/start.d.ts +10 -0
- package/dist/src/execution/tools/workflow/start.js +1 -1
- package/dist/src/execution/tools/workflow/step.js +1 -1
- package/dist/src/execution/tools/workflow/workflow-owner-blocking.d.ts +12 -0
- package/dist/src/execution/tools/workflow/workflow-owner-blocking.js +1 -1
- package/dist/src/execution/tools/workflow/workflow.js +1 -1
- package/dist/src/execution/workflow-runtime.d.ts +18 -3
- package/dist/src/execution/workflow-runtime.js +1 -1
- package/dist/src/harness/execute-tool.d.ts +2 -2
- package/dist/src/harness/hitl/approvals.d.ts +4 -7
- package/dist/src/harness/hitl/approvals.js +1 -1
- package/dist/src/harness/hitl/approved-calls.d.ts +11 -0
- package/dist/src/harness/hitl/approved-calls.js +1 -1
- package/dist/src/harness/hitl/candidates.d.ts +0 -4
- package/dist/src/harness/hitl/candidates.js +1 -1
- package/dist/src/harness/hitl/delivery.d.ts +0 -1
- package/dist/src/harness/hitl/delivery.js +1 -1
- package/dist/src/harness/hitl/index.d.ts +2 -2
- package/dist/src/harness/hitl/index.js +1 -1
- package/dist/src/harness/hitl/intake.d.ts +2 -2
- package/dist/src/harness/hitl/intake.js +1 -1
- package/dist/src/harness/input-extraction.js +1 -1
- package/dist/src/harness/model-call/call.js +1 -1
- package/dist/src/harness/model-call/run.js +1 -1
- package/dist/src/harness/session-machine/state.d.ts +0 -2
- package/dist/src/harness/session-machine/view.d.ts +0 -3
- package/dist/src/harness/session-machine/view.js +1 -1
- package/dist/src/harness/step/after-model.js +1 -1
- package/dist/src/harness/tool-loop.js +1 -1
- package/dist/src/harness/tools.js +1 -1
- package/dist/src/instrumentation/ai-sdk-hook-bridge.js +1 -1
- package/dist/src/instrumentation/dispatch.js +1 -1
- package/dist/src/instrumentation/memory.d.ts +1 -4
- package/dist/src/instrumentation/memory.js +1 -1
- package/dist/src/instrumentation/principal-summary.js +1 -1
- package/dist/src/instrumentation/providers.js +1 -1
- package/dist/src/instrumentation/runtime-context.d.ts +1 -1
- package/dist/src/instrumentation/runtime.d.ts +3 -3
- package/dist/src/instrumentation/runtime.js +1 -1
- package/dist/src/instrumentation/session-init.js +1 -1
- package/dist/src/internal/application/stamped-package-version.js +1 -1
- package/dist/src/internal/authored-definition/approval.d.ts +3 -1
- package/dist/src/internal/authored-definition/approval.js +1 -1
- package/dist/src/internal/authored-definition/schema-backed.js +1 -1
- package/dist/src/internal/logging.js +1 -1
- package/dist/src/internal/nitro/host/local-tracing-runtime-plugin.js +1 -1
- package/dist/src/internal/nitro/routes/channel-dispatch.js +1 -1
- package/dist/src/internal/nitro/routes/channel-request-instrumentation.d.ts +10 -31
- package/dist/src/internal/nitro/routes/channel-request-instrumentation.js +1 -1
- package/dist/src/internal/nitro/routes/route-invoke-tool.d.ts +1 -1
- package/dist/src/internal/workflow-bundle/dynamic-tool-transform.js +1 -1
- package/dist/src/public/channels/slack/hitl.js +1 -1
- package/dist/src/public/definitions/approval.d.ts +1 -1
- package/dist/src/public/instrumentation/otel.d.ts +2 -2
- package/dist/src/public/instrumentation/otel.js +1 -1
- package/dist/src/public/tools/approval/index.d.ts +1 -1
- package/dist/src/runtime/connections/mcp-tracing.d.ts +10 -18
- package/dist/src/runtime/connections/mcp-tracing.js +1 -1
- package/dist/src/runtime/resolve-tool.js +1 -1
- package/dist/src/runtime/types.d.ts +2 -2
- package/dist/src/self-modification/extension/subagents/agent/tools/inspect_trace.d.ts +12 -12
- package/dist/src/self-modification/extension/subagents/agent/tools/inspect_trace.js +1 -1
- package/dist/src/self-modification/extension/subagents/agent/tools/search_traces.d.ts +1 -1
- package/dist/src/self-modification/extension/subagents/agent/tools/search_traces.js +1 -1
- package/dist/src/self-modification/extension/trace-inspection.d.ts +1 -1
- package/dist/src/self-modification/extension/trace-inspection.js +1 -1
- package/dist/src/setup/scaffold/create/project.js +1 -1
- package/dist/src/shared/trace-policy.d.ts +3 -0
- package/dist/src/shared/trace-policy.js +1 -1
- package/dist/src/tools/definition.d.ts +3 -3
- package/dist/src/tools/durable-callbacks.d.ts +5 -3
- package/dist/src/tools/durable-callbacks.js +1 -1
- package/dist/src/tools/dynamic.d.ts +2 -2
- package/dist/src/tools/provided/ask-question-workflow.js +1 -1
- package/dist/src/tools/provided/sleep-workflow.js +1 -1
- package/dist/src/tools/workflow-definition.d.ts +2 -2
- package/dist/src/tracing/eve/agent-approval-instrumentation.d.ts +8 -0
- package/dist/src/tracing/eve/agent-approval-instrumentation.js +1 -0
- package/dist/src/tracing/{agent-channel-delivery-instrumentation.d.ts → eve/agent-channel-delivery-instrumentation.d.ts} +1 -1
- package/dist/src/tracing/{agent-channel-delivery-instrumentation.js → eve/agent-channel-delivery-instrumentation.js} +1 -1
- package/dist/src/tracing/eve/agent-invocation-coordinator.js +1 -0
- package/dist/src/tracing/eve/agent-memory-instrumentation.d.ts +12 -0
- package/dist/src/tracing/eve/agent-memory-instrumentation.js +1 -0
- package/dist/src/tracing/{agent-otel-provider.d.ts → eve/agent-otel-provider.d.ts} +6 -8
- package/dist/src/tracing/eve/agent-otel-provider.js +1 -0
- package/dist/src/tracing/eve/agent-tool-instrumentation.d.ts +44 -0
- package/dist/src/tracing/eve/agent-tool-instrumentation.js +1 -0
- package/dist/src/tracing/{agent-trace-context-store.d.ts → eve/agent-trace-context-store.d.ts} +8 -16
- package/dist/src/tracing/eve/agent-trace-context-store.js +1 -0
- package/dist/src/tracing/eve/agent-trace-context.js +1 -0
- package/dist/src/tracing/eve/agent-trace-state.d.ts +75 -0
- package/dist/src/tracing/{batch-span-processor.d.ts → eve/batch-span-processor.d.ts} +5 -1
- package/dist/src/tracing/eve/batch-span-processor.js +1 -0
- package/dist/src/tracing/{content-span-processor.d.ts → eve/content-span-processor.d.ts} +1 -1
- package/dist/src/tracing/eve/content-span-processor.js +1 -0
- package/dist/src/tracing/eve/delegation.d.ts +20 -0
- package/dist/src/tracing/eve/delegation.js +1 -0
- package/dist/src/tracing/eve/gateway.d.ts +16 -0
- package/dist/src/tracing/eve/gateway.js +1 -0
- package/dist/src/tracing/{install-instrumentation-runtime.d.ts → eve/install-instrumentation-runtime.d.ts} +1 -1
- package/dist/src/tracing/eve/install-instrumentation-runtime.js +1 -0
- package/dist/src/tracing/eve/invoke-tool-span.js +1 -0
- package/dist/src/tracing/eve/mcp.d.ts +29 -0
- package/dist/src/tracing/eve/mcp.js +1 -0
- package/dist/src/tracing/eve/metadata.d.ts +7 -0
- package/dist/src/tracing/eve/metadata.js +1 -0
- package/dist/src/tracing/eve/operation-input.d.ts +30 -0
- package/dist/src/tracing/eve/operation-input.js +1 -0
- package/dist/src/tracing/eve/otel-configuration.d.ts +70 -0
- package/dist/src/tracing/eve/otel-configuration.js +1 -0
- package/dist/src/tracing/{otel-declaration.d.ts → eve/otel-declaration.d.ts} +4 -11
- package/dist/src/tracing/eve/otel-declaration.js +1 -0
- package/dist/src/tracing/{otel-registration.d.ts → eve/otel-registration.d.ts} +12 -5
- package/dist/src/tracing/eve/otel-registration.js +1 -0
- package/dist/src/tracing/eve/policy.d.ts +15 -0
- package/dist/src/tracing/eve/policy.js +1 -0
- package/dist/src/tracing/eve/profile.d.ts +7 -0
- package/dist/src/tracing/eve/profile.js +1 -0
- package/dist/src/tracing/eve/session.d.ts +27 -0
- package/dist/src/tracing/eve/session.js +1 -0
- package/dist/src/tracing/{vercel-runtime-span-exporter.d.ts → eve/vercel-runtime-span-exporter.d.ts} +1 -1
- package/dist/src/tracing/local/agent-trace-span-processor.js +1 -0
- package/dist/src/tracing/local/inspection.d.ts +15 -0
- package/dist/src/tracing/local/inspection.js +1 -0
- package/dist/src/tracing/{local-instrumentation-runtime.d.ts → local/instrumentation-runtime.d.ts} +1 -1
- package/dist/src/tracing/local/instrumentation-runtime.js +1 -0
- package/dist/src/tracing/{local-trace-analysis.d.ts → local/trace-analysis.d.ts} +2 -2
- package/dist/src/tracing/{local-trace-analysis.js → local/trace-analysis.js} +1 -1
- package/dist/src/tracing/{local-trace-operations.d.ts → local/trace-operations.d.ts} +1 -1
- package/dist/src/tracing/{local-trace-query.d.ts → local/trace-query.d.ts} +1 -1
- package/dist/src/tracing/local/trace-reader.js +1 -0
- package/dist/src/tracing/{local-trace-retention.js → local/trace-retention.js} +1 -1
- package/dist/src/tracing/{local-trace-span-processor.js → local/trace-span-processor.js} +1 -1
- package/dist/src/tracing/{local-trace-summary.d.ts → local/trace-summary.d.ts} +1 -1
- package/dist/src/tracing/local/trace-summary.js +1 -0
- package/dist/src/tracing/{local-traces.d.ts → local/traces.d.ts} +1 -1
- package/dist/src/tracing/local/traces.js +1 -0
- package/docs/agent-config.md +12 -4
- package/docs/concepts/built-in-tools.md +9 -9
- package/docs/connections/mcp.mdx +22 -1
- package/docs/evals/assertions.mdx +50 -3
- package/docs/evals/overview.mdx +2 -1
- package/docs/evals/reporters.mdx +3 -1
- package/docs/guides/dynamic-capabilities.md +6 -2
- package/docs/sandbox/index.mdx +2 -0
- package/docs/tools/human-in-the-loop.md +24 -2
- package/package.json +2 -1
- package/dist/src/context/caller-scope.d.ts +0 -7
- package/dist/src/context/caller-scope.js +0 -1
- package/dist/src/execution/tools/workflow/emit-workflow-tool-run-report-step.d.ts +0 -13
- package/dist/src/execution/tools/workflow/emit-workflow-tool-run-report-step.js +0 -1
- package/dist/src/harness/hitl/approved-call-callers.d.ts +0 -14
- package/dist/src/harness/hitl/approved-call-callers.js +0 -1
- package/dist/src/tracing/agent-approval-instrumentation.d.ts +0 -11
- package/dist/src/tracing/agent-approval-instrumentation.js +0 -1
- package/dist/src/tracing/agent-invocation-coordinator.js +0 -1
- package/dist/src/tracing/agent-memory-instrumentation.d.ts +0 -16
- package/dist/src/tracing/agent-memory-instrumentation.js +0 -1
- package/dist/src/tracing/agent-otel-attributes.d.ts +0 -10
- package/dist/src/tracing/agent-otel-attributes.js +0 -1
- package/dist/src/tracing/agent-otel-content.js +0 -4
- package/dist/src/tracing/agent-otel-provider.js +0 -2
- package/dist/src/tracing/agent-otel-runtime-context.d.ts +0 -36
- package/dist/src/tracing/agent-otel-runtime-context.js +0 -1
- package/dist/src/tracing/agent-otel-session-context.d.ts +0 -22
- package/dist/src/tracing/agent-otel-session-context.js +0 -1
- package/dist/src/tracing/agent-otel-turn-state.d.ts +0 -6
- package/dist/src/tracing/agent-otel-turn-state.js +0 -1
- package/dist/src/tracing/agent-otel-usage.d.ts +0 -10
- package/dist/src/tracing/agent-otel-usage.js +0 -1
- package/dist/src/tracing/agent-span-contract.d.ts +0 -34
- package/dist/src/tracing/agent-span-contract.js +0 -1
- package/dist/src/tracing/agent-span-error.js +0 -1
- package/dist/src/tracing/agent-span-id-generator.js +0 -1
- package/dist/src/tracing/agent-span-naming.d.ts +0 -1
- package/dist/src/tracing/agent-span-naming.js +0 -1
- package/dist/src/tracing/agent-tool-instrumentation.d.ts +0 -40
- package/dist/src/tracing/agent-tool-instrumentation.js +0 -1
- package/dist/src/tracing/agent-tool-span-context.d.ts +0 -16
- package/dist/src/tracing/agent-tool-span-context.js +0 -1
- package/dist/src/tracing/agent-trace-context-codec.d.ts +0 -15
- package/dist/src/tracing/agent-trace-context-codec.js +0 -1
- package/dist/src/tracing/agent-trace-context-store.js +0 -1
- package/dist/src/tracing/agent-trace-context.js +0 -1
- package/dist/src/tracing/agent-trace-span-processor.js +0 -1
- package/dist/src/tracing/agent-trace-state.d.ts +0 -91
- package/dist/src/tracing/batch-span-processor.js +0 -1
- package/dist/src/tracing/bounded-error.d.ts +0 -1
- package/dist/src/tracing/bounded-error.js +0 -1
- package/dist/src/tracing/content-attributes.js +0 -1
- package/dist/src/tracing/content-span-processor.js +0 -1
- package/dist/src/tracing/error-content-context.d.ts +0 -3
- package/dist/src/tracing/error-content-context.js +0 -1
- package/dist/src/tracing/install-instrumentation-runtime.js +0 -1
- package/dist/src/tracing/invoke-tool-span.js +0 -1
- package/dist/src/tracing/local-instrumentation-runtime.js +0 -1
- package/dist/src/tracing/local-trace-reader.js +0 -1
- package/dist/src/tracing/local-trace-summary.js +0 -1
- package/dist/src/tracing/local-traces.js +0 -1
- package/dist/src/tracing/otel-declaration.js +0 -1
- package/dist/src/tracing/otel-registration.js +0 -1
- package/dist/src/tracing/sampled-trace.d.ts +0 -4
- package/dist/src/tracing/sampled-trace.js +0 -1
- package/dist/src/tracing/suppress-tracing.d.ts +0 -2
- package/dist/src/tracing/suppress-tracing.js +0 -1
- package/dist/src/tracing/telemetry-budget.d.ts +0 -3
- package/dist/src/tracing/telemetry-budget.js +0 -1
- /package/dist/src/{tracing/agent-span-error.d.ts → compiled/@eve/tracing/adapters/otel-error.d.ts} +0 -0
- /package/dist/src/tracing/{agent-invocation-coordinator.d.ts → eve/agent-invocation-coordinator.d.ts} +0 -0
- /package/dist/src/tracing/{agent-trace-context.d.ts → eve/agent-trace-context.d.ts} +0 -0
- /package/dist/src/tracing/{agent-trace-state.js → eve/agent-trace-state.js} +0 -0
- /package/dist/src/tracing/{channel-audience-context.d.ts → eve/channel-audience-context.d.ts} +0 -0
- /package/dist/src/tracing/{channel-audience-context.js → eve/channel-audience-context.js} +0 -0
- /package/dist/src/tracing/{conversation-context.d.ts → eve/conversation-context.d.ts} +0 -0
- /package/dist/src/tracing/{conversation-context.js → eve/conversation-context.js} +0 -0
- /package/dist/src/tracing/{invoke-tool-span.d.ts → eve/invoke-tool-span.d.ts} +0 -0
- /package/dist/src/tracing/{span-export-policy.d.ts → eve/span-export-policy.d.ts} +0 -0
- /package/dist/src/tracing/{span-export-policy.js → eve/span-export-policy.js} +0 -0
- /package/dist/src/tracing/{vercel-runtime-span-exporter.js → eve/vercel-runtime-span-exporter.js} +0 -0
- /package/dist/src/tracing/{agent-trace-span-processor.d.ts → local/agent-trace-span-processor.d.ts} +0 -0
- /package/dist/src/tracing/{local-trace-discovery-index.d.ts → local/trace-discovery-index.d.ts} +0 -0
- /package/dist/src/tracing/{local-trace-discovery-index.js → local/trace-discovery-index.js} +0 -0
- /package/dist/src/tracing/{local-trace-operations.js → local/trace-operations.js} +0 -0
- /package/dist/src/tracing/{local-trace-query.js → local/trace-query.js} +0 -0
- /package/dist/src/tracing/{local-trace-reader.d.ts → local/trace-reader.d.ts} +0 -0
- /package/dist/src/tracing/{local-trace-retention.d.ts → local/trace-retention.d.ts} +0 -0
- /package/dist/src/tracing/{local-trace-span-processor.d.ts → local/trace-span-processor.d.ts} +0 -0
package/docs/agent-config.md
CHANGED
|
@@ -220,16 +220,24 @@ timeout. Expiration does not delete stored session data.
|
|
|
220
220
|
Input tokens, output tokens, and model token cost are checked independently.
|
|
221
221
|
The model call that crosses a limit is allowed to finish because exact usage
|
|
222
222
|
arrives after the call completes. Before the next model call, eve pauses the
|
|
223
|
-
|
|
224
|
-
**
|
|
223
|
+
turn (`turn.waiting`) and sends a deterministic continuation prompt with two options:
|
|
224
|
+
**Continue** grants a fresh window of each configured size and resumes the same
|
|
225
|
+
turn, and **Stop**
|
|
225
226
|
cancels the in-flight turn through the standard cancellation path
|
|
226
227
|
(`turn.cancelled` → `session.waiting`) — a user decision, not an error. The session stays resumable; because it is
|
|
227
|
-
still over budget, the next message
|
|
228
|
+
still over budget after Stop, the next message raises a new prompt. Declining a
|
|
228
229
|
delegated child's prompt cancels the root turn, which cascades to the whole
|
|
229
230
|
delegation tree — the delegating parent never receives an error result it
|
|
230
231
|
could retry against a fresh quota share. A reply that answers neither option
|
|
231
232
|
is queued while the existing prompt stays pending; eve does not raise another
|
|
232
|
-
copy. The reply is
|
|
233
|
+
copy or start another turn. The reply is received (`message.received`) and
|
|
234
|
+
processed in the held turn once the budget is granted.
|
|
235
|
+
|
|
236
|
+
For sessions already waiting on a continuation prompt from an older deployment,
|
|
237
|
+
the previous turn has completed. A non-answer message still queues behind the
|
|
238
|
+
existing prompt, but emits no event until Continue is chosen. Continue starts a
|
|
239
|
+
new turn and receives the queued message once; it does not reopen the completed
|
|
240
|
+
turn or duplicate the prompt.
|
|
233
241
|
|
|
234
242
|
Sessions that cannot request input from a human, such as markdown schedules and
|
|
235
243
|
delegated runs without input proxying, skip the prompt and fail the next model
|
|
@@ -24,7 +24,7 @@ export default defineAgent({
|
|
|
24
24
|
|
|
25
25
|
This turns off the optional defaults described below. Add back only the tools the agent needs with the command in each tool's section. Existing files under `agent/tools/` remain available, including same-name replacements such as `agent/tools/bash.ts`.
|
|
26
26
|
|
|
27
|
-
`defaultTools` doesn't affect [the catalog tools](#eve__search-eve__tool-and-eve__skill): eve adds them from what the
|
|
27
|
+
`defaultTools` doesn't affect [the catalog tools](#eve__search-eve__tool-and-eve__skill): eve adds them from what the session can reach, as listed in that section.
|
|
28
28
|
|
|
29
29
|
### `bash`
|
|
30
30
|
|
|
@@ -301,16 +301,16 @@ export default disableTool();
|
|
|
301
301
|
|
|
302
302
|
### `eve__search`, `eve__tool`, and `eve__skill`
|
|
303
303
|
|
|
304
|
-
These tools let the model reach what isn't in its tool list. `eve__search` finds tools defined with `deferred: true`, agents defined with `tool: "deferred"`, every tool from the agent's [connections](../connections), and deferred [skills](../skills). `eve__tool` calls a deferred tool, agent, or connection tool by name, and `eve__skill` loads a skill, deferred or not. There is no add command; eve adds each one
|
|
304
|
+
These tools let the model reach what isn't in its tool list. `eve__search` finds tools defined with `deferred: true`, agents defined with `tool: "deferred"`, every tool from the agent's [connections](../connections), and deferred [skills](../skills). `eve__tool` calls a deferred tool, agent, or connection tool by name, and `eve__skill` loads a skill, deferred or not. There is no add command; eve adds each one when the session has something it reaches, even when `defaultTools` is `false`:
|
|
305
305
|
|
|
306
|
-
- `eve__tool` comes with anything it can call: a deferred tool or agent,
|
|
307
|
-
- `eve__skill` comes with any skill,
|
|
308
|
-
- `eve__search` comes with anything it could find: whatever `eve__tool` can call, plus deferred skills
|
|
309
|
-
-
|
|
306
|
+
- `eve__tool` comes with anything it can call: a deferred tool or agent, or a connection.
|
|
307
|
+
- `eve__skill` comes with any skill, so an agent whose only catalog entries are listed skills gets `eve__skill` alone.
|
|
308
|
+
- `eve__search` comes with anything it could find: whatever `eve__tool` can call, plus deferred skills.
|
|
309
|
+
- A session with none of these gets no catalog tools.
|
|
310
310
|
|
|
311
|
-
|
|
311
|
+
Entries the agent declares count from the session's first step. An entry a dynamic resolver returns counts from the step where it first appears, so a resolver in `agent/tools/` that returns only tools without `deferred: true` adds no catalog tools. Once a session's tool list includes a catalog tool, the session keeps it, even after a resolver stops returning the entry that brought it. The catalog tools change only when the session first reaches a new kind of entry, such as its first deferred tool or its first connection.
|
|
312
312
|
|
|
313
|
-
- `eve__search({ query, limit? })` returns the best matches, up to `limit` (default 10, at most 50). `query` is required and must contain a word. A tool match has its exact `tool` name, its `description`, and a TypeScript `signature` rendered from its schemas. A deferred skill's match has its `skill` name, its `description`, and the `path` of its `SKILL.md` when it has supporting files. A result with `tool` is called with `eve__tool`, and one with `skill` is loaded with `eve__skill`. Tools and skills rank together, in tiers: an entry whose name is exactly the query, then one whose name after its last `__` is the query, such as `linear__create_issue` or `sre__create_issue` for `create_issue`, then the tools of a connection named by the query, then entries whose names, or names after their last `__`, start with the query, then keyword matches in names, parameters, and descriptions. So `eve__search({ query: "linear" })` lists the `linear` connection's tools first. One-letter and filler words such as `a` and `the` don't count toward keyword matches unless the query has nothing else.
|
|
313
|
+
- `eve__search({ query, limit? })` returns the best matches, up to `limit` (default 10, at most 50). `query` is required and must contain a word. A tool match has its exact `tool` name, its `description`, and a TypeScript `signature` rendered from its schemas. A deferred skill's match has its `skill` name, its `description`, and the `path` of its `SKILL.md` when it has supporting files. A result with `tool` is called with `eve__tool`, and one with `skill` is loaded with `eve__skill`. Next to `results`, `searched` counts the entries the query covered, the whole catalog or one namespace, and `remaining` counts the matches past `limit`. A connection behind sign-in counts as its one sign-in result until the user signs in, and an unavailable connection counts as none. Tools and skills rank together, in tiers: an entry whose name is exactly the query, then one whose name after its last `__` is the query, such as `linear__create_issue` or `sre__create_issue` for `create_issue`, then the tools of a connection named by the query, then entries whose names, or names after their last `__`, start with the query, then keyword matches in names, parameters, and descriptions. So `eve__search({ query: "linear" })` lists the `linear` connection's tools first. One-letter and filler words such as `a` and `the` don't count toward keyword matches unless the query has nothing else.
|
|
314
314
|
- A query whose first word contains `__` searches one namespace: everything before its last `__`, without trailing underscores. The namespace only filters. `eve__search` keeps names under `<namespace>__` and anything named exactly the namespace, such as a connection's sign-in entry, then ranks them by the whole query, so `eve__search({ query: "linear__" })` returns only the `linear` connection's tools, or its sign-in result, and an exact name that contains `__` still ranks first. eve lists only the connections that can own names in that namespace, so other connections make no network call and don't appear in `unavailable`. A namespace that matches nothing fails with the closest connection names. Characters that can't appear in a name, such as a leading `^`, are ignored; `query` isn't a regular expression.
|
|
315
315
|
- `eve__search` never asks the user to sign in. A connection whose server won't list its tools until the user signs in appears as one result named after the connection, such as `linear`; a connection whose server lists its tools without a token is searched like any other. A connection whose tools fail to load, or don't load within 10 seconds, appears under `unavailable` with its `error`, such as `"crm" did not list its tools within 10s. Try again later.` Matches from everything else are still returned, so a slow server doesn't hold up the search. An `eve__tool` call on that connection's tools fails with the same error.
|
|
316
316
|
- `eve__tool({ name: "linear" })`, with a connection's own name, asks the user to sign in to that connection when its tools need it, and returns once they can be listed, so the next `eve__search` finds them. On a server that lists its tools without sign-in, it confirms the tools are available without asking.
|
|
@@ -319,7 +319,7 @@ The decision depends only on what the agent declares, never on what its resolver
|
|
|
319
319
|
|
|
320
320
|
After `eve__tool` or `eve__skill` resolves its entry, the call runs exactly like a direct call to that entry. A skill load reports a `load-skill` action and a `load-skill-result`, both with the skill's `name`, and an `execute_tool eve:load-skill` span. Approval policies and `approvedTools`, workflow tools and tasks, agents, `endsTurn`, `toModelOutput`, hooks, and stream events all see the entry's own name and input, such as `linear__list_issues`. Only model history records the call as `eve__tool` or `eve__skill`.
|
|
321
321
|
|
|
322
|
-
eve tells the model what it can reach in an append-only context message rather than in the system prompt: which kinds of deferred entries exist, up to 20 namespaces (the first `__` segment of deferred names, such as `sre` for `sre__list_alerts`), and up to 20 connections with their descriptions. It never names a deferred entry, so deferred entries stay out of context until `eve__search` finds them. A later step appends the listing again only when that changes; a deferred entry added to a listed namespace, or without one, changes nothing.
|
|
322
|
+
eve tells the model what it can reach in an append-only context message rather than in the system prompt: which kinds of deferred entries exist, up to 20 namespaces (the first `__` segment of deferred names, such as `sre` for `sre__list_alerts`), and up to 20 connections with their descriptions. It never names a deferred entry, so deferred entries stay out of context until `eve__search` finds them. A later step appends the listing again only when that changes; a deferred entry added to a listed namespace, or without one, changes nothing. After the session first reaches each kind of entry, adding or removing entries, signing in, or resolving more dynamic connections doesn't change the catalog tools, so the cached prompt prefix holds.
|
|
323
323
|
|
|
324
324
|
The tools eve adds itself all live in the `eve` namespace: `eve__search`, `eve__tool`, `eve__skill`, `eve__task_wait`, `eve__task_cancel`, and `eve__reply`. Nothing you author or resolve may be named `eve` or start with `eve__`: tools, subagents, skills, connections, and extension mounts, static or dynamic, since a connection or mount named `eve` would own every `eve__` name. The compiler rejects an authored tool, skill, connection, or mount, such as `agent/tools/eve__search.ts`; eve rejects an authored subagent when the agent loads, and a dynamic entry when its resolver returns it. Any other name is free, including `search` and `execute`. A connection's own name is also the entry that signs the user in to it.
|
|
325
325
|
|
package/docs/connections/mcp.mdx
CHANGED
|
@@ -115,7 +115,7 @@ Use the shared `approval` option to gate every tool served by an MCP connection.
|
|
|
115
115
|
|
|
116
116
|
### Gate specific tools by name or input
|
|
117
117
|
|
|
118
|
-
A remote MCP server usually mixes read tools with destructive or publishing ones, so a blanket `always()` would prompt on every harmless call. Pass a custom policy instead — the same [`Approval`](/docs/human-in-the-loop#approvals) shape authored tools use — to gate only the calls that matter. The policy receives `{ session, toolName, toolInput, approvedTools, callId, abortSignal }` and returns an approval status, synchronously or as a promise.
|
|
118
|
+
A remote MCP server usually mixes read tools with destructive or publishing ones, so a blanket `always()` would prompt on every harmless call. Pass a custom policy instead — the same [`Approval`](/docs/human-in-the-loop#approvals) shape authored tools use — to gate only the calls that matter. The policy receives `{ session, toolName, toolInput, toolAnnotations, approvedTools, callId, abortSignal }` and returns an approval status, synchronously or as a promise.
|
|
119
119
|
|
|
120
120
|
This connection always gates deletes, gates a publish only when the call actually schedules a post, and lets everything else through:
|
|
121
121
|
|
|
@@ -157,6 +157,27 @@ Two details are specific to connection tools:
|
|
|
157
157
|
|
|
158
158
|
Return `"user-approval"` (or `true`) to pause for a person and `"not-applicable"` (or `false`) to run without a prompt; return `"approved"` or `"denied"` to decide automatically without involving anyone.
|
|
159
159
|
|
|
160
|
+
### Gate by the server's tool annotations
|
|
161
|
+
|
|
162
|
+
Many MCP servers declare [tool annotations](https://modelcontextprotocol.io/specification/2025-06-18/server/tools#tool) such as `readOnlyHint` and `destructiveHint` for each tool. eve passes what the server declared for the called tool as `toolAnnotations`, typed as `ConnectionToolAnnotations` from `eve/tools/approval`. It is `undefined` when the server declares none, so keep a fallback:
|
|
163
|
+
|
|
164
|
+
```ts title="agent/connections/linear.ts"
|
|
165
|
+
import { connect } from "@vercel/connect/eve";
|
|
166
|
+
import { defineMcpClientConnection } from "eve/connections";
|
|
167
|
+
|
|
168
|
+
export default defineMcpClientConnection({
|
|
169
|
+
url: "https://mcp.linear.app/mcp",
|
|
170
|
+
description: "Linear issues and projects.",
|
|
171
|
+
auth: connect("mcp.linear.app/linear"),
|
|
172
|
+
approval: ({ toolAnnotations }) => {
|
|
173
|
+
if (toolAnnotations?.readOnlyHint === true) return "not-applicable";
|
|
174
|
+
return "user-approval";
|
|
175
|
+
},
|
|
176
|
+
});
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Annotations are hints from the server, not guarantees. Trust them only as far as you trust the server that sent them.
|
|
180
|
+
|
|
160
181
|
[Human-in-the-loop](/docs/human-in-the-loop#approvals) covers the full set of statuses, how `approvedTools` and `session.auth` factor in, and how a gated call pauses and resumes durably.
|
|
161
182
|
|
|
162
183
|
## Forward the caller to an eve agent
|
|
@@ -3,7 +3,7 @@ title: "Assertions"
|
|
|
3
3
|
description: "Scoped methods, value assertions, the matcher mini-language, and gate vs soft severity."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
Assertions are how an eval grades what its `test(t)` function produced. Each one records a result and returns a chainable handle. The runner reads every recorded result to compute the verdict, so a single run reports every failing assertion rather than dying on the first. There are
|
|
6
|
+
Assertions are how an eval grades what its `test(t)` function produced. Each one records a result and returns a chainable handle. The runner reads every recorded result to compute the verdict, so a single run reports every failing assertion rather than dying on the first. There are three deterministic surfaces: scoped methods, `t.check` for grading a specific value, and `t.score` for recording a measurement you computed yourself. For model-graded assertions, see [Judge](./judge).
|
|
7
7
|
|
|
8
8
|
## Scoped assertions
|
|
9
9
|
|
|
@@ -83,6 +83,53 @@ For example, a failed labeled equality check prints:
|
|
|
83
83
|
|
|
84
84
|
Pick the cheapest builder that captures what "correct" means. When exact match is too strict but a judge model is overkill, `similarity` is the middle ground. For nuanced grading, reach for the [judge](./judge).
|
|
85
85
|
|
|
86
|
+
## Scores with `t.score`
|
|
87
|
+
|
|
88
|
+
`t.score(evaluation)` records a score from 0 to 1. The evaluation is the number or `{ score, message?, metadata? }`, or a promise of either. Chain `.label(key)` to name it; reporters and artifacts expose the label as `key`. Without a label the score reports as `score`, and several unlabeled scores share that name.
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
const reply = turn.message ?? "";
|
|
92
|
+
t.score({
|
|
93
|
+
score: Math.max(0, 1 - reply.length / 400),
|
|
94
|
+
metadata: { length: reply.length },
|
|
95
|
+
}).label("reply-brevity");
|
|
96
|
+
t.score(gradeFaithfulness(reply, reference)).label("faithfulness"); // async
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
On its own a score has no verdict and does not affect the eval's outcome. Chain a rule when it should:
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
t.score(gradeFaithfulness(reply, reference)).label("faithfulness").gate(0.9); // fails below 0.9
|
|
103
|
+
t.score(brevity).label("reply-brevity").atLeast(0.5); // marks the eval `scored`, fatal under --strict
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The score is reported as measured. A `0.82` gated at `0.9` is reported as `score: 0.82`, `threshold: 0.9`, `passed: false`. An evaluator that throws or returns a score outside 0 to 1 records an errored outcome with the error message and no score, and fails the eval.
|
|
107
|
+
|
|
108
|
+
Every [reporter](./reporters#custom-reporters) receives the entries in `onEvalComplete` and can read them by `key`:
|
|
109
|
+
|
|
110
|
+
```ts
|
|
111
|
+
onEvalComplete(result) {
|
|
112
|
+
for (const { key, score, threshold, passed } of result.assertions) {
|
|
113
|
+
if (key === undefined) continue;
|
|
114
|
+
store.write({ evalId: result.id, key, score, threshold, passed });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The same entries land in the [run artifacts](./running#artifacts) under `.eve/evals/`, in `summary.json` and one line per eval in `results.jsonl`:
|
|
120
|
+
|
|
121
|
+
```json
|
|
122
|
+
{
|
|
123
|
+
"name": "score [reply-brevity]",
|
|
124
|
+
"key": "reply-brevity",
|
|
125
|
+
"score": 0.84,
|
|
126
|
+
"severity": "soft",
|
|
127
|
+
"errored": false
|
|
128
|
+
}
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`threshold` and `passed` are present only when a rule was chained.
|
|
132
|
+
|
|
86
133
|
## The matcher mini-language
|
|
87
134
|
|
|
88
135
|
`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ callId, childSessionId, remoteUrl, output, status, count }`. Calls match `status: "completed"` by default. Tool lifecycle checks also accept `"pending"`, `"failed"`, and `"rejected"`. Subagent checks accept `"working"`, `"completed"`, `"failed"`, and `"cancelled"`. A numeric `count` requires an exact number of calls matching every supplied constraint. Use a predicate for ranges or other custom count requirements; it receives the observed number of matching calls.
|
|
@@ -164,9 +211,9 @@ const request = session.requireInputRequest({ toolName: "guarded" });
|
|
|
164
211
|
Every assertion returns a chainable handle. Severity rides on the assertion, so there is no separate thresholds map to keep in sync.
|
|
165
212
|
|
|
166
213
|
- `.gate(threshold?)` is hard. A miss marks the eval `failed` and `eve eval` exits non-zero.
|
|
167
|
-
- `.soft(threshold?)` is tracked data. A below-threshold miss marks the eval `scored`, fatal only under `--strict`. With no threshold
|
|
214
|
+
- `.soft(threshold?)` is tracked data. A below-threshold miss marks the eval `scored`, fatal only under `--strict`. With no threshold it records no verdict and never fails.
|
|
168
215
|
- `.atLeast(threshold)` is soft with a bar (equivalent to `.soft(threshold)`).
|
|
169
|
-
- `.label(
|
|
216
|
+
- `.label(key)` adds a stable name when an eval records several assertions from the same family. Reporters and artifacts expose it as `key`. Braintrust and Datadog keep one score per name, so `eve eval` warns when soft assertions in one eval share a name with different scores.
|
|
170
217
|
|
|
171
218
|
The defaults are chosen so you rarely set severity. Run-level methods and `includes`/`equals`/`matches` are gates; `similarity` and every `t.judge(...)` assertion are soft. Annotate only when you deviate:
|
|
172
219
|
|
package/docs/evals/overview.mdx
CHANGED
|
@@ -154,6 +154,7 @@ Each surface matches a genuinely different kind of judgment:
|
|
|
154
154
|
|
|
155
155
|
- **Scoped methods** read the final whole run on `t`, snapshot one independent session when invoked there, or inspect one immutable `EveEvalTurn`. See [Assertions](./assertions).
|
|
156
156
|
- **`t.check(value, assertion)`** grades an explicit value with a deterministic builder from `eve/evals/expect`, such as `t.check(turn.message, includes("sunny"))`. Grade `turn.message`, an intermediate draft, parsed JSON, or anything else. See [Assertions](./assertions).
|
|
157
|
+
- **`t.score(evaluation)`** records a score, such as a length ratio or an evaluator's result. Chain `.label(key)` to name it, and `.gate(threshold)` or `.atLeast(threshold)` to let it decide the eval. See [Assertions](./assertions#scores-with-tscore).
|
|
157
158
|
- **`t.judge(...)`** is the LLM-as-judge surface, like `t.judge("cites a source")`. It grades the most recently settled turn’s assistant message by default; pass `{ on: turn.session.transcript }` to grade a multi-turn conversation. The judge uses the configured judge model, never the agent under test. See [Judge](./judge).
|
|
158
159
|
|
|
159
160
|
## Gate vs soft
|
|
@@ -161,7 +162,7 @@ Each surface matches a genuinely different kind of judgment:
|
|
|
161
162
|
Every assertion returns a chainable handle, so severity rides on the assertion itself. There is no separate thresholds map.
|
|
162
163
|
|
|
163
164
|
- **Gates** are hard. A failed gate marks the eval `failed` and `eve eval` exits non-zero. Run-level methods, `includes`, `equals`, and `matches` are gates by default.
|
|
164
|
-
- **Soft** assertions are tracked data. They land in reports and artifacts, and a below-threshold soft assertion marks the eval `scored` (visible but not fatal, unless you pass `--strict`). `similarity` and every `t.
|
|
165
|
+
- **Soft** assertions are tracked data. They land in reports and artifacts, and a below-threshold soft assertion marks the eval `scored` (visible but not fatal, unless you pass `--strict`). `similarity`, every `t.judge(...)` assertion, and every `t.score(...)` are soft by default. A soft assertion with no threshold records no verdict and never fails.
|
|
165
166
|
|
|
166
167
|
Override per assertion: `.gate(threshold?)` promotes to a hard gate, `.soft(threshold?)` demotes to tracked, and `.atLeast(threshold)` is a soft assertion with a bar.
|
|
167
168
|
|
package/docs/evals/reporters.mdx
CHANGED
|
@@ -38,7 +38,7 @@ export default defineEval({
|
|
|
38
38
|
});
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
-
The reporter config takes an optional `projectName` and `experimentName`, plus a base experiment (by name or id) to diff against. Gate assertions log as binary scores under a `gate:` prefix so experiments diff gate regressions the same way they diff soft-score regressions. Assertions that share a name, such as several unlabeled judges, log one score: the lowest.
|
|
41
|
+
The reporter config takes an optional `projectName` and `experimentName`, plus a base experiment (by name or id) to diff against. Gate assertions log as binary scores under a `gate:` prefix so experiments diff gate regressions the same way they diff soft-score regressions. Assertions that share a name, such as several unlabeled judges, log one score: the lowest. The console warns under the eval when merged soft scores differ; use `.label(name)` to group or separate them. Errored assertions log no score. Each assertion's score is stored in the Braintrust metadata under `eveAssertionScores` and failed assertion details under `eveFailedAssertions`; observed traces are stored under `eveTraceIds` and `eveTraceContexts`.
|
|
42
42
|
|
|
43
43
|
A reporter instance observes the evals that reference it. Share one instance across several evals (the config, a `shared.ts` export, or every entry of a dataset array) and their results land in a single experiment. Listing the same config reporter on an eval too does not double-report it.
|
|
44
44
|
|
|
@@ -97,6 +97,8 @@ interface EvalReporter {
|
|
|
97
97
|
- `onSessionStart` fires once for each session after eve receives its first trace context. It includes `sessionId`, `primary`, and `traceContext` with `traceId`, `spanId`, and `traceFlags`.
|
|
98
98
|
- `onEvalComplete` fires with the checks, scores, and verdict. The runner also supplies `context`, including every distinct trace context collected across the eval's sessions. The same list is available as `result.result.traceContexts`.
|
|
99
99
|
|
|
100
|
+
Each entry in `result.assertions` is an `AssertionResult`. `name` is the display label, `key` the identifier set by `.label(key)`, and `score` the measurement. `threshold` and `passed` describe the rule and are absent when no rule applies. When a scorer threw, `errored` is `true`, `message` carries the error, and `score` is absent. The `.eve/evals/` artifacts write the same fields.
|
|
101
|
+
|
|
100
102
|
An eval can create several sessions, and a long session can produce several traces, so completion exposes a list instead of a single trace id. `onEvalStart` does not include a trace because the agent session has not started yet. If the target has no tracing configured, `onSessionStart` does not fire and the completed trace list is empty.
|
|
101
103
|
|
|
102
104
|
`result.result.derived` holds the facts the runner derived from the eval's sessions, such as tool calls and `models`: the distinct ids of the models the steps started with, in session order. Each entry in `result.result.sessions` holds the same facts for one session, in first-use order, so a reporter can tell the primary session's models from those of the other sessions the eval created or attached. A delegated subagent runs in its own session, so its models appear only when the eval attaches that session with `t.target.attachSession`. Compaction and `auto` routing calls are not included.
|
|
@@ -195,7 +195,9 @@ arguments, and approval options as static [MCP](../connections/mcp) and
|
|
|
195
195
|
[OpenAPI](../connections/openapi) connections. Each resolved connection joins
|
|
196
196
|
the per-step connection registry. eve announces it to the model in an
|
|
197
197
|
append-only context message, and the model reaches its tools through
|
|
198
|
-
`eve__search` and `eve__tool
|
|
198
|
+
`eve__search` and `eve__tool`. If the session doesn't have those two tools yet,
|
|
199
|
+
eve adds them when its first connection resolves and keeps them for the rest of
|
|
200
|
+
the session, so later connections don't change the tool list.
|
|
199
201
|
|
|
200
202
|
Set `instanceKey` on every authenticated dynamic connection. Use a stable,
|
|
201
203
|
non-secret account or tenant identifier, and change it whenever the endpoint,
|
|
@@ -274,6 +276,8 @@ export default defineDynamic({
|
|
|
274
276
|
});
|
|
275
277
|
```
|
|
276
278
|
|
|
279
|
+
An entry with `deferred: true` stays out of the model's tool list, like a static [deferred tool](../tools#keep-a-tool-out-of-the-tool-list). The model finds it with `eve__search` and calls it with `eve__tool`. If the session doesn't have those two tools yet, eve adds them on the first step where a resolver returns a deferred entry, and keeps them for the rest of the session. A resolver whose entries are all listed adds neither, so the model doesn't spend steps searching for tools it already has.
|
|
280
|
+
|
|
277
281
|
### Author replayable callbacks
|
|
278
282
|
|
|
279
283
|
Write callback properties as inline function expressions, arrows, method shorthand, or module-level function references. eve transforms authored modules that import `defineTool`, including helper modules outside `agent/tools/`, and stores each callback's referenced closure values independently.
|
|
@@ -377,7 +381,7 @@ A dynamic connection, tool, or skill whose name matches an **authored** one **ov
|
|
|
377
381
|
|
|
378
382
|
At `turn.started`, model, tool, skill, and subagent resolvers receive the visible conversation history and incoming message in `ctx.messages`, oldest first. Request context is included, and history projection still applies. Read these messages from the handler's second argument; the event itself contains turn metadata. Instruction resolvers use the separate snapshot described under [Dynamic instructions](#dynamic-instructions).
|
|
379
383
|
|
|
380
|
-
A message that steers a held turn joins it without another `turn.started`, as does a completed sign-in, so `turn.started` resolvers do not see it; `step.started` resolvers do. A message sent while a session-limit prompt waits
|
|
384
|
+
A message that steers a held turn joins it without another `turn.started`, as does a completed sign-in, so `turn.started` resolvers do not see it; `step.started` resolvers do. A message sent while a session-limit prompt waits queues in the held turn without another `turn.started`. After Continue grants the budget, `step.started` resolvers see the queued message before the model runs. When memory recall runs, its results appear in the projected snapshot before incoming input.
|
|
381
385
|
|
|
382
386
|
### Execution order
|
|
383
387
|
|
package/docs/sandbox/index.mdx
CHANGED
|
@@ -28,6 +28,8 @@ export default defineTool({
|
|
|
28
28
|
|
|
29
29
|
Relative paths resolve under `/workspace`. `run()` returns `{ exitCode, stdout, stderr }`; a nonzero exit code does not throw automatically. Use `spawn()` for a long-running process.
|
|
30
30
|
|
|
31
|
+
Built-in environments set `AI_AGENT=eve` on every `run()` and `spawn()` call, including the `bash` tool and `prepare`, so agent-aware CLIs such as `vercel` skip interactive prompts. To use a different value, set `AI_AGENT` in the environment's `env` option or pass it in the `env` of one `run()` or `spawn()` call.
|
|
32
|
+
|
|
31
33
|
Pass the configured environment to preserve its specific session capabilities in authored runtime code:
|
|
32
34
|
|
|
33
35
|
```ts
|
|
@@ -64,7 +64,29 @@ approval: auto({
|
|
|
64
64
|
|
|
65
65
|
A reusable approval grant applies only after every matching request that is already pending has been resolved. If several calls to a `once()`-gated tool have each produced an approval prompt, approving one does not authorize the others; each visible prompt remains an independent decision. After those pending requests are resolved, later calls in the session are allowed automatically.
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
Use the object form of `approval` to replace the default `Approve <tool label>?` text with a tool-specific summary. The `prompt` callback receives the validated `input`, final `toolName`, and `callId`:
|
|
68
|
+
|
|
69
|
+
```ts title="agent/tools/refund_charge.ts"
|
|
70
|
+
import { defineTool } from "eve/tools";
|
|
71
|
+
import { always } from "eve/tools/approval";
|
|
72
|
+
import { z } from "zod";
|
|
73
|
+
|
|
74
|
+
export default defineTool({
|
|
75
|
+
description: "Refund a charge.",
|
|
76
|
+
inputSchema: z.object({ chargeId: z.string(), amount: z.number() }),
|
|
77
|
+
approval: {
|
|
78
|
+
request: always(),
|
|
79
|
+
prompt: ({ input }) => `Refund $${input.amount.toFixed(2)} for charge ${input.chargeId}?`,
|
|
80
|
+
},
|
|
81
|
+
async execute(input) {
|
|
82
|
+
return refund(input);
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Channels render this prompt in their approval UI. Slack also includes the complete tool input in a collapsed **Tool input** block so an approver can inspect the original structured value.
|
|
88
|
+
|
|
89
|
+
When the decision depends on the input, pass your own policy instead of a helper. It receives the same session context as tool execution, plus `{ toolName, toolInput, approvedTools, callId, abortSignal }` (connection tools also get `toolAnnotations`), and returns an AI SDK 7 approval status synchronously or as a promise. Use `abortSignal` for asynchronous policy work so cancellation stops it with the turn. Use `ctx.session.auth.current` to guard by the caller of the current turn and `ctx.session.auth.initiator` to guard by the caller that created the session. Return `"user-approval"` to pause for a person or `"not-applicable"` to continue without a prompt. `toolInput` can be undefined, so guard the access. This policy denies cross-tenant calls, then requires approval only when an amount crosses a threshold:
|
|
68
90
|
|
|
69
91
|
```ts
|
|
70
92
|
approval: ({ session, toolInput }) => {
|
|
@@ -232,7 +254,7 @@ The run picks back up exactly where it parked. Because the pause is durable, not
|
|
|
232
254
|
|
|
233
255
|
When a subagent requests input, eve emits the same `input.requested` event on its parent session. Answering through that parent session routes the response directly to the blocked child without invoking the parent model.
|
|
234
256
|
|
|
235
|
-
|
|
257
|
+
Approval is consent only. An approved tool call runs with the requesting user's identity, connections, and credentials, including its policy recheck. The approver's access is never lent to the call. Subsequent tool calls in the turn also stay with the original owner.
|
|
236
258
|
|
|
237
259
|
For approval requests, a follow-up message that doesn't match an option steers the turn instead of answering it. eve cancels the turn's pending approval, so the call doesn't run and `input.resolved` reports `outcome: "ignored"`, and the model reads the message next. This happens even when the message is sent with `turnPolicy: "queue"`, because a turn held on a person can't end until they act. Calls the person already approved in the same batch still run. A message from someone other than the person the turn serves waits until the turn ends. Cancelling the turn withdraws its approval: the call doesn't run, `input.resolved` reports `outcome: "cancelled"`, and a later answer to it approves nothing.
|
|
238
260
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "eve",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.79.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Filesystem-first framework for durable backend AI agents that run anywhere.",
|
|
6
6
|
"keywords": [
|
|
@@ -574,6 +574,7 @@
|
|
|
574
574
|
"@chat-adapter/twilio": "4.41.0",
|
|
575
575
|
"@clack/core": "1.3.1",
|
|
576
576
|
"@eve/catalog": "0.0.1",
|
|
577
|
+
"@eve/tracing": "0.0.0",
|
|
577
578
|
"@linqapp/chat-sdk-adapter": "^0.5.1",
|
|
578
579
|
"@modelcontextprotocol/server": "2.0.0",
|
|
579
580
|
"@nuxt/kit": "^4.0.0",
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { SessionAuthContext } from "#channel/types.js";
|
|
2
|
-
/**
|
|
3
|
-
* Runs `fn` with `caller` as the session's current caller. Writes still reach
|
|
4
|
-
* the outer context; only reads of the caller change, so work running beside
|
|
5
|
-
* `fn` in the same step keeps the outer caller.
|
|
6
|
-
*/
|
|
7
|
-
export declare function runAsCaller<T>(caller: SessionAuthContext, fn: () => T): T;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{contextStorage}from"#context/container.js";import{AuthKey,SessionKey}from"#context/keys.js";function runAsCaller(e,t){let n=contextStorage.getStore();return n===void 0?t():contextStorage.run(new CallerScope(n,e),t)}var CallerScope=class{#e;#t;constructor(e,t){this.#t=e,this.#e=t}get localDevRequest(){return this.#t.localDevRequest}get(e){if(e.name===AuthKey.name)return this.#e;if(e.name===SessionKey.name){let e=this.#t.get(SessionKey);return e===void 0?void 0:{...e,auth:{...e.auth,current:this.#e}}}return this.#t.get(e)}require(e){if(!this.has(e))throw Error(`Context key "${e.name}" is not set.`);return this.get(e)}has(e){return e.name===AuthKey.name||this.#t.has(e)}set(e,t){return this.#t.set(e,t)}ensure(e,t){return this.#t.ensure(e,t)}delete(e){return this.#t.delete(e)}entries(){return this.#t.entries()}setVirtualContext(e,t){this.#t.setVirtualContext(e,t)}};export{runAsCaller};
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { type SessionStepState } from "#execution/publish-session-events.js";
|
|
2
|
-
import { type SessionStateTransition } from "#execution/session/state-delta.js";
|
|
3
|
-
import type { WorkflowToolRunAgentStartedMessage, WorkflowToolRunRef } from "#execution/tools/workflow/messages.js";
|
|
4
|
-
import type { JsonValue } from "#shared/json.js";
|
|
5
|
-
/** Publishes a workflow tool run's `ctx.report()` update as `action.partial`. */
|
|
6
|
-
export declare function emitWorkflowToolRunReportStep(input: SessionStepState & {
|
|
7
|
-
readonly from: WorkflowToolRunRef;
|
|
8
|
-
readonly update: JsonValue;
|
|
9
|
-
}): Promise<SessionStateTransition>;
|
|
10
|
-
/** Publishes `agent.started`, in order, for sessions workflow tool runs opened. */
|
|
11
|
-
export declare function emitAgentStartedStep(input: SessionStepState & {
|
|
12
|
-
readonly messages: readonly WorkflowToolRunAgentStartedMessage[];
|
|
13
|
-
}): Promise<SessionStateTransition>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{createActionPartialEvent,createAgentStartedEvent}from"#protocol/message.js";import{publishSessionEvents}from"#execution/publish-session-events.js";import{withSessionStateDelta}from"#execution/session/state-delta.js";import{createRuntimeToolResultFromValue}from"#harness/action-result-helpers.js";async function emitWorkflowToolRunReportStep(e){"use step";let t=createActionPartialEvent({result:createRuntimeToolResultFromValue({callId:e.from.callId,output:e.update,toolName:e.from.toolName}),sequence:e.from.sequence,stepIndex:e.from.stepIndex,turnId:e.from.turnId});return await withSessionStateDelta(e,e=>publishSessionEvents(e,[t]))}async function emitAgentStartedStep(e){"use step";let t=e.messages.map(({from:t,session:n})=>createAgentStartedEvent({callId:t.callId,name:n.name,parentSessionId:e.sessionState.sessionId,remote:n.kind===`remote`?{url:n.url,...n.resolverId!==void 0&&{resolverId:n.resolverId}}:void 0,sessionId:n.sessionId,...t.taskId!==void 0&&{taskId:t.taskId},turnId:t.turnId}));return await withSessionStateDelta(e,e=>publishSessionEvents(e,t))}export{emitAgentStartedStep,emitWorkflowToolRunReportStep};
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { SessionAuthContext } from "#channel/types.js";
|
|
2
|
-
import type { InputRequest } from "#shared/input.js";
|
|
3
|
-
import type { SessionStateMap } from "#harness/types.js";
|
|
4
|
-
/** Who approved each of the `approved` calls, by call id. */
|
|
5
|
-
export declare function approversOf(approved: readonly InputRequest[], state: SessionStateMap | undefined): Readonly<Record<string, SessionAuthContext>>;
|
|
6
|
-
/**
|
|
7
|
-
* Records who approved each of the `approved` calls, so each runs as its approver while the rest
|
|
8
|
-
* of the turn keeps the turn's own caller.
|
|
9
|
-
*/
|
|
10
|
-
export declare function setApprovedCallCallers(approved: readonly InputRequest[], state: SessionStateMap | undefined): void;
|
|
11
|
-
/** Runs `fn` as the call's approver when `callId` is an approved call; otherwise as is. */
|
|
12
|
-
export declare function runAsApprover<T>(callId: string | undefined, fn: () => T): T;
|
|
13
|
-
/** A streaming call's body runs on each `next()`, so each one runs as the approver too. */
|
|
14
|
-
export declare function iterateAsApprover<T>(callId: string | undefined, iterable: AsyncIterable<T>): AsyncIterable<T>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{approverOfRequest}from"./candidates.js";import{contextStorage}from"#context/container.js";import{ContextKey}from"#context/key.js";import{runAsCaller}from"#context/caller-scope.js";const ApprovedCallCallersKey=new ContextKey(`eve.approvedCallCallers`);function approversOf(e,t){let n={};for(let r of e){if(r.action===void 0)continue;let e=approverOfRequest(t,r.requestId);e!==void 0&&(n[r.action.callId]=e)}return n}function setApprovedCallCallers(e,t){let n=contextStorage.getStore();n!==void 0&&n.setVirtualContext(ApprovedCallCallersKey,new Map(Object.entries(approversOf(e,t))))}function runAsApprover(e,t){let n=e===void 0?void 0:contextStorage.getStore()?.get(ApprovedCallCallersKey)?.get(e);return n===void 0?t():runAsCaller(n,t)}async function*iterateAsApprover(e,t){let n=t[Symbol.asyncIterator]();try{for(;;){let t=await runAsApprover(e,()=>n.next());if(t.done===!0)return;yield t.value}}finally{await n.return?.()}}export{approversOf,iterateAsApprover,runAsApprover,setApprovedCallCallers};
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { type Tracer } from "#compiled/@opentelemetry/api/index.js";
|
|
2
|
-
import type { InstrumentationProviderDefinition } from "#instrumentation/lifecycle.js";
|
|
3
|
-
import type { AgentToolContext } from "#tracing/agent-tool-instrumentation.js";
|
|
4
|
-
import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
|
|
5
|
-
/** Builds durable approval wait spans under their originating runtime action. */
|
|
6
|
-
export declare function createAgentApprovalInstrumentation(input: {
|
|
7
|
-
readonly actionContextFor: (sessionId: string, turnId: string, callId: string) => Promise<AgentToolContext | undefined>;
|
|
8
|
-
readonly frameworkVersion: string;
|
|
9
|
-
readonly idGenerator: AgentSpanIdGenerator;
|
|
10
|
-
readonly tracer: Tracer;
|
|
11
|
-
}): Pick<NonNullable<InstrumentationProviderDefinition["events"]>, "input.requested" | "input.resolved">;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{normalizeChannelAudience}from"#shared/channel-audience.js";import{AGENT_SPAN_NAMES}from"#tracing/agent-span-contract.js";import{ROOT_CONTEXT,trace}from"#compiled/@opentelemetry/api/index.js";import{agentSpanNamingAttributes}from"#tracing/agent-span-naming.js";import{contentAttribute}from"#tracing/agent-otel-content.js";import{recordAgentSpanError}from"#tracing/agent-span-error.js";import{agentTraceIdentityAttributes,traceSessionIdOf}from"#tracing/agent-otel-attributes.js";import{decodeTraceSessionId}from"#tracing/agent-trace-context-codec.js";import{withChannelAudience}from"#tracing/channel-audience-context.js";function createAgentApprovalInstrumentation(e){return{"input.requested":async(t,n)=>{if(t.kind!==`tool-approval`||readState(n.state.get())!==void 0)return;let r=await e.actionContextFor(t.scope.sessionId,t.scope.turnId,t.action.callId);if(r===void 0)return;let i={actionCallId:t.action.callId,actionName:t.action.name,attemptIndex:t.scope.attemptIndex,channelAudience:normalizeChannelAudience(t.scope.channelAudience),parent:{spanId:r.spanContext.spanId,traceFlags:r.spanContext.traceFlags,traceId:r.spanContext.traceId},requestId:t.requestId,rootSessionId:t.scope.rootSessionId??t.scope.sessionId,traceSessionId:traceSessionIdOf(t.scope),sessionId:t.scope.sessionId,startTimeMs:Date.now(),stepIndex:t.scope.stepIndex,turnId:t.scope.turnId},a=contentAttribute(t.request);a!==void 0&&(i.requestAttribute=a),n.state.set(i)},"input.resolved":(t,n)=>{let r=readState(n.state.get());if(r===void 0)return;let i=e.idGenerator.withSpanId(e.idGenerator.deriveSpanId(`approval:${t.idempotencyKey}`),()=>e.tracer.startSpan(AGENT_SPAN_NAMES.approval,{attributes:{"gen_ai.tool.call.id":r.actionCallId,"agent.action.name":r.actionName,"agent.approval.kind":`tool-approval`,"agent.approval.outcome":t.outcome,"agent.approval.request_id":r.requestId,"agent.framework.name":`eve`,"agent.framework.version":e.frameworkVersion,"agent.step.attempt":r.attemptIndex,"agent.step.index":r.stepIndex,"agent.turn.id":r.turnId,"gen_ai.operation.name":`workflow`,...agentSpanNamingAttributes(`agent.approval`,`workflow`),...agentTraceIdentityAttributes({rootSessionId:r.rootSessionId,traceSessionId:r.traceSessionId,sessionId:r.sessionId})},startTime:r.startTimeMs},withChannelAudience(trace.setSpan(ROOT_CONTEXT,trace.wrapSpanContext({...r.parent,isRemote:!1})),r.channelAudience)));if(r.requestAttribute!==void 0&&i.setAttribute(`agent.approval.request`,r.requestAttribute),t.response!==void 0){let e=contentAttribute(t.response);e!==void 0&&i.setAttribute(`agent.approval.response`,e)}t.outcome===`failed`&&recordAgentSpanError(i,t.error),i.end()}}}function readState(e){if(typeof e!=`object`||!e||Array.isArray(e))return;let t=e,n=t.parent;if(typeof n!=`object`||!n||Array.isArray(n))return;let r=n;if(typeof t.actionCallId!=`string`||typeof t.actionName!=`string`||typeof t.attemptIndex!=`number`||typeof r.spanId!=`string`||typeof r.traceFlags!=`number`||typeof r.traceId!=`string`||typeof t.requestId!=`string`||typeof t.rootSessionId!=`string`||typeof t.sessionId!=`string`||typeof t.startTimeMs!=`number`||typeof t.stepIndex!=`number`||typeof t.turnId!=`string`)return;let i=t.requestAttribute;if(i===void 0||typeof i==`string`)return{actionCallId:t.actionCallId,actionName:t.actionName,attemptIndex:t.attemptIndex,channelAudience:normalizeChannelAudience(t.channelAudience),parent:{isRemote:!1,spanId:r.spanId,traceFlags:r.traceFlags,traceId:r.traceId},requestAttribute:i,requestId:t.requestId,rootSessionId:t.rootSessionId,traceSessionId:decodeTraceSessionId(t),sessionId:t.sessionId,startTimeMs:t.startTimeMs,stepIndex:t.stepIndex,turnId:t.turnId}}export{createAgentApprovalInstrumentation};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{applyLiveDeliveryAudienceCeiling,readForwardedTraceAssertion}from"#shared/forwarded-trace-policy.js";import{ConversationIdKey}from"#context/keys.js";import{readConversationId}from"#shared/conversation-identity.js";import{readActionTraceContext,readTurnTraceContext}from"#tracing/agent-trace-context-store.js";function resolveToolCallAgentTrace(e){let{serializedContext:t,sessionId:n,turnId:r}=e;return toChildTraceDispatch({conversation:e.conversation,conversationId:readConversationId(t[ConversationIdKey.name]),stored:readActionTraceContext(t,n,r,e.callId)??readTurnTraceContext(t,n,r)})}function toChildTraceDispatch(e){let{stored:t}=e,n=e.conversation?.audience??`unknown`,r=e.conversation?.environment??`production`,i=readForwardedTraceAssertion(t?.forwardedTracePolicy),a=t?.decision===void 0?t:{...t,decision:applyLiveDeliveryAudienceCeiling(t.decision,n,i,r)};return{conversationId:e.conversationId,originAudience:i?.originAudience??n,parentTraceContext:a}}export{resolveToolCallAgentTrace};
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { type Tracer } from "#compiled/@opentelemetry/api/index.js";
|
|
2
|
-
import type { InstrumentationProviderDefinition } from "#instrumentation/lifecycle.js";
|
|
3
|
-
import type { InstrumentationMemoryExecutionOperation } from "#instrumentation/memory.js";
|
|
4
|
-
import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";
|
|
5
|
-
import type { ConversationEnvironment } from "#shared/conversation-context.js";
|
|
6
|
-
interface AgentMemoryInstrumentation {
|
|
7
|
-
readonly events: Pick<NonNullable<InstrumentationProviderDefinition["events"]>, "memory.operation.completed" | "memory.operation.failed" | "memory.operation.started">;
|
|
8
|
-
runInContext<T>(operation: InstrumentationMemoryExecutionOperation, execute: () => PromiseLike<T>): Promise<T>;
|
|
9
|
-
}
|
|
10
|
-
export declare function createAgentMemoryInstrumentation(input: {
|
|
11
|
-
readonly environment: ConversationEnvironment;
|
|
12
|
-
readonly recordOutputs?: boolean;
|
|
13
|
-
readonly stateStore: AgentTraceStateStore;
|
|
14
|
-
readonly tracer: Tracer;
|
|
15
|
-
}): AgentMemoryInstrumentation;
|
|
16
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{normalizeChannelAudience}from"#shared/channel-audience.js";import{applyLiveDeliveryAudienceCeiling,resolveForwardedTraceSeed}from"#shared/forwarded-trace-policy.js";import{contextStorage}from"#context/container.js";import{SessionTraceSeedKey}from"#context/keys.js";import{withOperationConversation}from"#tracing/conversation-context.js";import{isSampledTrace}from"#tracing/sampled-trace.js";import{ROOT_CONTEXT,SpanKind,context,trace}from"#compiled/@opentelemetry/api/index.js";import{isAgentTraceContext,markAgentTraceContext}from"#tracing/agent-trace-context.js";import{withErrorContent}from"#tracing/error-content-context.js";import{agentSpanNamingAttributes}from"#tracing/agent-span-naming.js";import{genAiMemoryRecordsAttribute}from"#tracing/agent-otel-content.js";import{recordAgentSpanError}from"#tracing/agent-span-error.js";import{agentTraceIdentityAttributes,traceSessionIdOf}from"#tracing/agent-otel-attributes.js";import{withChannelAudience}from"#tracing/channel-audience-context.js";import{suppressTracing}from"#tracing/suppress-tracing.js";function createAgentMemoryInstrumentation(e){let t=e.recordOutputs??!1,n=new Map,parentContext=async t=>{let n=context.active();if(isAgentTraceContext(n))return n;let r=await e.stateStore.getSession(t.sessionId);if(t.turnId!==void 0){let n=await e.stateStore.getTurn(t.sessionId,t.turnId);if(n!==void 0)return withChannelAudience(contextFromSpanContext(n.context),r?.channelAudience)}return r===void 0?void 0:withChannelAudience(contextFromSpanContext(r.context),r.channelAudience)},onStarted=async t=>{if(n.has(t.idempotencyKey))return;let r=await parentContext(t),i=r===void 0?void 0:trace.getSpan(r)?.spanContext();if(r===void 0||i===void 0||!isSampledTrace(i))return;let a=e.tracer.startSpan(t.operationName,{attributes:memorySpanAttributes(t),kind:SpanKind.CLIENT},r);n.set(t.idempotencyKey,{context:trace.setSpan(r,a),span:a})},onTerminal=e=>{let t=n.get(e.idempotencyKey);t!==void 0&&(n.delete(e.idempotencyKey),updateMemorySpan(t.span,e),t.span.end())};return{events:{"memory.operation.completed":onTerminal,"memory.operation.failed":onTerminal,"memory.operation.started":onStarted},async runInContext(r,i){let a=await e.stateStore.getSession(r.sessionId),o=n.get(r.idempotencyKey)?.context;if(o===void 0&&r.turnId!==void 0){let t=await e.stateStore.getTurn(r.sessionId,r.turnId);t!==void 0&&(o=withChannelAudience(contextFromSpanContext(t.context),a?.channelAudience),isSampledTrace(t.context)||(o=suppressTracing(o)))}o===void 0&&a!==void 0&&(o=withChannelAudience(contextFromSpanContext(a.context),a.channelAudience),isSampledTrace(a.context)||(o=suppressTracing(o)));let s=resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey)),c=s?.decision??a?.decision,l=c===void 0?void 0:applyLiveDeliveryAudienceCeiling(c,normalizeChannelAudience(a?.channelAudience),s?.forwardedTracePolicy,e.environment);return o===void 0?await i():await context.with(markAgentTraceContext(withErrorContent(withOperationConversation(o,{rootSessionId:a?.rootSessionId,sessionId:r.sessionId}),t&&l?.action===`record`&&l.recordOutputs)),i)}}}function memorySpanAttributes(e){let t={"agent.memory.phase":e.phase,"agent.memory.slot":e.slot,"gen_ai.memory.store.id":e.storeId,"gen_ai.operation.name":e.operationName,...agentSpanNamingAttributes(e.operationName,e.operationName),...agentTraceIdentityAttributes({rootSessionId:e.rootSessionId,traceSessionId:traceSessionIdOf(e),sessionId:e.sessionId})};return e.turnId!==void 0&&(t[`agent.turn.id`]=e.turnId),t}function updateMemorySpan(e,t){if(t.type===`memory.operation.failed`)recordAgentSpanError(e,t.error);else{t.recordCount!==void 0&&e.setAttribute(`gen_ai.memory.record.count`,t.recordCount);let n=genAiMemoryRecordsAttribute(t.outputRecords??[]);n!==void 0&&t.outputRecords!==void 0&&e.setAttribute(`gen_ai.memory.records`,n)}}function contextFromSpanContext(e){return trace.setSpan(ROOT_CONTEXT,trace.wrapSpanContext(e))}export{createAgentMemoryInstrumentation};
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export declare function traceSessionIdOf(scope: {
|
|
2
|
-
readonly traceSessionId?: string;
|
|
3
|
-
readonly rootSessionId?: string;
|
|
4
|
-
readonly sessionId: string;
|
|
5
|
-
}): string;
|
|
6
|
-
export declare function agentTraceIdentityAttributes(input: {
|
|
7
|
-
readonly rootSessionId: string;
|
|
8
|
-
readonly sessionId: string;
|
|
9
|
-
readonly traceSessionId: string;
|
|
10
|
-
}): Record<string, string | number>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{AGENT_TRACE_SCHEMA_VERSION}from"#tracing/agent-span-contract.js";import{operationConversationId}from"#tracing/conversation-context.js";function traceSessionIdOf(e){return e.traceSessionId??e.rootSessionId??e.sessionId}function agentTraceIdentityAttributes(e){let t=operationConversationId(e),n={"agent.run.id":e.sessionId,"agent.trace.schema.version":AGENT_TRACE_SCHEMA_VERSION,"gen_ai.conversation.id":t};return process.env.VERCEL_ENV!==void 0&&(n[`vercel.session_id`]=e.traceSessionId),n}export{agentTraceIdentityAttributes,traceSessionIdOf};
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import{isUserMessageKind}from"#harness/messages.js";const CONTENT_ATTRIBUTE_LIMIT=32768;function contentAttribute(e){if(e===void 0)return;let t;try{t=JSON.stringify(e)}catch{return}if(t!==void 0)return textContentAttribute(t)}function genAiInputMessagesAttribute(e){if(!Array.isArray(e))return;let t=e.flatMap(e=>{if(!isRecord(e)||e.role===`system`||typeof e.role!=`string`)return[];let t=semanticParts(e.content);if(t.length===0)return[];let n=e.role===`user`&&isUserMessageKind(e.kind)?e.kind:void 0;return[n===void 0?{parts:t,role:e.role}:{kind:n,parts:t,role:e.role}]});for(let e=0;e<t.length;e+=1){let n=semanticJsonAttribute(t.slice(e));if(n!==void 0)return n}return truncateSingleSemanticMessage(t)??semanticJsonAttribute([])}function genAiSystemInstructionsAttribute(e){let t=systemPromptText(e);if(t!==void 0)return fitSemanticText(t,e=>[{content:e,type:`text`}])}function genAiToolDefinitionsAttribute(e){if(!Array.isArray(e))return;let t=e.flatMap(e=>{if(!isRecord(e)||typeof e.name!=`string`)return[];let t={name:e.name};return typeof e.description==`string`&&(t.description=e.description),e.inputSchema!==void 0&&(t.parameters=e.inputSchema),[t]});if(t.length===0)return;let n=[],r=2;for(let e of t){let t=stringifyContent(e);if(t===void 0)break;let i=r+(n.length===0?0:1)+t.length;if(i>32768)break;n.push(t),r=i}return`[${n.join(`,`)}]`}function genAiOutputMessagesAttribute(e,t){let n=semanticParts(e);return semanticJsonAttribute([{finish_reason:t===`tool-calls`?`tool_call`:t,parts:n,role:`assistant`}])}function genAiMemoryRecordsAttribute(e){return semanticJsonAttribute(e)}function toolResultsContentAttribute(e){if(e.length===0)return;let t=stringifyContent(e);if(t!==void 0&&t.length<=32768)return t;for(let t=CONTENT_ATTRIBUTE_LIMIT;t>=0;t=t>=256?Math.floor(t/2):-1){let n=stringifyContent(e.map(e=>cappedToolResult(e,t)));if(n!==void 0&&n.length<=32768)return n}}function cappedToolResult(e,t){let n={toolName:e.toolName};for(let r of[`input`,`output`,`error`]){if(!(r in e))continue;let i=e[r],a=typeof i==`string`?i:stringifyContent(i)??``;n[r]=a.length<=t?a:`${a.slice(0,t)}… [truncated]`}return n}function systemPromptText(e){if(typeof e==`string`)return e.length===0?void 0:e;if(!isRecord(e)&&!Array.isArray(e))return;let t=Array.isArray(e)?e:[e],n=[];for(let e of t)if(isRecord(e)){if(typeof e.content==`string`)n.push(e.content);else if(Array.isArray(e.content))for(let t of e.content)isRecord(t)&&t.type===`text`&&typeof t.text==`string`&&n.push(t.text)}let r=n.join(`
|
|
2
|
-
|
|
3
|
-
`).trim();return r.length===0?void 0:r}function textContentAttribute(e){if(e.length!==0)return e.length<=32768?e:`${e.slice(0,CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`}function stringifyContent(e){try{return JSON.stringify(e)}catch{return}}function semanticJsonAttribute(e){let t=stringifyContent(e);return t!==void 0&&t.length<=32768?t:void 0}function truncateSingleSemanticMessage(e){let t=e.at(-1);if(t===void 0||typeof t.role!=`string`)return;let n=semanticMessageText(t.parts),r=typeof t.kind==`string`?{kind:t.kind,role:t.role}:{role:t.role};return fitSemanticText(n,e=>[{...r,parts:[{content:e,type:`text`}]}],!0)}function fitSemanticText(e,t,n=!1){if(!n&&e.length<=32768){let n=semanticJsonAttribute(t(e));if(n!==void 0)return n}for(let n=Math.min(e.length,CONTENT_ATTRIBUTE_LIMIT);n>0;n-=256){let r=semanticJsonAttribute(t(`${e.slice(0,n)}… [truncated]`));if(r!==void 0)return r}return semanticJsonAttribute(t(`… [truncated]`))}function semanticMessageText(e){if(!Array.isArray(e))return``;let t=``;for(let n of e){if(!isRecord(n)||n.type!==`text`||typeof n.content!=`string`)continue;let e=t.length===0?``:`
|
|
4
|
-
`,r=32769-t.length-e.length;if(r<=0)break;t+=e+n.content.slice(0,r)}return t}function semanticParts(e){if(typeof e==`string`)return[{content:e,type:`text`}];if(!Array.isArray(e))return[];let t=[];for(let n of e){let e=semanticPart(n);e!==void 0&&t.push(e)}return t}function semanticPart(e){if(isRecord(e)&&typeof e.type==`string`)switch(e.type){case`text`:case`reasoning`:{let t=typeof e.text==`string`?e.text:e.content;return typeof t==`string`?{content:t,type:e.type}:void 0}case`tool-call`:return{arguments:e.input,id:stringValue(e.toolCallId)??stringValue(e.callId)??null,name:e.toolName,type:`tool_call`};case`tool-result`:return{id:stringValue(e.toolCallId)??stringValue(e.callId)??null,response:toolResponse(e.output),type:`tool_call_response`};case`tool-error`:return{id:stringValue(e.toolCallId)??stringValue(e.callId)??null,response:e.error instanceof Error?e.error.message:e.error,type:`tool_call_response`};default:return{type:e.type}}}function toolResponse(e){return isRecord(e)?(e.type===`text`||e.type===`error-text`||e.type===`json`||e.type===`error-json`)&&`value`in e?e.value:e.type===`execution-denied`?{denied:!0,reason:e.reason}:e:e}function stringValue(e){return typeof e==`string`?e:void 0}function isRecord(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{CONTENT_ATTRIBUTE_LIMIT,contentAttribute,genAiInputMessagesAttribute,genAiMemoryRecordsAttribute,genAiOutputMessagesAttribute,genAiSystemInstructionsAttribute,genAiToolDefinitionsAttribute,textContentAttribute,toolResultsContentAttribute};
|
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import{normalizeChannelAudience}from"#shared/channel-audience.js";import{applyLiveDeliveryAudienceCeiling,resolveForwardedTraceSeed}from"#shared/forwarded-trace-policy.js";import{contextStorage}from"#context/container.js";import{SessionTraceSeedKey}from"#context/keys.js";import{AGENT_SPAN_NAMES,agentInvocationSpanName,modelSpanName}from"#tracing/agent-span-contract.js";import{resolveInstrumentationEnvironment}from"#internal/application/dev-environment.js";import{withOperationConversation}from"#tracing/conversation-context.js";import{isSampledTrace,resolveTracePolicyDecision}from"#tracing/sampled-trace.js";import{ROOT_CONTEXT,SpanKind,context,trace}from"#compiled/@opentelemetry/api/index.js";import{markAgentTraceContext}from"#tracing/agent-trace-context.js";import{attemptIdempotencyKey}from"#instrumentation/lifecycle.js";import{withoutInstrumentationContent}from"#instrumentation/content.js";import{withErrorContent}from"#tracing/error-content-context.js";import{readInstrumentationDecision}from"#shared/instrumentation-decision.js";import{agentSpanNamingAttributes}from"#tracing/agent-span-naming.js";import{contentAttribute,genAiInputMessagesAttribute,genAiOutputMessagesAttribute,genAiSystemInstructionsAttribute,genAiToolDefinitionsAttribute,textContentAttribute,toolResultsContentAttribute}from"#tracing/agent-otel-content.js";import{recordAgentSpanError}from"#tracing/agent-span-error.js";import{withAgentToolContentPolicy}from"#tracing/agent-tool-span-context.js";import{agentTraceIdentityAttributes,traceSessionIdOf}from"#tracing/agent-otel-attributes.js";import{withChannelAudience}from"#tracing/channel-audience-context.js";import{suppressTracing}from"#tracing/suppress-tracing.js";import{instrumentationEventForTraceDecision}from"#instrumentation/content-policy.js";import{createAgentApprovalInstrumentation}from"#tracing/agent-approval-instrumentation.js";import{createAgentChannelDeliveryInstrumentation}from"#tracing/agent-channel-delivery-instrumentation.js";import{createAgentToolInstrumentation}from"#tracing/agent-tool-instrumentation.js";import*as runtimeAttributes from"#tracing/agent-otel-runtime-context.js";import{createAgentMemoryInstrumentation}from"#tracing/agent-memory-instrumentation.js";import{readGatewayCost,setAgentInvocationUsage,setAgentUsage,setGenAiUsage}from"#tracing/agent-otel-usage.js";import{createAgentOtelSessionContext}from"#tracing/agent-otel-session-context.js";import{rememberTurnInputMessages,rememberTurnOutputMessages}from"#tracing/agent-otel-turn-state.js";function createAgentOtelInstrumentation(e){let t=e.environment??resolveInstrumentationEnvironment(),n=e.recordInputs??!1,r=e.recordOutputs??!1,i=new WeakMap,a=new Map,o=new WeakMap,s=new WeakMap,c=createAgentToolInstrumentation({frameworkVersion:e.frameworkVersion,idGenerator:e.idGenerator,recordInputs:n,recordOutputs:r,resolveTraceContext:async t=>(await e.stateStore.getTurn(t.scope.sessionId,t.scope.turnId))?.context,stateStore:e.stateStore,resolveFallback:e=>{let t=a.get(e.scope.attemptId)??e.scope,n=o.get(t);return n===void 0?void 0:{context:n.context,spanContext:n.span.spanContext()}},tracer:e.tracer}),l=createAgentApprovalInstrumentation({actionContextFor:c.dispatchContextFor,frameworkVersion:e.frameworkVersion,idGenerator:e.idGenerator,tracer:e.tracer}),u=createAgentMemoryInstrumentation({...e,environment:t}),{prepareSessionTrace:d,prepareTurnTrace:f}=createAgentOtelSessionContext({...e,environment:t}),projectEvent=async i=>{let a=await e.stateStore.getSession(sessionIdForEvent(i)),o=audienceForEvent(i,a?.channelAudience),s=`traceSeed`in i?i.traceSeed:void 0,c=contextStorage.getStore()?.get(SessionTraceSeedKey),l=resolveForwardedTraceSeed(c),u=resolveForwardedTraceSeed(s,l?.forwardedTracePolicy),decisionForTrace=e=>e===void 0?void 0:resolveTracePolicyDecision(isSampledTrace(e),{audience:o,environment:t}),d=u?.decision??l?.decision??readInstrumentationDecision(a?.decision)??decisionForTrace(s)??decisionForTrace(c)??decisionForTrace(a?.context);if(d===void 0)return withoutInstrumentationContent(i);let f=u===void 0||!(`traceSeed`in i)||i.traceSeed===void 0?i:{...i,traceSeed:{...i.traceSeed,decision:u.decision,traceFlags:u.traceFlags}};return instrumentationEventForTraceDecision(f,applyLiveDeliveryAudienceCeiling(d.action===`drop`?d:{action:`record`,recordInputs:n&&d.recordInputs,recordOutputs:r&&d.recordOutputs},o,u?.forwardedTracePolicy??l?.forwardedTracePolicy,t),{audience:o,environment:t},{applyAudienceCeiling:!1})},onSessionStarted=async e=>{await d(e)},onTurnStarted=async e=>{await f(e)},onStepStarted=async t=>{let n=await e.stateStore.getTurn(t.scope.sessionId,t.scope.turnId);if(n===void 0||!isSampledTrace(n.context))return;let r=await e.stateStore.getSession(t.scope.sessionId),i=withChannelAudience(contextFromSpanContext(n.context),t.scope.channelAudience),s=trace.getSpan(context.active())?.spanContext(),c=e.idGenerator.withSpanId(e.idGenerator.deriveSpanId(attemptIdempotencyKey(t.scope)),()=>e.tracer.startSpan(AGENT_SPAN_NAMES.step,{attributes:runtimeAttributes.agentStepAttributes({event:t,frameworkVersion:e.frameworkVersion,session:r,turn:n}),links:s===void 0||s.traceId===n.context.traceId?void 0:[{attributes:{"eve.link.type":`workflow.delivery`},context:s}]},i));c.addEvent(`step.started`);let l=trace.setSpan(i,c);o.set(t.scope,{context:l,span:c}),a.set(t.scope.attemptId,t.scope)},onStepTerminal=async e=>{let t=a.get(e.scope.attemptId)??e.scope;i.delete(t),drainOpenSpans({...e,scope:t}),c.drain(e.scope.attemptId,e.type===`step.attempt.failed`?{error:e.error}:void 0),e.type===`step.attempt.failed`&&await c.failForAttempt(t,e.error),a.delete(e.scope.attemptId);let n=o.get(t);n!==void 0&&(n.span.addEvent(e.type===`step.attempt.completed`?`step.completed`:`step.failed`),e.type===`step.attempt.failed`&&recordAgentSpanError(n.span,e.error),n.span.end(),o.delete(t))},onTurnTerminal=async t=>{await e.stateStore.updateTurn(t.sessionId,t.turnId,e=>({...e,terminal:t.type===`turn.failed`?{error:t.error,type:t.type}:{type:t.type}}))},onSessionTransition=async t=>{if(t.type===`session.failed`&&t.turnId!==void 0&&await e.stateStore.updateTurn(t.sessionId,t.turnId,e=>({...e,terminal:e.terminal??{error:t.error,type:`turn.failed`}})),t.turnId!==void 0){let n=await e.stateStore.getTurn(t.sessionId,t.turnId);if(n!==void 0){let r=await e.stateStore.getSession(t.sessionId);if(isSampledTrace(n.context)){let i=r?.agentName??n.subagentName,a=withChannelAudience(n.caller===void 0||n.caller.isRemote===!0?ROOT_CONTEXT:contextFromSpanContext(n.caller),r?.channelAudience),startSpan=()=>e.tracer.startSpan(agentInvocationSpanName(i),{attributes:runtimeAttributes.agentActivationAttributes({agentName:i,frameworkVersion:e.frameworkVersion,session:r,sessionId:t.sessionId,turnId:t.turnId,turn:n}),kind:SpanKind.INTERNAL,links:agentActivationLinks(n),root:n.caller===void 0||n.caller.isRemote===!0,startTime:n.startTimeMs},a),o=e.idGenerator.withSpanId(n.context.spanId,()=>e.idGenerator.withTraceId(n.context.traceId,startSpan));setAgentInvocationUsage(o,n.modelUsage),o.addEvent(`turn.started`,void 0,n.startTimeMs),n.terminal!==void 0&&(o.setAttribute(`agent.turn.outcome`,n.terminal.type===`turn.completed`?`completed`:n.terminal.type===`turn.cancelled`?`cancelled`:`failed`),o.addEvent(n.terminal.type),n.terminal.type===`turn.failed`&&recordAgentSpanError(o,n.terminal.error)),o.end()}await e.stateStore.deleteTurn(t.sessionId,t.turnId)}}(t.type===`session.completed`||t.type===`session.failed`)&&(await c.deleteForSession(t.sessionId),await e.stateStore.deleteSession(t.sessionId))},onModelCallStarted=t=>{let r=o.get(t.scope);if(r===void 0)return;r.span.setAttribute(`agent.model.id`,t.model.modelId);let i=e.tracer.startSpan(modelSpanName(t.model.modelId),{attributes:{"gen_ai.agent.name":t.scope.functionId,"gen_ai.operation.name":`chat`,"gen_ai.provider.name":t.model.provider,"gen_ai.request.model":t.model.modelId,...agentSpanNamingAttributes(modelSpanName(t.model.modelId),`chat`),...agentTraceIdentityAttributes({rootSessionId:t.scope.rootSessionId??t.scope.sessionId,traceSessionId:traceSessionIdOf(t.scope),sessionId:t.scope.sessionId}),...runtimeAttributes.runtimeContextAttributes(t.runtimeContext)},kind:SpanKind.CLIENT},r.context),a;if(n&&t.input!==void 0){a=genAiInputMessagesAttribute(t.input.messages),a!==void 0&&i.setAttribute(`gen_ai.input.messages`,a);let e=genAiSystemInstructionsAttribute(t.input.instructions);e!==void 0&&i.setAttribute(`gen_ai.system_instructions`,e);let n=genAiToolDefinitionsAttribute(t.input.tools);n!==void 0&&i.setAttribute(`gen_ai.tool.definitions`,n)}let c={context:trace.setSpan(r.context,i),span:i};return getExecutionContexts(t.scope).set(t.idempotencyKey,c.context),getSpanStates(s,t.scope).set(t.idempotencyKey,c),rememberTurnInputMessages(e.stateStore,t.scope,a)},onModelCallTerminal=async t=>{i.get(t.scope)?.delete(t.idempotencyKey);let n=takeSpanState(s,t.scope,t.idempotencyKey);if(n!==void 0){if(t.type===`model.call.failed`)recordAgentSpanError(n.span,t.error);else{await recordTurnUsage(t),setGenAiUsage(n.span,t.usage),t.responseId!==void 0&&n.span.setAttribute(`gen_ai.response.id`,t.responseId),t.responseModelId!==void 0&&n.span.setAttribute(`gen_ai.response.model`,t.responseModelId),n.span.setAttribute(`gen_ai.response.finish_reasons`,[t.finishReason]);let i=o.get(t.scope);if(i!==void 0&&setAgentUsage(i.span,t.usage),r){n.span.setAttribute(`ai.response.finish_reason`,t.finishReason);let r=t.content;if(r===void 0){n.span.end();return}let i=genAiOutputMessagesAttribute(r,t.finishReason);i!==void 0&&(n.span.setAttribute(`gen_ai.output.messages`,i),await rememberTurnOutputMessages(e.stateStore,t.scope,i));let a=textContentAttribute(r.filter(e=>e.type===`reasoning`).map(e=>e.text).filter(e=>e.trim().length>0).join(`
|
|
2
|
-
`));a!==void 0&&n.span.setAttribute(`ai.response.reasoning`,a);let o=textContentAttribute(r.filter(e=>e.type===`text`).map(e=>e.text).join(``));o!==void 0&&n.span.setAttribute(`ai.response.text`,o);let s=r.filter(e=>e.type===`tool-call`).map(e=>({callId:e.callId,input:e.input,toolName:e.toolName}));if(s.length>0){let e=contentAttribute(s);e!==void 0&&n.span.setAttribute(`ai.response.tool_calls`,e)}let c=r.filter(e=>e.type===`tool-result`||e.type===`tool-error`).map(e=>e.type===`tool-result`?{callId:e.callId,input:e.input,output:e.output,toolName:e.toolName}:{callId:e.callId,error:errorText(e.error),input:e.input,toolName:e.toolName});if(c.length>0){let e=toolResultsContentAttribute(c);e!==void 0&&n.span.setAttribute(`ai.response.tool_results`,e)}}}n.span.end()}},recordTurnUsage=async t=>{(t.usage.inputTokens!==void 0||t.usage.outputTokens!==void 0)&&await e.stateStore.updateTurn(t.scope.sessionId,t.scope.turnId,e=>({...e,modelUsage:{inputTokens:t.usage.inputTokens===void 0?e.modelUsage?.inputTokens:(e.modelUsage?.inputTokens??0)+t.usage.inputTokens,outputTokens:t.usage.outputTokens===void 0?e.modelUsage?.outputTokens:(e.modelUsage?.outputTokens??0)+t.usage.outputTokens}}))},p=createAgentChannelDeliveryInstrumentation({recordInputs:n,stateStore:e.stateStore}),onStepMetadata=e=>{let t=o.get(e.scope);if(t===void 0)return;let n=readGatewayCost(e.providerMetadata);if(n!==void 0)for(let[e,r]of Object.entries(n))t.span.setAttribute(e,r)};return{hook:{events:{...p,...l,...u.events,"step.attempt.completed":onStepTerminal,"step.attempt.failed":onStepTerminal,"step.attempt.metadata":onStepMetadata,"step.attempt.started":onStepStarted,"model.call.completed":onModelCallTerminal,"model.call.failed":onModelCallTerminal,"model.call.started":onModelCallStarted,"session.completed":onSessionTransition,"session.failed":onSessionTransition,"session.started":onSessionStarted,"session.waiting":onSessionTransition,...c.events,"turn.cancelled":onTurnTerminal,"turn.completed":onTurnTerminal,"turn.failed":onTurnTerminal,"turn.started":onTurnStarted},name:`eve.otel`,projectEvent,tracePolicy:()=>({emit:!0,recordInputs:n,recordOutputs:r})},prepareSessionTrace:d,prepareTurnTrace:f,async runInContext(o,s){if(o.type===`memory.operation`)return u.runInContext(o,s);let l=a.get(o.scope.attemptId)??o.scope,d=i.get(l),f=o.type===`model.call`?d?.get(o.idempotencyKey):c.contextFor(o.scope.attemptId,o.idempotencyKey);if(f===void 0){let t=await e.stateStore.getTurn(o.scope.sessionId,o.scope.turnId);t!==void 0&&(f=withChannelAudience(contextFromSpanContext(t.context),o.scope.channelAudience),isSampledTrace(t.context)||(f=suppressTracing(f)))}let p=await e.stateStore.getSession(o.scope.sessionId),m=resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey)),h=m?.decision??p?.decision,g=h===void 0?void 0:applyLiveDeliveryAudienceCeiling(h,normalizeChannelAudience(o.scope.channelAudience),m?.forwardedTracePolicy,t),_={recordInputs:n&&g?.action===`record`&&g.recordInputs,recordOutputs:r&&g?.action===`record`&&g.recordOutputs},run=e=>{let t=e??f;if(t===void 0)return s();let n=withErrorContent(t,_.recordOutputs),r=o.type===`tool.call`?withAgentToolContentPolicy(n,_):n;return context.with(markAgentTraceContext(withOperationConversation(r,o.scope)),s)};return o.type===`tool.call`?c.runInContext({...o,input:_.recordInputs?o.input:void 0},run,_.recordOutputs):run(void 0)}};function getExecutionContexts(e){let t=i.get(e);return t===void 0&&(t=new Map,i.set(e,t)),t}function drainOpenSpans(e){for(let t of s.get(e.scope)?.values()??[])e.type===`step.attempt.failed`&&recordAgentSpanError(t.span,e.error),t.span.end();s.delete(e.scope)}}function agentActivationLinks(e){let t=[];return e.caller?.isRemote===!0&&t.push({context:e.caller,attributes:{"eve.link.type":`agent.dispatch`}}),e.channelDelivery?.requestTraceContext!==void 0&&t.push({context:e.channelDelivery.requestTraceContext,attributes:{"eve.link.type":`channel.request`}}),t.length===0?void 0:t}function sessionIdForEvent(e){return`scope`in e?e.scope.sessionId:e.sessionId}function audienceForEvent(e,t){return`delivery`in e?normalizeChannelAudience(e.delivery.channelAudience):`scope`in e&&e.scope.channelAudience!==void 0?normalizeChannelAudience(e.scope.channelAudience):e.type===`session.started`?normalizeChannelAudience(e.channelAudience):normalizeChannelAudience(t)}function getSpanStates(e,t){let n=e.get(t);return n===void 0&&(n=new Map,e.set(t,n)),n}function takeSpanState(e,t,n){let r=e.get(t),i=r?.get(n);if(r!==void 0)return r.delete(n),r.size===0&&e.delete(t),i}function contextFromSpanContext(e){return trace.setSpan(ROOT_CONTEXT,trace.wrapSpanContext(e))}const errorText=e=>e instanceof Error?e.message:e;export{createAgentOtelInstrumentation};
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";
|
|
2
|
-
import type { InstrumentationStepAttemptStartedEvent } from "#instrumentation/lifecycle.js";
|
|
3
|
-
type SpanAttributePrimitive = string | number | boolean;
|
|
4
|
-
type SpanAttributeValue = SpanAttributePrimitive | SpanAttributePrimitive[];
|
|
5
|
-
export declare function agentActivationAttributes(input: {
|
|
6
|
-
readonly agentName?: string;
|
|
7
|
-
readonly frameworkVersion: string;
|
|
8
|
-
readonly session?: AgentSessionTraceState;
|
|
9
|
-
readonly sessionId: string;
|
|
10
|
-
readonly turnId: string;
|
|
11
|
-
readonly turn: AgentTurnTraceState;
|
|
12
|
-
}): Record<string, string | number | boolean | undefined>;
|
|
13
|
-
export declare function agentChannelClassificationAttributes(session: AgentSessionTraceState | undefined, turn: AgentTurnTraceState, sessionId: string): {
|
|
14
|
-
readonly "agent.channel.kind": string | undefined;
|
|
15
|
-
readonly "agent.session.origin": string | undefined;
|
|
16
|
-
};
|
|
17
|
-
export declare function agentStepAttributes(input: {
|
|
18
|
-
readonly event: InstrumentationStepAttemptStartedEvent;
|
|
19
|
-
readonly frameworkVersion: string;
|
|
20
|
-
readonly session?: AgentSessionTraceState;
|
|
21
|
-
readonly turn: AgentTurnTraceState;
|
|
22
|
-
}): {
|
|
23
|
-
"agent.channel.kind"?: string | undefined;
|
|
24
|
-
"agent.session.origin"?: string | undefined;
|
|
25
|
-
"agent.framework.name": string;
|
|
26
|
-
"agent.framework.version": string;
|
|
27
|
-
"agent.step.attempt": number;
|
|
28
|
-
"agent.step.index": number;
|
|
29
|
-
"agent.turn.id": string;
|
|
30
|
-
"agent.name": string | undefined;
|
|
31
|
-
"gen_ai.operation.name": string;
|
|
32
|
-
};
|
|
33
|
-
export declare function agentPrincipalAttributes(turn: AgentTurnTraceState): Record<string, string>;
|
|
34
|
-
/** Flattens merged runtime context into AI SDK-compatible span attributes. */
|
|
35
|
-
export declare function runtimeContextAttributes(runtimeContext: Readonly<Record<string, unknown>> | undefined): Record<string, SpanAttributeValue>;
|
|
36
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{normalizeInstrumentationChannelKind}from"#internal/instrumentation.js";import{agentInvocationSpanName}from"#tracing/agent-span-contract.js";import{agentSpanNamingAttributes}from"#tracing/agent-span-naming.js";import{agentTraceIdentityAttributes,traceSessionIdOf}from"#tracing/agent-otel-attributes.js";function agentActivationAttributes(e){let t=e.session?.decision?.action===`record`,n=t&&e.session?.decision?.recordInputs===!0,r=t&&e.session?.decision?.recordOutputs===!0,i=e.turn.parentLineage??e.session?.parentLineage,a=i!==void 0,o=!a||e.turn.traceSessionId===e.sessionId,s=agentChannelClassificationAttributes(e.session,e.turn,e.sessionId),c=a?void 0:e.session?.scheduleId;return{"agent.framework.name":`eve`,"agent.framework.version":e.frameworkVersion,"agent.name":e.agentName,"agent.channel.audience":e.session?.channelAudience,...agentPrincipalAttributes(e.turn),"agent.channel.delivery.id":e.turn.channelDelivery?.deliveryId,"agent.channel.delivery.input":e.turn.channelDelivery?.inputAttribute,...s,"agent.channel.name":e.turn.channelDelivery?.channelName,"agent.channel.request.id":e.turn.channelDelivery?.requestId,"agent.parent_call.id":i?.callId,"agent.parent_run.id":i?.sessionId,"agent.run.type":a?`subagent`:`session`,"agent.schedule.id":c,"agent.session.title":o&&n?e.session?.title:void 0,"agent.subagent.name":e.turn.subagentName,"agent.trace.content.input":n,"agent.trace.content.output":r,"agent.turn.id":e.turnId,"agent.turn.sequence":e.turn.sequence,"gen_ai.agent.name":e.agentName,"gen_ai.input.messages":e.turn.inputMessagesAttribute,"gen_ai.operation.name":`invoke_agent`,"gen_ai.output.messages":e.turn.outputMessagesAttribute,...agentSpanNamingAttributes(agentInvocationSpanName(e.agentName),`invoke_agent`),...agentTraceIdentityAttributes({rootSessionId:e.turn.rootSessionId,traceSessionId:e.turn.traceSessionId,sessionId:e.sessionId})}}function agentChannelClassificationAttributes(e,t,n){let r=(t.parentLineage??e?.parentLineage)===void 0||t.traceSessionId===n,i=t.channelDelivery?.channelKind??(r?e?.channelKind??(e?.channelType===void 0?void 0:normalizeInstrumentationChannelKind(e.channelType)):void 0);return{"agent.channel.kind":i,"agent.session.origin":!r||i===void 0?void 0:e?.scheduleId===void 0?`channel`:`schedule`}}function agentStepAttributes(e){let{event:t,session:n,turn:r}=e,i=agentChannelClassificationAttributes(n,r,t.scope.sessionId);return{"agent.framework.name":`eve`,"agent.framework.version":e.frameworkVersion,"agent.step.attempt":t.scope.attemptIndex,"agent.step.index":t.scope.stepIndex,"agent.turn.id":t.scope.turnId,"agent.name":t.scope.functionId,...i[`agent.channel.kind`]===`unknown`?void 0:i,"gen_ai.operation.name":`workflow`,...agentSpanNamingAttributes(`agent.step`,`workflow`),...agentTraceIdentityAttributes({rootSessionId:t.scope.rootSessionId??t.scope.sessionId,traceSessionId:traceSessionIdOf(t.scope),sessionId:t.scope.sessionId}),...runtimeContextAttributes(t.runtimeContext)}}function agentPrincipalAttributes(e){let t={};return setOptionalAttribute(t,`agent.principal.current.id`,e.currentPrincipal?.id),setOptionalAttribute(t,`agent.principal.current.type`,e.currentPrincipal?.type),setOptionalAttribute(t,`agent.principal.initiator.id`,e.initiatorPrincipal?.id),setOptionalAttribute(t,`agent.principal.initiator.type`,e.initiatorPrincipal?.type),t}function runtimeContextAttributes(e){let t={};if(e===void 0)return t;for(let[n,r]of Object.entries(e))flattenContextAttribute(t,`ai.settings.context.${n}`,r);return t}function flattenContextAttribute(e,t,n){if(n!=null){if(typeof n==`string`||typeof n==`number`||typeof n==`boolean`){e[t]=n;return}if(Array.isArray(n)){let r=n.filter(e=>typeof e==`string`||typeof e==`number`||typeof e==`boolean`);if(r.length!==n.length||new Set(r.map(e=>typeof e)).size!==1)return;e[t]=r;return}if(typeof n==`object`)for(let[r,i]of Object.entries(n))flattenContextAttribute(e,`${t}.${r}`,i)}}function setOptionalAttribute(e,t,n){n!==void 0&&(e[t]=n)}export{agentActivationAttributes,agentChannelClassificationAttributes,agentPrincipalAttributes,agentStepAttributes,runtimeContextAttributes};
|