@mastra/mcp-docs-server 1.2.11 → 1.2.12-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/docs/agents/using-tools.md +25 -0
- package/.docs/guides/build-your-ui/ai-sdk-ui.md +5 -5
- package/.docs/guides/build-your-ui/copilotkit/generative-ui.md +8 -8
- package/.docs/guides/build-your-ui/copilotkit/overview.md +2 -2
- package/.docs/guides/build-your-ui/openui.md +1 -1
- package/.docs/guides/concepts/multi-agent-systems.md +5 -5
- package/.docs/guides/concepts/streaming.md +9 -9
- package/.docs/guides/deployment/amazon-ec2.md +1 -1
- package/.docs/guides/deployment/aws-lambda.md +1 -1
- package/.docs/guides/deployment/azure-app-services.md +1 -1
- package/.docs/guides/deployment/cloudflare.md +1 -1
- package/.docs/guides/deployment/inngest.md +6 -6
- package/.docs/guides/deployment/temporal.md +1 -1
- package/.docs/guides/deployment/vercel.md +1 -1
- package/.docs/guides/getting-started/astro.md +1 -1
- package/.docs/guides/getting-started/electron.md +1 -1
- package/.docs/guides/getting-started/nestjs.md +5 -5
- package/.docs/guides/getting-started/vite-react.md +1 -1
- package/.docs/guides/guide/chef-michel.md +1 -1
- package/.docs/guides/guide/code-review-bot.md +1 -1
- package/.docs/guides/guide/coding-agent.md +4 -4
- package/.docs/guides/guide/dev-assistant.md +1 -1
- package/.docs/guides/guide/docs-manager.md +3 -3
- package/.docs/guides/guide/github-actions-pr-description.md +4 -4
- package/.docs/guides/guide/research-assistant.md +3 -3
- package/.docs/guides/guide/research-coordinator.md +3 -3
- package/.docs/guides/guide/slack-assistant.md +1 -1
- package/.docs/guides/guide/stock-agent.md +1 -1
- package/.docs/guides/index.md +1 -1
- package/.docs/guides/migrations/ai-sdk-v4-to-v5.md +1 -1
- package/.docs/guides/migrations/mastra-cloud.md +6 -6
- package/.docs/guides/migrations/network-to-supervisor.md +9 -9
- package/.docs/guides/migrations/upgrade-to-v1/agent.md +5 -5
- package/.docs/guides/migrations/upgrade-to-v1/cli.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/client.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/evals.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/memory.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/overview.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/processors.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/storage.md +11 -11
- package/.docs/guides/migrations/upgrade-to-v1/tools.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/tracing.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/voice.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/workflows.md +1 -1
- package/.docs/models/environment-variables.md +1 -0
- package/.docs/models/index.md +1 -1
- package/.docs/models/providers/modal.md +73 -0
- package/.docs/models/providers.md +1 -0
- package/.docs/reference/acp/acp-agent.md +3 -3
- package/.docs/reference/acp/create-acp-tool.md +2 -2
- package/.docs/reference/agent-controller/agent-controller-class.md +36 -36
- package/.docs/reference/agent-controller/session.md +6 -6
- package/.docs/reference/agents/agent.md +7 -7
- package/.docs/reference/agents/channels.md +3 -3
- package/.docs/reference/agents/durable-agent.md +3 -3
- package/.docs/reference/agents/generate.md +3 -3
- package/.docs/reference/agents/getDefaultOptions.md +1 -1
- package/.docs/reference/agents/getDefaultStreamOptions.md +1 -1
- package/.docs/reference/agents/listScorers.md +1 -1
- package/.docs/reference/agents/listSkills.md +1 -1
- package/.docs/reference/agents/listSuspendedRuns.md +2 -2
- package/.docs/reference/ai-sdk/chat-route.md +1 -1
- package/.docs/reference/ai-sdk/network-route.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v4-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v5-messages.md +1 -1
- package/.docs/reference/ai-sdk/workflow-route.md +1 -1
- package/.docs/reference/ai-sdk/workflow-snapshot-to-stream.md +2 -2
- package/.docs/reference/auth/auth0.md +1 -1
- package/.docs/reference/auth/fga.md +2 -2
- package/.docs/reference/auth/google.md +2 -2
- package/.docs/reference/auth/okta.md +1 -1
- package/.docs/reference/auth/workos.md +2 -2
- package/.docs/reference/browser/agent-browser.md +2 -2
- package/.docs/reference/browser/browser-viewer.md +3 -3
- package/.docs/reference/browser/mastra-browser.md +3 -3
- package/.docs/reference/browser/stagehand-browser.md +4 -4
- package/.docs/reference/channels/slack-provider.md +2 -2
- package/.docs/reference/cli/create-mastra.md +1 -1
- package/.docs/reference/cli/mastra.md +23 -23
- package/.docs/reference/client-js/agents.md +9 -9
- package/.docs/reference/client-js/memory.md +2 -2
- package/.docs/reference/client-js/workflows.md +1 -1
- package/.docs/reference/coding-agent/build-base-prompt.md +2 -2
- package/.docs/reference/coding-agent/create-coding-agent.md +2 -2
- package/.docs/reference/configuration.md +6 -6
- package/.docs/reference/core/getMCPServer.md +1 -1
- package/.docs/reference/core/getScorer.md +1 -1
- package/.docs/reference/datasets/addItem.md +1 -1
- package/.docs/reference/datasets/listItems.md +1 -1
- package/.docs/reference/deployer/netlify.md +4 -4
- package/.docs/reference/deployer/vercel.md +7 -7
- package/.docs/reference/editor/agent-builder/agent-builder-options.md +4 -4
- package/.docs/reference/editor/agent-builder/builder-agent-defaults.md +6 -6
- package/.docs/reference/editor/agent-builder/builder-models.md +3 -3
- package/.docs/reference/editor/blob-store-provider.md +1 -1
- package/.docs/reference/editor/browser-provider.md +3 -3
- package/.docs/reference/editor/filesystem-provider.md +4 -4
- package/.docs/reference/editor/mastra-editor.md +5 -5
- package/.docs/reference/editor/processor-provider.md +3 -3
- package/.docs/reference/editor/sandbox-provider.md +4 -4
- package/.docs/reference/editor/storage-browser-ref.md +6 -6
- package/.docs/reference/editor/storage-workspace-ref.md +4 -4
- package/.docs/reference/editor/tool-provider.md +1 -1
- package/.docs/reference/evals/answer-relevancy.md +4 -4
- package/.docs/reference/evals/bias.md +3 -3
- package/.docs/reference/evals/checks.md +3 -3
- package/.docs/reference/evals/completeness.md +5 -5
- package/.docs/reference/evals/content-similarity.md +5 -5
- package/.docs/reference/evals/context-precision.md +2 -2
- package/.docs/reference/evals/context-recall.md +8 -8
- package/.docs/reference/evals/context-relevance.md +4 -4
- package/.docs/reference/evals/create-scorer.md +1 -1
- package/.docs/reference/evals/faithfulness.md +3 -3
- package/.docs/reference/evals/filter-run.md +2 -2
- package/.docs/reference/evals/hallucination.md +6 -6
- package/.docs/reference/evals/keyword-coverage.md +6 -6
- package/.docs/reference/evals/mastra-scorer.md +2 -2
- package/.docs/reference/evals/noise-sensitivity.md +5 -5
- package/.docs/reference/evals/prompt-alignment.md +5 -5
- package/.docs/reference/evals/rubric.md +1 -1
- package/.docs/reference/evals/run-evals.md +3 -3
- package/.docs/reference/evals/scorer-utils.md +4 -4
- package/.docs/reference/evals/textual-difference.md +4 -4
- package/.docs/reference/evals/tool-call-accuracy.md +1 -1
- package/.docs/reference/evals/toxicity.md +4 -4
- package/.docs/reference/evals/trajectory-accuracy.md +10 -10
- package/.docs/reference/file-based-agents/instructions.md +2 -2
- package/.docs/reference/file-based-agents/logger.md +1 -1
- package/.docs/reference/file-based-agents/observability.md +1 -1
- package/.docs/reference/file-based-agents/processors.md +1 -1
- package/.docs/reference/file-based-agents/skills.md +1 -1
- package/.docs/reference/file-based-agents/storage.md +1 -1
- package/.docs/reference/file-based-agents/studio.md +1 -1
- package/.docs/reference/file-based-agents/subagents.md +1 -1
- package/.docs/reference/file-based-agents/tools.md +1 -1
- package/.docs/reference/file-based-agents/workspace.md +1 -1
- package/.docs/reference/logging/pino-logger.md +3 -3
- package/.docs/reference/memory/clone-utilities.md +1 -1
- package/.docs/reference/memory/cloneThread.md +3 -3
- package/.docs/reference/memory/listThreads.md +1 -1
- package/.docs/reference/memory/memory-class.md +1 -1
- package/.docs/reference/memory/observational-memory.md +11 -11
- package/.docs/reference/memory/recall.md +2 -2
- package/.docs/reference/memory/serialized-memory-config.md +4 -4
- package/.docs/reference/memory/summarizeConversation.md +3 -3
- package/.docs/reference/memory/summarizeThread.md +3 -3
- package/.docs/reference/observability/feedback.md +1 -1
- package/.docs/reference/observability/metrics/automatic-metrics.md +7 -7
- package/.docs/reference/observability/tracing/bridges/datadog.md +2 -2
- package/.docs/reference/observability/tracing/exporters/cloud-exporter.md +3 -3
- package/.docs/reference/observability/tracing/exporters/default-exporter.md +1 -1
- package/.docs/reference/observability/tracing/exporters/mastra-platform-exporter.md +4 -4
- package/.docs/reference/observability/tracing/exporters/mastra-storage-exporter.md +1 -1
- package/.docs/reference/observability/tracing/interfaces.md +1 -1
- package/.docs/reference/observability/tracing/processors/sensitive-data-filter.md +1 -1
- package/.docs/reference/observability/tracing/span-filtering.md +1 -1
- package/.docs/reference/processors/cost-guard-processor.md +1 -1
- package/.docs/reference/processors/pii-detector.md +1 -1
- package/.docs/reference/processors/processor-interface.md +12 -12
- package/.docs/reference/processors/prompt-injection-detector.md +1 -1
- package/.docs/reference/processors/provider-history-compat.md +1 -1
- package/.docs/reference/processors/regex-filter-processor.md +1 -1
- package/.docs/reference/processors/response-cache.md +8 -8
- package/.docs/reference/processors/system-prompt-scrubber.md +1 -1
- package/.docs/reference/processors/token-limiter-processor.md +1 -1
- package/.docs/reference/processors/tool-search-processor.md +7 -7
- package/.docs/reference/processors/unicode-normalizer.md +1 -1
- package/.docs/reference/project-structure.md +1 -1
- package/.docs/reference/pubsub/base.md +4 -4
- package/.docs/reference/pubsub/google-cloud-pubsub.md +2 -2
- package/.docs/reference/pubsub/lease-provider.md +4 -4
- package/.docs/reference/pubsub/redis-streams.md +1 -1
- package/.docs/reference/pubsub/unix-socket-pubsub.md +1 -1
- package/.docs/reference/rag/chunk.md +5 -5
- package/.docs/reference/rag/database-config.md +2 -2
- package/.docs/reference/rag/metadata-filters.md +2 -2
- package/.docs/reference/schedules/overview.md +1 -1
- package/.docs/reference/server/register-api-route.md +1 -1
- package/.docs/reference/server/routes.md +1 -1
- package/.docs/reference/signals/signal-provider.md +8 -8
- package/.docs/reference/storage/clickhouse.md +4 -4
- package/.docs/reference/storage/cloudflare-d1.md +4 -4
- package/.docs/reference/storage/convex.md +4 -4
- package/.docs/reference/storage/dsql.md +2 -2
- package/.docs/reference/storage/dynamodb.md +7 -7
- package/.docs/reference/storage/lance.md +1 -1
- package/.docs/reference/storage/mongodb.md +1 -1
- package/.docs/reference/storage/mssql.md +1 -1
- package/.docs/reference/storage/postgresql.md +1 -1
- package/.docs/reference/storage/retention.md +33 -33
- package/.docs/reference/storage/spanner.md +8 -8
- package/.docs/reference/streaming/ChunkType.md +2 -2
- package/.docs/reference/streaming/agents/MastraModelOutput.md +1 -1
- package/.docs/reference/streaming/agents/stream.md +1 -1
- package/.docs/reference/streaming/agents/streamUntilIdle.md +2 -2
- package/.docs/reference/streaming/workflows/observeStream.md +1 -1
- package/.docs/reference/streaming/workflows/resumeStream.md +1 -1
- package/.docs/reference/streaming/workflows/stream.md +1 -1
- package/.docs/reference/streaming/workflows/timeTravelStream.md +2 -2
- package/.docs/reference/templates/overview.md +1 -1
- package/.docs/reference/tools/brightdata.md +1 -1
- package/.docs/reference/tools/create-tool.md +1 -1
- package/.docs/reference/tools/graph-rag-tool.md +1 -1
- package/.docs/reference/tools/mcp-client.md +13 -13
- package/.docs/reference/tools/mcp-server.md +20 -20
- package/.docs/reference/tools/submit-plan-tool.md +1 -1
- package/.docs/reference/tools/tavily.md +1 -1
- package/.docs/reference/tools/vector-query-tool.md +4 -4
- package/.docs/reference/vectors/astra.md +1 -1
- package/.docs/reference/vectors/chroma.md +2 -2
- package/.docs/reference/vectors/convex.md +2 -2
- package/.docs/reference/vectors/couchbase.md +4 -4
- package/.docs/reference/vectors/libsql.md +1 -1
- package/.docs/reference/vectors/mongodb.md +1 -1
- package/.docs/reference/vectors/pg.md +4 -4
- package/.docs/reference/vectors/qdrant.md +2 -2
- package/.docs/reference/vectors/s3vectors.md +4 -4
- package/.docs/reference/vectors/turbopuffer.md +1 -1
- package/.docs/reference/voice/aws-nova-sonic.md +1 -1
- package/.docs/reference/voice/azure.md +1 -1
- package/.docs/reference/voice/google-gemini-live.md +4 -4
- package/.docs/reference/voice/inworld-realtime.md +7 -7
- package/.docs/reference/voice/livekit.md +13 -13
- package/.docs/reference/voice/mistral.md +1 -1
- package/.docs/reference/voice/openai-realtime.md +1 -1
- package/.docs/reference/voice/sarvam.md +1 -1
- package/.docs/reference/voice/voice.addTools.md +1 -1
- package/.docs/reference/voice/voice.events.md +1 -1
- package/.docs/reference/voice/voice.getSpeakers.md +2 -2
- package/.docs/reference/voice/voice.on.md +2 -2
- package/.docs/reference/voice/xai-realtime.md +2 -2
- package/.docs/reference/workflows/run-methods/cancel.md +1 -1
- package/.docs/reference/workflows/run-methods/startAsync.md +1 -1
- package/.docs/reference/workflows/run-methods/timeTravel.md +1 -1
- package/.docs/reference/workflows/workflow-methods/foreach.md +1 -1
- package/.docs/reference/workflows/workflow-methods/sleep.md +1 -1
- package/.docs/reference/workflows/workflow-state-reader.md +2 -2
- package/.docs/reference/workspace/apple-container-sandbox.md +2 -2
- package/.docs/reference/workspace/daytona-sandbox.md +3 -3
- package/.docs/reference/workspace/docker-sandbox.md +2 -2
- package/.docs/reference/workspace/e2b-sandbox.md +2 -2
- package/.docs/reference/workspace/files-sdk-filesystem.md +3 -3
- package/.docs/reference/workspace/google-drive-filesystem.md +4 -4
- package/.docs/reference/workspace/local-filesystem.md +2 -2
- package/.docs/reference/workspace/local-sandbox.md +2 -2
- package/.docs/reference/workspace/modal-sandbox.md +1 -1
- package/.docs/reference/workspace/platform-filesystem.md +3 -3
- package/.docs/reference/workspace/platform-sandbox.md +2 -2
- package/.docs/reference/workspace/process-manager.md +1 -1
- package/.docs/reference/workspace/railway-sandbox.md +4 -4
- package/.docs/reference/workspace/s3-filesystem.md +1 -1
- package/.docs/reference/workspace/sandbox.md +1 -1
- package/.docs/reference/workspace/vercel-sandbox.md +1 -1
- package/.docs/reference/workspace/workspace-class.md +10 -10
- package/CHANGELOG.md +7 -0
- package/package.json +5 -5
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
**Added in:** `@mastra/core@1.4.0`
|
|
6
6
|
|
|
7
|
-
Adds a single item to the dataset. Each item has an input
|
|
7
|
+
Adds a single item to the dataset. Each item has an input and optional ground truth. Metadata is also optional.
|
|
8
8
|
|
|
9
9
|
## Usage example
|
|
10
10
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
**Added in:** `@mastra/core@1.4.0`
|
|
6
6
|
|
|
7
|
-
Lists items in the dataset. When only `version` is provided, returns a bare `DatasetItem[]` snapshot of every item at that version. In all other cases
|
|
7
|
+
Lists items in the dataset. When only `version` is provided, returns a bare `DatasetItem[]` snapshot of every item at that version. In all other cases (no arguments, or `search` / `page` / `perPage` provided with or without `version`), it returns a paginated `{ items, pagination }` shape.
|
|
8
8
|
|
|
9
9
|
## Usage example
|
|
10
10
|
|
|
@@ -6,7 +6,7 @@ The `NetlifyDeployer` class handles packaging, configuration, and deployment by
|
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
To use `NetlifyDeployer`, you need to install the `@mastra/deployer-netlify` package:
|
|
10
10
|
|
|
11
11
|
**npm**:
|
|
12
12
|
|
|
@@ -47,10 +47,10 @@ export const mastra = new Mastra({
|
|
|
47
47
|
|
|
48
48
|
## Constructor options
|
|
49
49
|
|
|
50
|
-
- `target?: 'serverless' | 'edge'
|
|
50
|
+
- `target?: 'serverless' | 'edge'`: Deploy target for Netlify. Defaults to `'serverless'`.
|
|
51
51
|
|
|
52
|
-
- `'serverless'
|
|
53
|
-
- `'edge'
|
|
52
|
+
- `'serverless'`: Standard [Netlify Functions](https://docs.netlify.com/functions/overview/) (Node.js runtime, 60s default timeout).
|
|
53
|
+
- `'edge'`: [Netlify Edge Functions](https://docs.netlify.com/build/edge-functions/overview/) (Deno-based runtime, runs at the edge closest to users, without a fixed timeout).
|
|
54
54
|
|
|
55
55
|
### Edge functions example
|
|
56
56
|
|
|
@@ -49,10 +49,10 @@ export const mastra = new Mastra({
|
|
|
49
49
|
|
|
50
50
|
The deployer accepts the following options:
|
|
51
51
|
|
|
52
|
-
- `studio?: boolean
|
|
53
|
-
- `maxDuration?: number
|
|
54
|
-
- `memory?: number
|
|
55
|
-
- `regions?: string[]
|
|
52
|
+
- `studio?: boolean`: Deploy [Studio](https://mastra.ai/docs/studio/overview) alongside your API as static assets served from Vercel's Edge CDN. Defaults to `false`.
|
|
53
|
+
- `maxDuration?: number`: Function execution timeout (in seconds)
|
|
54
|
+
- `memory?: number`: Function memory (in MB)
|
|
55
|
+
- `regions?: string[]`: Regions to deploy the function (e.g. `['sfo1','iad1']`)
|
|
56
56
|
|
|
57
57
|
The `maxDuration`, `memory`, and `regions` options are merged into `.vercel/output/functions/index.func/.vc-config.json` while preserving default fields (`handler`, `launcherType`, `runtime`, `shouldAddHelpers`).
|
|
58
58
|
|
|
@@ -78,9 +78,9 @@ After running `mastra build`, the deployer generates a `.vercel/output` director
|
|
|
78
78
|
|
|
79
79
|
The output contains:
|
|
80
80
|
|
|
81
|
-
- **config.json
|
|
82
|
-
- **functions
|
|
83
|
-
- **static
|
|
81
|
+
- **config.json**: Routing configuration that directs requests to the appropriate handler
|
|
82
|
+
- **functions/**: Your Mastra server bundled as a serverless function
|
|
83
|
+
- **static/**: Studio SPA assets (only when `studio: true`)
|
|
84
84
|
|
|
85
85
|
Without studio:
|
|
86
86
|
|
|
@@ -70,7 +70,7 @@ new MastraEditor({
|
|
|
70
70
|
|
|
71
71
|
## Related
|
|
72
72
|
|
|
73
|
-
- [Configuration](https://mastra.ai/docs/agent-builder/configuration)
|
|
74
|
-
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults)
|
|
75
|
-
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models)
|
|
76
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
73
|
+
- [Configuration](https://mastra.ai/docs/agent-builder/configuration): Concept and worked examples.
|
|
74
|
+
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults): Full schema for `configuration.agent`.
|
|
75
|
+
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models): Model allowlist and default model.
|
|
76
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): The parent surface that hosts `builder`.
|
|
@@ -71,9 +71,9 @@ new MastraEditor({
|
|
|
71
71
|
|
|
72
72
|
## Related
|
|
73
73
|
|
|
74
|
-
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options)
|
|
75
|
-
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models)
|
|
76
|
-
- [SerializedMemoryConfig](https://mastra.ai/reference/memory/serialized-memory-config)
|
|
77
|
-
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref)
|
|
78
|
-
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref)
|
|
79
|
-
- [Configuration](https://mastra.ai/docs/agent-builder/configuration)
|
|
74
|
+
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options): The parent options object.
|
|
75
|
+
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models): Admin-facing model allowlist and default.
|
|
76
|
+
- [SerializedMemoryConfig](https://mastra.ai/reference/memory/serialized-memory-config): Memory shape for new agents.
|
|
77
|
+
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref): Workspace reference shape.
|
|
78
|
+
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref): Browser reference shape.
|
|
79
|
+
- [Configuration](https://mastra.ai/docs/agent-builder/configuration): Concept and worked examples.
|
|
@@ -61,6 +61,6 @@ Mastra validates the admin policy at server boot. Violations surface as warnings
|
|
|
61
61
|
|
|
62
62
|
## Related
|
|
63
63
|
|
|
64
|
-
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults)
|
|
65
|
-
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options)
|
|
66
|
-
- [Model policy](https://mastra.ai/docs/agent-builder/model-policy)
|
|
64
|
+
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults): Parent object containing `models`.
|
|
65
|
+
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options): The top-level Builder options.
|
|
66
|
+
- [Model policy](https://mastra.ai/docs/agent-builder/model-policy): Concept and worked examples.
|
|
@@ -58,4 +58,4 @@ When no `providerId` is passed to `MastraEditor.resolveBlobStore()`, the editor
|
|
|
58
58
|
|
|
59
59
|
## Related
|
|
60
60
|
|
|
61
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
61
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Provider registry and `resolveBlobStore()` method.
|
|
@@ -72,6 +72,6 @@ new MastraEditor({
|
|
|
72
72
|
|
|
73
73
|
## Related
|
|
74
74
|
|
|
75
|
-
- [Browser](https://mastra.ai/docs/agent-builder/browser)
|
|
76
|
-
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref)
|
|
77
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
75
|
+
- [Browser](https://mastra.ai/docs/agent-builder/browser): Concept and worked examples.
|
|
76
|
+
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref): Stored configuration consumed by `createBrowser`.
|
|
77
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Provider registry.
|
|
@@ -58,7 +58,7 @@ Once registered, admins can reference the provider from an inline workspace conf
|
|
|
58
58
|
|
|
59
59
|
## Related
|
|
60
60
|
|
|
61
|
-
- [Workspace](https://mastra.ai/docs/agent-builder/workspace)
|
|
62
|
-
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref)
|
|
63
|
-
- [SandboxProvider](https://mastra.ai/reference/editor/sandbox-provider)
|
|
64
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
61
|
+
- [Workspace](https://mastra.ai/docs/agent-builder/workspace): Concept and worked examples.
|
|
62
|
+
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref): Stored configuration consumed by `createFilesystem`.
|
|
63
|
+
- [SandboxProvider](https://mastra.ai/reference/editor/sandbox-provider): Sibling provider for command execution.
|
|
64
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Provider registry.
|
|
@@ -61,10 +61,10 @@ Each provider field above takes a record keyed by provider id. See the per-provi
|
|
|
61
61
|
|
|
62
62
|
The `builder` field enables the [Agent Builder](https://mastra.ai/docs/agent-builder/overview), a browser-based UI for creating and editing stored agents. See:
|
|
63
63
|
|
|
64
|
-
- [Agent Builder overview](https://mastra.ai/docs/agent-builder/overview)
|
|
65
|
-
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options)
|
|
66
|
-
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults)
|
|
67
|
-
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models)
|
|
64
|
+
- [Agent Builder overview](https://mastra.ai/docs/agent-builder/overview): Concepts and getting started.
|
|
65
|
+
- [AgentBuilderOptions](https://mastra.ai/reference/editor/agent-builder/agent-builder-options): Full options schema.
|
|
66
|
+
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults): Admin-pinned defaults for new agents.
|
|
67
|
+
- [builder.configuration.agent.models](https://mastra.ai/reference/editor/agent-builder/builder-models): Model allowlist and default model.
|
|
68
68
|
|
|
69
69
|
### Registering the Builder agent
|
|
70
70
|
|
|
@@ -83,7 +83,7 @@ export const mastra = new Mastra({
|
|
|
83
83
|
})
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
-
The key name (`builderAgent`) is conventional
|
|
86
|
+
The key name (`builderAgent`) is conventional. Any key works. The `@mastra/editor/ee` subpath is gated by the Mastra Enterprise Edition license at runtime.
|
|
87
87
|
|
|
88
88
|
See the [Agent Builder overview](https://mastra.ai/docs/agent-builder/overview) for the full setup checklist.
|
|
89
89
|
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
`ProcessorProvider` is the interface a package implements to register a configurable processor with [`MastraEditor`](https://mastra.ai/reference/editor/mastra-editor). Processor providers serve two purposes:
|
|
6
6
|
|
|
7
|
-
1. **Discovery
|
|
8
|
-
2. **Runtime
|
|
7
|
+
1. **Discovery:** The UI uses `info`, `configSchema`, and `availablePhases` to render configuration forms.
|
|
8
|
+
2. **Runtime:** The editor calls `createProcessor(config)` during agent hydration to instantiate processors from stored configuration.
|
|
9
9
|
|
|
10
10
|
Providers are supplied via `MastraEditorConfig.processorProviders`.
|
|
11
11
|
|
|
@@ -63,4 +63,4 @@ export const myProcessorProvider: ProcessorProvider = {
|
|
|
63
63
|
|
|
64
64
|
## Related
|
|
65
65
|
|
|
66
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
66
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Provider registry.
|
|
@@ -57,7 +57,7 @@ Once registered, admins can reference the provider from an inline workspace conf
|
|
|
57
57
|
|
|
58
58
|
## Related
|
|
59
59
|
|
|
60
|
-
- [Workspace](https://mastra.ai/docs/agent-builder/workspace)
|
|
61
|
-
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref)
|
|
62
|
-
- [FilesystemProvider](https://mastra.ai/reference/editor/filesystem-provider)
|
|
63
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
60
|
+
- [Workspace](https://mastra.ai/docs/agent-builder/workspace): Concept and worked examples.
|
|
61
|
+
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref): Stored configuration consumed by `createSandbox`.
|
|
62
|
+
- [FilesystemProvider](https://mastra.ai/reference/editor/filesystem-provider): Sibling provider for file access.
|
|
63
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Provider registry.
|
|
@@ -38,7 +38,7 @@ new MastraEditor({
|
|
|
38
38
|
type StorageBrowserRef = { type: 'inline'; config: StorageBrowserConfig }
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
-
A `{ type: 'id' }` variant for browsers isn't available
|
|
41
|
+
A `{ type: 'id' }` variant for browsers isn't available: they're always inlined.
|
|
42
42
|
|
|
43
43
|
## Properties
|
|
44
44
|
|
|
@@ -72,11 +72,11 @@ The shape embedded under `config`. Defined in `@mastra/core/storage`.
|
|
|
72
72
|
|
|
73
73
|
## Hydration
|
|
74
74
|
|
|
75
|
-
`StorageBrowserRef` is resolved lazily on `mastra.editor.agent.getById()`. The editor looks up `config.provider` on `MastraEditor.browsers` and calls `provider.createBrowser(config)`. If the provider isn't registered, the editor logs a warning and returns `undefined
|
|
75
|
+
`StorageBrowserRef` is resolved lazily on `mastra.editor.agent.getById()`. The editor looks up `config.provider` on `MastraEditor.browsers` and calls `provider.createBrowser(config)`. If the provider isn't registered, the editor logs a warning and returns `undefined`: the agent still loads, but without a browser.
|
|
76
76
|
|
|
77
77
|
## Related
|
|
78
78
|
|
|
79
|
-
- [Browser](https://mastra.ai/docs/agent-builder/browser)
|
|
80
|
-
- [BrowserProvider](https://mastra.ai/reference/editor/browser-provider)
|
|
81
|
-
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults)
|
|
82
|
-
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref)
|
|
79
|
+
- [Browser](https://mastra.ai/docs/agent-builder/browser): Concept and worked examples.
|
|
80
|
+
- [BrowserProvider](https://mastra.ai/reference/editor/browser-provider): Implementer-facing provider interface.
|
|
81
|
+
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults): Where this type is pinned as the Builder default.
|
|
82
|
+
- [StorageWorkspaceRef](https://mastra.ai/reference/editor/storage-workspace-ref): Sibling reference type for workspace configuration.
|
|
@@ -89,7 +89,7 @@ The shape embedded under `{ type: 'inline', config }`. Defined in `@mastra/core/
|
|
|
89
89
|
|
|
90
90
|
## Related
|
|
91
91
|
|
|
92
|
-
- [Workspace](https://mastra.ai/docs/agent-builder/workspace)
|
|
93
|
-
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults)
|
|
94
|
-
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor)
|
|
95
|
-
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref)
|
|
92
|
+
- [Workspace](https://mastra.ai/docs/agent-builder/workspace): Concept and worked examples.
|
|
93
|
+
- [BuilderAgentDefaults](https://mastra.ai/reference/editor/agent-builder/builder-agent-defaults): Where this type is pinned as the Builder default.
|
|
94
|
+
- [MastraEditor class](https://mastra.ai/reference/editor/mastra-editor): Registers filesystem and sandbox providers.
|
|
95
|
+
- [StorageBrowserRef](https://mastra.ai/reference/editor/storage-browser-ref): Sibling reference type for browser configuration.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# ToolProvider
|
|
4
4
|
|
|
5
|
-
The `ToolProvider` interface defines how the editor discovers and resolves integration tools from external platforms. Mastra
|
|
5
|
+
The `ToolProvider` interface defines how the editor discovers and resolves integration tools from external platforms. Mastra includes two built-in implementations: `ComposioToolProvider` and `ArcadeToolProvider`.
|
|
6
6
|
|
|
7
7
|
See [Tools](https://mastra.ai/docs/editor/tools) for a guide on setting up tool providers.
|
|
8
8
|
|
|
@@ -39,7 +39,7 @@ The scorer evaluates relevancy through query-answer alignment, considering compl
|
|
|
39
39
|
### Scoring Process
|
|
40
40
|
|
|
41
41
|
1. **Statement Preprocess:**
|
|
42
|
-
- Breaks output into
|
|
42
|
+
- Breaks output into useful statements while preserving context.
|
|
43
43
|
|
|
44
44
|
2. **Relevance Analysis:**
|
|
45
45
|
|
|
@@ -57,9 +57,9 @@ The scorer evaluates relevancy through query-answer alignment, considering compl
|
|
|
57
57
|
A relevancy score between 0 and 1:
|
|
58
58
|
|
|
59
59
|
- **1.0**: The response fully answers the query with relevant and focused information.
|
|
60
|
-
- **0.7
|
|
61
|
-
- **0.4
|
|
62
|
-
- **0.1
|
|
60
|
+
- **0.7 to 0.9**: The response mostly answers the query but may include minor unrelated content.
|
|
61
|
+
- **0.4 to 0.6**: The response partially answers the query, mixing relevant and unrelated information.
|
|
62
|
+
- **0.1 to 0.3**: The response includes minimal relevant content and largely misses the intent of the query.
|
|
63
63
|
- **0.0**: The response is entirely unrelated and doesn't answer the query.
|
|
64
64
|
|
|
65
65
|
## Example
|
|
@@ -70,9 +70,9 @@ Final score: `(biased_opinions / total_opinions) * scale`
|
|
|
70
70
|
A bias score between 0 and 1:
|
|
71
71
|
|
|
72
72
|
- **1.0**: Contains explicit discriminatory or stereotypical statements.
|
|
73
|
-
- **0.7
|
|
74
|
-
- **0.4
|
|
75
|
-
- **0.1
|
|
73
|
+
- **0.7 to 0.9**: Includes strong prejudiced assumptions or generalizations.
|
|
74
|
+
- **0.4 to 0.6**: Mixes reasonable points with subtle bias or stereotypes.
|
|
75
|
+
- **0.1 to 0.3**: Mostly neutral with minor biased language or assumptions.
|
|
76
76
|
- **0.0**: Completely objective and free from bias.
|
|
77
77
|
|
|
78
78
|
## Example
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Quick Checks
|
|
4
4
|
|
|
5
|
-
Quick Checks are zero-LLM, composable micro-scorers for common assertions. They plug into the existing `scorers: [...]` array anywhere scorers are used
|
|
5
|
+
Quick Checks are zero-LLM, composable micro-scorers for common assertions. They plug into the existing `scorers: [...]` array anywhere scorers are used: in `runEvals`, live scoring, experiments, and Studio.
|
|
6
6
|
|
|
7
7
|
Internally they're standard `createScorer()` instances, so they have the same observability, storage, and pipeline integration as any other scorer.
|
|
8
8
|
|
|
@@ -76,7 +76,7 @@ Returns: `1` if equal, `0` otherwise.
|
|
|
76
76
|
|
|
77
77
|
### `checks.matches(pattern, options?)`
|
|
78
78
|
|
|
79
|
-
Scores 1 if the output matches the
|
|
79
|
+
Scores 1 if the output matches the regular expression.
|
|
80
80
|
|
|
81
81
|
```typescript
|
|
82
82
|
checks.matches(/\d+°[FC]/)
|
|
@@ -137,7 +137,7 @@ Returns: `1` if the tool wasn't called, `0` otherwise.
|
|
|
137
137
|
|
|
138
138
|
### `checks.toolOrder(expectedOrder)`
|
|
139
139
|
|
|
140
|
-
Scores 1 if the tools were called in the specified order. Uses relaxed matching
|
|
140
|
+
Scores 1 if the tools were called in the specified order. Uses relaxed matching. Other tool calls between the expected tools are allowed.
|
|
141
141
|
|
|
142
142
|
```typescript
|
|
143
143
|
checks.toolOrder(['search', 'summarize', 'respond'])
|
|
@@ -40,7 +40,7 @@ The scorer extracts and analyzes several types of elements:
|
|
|
40
40
|
- Nouns: Key objects, concepts, and entities
|
|
41
41
|
- Verbs: Actions and states (converted to infinitive form)
|
|
42
42
|
- Topics: Main subjects and themes
|
|
43
|
-
- Terms: Individual
|
|
43
|
+
- Terms: Individual substantial words
|
|
44
44
|
|
|
45
45
|
The extraction process includes:
|
|
46
46
|
|
|
@@ -83,10 +83,10 @@ Final score: `(covered_elements / total_input_elements) * scale`
|
|
|
83
83
|
|
|
84
84
|
A completeness score between 0 and 1:
|
|
85
85
|
|
|
86
|
-
- **1.0**: Thoroughly addresses all aspects of the query with
|
|
87
|
-
- **0.7
|
|
88
|
-
- **0.4
|
|
89
|
-
- **0.1
|
|
86
|
+
- **1.0**: Thoroughly addresses all aspects of the query with detailed detail.
|
|
87
|
+
- **0.7 to 0.9**: Covers most important aspects with good detail, minor gaps.
|
|
88
|
+
- **0.4 to 0.6**: Addresses some key points but missing important aspects or lacking detail.
|
|
89
|
+
- **0.1 to 0.3**: Only partially addresses the query with substantial gaps.
|
|
90
90
|
- **0.0**: Fails to address the query or provides irrelevant information.
|
|
91
91
|
|
|
92
92
|
## Example
|
|
@@ -89,11 +89,11 @@ To add this scorer to an agent, see the [Scorers overview](https://mastra.ai/doc
|
|
|
89
89
|
|
|
90
90
|
A similarity score between 0 and 1:
|
|
91
91
|
|
|
92
|
-
- **1.0**: Perfect match
|
|
93
|
-
- **0.7
|
|
94
|
-
- **0.4
|
|
95
|
-
- **0.1
|
|
96
|
-
- **0.0**: No similarity
|
|
92
|
+
- **1.0**: Perfect match, content is nearly identical.
|
|
93
|
+
- **0.7 to 0.9**: High similarity, minor differences in word choice or structure.
|
|
94
|
+
- **0.4 to 0.6**: Moderate similarity, general overlap with noticeable variation.
|
|
95
|
+
- **0.1 to 0.3**: Low similarity, few common elements or shared meaning.
|
|
96
|
+
- **0.0**: No similarity, completely different content.
|
|
97
97
|
|
|
98
98
|
## Related
|
|
99
99
|
|
|
@@ -28,7 +28,7 @@ Use when optimizing context selection for:
|
|
|
28
28
|
|
|
29
29
|
**options** (`ContextPrecisionMetricOptions`): Configuration options for the scorer
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
Either `context` or `contextExtractor` must be provided. If both are provided, `contextExtractor` takes precedence.
|
|
32
32
|
|
|
33
33
|
## `.run()` returns
|
|
34
34
|
|
|
@@ -85,7 +85,7 @@ Use results to:
|
|
|
85
85
|
|
|
86
86
|
### Example Calculation
|
|
87
87
|
|
|
88
|
-
|
|
88
|
+
context: `[relevant, irrelevant, relevant, irrelevant]`
|
|
89
89
|
|
|
90
90
|
- Position 0: Relevant → Precision = 1/1 = 1.0
|
|
91
91
|
- Position 1: Skip (irrelevant)
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The `createContextRecallScorer()` function creates a scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer. It measures retrieval completeness by checking what fraction of the ground truth's claims are attributable to the retrieved context.
|
|
6
6
|
|
|
7
|
-
This scorer requires a ground-truth reference answer, making it suitable for labeled datasets in CI or test environments. When `groundTruth`
|
|
7
|
+
This scorer requires a ground-truth reference answer, making it suitable for labeled datasets in CI or test environments. When `groundTruth` isn't provided in the run, the scorer returns a score of 0 rather than throwing an error.
|
|
8
8
|
|
|
9
9
|
## RAG retrieval evaluation
|
|
10
10
|
|
|
@@ -28,7 +28,7 @@ Use when running evaluations against curated test sets:
|
|
|
28
28
|
|
|
29
29
|
**options** (`ContextRecallMetricOptions`): Configuration options for the scorer
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
Either `context` or `contextExtractor` must be provided. When both are provided, `contextExtractor` is used only if the run input and output are agent-shaped (`MastraDBMessage[]`); otherwise the scorer falls back to `context`.
|
|
32
32
|
|
|
33
33
|
## `.run()` returns
|
|
34
34
|
|
|
@@ -62,11 +62,11 @@ Where:
|
|
|
62
62
|
|
|
63
63
|
These ranges assume the default `scale` of 1. When using a custom scale, multiply accordingly.
|
|
64
64
|
|
|
65
|
-
- **0.9-1.0**: Excellent recall
|
|
66
|
-
- **0.7-0.8**: Good recall
|
|
67
|
-
- **0.4-0.6**: Moderate recall
|
|
68
|
-
- **0.1-0.3**: Poor recall
|
|
69
|
-
- **0.0**: No recall
|
|
65
|
+
- **0.9-1.0**: Excellent recall, context covers nearly all ground-truth claims
|
|
66
|
+
- **0.7-0.8**: Good recall. Most claims are covered, with minor gaps
|
|
67
|
+
- **0.4-0.6**: Moderate recall, substantial information missing from context
|
|
68
|
+
- **0.1-0.3**: Poor recall, most ground-truth claims not found in context
|
|
69
|
+
- **0.0**: No recall, none of the ground-truth claims are in the context
|
|
70
70
|
|
|
71
71
|
### Reason analysis
|
|
72
72
|
|
|
@@ -87,7 +87,7 @@ Use results to:
|
|
|
87
87
|
|
|
88
88
|
### Example calculation
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
ground truth: "Einstein was born in 1879. He developed relativity. He won the Nobel Prize."
|
|
91
91
|
|
|
92
92
|
Claims extracted: 3
|
|
93
93
|
|
|
@@ -11,7 +11,7 @@ It's especially useful for these use cases:
|
|
|
11
11
|
Best for evaluating context quality in:
|
|
12
12
|
|
|
13
13
|
- Chat systems where context usage matters
|
|
14
|
-
- RAG pipelines needing
|
|
14
|
+
- RAG pipelines needing detailed relevance assessment
|
|
15
15
|
- Systems where missing context affects quality
|
|
16
16
|
|
|
17
17
|
## Context selection optimization
|
|
@@ -40,7 +40,7 @@ Note: Either `context` or `contextExtractor` must be provided. If both are provi
|
|
|
40
40
|
|
|
41
41
|
### Weighted Relevance Scoring
|
|
42
42
|
|
|
43
|
-
Context Relevance uses a
|
|
43
|
+
Context Relevance uses a advanced scoring algorithm that considers:
|
|
44
44
|
|
|
45
45
|
1. **Relevance Levels**: Each context piece is classified with weighted values:
|
|
46
46
|
|
|
@@ -77,7 +77,7 @@ Final Score = max(0, Base Score - Usage Penalty - Missing Penalty) × scale
|
|
|
77
77
|
|
|
78
78
|
- **0.9-1.0**: Excellent - all context highly relevant and used
|
|
79
79
|
- **0.7-0.8**: Good - mostly relevant with minor gaps
|
|
80
|
-
- **0.4-0.6**: Mixed -
|
|
80
|
+
- **0.4-0.6**: Mixed - substantial irrelevant or unused context
|
|
81
81
|
- **0.2-0.3**: Poor - mostly irrelevant context
|
|
82
82
|
- **0.0-0.1**: Very poor - no relevant context found
|
|
83
83
|
|
|
@@ -428,7 +428,7 @@ console.log(result)
|
|
|
428
428
|
|
|
429
429
|
### Dynamic context extraction
|
|
430
430
|
|
|
431
|
-
Extract context
|
|
431
|
+
Extract context at runtime based on the run input:
|
|
432
432
|
|
|
433
433
|
```typescript
|
|
434
434
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
@@ -110,7 +110,7 @@ export const responseQuality = createScorer({
|
|
|
110
110
|
|
|
111
111
|
With this configuration, a failed request has at most three provider attempts: the initial request plus two processor retries. If `generateScore` completes and `generateReason` receives a retryable failure, only `generateReason` retries.
|
|
112
112
|
|
|
113
|
-
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default
|
|
113
|
+
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default. Use `maxRetryAfterMs` to change that cap.
|
|
114
114
|
|
|
115
115
|
Avoid adding outer scorer or workflow retries. Avoid combining a nonzero model retry setting with this processor unless you intentionally accept additional attempts.
|
|
116
116
|
|
|
@@ -65,9 +65,9 @@ Final score: `(supported_claims / total_claims) * scale`
|
|
|
65
65
|
A faithfulness score between 0 and 1:
|
|
66
66
|
|
|
67
67
|
- **1.0**: All claims are accurate and directly supported by the context.
|
|
68
|
-
- **0.7
|
|
69
|
-
- **0.4
|
|
70
|
-
- **0.1
|
|
68
|
+
- **0.7 to 0.9**: Most claims are correct, with minor additions or omissions.
|
|
69
|
+
- **0.4 to 0.6**: Some claims are supported, but others are unverifiable.
|
|
70
|
+
- **0.1 to 0.3**: Most of the content is inaccurate or unsupported.
|
|
71
71
|
- **0.0**: All claims are false or contradict the context.
|
|
72
72
|
|
|
73
73
|
## Example
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# filterRun()
|
|
4
4
|
|
|
5
|
-
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages
|
|
5
|
+
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages and limit context size. It also drops unnecessary fields before the scorer pipeline runs.
|
|
6
6
|
|
|
7
7
|
Use [`filterRun()`](#usage-example) for declarative filtering. Write a custom `prepareRun` function directly when you need imperative logic that `filterRun()` doesn't cover. See [Custom scorers: input filtering](https://mastra.ai/docs/evals/custom-scorers) for more.
|
|
8
8
|
|
|
@@ -88,7 +88,7 @@ const simpleScorer = createScorer({
|
|
|
88
88
|
|
|
89
89
|
**options.dropGroundTruth** (`boolean`): Remove ground truth from the run.
|
|
90
90
|
|
|
91
|
-
**Returns:** `(run: ScorerRun) => ScorerRun
|
|
91
|
+
**Returns:** `(run: ScorerRun) => ScorerRun`. A function suitable for the `prepareRun` option on [`createScorer()`](https://mastra.ai/reference/evals/create-scorer).
|
|
92
92
|
|
|
93
93
|
## Part types
|
|
94
94
|
|
|
@@ -88,13 +88,13 @@ Final score: `(hallucinated_statements / total_statements) * scale`
|
|
|
88
88
|
|
|
89
89
|
A hallucination score between 0 and 1:
|
|
90
90
|
|
|
91
|
-
- **0.0**: No hallucination
|
|
92
|
-
- **0.3
|
|
93
|
-
- **0.5
|
|
94
|
-
- **0.7
|
|
95
|
-
- **0.9
|
|
91
|
+
- **0.0**: No hallucination, all claims match the context.
|
|
92
|
+
- **0.3 to 0.4**: Low hallucination, a few contradictions.
|
|
93
|
+
- **0.5 to 0.6**: Mixed hallucination, several contradictions.
|
|
94
|
+
- **0.7 to 0.8**: High hallucination, many contradictions.
|
|
95
|
+
- **0.9 to 1.0**: Complete hallucination, most or all claims contradict the context.
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
The score represents the degree of hallucination - lower scores indicate better factual alignment with the provided context
|
|
98
98
|
|
|
99
99
|
## Examples
|
|
100
100
|
|
|
@@ -66,11 +66,11 @@ Final score: `(matched_keywords / total_keywords) * scale`
|
|
|
66
66
|
|
|
67
67
|
A coverage score between 0 and 1:
|
|
68
68
|
|
|
69
|
-
- **1.0**: Complete coverage
|
|
70
|
-
- **0.7
|
|
71
|
-
- **0.4
|
|
72
|
-
- **0.1
|
|
73
|
-
- **0.0**:
|
|
69
|
+
- **1.0**: Complete coverage, all keywords present.
|
|
70
|
+
- **0.7 to 0.9**: High coverage, most keywords included.
|
|
71
|
+
- **0.4 to 0.6**: Partial coverage, some keywords present.
|
|
72
|
+
- **0.1 to 0.3**: Low coverage, few keywords matched.
|
|
73
|
+
- **0.0**: The response contains none of the keywords.
|
|
74
74
|
|
|
75
75
|
### Special Cases
|
|
76
76
|
|
|
@@ -80,7 +80,7 @@ The scorer handles several special cases:
|
|
|
80
80
|
- Single word: Treated as a single keyword
|
|
81
81
|
- Technical terms: Preserves compound technical terms (e.g., "React.js", "machine learning")
|
|
82
82
|
- Case differences: "JavaScript" matches "javascript"
|
|
83
|
-
- Common words: Ignored in scoring to focus on
|
|
83
|
+
- Common words: Ignored in scoring to focus on useful keywords
|
|
84
84
|
|
|
85
85
|
## Example
|
|
86
86
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The `MastraScorer` class is the base class for all scorers in Mastra. It provides a standard `.run()` method for evaluating input/output pairs and supports multi-step scoring workflows with preprocess → analyze → generateScore → generateReason execution flow.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Most users should use [`createScorer`](https://mastra.ai/reference/evals/create-scorer) to create scorer instances. Direct instantiation of `MastraScorer` isn't recommended.
|
|
8
8
|
|
|
9
9
|
## How to get a `MastraScorer` instance
|
|
10
10
|
|
|
@@ -26,7 +26,7 @@ const scorer = createScorer({
|
|
|
26
26
|
|
|
27
27
|
## `.run()` method
|
|
28
28
|
|
|
29
|
-
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a
|
|
29
|
+
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a detailed result object with the score, reasoning, and intermediate results.
|
|
30
30
|
|
|
31
31
|
```typescript
|
|
32
32
|
const result = await scorer.run({
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
# Noise sensitivity scorer
|
|
4
4
|
|
|
5
|
-
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how
|
|
5
|
+
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how reliable an agent is when exposed to irrelevant, distracting, or misleading information. Unlike live scorers that evaluate single production runs, this scorer requires predetermined test data including both baseline responses and noisy variations.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
This isn't a live scorer. It requires pre-computed baseline responses and can't be used for real-time agent evaluation. Use this scorer in your CI/CD pipeline or testing suites only.
|
|
8
8
|
|
|
9
9
|
Before using the noise sensitivity scorer, prepare your test data:
|
|
10
10
|
|
|
@@ -177,7 +177,7 @@ Include in your test harness to:
|
|
|
177
177
|
|
|
178
178
|
- Benchmark different models' noise resistance before deployment
|
|
179
179
|
- Identify agents vulnerable to manipulation during development
|
|
180
|
-
- Create
|
|
180
|
+
- Create detailed test coverage for various noise types
|
|
181
181
|
- Ensure consistent behavior across updates
|
|
182
182
|
|
|
183
183
|
### Security Testing
|
|
@@ -194,7 +194,7 @@ Evaluate resistance in controlled environments:
|
|
|
194
194
|
- **1.0**: Perfect robustness - no impact detected
|
|
195
195
|
- **0.8-0.9**: Excellent - minimal impact, core functionality preserved
|
|
196
196
|
- **0.6-0.7**: Good - some impact but acceptable for most use cases
|
|
197
|
-
- **0.4-0.5**: Concerning -
|
|
197
|
+
- **0.4-0.5**: Concerning - substantial vulnerabilities detected
|
|
198
198
|
- **0.0-0.3**: Critical - agent severely compromised by noise
|
|
199
199
|
|
|
200
200
|
### Dimension analysis
|
|
@@ -472,7 +472,7 @@ const strictScorer = createNoiseSensitivityScorerLLM({
|
|
|
472
472
|
|
|
473
473
|
## CI test suite: Testing different noise types
|
|
474
474
|
|
|
475
|
-
Create
|
|
475
|
+
Create detailed test suites to evaluate agent performance across noise categories in your CI pipeline:
|
|
476
476
|
|
|
477
477
|
```typescript
|
|
478
478
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|