@mastra/mcp-docs-server 1.2.11-alpha.6 → 1.2.12-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/docs/agents/using-tools.md +25 -0
- package/.docs/guides/build-your-ui/ai-sdk-ui.md +5 -5
- package/.docs/guides/build-your-ui/copilotkit/generative-ui.md +8 -8
- package/.docs/guides/build-your-ui/copilotkit/overview.md +2 -2
- package/.docs/guides/build-your-ui/openui.md +1 -1
- package/.docs/guides/concepts/multi-agent-systems.md +5 -5
- package/.docs/guides/concepts/streaming.md +9 -9
- package/.docs/guides/deployment/amazon-ec2.md +1 -1
- package/.docs/guides/deployment/aws-lambda.md +1 -1
- package/.docs/guides/deployment/azure-app-services.md +1 -1
- package/.docs/guides/deployment/cloudflare.md +1 -1
- package/.docs/guides/deployment/inngest.md +6 -6
- package/.docs/guides/deployment/temporal.md +1 -1
- package/.docs/guides/deployment/vercel.md +1 -1
- package/.docs/guides/getting-started/astro.md +1 -1
- package/.docs/guides/getting-started/electron.md +1 -1
- package/.docs/guides/getting-started/nestjs.md +5 -5
- package/.docs/guides/getting-started/vite-react.md +1 -1
- package/.docs/guides/guide/chef-michel.md +1 -1
- package/.docs/guides/guide/code-review-bot.md +1 -1
- package/.docs/guides/guide/coding-agent.md +4 -4
- package/.docs/guides/guide/dev-assistant.md +1 -1
- package/.docs/guides/guide/docs-manager.md +3 -3
- package/.docs/guides/guide/github-actions-pr-description.md +4 -4
- package/.docs/guides/guide/research-assistant.md +3 -3
- package/.docs/guides/guide/research-coordinator.md +3 -3
- package/.docs/guides/guide/slack-assistant.md +1 -1
- package/.docs/guides/guide/stock-agent.md +1 -1
- package/.docs/guides/index.md +1 -1
- package/.docs/guides/migrations/ai-sdk-v4-to-v5.md +1 -1
- package/.docs/guides/migrations/mastra-cloud.md +6 -6
- package/.docs/guides/migrations/network-to-supervisor.md +9 -9
- package/.docs/guides/migrations/upgrade-to-v1/agent.md +5 -5
- package/.docs/guides/migrations/upgrade-to-v1/cli.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/client.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/evals.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/memory.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/overview.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/processors.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/storage.md +11 -11
- package/.docs/guides/migrations/upgrade-to-v1/tools.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/tracing.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/voice.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/workflows.md +1 -1
- package/.docs/models/environment-variables.md +1 -0
- package/.docs/models/gateways/vercel.md +1 -2
- package/.docs/models/index.md +1 -1
- package/.docs/models/providers/aiand.md +1 -3
- package/.docs/models/providers/berget.md +2 -1
- package/.docs/models/providers/hyper.md +7 -7
- package/.docs/models/providers/modal.md +73 -0
- package/.docs/models/providers/xpersona.md +12 -2
- package/.docs/models/providers.md +1 -0
- package/.docs/reference/acp/acp-agent.md +3 -3
- package/.docs/reference/acp/create-acp-tool.md +2 -2
- package/.docs/reference/agent-controller/agent-controller-class.md +36 -36
- package/.docs/reference/agent-controller/session.md +6 -6
- package/.docs/reference/agents/agent.md +7 -7
- package/.docs/reference/agents/channels.md +3 -3
- package/.docs/reference/agents/durable-agent.md +3 -3
- package/.docs/reference/agents/generate.md +3 -3
- package/.docs/reference/agents/getDefaultOptions.md +1 -1
- package/.docs/reference/agents/getDefaultStreamOptions.md +1 -1
- package/.docs/reference/agents/listScorers.md +1 -1
- package/.docs/reference/agents/listSkills.md +1 -1
- package/.docs/reference/agents/listSuspendedRuns.md +2 -2
- package/.docs/reference/ai-sdk/chat-route.md +1 -1
- package/.docs/reference/ai-sdk/network-route.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v4-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v5-messages.md +1 -1
- package/.docs/reference/ai-sdk/workflow-route.md +1 -1
- package/.docs/reference/ai-sdk/workflow-snapshot-to-stream.md +2 -2
- package/.docs/reference/auth/auth0.md +1 -1
- package/.docs/reference/auth/fga.md +2 -2
- package/.docs/reference/auth/google.md +2 -2
- package/.docs/reference/auth/okta.md +1 -1
- package/.docs/reference/auth/workos.md +2 -2
- package/.docs/reference/browser/agent-browser.md +2 -2
- package/.docs/reference/browser/browser-viewer.md +3 -3
- package/.docs/reference/browser/mastra-browser.md +3 -3
- package/.docs/reference/browser/stagehand-browser.md +4 -4
- package/.docs/reference/channels/slack-provider.md +2 -2
- package/.docs/reference/cli/create-mastra.md +1 -1
- package/.docs/reference/cli/mastra.md +23 -23
- package/.docs/reference/client-js/agents.md +9 -9
- package/.docs/reference/client-js/memory.md +2 -2
- package/.docs/reference/client-js/workflows.md +1 -1
- package/.docs/reference/coding-agent/build-base-prompt.md +2 -2
- package/.docs/reference/coding-agent/create-coding-agent.md +2 -2
- package/.docs/reference/configuration.md +6 -6
- package/.docs/reference/core/getMCPServer.md +1 -1
- package/.docs/reference/core/getScorer.md +1 -1
- package/.docs/reference/datasets/addItem.md +1 -1
- package/.docs/reference/datasets/listItems.md +1 -1
- package/.docs/reference/deployer/netlify.md +4 -4
- package/.docs/reference/deployer/vercel.md +7 -7
- package/.docs/reference/editor/agent-builder/agent-builder-options.md +4 -4
- package/.docs/reference/editor/agent-builder/builder-agent-defaults.md +6 -6
- package/.docs/reference/editor/agent-builder/builder-models.md +3 -3
- package/.docs/reference/editor/blob-store-provider.md +1 -1
- package/.docs/reference/editor/browser-provider.md +3 -3
- package/.docs/reference/editor/filesystem-provider.md +4 -4
- package/.docs/reference/editor/mastra-editor.md +5 -5
- package/.docs/reference/editor/processor-provider.md +3 -3
- package/.docs/reference/editor/sandbox-provider.md +4 -4
- package/.docs/reference/editor/storage-browser-ref.md +6 -6
- package/.docs/reference/editor/storage-workspace-ref.md +4 -4
- package/.docs/reference/editor/tool-provider.md +1 -1
- package/.docs/reference/evals/answer-relevancy.md +4 -4
- package/.docs/reference/evals/bias.md +3 -3
- package/.docs/reference/evals/checks.md +3 -3
- package/.docs/reference/evals/completeness.md +5 -5
- package/.docs/reference/evals/content-similarity.md +5 -5
- package/.docs/reference/evals/context-precision.md +2 -2
- package/.docs/reference/evals/context-recall.md +8 -8
- package/.docs/reference/evals/context-relevance.md +4 -4
- package/.docs/reference/evals/create-scorer.md +1 -1
- package/.docs/reference/evals/faithfulness.md +3 -3
- package/.docs/reference/evals/filter-run.md +2 -2
- package/.docs/reference/evals/hallucination.md +6 -6
- package/.docs/reference/evals/keyword-coverage.md +6 -6
- package/.docs/reference/evals/mastra-scorer.md +9 -3
- package/.docs/reference/evals/noise-sensitivity.md +5 -5
- package/.docs/reference/evals/prompt-alignment.md +5 -5
- package/.docs/reference/evals/rubric.md +1 -1
- package/.docs/reference/evals/run-evals.md +3 -3
- package/.docs/reference/evals/scorer-utils.md +4 -4
- package/.docs/reference/evals/textual-difference.md +4 -4
- package/.docs/reference/evals/tool-call-accuracy.md +1 -1
- package/.docs/reference/evals/toxicity.md +4 -4
- package/.docs/reference/evals/trajectory-accuracy.md +10 -10
- package/.docs/reference/file-based-agents/instructions.md +2 -2
- package/.docs/reference/file-based-agents/logger.md +1 -1
- package/.docs/reference/file-based-agents/observability.md +1 -1
- package/.docs/reference/file-based-agents/processors.md +1 -1
- package/.docs/reference/file-based-agents/skills.md +1 -1
- package/.docs/reference/file-based-agents/storage.md +1 -1
- package/.docs/reference/file-based-agents/studio.md +1 -1
- package/.docs/reference/file-based-agents/subagents.md +1 -1
- package/.docs/reference/file-based-agents/tools.md +1 -1
- package/.docs/reference/file-based-agents/workspace.md +1 -1
- package/.docs/reference/logging/pino-logger.md +3 -3
- package/.docs/reference/memory/clone-utilities.md +1 -1
- package/.docs/reference/memory/cloneThread.md +3 -3
- package/.docs/reference/memory/listThreads.md +1 -1
- package/.docs/reference/memory/memory-class.md +1 -1
- package/.docs/reference/memory/observational-memory.md +11 -11
- package/.docs/reference/memory/recall.md +2 -2
- package/.docs/reference/memory/serialized-memory-config.md +4 -4
- package/.docs/reference/memory/summarizeConversation.md +3 -3
- package/.docs/reference/memory/summarizeThread.md +3 -3
- package/.docs/reference/observability/feedback.md +1 -1
- package/.docs/reference/observability/metrics/automatic-metrics.md +7 -7
- package/.docs/reference/observability/tracing/bridges/datadog.md +2 -2
- package/.docs/reference/observability/tracing/exporters/cloud-exporter.md +3 -3
- package/.docs/reference/observability/tracing/exporters/default-exporter.md +1 -1
- package/.docs/reference/observability/tracing/exporters/mastra-platform-exporter.md +4 -4
- package/.docs/reference/observability/tracing/exporters/mastra-storage-exporter.md +1 -1
- package/.docs/reference/observability/tracing/interfaces.md +1 -1
- package/.docs/reference/observability/tracing/processors/sensitive-data-filter.md +1 -1
- package/.docs/reference/observability/tracing/span-filtering.md +1 -1
- package/.docs/reference/processors/cost-guard-processor.md +1 -1
- package/.docs/reference/processors/pii-detector.md +1 -1
- package/.docs/reference/processors/processor-interface.md +12 -12
- package/.docs/reference/processors/prompt-injection-detector.md +1 -1
- package/.docs/reference/processors/provider-history-compat.md +1 -1
- package/.docs/reference/processors/regex-filter-processor.md +1 -1
- package/.docs/reference/processors/response-cache.md +8 -8
- package/.docs/reference/processors/system-prompt-scrubber.md +1 -1
- package/.docs/reference/processors/token-limiter-processor.md +1 -1
- package/.docs/reference/processors/tool-search-processor.md +7 -7
- package/.docs/reference/processors/unicode-normalizer.md +1 -1
- package/.docs/reference/project-structure.md +1 -1
- package/.docs/reference/pubsub/base.md +4 -4
- package/.docs/reference/pubsub/google-cloud-pubsub.md +2 -2
- package/.docs/reference/pubsub/lease-provider.md +4 -4
- package/.docs/reference/pubsub/redis-streams.md +1 -1
- package/.docs/reference/pubsub/unix-socket-pubsub.md +1 -1
- package/.docs/reference/rag/chunk.md +5 -5
- package/.docs/reference/rag/database-config.md +2 -2
- package/.docs/reference/rag/metadata-filters.md +2 -2
- package/.docs/reference/schedules/overview.md +1 -1
- package/.docs/reference/server/register-api-route.md +1 -1
- package/.docs/reference/server/routes.md +1 -1
- package/.docs/reference/signals/signal-provider.md +8 -8
- package/.docs/reference/storage/clickhouse.md +4 -4
- package/.docs/reference/storage/cloudflare-d1.md +4 -4
- package/.docs/reference/storage/convex.md +4 -4
- package/.docs/reference/storage/dsql.md +2 -2
- package/.docs/reference/storage/dynamodb.md +7 -7
- package/.docs/reference/storage/lance.md +1 -1
- package/.docs/reference/storage/mongodb.md +1 -1
- package/.docs/reference/storage/mssql.md +1 -1
- package/.docs/reference/storage/postgresql.md +1 -1
- package/.docs/reference/storage/retention.md +33 -33
- package/.docs/reference/storage/spanner.md +8 -8
- package/.docs/reference/streaming/ChunkType.md +2 -2
- package/.docs/reference/streaming/agents/MastraModelOutput.md +1 -1
- package/.docs/reference/streaming/agents/stream.md +1 -1
- package/.docs/reference/streaming/agents/streamUntilIdle.md +2 -2
- package/.docs/reference/streaming/workflows/observeStream.md +1 -1
- package/.docs/reference/streaming/workflows/resumeStream.md +1 -1
- package/.docs/reference/streaming/workflows/stream.md +1 -1
- package/.docs/reference/streaming/workflows/timeTravelStream.md +2 -2
- package/.docs/reference/templates/overview.md +1 -1
- package/.docs/reference/tools/brightdata.md +1 -1
- package/.docs/reference/tools/create-tool.md +1 -1
- package/.docs/reference/tools/graph-rag-tool.md +1 -1
- package/.docs/reference/tools/mcp-client.md +13 -13
- package/.docs/reference/tools/mcp-server.md +20 -20
- package/.docs/reference/tools/submit-plan-tool.md +1 -1
- package/.docs/reference/tools/tavily.md +1 -1
- package/.docs/reference/tools/vector-query-tool.md +4 -4
- package/.docs/reference/vectors/astra.md +1 -1
- package/.docs/reference/vectors/chroma.md +2 -2
- package/.docs/reference/vectors/convex.md +2 -2
- package/.docs/reference/vectors/couchbase.md +4 -4
- package/.docs/reference/vectors/libsql.md +1 -1
- package/.docs/reference/vectors/mongodb.md +1 -1
- package/.docs/reference/vectors/pg.md +4 -4
- package/.docs/reference/vectors/qdrant.md +2 -2
- package/.docs/reference/vectors/s3vectors.md +4 -4
- package/.docs/reference/vectors/turbopuffer.md +1 -1
- package/.docs/reference/voice/aws-nova-sonic.md +1 -1
- package/.docs/reference/voice/azure.md +1 -1
- package/.docs/reference/voice/google-gemini-live.md +4 -4
- package/.docs/reference/voice/inworld-realtime.md +7 -7
- package/.docs/reference/voice/livekit.md +13 -13
- package/.docs/reference/voice/mistral.md +1 -1
- package/.docs/reference/voice/openai-realtime.md +1 -1
- package/.docs/reference/voice/sarvam.md +1 -1
- package/.docs/reference/voice/voice.addTools.md +1 -1
- package/.docs/reference/voice/voice.events.md +1 -1
- package/.docs/reference/voice/voice.getSpeakers.md +2 -2
- package/.docs/reference/voice/voice.on.md +2 -2
- package/.docs/reference/voice/xai-realtime.md +2 -2
- package/.docs/reference/workflows/run-methods/cancel.md +1 -1
- package/.docs/reference/workflows/run-methods/startAsync.md +1 -1
- package/.docs/reference/workflows/run-methods/timeTravel.md +1 -1
- package/.docs/reference/workflows/workflow-methods/foreach.md +1 -1
- package/.docs/reference/workflows/workflow-methods/sleep.md +1 -1
- package/.docs/reference/workflows/workflow-state-reader.md +2 -2
- package/.docs/reference/workspace/apple-container-sandbox.md +2 -2
- package/.docs/reference/workspace/daytona-sandbox.md +3 -3
- package/.docs/reference/workspace/docker-sandbox.md +2 -2
- package/.docs/reference/workspace/e2b-sandbox.md +2 -2
- package/.docs/reference/workspace/files-sdk-filesystem.md +3 -3
- package/.docs/reference/workspace/google-drive-filesystem.md +4 -4
- package/.docs/reference/workspace/local-filesystem.md +2 -2
- package/.docs/reference/workspace/local-sandbox.md +2 -2
- package/.docs/reference/workspace/modal-sandbox.md +1 -1
- package/.docs/reference/workspace/platform-filesystem.md +3 -3
- package/.docs/reference/workspace/platform-sandbox.md +2 -2
- package/.docs/reference/workspace/process-manager.md +1 -1
- package/.docs/reference/workspace/railway-sandbox.md +4 -4
- package/.docs/reference/workspace/s3-filesystem.md +1 -1
- package/.docs/reference/workspace/sandbox.md +1 -1
- package/.docs/reference/workspace/vercel-sandbox.md +1 -1
- package/.docs/reference/workspace/workspace-class.md +10 -10
- package/CHANGELOG.md +14 -0
- package/package.json +6 -6
|
@@ -110,7 +110,7 @@ export const responseQuality = createScorer({
|
|
|
110
110
|
|
|
111
111
|
With this configuration, a failed request has at most three provider attempts: the initial request plus two processor retries. If `generateScore` completes and `generateReason` receives a retryable failure, only `generateReason` retries.
|
|
112
112
|
|
|
113
|
-
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default
|
|
113
|
+
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default. Use `maxRetryAfterMs` to change that cap.
|
|
114
114
|
|
|
115
115
|
Avoid adding outer scorer or workflow retries. Avoid combining a nonzero model retry setting with this processor unless you intentionally accept additional attempts.
|
|
116
116
|
|
|
@@ -65,9 +65,9 @@ Final score: `(supported_claims / total_claims) * scale`
|
|
|
65
65
|
A faithfulness score between 0 and 1:
|
|
66
66
|
|
|
67
67
|
- **1.0**: All claims are accurate and directly supported by the context.
|
|
68
|
-
- **0.7
|
|
69
|
-
- **0.4
|
|
70
|
-
- **0.1
|
|
68
|
+
- **0.7 to 0.9**: Most claims are correct, with minor additions or omissions.
|
|
69
|
+
- **0.4 to 0.6**: Some claims are supported, but others are unverifiable.
|
|
70
|
+
- **0.1 to 0.3**: Most of the content is inaccurate or unsupported.
|
|
71
71
|
- **0.0**: All claims are false or contradict the context.
|
|
72
72
|
|
|
73
73
|
## Example
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# filterRun()
|
|
4
4
|
|
|
5
|
-
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages
|
|
5
|
+
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages and limit context size. It also drops unnecessary fields before the scorer pipeline runs.
|
|
6
6
|
|
|
7
7
|
Use [`filterRun()`](#usage-example) for declarative filtering. Write a custom `prepareRun` function directly when you need imperative logic that `filterRun()` doesn't cover. See [Custom scorers: input filtering](https://mastra.ai/docs/evals/custom-scorers) for more.
|
|
8
8
|
|
|
@@ -88,7 +88,7 @@ const simpleScorer = createScorer({
|
|
|
88
88
|
|
|
89
89
|
**options.dropGroundTruth** (`boolean`): Remove ground truth from the run.
|
|
90
90
|
|
|
91
|
-
**Returns:** `(run: ScorerRun) => ScorerRun
|
|
91
|
+
**Returns:** `(run: ScorerRun) => ScorerRun`. A function suitable for the `prepareRun` option on [`createScorer()`](https://mastra.ai/reference/evals/create-scorer).
|
|
92
92
|
|
|
93
93
|
## Part types
|
|
94
94
|
|
|
@@ -88,13 +88,13 @@ Final score: `(hallucinated_statements / total_statements) * scale`
|
|
|
88
88
|
|
|
89
89
|
A hallucination score between 0 and 1:
|
|
90
90
|
|
|
91
|
-
- **0.0**: No hallucination
|
|
92
|
-
- **0.3
|
|
93
|
-
- **0.5
|
|
94
|
-
- **0.7
|
|
95
|
-
- **0.9
|
|
91
|
+
- **0.0**: No hallucination, all claims match the context.
|
|
92
|
+
- **0.3 to 0.4**: Low hallucination, a few contradictions.
|
|
93
|
+
- **0.5 to 0.6**: Mixed hallucination, several contradictions.
|
|
94
|
+
- **0.7 to 0.8**: High hallucination, many contradictions.
|
|
95
|
+
- **0.9 to 1.0**: Complete hallucination, most or all claims contradict the context.
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
The score represents the degree of hallucination - lower scores indicate better factual alignment with the provided context
|
|
98
98
|
|
|
99
99
|
## Examples
|
|
100
100
|
|
|
@@ -66,11 +66,11 @@ Final score: `(matched_keywords / total_keywords) * scale`
|
|
|
66
66
|
|
|
67
67
|
A coverage score between 0 and 1:
|
|
68
68
|
|
|
69
|
-
- **1.0**: Complete coverage
|
|
70
|
-
- **0.7
|
|
71
|
-
- **0.4
|
|
72
|
-
- **0.1
|
|
73
|
-
- **0.0**:
|
|
69
|
+
- **1.0**: Complete coverage, all keywords present.
|
|
70
|
+
- **0.7 to 0.9**: High coverage, most keywords included.
|
|
71
|
+
- **0.4 to 0.6**: Partial coverage, some keywords present.
|
|
72
|
+
- **0.1 to 0.3**: Low coverage, few keywords matched.
|
|
73
|
+
- **0.0**: The response contains none of the keywords.
|
|
74
74
|
|
|
75
75
|
### Special Cases
|
|
76
76
|
|
|
@@ -80,7 +80,7 @@ The scorer handles several special cases:
|
|
|
80
80
|
- Single word: Treated as a single keyword
|
|
81
81
|
- Technical terms: Preserves compound technical terms (e.g., "React.js", "machine learning")
|
|
82
82
|
- Case differences: "JavaScript" matches "javascript"
|
|
83
|
-
- Common words: Ignored in scoring to focus on
|
|
83
|
+
- Common words: Ignored in scoring to focus on useful keywords
|
|
84
84
|
|
|
85
85
|
## Example
|
|
86
86
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The `MastraScorer` class is the base class for all scorers in Mastra. It provides a standard `.run()` method for evaluating input/output pairs and supports multi-step scoring workflows with preprocess → analyze → generateScore → generateReason execution flow.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Most users should use [`createScorer`](https://mastra.ai/reference/evals/create-scorer) to create scorer instances. Direct instantiation of `MastraScorer` isn't recommended.
|
|
8
8
|
|
|
9
9
|
## How to get a `MastraScorer` instance
|
|
10
10
|
|
|
@@ -26,7 +26,7 @@ const scorer = createScorer({
|
|
|
26
26
|
|
|
27
27
|
## `.run()` method
|
|
28
28
|
|
|
29
|
-
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a
|
|
29
|
+
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a detailed result object with the score, reasoning, and intermediate results.
|
|
30
30
|
|
|
31
31
|
```typescript
|
|
32
32
|
const result = await scorer.run({
|
|
@@ -76,7 +76,8 @@ const result = await scorer.run({
|
|
|
76
76
|
The optional `judge` record contains details about the judge model calls made by prompt-based scorer steps. Its known keys are `preprocess`, `analyze`, `generateScore`, and `generateReason`. Each key contains an ordered `executions` array.
|
|
77
77
|
|
|
78
78
|
```typescript
|
|
79
|
-
interface
|
|
79
|
+
interface ScorerJudgeExecutionSuccess {
|
|
80
|
+
status: 'success'
|
|
80
81
|
prompt: string
|
|
81
82
|
output: JSONValue
|
|
82
83
|
judgeModelId: string
|
|
@@ -92,6 +93,8 @@ interface ScorerJudgeExecution {
|
|
|
92
93
|
}
|
|
93
94
|
}
|
|
94
95
|
|
|
96
|
+
type ScorerJudgeExecution = ScorerJudgeExecutionSuccess
|
|
97
|
+
|
|
95
98
|
interface ScorerJudgeUsage {
|
|
96
99
|
inputTokens?: number
|
|
97
100
|
outputTokens?: number
|
|
@@ -114,11 +117,14 @@ Use the step key to access its judge execution details:
|
|
|
114
117
|
```typescript
|
|
115
118
|
const execution = result.judge?.generateScore?.executions[0]
|
|
116
119
|
|
|
120
|
+
console.log(execution?.status)
|
|
117
121
|
console.log(execution?.judgeModelId)
|
|
118
122
|
console.log(execution?.usage.totalTokens)
|
|
119
123
|
console.log(execution?.durationMs)
|
|
120
124
|
```
|
|
121
125
|
|
|
126
|
+
The `status` value describes the outcome of the logical prompt-step execution, not the quality of the evaluated response. A structured-output fallback that eventually succeeds creates one `success` execution with an `attemptCount` greater than one.
|
|
127
|
+
|
|
122
128
|
`attemptCount` counts judge invocations, including a structured-output fallback. `modelCallCount` counts the completed model steps across those attempts. `durationMs` covers the full prompt-step execution.
|
|
123
129
|
|
|
124
130
|
Function steps don't create `judge` entries. Usage in this record belongs to the scorer's judge model, not the agent or workflow being evaluated. The optional `cost` field is present only when the execution directly reports an authoritative cost, source, and unit.
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
# Noise sensitivity scorer
|
|
4
4
|
|
|
5
|
-
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how
|
|
5
|
+
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how reliable an agent is when exposed to irrelevant, distracting, or misleading information. Unlike live scorers that evaluate single production runs, this scorer requires predetermined test data including both baseline responses and noisy variations.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
This isn't a live scorer. It requires pre-computed baseline responses and can't be used for real-time agent evaluation. Use this scorer in your CI/CD pipeline or testing suites only.
|
|
8
8
|
|
|
9
9
|
Before using the noise sensitivity scorer, prepare your test data:
|
|
10
10
|
|
|
@@ -177,7 +177,7 @@ Include in your test harness to:
|
|
|
177
177
|
|
|
178
178
|
- Benchmark different models' noise resistance before deployment
|
|
179
179
|
- Identify agents vulnerable to manipulation during development
|
|
180
|
-
- Create
|
|
180
|
+
- Create detailed test coverage for various noise types
|
|
181
181
|
- Ensure consistent behavior across updates
|
|
182
182
|
|
|
183
183
|
### Security Testing
|
|
@@ -194,7 +194,7 @@ Evaluate resistance in controlled environments:
|
|
|
194
194
|
- **1.0**: Perfect robustness - no impact detected
|
|
195
195
|
- **0.8-0.9**: Excellent - minimal impact, core functionality preserved
|
|
196
196
|
- **0.6-0.7**: Good - some impact but acceptable for most use cases
|
|
197
|
-
- **0.4-0.5**: Concerning -
|
|
197
|
+
- **0.4-0.5**: Concerning - substantial vulnerabilities detected
|
|
198
198
|
- **0.0-0.3**: Critical - agent severely compromised by noise
|
|
199
199
|
|
|
200
200
|
### Dimension analysis
|
|
@@ -472,7 +472,7 @@ const strictScorer = createNoiseSensitivityScorerLLM({
|
|
|
472
472
|
|
|
473
473
|
## CI test suite: Testing different noise types
|
|
474
474
|
|
|
475
|
-
Create
|
|
475
|
+
Create detailed test suites to evaluate agent performance across noise categories in your CI pipeline:
|
|
476
476
|
|
|
477
477
|
```typescript
|
|
478
478
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Prompt alignment scorer
|
|
4
4
|
|
|
5
|
-
The `createPromptAlignmentScorerLLM()` function creates a scorer that evaluates how well agent responses align with user prompts across
|
|
5
|
+
The `createPromptAlignmentScorerLLM()` function creates a scorer that evaluates how well agent responses align with user prompts across intent understanding and requirement fulfillment, plus response completeness, and format appropriateness.
|
|
6
6
|
|
|
7
7
|
## Parameters
|
|
8
8
|
|
|
@@ -80,7 +80,7 @@ Evaluates alignment with user prompts only:
|
|
|
80
80
|
|
|
81
81
|
1. **Intent Alignment** (40% weight): Whether the response addresses the user's core request
|
|
82
82
|
2. **Requirements Fulfillment** (30% weight): If all user requirements are met
|
|
83
|
-
3. **Completeness** (20% weight): Whether the response is
|
|
83
|
+
3. **Completeness** (20% weight): Whether the response is detailed for user needs
|
|
84
84
|
4. **Response Appropriateness** (10% weight): If format and tone match user expectations
|
|
85
85
|
|
|
86
86
|
#### System Mode ('system')
|
|
@@ -139,7 +139,7 @@ Final Score = Weighted Score × scale
|
|
|
139
139
|
- **0.8-0.9** = Very good alignment with minor gaps
|
|
140
140
|
- **0.7-0.8** = Good alignment but missing some requirements or completeness
|
|
141
141
|
- **0.6-0.7** = Moderate alignment with noticeable gaps
|
|
142
|
-
- **0.4-0.6** = Poor alignment with
|
|
142
|
+
- **0.4-0.6** = Poor alignment with substantial issues
|
|
143
143
|
- **0.0-0.4** = Very poor alignment, response doesn't address the prompt effectively
|
|
144
144
|
|
|
145
145
|
### When to Use Each Mode
|
|
@@ -160,7 +160,7 @@ Final Score = Weighted Score × scale
|
|
|
160
160
|
|
|
161
161
|
**Both Mode (`'both'`)** - Use when (default, recommended):
|
|
162
162
|
|
|
163
|
-
- Comprehensive evaluation of
|
|
163
|
+
- Comprehensive evaluation of AI agent performance
|
|
164
164
|
- Balancing user satisfaction with system compliance
|
|
165
165
|
- Production monitoring where both user and system requirements matter
|
|
166
166
|
- Holistic assessment of prompt-response alignment
|
|
@@ -403,7 +403,7 @@ console.log(result)
|
|
|
403
403
|
|
|
404
404
|
### Excellent alignment output
|
|
405
405
|
|
|
406
|
-
The output receives a high score because it perfectly addresses the intent
|
|
406
|
+
The output receives a high score because it perfectly addresses the intent and fulfills all requirements. It also uses the appropriate format.
|
|
407
407
|
|
|
408
408
|
```typescript
|
|
409
409
|
{
|
|
@@ -106,7 +106,7 @@ The scorer runs in two phases:
|
|
|
106
106
|
1. **Grade**: The judge model evaluates each criterion independently and returns a per-criterion verdict (`satisfied` / not) with reasoning.
|
|
107
107
|
2. **Score**: The result is `1` only when every required criterion is `satisfied`, otherwise `0`. If no criteria are marked required, all criteria are treated as required.
|
|
108
108
|
|
|
109
|
-
The `reason` summarizes the
|
|
109
|
+
The `reason` summarizes the result and lists each criterion with its verdict, so a failing grade gives the agent targeted, useful feedback rather than a generic "try again".
|
|
110
110
|
|
|
111
111
|
## Related
|
|
112
112
|
|
|
@@ -371,9 +371,9 @@ const result = await runEvals({
|
|
|
371
371
|
// result.verdict: 'passed' | 'scored' | 'failed'
|
|
372
372
|
```
|
|
373
373
|
|
|
374
|
-
Each turn runs `agent.generate()` with the same `threadId`, so the agent sees the full conversation history. `runEvals` also injects a `resourceId` (Mastra memory scopes messages by resource + thread), defaulting it to the generated thread
|
|
374
|
+
Each turn runs `agent.generate()` with the same `threadId`, so the agent sees the full conversation history. `runEvals` also injects a `resourceId` (Mastra memory scopes messages by resource + thread), defaulting it to the generated thread. Pass `targetOptions.memory.resource` to pin a specific one. Cross-turn recall requires the agent to have a memory store configured. Otherwise turns run in isolation. Mix single-turn (`input`) and multi-turn (`inputs`) items in the same `data` array. When using `inputs`, `input` can be omitted.
|
|
375
375
|
|
|
376
|
-
Scoring uses the accumulated output from all turns as `run.output`, but only the first turn as `run.input`. Prefer output-based scorers (`checks.includes`, `checks.calledTool`, `checks.similarity`) for multi-turn
|
|
376
|
+
Scoring uses the accumulated output from all turns as `run.output`, but only the first turn as `run.input`. Prefer output-based scorers (`checks.includes`, `checks.calledTool`, `checks.similarity`) for multi-turn. Input-relative scorers (e.g. faithfulness) only see the first turn's input. Trajectory scorers that read from the trace (`AgentScorerConfig.trajectory`) resolve against the last turn's span. Tool-call checks that read `run.output` (like `checks.calledTool`) still see every turn.
|
|
377
377
|
|
|
378
378
|
### Per-turn assertions
|
|
379
379
|
|
|
@@ -403,7 +403,7 @@ result.verdict // folds in per-turn gate/threshold outcomes
|
|
|
403
403
|
result.turnResults // [{ index, gateResults, thresholdResults, scores }]
|
|
404
404
|
```
|
|
405
405
|
|
|
406
|
-
Per-turn gates/scorers evaluate only that turn (`run.input`/`run.output` are that turn's). A failing turn gate makes the verdict `failed
|
|
406
|
+
Per-turn gates/scorers evaluate only that turn (`run.input`/`run.output` are that turn's). A failing turn gate makes the verdict `failed`. A missed turn threshold (gates passing) makes it `scored`. Top-level `scorers`/`gates` still score the accumulated conversation as a whole. `turns` is Agent-only and can't be combined with `input` or `inputs`.
|
|
407
407
|
|
|
408
408
|
## Related
|
|
409
409
|
|
|
@@ -284,7 +284,7 @@ const result = await myScorer.run({
|
|
|
284
284
|
|
|
285
285
|
### `extractTrajectory`
|
|
286
286
|
|
|
287
|
-
Extracts a `Trajectory` from agent output messages (`MastraDBMessage[]`). Converts tool invocations into `ToolCallStep` objects. The `runEvals` pipeline calls this automatically for trajectory scorers
|
|
287
|
+
Extracts a `Trajectory` from agent output messages (`MastraDBMessage[]`). Converts tool invocations into `ToolCallStep` objects. The `runEvals` pipeline calls this automatically for trajectory scorers: you only need it for direct testing.
|
|
288
288
|
|
|
289
289
|
Available from `@mastra/core/evals`.
|
|
290
290
|
|
|
@@ -296,7 +296,7 @@ const trajectory = extractTrajectory(agentOutputMessages)
|
|
|
296
296
|
// trajectory.rawOutput — the original MastraDBMessage[] array
|
|
297
297
|
```
|
|
298
298
|
|
|
299
|
-
**Returns:** `Trajectory
|
|
299
|
+
**Returns:** `Trajectory`: Contains `steps: TrajectoryStep[]`, `totalDurationMs`, and `rawOutput`.
|
|
300
300
|
|
|
301
301
|
### `extractWorkflowTrajectory`
|
|
302
302
|
|
|
@@ -314,7 +314,7 @@ const trajectory = extractWorkflowTrajectory(
|
|
|
314
314
|
// trajectory.steps — WorkflowStepStep[] in execution order
|
|
315
315
|
```
|
|
316
316
|
|
|
317
|
-
**Returns:** `Trajectory
|
|
317
|
+
**Returns:** `Trajectory`: Contains `steps: TrajectoryStep[]`, `totalDurationMs`, and `rawWorkflowResult`.
|
|
318
318
|
|
|
319
319
|
### `extractTrajectoryFromTrace`
|
|
320
320
|
|
|
@@ -363,7 +363,7 @@ Spans with types `GENERIC`, `MODEL_STEP`, `MODEL_CHUNK`, and `WORKFLOW_CONDITION
|
|
|
363
363
|
|
|
364
364
|
Compares an actual trajectory against an expected trajectory and returns a detailed comparison result. Used internally by `createTrajectoryAccuracyScorerCode`.
|
|
365
365
|
|
|
366
|
-
The `expected` parameter accepts either a `Trajectory` (actual trajectory) or `{ steps: ExpectedStep[] }`. When using `ExpectedStep[]`, you can match by name only
|
|
366
|
+
The `expected` parameter accepts either a `Trajectory` (actual trajectory) or `{ steps: ExpectedStep[] }`. When using `ExpectedStep[]`, you can match by name only or by name + stepType. You can also include data for comparison. See [Expected steps](https://mastra.ai/reference/evals/trajectory-accuracy) for details.
|
|
367
367
|
|
|
368
368
|
```typescript
|
|
369
369
|
import { compareTrajectories } from '@mastra/evals/scorers/utils'
|
|
@@ -62,10 +62,10 @@ Final score: `(similarity_ratio * confidence) * scale`
|
|
|
62
62
|
|
|
63
63
|
A textual difference score between 0 and 1:
|
|
64
64
|
|
|
65
|
-
- **1.0**:
|
|
66
|
-
- **0.7
|
|
67
|
-
- **0.4
|
|
68
|
-
- **0.1
|
|
65
|
+
- **1.0**: The texts are identical.
|
|
66
|
+
- **0.7 to 0.9**: Minor differences, few changes needed.
|
|
67
|
+
- **0.4 to 0.6**: Moderate differences, noticeable changes required.
|
|
68
|
+
- **0.1 to 0.3**: Major differences, extensive changes needed.
|
|
69
69
|
- **0.0**: Completely different texts.
|
|
70
70
|
|
|
71
71
|
## Example
|
|
@@ -275,7 +275,7 @@ console.log(result.score) // 1 - auth-tool comes before fetch-tool
|
|
|
275
275
|
|
|
276
276
|
## LLM-based tool call accuracy scorer
|
|
277
277
|
|
|
278
|
-
The `createToolCallAccuracyScorerLLM()` function from `@mastra/evals/scorers/prebuilt` uses an LLM to evaluate whether the tools called by an agent are appropriate for the
|
|
278
|
+
The `createToolCallAccuracyScorerLLM()` function from `@mastra/evals/scorers/prebuilt` uses an LLM to evaluate whether the tools called by an agent are appropriate for the user request, providing semantic evaluation rather than exact matching.
|
|
279
279
|
|
|
280
280
|
### Parameters
|
|
281
281
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Toxicity scorer
|
|
4
4
|
|
|
5
|
-
The `createToxicityScorer()` function evaluates whether an LLM's output contains racist, biased, or toxic elements. It uses a judge-based system to analyze responses for
|
|
5
|
+
The `createToxicityScorer()` function evaluates whether an LLM's output contains racist, biased, or toxic elements. It uses a judge-based system to analyze responses for forms of toxicity including personal attacks, mockery, hate speech, dismissive statements, and threats.
|
|
6
6
|
|
|
7
7
|
## Parameters
|
|
8
8
|
|
|
@@ -74,9 +74,9 @@ Final score: `(toxicity_weighted_sum / max_toxicity) * scale`
|
|
|
74
74
|
|
|
75
75
|
A toxicity score between 0 and 1:
|
|
76
76
|
|
|
77
|
-
- **0.8
|
|
78
|
-
- **0.4
|
|
79
|
-
- **0.1
|
|
77
|
+
- **0.8 to 1.0**: Severe toxicity.
|
|
78
|
+
- **0.4 to 0.7**: Moderate toxicity.
|
|
79
|
+
- **0.1 to 0.3**: Mild toxicity.
|
|
80
80
|
- **0.0**: No toxic elements detected.
|
|
81
81
|
|
|
82
82
|
## Example
|
|
@@ -15,7 +15,7 @@ The `runEvals` pipeline uses two extraction strategies, depending on whether obs
|
|
|
15
15
|
|
|
16
16
|
### Trace-based extraction (preferred)
|
|
17
17
|
|
|
18
|
-
When the target's `Mastra` instance has storage configured, the pipeline fetches the full execution trace from the observability store and calls `extractTrajectoryFromTrace()`. This produces a hierarchical trajectory with nested `children
|
|
18
|
+
When the target's `Mastra` instance has storage configured, the pipeline fetches the full execution trace from the observability store and calls `extractTrajectoryFromTrace()`. This produces a hierarchical trajectory with nested `children` that captures the complete execution tree. The tree includes nested agent runs and tool calls within workflow steps. It also includes model generations.
|
|
19
19
|
|
|
20
20
|
For example, a workflow that calls an agent, which in turn calls tools, produces:
|
|
21
21
|
|
|
@@ -35,8 +35,8 @@ workflow_run
|
|
|
35
35
|
|
|
36
36
|
When storage isn't available, the pipeline falls back to:
|
|
37
37
|
|
|
38
|
-
- **Agents:** `extractTrajectory()
|
|
39
|
-
- **Workflows:** `extractWorkflowTrajectory()
|
|
38
|
+
- **Agents:** `extractTrajectory()`, Extracts `ToolCallStep` entries from `toolInvocations` in the agent's message output. Produces a flat list of tool calls.
|
|
39
|
+
- **Workflows:** `extractWorkflowTrajectory()`, Extracts `WorkflowStepStep` entries from `stepResults`. Produces a flat list of workflow steps.
|
|
40
40
|
|
|
41
41
|
These fallbacks don't capture nested execution or non-tool-call spans.
|
|
42
42
|
|
|
@@ -105,7 +105,7 @@ All step types share the base properties `name`, `durationMs`, `metadata`, and `
|
|
|
105
105
|
|
|
106
106
|
## Expected steps
|
|
107
107
|
|
|
108
|
-
When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a discriminated union that mirrors `TrajectoryStep
|
|
108
|
+
When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a discriminated union that mirrors `TrajectoryStep`: when you specify a `stepType`, you get autocomplete for that variant's fields (e.g., `toolArgs` for `tool_call`, `modelId` for `model_generation`). All variant-specific fields are optional, so you only assert against what you care about.
|
|
109
109
|
|
|
110
110
|
Omit `stepType` entirely to match any step by name only.
|
|
111
111
|
|
|
@@ -199,8 +199,8 @@ This function returns an instance of the MastraScorer class. See the [MastraScor
|
|
|
199
199
|
|
|
200
200
|
The code-based scorer resolves `expectedTrajectory` from two sources, in order of priority:
|
|
201
201
|
|
|
202
|
-
1. **Constructor option
|
|
203
|
-
2. **Dataset item
|
|
202
|
+
1. **Constructor option**: A static trajectory passed when creating the scorer. Used for all dataset items.
|
|
203
|
+
2. **Dataset item**: An `expectedTrajectory` field on the dataset item, passed through the `runEvals` pipeline. Allows different expected trajectories per item.
|
|
204
204
|
|
|
205
205
|
```typescript
|
|
206
206
|
// Static: same expected trajectory for all items
|
|
@@ -474,10 +474,10 @@ The `createTrajectoryScorerCode()` function from `@mastra/evals/scorers/prebuilt
|
|
|
474
474
|
|
|
475
475
|
The unified scorer evaluates four dimensions:
|
|
476
476
|
|
|
477
|
-
1. **Accuracy
|
|
478
|
-
2. **Efficiency
|
|
479
|
-
3. **Blacklist
|
|
480
|
-
4. **Tool failures
|
|
477
|
+
1. **Accuracy**: Matches actual steps against expected steps (if `steps` is configured). Uses the `ordering` mode.
|
|
478
|
+
2. **Efficiency**: Checks step budgets (`maxSteps`, `maxTotalTokens`, `maxTotalDurationMs`) and redundant calls (`noRedundantCalls`).
|
|
479
|
+
3. **Blacklist**: Checks for forbidden tools or sequences. Any violation immediately results in a score of **0.0** regardless of other dimensions.
|
|
480
|
+
4. **Tool failures**: Detects retry and fallback patterns. It also detects argument correction patterns.
|
|
481
481
|
|
|
482
482
|
The final score is a weighted combination of active dimensions, normalized by which dimensions are active. Default weights are accuracy 0.4, efficiency 0.3, tool failures 0.2, blacklist 0.1, but you can customize them via the `weights` option. Blacklist violations override everything to 0. When nested evaluations are present, the score is 70% top-level and 30% nested average.
|
|
483
483
|
|
|
@@ -27,7 +27,7 @@ Move conditional, large, or action-oriented guidance into [`tools/`](https://mas
|
|
|
27
27
|
|
|
28
28
|
## Dynamic instructions
|
|
29
29
|
|
|
30
|
-
When the prompt needs to change per request, for example based on the current user or runtime context, set a
|
|
30
|
+
When the prompt needs to change per request, for example based on the current user or runtime context, set a runtime-defined `instructions` function in [`config.ts`](https://mastra.ai/reference/file-based-agents/config) instead of using `instructions.md`. A function `instructions` wins over `instructions.md`, so the static file is ignored when both are present.
|
|
31
31
|
|
|
32
32
|
```typescript
|
|
33
33
|
import { agentConfig } from '@mastra/core/agent'
|
|
@@ -49,6 +49,6 @@ Mastra reads `instructions.md` and inlines its contents into the generated code
|
|
|
49
49
|
|
|
50
50
|
Instructions can come from `instructions.md` or from the `instructions` field in [`config.ts`](https://mastra.ai/reference/file-based-agents/config):
|
|
51
51
|
|
|
52
|
-
- A
|
|
52
|
+
- A runtime-defined (function) `instructions` in `config.ts` wins over `instructions.md`.
|
|
53
53
|
- Otherwise `instructions.md` wins over a static `instructions` string.
|
|
54
54
|
- If neither is present, the build fails and names the agent directory.
|
|
@@ -23,4 +23,4 @@ Mastra registers the logger before storage, observability, and file-based agents
|
|
|
23
23
|
|
|
24
24
|
## Precedence with code
|
|
25
25
|
|
|
26
|
-
Code-registered logging wins over `logger.ts`. If you pass `logger` to `new Mastra({ logger })` in `src/mastra/index.ts` (or set `logger: false` to disable logging), `logger.ts` is ignored with a warning. Use `logger.ts` when one project-wide logger is enough
|
|
26
|
+
Code-registered logging wins over `logger.ts`. If you pass `logger` to `new Mastra({ logger })` in `src/mastra/index.ts` (or set `logger: false` to disable logging), `logger.ts` is ignored with a warning. Use `logger.ts` when one project-wide logger is enough. Use code registration when setup depends on runtime wiring.
|
|
@@ -29,4 +29,4 @@ export default new Observability({
|
|
|
29
29
|
|
|
30
30
|
## Precedence with code
|
|
31
31
|
|
|
32
|
-
Code-registered observability wins over `observability.ts`. Use `observability.ts` when one project-wide configuration is enough
|
|
32
|
+
Code-registered observability wins over `observability.ts`. Use `observability.ts` when one project-wide configuration is enough. Use code registration when setup depends on runtime wiring in `src/mastra/index.ts`.
|
|
@@ -45,7 +45,7 @@ If `config.inputProcessors` or `config.outputProcessors` is a function, discover
|
|
|
45
45
|
|
|
46
46
|
## Precedence with config
|
|
47
47
|
|
|
48
|
-
Use `config.ts` for processors that need to be created
|
|
48
|
+
Use `config.ts` for processors that need to be created at runtime or shared across agents. Use file-based processors for static, per-agent processors that should be reviewed alongside the agent.
|
|
49
49
|
|
|
50
50
|
Precedence is:
|
|
51
51
|
|
|
@@ -40,7 +40,7 @@ For packaged skills, `description` is required in `SKILL.md` frontmatter. Flat m
|
|
|
40
40
|
|
|
41
41
|
## What gets loaded
|
|
42
42
|
|
|
43
|
-
File-based
|
|
43
|
+
File-based Agent skills are added to the generated bundle at build time, so the deployed agent doesn't read them from disk.
|
|
44
44
|
|
|
45
45
|
For packaged skills, Mastra reads:
|
|
46
46
|
|
|
@@ -27,4 +27,4 @@ Mastra registers the store before file-based agents and workflows, so storage-de
|
|
|
27
27
|
|
|
28
28
|
## Precedence with code
|
|
29
29
|
|
|
30
|
-
Code-registered storage wins over `storage.ts`. Use `storage.ts` when one project-wide store is enough
|
|
30
|
+
Code-registered storage wins over `storage.ts`. Use `storage.ts` when one project-wide store is enough. Use code registration when setup depends on runtime wiring in `src/mastra/index.ts`.
|
|
@@ -53,4 +53,4 @@ The auth provider handles who the user is. For what the user can do after sign-i
|
|
|
53
53
|
|
|
54
54
|
## Precedence with code
|
|
55
55
|
|
|
56
|
-
Code-registered Studio config wins over `studio.ts`. Use `studio.ts` when one project-wide Studio config is enough
|
|
56
|
+
Code-registered Studio config wins over `studio.ts`. Use `studio.ts` when one project-wide Studio config is enough. Use code registration when auth depends on runtime wiring in `src/mastra/index.ts`.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Subagents
|
|
4
4
|
|
|
5
|
-
A file-based agent can declare **subagents**, specialist child agents it delegates to. The parent model sees each subagent as a delegation tool named after the subagent directory
|
|
5
|
+
A file-based agent can declare **subagents**, specialist child agents it delegates to. The parent model sees each subagent as a delegation tool named after the subagent directory and calls that tool to hand off a task. The subagent's result returns to the parent conversation.
|
|
6
6
|
|
|
7
7
|
Use this page for the file-based convention. For broader delegation patterns, hooks, memory isolation, tool approval propagation, and scoring, see [Supervisor agents](https://mastra.ai/docs/agents/supervisor-agents).
|
|
8
8
|
|
|
@@ -37,7 +37,7 @@ Use one file per tool. Name the file after the action the model should take:
|
|
|
37
37
|
|
|
38
38
|
- Use `get_weather.ts`, `search_docs.ts`, or `create_ticket.ts`.
|
|
39
39
|
- Avoid vague names like `helper.ts`, `api.ts`, or `utils.ts`.
|
|
40
|
-
- Co-locate tests next to the tool as `*.test.ts` or `*.spec.ts
|
|
40
|
+
- Co-locate tests next to the tool as `*.test.ts` or `*.spec.ts`. Discovery ignores those files.
|
|
41
41
|
- Keep shared helper code outside `tools/` or in files that don't match the discovered extensions under `tools/`.
|
|
42
42
|
|
|
43
43
|
Because nested directories aren't discovered as tools, group related tools with clear filename prefixes instead of subfolders, such as `calendar_create_event.ts` and `calendar_list_events.ts`.
|
|
@@ -56,7 +56,7 @@ The workspace filesystem controls what file tools can read and write. The sandbo
|
|
|
56
56
|
|
|
57
57
|
## Seed files
|
|
58
58
|
|
|
59
|
-
Add a `workspace/` directory to
|
|
59
|
+
Add a `workspace/` directory to include starting files with the agent. Mastra mirrors files under `workspace/` into the agent's default runtime workspace at build time, so the agent starts with those files on disk.
|
|
60
60
|
|
|
61
61
|
```text
|
|
62
62
|
src/mastra/agents/weather/
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# PinoLogger
|
|
4
4
|
|
|
5
|
-
A Logger instance is created using `new PinoLogger()` and provides methods to record events at
|
|
5
|
+
A Logger instance is created using `new PinoLogger()` and provides methods to record events at severity levels.
|
|
6
6
|
|
|
7
7
|
## Usage example
|
|
8
8
|
|
|
@@ -86,7 +86,7 @@ const logger = new MastraPinoWithAudit({
|
|
|
86
86
|
export const mastra = new Mastra({ logger })
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
-
Numeric values follow Pino’s ordering (built-in levels use 10
|
|
89
|
+
Numeric values follow Pino’s ordering (built-in levels use 10 to 60). A level of `35` sits between `info` (30) and `warn` (40), so with `level: 'info'` both `info` and `audit` lines are emitted.
|
|
90
90
|
|
|
91
91
|
## File transport (structured logs)
|
|
92
92
|
|
|
@@ -114,7 +114,7 @@ fileLogger.warn('Low disk space', {
|
|
|
114
114
|
|
|
115
115
|
## Upstash transport (remote log drain)
|
|
116
116
|
|
|
117
|
-
Streams structured logs to a remote Redis list using the `UpstashTransport`. The logger accepts a string message and a structured metadata object.
|
|
117
|
+
Streams structured logs to a remote Redis list using the `UpstashTransport`. The logger accepts a string message and a structured metadata object. It supports centralized logging for distributed environments, supporting filtering by `destinationPath`, `type`, and `runId`.
|
|
118
118
|
|
|
119
119
|
```typescript
|
|
120
120
|
import { UpstashTransport } from '@mastra/loggers/upstash'
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Cloned thread utilities
|
|
4
4
|
|
|
5
|
-
The Memory class provides utility methods for working with cloned threads. These methods help you check clone status, retrieve clone metadata,
|
|
5
|
+
The Memory class provides utility methods for working with cloned threads. These methods help you check clone status, retrieve clone metadata, traverse clone relationships, and track clone history.
|
|
6
6
|
|
|
7
7
|
## `isClone()`
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Memory.cloneThread()
|
|
4
4
|
|
|
5
|
-
The `.cloneThread()` method creates a copy of an existing conversation thread, including all its messages.
|
|
5
|
+
The `.cloneThread()` method creates a copy of an existing conversation thread, including all its messages. It supports creating divergent conversation paths from a specific point in a conversation. When semantic recall is enabled, the method also creates vector embeddings for the cloned messages.
|
|
6
6
|
|
|
7
7
|
## Usage example
|
|
8
8
|
|
|
@@ -125,7 +125,7 @@ Pass the cloned `thread.id` and `thread.resourceId` to `agent.generate()` to con
|
|
|
125
125
|
|
|
126
126
|
## Vector embeddings
|
|
127
127
|
|
|
128
|
-
When the Memory instance has semantic recall enabled with a vector store and embedder configured, `cloneThread()` automatically creates vector embeddings for all cloned messages.
|
|
128
|
+
When the Memory instance has semantic recall enabled with a vector store and embedder configured, `cloneThread()` automatically creates vector embeddings for all cloned messages. It ensures that semantic search works correctly on the cloned thread.
|
|
129
129
|
|
|
130
130
|
In this example, `embeddingModel` is the embedding model configured for the project.
|
|
131
131
|
|
|
@@ -170,4 +170,4 @@ When [Observational Memory](https://mastra.ai/docs/memory/observational-memory)
|
|
|
170
170
|
- **Resource-scoped OM (same `resourceId`)**: The OM record is shared between the source and cloned threads since they belong to the same resource. No duplication occurs.
|
|
171
171
|
- **Resource-scoped OM (different `resourceId`)**: The OM record is cloned to the new resource. Message IDs are remapped and any thread-identifying tags within observations are updated to reference the cloned thread.
|
|
172
172
|
|
|
173
|
-
Only the current (most recent) OM generation is cloned
|
|
173
|
+
Only the current (most recent) OM generation is cloned. Older history generations aren't copied. Transient processing state (observation/reflection in-progress flags) is reset on the cloned record.
|
|
@@ -19,7 +19,7 @@ const result = await memory.listThreads({
|
|
|
19
19
|
|
|
20
20
|
Use `perPage: false` to retrieve all matching threads at once.
|
|
21
21
|
|
|
22
|
-
> **Warning:**
|
|
22
|
+
> **Warning:** Use pagination, especially for large datasets. Use this option cautiously.
|
|
23
23
|
|
|
24
24
|
```typescript
|
|
25
25
|
const result = await memory.listThreads({
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Memory class
|
|
4
4
|
|
|
5
|
-
The `Memory` class provides a
|
|
5
|
+
The `Memory` class provides a reliable system for managing conversation history and thread-based message storage in Mastra. It enables persistent storage of conversations, semantic search capabilities, and efficient message retrieval. You must configure a storage provider for conversation history, and if you enable semantic recall you will also need to provide a vector store and embedder.
|
|
6
6
|
|
|
7
7
|
## Usage example
|
|
8
8
|
|