@mastra/mcp-docs-server 1.2.12-alpha.0 → 1.2.12-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/docs/agent-builder/access-control.md +3 -3
- package/.docs/docs/agent-builder/browser.md +1 -1
- package/.docs/docs/agent-builder/channels.md +1 -1
- package/.docs/docs/agent-builder/configuration.md +3 -3
- package/.docs/docs/agent-builder/deploying.md +2 -2
- package/.docs/docs/agent-builder/integrations.md +8 -8
- package/.docs/docs/agent-builder/memory.md +2 -2
- package/.docs/docs/agent-builder/overview.md +1 -1
- package/.docs/docs/agent-controller/channels.md +8 -6
- package/.docs/docs/agent-controller/modes.md +6 -6
- package/.docs/docs/agent-controller/overview.md +13 -9
- package/.docs/docs/agent-controller/session.md +27 -9
- package/.docs/docs/agent-controller/subagents.md +9 -6
- package/.docs/docs/agent-controller/threads-and-state.md +12 -8
- package/.docs/docs/agent-controller/tool-approvals.md +7 -3
- package/.docs/docs/agents/a2a.md +4 -2
- package/.docs/docs/agents/acp.md +5 -5
- package/.docs/docs/agents/agent-approval.md +13 -11
- package/.docs/docs/agents/code-mode.md +29 -6
- package/.docs/docs/agents/guardrails.md +8 -8
- package/.docs/docs/agents/networks.md +2 -2
- package/.docs/docs/agents/overview.md +9 -7
- package/.docs/docs/agents/processors.md +15 -15
- package/.docs/docs/agents/skills.md +6 -6
- package/.docs/docs/agents/structured-output.md +6 -6
- package/.docs/docs/agents/supervisor-agents.md +19 -13
- package/.docs/docs/agents/using-tools.md +25 -23
- package/.docs/docs/browser/agent-browser.md +4 -4
- package/.docs/docs/browser/browser-viewer.md +2 -2
- package/.docs/docs/browser/firecrawl.md +1 -1
- package/.docs/docs/browser/overview.md +2 -2
- package/.docs/docs/browser/recording.md +1 -1
- package/.docs/docs/browser/stagehand.md +7 -7
- package/.docs/docs/capabilities/channels/discord.md +1 -1
- package/.docs/docs/capabilities/channels/other-adapters.md +1 -1
- package/.docs/docs/capabilities/channels/overview.md +9 -5
- package/.docs/docs/capabilities/channels/slack.md +2 -2
- package/.docs/docs/capabilities/channels/teams.md +1 -1
- package/.docs/docs/capabilities/channels/telegram.md +1 -1
- package/.docs/docs/capabilities/channels/whatsapp.md +1 -1
- package/.docs/docs/deployment/mastra-server.md +1 -1
- package/.docs/docs/deployment/overview.md +6 -2
- package/.docs/docs/deployment/sandbox.md +3 -1
- package/.docs/docs/deployment/workers.md +137 -0
- package/.docs/docs/editor/overview.md +37 -31
- package/.docs/docs/editor/prompts.md +8 -4
- package/.docs/docs/editor/tools.md +3 -3
- package/.docs/docs/evals/built-in-scorers.md +4 -4
- package/.docs/docs/evals/custom-scorers.md +5 -5
- package/.docs/docs/evals/datasets/running-experiments.md +7 -7
- package/.docs/docs/evals/evals-with-memory.md +4 -4
- package/.docs/docs/evals/gates-and-verdicts.md +4 -4
- package/.docs/docs/evals/multi-turn.md +12 -12
- package/.docs/docs/evals/overview.md +6 -4
- package/.docs/docs/evals/quick-checks.md +2 -2
- package/.docs/docs/getting-started/build-with-ai.md +6 -4
- package/.docs/docs/getting-started/file-based-agents.md +6 -4
- package/.docs/docs/getting-started/manual-install.md +1 -1
- package/.docs/docs/index.md +9 -90
- package/.docs/docs/long-running-agents/background-tasks.md +10 -10
- package/.docs/docs/long-running-agents/durable-agents.md +46 -5
- package/.docs/docs/long-running-agents/goals.md +15 -11
- package/.docs/docs/long-running-agents/schedules.md +3 -3
- package/.docs/docs/long-running-agents/signal-providers.md +8 -6
- package/.docs/docs/long-running-agents/signals.md +9 -5
- package/.docs/docs/mastra-platform/configuration.md +3 -3
- package/.docs/docs/mastra-platform/database.md +8 -8
- package/.docs/docs/mastra-platform/deploy.md +8 -4
- package/.docs/docs/mastra-platform/environments.md +8 -4
- package/.docs/docs/mastra-platform/github.md +3 -1
- package/.docs/docs/mastra-platform/server.md +1 -1
- package/.docs/docs/mastra-platform/studio.md +2 -2
- package/.docs/docs/mastra-platform/workspace.md +6 -6
- package/.docs/docs/mcp/mcp-apps.md +1 -1
- package/.docs/docs/mcp/overview.md +9 -9
- package/.docs/docs/memory/memory-processors.md +10 -10
- package/.docs/docs/memory/message-history.md +11 -5
- package/.docs/docs/memory/multi-user-threads.md +6 -6
- package/.docs/docs/memory/observational-memory.md +27 -25
- package/.docs/docs/memory/overview.md +10 -10
- package/.docs/docs/memory/semantic-recall.md +3 -3
- package/.docs/docs/memory/working-memory.md +8 -8
- package/.docs/docs/observability/integrations/bridges/datadog.md +6 -6
- package/.docs/docs/observability/integrations/bridges/otel.md +5 -5
- package/.docs/docs/observability/integrations/exporters/datadog.md +1 -1
- package/.docs/docs/observability/integrations/exporters/laminar.md +1 -1
- package/.docs/docs/observability/integrations/exporters/langfuse.md +3 -3
- package/.docs/docs/observability/integrations/exporters/langsmith.md +4 -4
- package/.docs/docs/observability/integrations/exporters/mastra-platform.md +1 -1
- package/.docs/docs/observability/integrations/exporters/mastra-storage.md +1 -1
- package/.docs/docs/observability/integrations/exporters/otel.md +26 -4
- package/.docs/docs/observability/integrations/exporters/sentry.md +1 -1
- package/.docs/docs/observability/integrations/overview.md +1 -1
- package/.docs/docs/observability/integrations/processors/sensitive-data-filter.md +1 -1
- package/.docs/docs/observability/logging.md +1 -1
- package/.docs/docs/observability/metrics/overview.md +1 -1
- package/.docs/docs/observability/metrics/querying.md +8 -8
- package/.docs/docs/observability/overview.md +4 -4
- package/.docs/docs/observability/tracing/overview.md +18 -12
- package/.docs/docs/rag/chunking-and-embedding.md +1 -1
- package/.docs/docs/rag/graph-rag.md +7 -7
- package/.docs/docs/rag/overview.md +2 -2
- package/.docs/docs/rag/retrieval.md +18 -18
- package/.docs/docs/rag/vector-databases.md +2 -2
- package/.docs/docs/server/auth/composite-auth.md +2 -2
- package/.docs/docs/server/auth/fga.md +6 -6
- package/.docs/docs/server/auth/firebase.md +2 -2
- package/.docs/docs/server/auth/simple-auth.md +1 -1
- package/.docs/docs/server/auth/workers.md +133 -0
- package/.docs/docs/server/custom-adapters.md +4 -4
- package/.docs/docs/server/mastra-client.md +3 -3
- package/.docs/docs/server/mastra-server.md +2 -2
- package/.docs/docs/server/middleware.md +1 -1
- package/.docs/docs/server/pubsub.md +7 -4
- package/.docs/docs/server/request-context.md +11 -11
- package/.docs/docs/server/server-adapters.md +2 -2
- package/.docs/docs/storage/overview.md +2 -2
- package/.docs/docs/studio/auth.md +5 -5
- package/.docs/docs/studio/deployment.md +1 -1
- package/.docs/docs/studio/observability.md +2 -2
- package/.docs/docs/studio/overview.md +11 -9
- package/.docs/docs/voice/overview.md +26 -26
- package/.docs/docs/voice/realtime-voice.md +8 -8
- package/.docs/docs/voice/speech-to-speech.md +9 -9
- package/.docs/docs/voice/speech-to-text.md +3 -3
- package/.docs/docs/voice/text-to-speech.md +6 -6
- package/.docs/docs/what-is-mastra.md +161 -0
- package/.docs/docs/workflows/agents-and-tools.md +3 -3
- package/.docs/docs/workflows/control-flow.md +10 -8
- package/.docs/docs/workflows/error-handling.md +2 -2
- package/.docs/docs/workflows/overview.md +7 -6
- package/.docs/docs/workflows/scheduled-workflows.md +12 -12
- package/.docs/docs/workflows/snapshots.md +3 -3
- package/.docs/docs/workflows/suspend-and-resume.md +1 -1
- package/.docs/docs/workflows/time-travel.md +3 -1
- package/.docs/docs/workflows/workflow-state.md +1 -1
- package/.docs/docs/workspace/filesystem.md +10 -10
- package/.docs/docs/workspace/lsp.md +1 -1
- package/.docs/docs/workspace/overview.md +12 -12
- package/.docs/docs/workspace/sandbox.md +9 -7
- package/.docs/docs/workspace/search.md +3 -3
- package/.docs/docs/workspace/skills.md +7 -7
- package/.docs/guides/build-your-ui/ai-sdk-ui.md +1 -1
- package/.docs/guides/build-your-ui/copilotkit/generative-ui.md +1 -1
- package/.docs/guides/build-your-ui/openui.md +1 -1
- package/.docs/guides/concepts/streaming.md +1 -1
- package/.docs/guides/deployment/kubernetes.md +1 -1
- package/.docs/guides/deployment/mastra-workers.md +232 -0
- package/.docs/guides/guide/ai-recruiter.md +1 -1
- package/.docs/guides/guide/chef-michel.md +1 -1
- package/.docs/guides/guide/code-review-bot.md +1 -1
- package/.docs/guides/guide/coding-agent.md +1 -1
- package/.docs/guides/guide/dev-assistant.md +1 -1
- package/.docs/guides/guide/docs-manager.md +1 -1
- package/.docs/guides/guide/firecrawl.md +1 -1
- package/.docs/guides/guide/github-actions-pr-description.md +1 -1
- package/.docs/guides/guide/research-assistant.md +1 -1
- package/.docs/guides/guide/research-coordinator.md +2 -2
- package/.docs/guides/guide/signal-provider.md +1 -1
- package/.docs/guides/guide/stock-agent.md +2 -2
- package/.docs/guides/guide/web-search.md +4 -4
- package/.docs/guides/guide/whatsapp-chat-bot.md +2 -2
- package/.docs/guides/migrations/agentnetwork.md +4 -4
- package/.docs/guides/migrations/upgrade-to-v1/agent.md +1 -1
- package/.docs/guides/migrations/vnext-to-standard-apis.md +2 -2
- package/.docs/models/gateways/custom-gateways.md +3 -3
- package/.docs/models/gateways/openrouter.md +1 -3
- package/.docs/models/gateways/vercel.md +2 -1
- package/.docs/models/index.md +6 -6
- package/.docs/models/providers/baseten.md +1 -1
- package/.docs/models/providers/chutes.md +2 -1
- package/.docs/models/providers/crossmodel.md +2 -2
- package/.docs/models/providers/deepinfra.md +1 -1
- package/.docs/models/providers/hyper.md +3 -3
- package/.docs/models/providers/llmgateway.md +4 -3
- package/.docs/models/providers/wandb.md +2 -1
- package/.docs/reference/acp/acp-agent.md +1 -1
- package/.docs/reference/acp/create-acp-tool.md +1 -1
- package/.docs/reference/agents/agent.md +7 -7
- package/.docs/reference/agents/channels.md +26 -10
- package/.docs/reference/agents/durable-agent.md +51 -2
- package/.docs/reference/agents/generateLegacy.md +1 -1
- package/.docs/reference/agents/getLLM.md +2 -2
- package/.docs/reference/agents/getMetadata.md +2 -2
- package/.docs/reference/agents/inngest-agent.md +1 -1
- package/.docs/reference/agents/network.md +1 -1
- package/.docs/reference/browser/agent-browser.md +1 -1
- package/.docs/reference/browser/browser-viewer.md +1 -1
- package/.docs/reference/browser/firecrawl-browser.md +1 -1
- package/.docs/reference/browser/mastra-browser.md +1 -1
- package/.docs/reference/browser/stagehand-browser.md +4 -4
- package/.docs/reference/cli/mastra.md +132 -1
- package/.docs/reference/client-js/agents.md +17 -0
- package/.docs/reference/configuration.md +1 -1
- package/.docs/reference/core/getAgentById.md +1 -1
- package/.docs/reference/core/mastra-class.md +48 -1
- package/.docs/reference/core/mastra-model-gateway.md +1 -1
- package/.docs/reference/evals/answer-relevancy.md +1 -1
- package/.docs/reference/evals/answer-similarity.md +1 -1
- package/.docs/reference/evals/bias.md +1 -1
- package/.docs/reference/evals/context-precision.md +3 -3
- package/.docs/reference/evals/context-recall.md +3 -3
- package/.docs/reference/evals/context-relevance.md +11 -11
- package/.docs/reference/evals/faithfulness.md +1 -1
- package/.docs/reference/evals/hallucination.md +5 -5
- package/.docs/reference/evals/noise-sensitivity.md +10 -10
- package/.docs/reference/evals/prompt-alignment.md +15 -15
- package/.docs/reference/evals/rubric.md +1 -1
- package/.docs/reference/evals/tool-call-accuracy.md +3 -3
- package/.docs/reference/evals/toxicity.md +1 -1
- package/.docs/reference/file-based-agents/config.md +3 -3
- package/.docs/reference/file-based-agents/instructions.md +1 -1
- package/.docs/reference/file-based-agents/subagents.md +3 -3
- package/.docs/reference/index.md +2 -0
- package/.docs/reference/memory/memory-class.md +3 -3
- package/.docs/reference/memory/observational-memory.md +4 -4
- package/.docs/reference/observability/tracing/exporters/langfuse.md +1 -1
- package/.docs/reference/processors/batch-parts-processor.md +1 -1
- package/.docs/reference/processors/language-detector.md +1 -1
- package/.docs/reference/processors/message-history-processor.md +1 -1
- package/.docs/reference/processors/moderation-processor.md +2 -2
- package/.docs/reference/processors/pii-detector.md +2 -2
- package/.docs/reference/processors/processor-interface.md +1 -1
- package/.docs/reference/processors/prompt-injection-detector.md +1 -1
- package/.docs/reference/processors/semantic-recall-processor.md +1 -1
- package/.docs/reference/processors/skill-search-processor.md +1 -1
- package/.docs/reference/processors/system-prompt-scrubber.md +1 -1
- package/.docs/reference/processors/token-limiter-processor.md +3 -3
- package/.docs/reference/processors/tool-call-filter.md +2 -2
- package/.docs/reference/processors/tool-search-processor.md +2 -2
- package/.docs/reference/processors/unicode-normalizer.md +1 -1
- package/.docs/reference/processors/working-memory-processor.md +1 -1
- package/.docs/reference/rag/rerank.md +1 -1
- package/.docs/reference/server/routes.md +7 -7
- package/.docs/reference/signals/create-notification-inbox-tool.md +1 -1
- package/.docs/reference/storage/dsql.md +1 -1
- package/.docs/reference/storage/mongodb.md +1 -1
- package/.docs/reference/storage/postgresql.md +1 -1
- package/.docs/reference/storage/redis.md +1 -1
- package/.docs/reference/storage/upstash.md +1 -1
- package/.docs/reference/streaming/agents/stream.md +1 -1
- package/.docs/reference/tools/create-code-mode.md +3 -3
- package/.docs/reference/tools/isolated-vm-transport.md +74 -0
- package/.docs/reference/tools/mcp-client.md +2 -2
- package/.docs/reference/tools/mcp-server.md +1 -1
- package/.docs/reference/tools/vector-query-tool.md +1 -1
- package/.docs/reference/vectors/libsql.md +1 -1
- package/.docs/reference/vectors/mongodb.md +2 -2
- package/.docs/reference/vectors/pg.md +1 -1
- package/.docs/reference/vectors/upstash.md +1 -1
- package/.docs/reference/voice/voice.addInstructions.md +1 -1
- package/.docs/reference/workers/overview.md +85 -0
- package/.docs/reference/workspace/agentfs-filesystem.md +1 -1
- package/.docs/reference/workspace/local-filesystem.md +1 -1
- package/.docs/reference/workspace/local-sandbox.md +1 -1
- package/CHANGELOG.md +14 -0
- package/package.json +5 -5
|
@@ -169,6 +169,55 @@ The path to your built Mastra output directory. Defaults to `.mastra/output`.
|
|
|
169
169
|
|
|
170
170
|
Comma-separated list of custom arguments to pass to the Node.js process, e.g. `--require=newrelic` or `--experimental-transform-types`.
|
|
171
171
|
|
|
172
|
+
## `mastra worker build`
|
|
173
|
+
|
|
174
|
+
Bundles your Mastra application for worker deployment. Produces the same output as `mastra build` — a self-contained `.mastra/output/` directory.
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
mastra worker build [options]
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Flags
|
|
181
|
+
|
|
182
|
+
| Flag | Description |
|
|
183
|
+
| --------------------- | ------------------------------------------------------------ |
|
|
184
|
+
| `--dir <path>` | Path to your Mastra source directory (default: `src/mastra`) |
|
|
185
|
+
| `--root <path>` | Project root directory (default: current directory) |
|
|
186
|
+
| `--tools <paths>` | Comma-separated tool paths to include |
|
|
187
|
+
| `--output-dir <path>` | Custom output directory (default: `.mastra/output`) |
|
|
188
|
+
| `--debug` | Enable debug logging |
|
|
189
|
+
|
|
190
|
+
## `mastra worker start`
|
|
191
|
+
|
|
192
|
+
> **Info:** You need to run `mastra worker build` or `mastra build` before using `mastra worker start`.
|
|
193
|
+
|
|
194
|
+
Starts a worker process from a previously built bundle.
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
mastra worker start [name] [options]
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
| Argument/Flag | Description |
|
|
201
|
+
| -------------- | ------------------------------------------------------------------- |
|
|
202
|
+
| `name` | Worker name to start. Sets `MASTRA_WORKERS` in the spawned process. |
|
|
203
|
+
| `--dir <path>` | Path to the build output (default: `.mastra/output`) |
|
|
204
|
+
| `--env <path>` | Path to environment file (default: `.env.production`, then `.env`) |
|
|
205
|
+
|
|
206
|
+
### Examples
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
# Start only the orchestration worker
|
|
210
|
+
mastra worker start orchestration
|
|
211
|
+
|
|
212
|
+
# Start only the scheduler
|
|
213
|
+
mastra worker start scheduler
|
|
214
|
+
|
|
215
|
+
# Start from a custom build directory
|
|
216
|
+
mastra worker start orchestration --dir ./dist
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
See [Workers](https://mastra.ai/docs/deployment/workers) for deployment topologies and configuration.
|
|
220
|
+
|
|
172
221
|
## `mastra studio`
|
|
173
222
|
|
|
174
223
|
Starts [Studio](https://mastra.ai/docs/studio/overview) as a static server. After starting, you can enter your Mastra instance URL (e.g. `http://localhost:4111`) to connect Studio to your Mastra backend. Looks for `.env` and `.env.production` files in the current working directory for configuration.
|
|
@@ -968,13 +1017,15 @@ For runtime commands, the command resolves the target server in this order:
|
|
|
968
1017
|
|
|
969
1018
|
Automatic platform auth is only used when the CLI resolves a Mastra platform target from `.mastra-project.json`. Localhost targets and explicit `--url` targets don't receive automatic credentials. Headers passed with `--header` are sent to any target, including localhost.
|
|
970
1019
|
|
|
971
|
-
For observability commands (`trace`, `log`, `score`, and `metric`), the CLI targets `https://observability.mastra.ai` by default instead of a project deployment URL.
|
|
1020
|
+
For observability commands (`trace`, `log`, `score`, and `metric`), the CLI targets `https://observability.mastra.ai` by default instead of a project deployment URL. Trace Intelligence commands (`learning`) work the same way but target `https://output.signals.mastra.ai`. Both resolve credentials in this order:
|
|
972
1021
|
|
|
973
1022
|
1. Explicit `Authorization` and `X-Mastra-Project-Id` headers passed with `--header`.
|
|
974
1023
|
2. `MASTRA_PLATFORM_ACCESS_TOKEN` and `MASTRA_PROJECT_ID` from your environment.
|
|
975
1024
|
3. Project metadata from `.mastra-project.json` for the project ID.
|
|
976
1025
|
4. Your Mastra CLI login token as an auth fallback.
|
|
977
1026
|
|
|
1027
|
+
Learning commands also send `X-Mastra-Organization-Id`, resolved from an explicit `--header`, `MASTRA_ORGANIZATION_ID` in your environment, or `.mastra-project.json`, in that order.
|
|
1028
|
+
|
|
978
1029
|
Use `--url` and `--header` when you need to override the default hosted observability target or credentials.
|
|
979
1030
|
|
|
980
1031
|
### Flags
|
|
@@ -1503,6 +1554,86 @@ Lists results for an experiment. Pass optional JSON input for route-supported fi
|
|
|
1503
1554
|
mastra api experiment results <datasetId> <experimentId> [input]
|
|
1504
1555
|
```
|
|
1505
1556
|
|
|
1557
|
+
#### `mastra api learning entities`
|
|
1558
|
+
|
|
1559
|
+
Lists entities (agents) with Trace Intelligence output, including which trace signals are available per entity. Requires enrollment in the Trace Intelligence private beta.
|
|
1560
|
+
|
|
1561
|
+
```bash
|
|
1562
|
+
mastra api learning entities '{"entityType":"agent"}'
|
|
1563
|
+
```
|
|
1564
|
+
|
|
1565
|
+
#### `mastra api learning snapshots`
|
|
1566
|
+
|
|
1567
|
+
Lists analysis snapshots for an entity and an ordered, comma-separated list of trace signals. Later commands need a `snapshotId` from this list.
|
|
1568
|
+
|
|
1569
|
+
```bash
|
|
1570
|
+
mastra api learning snapshots <entityId> '{"entityType":"agent","signalNames":"goal,outcome,behavior,sentiment","limit":10}'
|
|
1571
|
+
```
|
|
1572
|
+
|
|
1573
|
+
#### `mastra api learning flow`
|
|
1574
|
+
|
|
1575
|
+
Gets the cross-signal theme flow for one snapshot: stages and links for a Sankey-style view where counts are distinct traces.
|
|
1576
|
+
|
|
1577
|
+
```bash
|
|
1578
|
+
mastra api learning flow <entityId> '{"entityType":"agent","signalNames":"goal,outcome","snapshotId":"<snapshotId>"}'
|
|
1579
|
+
```
|
|
1580
|
+
|
|
1581
|
+
#### `mastra api learning paths`
|
|
1582
|
+
|
|
1583
|
+
Gets per-trace theme assignments across the ordered trace signals in one snapshot. Paginate with `limit` and `offset`.
|
|
1584
|
+
|
|
1585
|
+
```bash
|
|
1586
|
+
mastra api learning paths <entityId> '{"entityType":"agent","signalNames":"goal,outcome","snapshotId":"<snapshotId>","limit":100}'
|
|
1587
|
+
```
|
|
1588
|
+
|
|
1589
|
+
#### `mastra api learning theme list`
|
|
1590
|
+
|
|
1591
|
+
Lists themes for one trace signal in one snapshot.
|
|
1592
|
+
|
|
1593
|
+
```bash
|
|
1594
|
+
mastra api learning theme list <entityId> '{"entityType":"agent","signalName":"goal","snapshotId":"<snapshotId>"}'
|
|
1595
|
+
```
|
|
1596
|
+
|
|
1597
|
+
#### `mastra api learning theme get`
|
|
1598
|
+
|
|
1599
|
+
Gets one theme in one snapshot by its numeric theme ID.
|
|
1600
|
+
|
|
1601
|
+
```bash
|
|
1602
|
+
mastra api learning theme get <entityId> <themeId> '{"entityType":"agent","signalName":"goal","snapshotId":"<snapshotId>"}'
|
|
1603
|
+
```
|
|
1604
|
+
|
|
1605
|
+
#### `mastra api learning theme examples`
|
|
1606
|
+
|
|
1607
|
+
Lists trace examples for one theme in one snapshot. Paginate with `limit` and `offset`.
|
|
1608
|
+
|
|
1609
|
+
```bash
|
|
1610
|
+
mastra api learning theme examples <entityId> <themeId> '{"entityType":"agent","signalName":"goal","snapshotId":"<snapshotId>","limit":10}'
|
|
1611
|
+
```
|
|
1612
|
+
|
|
1613
|
+
#### `mastra api learning theme history`
|
|
1614
|
+
|
|
1615
|
+
Gets the lifecycle history for one durable theme across snapshots, including split and merge relationships. Takes no `snapshotId`.
|
|
1616
|
+
|
|
1617
|
+
```bash
|
|
1618
|
+
mastra api learning theme history <entityId> <themeId> '{"entityType":"agent","signalName":"goal"}'
|
|
1619
|
+
```
|
|
1620
|
+
|
|
1621
|
+
#### `mastra api learning noise get`
|
|
1622
|
+
|
|
1623
|
+
Gets the unclustered (noise) bucket for one trace signal in one snapshot.
|
|
1624
|
+
|
|
1625
|
+
```bash
|
|
1626
|
+
mastra api learning noise get <entityId> '{"entityType":"agent","signalName":"goal","snapshotId":"<snapshotId>"}'
|
|
1627
|
+
```
|
|
1628
|
+
|
|
1629
|
+
#### `mastra api learning noise examples`
|
|
1630
|
+
|
|
1631
|
+
Lists trace examples for the noise bucket in one snapshot. Paginate with `limit` and `offset`.
|
|
1632
|
+
|
|
1633
|
+
```bash
|
|
1634
|
+
mastra api learning noise examples <entityId> '{"entityType":"agent","signalName":"goal","snapshotId":"<snapshotId>","limit":10}'
|
|
1635
|
+
```
|
|
1636
|
+
|
|
1506
1637
|
## Common flags
|
|
1507
1638
|
|
|
1508
1639
|
### `--dir`
|
|
@@ -727,6 +727,23 @@ const agent = await mastraClient.createStoredAgent({
|
|
|
727
727
|
})
|
|
728
728
|
```
|
|
729
729
|
|
|
730
|
+
By default, `createStoredAgent()` publishes the initial version immediately. Set `autoPublish` to `false` to create an unpublished draft that you can review before calling [`activateVersion()`](#activateversion):
|
|
731
|
+
|
|
732
|
+
```typescript
|
|
733
|
+
const draft = await mastraClient.createStoredAgent({
|
|
734
|
+
id: 'draft-agent',
|
|
735
|
+
name: 'Draft Assistant',
|
|
736
|
+
instructions: 'You are a helpful assistant.',
|
|
737
|
+
model: {
|
|
738
|
+
provider: 'openai',
|
|
739
|
+
name: 'gpt-5',
|
|
740
|
+
},
|
|
741
|
+
autoPublish: false,
|
|
742
|
+
})
|
|
743
|
+
```
|
|
744
|
+
|
|
745
|
+
Editors configured with the `code` source always publish the initial version because saving writes the agent configuration to the file system.
|
|
746
|
+
|
|
730
747
|
With all options:
|
|
731
748
|
|
|
732
749
|
```typescript
|
|
@@ -18,7 +18,7 @@ const supportAgent = new Agent({
|
|
|
18
18
|
id: 'support-agent-id',
|
|
19
19
|
name: 'Support Agent',
|
|
20
20
|
instructions: 'Answer support questions clearly and concisely.',
|
|
21
|
-
model: 'openai/gpt-5.
|
|
21
|
+
model: 'openai/gpt-5.6-sol',
|
|
22
22
|
})
|
|
23
23
|
|
|
24
24
|
export const mastra = new Mastra({
|
|
@@ -103,4 +103,51 @@ Visit the [Configuration reference](https://mastra.ai/reference/configuration) f
|
|
|
103
103
|
|
|
104
104
|
**versions.agents.versionId** (`string`): The ID of a specific version to use.
|
|
105
105
|
|
|
106
|
-
**versions.agents.status** (`'draft' | 'published'`): Select the latest version with this publication status.
|
|
106
|
+
**versions.agents.status** (`'draft' | 'published'`): Select the latest version with this publication status.
|
|
107
|
+
|
|
108
|
+
**workers** (`MastraWorker[] | false`): Configure which workers run in this Mastra instance. When omitted, Mastra auto-creates default workers based on your PubSub and config. Pass false to disable all event processing (useful when running standalone workers separately). Pass a MastraWorker\[] to add custom workers — they are merged with the auto-created defaults, and a custom worker with the same name as a default replaces it.
|
|
109
|
+
|
|
110
|
+
**backgroundTasks** (`BackgroundTaskManagerConfig`): Configure background task execution for agents. See background tasks configuration reference for all options.
|
|
111
|
+
|
|
112
|
+
**backgroundTasks.enabled** (`boolean`): Enable background task dispatch.
|
|
113
|
+
|
|
114
|
+
**backgroundTasks.globalConcurrency** (`number`): Max concurrent tasks across all agents.
|
|
115
|
+
|
|
116
|
+
**backgroundTasks.perAgentConcurrency** (`number`): Max concurrent tasks per agent.
|
|
117
|
+
|
|
118
|
+
**backgroundTasks.backpressure** (`'queue' | 'reject' | 'fallback-sync'`): Behavior when concurrency limit is reached.
|
|
119
|
+
|
|
120
|
+
**backgroundTasks.defaultTimeoutMs** (`number`): Default task timeout in milliseconds.
|
|
121
|
+
|
|
122
|
+
**backgroundTasks.defaultRetries** (`RetryConfig`): Default retry configuration.
|
|
123
|
+
|
|
124
|
+
**scheduler** (`object`): Configure the scheduler worker for cron-driven workflow triggers. Auto-enables when any workflow declares a schedule. See Scheduled workflows.
|
|
125
|
+
|
|
126
|
+
**scheduler.enabled** (`boolean`): Explicitly enable or disable the scheduler.
|
|
127
|
+
|
|
128
|
+
**recovery** (`MastraRecoveryConfig`): Boot-time recovery behavior for orphaned agent and workflow runs. See Crash recovery. (Default: `{ durableAgents: 'off' }`)
|
|
129
|
+
|
|
130
|
+
**recovery.durableAgents** (`'auto' | 'off'`): Set to 'auto' to automatically re-drive orphaned RUNNING durable agent runs on server boot. Recovery re-issues LLM calls and re-executes tool calls, so tools must be idempotent. See Crash recovery.
|
|
131
|
+
|
|
132
|
+
## Methods
|
|
133
|
+
|
|
134
|
+
### `recoverAllDurableAgents()`
|
|
135
|
+
|
|
136
|
+
Re-drives every orphaned `running` durable-agent run across all registered durable agents. Called automatically on boot when `recovery.durableAgents` is `'auto'`. You can also call it directly for manual recovery or from a scheduled task.
|
|
137
|
+
|
|
138
|
+
Requires persistent storage — with an in-memory store there is nothing to recover after a process restart.
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
const result = await mastra.recoverAllDurableAgents()
|
|
142
|
+
// { agents: 2, recovered: 3, succeeded: 3, failed: 0 }
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
|
|
147
|
+
**agents** (`number`): Number of durable agents scanned.
|
|
148
|
+
|
|
149
|
+
**recovered** (`number`): Total number of runs that were re-driven.
|
|
150
|
+
|
|
151
|
+
**succeeded** (`number`): Runs that restarted successfully.
|
|
152
|
+
|
|
153
|
+
**failed** (`number`): Runs whose restart threw an error.
|
|
@@ -21,7 +21,7 @@ class MyCustomGateway extends MastraModelGateway {
|
|
|
21
21
|
return {
|
|
22
22
|
'my-provider': {
|
|
23
23
|
name: 'My Provider',
|
|
24
|
-
models: ['openai/gpt-5.
|
|
24
|
+
models: ['openai/gpt-5.6-sol', 'anthropic/claude-sonnet-4-6'],
|
|
25
25
|
apiKeyEnvVar: 'MY_API_KEY',
|
|
26
26
|
gateway: this.id,
|
|
27
27
|
},
|
|
@@ -71,7 +71,7 @@ import { runEvals } from '@mastra/core/evals'
|
|
|
71
71
|
import { createAnswerRelevancyScorer } from '@mastra/evals/scorers/prebuilt'
|
|
72
72
|
import { myAgent } from './agent'
|
|
73
73
|
|
|
74
|
-
const scorer = createAnswerRelevancyScorer({ model: 'openai/gpt-5.
|
|
74
|
+
const scorer = createAnswerRelevancyScorer({ model: 'openai/gpt-5.6-sol' })
|
|
75
75
|
|
|
76
76
|
const result = await runEvals({
|
|
77
77
|
data: [
|
|
@@ -64,7 +64,7 @@ import { runEvals } from '@mastra/core/evals'
|
|
|
64
64
|
import { createAnswerSimilarityScorer } from '@mastra/evals/scorers/prebuilt'
|
|
65
65
|
import { myAgent } from './agent'
|
|
66
66
|
|
|
67
|
-
const scorer = createAnswerSimilarityScorer({ model: 'openai/gpt-5.
|
|
67
|
+
const scorer = createAnswerSimilarityScorer({ model: 'openai/gpt-5.6-sol' })
|
|
68
68
|
|
|
69
69
|
const result = await runEvals({
|
|
70
70
|
data: [
|
|
@@ -84,7 +84,7 @@ import { runEvals } from '@mastra/core/evals'
|
|
|
84
84
|
import { createBiasScorer } from '@mastra/evals/scorers/prebuilt'
|
|
85
85
|
import { myAgent } from './agent'
|
|
86
86
|
|
|
87
|
-
const scorer = createBiasScorer({ model: 'openai/gpt-5.
|
|
87
|
+
const scorer = createBiasScorer({ model: 'openai/gpt-5.6-sol' })
|
|
88
88
|
|
|
89
89
|
const result = await runEvals({
|
|
90
90
|
data: [
|
|
@@ -100,7 +100,7 @@ MAP = (1.0 + 0.67) / 2 = 0.835 ≈ **0.83**
|
|
|
100
100
|
|
|
101
101
|
```typescript
|
|
102
102
|
const scorer = createContextPrecisionScorer({
|
|
103
|
-
model: 'openai/gpt-5.
|
|
103
|
+
model: 'openai/gpt-5.6-sol',
|
|
104
104
|
options: {
|
|
105
105
|
contextExtractor: (input, output) => {
|
|
106
106
|
// Extract context dynamically based on the query
|
|
@@ -119,7 +119,7 @@ const scorer = createContextPrecisionScorer({
|
|
|
119
119
|
|
|
120
120
|
```typescript
|
|
121
121
|
const scorer = createContextPrecisionScorer({
|
|
122
|
-
model: 'openai/gpt-5.
|
|
122
|
+
model: 'openai/gpt-5.6-sol',
|
|
123
123
|
options: {
|
|
124
124
|
context: [
|
|
125
125
|
// Simulate retrieved documents from vector database
|
|
@@ -144,7 +144,7 @@ import { createContextPrecisionScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
144
144
|
import { myAgent } from './agent'
|
|
145
145
|
|
|
146
146
|
const scorer = createContextPrecisionScorer({
|
|
147
|
-
model: 'openai/gpt-5.
|
|
147
|
+
model: 'openai/gpt-5.6-sol',
|
|
148
148
|
options: {
|
|
149
149
|
contextExtractor: (input, output) => {
|
|
150
150
|
// Extract context from agent's retrieved documents
|
|
@@ -103,7 +103,7 @@ Recall = 2/3 = 0.67
|
|
|
103
103
|
|
|
104
104
|
```typescript
|
|
105
105
|
const scorer = createContextRecallScorer({
|
|
106
|
-
model: 'openai/gpt-5.
|
|
106
|
+
model: 'openai/gpt-5.6-sol',
|
|
107
107
|
options: {
|
|
108
108
|
contextExtractor: (input, output) => {
|
|
109
109
|
const query = input?.inputMessages?.[0]?.content || ''
|
|
@@ -119,7 +119,7 @@ const scorer = createContextRecallScorer({
|
|
|
119
119
|
|
|
120
120
|
```typescript
|
|
121
121
|
const scorer = createContextRecallScorer({
|
|
122
|
-
model: 'openai/gpt-5.
|
|
122
|
+
model: 'openai/gpt-5.6-sol',
|
|
123
123
|
options: {
|
|
124
124
|
context: [
|
|
125
125
|
'Document 1: Einstein was born on 14 March 1879 in Ulm, Germany.',
|
|
@@ -140,7 +140,7 @@ import { createContextRecallScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
140
140
|
import { myAgent } from './agent'
|
|
141
141
|
|
|
142
142
|
const scorer = createContextRecallScorer({
|
|
143
|
-
model: 'openai/gpt-5.
|
|
143
|
+
model: 'openai/gpt-5.6-sol',
|
|
144
144
|
options: {
|
|
145
145
|
contextExtractor: (input, output) => {
|
|
146
146
|
// Extract context from tool invocation results in the agent output
|
|
@@ -121,7 +121,7 @@ import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
|
121
121
|
|
|
122
122
|
// Stricter penalty configuration
|
|
123
123
|
const strictScorer = createContextRelevanceScorerLLM({
|
|
124
|
-
model: 'openai/gpt-5.
|
|
124
|
+
model: 'openai/gpt-5.6-sol',
|
|
125
125
|
options: {
|
|
126
126
|
context: [
|
|
127
127
|
'Einstein won the Nobel Prize for photoelectric effect',
|
|
@@ -139,7 +139,7 @@ const strictScorer = createContextRelevanceScorerLLM({
|
|
|
139
139
|
|
|
140
140
|
// Lenient penalty configuration
|
|
141
141
|
const lenientScorer = createContextRelevanceScorerLLM({
|
|
142
|
-
model: 'openai/gpt-5.
|
|
142
|
+
model: 'openai/gpt-5.6-sol',
|
|
143
143
|
options: {
|
|
144
144
|
context: [
|
|
145
145
|
'Einstein won the Nobel Prize for photoelectric effect',
|
|
@@ -185,7 +185,7 @@ console.log('Lenient penalties:', lenientResult.score) // Higher score, less pen
|
|
|
185
185
|
|
|
186
186
|
```typescript
|
|
187
187
|
const scorer = createContextRelevanceScorerLLM({
|
|
188
|
-
model: 'openai/gpt-5.
|
|
188
|
+
model: 'openai/gpt-5.6-sol',
|
|
189
189
|
options: {
|
|
190
190
|
contextExtractor: (input, output) => {
|
|
191
191
|
// Extract context based on the query
|
|
@@ -209,7 +209,7 @@ const scorer = createContextRelevanceScorerLLM({
|
|
|
209
209
|
|
|
210
210
|
```typescript
|
|
211
211
|
const scorer = createContextRelevanceScorerLLM({
|
|
212
|
-
model: 'openai/gpt-5.
|
|
212
|
+
model: 'openai/gpt-5.6-sol',
|
|
213
213
|
options: {
|
|
214
214
|
context: ['Relevant information...', 'Supporting details...'],
|
|
215
215
|
scale: 100, // Scale scores from 0-100 instead of 0-1
|
|
@@ -223,7 +223,7 @@ const scorer = createContextRelevanceScorerLLM({
|
|
|
223
223
|
|
|
224
224
|
```typescript
|
|
225
225
|
const scorer = createContextRelevanceScorerLLM({
|
|
226
|
-
model: 'openai/gpt-5.
|
|
226
|
+
model: 'openai/gpt-5.6-sol',
|
|
227
227
|
options: {
|
|
228
228
|
contextExtractor: (input, output) => {
|
|
229
229
|
const query = input?.inputMessages?.[0]?.content || ''
|
|
@@ -250,7 +250,7 @@ This example shows excellent context relevance where all context directly suppor
|
|
|
250
250
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
251
251
|
|
|
252
252
|
const scorer = createContextRelevanceScorerLLM({
|
|
253
|
-
model: 'openai/gpt-5.
|
|
253
|
+
model: 'openai/gpt-5.6-sol',
|
|
254
254
|
options: {
|
|
255
255
|
context: [
|
|
256
256
|
'Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1921.',
|
|
@@ -297,7 +297,7 @@ This example shows moderate relevance with some context being irrelevant or unus
|
|
|
297
297
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
298
298
|
|
|
299
299
|
const scorer = createContextRelevanceScorerLLM({
|
|
300
|
-
model: 'openai/gpt-5.
|
|
300
|
+
model: 'openai/gpt-5.6-sol',
|
|
301
301
|
options: {
|
|
302
302
|
context: [
|
|
303
303
|
'Solar eclipses occur when the Moon blocks the Sun.',
|
|
@@ -339,7 +339,7 @@ console.log(result)
|
|
|
339
339
|
|
|
340
340
|
// With custom penalty configuration
|
|
341
341
|
const customScorer = createContextRelevanceScorerLLM({
|
|
342
|
-
model: 'openai/gpt-5.
|
|
342
|
+
model: 'openai/gpt-5.6-sol',
|
|
343
343
|
options: {
|
|
344
344
|
context: [
|
|
345
345
|
'Solar eclipses occur when the Moon blocks the Sun.',
|
|
@@ -386,7 +386,7 @@ This example shows poor context relevance with mostly irrelevant information:
|
|
|
386
386
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
387
387
|
|
|
388
388
|
const scorer = createContextRelevanceScorerLLM({
|
|
389
|
-
model: 'openai/gpt-5.
|
|
389
|
+
model: 'openai/gpt-5.6-sol',
|
|
390
390
|
options: {
|
|
391
391
|
context: [
|
|
392
392
|
'The Great Barrier Reef is located in Australia.',
|
|
@@ -434,7 +434,7 @@ Extract context at runtime based on the run input:
|
|
|
434
434
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
435
435
|
|
|
436
436
|
const scorer = createContextRelevanceScorerLLM({
|
|
437
|
-
model: 'openai/gpt-5.
|
|
437
|
+
model: 'openai/gpt-5.6-sol',
|
|
438
438
|
options: {
|
|
439
439
|
contextExtractor: (input, output) => {
|
|
440
440
|
// Extract query from input
|
|
@@ -477,7 +477,7 @@ Integrate with RAG pipelines to evaluate retrieved context:
|
|
|
477
477
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
478
478
|
|
|
479
479
|
const scorer = createContextRelevanceScorerLLM({
|
|
480
|
-
model: 'openai/gpt-5.
|
|
480
|
+
model: 'openai/gpt-5.6-sol',
|
|
481
481
|
options: {
|
|
482
482
|
contextExtractor: (input, output) => {
|
|
483
483
|
// Extract from RAG retrieval results
|
|
@@ -81,7 +81,7 @@ import { myAgent } from './agent'
|
|
|
81
81
|
|
|
82
82
|
// Context is typically populated from agent tool calls or RAG retrieval
|
|
83
83
|
const scorer = createFaithfulnessScorer({
|
|
84
|
-
model: 'openai/gpt-5.
|
|
84
|
+
model: 'openai/gpt-5.6-sol',
|
|
85
85
|
})
|
|
86
86
|
|
|
87
87
|
const result = await runEvals({
|
|
@@ -106,7 +106,7 @@ Use static context when you have known ground truth to compare against:
|
|
|
106
106
|
import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
107
107
|
|
|
108
108
|
const scorer = createHallucinationScorer({
|
|
109
|
-
model: 'openai/gpt-5.
|
|
109
|
+
model: 'openai/gpt-5.6-sol',
|
|
110
110
|
options: {
|
|
111
111
|
context: [
|
|
112
112
|
'The first iPhone was announced on January 9, 2007.',
|
|
@@ -126,7 +126,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
126
126
|
import { extractToolResults } from '@mastra/evals/scorers'
|
|
127
127
|
|
|
128
128
|
const scorer = createHallucinationScorer({
|
|
129
|
-
model: 'openai/gpt-5.
|
|
129
|
+
model: 'openai/gpt-5.6-sol',
|
|
130
130
|
options: {
|
|
131
131
|
getContext: ({ run, step }) => {
|
|
132
132
|
// Extract tool results as context
|
|
@@ -147,7 +147,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
147
147
|
import { extractToolResults } from '@mastra/evals/scorers'
|
|
148
148
|
|
|
149
149
|
const hallucinationScorer = createHallucinationScorer({
|
|
150
|
-
model: 'openai/gpt-5.
|
|
150
|
+
model: 'openai/gpt-5.6-sol',
|
|
151
151
|
options: {
|
|
152
152
|
getContext: ({ run }) => {
|
|
153
153
|
const toolResults = extractToolResults(run.output)
|
|
@@ -159,7 +159,7 @@ const hallucinationScorer = createHallucinationScorer({
|
|
|
159
159
|
const agent = new Agent({
|
|
160
160
|
id: 'my-agent',
|
|
161
161
|
name: 'my-agent',
|
|
162
|
-
model: 'openai/gpt-5.
|
|
162
|
+
model: 'openai/gpt-5.6-sol',
|
|
163
163
|
instructions: 'You are a helpful assistant.',
|
|
164
164
|
evals: {
|
|
165
165
|
scorers: [hallucinationScorer],
|
|
@@ -175,7 +175,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
175
175
|
import { myAgent } from './agent'
|
|
176
176
|
|
|
177
177
|
const scorer = createHallucinationScorer({
|
|
178
|
-
model: 'openai/gpt-5.
|
|
178
|
+
model: 'openai/gpt-5.6-sol',
|
|
179
179
|
options: {
|
|
180
180
|
context: ['Known fact 1', 'Known fact 2'],
|
|
181
181
|
},
|
|
@@ -63,7 +63,7 @@ describe('Agent Noise Resistance Tests', () => {
|
|
|
63
63
|
|
|
64
64
|
// Step 4: Evaluate using noise sensitivity scorer
|
|
65
65
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
66
|
-
model: 'openai/gpt-5.
|
|
66
|
+
model: 'openai/gpt-5.6-sol',
|
|
67
67
|
options: {
|
|
68
68
|
baselineResponse,
|
|
69
69
|
noisyQuery,
|
|
@@ -258,7 +258,7 @@ describe('Agent Noise Resistance CI Tests', () => {
|
|
|
258
258
|
|
|
259
259
|
// Evaluate using noise sensitivity scorer
|
|
260
260
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
261
|
-
model: 'openai/gpt-5.
|
|
261
|
+
model: 'openai/gpt-5.6-sol',
|
|
262
262
|
options: {
|
|
263
263
|
baselineResponse: testCase.baselineResponse,
|
|
264
264
|
noisyQuery: testCase.noisyQuery,
|
|
@@ -293,7 +293,7 @@ This example shows an agent that completely resists misinformation in a test sce
|
|
|
293
293
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
294
294
|
|
|
295
295
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
296
|
-
model: 'openai/gpt-5.
|
|
296
|
+
model: 'openai/gpt-5.6-sol',
|
|
297
297
|
options: {
|
|
298
298
|
baselineResponse:
|
|
299
299
|
'Regular exercise improves cardiovascular health, strengthens muscles, and enhances mental wellbeing.',
|
|
@@ -339,7 +339,7 @@ This example shows an agent partially distracted by irrelevant requests:
|
|
|
339
339
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals/scorers/prebuilt'
|
|
340
340
|
|
|
341
341
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
342
|
-
model: 'openai/gpt-5.
|
|
342
|
+
model: 'openai/gpt-5.6-sol',
|
|
343
343
|
options: {
|
|
344
344
|
baselineResponse:
|
|
345
345
|
'To bake a cake: Mix flour, sugar, eggs, and butter. Bake at 350°F for 30 minutes.',
|
|
@@ -384,7 +384,7 @@ This example shows an agent that incorporates misinformation:
|
|
|
384
384
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
385
385
|
|
|
386
386
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
387
|
-
model: 'openai/gpt-5.
|
|
387
|
+
model: 'openai/gpt-5.6-sol',
|
|
388
388
|
options: {
|
|
389
389
|
baselineResponse: 'Climate change is caused by greenhouse gas emissions from human activities.',
|
|
390
390
|
noisyQuery:
|
|
@@ -430,7 +430,7 @@ import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
|
430
430
|
|
|
431
431
|
// Lenient scoring - more forgiving of minor issues
|
|
432
432
|
const lenientScorer = createNoiseSensitivityScorerLLM({
|
|
433
|
-
model: 'openai/gpt-5.
|
|
433
|
+
model: 'openai/gpt-5.6-sol',
|
|
434
434
|
options: {
|
|
435
435
|
baselineResponse: 'Python is a high-level programming language.',
|
|
436
436
|
noisyQuery: 'What is Python? Also, snakes are dangerous!',
|
|
@@ -450,7 +450,7 @@ const lenientScorer = createNoiseSensitivityScorerLLM({
|
|
|
450
450
|
|
|
451
451
|
// Strict scoring - harsh on any deviation
|
|
452
452
|
const strictScorer = createNoiseSensitivityScorerLLM({
|
|
453
|
-
model: 'openai/gpt-5.
|
|
453
|
+
model: 'openai/gpt-5.6-sol',
|
|
454
454
|
options: {
|
|
455
455
|
baselineResponse: 'Python is a high-level programming language.',
|
|
456
456
|
noisyQuery: 'What is Python? Also, snakes are dangerous!',
|
|
@@ -501,7 +501,7 @@ async function evaluateNoiseResistance(testCases) {
|
|
|
501
501
|
|
|
502
502
|
for (const testCase of testCases) {
|
|
503
503
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
504
|
-
model: 'openai/gpt-5.
|
|
504
|
+
model: 'openai/gpt-5.6-sol',
|
|
505
505
|
options: {
|
|
506
506
|
baselineResponse: testCase.baseline,
|
|
507
507
|
noisyQuery: testCase.noisyQuery,
|
|
@@ -548,7 +548,7 @@ import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
|
548
548
|
|
|
549
549
|
async function compareModelRobustness() {
|
|
550
550
|
const models = [
|
|
551
|
-
{ name: 'GPT-5.4', model: 'openai/gpt-5.
|
|
551
|
+
{ name: 'GPT-5.4', model: 'openai/gpt-5.6-sol' },
|
|
552
552
|
{ name: 'GPT-5.4-mini', model: 'openai/gpt-5-mini' },
|
|
553
553
|
{ name: 'Claude', model: 'anthropic/claude-opus-4-7' },
|
|
554
554
|
]
|
|
@@ -600,7 +600,7 @@ Include noise sensitivity tests in your security test suite to validate prompt i
|
|
|
600
600
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
601
601
|
|
|
602
602
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
603
|
-
model: 'openai/gpt-5.
|
|
603
|
+
model: 'openai/gpt-5.6-sol',
|
|
604
604
|
options: {
|
|
605
605
|
baselineResponse: 'I can help you with programming questions.',
|
|
606
606
|
noisyQuery:
|