@mastra/mcp-docs-server 1.2.11 → 1.2.12-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/docs/agent-builder/access-control.md +3 -3
- package/.docs/docs/agent-builder/browser.md +1 -1
- package/.docs/docs/agent-builder/channels.md +1 -1
- package/.docs/docs/agent-builder/configuration.md +3 -3
- package/.docs/docs/agent-builder/deploying.md +2 -2
- package/.docs/docs/agent-builder/integrations.md +8 -8
- package/.docs/docs/agent-builder/memory.md +2 -2
- package/.docs/docs/agent-builder/overview.md +1 -1
- package/.docs/docs/agent-controller/channels.md +8 -6
- package/.docs/docs/agent-controller/modes.md +6 -6
- package/.docs/docs/agent-controller/overview.md +13 -9
- package/.docs/docs/agent-controller/session.md +27 -9
- package/.docs/docs/agent-controller/subagents.md +9 -6
- package/.docs/docs/agent-controller/threads-and-state.md +12 -8
- package/.docs/docs/agent-controller/tool-approvals.md +7 -3
- package/.docs/docs/agents/a2a.md +4 -2
- package/.docs/docs/agents/acp.md +5 -5
- package/.docs/docs/agents/agent-approval.md +13 -11
- package/.docs/docs/agents/code-mode.md +29 -6
- package/.docs/docs/agents/guardrails.md +8 -8
- package/.docs/docs/agents/networks.md +2 -2
- package/.docs/docs/agents/overview.md +9 -7
- package/.docs/docs/agents/processors.md +15 -15
- package/.docs/docs/agents/skills.md +6 -6
- package/.docs/docs/agents/structured-output.md +6 -6
- package/.docs/docs/agents/supervisor-agents.md +19 -13
- package/.docs/docs/agents/using-tools.md +48 -21
- package/.docs/docs/browser/agent-browser.md +4 -4
- package/.docs/docs/browser/browser-viewer.md +2 -2
- package/.docs/docs/browser/firecrawl.md +1 -1
- package/.docs/docs/browser/overview.md +2 -2
- package/.docs/docs/browser/recording.md +1 -1
- package/.docs/docs/browser/stagehand.md +7 -7
- package/.docs/docs/capabilities/channels/discord.md +1 -1
- package/.docs/docs/capabilities/channels/other-adapters.md +1 -1
- package/.docs/docs/capabilities/channels/overview.md +9 -5
- package/.docs/docs/capabilities/channels/slack.md +2 -2
- package/.docs/docs/capabilities/channels/teams.md +1 -1
- package/.docs/docs/capabilities/channels/telegram.md +1 -1
- package/.docs/docs/capabilities/channels/whatsapp.md +1 -1
- package/.docs/docs/deployment/mastra-server.md +1 -1
- package/.docs/docs/deployment/overview.md +6 -2
- package/.docs/docs/deployment/sandbox.md +3 -1
- package/.docs/docs/deployment/workers.md +137 -0
- package/.docs/docs/editor/overview.md +37 -31
- package/.docs/docs/editor/prompts.md +8 -4
- package/.docs/docs/editor/tools.md +3 -3
- package/.docs/docs/evals/built-in-scorers.md +4 -4
- package/.docs/docs/evals/custom-scorers.md +5 -5
- package/.docs/docs/evals/datasets/running-experiments.md +7 -7
- package/.docs/docs/evals/evals-with-memory.md +4 -4
- package/.docs/docs/evals/gates-and-verdicts.md +4 -4
- package/.docs/docs/evals/multi-turn.md +12 -12
- package/.docs/docs/evals/overview.md +6 -4
- package/.docs/docs/evals/quick-checks.md +2 -2
- package/.docs/docs/getting-started/build-with-ai.md +6 -4
- package/.docs/docs/getting-started/file-based-agents.md +6 -4
- package/.docs/docs/getting-started/manual-install.md +1 -1
- package/.docs/docs/index.md +9 -90
- package/.docs/docs/long-running-agents/background-tasks.md +10 -10
- package/.docs/docs/long-running-agents/durable-agents.md +46 -5
- package/.docs/docs/long-running-agents/goals.md +15 -11
- package/.docs/docs/long-running-agents/schedules.md +3 -3
- package/.docs/docs/long-running-agents/signal-providers.md +8 -6
- package/.docs/docs/long-running-agents/signals.md +9 -5
- package/.docs/docs/mastra-platform/configuration.md +3 -3
- package/.docs/docs/mastra-platform/database.md +8 -8
- package/.docs/docs/mastra-platform/deploy.md +8 -4
- package/.docs/docs/mastra-platform/environments.md +8 -4
- package/.docs/docs/mastra-platform/github.md +3 -1
- package/.docs/docs/mastra-platform/server.md +1 -1
- package/.docs/docs/mastra-platform/studio.md +2 -2
- package/.docs/docs/mastra-platform/workspace.md +6 -6
- package/.docs/docs/mcp/mcp-apps.md +1 -1
- package/.docs/docs/mcp/overview.md +9 -9
- package/.docs/docs/memory/memory-processors.md +10 -10
- package/.docs/docs/memory/message-history.md +11 -5
- package/.docs/docs/memory/multi-user-threads.md +6 -6
- package/.docs/docs/memory/observational-memory.md +27 -25
- package/.docs/docs/memory/overview.md +10 -10
- package/.docs/docs/memory/semantic-recall.md +3 -3
- package/.docs/docs/memory/working-memory.md +8 -8
- package/.docs/docs/observability/integrations/bridges/datadog.md +6 -6
- package/.docs/docs/observability/integrations/bridges/otel.md +5 -5
- package/.docs/docs/observability/integrations/exporters/datadog.md +1 -1
- package/.docs/docs/observability/integrations/exporters/laminar.md +1 -1
- package/.docs/docs/observability/integrations/exporters/langfuse.md +3 -3
- package/.docs/docs/observability/integrations/exporters/langsmith.md +4 -4
- package/.docs/docs/observability/integrations/exporters/mastra-platform.md +1 -1
- package/.docs/docs/observability/integrations/exporters/mastra-storage.md +1 -1
- package/.docs/docs/observability/integrations/exporters/otel.md +26 -4
- package/.docs/docs/observability/integrations/exporters/sentry.md +1 -1
- package/.docs/docs/observability/integrations/overview.md +1 -1
- package/.docs/docs/observability/integrations/processors/sensitive-data-filter.md +1 -1
- package/.docs/docs/observability/logging.md +1 -1
- package/.docs/docs/observability/metrics/overview.md +1 -1
- package/.docs/docs/observability/metrics/querying.md +8 -8
- package/.docs/docs/observability/overview.md +4 -4
- package/.docs/docs/observability/tracing/overview.md +18 -12
- package/.docs/docs/rag/chunking-and-embedding.md +1 -1
- package/.docs/docs/rag/graph-rag.md +7 -7
- package/.docs/docs/rag/overview.md +2 -2
- package/.docs/docs/rag/retrieval.md +18 -18
- package/.docs/docs/rag/vector-databases.md +2 -2
- package/.docs/docs/server/auth/composite-auth.md +2 -2
- package/.docs/docs/server/auth/fga.md +6 -6
- package/.docs/docs/server/auth/firebase.md +2 -2
- package/.docs/docs/server/auth/simple-auth.md +1 -1
- package/.docs/docs/server/auth/workers.md +133 -0
- package/.docs/docs/server/custom-adapters.md +4 -4
- package/.docs/docs/server/mastra-client.md +3 -3
- package/.docs/docs/server/mastra-server.md +2 -2
- package/.docs/docs/server/middleware.md +1 -1
- package/.docs/docs/server/pubsub.md +7 -4
- package/.docs/docs/server/request-context.md +11 -11
- package/.docs/docs/server/server-adapters.md +2 -2
- package/.docs/docs/storage/overview.md +2 -2
- package/.docs/docs/studio/auth.md +5 -5
- package/.docs/docs/studio/deployment.md +1 -1
- package/.docs/docs/studio/observability.md +2 -2
- package/.docs/docs/studio/overview.md +11 -9
- package/.docs/docs/voice/overview.md +26 -26
- package/.docs/docs/voice/realtime-voice.md +8 -8
- package/.docs/docs/voice/speech-to-speech.md +9 -9
- package/.docs/docs/voice/speech-to-text.md +3 -3
- package/.docs/docs/voice/text-to-speech.md +6 -6
- package/.docs/docs/what-is-mastra.md +161 -0
- package/.docs/docs/workflows/agents-and-tools.md +3 -3
- package/.docs/docs/workflows/control-flow.md +10 -8
- package/.docs/docs/workflows/error-handling.md +2 -2
- package/.docs/docs/workflows/overview.md +7 -6
- package/.docs/docs/workflows/scheduled-workflows.md +12 -12
- package/.docs/docs/workflows/snapshots.md +3 -3
- package/.docs/docs/workflows/suspend-and-resume.md +1 -1
- package/.docs/docs/workflows/time-travel.md +3 -1
- package/.docs/docs/workflows/workflow-state.md +1 -1
- package/.docs/docs/workspace/filesystem.md +10 -10
- package/.docs/docs/workspace/lsp.md +1 -1
- package/.docs/docs/workspace/overview.md +12 -12
- package/.docs/docs/workspace/sandbox.md +9 -7
- package/.docs/docs/workspace/search.md +3 -3
- package/.docs/docs/workspace/skills.md +7 -7
- package/.docs/guides/build-your-ui/ai-sdk-ui.md +6 -6
- package/.docs/guides/build-your-ui/copilotkit/generative-ui.md +9 -9
- package/.docs/guides/build-your-ui/copilotkit/overview.md +2 -2
- package/.docs/guides/build-your-ui/openui.md +2 -2
- package/.docs/guides/concepts/multi-agent-systems.md +5 -5
- package/.docs/guides/concepts/streaming.md +10 -10
- package/.docs/guides/deployment/amazon-ec2.md +1 -1
- package/.docs/guides/deployment/aws-lambda.md +1 -1
- package/.docs/guides/deployment/azure-app-services.md +1 -1
- package/.docs/guides/deployment/cloudflare.md +1 -1
- package/.docs/guides/deployment/inngest.md +6 -6
- package/.docs/guides/deployment/kubernetes.md +1 -1
- package/.docs/guides/deployment/mastra-workers.md +232 -0
- package/.docs/guides/deployment/temporal.md +1 -1
- package/.docs/guides/deployment/vercel.md +1 -1
- package/.docs/guides/getting-started/astro.md +1 -1
- package/.docs/guides/getting-started/electron.md +1 -1
- package/.docs/guides/getting-started/nestjs.md +5 -5
- package/.docs/guides/getting-started/vite-react.md +1 -1
- package/.docs/guides/guide/ai-recruiter.md +1 -1
- package/.docs/guides/guide/chef-michel.md +2 -2
- package/.docs/guides/guide/code-review-bot.md +2 -2
- package/.docs/guides/guide/coding-agent.md +5 -5
- package/.docs/guides/guide/dev-assistant.md +2 -2
- package/.docs/guides/guide/docs-manager.md +4 -4
- package/.docs/guides/guide/firecrawl.md +1 -1
- package/.docs/guides/guide/github-actions-pr-description.md +5 -5
- package/.docs/guides/guide/research-assistant.md +4 -4
- package/.docs/guides/guide/research-coordinator.md +5 -5
- package/.docs/guides/guide/signal-provider.md +1 -1
- package/.docs/guides/guide/slack-assistant.md +1 -1
- package/.docs/guides/guide/stock-agent.md +3 -3
- package/.docs/guides/guide/web-search.md +4 -4
- package/.docs/guides/guide/whatsapp-chat-bot.md +2 -2
- package/.docs/guides/index.md +1 -1
- package/.docs/guides/migrations/agentnetwork.md +4 -4
- package/.docs/guides/migrations/ai-sdk-v4-to-v5.md +1 -1
- package/.docs/guides/migrations/mastra-cloud.md +6 -6
- package/.docs/guides/migrations/network-to-supervisor.md +9 -9
- package/.docs/guides/migrations/upgrade-to-v1/agent.md +6 -6
- package/.docs/guides/migrations/upgrade-to-v1/cli.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/client.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/evals.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/memory.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/overview.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/processors.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/storage.md +11 -11
- package/.docs/guides/migrations/upgrade-to-v1/tools.md +3 -3
- package/.docs/guides/migrations/upgrade-to-v1/tracing.md +2 -2
- package/.docs/guides/migrations/upgrade-to-v1/voice.md +1 -1
- package/.docs/guides/migrations/upgrade-to-v1/workflows.md +1 -1
- package/.docs/guides/migrations/vnext-to-standard-apis.md +2 -2
- package/.docs/models/environment-variables.md +1 -0
- package/.docs/models/gateways/custom-gateways.md +3 -3
- package/.docs/models/gateways/openrouter.md +1 -3
- package/.docs/models/gateways/vercel.md +2 -1
- package/.docs/models/index.md +6 -6
- package/.docs/models/providers/baseten.md +1 -1
- package/.docs/models/providers/chutes.md +2 -1
- package/.docs/models/providers/crossmodel.md +2 -2
- package/.docs/models/providers/deepinfra.md +1 -1
- package/.docs/models/providers/hyper.md +3 -3
- package/.docs/models/providers/llmgateway.md +4 -3
- package/.docs/models/providers/modal.md +73 -0
- package/.docs/models/providers/wandb.md +2 -1
- package/.docs/models/providers.md +1 -0
- package/.docs/reference/acp/acp-agent.md +4 -4
- package/.docs/reference/acp/create-acp-tool.md +3 -3
- package/.docs/reference/agent-controller/agent-controller-class.md +36 -36
- package/.docs/reference/agent-controller/session.md +6 -6
- package/.docs/reference/agents/agent.md +14 -14
- package/.docs/reference/agents/channels.md +29 -13
- package/.docs/reference/agents/durable-agent.md +54 -5
- package/.docs/reference/agents/generate.md +3 -3
- package/.docs/reference/agents/generateLegacy.md +1 -1
- package/.docs/reference/agents/getDefaultOptions.md +1 -1
- package/.docs/reference/agents/getDefaultStreamOptions.md +1 -1
- package/.docs/reference/agents/getLLM.md +2 -2
- package/.docs/reference/agents/getMetadata.md +2 -2
- package/.docs/reference/agents/inngest-agent.md +1 -1
- package/.docs/reference/agents/listScorers.md +1 -1
- package/.docs/reference/agents/listSkills.md +1 -1
- package/.docs/reference/agents/listSuspendedRuns.md +2 -2
- package/.docs/reference/agents/network.md +1 -1
- package/.docs/reference/ai-sdk/chat-route.md +1 -1
- package/.docs/reference/ai-sdk/network-route.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v4-messages.md +1 -1
- package/.docs/reference/ai-sdk/to-ai-sdk-v5-messages.md +1 -1
- package/.docs/reference/ai-sdk/workflow-route.md +1 -1
- package/.docs/reference/ai-sdk/workflow-snapshot-to-stream.md +2 -2
- package/.docs/reference/auth/auth0.md +1 -1
- package/.docs/reference/auth/fga.md +2 -2
- package/.docs/reference/auth/google.md +2 -2
- package/.docs/reference/auth/okta.md +1 -1
- package/.docs/reference/auth/workos.md +2 -2
- package/.docs/reference/browser/agent-browser.md +3 -3
- package/.docs/reference/browser/browser-viewer.md +4 -4
- package/.docs/reference/browser/firecrawl-browser.md +1 -1
- package/.docs/reference/browser/mastra-browser.md +4 -4
- package/.docs/reference/browser/stagehand-browser.md +8 -8
- package/.docs/reference/channels/slack-provider.md +2 -2
- package/.docs/reference/cli/create-mastra.md +1 -1
- package/.docs/reference/cli/mastra.md +155 -24
- package/.docs/reference/client-js/agents.md +26 -9
- package/.docs/reference/client-js/memory.md +2 -2
- package/.docs/reference/client-js/workflows.md +1 -1
- package/.docs/reference/coding-agent/build-base-prompt.md +2 -2
- package/.docs/reference/coding-agent/create-coding-agent.md +2 -2
- package/.docs/reference/configuration.md +7 -7
- package/.docs/reference/core/getAgentById.md +1 -1
- package/.docs/reference/core/getMCPServer.md +1 -1
- package/.docs/reference/core/getScorer.md +1 -1
- package/.docs/reference/core/mastra-class.md +48 -1
- package/.docs/reference/core/mastra-model-gateway.md +1 -1
- package/.docs/reference/datasets/addItem.md +1 -1
- package/.docs/reference/datasets/listItems.md +1 -1
- package/.docs/reference/deployer/netlify.md +4 -4
- package/.docs/reference/deployer/vercel.md +7 -7
- package/.docs/reference/editor/agent-builder/agent-builder-options.md +4 -4
- package/.docs/reference/editor/agent-builder/builder-agent-defaults.md +6 -6
- package/.docs/reference/editor/agent-builder/builder-models.md +3 -3
- package/.docs/reference/editor/blob-store-provider.md +1 -1
- package/.docs/reference/editor/browser-provider.md +3 -3
- package/.docs/reference/editor/filesystem-provider.md +4 -4
- package/.docs/reference/editor/mastra-editor.md +5 -5
- package/.docs/reference/editor/processor-provider.md +3 -3
- package/.docs/reference/editor/sandbox-provider.md +4 -4
- package/.docs/reference/editor/storage-browser-ref.md +6 -6
- package/.docs/reference/editor/storage-workspace-ref.md +4 -4
- package/.docs/reference/editor/tool-provider.md +1 -1
- package/.docs/reference/evals/answer-relevancy.md +5 -5
- package/.docs/reference/evals/answer-similarity.md +1 -1
- package/.docs/reference/evals/bias.md +4 -4
- package/.docs/reference/evals/checks.md +3 -3
- package/.docs/reference/evals/completeness.md +5 -5
- package/.docs/reference/evals/content-similarity.md +5 -5
- package/.docs/reference/evals/context-precision.md +5 -5
- package/.docs/reference/evals/context-recall.md +11 -11
- package/.docs/reference/evals/context-relevance.md +15 -15
- package/.docs/reference/evals/create-scorer.md +1 -1
- package/.docs/reference/evals/faithfulness.md +4 -4
- package/.docs/reference/evals/filter-run.md +2 -2
- package/.docs/reference/evals/hallucination.md +11 -11
- package/.docs/reference/evals/keyword-coverage.md +6 -6
- package/.docs/reference/evals/mastra-scorer.md +2 -2
- package/.docs/reference/evals/noise-sensitivity.md +15 -15
- package/.docs/reference/evals/prompt-alignment.md +20 -20
- package/.docs/reference/evals/rubric.md +2 -2
- package/.docs/reference/evals/run-evals.md +3 -3
- package/.docs/reference/evals/scorer-utils.md +4 -4
- package/.docs/reference/evals/textual-difference.md +4 -4
- package/.docs/reference/evals/tool-call-accuracy.md +4 -4
- package/.docs/reference/evals/toxicity.md +5 -5
- package/.docs/reference/evals/trajectory-accuracy.md +10 -10
- package/.docs/reference/file-based-agents/config.md +3 -3
- package/.docs/reference/file-based-agents/instructions.md +3 -3
- package/.docs/reference/file-based-agents/logger.md +1 -1
- package/.docs/reference/file-based-agents/observability.md +1 -1
- package/.docs/reference/file-based-agents/processors.md +1 -1
- package/.docs/reference/file-based-agents/skills.md +1 -1
- package/.docs/reference/file-based-agents/storage.md +1 -1
- package/.docs/reference/file-based-agents/studio.md +1 -1
- package/.docs/reference/file-based-agents/subagents.md +4 -4
- package/.docs/reference/file-based-agents/tools.md +1 -1
- package/.docs/reference/file-based-agents/workspace.md +1 -1
- package/.docs/reference/index.md +2 -0
- package/.docs/reference/logging/pino-logger.md +3 -3
- package/.docs/reference/memory/clone-utilities.md +1 -1
- package/.docs/reference/memory/cloneThread.md +3 -3
- package/.docs/reference/memory/listThreads.md +1 -1
- package/.docs/reference/memory/memory-class.md +4 -4
- package/.docs/reference/memory/observational-memory.md +15 -15
- package/.docs/reference/memory/recall.md +2 -2
- package/.docs/reference/memory/serialized-memory-config.md +4 -4
- package/.docs/reference/memory/summarizeConversation.md +3 -3
- package/.docs/reference/memory/summarizeThread.md +3 -3
- package/.docs/reference/observability/feedback.md +1 -1
- package/.docs/reference/observability/metrics/automatic-metrics.md +7 -7
- package/.docs/reference/observability/tracing/bridges/datadog.md +2 -2
- package/.docs/reference/observability/tracing/exporters/cloud-exporter.md +3 -3
- package/.docs/reference/observability/tracing/exporters/default-exporter.md +1 -1
- package/.docs/reference/observability/tracing/exporters/langfuse.md +1 -1
- package/.docs/reference/observability/tracing/exporters/mastra-platform-exporter.md +4 -4
- package/.docs/reference/observability/tracing/exporters/mastra-storage-exporter.md +1 -1
- package/.docs/reference/observability/tracing/interfaces.md +1 -1
- package/.docs/reference/observability/tracing/processors/sensitive-data-filter.md +1 -1
- package/.docs/reference/observability/tracing/span-filtering.md +1 -1
- package/.docs/reference/processors/batch-parts-processor.md +1 -1
- package/.docs/reference/processors/cost-guard-processor.md +1 -1
- package/.docs/reference/processors/language-detector.md +1 -1
- package/.docs/reference/processors/message-history-processor.md +1 -1
- package/.docs/reference/processors/moderation-processor.md +2 -2
- package/.docs/reference/processors/pii-detector.md +3 -3
- package/.docs/reference/processors/processor-interface.md +13 -13
- package/.docs/reference/processors/prompt-injection-detector.md +2 -2
- package/.docs/reference/processors/provider-history-compat.md +1 -1
- package/.docs/reference/processors/regex-filter-processor.md +1 -1
- package/.docs/reference/processors/response-cache.md +8 -8
- package/.docs/reference/processors/semantic-recall-processor.md +1 -1
- package/.docs/reference/processors/skill-search-processor.md +1 -1
- package/.docs/reference/processors/system-prompt-scrubber.md +2 -2
- package/.docs/reference/processors/token-limiter-processor.md +4 -4
- package/.docs/reference/processors/tool-call-filter.md +2 -2
- package/.docs/reference/processors/tool-search-processor.md +9 -9
- package/.docs/reference/processors/unicode-normalizer.md +2 -2
- package/.docs/reference/processors/working-memory-processor.md +1 -1
- package/.docs/reference/project-structure.md +1 -1
- package/.docs/reference/pubsub/base.md +4 -4
- package/.docs/reference/pubsub/google-cloud-pubsub.md +2 -2
- package/.docs/reference/pubsub/lease-provider.md +4 -4
- package/.docs/reference/pubsub/redis-streams.md +1 -1
- package/.docs/reference/pubsub/unix-socket-pubsub.md +1 -1
- package/.docs/reference/rag/chunk.md +5 -5
- package/.docs/reference/rag/database-config.md +2 -2
- package/.docs/reference/rag/metadata-filters.md +2 -2
- package/.docs/reference/rag/rerank.md +1 -1
- package/.docs/reference/schedules/overview.md +1 -1
- package/.docs/reference/server/register-api-route.md +1 -1
- package/.docs/reference/server/routes.md +7 -7
- package/.docs/reference/signals/create-notification-inbox-tool.md +1 -1
- package/.docs/reference/signals/signal-provider.md +8 -8
- package/.docs/reference/storage/clickhouse.md +4 -4
- package/.docs/reference/storage/cloudflare-d1.md +4 -4
- package/.docs/reference/storage/convex.md +4 -4
- package/.docs/reference/storage/dsql.md +3 -3
- package/.docs/reference/storage/dynamodb.md +7 -7
- package/.docs/reference/storage/lance.md +1 -1
- package/.docs/reference/storage/mongodb.md +2 -2
- package/.docs/reference/storage/mssql.md +1 -1
- package/.docs/reference/storage/postgresql.md +2 -2
- package/.docs/reference/storage/redis.md +1 -1
- package/.docs/reference/storage/retention.md +33 -33
- package/.docs/reference/storage/spanner.md +8 -8
- package/.docs/reference/storage/upstash.md +1 -1
- package/.docs/reference/streaming/ChunkType.md +2 -2
- package/.docs/reference/streaming/agents/MastraModelOutput.md +1 -1
- package/.docs/reference/streaming/agents/stream.md +2 -2
- package/.docs/reference/streaming/agents/streamUntilIdle.md +2 -2
- package/.docs/reference/streaming/workflows/observeStream.md +1 -1
- package/.docs/reference/streaming/workflows/resumeStream.md +1 -1
- package/.docs/reference/streaming/workflows/stream.md +1 -1
- package/.docs/reference/streaming/workflows/timeTravelStream.md +2 -2
- package/.docs/reference/templates/overview.md +1 -1
- package/.docs/reference/tools/brightdata.md +1 -1
- package/.docs/reference/tools/create-code-mode.md +3 -3
- package/.docs/reference/tools/create-tool.md +1 -1
- package/.docs/reference/tools/graph-rag-tool.md +1 -1
- package/.docs/reference/tools/isolated-vm-transport.md +74 -0
- package/.docs/reference/tools/mcp-client.md +15 -15
- package/.docs/reference/tools/mcp-server.md +21 -21
- package/.docs/reference/tools/submit-plan-tool.md +1 -1
- package/.docs/reference/tools/tavily.md +1 -1
- package/.docs/reference/tools/vector-query-tool.md +5 -5
- package/.docs/reference/vectors/astra.md +1 -1
- package/.docs/reference/vectors/chroma.md +2 -2
- package/.docs/reference/vectors/convex.md +2 -2
- package/.docs/reference/vectors/couchbase.md +4 -4
- package/.docs/reference/vectors/libsql.md +2 -2
- package/.docs/reference/vectors/mongodb.md +3 -3
- package/.docs/reference/vectors/pg.md +5 -5
- package/.docs/reference/vectors/qdrant.md +2 -2
- package/.docs/reference/vectors/s3vectors.md +4 -4
- package/.docs/reference/vectors/turbopuffer.md +1 -1
- package/.docs/reference/vectors/upstash.md +1 -1
- package/.docs/reference/voice/aws-nova-sonic.md +1 -1
- package/.docs/reference/voice/azure.md +1 -1
- package/.docs/reference/voice/google-gemini-live.md +4 -4
- package/.docs/reference/voice/inworld-realtime.md +7 -7
- package/.docs/reference/voice/livekit.md +13 -13
- package/.docs/reference/voice/mistral.md +1 -1
- package/.docs/reference/voice/openai-realtime.md +1 -1
- package/.docs/reference/voice/sarvam.md +1 -1
- package/.docs/reference/voice/voice.addInstructions.md +1 -1
- package/.docs/reference/voice/voice.addTools.md +1 -1
- package/.docs/reference/voice/voice.events.md +1 -1
- package/.docs/reference/voice/voice.getSpeakers.md +2 -2
- package/.docs/reference/voice/voice.on.md +2 -2
- package/.docs/reference/voice/xai-realtime.md +2 -2
- package/.docs/reference/workers/overview.md +85 -0
- package/.docs/reference/workflows/run-methods/cancel.md +1 -1
- package/.docs/reference/workflows/run-methods/startAsync.md +1 -1
- package/.docs/reference/workflows/run-methods/timeTravel.md +1 -1
- package/.docs/reference/workflows/workflow-methods/foreach.md +1 -1
- package/.docs/reference/workflows/workflow-methods/sleep.md +1 -1
- package/.docs/reference/workflows/workflow-state-reader.md +2 -2
- package/.docs/reference/workspace/agentfs-filesystem.md +1 -1
- package/.docs/reference/workspace/apple-container-sandbox.md +2 -2
- package/.docs/reference/workspace/daytona-sandbox.md +3 -3
- package/.docs/reference/workspace/docker-sandbox.md +2 -2
- package/.docs/reference/workspace/e2b-sandbox.md +2 -2
- package/.docs/reference/workspace/files-sdk-filesystem.md +3 -3
- package/.docs/reference/workspace/google-drive-filesystem.md +4 -4
- package/.docs/reference/workspace/local-filesystem.md +3 -3
- package/.docs/reference/workspace/local-sandbox.md +3 -3
- package/.docs/reference/workspace/modal-sandbox.md +1 -1
- package/.docs/reference/workspace/platform-filesystem.md +3 -3
- package/.docs/reference/workspace/platform-sandbox.md +2 -2
- package/.docs/reference/workspace/process-manager.md +1 -1
- package/.docs/reference/workspace/railway-sandbox.md +4 -4
- package/.docs/reference/workspace/s3-filesystem.md +1 -1
- package/.docs/reference/workspace/sandbox.md +1 -1
- package/.docs/reference/workspace/vercel-sandbox.md +1 -1
- package/.docs/reference/workspace/workspace-class.md +10 -10
- package/CHANGELOG.md +21 -0
- package/package.json +3 -3
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The `createContextRecallScorer()` function creates a scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer. It measures retrieval completeness by checking what fraction of the ground truth's claims are attributable to the retrieved context.
|
|
6
6
|
|
|
7
|
-
This scorer requires a ground-truth reference answer, making it suitable for labeled datasets in CI or test environments. When `groundTruth`
|
|
7
|
+
This scorer requires a ground-truth reference answer, making it suitable for labeled datasets in CI or test environments. When `groundTruth` isn't provided in the run, the scorer returns a score of 0 rather than throwing an error.
|
|
8
8
|
|
|
9
9
|
## RAG retrieval evaluation
|
|
10
10
|
|
|
@@ -28,7 +28,7 @@ Use when running evaluations against curated test sets:
|
|
|
28
28
|
|
|
29
29
|
**options** (`ContextRecallMetricOptions`): Configuration options for the scorer
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
Either `context` or `contextExtractor` must be provided. When both are provided, `contextExtractor` is used only if the run input and output are agent-shaped (`MastraDBMessage[]`); otherwise the scorer falls back to `context`.
|
|
32
32
|
|
|
33
33
|
## `.run()` returns
|
|
34
34
|
|
|
@@ -62,11 +62,11 @@ Where:
|
|
|
62
62
|
|
|
63
63
|
These ranges assume the default `scale` of 1. When using a custom scale, multiply accordingly.
|
|
64
64
|
|
|
65
|
-
- **0.9-1.0**: Excellent recall
|
|
66
|
-
- **0.7-0.8**: Good recall
|
|
67
|
-
- **0.4-0.6**: Moderate recall
|
|
68
|
-
- **0.1-0.3**: Poor recall
|
|
69
|
-
- **0.0**: No recall
|
|
65
|
+
- **0.9-1.0**: Excellent recall, context covers nearly all ground-truth claims
|
|
66
|
+
- **0.7-0.8**: Good recall. Most claims are covered, with minor gaps
|
|
67
|
+
- **0.4-0.6**: Moderate recall, substantial information missing from context
|
|
68
|
+
- **0.1-0.3**: Poor recall, most ground-truth claims not found in context
|
|
69
|
+
- **0.0**: No recall, none of the ground-truth claims are in the context
|
|
70
70
|
|
|
71
71
|
### Reason analysis
|
|
72
72
|
|
|
@@ -87,7 +87,7 @@ Use results to:
|
|
|
87
87
|
|
|
88
88
|
### Example calculation
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
ground truth: "Einstein was born in 1879. He developed relativity. He won the Nobel Prize."
|
|
91
91
|
|
|
92
92
|
Claims extracted: 3
|
|
93
93
|
|
|
@@ -103,7 +103,7 @@ Recall = 2/3 = 0.67
|
|
|
103
103
|
|
|
104
104
|
```typescript
|
|
105
105
|
const scorer = createContextRecallScorer({
|
|
106
|
-
model: 'openai/gpt-5.
|
|
106
|
+
model: 'openai/gpt-5.6-sol',
|
|
107
107
|
options: {
|
|
108
108
|
contextExtractor: (input, output) => {
|
|
109
109
|
const query = input?.inputMessages?.[0]?.content || ''
|
|
@@ -119,7 +119,7 @@ const scorer = createContextRecallScorer({
|
|
|
119
119
|
|
|
120
120
|
```typescript
|
|
121
121
|
const scorer = createContextRecallScorer({
|
|
122
|
-
model: 'openai/gpt-5.
|
|
122
|
+
model: 'openai/gpt-5.6-sol',
|
|
123
123
|
options: {
|
|
124
124
|
context: [
|
|
125
125
|
'Document 1: Einstein was born on 14 March 1879 in Ulm, Germany.',
|
|
@@ -140,7 +140,7 @@ import { createContextRecallScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
140
140
|
import { myAgent } from './agent'
|
|
141
141
|
|
|
142
142
|
const scorer = createContextRecallScorer({
|
|
143
|
-
model: 'openai/gpt-5.
|
|
143
|
+
model: 'openai/gpt-5.6-sol',
|
|
144
144
|
options: {
|
|
145
145
|
contextExtractor: (input, output) => {
|
|
146
146
|
// Extract context from tool invocation results in the agent output
|
|
@@ -11,7 +11,7 @@ It's especially useful for these use cases:
|
|
|
11
11
|
Best for evaluating context quality in:
|
|
12
12
|
|
|
13
13
|
- Chat systems where context usage matters
|
|
14
|
-
- RAG pipelines needing
|
|
14
|
+
- RAG pipelines needing detailed relevance assessment
|
|
15
15
|
- Systems where missing context affects quality
|
|
16
16
|
|
|
17
17
|
## Context selection optimization
|
|
@@ -40,7 +40,7 @@ Note: Either `context` or `contextExtractor` must be provided. If both are provi
|
|
|
40
40
|
|
|
41
41
|
### Weighted Relevance Scoring
|
|
42
42
|
|
|
43
|
-
Context Relevance uses a
|
|
43
|
+
Context Relevance uses a advanced scoring algorithm that considers:
|
|
44
44
|
|
|
45
45
|
1. **Relevance Levels**: Each context piece is classified with weighted values:
|
|
46
46
|
|
|
@@ -77,7 +77,7 @@ Final Score = max(0, Base Score - Usage Penalty - Missing Penalty) × scale
|
|
|
77
77
|
|
|
78
78
|
- **0.9-1.0**: Excellent - all context highly relevant and used
|
|
79
79
|
- **0.7-0.8**: Good - mostly relevant with minor gaps
|
|
80
|
-
- **0.4-0.6**: Mixed -
|
|
80
|
+
- **0.4-0.6**: Mixed - substantial irrelevant or unused context
|
|
81
81
|
- **0.2-0.3**: Poor - mostly irrelevant context
|
|
82
82
|
- **0.0-0.1**: Very poor - no relevant context found
|
|
83
83
|
|
|
@@ -121,7 +121,7 @@ import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
|
121
121
|
|
|
122
122
|
// Stricter penalty configuration
|
|
123
123
|
const strictScorer = createContextRelevanceScorerLLM({
|
|
124
|
-
model: 'openai/gpt-5.
|
|
124
|
+
model: 'openai/gpt-5.6-sol',
|
|
125
125
|
options: {
|
|
126
126
|
context: [
|
|
127
127
|
'Einstein won the Nobel Prize for photoelectric effect',
|
|
@@ -139,7 +139,7 @@ const strictScorer = createContextRelevanceScorerLLM({
|
|
|
139
139
|
|
|
140
140
|
// Lenient penalty configuration
|
|
141
141
|
const lenientScorer = createContextRelevanceScorerLLM({
|
|
142
|
-
model: 'openai/gpt-5.
|
|
142
|
+
model: 'openai/gpt-5.6-sol',
|
|
143
143
|
options: {
|
|
144
144
|
context: [
|
|
145
145
|
'Einstein won the Nobel Prize for photoelectric effect',
|
|
@@ -185,7 +185,7 @@ console.log('Lenient penalties:', lenientResult.score) // Higher score, less pen
|
|
|
185
185
|
|
|
186
186
|
```typescript
|
|
187
187
|
const scorer = createContextRelevanceScorerLLM({
|
|
188
|
-
model: 'openai/gpt-5.
|
|
188
|
+
model: 'openai/gpt-5.6-sol',
|
|
189
189
|
options: {
|
|
190
190
|
contextExtractor: (input, output) => {
|
|
191
191
|
// Extract context based on the query
|
|
@@ -209,7 +209,7 @@ const scorer = createContextRelevanceScorerLLM({
|
|
|
209
209
|
|
|
210
210
|
```typescript
|
|
211
211
|
const scorer = createContextRelevanceScorerLLM({
|
|
212
|
-
model: 'openai/gpt-5.
|
|
212
|
+
model: 'openai/gpt-5.6-sol',
|
|
213
213
|
options: {
|
|
214
214
|
context: ['Relevant information...', 'Supporting details...'],
|
|
215
215
|
scale: 100, // Scale scores from 0-100 instead of 0-1
|
|
@@ -223,7 +223,7 @@ const scorer = createContextRelevanceScorerLLM({
|
|
|
223
223
|
|
|
224
224
|
```typescript
|
|
225
225
|
const scorer = createContextRelevanceScorerLLM({
|
|
226
|
-
model: 'openai/gpt-5.
|
|
226
|
+
model: 'openai/gpt-5.6-sol',
|
|
227
227
|
options: {
|
|
228
228
|
contextExtractor: (input, output) => {
|
|
229
229
|
const query = input?.inputMessages?.[0]?.content || ''
|
|
@@ -250,7 +250,7 @@ This example shows excellent context relevance where all context directly suppor
|
|
|
250
250
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
251
251
|
|
|
252
252
|
const scorer = createContextRelevanceScorerLLM({
|
|
253
|
-
model: 'openai/gpt-5.
|
|
253
|
+
model: 'openai/gpt-5.6-sol',
|
|
254
254
|
options: {
|
|
255
255
|
context: [
|
|
256
256
|
'Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1921.',
|
|
@@ -297,7 +297,7 @@ This example shows moderate relevance with some context being irrelevant or unus
|
|
|
297
297
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
298
298
|
|
|
299
299
|
const scorer = createContextRelevanceScorerLLM({
|
|
300
|
-
model: 'openai/gpt-5.
|
|
300
|
+
model: 'openai/gpt-5.6-sol',
|
|
301
301
|
options: {
|
|
302
302
|
context: [
|
|
303
303
|
'Solar eclipses occur when the Moon blocks the Sun.',
|
|
@@ -339,7 +339,7 @@ console.log(result)
|
|
|
339
339
|
|
|
340
340
|
// With custom penalty configuration
|
|
341
341
|
const customScorer = createContextRelevanceScorerLLM({
|
|
342
|
-
model: 'openai/gpt-5.
|
|
342
|
+
model: 'openai/gpt-5.6-sol',
|
|
343
343
|
options: {
|
|
344
344
|
context: [
|
|
345
345
|
'Solar eclipses occur when the Moon blocks the Sun.',
|
|
@@ -386,7 +386,7 @@ This example shows poor context relevance with mostly irrelevant information:
|
|
|
386
386
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
387
387
|
|
|
388
388
|
const scorer = createContextRelevanceScorerLLM({
|
|
389
|
-
model: 'openai/gpt-5.
|
|
389
|
+
model: 'openai/gpt-5.6-sol',
|
|
390
390
|
options: {
|
|
391
391
|
context: [
|
|
392
392
|
'The Great Barrier Reef is located in Australia.',
|
|
@@ -428,13 +428,13 @@ console.log(result)
|
|
|
428
428
|
|
|
429
429
|
### Dynamic context extraction
|
|
430
430
|
|
|
431
|
-
Extract context
|
|
431
|
+
Extract context at runtime based on the run input:
|
|
432
432
|
|
|
433
433
|
```typescript
|
|
434
434
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
435
435
|
|
|
436
436
|
const scorer = createContextRelevanceScorerLLM({
|
|
437
|
-
model: 'openai/gpt-5.
|
|
437
|
+
model: 'openai/gpt-5.6-sol',
|
|
438
438
|
options: {
|
|
439
439
|
contextExtractor: (input, output) => {
|
|
440
440
|
// Extract query from input
|
|
@@ -477,7 +477,7 @@ Integrate with RAG pipelines to evaluate retrieved context:
|
|
|
477
477
|
import { createContextRelevanceScorerLLM } from '@mastra/evals'
|
|
478
478
|
|
|
479
479
|
const scorer = createContextRelevanceScorerLLM({
|
|
480
|
-
model: 'openai/gpt-5.
|
|
480
|
+
model: 'openai/gpt-5.6-sol',
|
|
481
481
|
options: {
|
|
482
482
|
contextExtractor: (input, output) => {
|
|
483
483
|
// Extract from RAG retrieval results
|
|
@@ -110,7 +110,7 @@ export const responseQuality = createScorer({
|
|
|
110
110
|
|
|
111
111
|
With this configuration, a failed request has at most three provider attempts: the initial request plus two processor retries. If `generateScore` completes and `generateReason` receives a retryable failure, only `generateReason` retries.
|
|
112
112
|
|
|
113
|
-
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default
|
|
113
|
+
`StreamErrorRetryProcessor` honors retryable provider metadata and narrow custom matchers. It keeps `retryUnknownErrors` disabled by default, so authentication, invalid-request, and context-length errors fail immediately unless you explicitly match them. It bounds `Retry-After` values to `30_000` milliseconds by default. Use `maxRetryAfterMs` to change that cap.
|
|
114
114
|
|
|
115
115
|
Avoid adding outer scorer or workflow retries. Avoid combining a nonzero model retry setting with this processor unless you intentionally accept additional attempts.
|
|
116
116
|
|
|
@@ -65,9 +65,9 @@ Final score: `(supported_claims / total_claims) * scale`
|
|
|
65
65
|
A faithfulness score between 0 and 1:
|
|
66
66
|
|
|
67
67
|
- **1.0**: All claims are accurate and directly supported by the context.
|
|
68
|
-
- **0.7
|
|
69
|
-
- **0.4
|
|
70
|
-
- **0.1
|
|
68
|
+
- **0.7 to 0.9**: Most claims are correct, with minor additions or omissions.
|
|
69
|
+
- **0.4 to 0.6**: Some claims are supported, but others are unverifiable.
|
|
70
|
+
- **0.1 to 0.3**: Most of the content is inaccurate or unsupported.
|
|
71
71
|
- **0.0**: All claims are false or contradict the context.
|
|
72
72
|
|
|
73
73
|
## Example
|
|
@@ -81,7 +81,7 @@ import { myAgent } from './agent'
|
|
|
81
81
|
|
|
82
82
|
// Context is typically populated from agent tool calls or RAG retrieval
|
|
83
83
|
const scorer = createFaithfulnessScorer({
|
|
84
|
-
model: 'openai/gpt-5.
|
|
84
|
+
model: 'openai/gpt-5.6-sol',
|
|
85
85
|
})
|
|
86
86
|
|
|
87
87
|
const result = await runEvals({
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# filterRun()
|
|
4
4
|
|
|
5
|
-
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages
|
|
5
|
+
Creates a `prepareRun` function from declarative options. Pass the result to `createScorer()` to filter messages and limit context size. It also drops unnecessary fields before the scorer pipeline runs.
|
|
6
6
|
|
|
7
7
|
Use [`filterRun()`](#usage-example) for declarative filtering. Write a custom `prepareRun` function directly when you need imperative logic that `filterRun()` doesn't cover. See [Custom scorers: input filtering](https://mastra.ai/docs/evals/custom-scorers) for more.
|
|
8
8
|
|
|
@@ -88,7 +88,7 @@ const simpleScorer = createScorer({
|
|
|
88
88
|
|
|
89
89
|
**options.dropGroundTruth** (`boolean`): Remove ground truth from the run.
|
|
90
90
|
|
|
91
|
-
**Returns:** `(run: ScorerRun) => ScorerRun
|
|
91
|
+
**Returns:** `(run: ScorerRun) => ScorerRun`. A function suitable for the `prepareRun` option on [`createScorer()`](https://mastra.ai/reference/evals/create-scorer).
|
|
92
92
|
|
|
93
93
|
## Part types
|
|
94
94
|
|
|
@@ -88,13 +88,13 @@ Final score: `(hallucinated_statements / total_statements) * scale`
|
|
|
88
88
|
|
|
89
89
|
A hallucination score between 0 and 1:
|
|
90
90
|
|
|
91
|
-
- **0.0**: No hallucination
|
|
92
|
-
- **0.3
|
|
93
|
-
- **0.5
|
|
94
|
-
- **0.7
|
|
95
|
-
- **0.9
|
|
91
|
+
- **0.0**: No hallucination, all claims match the context.
|
|
92
|
+
- **0.3 to 0.4**: Low hallucination, a few contradictions.
|
|
93
|
+
- **0.5 to 0.6**: Mixed hallucination, several contradictions.
|
|
94
|
+
- **0.7 to 0.8**: High hallucination, many contradictions.
|
|
95
|
+
- **0.9 to 1.0**: Complete hallucination, most or all claims contradict the context.
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
The score represents the degree of hallucination - lower scores indicate better factual alignment with the provided context
|
|
98
98
|
|
|
99
99
|
## Examples
|
|
100
100
|
|
|
@@ -106,7 +106,7 @@ Use static context when you have known ground truth to compare against:
|
|
|
106
106
|
import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
107
107
|
|
|
108
108
|
const scorer = createHallucinationScorer({
|
|
109
|
-
model: 'openai/gpt-5.
|
|
109
|
+
model: 'openai/gpt-5.6-sol',
|
|
110
110
|
options: {
|
|
111
111
|
context: [
|
|
112
112
|
'The first iPhone was announced on January 9, 2007.',
|
|
@@ -126,7 +126,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
126
126
|
import { extractToolResults } from '@mastra/evals/scorers'
|
|
127
127
|
|
|
128
128
|
const scorer = createHallucinationScorer({
|
|
129
|
-
model: 'openai/gpt-5.
|
|
129
|
+
model: 'openai/gpt-5.6-sol',
|
|
130
130
|
options: {
|
|
131
131
|
getContext: ({ run, step }) => {
|
|
132
132
|
// Extract tool results as context
|
|
@@ -147,7 +147,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
147
147
|
import { extractToolResults } from '@mastra/evals/scorers'
|
|
148
148
|
|
|
149
149
|
const hallucinationScorer = createHallucinationScorer({
|
|
150
|
-
model: 'openai/gpt-5.
|
|
150
|
+
model: 'openai/gpt-5.6-sol',
|
|
151
151
|
options: {
|
|
152
152
|
getContext: ({ run }) => {
|
|
153
153
|
const toolResults = extractToolResults(run.output)
|
|
@@ -159,7 +159,7 @@ const hallucinationScorer = createHallucinationScorer({
|
|
|
159
159
|
const agent = new Agent({
|
|
160
160
|
id: 'my-agent',
|
|
161
161
|
name: 'my-agent',
|
|
162
|
-
model: 'openai/gpt-5.
|
|
162
|
+
model: 'openai/gpt-5.6-sol',
|
|
163
163
|
instructions: 'You are a helpful assistant.',
|
|
164
164
|
evals: {
|
|
165
165
|
scorers: [hallucinationScorer],
|
|
@@ -175,7 +175,7 @@ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt'
|
|
|
175
175
|
import { myAgent } from './agent'
|
|
176
176
|
|
|
177
177
|
const scorer = createHallucinationScorer({
|
|
178
|
-
model: 'openai/gpt-5.
|
|
178
|
+
model: 'openai/gpt-5.6-sol',
|
|
179
179
|
options: {
|
|
180
180
|
context: ['Known fact 1', 'Known fact 2'],
|
|
181
181
|
},
|
|
@@ -66,11 +66,11 @@ Final score: `(matched_keywords / total_keywords) * scale`
|
|
|
66
66
|
|
|
67
67
|
A coverage score between 0 and 1:
|
|
68
68
|
|
|
69
|
-
- **1.0**: Complete coverage
|
|
70
|
-
- **0.7
|
|
71
|
-
- **0.4
|
|
72
|
-
- **0.1
|
|
73
|
-
- **0.0**:
|
|
69
|
+
- **1.0**: Complete coverage, all keywords present.
|
|
70
|
+
- **0.7 to 0.9**: High coverage, most keywords included.
|
|
71
|
+
- **0.4 to 0.6**: Partial coverage, some keywords present.
|
|
72
|
+
- **0.1 to 0.3**: Low coverage, few keywords matched.
|
|
73
|
+
- **0.0**: The response contains none of the keywords.
|
|
74
74
|
|
|
75
75
|
### Special Cases
|
|
76
76
|
|
|
@@ -80,7 +80,7 @@ The scorer handles several special cases:
|
|
|
80
80
|
- Single word: Treated as a single keyword
|
|
81
81
|
- Technical terms: Preserves compound technical terms (e.g., "React.js", "machine learning")
|
|
82
82
|
- Case differences: "JavaScript" matches "javascript"
|
|
83
|
-
- Common words: Ignored in scoring to focus on
|
|
83
|
+
- Common words: Ignored in scoring to focus on useful keywords
|
|
84
84
|
|
|
85
85
|
## Example
|
|
86
86
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The `MastraScorer` class is the base class for all scorers in Mastra. It provides a standard `.run()` method for evaluating input/output pairs and supports multi-step scoring workflows with preprocess → analyze → generateScore → generateReason execution flow.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Most users should use [`createScorer`](https://mastra.ai/reference/evals/create-scorer) to create scorer instances. Direct instantiation of `MastraScorer` isn't recommended.
|
|
8
8
|
|
|
9
9
|
## How to get a `MastraScorer` instance
|
|
10
10
|
|
|
@@ -26,7 +26,7 @@ const scorer = createScorer({
|
|
|
26
26
|
|
|
27
27
|
## `.run()` method
|
|
28
28
|
|
|
29
|
-
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a
|
|
29
|
+
The `.run()` method is the primary way to execute your scorer and evaluate input/output pairs. It processes the data through your defined steps (preprocess → analyze → generateScore → generateReason) and returns a detailed result object with the score, reasoning, and intermediate results.
|
|
30
30
|
|
|
31
31
|
```typescript
|
|
32
32
|
const result = await scorer.run({
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
# Noise sensitivity scorer
|
|
4
4
|
|
|
5
|
-
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how
|
|
5
|
+
The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how reliable an agent is when exposed to irrelevant, distracting, or misleading information. Unlike live scorers that evaluate single production runs, this scorer requires predetermined test data including both baseline responses and noisy variations.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
This isn't a live scorer. It requires pre-computed baseline responses and can't be used for real-time agent evaluation. Use this scorer in your CI/CD pipeline or testing suites only.
|
|
8
8
|
|
|
9
9
|
Before using the noise sensitivity scorer, prepare your test data:
|
|
10
10
|
|
|
@@ -63,7 +63,7 @@ describe('Agent Noise Resistance Tests', () => {
|
|
|
63
63
|
|
|
64
64
|
// Step 4: Evaluate using noise sensitivity scorer
|
|
65
65
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
66
|
-
model: 'openai/gpt-5.
|
|
66
|
+
model: 'openai/gpt-5.6-sol',
|
|
67
67
|
options: {
|
|
68
68
|
baselineResponse,
|
|
69
69
|
noisyQuery,
|
|
@@ -177,7 +177,7 @@ Include in your test harness to:
|
|
|
177
177
|
|
|
178
178
|
- Benchmark different models' noise resistance before deployment
|
|
179
179
|
- Identify agents vulnerable to manipulation during development
|
|
180
|
-
- Create
|
|
180
|
+
- Create detailed test coverage for various noise types
|
|
181
181
|
- Ensure consistent behavior across updates
|
|
182
182
|
|
|
183
183
|
### Security Testing
|
|
@@ -194,7 +194,7 @@ Evaluate resistance in controlled environments:
|
|
|
194
194
|
- **1.0**: Perfect robustness - no impact detected
|
|
195
195
|
- **0.8-0.9**: Excellent - minimal impact, core functionality preserved
|
|
196
196
|
- **0.6-0.7**: Good - some impact but acceptable for most use cases
|
|
197
|
-
- **0.4-0.5**: Concerning -
|
|
197
|
+
- **0.4-0.5**: Concerning - substantial vulnerabilities detected
|
|
198
198
|
- **0.0-0.3**: Critical - agent severely compromised by noise
|
|
199
199
|
|
|
200
200
|
### Dimension analysis
|
|
@@ -258,7 +258,7 @@ describe('Agent Noise Resistance CI Tests', () => {
|
|
|
258
258
|
|
|
259
259
|
// Evaluate using noise sensitivity scorer
|
|
260
260
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
261
|
-
model: 'openai/gpt-5.
|
|
261
|
+
model: 'openai/gpt-5.6-sol',
|
|
262
262
|
options: {
|
|
263
263
|
baselineResponse: testCase.baselineResponse,
|
|
264
264
|
noisyQuery: testCase.noisyQuery,
|
|
@@ -293,7 +293,7 @@ This example shows an agent that completely resists misinformation in a test sce
|
|
|
293
293
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
294
294
|
|
|
295
295
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
296
|
-
model: 'openai/gpt-5.
|
|
296
|
+
model: 'openai/gpt-5.6-sol',
|
|
297
297
|
options: {
|
|
298
298
|
baselineResponse:
|
|
299
299
|
'Regular exercise improves cardiovascular health, strengthens muscles, and enhances mental wellbeing.',
|
|
@@ -339,7 +339,7 @@ This example shows an agent partially distracted by irrelevant requests:
|
|
|
339
339
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals/scorers/prebuilt'
|
|
340
340
|
|
|
341
341
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
342
|
-
model: 'openai/gpt-5.
|
|
342
|
+
model: 'openai/gpt-5.6-sol',
|
|
343
343
|
options: {
|
|
344
344
|
baselineResponse:
|
|
345
345
|
'To bake a cake: Mix flour, sugar, eggs, and butter. Bake at 350°F for 30 minutes.',
|
|
@@ -384,7 +384,7 @@ This example shows an agent that incorporates misinformation:
|
|
|
384
384
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
385
385
|
|
|
386
386
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
387
|
-
model: 'openai/gpt-5.
|
|
387
|
+
model: 'openai/gpt-5.6-sol',
|
|
388
388
|
options: {
|
|
389
389
|
baselineResponse: 'Climate change is caused by greenhouse gas emissions from human activities.',
|
|
390
390
|
noisyQuery:
|
|
@@ -430,7 +430,7 @@ import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
|
430
430
|
|
|
431
431
|
// Lenient scoring - more forgiving of minor issues
|
|
432
432
|
const lenientScorer = createNoiseSensitivityScorerLLM({
|
|
433
|
-
model: 'openai/gpt-5.
|
|
433
|
+
model: 'openai/gpt-5.6-sol',
|
|
434
434
|
options: {
|
|
435
435
|
baselineResponse: 'Python is a high-level programming language.',
|
|
436
436
|
noisyQuery: 'What is Python? Also, snakes are dangerous!',
|
|
@@ -450,7 +450,7 @@ const lenientScorer = createNoiseSensitivityScorerLLM({
|
|
|
450
450
|
|
|
451
451
|
// Strict scoring - harsh on any deviation
|
|
452
452
|
const strictScorer = createNoiseSensitivityScorerLLM({
|
|
453
|
-
model: 'openai/gpt-5.
|
|
453
|
+
model: 'openai/gpt-5.6-sol',
|
|
454
454
|
options: {
|
|
455
455
|
baselineResponse: 'Python is a high-level programming language.',
|
|
456
456
|
noisyQuery: 'What is Python? Also, snakes are dangerous!',
|
|
@@ -472,7 +472,7 @@ const strictScorer = createNoiseSensitivityScorerLLM({
|
|
|
472
472
|
|
|
473
473
|
## CI test suite: Testing different noise types
|
|
474
474
|
|
|
475
|
-
Create
|
|
475
|
+
Create detailed test suites to evaluate agent performance across noise categories in your CI pipeline:
|
|
476
476
|
|
|
477
477
|
```typescript
|
|
478
478
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
@@ -501,7 +501,7 @@ async function evaluateNoiseResistance(testCases) {
|
|
|
501
501
|
|
|
502
502
|
for (const testCase of testCases) {
|
|
503
503
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
504
|
-
model: 'openai/gpt-5.
|
|
504
|
+
model: 'openai/gpt-5.6-sol',
|
|
505
505
|
options: {
|
|
506
506
|
baselineResponse: testCase.baseline,
|
|
507
507
|
noisyQuery: testCase.noisyQuery,
|
|
@@ -548,7 +548,7 @@ import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
|
548
548
|
|
|
549
549
|
async function compareModelRobustness() {
|
|
550
550
|
const models = [
|
|
551
|
-
{ name: 'GPT-5.4', model: 'openai/gpt-5.
|
|
551
|
+
{ name: 'GPT-5.4', model: 'openai/gpt-5.6-sol' },
|
|
552
552
|
{ name: 'GPT-5.4-mini', model: 'openai/gpt-5-mini' },
|
|
553
553
|
{ name: 'Claude', model: 'anthropic/claude-opus-4-7' },
|
|
554
554
|
]
|
|
@@ -600,7 +600,7 @@ Include noise sensitivity tests in your security test suite to validate prompt i
|
|
|
600
600
|
import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
601
601
|
|
|
602
602
|
const scorer = createNoiseSensitivityScorerLLM({
|
|
603
|
-
model: 'openai/gpt-5.
|
|
603
|
+
model: 'openai/gpt-5.6-sol',
|
|
604
604
|
options: {
|
|
605
605
|
baselineResponse: 'I can help you with programming questions.',
|
|
606
606
|
noisyQuery:
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# Prompt alignment scorer
|
|
4
4
|
|
|
5
|
-
The `createPromptAlignmentScorerLLM()` function creates a scorer that evaluates how well agent responses align with user prompts across
|
|
5
|
+
The `createPromptAlignmentScorerLLM()` function creates a scorer that evaluates how well agent responses align with user prompts across intent understanding and requirement fulfillment, plus response completeness, and format appropriateness.
|
|
6
6
|
|
|
7
7
|
## Parameters
|
|
8
8
|
|
|
@@ -62,7 +62,7 @@ You can customize the Prompt Alignment Scorer by adjusting the scale parameter a
|
|
|
62
62
|
|
|
63
63
|
```typescript
|
|
64
64
|
const scorer = createPromptAlignmentScorerLLM({
|
|
65
|
-
model: 'openai/gpt-5.
|
|
65
|
+
model: 'openai/gpt-5.6-sol',
|
|
66
66
|
options: {
|
|
67
67
|
scale: 10, // Score from 0-10 instead of 0-1
|
|
68
68
|
evaluationMode: 'both', // 'user', 'system', or 'both' (default)
|
|
@@ -80,7 +80,7 @@ Evaluates alignment with user prompts only:
|
|
|
80
80
|
|
|
81
81
|
1. **Intent Alignment** (40% weight): Whether the response addresses the user's core request
|
|
82
82
|
2. **Requirements Fulfillment** (30% weight): If all user requirements are met
|
|
83
|
-
3. **Completeness** (20% weight): Whether the response is
|
|
83
|
+
3. **Completeness** (20% weight): Whether the response is detailed for user needs
|
|
84
84
|
4. **Response Appropriateness** (10% weight): If format and tone match user expectations
|
|
85
85
|
|
|
86
86
|
#### System Mode ('system')
|
|
@@ -139,7 +139,7 @@ Final Score = Weighted Score × scale
|
|
|
139
139
|
- **0.8-0.9** = Very good alignment with minor gaps
|
|
140
140
|
- **0.7-0.8** = Good alignment but missing some requirements or completeness
|
|
141
141
|
- **0.6-0.7** = Moderate alignment with noticeable gaps
|
|
142
|
-
- **0.4-0.6** = Poor alignment with
|
|
142
|
+
- **0.4-0.6** = Poor alignment with substantial issues
|
|
143
143
|
- **0.0-0.4** = Very poor alignment, response doesn't address the prompt effectively
|
|
144
144
|
|
|
145
145
|
### When to Use Each Mode
|
|
@@ -160,7 +160,7 @@ Final Score = Weighted Score × scale
|
|
|
160
160
|
|
|
161
161
|
**Both Mode (`'both'`)** - Use when (default, recommended):
|
|
162
162
|
|
|
163
|
-
- Comprehensive evaluation of
|
|
163
|
+
- Comprehensive evaluation of AI agent performance
|
|
164
164
|
- Balancing user satisfaction with system compliance
|
|
165
165
|
- Production monitoring where both user and system requirements matter
|
|
166
166
|
- Holistic assessment of prompt-response alignment
|
|
@@ -224,24 +224,24 @@ const agent = new Agent({
|
|
|
224
224
|
id: 'coding-assistant',
|
|
225
225
|
name: 'CodingAssistant',
|
|
226
226
|
instructions: 'You are a helpful coding assistant. Always provide working code examples.',
|
|
227
|
-
model: 'openai/gpt-5.
|
|
227
|
+
model: 'openai/gpt-5.6-sol',
|
|
228
228
|
})
|
|
229
229
|
|
|
230
230
|
// Evaluate comprehensive alignment (default)
|
|
231
231
|
const scorer = createPromptAlignmentScorerLLM({
|
|
232
|
-
model: 'openai/gpt-5.
|
|
232
|
+
model: 'openai/gpt-5.6-sol',
|
|
233
233
|
options: { evaluationMode: 'both' }, // Evaluates both user intent and system guidelines
|
|
234
234
|
})
|
|
235
235
|
|
|
236
236
|
// Evaluate just user satisfaction
|
|
237
237
|
const userScorer = createPromptAlignmentScorerLLM({
|
|
238
|
-
model: 'openai/gpt-5.
|
|
238
|
+
model: 'openai/gpt-5.6-sol',
|
|
239
239
|
options: { evaluationMode: 'user' }, // Focus only on user request fulfillment
|
|
240
240
|
})
|
|
241
241
|
|
|
242
242
|
// Evaluate system compliance
|
|
243
243
|
const systemScorer = createPromptAlignmentScorerLLM({
|
|
244
|
-
model: 'openai/gpt-5.
|
|
244
|
+
model: 'openai/gpt-5.6-sol',
|
|
245
245
|
options: { evaluationMode: 'system' }, // Check adherence to system instructions
|
|
246
246
|
})
|
|
247
247
|
|
|
@@ -293,7 +293,7 @@ for (const agent of agents) {
|
|
|
293
293
|
import { createPromptAlignmentScorerLLM } from '@mastra/evals'
|
|
294
294
|
|
|
295
295
|
const scorer = createPromptAlignmentScorerLLM({
|
|
296
|
-
model: 'openai/gpt-5.
|
|
296
|
+
model: 'openai/gpt-5.6-sol',
|
|
297
297
|
})
|
|
298
298
|
|
|
299
299
|
// Evaluate a code generation task
|
|
@@ -322,7 +322,7 @@ const result = await scorer.run({
|
|
|
322
322
|
```typescript
|
|
323
323
|
// Configure scale and evaluation mode
|
|
324
324
|
const scorer = createPromptAlignmentScorerLLM({
|
|
325
|
-
model: 'openai/gpt-5.
|
|
325
|
+
model: 'openai/gpt-5.6-sol',
|
|
326
326
|
options: {
|
|
327
327
|
scale: 10, // Score from 0-10 instead of 0-1
|
|
328
328
|
evaluationMode: 'both', // 'user', 'system', or 'both' (default)
|
|
@@ -331,13 +331,13 @@ const scorer = createPromptAlignmentScorerLLM({
|
|
|
331
331
|
|
|
332
332
|
// User-only evaluation - focus on user satisfaction
|
|
333
333
|
const userScorer = createPromptAlignmentScorerLLM({
|
|
334
|
-
model: 'openai/gpt-5.
|
|
334
|
+
model: 'openai/gpt-5.6-sol',
|
|
335
335
|
options: { evaluationMode: 'user' },
|
|
336
336
|
})
|
|
337
337
|
|
|
338
338
|
// System-only evaluation - focus on compliance
|
|
339
339
|
const systemScorer = createPromptAlignmentScorerLLM({
|
|
340
|
-
model: 'openai/gpt-5.
|
|
340
|
+
model: 'openai/gpt-5.6-sol',
|
|
341
341
|
options: { evaluationMode: 'system' },
|
|
342
342
|
})
|
|
343
343
|
|
|
@@ -372,7 +372,7 @@ In this example, the response fully addresses the user's prompt with all require
|
|
|
372
372
|
import { createPromptAlignmentScorerLLM } from '@mastra/evals/scorers/prebuilt'
|
|
373
373
|
|
|
374
374
|
const scorer = createPromptAlignmentScorerLLM({
|
|
375
|
-
model: 'openai/gpt-5.
|
|
375
|
+
model: 'openai/gpt-5.6-sol',
|
|
376
376
|
})
|
|
377
377
|
|
|
378
378
|
const inputMessages = [
|
|
@@ -403,7 +403,7 @@ console.log(result)
|
|
|
403
403
|
|
|
404
404
|
### Excellent alignment output
|
|
405
405
|
|
|
406
|
-
The output receives a high score because it perfectly addresses the intent
|
|
406
|
+
The output receives a high score because it perfectly addresses the intent and fulfills all requirements. It also uses the appropriate format.
|
|
407
407
|
|
|
408
408
|
```typescript
|
|
409
409
|
{
|
|
@@ -420,7 +420,7 @@ In this example, the response addresses the core intent but misses some requirem
|
|
|
420
420
|
import { createPromptAlignmentScorerLLM } from '@mastra/evals/scorers/prebuilt'
|
|
421
421
|
|
|
422
422
|
const scorer = createPromptAlignmentScorerLLM({
|
|
423
|
-
model: 'openai/gpt-5.
|
|
423
|
+
model: 'openai/gpt-5.6-sol',
|
|
424
424
|
})
|
|
425
425
|
|
|
426
426
|
const inputMessages = [
|
|
@@ -461,7 +461,7 @@ In this example, the response fails to address the user's specific requirements.
|
|
|
461
461
|
import { createPromptAlignmentScorerLLM } from '@mastra/evals/scorers/prebuilt'
|
|
462
462
|
|
|
463
463
|
const scorer = createPromptAlignmentScorerLLM({
|
|
464
|
-
model: 'openai/gpt-5.
|
|
464
|
+
model: 'openai/gpt-5.6-sol',
|
|
465
465
|
})
|
|
466
466
|
|
|
467
467
|
const inputMessages = [
|
|
@@ -505,7 +505,7 @@ Evaluates how well the response addresses the user's request, ignoring system in
|
|
|
505
505
|
|
|
506
506
|
```typescript
|
|
507
507
|
const scorer = createPromptAlignmentScorerLLM({
|
|
508
|
-
model: 'openai/gpt-5.
|
|
508
|
+
model: 'openai/gpt-5.6-sol',
|
|
509
509
|
options: { evaluationMode: 'user' },
|
|
510
510
|
})
|
|
511
511
|
|
|
@@ -537,7 +537,7 @@ Evaluates compliance with system behavioral guidelines and constraints:
|
|
|
537
537
|
|
|
538
538
|
```typescript
|
|
539
539
|
const scorer = createPromptAlignmentScorerLLM({
|
|
540
|
-
model: 'openai/gpt-5.
|
|
540
|
+
model: 'openai/gpt-5.6-sol',
|
|
541
541
|
options: { evaluationMode: 'system' },
|
|
542
542
|
})
|
|
543
543
|
|
|
@@ -569,7 +569,7 @@ Evaluates both user intent fulfillment and system compliance with weighted scori
|
|
|
569
569
|
|
|
570
570
|
```typescript
|
|
571
571
|
const scorer = createPromptAlignmentScorerLLM({
|
|
572
|
-
model: 'openai/gpt-5.
|
|
572
|
+
model: 'openai/gpt-5.6-sol',
|
|
573
573
|
options: { evaluationMode: 'both' }, // This is the default
|
|
574
574
|
})
|
|
575
575
|
|