@mastra/mcp-docs-server 0.13.39 → 0.13.40-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/organized/changelogs/%40mastra%2Fai-sdk.md +18 -18
- package/.docs/organized/changelogs/%40mastra%2Fclient-js.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fcore.md +32 -32
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-cloud.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-cloudflare.md +9 -9
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-netlify.md +9 -9
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-vercel.md +9 -9
- package/.docs/organized/changelogs/%40mastra%2Fdeployer.md +13 -13
- package/.docs/organized/changelogs/%40mastra%2Fevals.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fmcp-docs-server.md +8 -8
- package/.docs/organized/changelogs/%40mastra%2Fmssql.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fplayground-ui.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Freact.md +8 -8
- package/.docs/organized/changelogs/%40mastra%2Fserver.md +8 -8
- package/.docs/organized/changelogs/create-mastra.md +3 -3
- package/.docs/organized/changelogs/mastra.md +13 -13
- package/.docs/raw/agents/adding-voice.mdx +1 -1
- package/.docs/raw/agents/agent-memory.mdx +1 -1
- package/.docs/raw/agents/guardrails.mdx +1 -1
- package/.docs/raw/agents/networks.mdx +1 -1
- package/.docs/raw/agents/overview.mdx +120 -64
- package/.docs/raw/agents/using-tools.mdx +138 -15
- package/.docs/raw/auth/auth0.mdx +1 -1
- package/.docs/raw/auth/clerk.mdx +1 -1
- package/.docs/raw/auth/firebase.mdx +1 -1
- package/.docs/raw/auth/index.mdx +1 -1
- package/.docs/raw/auth/jwt.mdx +1 -1
- package/.docs/raw/auth/supabase.mdx +1 -1
- package/.docs/raw/auth/workos.mdx +1 -1
- package/.docs/raw/community/contributing-templates.mdx +1 -1
- package/.docs/raw/community/discord.mdx +2 -2
- package/.docs/raw/community/licensing.mdx +1 -1
- package/.docs/raw/course/02-agent-tools-mcp/04-initializing-mcp-tools.md +2 -2
- package/.docs/raw/deployment/{server-deployment.mdx → building-mastra.mdx} +3 -3
- package/.docs/raw/deployment/cloud-providers/amazon-ec2.mdx +1 -1
- package/.docs/raw/deployment/cloud-providers/aws-lambda.mdx +2 -2
- package/.docs/raw/deployment/cloud-providers/azure-app-services.mdx +1 -1
- package/.docs/raw/deployment/{serverless-platforms → cloud-providers}/cloudflare-deployer.mdx +1 -1
- package/.docs/raw/deployment/cloud-providers/digital-ocean.mdx +1 -1
- package/.docs/raw/deployment/cloud-providers/index.mdx +6 -3
- package/.docs/raw/deployment/{serverless-platforms → cloud-providers}/netlify-deployer.mdx +1 -1
- package/.docs/raw/deployment/{serverless-platforms → cloud-providers}/vercel-deployer.mdx +1 -1
- package/.docs/raw/{mastra-cloud → deployment/mastra-cloud}/dashboard.mdx +3 -3
- package/.docs/raw/{mastra-cloud → deployment/mastra-cloud}/observability.mdx +2 -2
- package/.docs/raw/{mastra-cloud → deployment/mastra-cloud}/overview.mdx +3 -3
- package/.docs/raw/{mastra-cloud → deployment/mastra-cloud}/setting-up.mdx +3 -3
- package/.docs/raw/deployment/monorepo.mdx +3 -3
- package/.docs/raw/deployment/overview.mdx +5 -5
- package/.docs/raw/deployment/web-framework.mdx +1 -1
- package/.docs/raw/frameworks/agentic-uis/ai-sdk.mdx +5 -1
- package/.docs/raw/frameworks/agentic-uis/assistant-ui.mdx +1 -1
- package/.docs/raw/frameworks/agentic-uis/cedar-os.mdx +1 -1
- package/.docs/raw/frameworks/agentic-uis/copilotkit.mdx +1 -1
- package/.docs/raw/frameworks/agentic-uis/openrouter.mdx +1 -1
- package/.docs/raw/frameworks/servers/express.mdx +1 -1
- package/.docs/raw/frameworks/web-frameworks/astro.mdx +1 -1
- package/.docs/raw/frameworks/web-frameworks/next-js.mdx +1 -1
- package/.docs/raw/frameworks/web-frameworks/sveltekit.mdx +1 -1
- package/.docs/raw/frameworks/web-frameworks/vite-react.mdx +1 -1
- package/.docs/raw/getting-started/installation.mdx +1 -1
- package/.docs/raw/getting-started/mcp-docs-server.mdx +1 -1
- package/.docs/raw/getting-started/project-structure.mdx +1 -1
- package/.docs/raw/getting-started/studio.mdx +2 -2
- package/.docs/raw/getting-started/templates.mdx +1 -1
- package/.docs/raw/index.mdx +2 -2
- package/.docs/raw/{tools-mcp/mcp-overview.mdx → mcp/overview.mdx} +2 -2
- package/.docs/raw/mcp/publishing-mcp-server.mdx +137 -0
- package/.docs/raw/memory/conversation-history.mdx +1 -1
- package/.docs/raw/memory/memory-processors.mdx +1 -1
- package/.docs/raw/memory/overview.mdx +2 -2
- package/.docs/raw/memory/semantic-recall.mdx +2 -2
- package/.docs/raw/memory/storage/memory-with-libsql.mdx +1 -1
- package/.docs/raw/memory/storage/memory-with-mongodb.mdx +150 -0
- package/.docs/raw/memory/storage/memory-with-pg.mdx +1 -1
- package/.docs/raw/memory/storage/memory-with-upstash.mdx +1 -1
- package/.docs/raw/memory/threads-and-resources.mdx +1 -1
- package/.docs/raw/memory/working-memory.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/arize.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/braintrust.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/cloud.mdx +2 -2
- package/.docs/raw/observability/ai-tracing/exporters/default.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/langfuse.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/langsmith.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/exporters/otel.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/overview.mdx +1 -1
- package/.docs/raw/observability/ai-tracing/processors/sensitive-data-filter.mdx +1 -1
- package/.docs/raw/observability/logging.mdx +5 -5
- package/.docs/raw/observability/nextjs-tracing.mdx +1 -1
- package/.docs/raw/observability/otel-tracing.mdx +1 -1
- package/.docs/raw/observability/overview.mdx +1 -1
- package/.docs/raw/rag/overview.mdx +1 -1
- package/.docs/raw/rag/retrieval.mdx +1 -1
- package/.docs/raw/rag/vector-databases.mdx +1 -1
- package/.docs/raw/reference/agents/agent.mdx +1 -1
- package/.docs/raw/reference/agents/generate.mdx +1 -1
- package/.docs/raw/reference/agents/generateLegacy.mdx +1 -1
- package/.docs/raw/reference/agents/getDefaultGenerateOptions.mdx +1 -1
- package/.docs/raw/reference/agents/getDefaultStreamOptions.mdx +1 -1
- package/.docs/raw/reference/agents/getDescription.mdx +1 -1
- package/.docs/raw/reference/agents/getInstructions.mdx +1 -1
- package/.docs/raw/reference/agents/getLLM.mdx +1 -1
- package/.docs/raw/reference/agents/getMemory.mdx +1 -1
- package/.docs/raw/reference/agents/getModel.mdx +1 -1
- package/.docs/raw/reference/agents/getScorers.mdx +1 -1
- package/.docs/raw/reference/agents/getTools.mdx +2 -2
- package/.docs/raw/reference/agents/getVoice.mdx +1 -1
- package/.docs/raw/reference/agents/getWorkflows.mdx +1 -1
- package/.docs/raw/reference/agents/listAgents.mdx +1 -1
- package/.docs/raw/reference/agents/listScorers.mdx +1 -1
- package/.docs/raw/reference/agents/listWorkflows.mdx +1 -1
- package/.docs/raw/reference/agents/network.mdx +1 -1
- package/.docs/raw/reference/auth/auth0.mdx +1 -1
- package/.docs/raw/reference/auth/clerk.mdx +1 -1
- package/.docs/raw/reference/auth/firebase.mdx +1 -1
- package/.docs/raw/reference/auth/jwt.mdx +1 -1
- package/.docs/raw/reference/auth/supabase.mdx +1 -1
- package/.docs/raw/reference/auth/workos.mdx +1 -1
- package/.docs/raw/reference/cli/create-mastra.mdx +1 -1
- package/.docs/raw/reference/cli/mastra.mdx +2 -2
- package/.docs/raw/reference/client-js/agents.mdx +2 -2
- package/.docs/raw/reference/client-js/error-handling.mdx +1 -1
- package/.docs/raw/reference/client-js/logs.mdx +1 -1
- package/.docs/raw/reference/client-js/mastra-client.mdx +5 -5
- package/.docs/raw/reference/client-js/memory.mdx +1 -1
- package/.docs/raw/reference/client-js/observability.mdx +1 -1
- package/.docs/raw/reference/client-js/telemetry.mdx +1 -1
- package/.docs/raw/reference/client-js/tools.mdx +2 -2
- package/.docs/raw/reference/client-js/vectors.mdx +1 -1
- package/.docs/raw/reference/client-js/workflows-legacy.mdx +1 -1
- package/.docs/raw/reference/client-js/workflows.mdx +1 -1
- package/.docs/raw/reference/core/getAgent.mdx +1 -1
- package/.docs/raw/reference/core/getAgentById.mdx +1 -1
- package/.docs/raw/reference/core/getAgents.mdx +1 -1
- package/.docs/raw/reference/core/getDeployer.mdx +1 -1
- package/.docs/raw/reference/core/getLogger.mdx +1 -1
- package/.docs/raw/reference/core/getLogs.mdx +1 -1
- package/.docs/raw/reference/core/getLogsByRunId.mdx +1 -1
- package/.docs/raw/reference/core/getMCPServer.mdx +2 -2
- package/.docs/raw/reference/core/getMCPServers.mdx +2 -2
- package/.docs/raw/reference/core/getMemory.mdx +1 -1
- package/.docs/raw/reference/core/getScorer.mdx +1 -1
- package/.docs/raw/reference/core/getScorerByName.mdx +1 -1
- package/.docs/raw/reference/core/getScorers.mdx +1 -1
- package/.docs/raw/reference/core/getServer.mdx +2 -2
- package/.docs/raw/reference/core/getStorage.mdx +1 -1
- package/.docs/raw/reference/core/getTelemetry.mdx +1 -1
- package/.docs/raw/reference/core/getVector.mdx +1 -1
- package/.docs/raw/reference/core/getVectors.mdx +1 -1
- package/.docs/raw/reference/core/getWorkflow.mdx +1 -1
- package/.docs/raw/reference/core/getWorkflows.mdx +1 -1
- package/.docs/raw/reference/core/listLogs.mdx +1 -1
- package/.docs/raw/reference/core/listLogsByRunId.mdx +1 -1
- package/.docs/raw/reference/core/listScorers.mdx +1 -1
- package/.docs/raw/reference/core/listWorkflows.mdx +1 -1
- package/.docs/raw/reference/core/mastra-class.mdx +1 -1
- package/.docs/raw/reference/core/setLogger.mdx +1 -1
- package/.docs/raw/reference/core/setStorage.mdx +1 -1
- package/.docs/raw/reference/core/setTelemetry.mdx +1 -1
- package/.docs/raw/reference/deployer/cloudflare.mdx +1 -1
- package/.docs/raw/reference/deployer/deployer.mdx +1 -1
- package/.docs/raw/reference/deployer/netlify.mdx +1 -1
- package/.docs/raw/reference/deployer/vercel.mdx +1 -1
- package/.docs/raw/reference/evals/answer-relevancy.mdx +1 -1
- package/.docs/raw/reference/evals/bias.mdx +1 -1
- package/.docs/raw/reference/evals/completeness.mdx +1 -1
- package/.docs/raw/reference/evals/content-similarity.mdx +1 -1
- package/.docs/raw/reference/evals/context-position.mdx +1 -1
- package/.docs/raw/reference/evals/context-precision.mdx +1 -1
- package/.docs/raw/reference/evals/context-relevancy.mdx +1 -1
- package/.docs/raw/reference/evals/contextual-recall.mdx +1 -1
- package/.docs/raw/reference/evals/faithfulness.mdx +1 -1
- package/.docs/raw/reference/evals/hallucination.mdx +1 -1
- package/.docs/raw/reference/evals/keyword-coverage.mdx +1 -1
- package/.docs/raw/reference/evals/prompt-alignment.mdx +1 -1
- package/.docs/raw/reference/evals/summarization.mdx +1 -1
- package/.docs/raw/reference/evals/textual-difference.mdx +1 -1
- package/.docs/raw/reference/evals/tone-consistency.mdx +1 -1
- package/.docs/raw/reference/evals/toxicity.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/after.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/afterEvent.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/commit.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/createRun.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/else.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/events.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/execute.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/if.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/resume.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/resumeWithEvent.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/snapshots.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/start.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/step-class.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/step-condition.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/step-function.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/step-options.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/step-retries.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/suspend.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/then.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/until.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/watch.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/while.mdx +1 -1
- package/.docs/raw/reference/legacyWorkflows/workflow.mdx +1 -1
- package/.docs/raw/reference/memory/createThread.mdx +1 -1
- package/.docs/raw/reference/memory/deleteMessages.mdx +1 -1
- package/.docs/raw/reference/memory/getThreadById.mdx +1 -1
- package/.docs/raw/reference/memory/getThreadsByResourceId.mdx +1 -1
- package/.docs/raw/reference/memory/getThreadsByResourceIdPaginated.mdx +1 -1
- package/.docs/raw/reference/memory/memory-class.mdx +1 -1
- package/.docs/raw/reference/memory/query.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/ai-tracing.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/configuration.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/arize.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/braintrust.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/cloud-exporter.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/console-exporter.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/default-exporter.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/langfuse.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/langsmith.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/exporters/otel.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/interfaces.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/processors/sensitive-data-filter.mdx +1 -1
- package/.docs/raw/reference/observability/ai-tracing/span.mdx +1 -1
- package/.docs/raw/reference/observability/logging/pino-logger.mdx +2 -2
- package/.docs/raw/reference/observability/otel-tracing/otel-config.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/arize-ax.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/arize-phoenix.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/braintrust.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/dash0.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/index.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/laminar.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/langfuse.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/langsmith.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/langwatch.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/new-relic.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/signoz.mdx +1 -1
- package/.docs/raw/reference/observability/otel-tracing/providers/traceloop.mdx +1 -1
- package/.docs/raw/reference/processors/batch-parts-processor.mdx +1 -1
- package/.docs/raw/reference/processors/language-detector.mdx +1 -1
- package/.docs/raw/reference/processors/moderation-processor.mdx +1 -1
- package/.docs/raw/reference/processors/pii-detector.mdx +1 -1
- package/.docs/raw/reference/processors/prompt-injection-detector.mdx +1 -1
- package/.docs/raw/reference/processors/system-prompt-scrubber.mdx +1 -1
- package/.docs/raw/reference/processors/token-limiter-processor.mdx +1 -1
- package/.docs/raw/reference/processors/unicode-normalizer.mdx +1 -1
- package/.docs/raw/reference/rag/chunk.mdx +1 -1
- package/.docs/raw/reference/rag/database-config.mdx +1 -1
- package/.docs/raw/reference/rag/document.mdx +1 -1
- package/.docs/raw/reference/rag/embeddings.mdx +1 -1
- package/.docs/raw/reference/rag/extract-params.mdx +1 -1
- package/.docs/raw/reference/rag/graph-rag.mdx +1 -1
- package/.docs/raw/reference/rag/metadata-filters.mdx +1 -1
- package/.docs/raw/reference/rag/rerank.mdx +1 -1
- package/.docs/raw/reference/rag/rerankWithScorer.mdx +1 -1
- package/.docs/raw/reference/scorers/answer-relevancy.mdx +1 -1
- package/.docs/raw/reference/scorers/answer-similarity.mdx +1 -1
- package/.docs/raw/reference/scorers/bias.mdx +1 -1
- package/.docs/raw/reference/scorers/completeness.mdx +1 -1
- package/.docs/raw/reference/scorers/content-similarity.mdx +1 -1
- package/.docs/raw/reference/scorers/context-precision.mdx +1 -1
- package/.docs/raw/reference/scorers/context-relevance.mdx +1 -1
- package/.docs/raw/reference/scorers/create-scorer.mdx +1 -1
- package/.docs/raw/reference/scorers/faithfulness.mdx +1 -1
- package/.docs/raw/reference/scorers/hallucination.mdx +1 -1
- package/.docs/raw/reference/scorers/keyword-coverage.mdx +1 -1
- package/.docs/raw/reference/scorers/mastra-scorer.mdx +1 -1
- package/.docs/raw/reference/scorers/noise-sensitivity.mdx +1 -1
- package/.docs/raw/reference/scorers/prompt-alignment.mdx +1 -1
- package/.docs/raw/reference/scorers/run-experiment.mdx +1 -1
- package/.docs/raw/reference/scorers/textual-difference.mdx +1 -1
- package/.docs/raw/reference/scorers/tone-consistency.mdx +1 -1
- package/.docs/raw/reference/scorers/tool-call-accuracy.mdx +1 -1
- package/.docs/raw/reference/scorers/toxicity.mdx +1 -1
- package/.docs/raw/reference/storage/cloudflare-d1.mdx +1 -1
- package/.docs/raw/reference/storage/cloudflare.mdx +1 -1
- package/.docs/raw/reference/storage/dynamodb.mdx +1 -1
- package/.docs/raw/reference/storage/lance.mdx +1 -1
- package/.docs/raw/reference/storage/libsql.mdx +1 -1
- package/.docs/raw/reference/storage/mongodb.mdx +1 -1
- package/.docs/raw/reference/storage/mssql.mdx +1 -1
- package/.docs/raw/reference/storage/postgresql.mdx +1 -1
- package/.docs/raw/reference/storage/upstash.mdx +1 -1
- package/.docs/raw/reference/streaming/ChunkType.mdx +1 -1
- package/.docs/raw/reference/streaming/agents/MastraModelOutput.mdx +1 -1
- package/.docs/raw/reference/streaming/agents/stream.mdx +1 -1
- package/.docs/raw/reference/streaming/agents/streamLegacy.mdx +1 -1
- package/.docs/raw/reference/streaming/workflows/observeStream.mdx +1 -1
- package/.docs/raw/reference/streaming/workflows/observeStreamVNext.mdx +1 -1
- package/.docs/raw/reference/streaming/workflows/resumeStreamVNext.mdx +1 -1
- package/.docs/raw/reference/streaming/workflows/stream.mdx +1 -1
- package/.docs/raw/reference/streaming/workflows/streamVNext.mdx +1 -1
- package/.docs/raw/reference/templates/overview.mdx +1 -1
- package/.docs/raw/reference/tools/client.mdx +1 -1
- package/.docs/raw/reference/tools/create-tool.mdx +3 -4
- package/.docs/raw/reference/tools/document-chunker-tool.mdx +1 -1
- package/.docs/raw/reference/tools/graph-rag-tool.mdx +2 -2
- package/.docs/raw/reference/tools/mcp-client.mdx +9 -9
- package/.docs/raw/reference/tools/mcp-server.mdx +2 -2
- package/.docs/raw/reference/tools/vector-query-tool.mdx +3 -3
- package/.docs/raw/reference/vectors/astra.mdx +1 -1
- package/.docs/raw/reference/vectors/chroma.mdx +1 -1
- package/.docs/raw/reference/vectors/couchbase.mdx +1 -1
- package/.docs/raw/reference/vectors/lance.mdx +1 -1
- package/.docs/raw/reference/vectors/libsql.mdx +1 -1
- package/.docs/raw/reference/vectors/mongodb.mdx +1 -1
- package/.docs/raw/reference/vectors/opensearch.mdx +1 -1
- package/.docs/raw/reference/vectors/pg.mdx +1 -1
- package/.docs/raw/reference/vectors/pinecone.mdx +1 -1
- package/.docs/raw/reference/vectors/qdrant.mdx +1 -1
- package/.docs/raw/reference/vectors/s3vectors.mdx +1 -1
- package/.docs/raw/reference/vectors/turbopuffer.mdx +1 -1
- package/.docs/raw/reference/vectors/upstash.mdx +1 -1
- package/.docs/raw/reference/vectors/vectorize.mdx +1 -1
- package/.docs/raw/reference/voice/azure.mdx +1 -1
- package/.docs/raw/reference/voice/cloudflare.mdx +1 -1
- package/.docs/raw/reference/voice/composite-voice.mdx +1 -1
- package/.docs/raw/reference/voice/deepgram.mdx +1 -1
- package/.docs/raw/reference/voice/elevenlabs.mdx +1 -1
- package/.docs/raw/reference/voice/google-gemini-live.mdx +1 -1
- package/.docs/raw/reference/voice/google.mdx +1 -1
- package/.docs/raw/reference/voice/mastra-voice.mdx +1 -1
- package/.docs/raw/reference/voice/murf.mdx +1 -1
- package/.docs/raw/reference/voice/openai-realtime.mdx +1 -1
- package/.docs/raw/reference/voice/openai.mdx +1 -1
- package/.docs/raw/reference/voice/playai.mdx +1 -1
- package/.docs/raw/reference/voice/sarvam.mdx +1 -1
- package/.docs/raw/reference/voice/speechify.mdx +1 -1
- package/.docs/raw/reference/voice/voice.addInstructions.mdx +1 -1
- package/.docs/raw/reference/voice/voice.addTools.mdx +1 -1
- package/.docs/raw/reference/voice/voice.answer.mdx +1 -1
- package/.docs/raw/reference/voice/voice.close.mdx +1 -1
- package/.docs/raw/reference/voice/voice.connect.mdx +1 -1
- package/.docs/raw/reference/voice/voice.events.mdx +1 -1
- package/.docs/raw/reference/voice/voice.getSpeakers.mdx +1 -1
- package/.docs/raw/reference/voice/voice.listen.mdx +1 -1
- package/.docs/raw/reference/voice/voice.off.mdx +1 -1
- package/.docs/raw/reference/voice/voice.on.mdx +1 -1
- package/.docs/raw/reference/voice/voice.send.mdx +1 -1
- package/.docs/raw/reference/voice/voice.speak.mdx +1 -1
- package/.docs/raw/reference/voice/voice.updateConfig.mdx +1 -1
- package/.docs/raw/reference/workflows/run-methods/cancel.mdx +1 -1
- package/.docs/raw/reference/workflows/run-methods/resume.mdx +1 -1
- package/.docs/raw/reference/workflows/run-methods/start.mdx +1 -1
- package/.docs/raw/reference/workflows/run-methods/watch.mdx +1 -1
- package/.docs/raw/reference/workflows/run.mdx +1 -1
- package/.docs/raw/reference/workflows/step.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/branch.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/commit.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/create-run.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/dountil.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/dowhile.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/foreach.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/map.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/parallel.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/sendEvent.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/sleep.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/sleepUntil.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/then.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow-methods/waitForEvent.mdx +1 -1
- package/.docs/raw/reference/workflows/workflow.mdx +1 -1
- package/.docs/raw/scorers/{off-the-shelf-scorers.mdx → built-in-scorers.mdx} +7 -5
- package/.docs/raw/scorers/custom-scorers.mdx +16 -16
- package/.docs/raw/scorers/{evals-old-api → evals-legacy}/custom-eval.mdx +1 -1
- package/.docs/raw/scorers/{evals-old-api → evals-legacy}/overview.mdx +13 -13
- package/.docs/raw/scorers/{evals-old-api → evals-legacy}/running-in-ci.mdx +5 -5
- package/.docs/raw/scorers/{evals-old-api → evals-legacy}/textual-evals.mdx +7 -7
- package/.docs/raw/scorers/overview.mdx +5 -5
- package/.docs/raw/server-db/custom-api-routes.mdx +1 -1
- package/.docs/raw/server-db/mastra-client.mdx +2 -2
- package/.docs/raw/server-db/{production-server.mdx → mastra-server.mdx} +2 -2
- package/.docs/raw/server-db/middleware.mdx +30 -30
- package/.docs/raw/server-db/runtime-context.mdx +3 -3
- package/.docs/raw/server-db/storage.mdx +1 -1
- package/.docs/raw/streaming/events.mdx +1 -1
- package/.docs/raw/streaming/overview.mdx +1 -1
- package/.docs/raw/streaming/tool-streaming.mdx +1 -1
- package/.docs/raw/streaming/workflow-streaming.mdx +1 -1
- package/.docs/raw/voice/overview.mdx +1 -1
- package/.docs/raw/voice/speech-to-speech.mdx +1 -1
- package/.docs/raw/voice/speech-to-text.mdx +1 -1
- package/.docs/raw/voice/text-to-speech.mdx +1 -1
- package/.docs/raw/workflows/agents-and-tools.mdx +2 -2
- package/.docs/raw/workflows/control-flow.mdx +7 -1
- package/.docs/raw/workflows/error-handling.mdx +4 -4
- package/.docs/raw/workflows/human-in-the-loop.mdx +160 -210
- package/.docs/raw/workflows/inngest-workflow.mdx +1 -1
- package/.docs/raw/workflows/overview.mdx +11 -2
- package/.docs/raw/workflows/snapshots.mdx +4 -4
- package/.docs/raw/workflows/suspend-and-resume.mdx +72 -215
- package/.docs/raw/workflows-legacy/control-flow.mdx +1 -1
- package/.docs/raw/workflows-legacy/dynamic-workflows.mdx +1 -1
- package/.docs/raw/workflows-legacy/error-handling.mdx +1 -1
- package/.docs/raw/workflows-legacy/nested-workflows.mdx +1 -1
- package/.docs/raw/workflows-legacy/overview.mdx +1 -1
- package/.docs/raw/workflows-legacy/runtime-variables.mdx +1 -1
- package/.docs/raw/workflows-legacy/steps.mdx +1 -1
- package/.docs/raw/workflows-legacy/suspend-and-resume.mdx +1 -1
- package/.docs/raw/workflows-legacy/variables.mdx +1 -1
- package/CHANGELOG.md +7 -0
- package/package.json +4 -4
- package/.docs/raw/deployment/serverless-platforms/index.mdx +0 -55
- package/.docs/raw/reference/agents/listTools.mdx +0 -69
- package/.docs/raw/reference/core/listAgents.mdx +0 -35
- package/.docs/raw/tools-mcp/advanced-usage.mdx +0 -128
- package/.docs/raw/tools-mcp/overview.mdx +0 -88
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow.dountil() | Workflows
|
|
2
|
+
title: "Reference: Workflow.dountil() | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow.dountil()` method in workflows, which creates a loop that executes a step until a condition is met.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow.dowhile() | Workflows
|
|
2
|
+
title: "Reference: Workflow.dowhile() | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow.dowhile()` method in workflows, which creates a loop that executes a step while a condition is met.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow.foreach() | Workflows
|
|
2
|
+
title: "Reference: Workflow.foreach() | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow.foreach()` method in workflows, which creates a loop that executes a step for each item in an array.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow.map() | Workflows
|
|
2
|
+
title: "Reference: Workflow.map() | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow.map()` method in workflows, which maps output data from a previous step to the input of a subsequent step.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow.waitForEvent() | Workflows
|
|
2
|
+
title: "Reference: Workflow.waitForEvent() | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow.waitForEvent()` method in workflows, which pauses execution until an event is received.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Reference: Workflow Class | Workflows
|
|
2
|
+
title: "Reference: Workflow Class | Workflows"
|
|
3
3
|
description: Documentation for the `Workflow` class in Mastra, which enables you to create state machines for complex sequences of operations with conditional branching and data validation.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Built-in Scorers | Scorers
|
|
2
|
+
title: "Built-in Scorers | Scorers"
|
|
3
3
|
description: "Overview of Mastra's ready-to-use scorers for evaluating AI outputs across quality, safety, and performance dimensions."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -7,9 +7,11 @@ description: "Overview of Mastra's ready-to-use scorers for evaluating AI output
|
|
|
7
7
|
|
|
8
8
|
Mastra provides a comprehensive set of built-in scorers for evaluating AI outputs. These scorers are optimized for common evaluation scenarios and are ready to use in your agents and workflows.
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
To create your own scorers, please see the [Custom Scorers](/docs/scorers/custom-scorers) documentation.
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
## Available scorers
|
|
13
|
+
|
|
14
|
+
### Accuracy and reliability
|
|
13
15
|
|
|
14
16
|
These scorers evaluate how correct, truthful, and complete your agent's answers are:
|
|
15
17
|
|
|
@@ -23,7 +25,7 @@ These scorers evaluate how correct, truthful, and complete your agent's answers
|
|
|
23
25
|
- [`tool-call-accuracy`](/reference/scorers/tool-call-accuracy): Evaluates whether the LLM selects the correct tool from available options (`0-1`, higher is better)
|
|
24
26
|
- [`prompt-alignment`](/reference/scorers/prompt-alignment): Measures how well agent responses align with user prompt intent, requirements, completeness, and format (`0-1`, higher is better)
|
|
25
27
|
|
|
26
|
-
### Context
|
|
28
|
+
### Context quality
|
|
27
29
|
|
|
28
30
|
These scorers evaluate the quality and relevance of context used in generating responses:
|
|
29
31
|
|
|
@@ -40,7 +42,7 @@ Both context scorers support:
|
|
|
40
42
|
- **Static context**: Pre-defined context arrays
|
|
41
43
|
- **Dynamic context extraction**: Extract context from runs using custom functions (ideal for RAG systems, vector databases, etc.)
|
|
42
44
|
|
|
43
|
-
### Output
|
|
45
|
+
### Output quality
|
|
44
46
|
|
|
45
47
|
These scorers evaluate adherence to format, style, and safety requirements:
|
|
46
48
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Custom Scorers | Scorers
|
|
2
|
+
title: "Custom Scorers | Scorers"
|
|
3
3
|
---
|
|
4
4
|
|
|
5
|
-
# Custom
|
|
5
|
+
# Custom Scorers
|
|
6
6
|
|
|
7
7
|
Mastra provides a unified `createScorer` factory that allows you to build custom evaluation logic using either JavaScript functions or LLM-based prompt objects for each step. This flexibility lets you choose the best approach for each part of your evaluation pipeline.
|
|
8
8
|
|
|
9
|
-
### The
|
|
9
|
+
### The four-step pipeline
|
|
10
10
|
|
|
11
11
|
All scorers in Mastra follow a consistent four-step evaluation pipeline:
|
|
12
12
|
|
|
@@ -17,7 +17,7 @@ All scorers in Mastra follow a consistent four-step evaluation pipeline:
|
|
|
17
17
|
|
|
18
18
|
Each step can use either **functions** or **prompt objects** (LLM-based evaluation), giving you the flexibility to combine deterministic algorithms with AI judgment as needed.
|
|
19
19
|
|
|
20
|
-
### Functions vs
|
|
20
|
+
### Functions vs prompt objects
|
|
21
21
|
|
|
22
22
|
**Functions** use JavaScript for deterministic logic. They're ideal for:
|
|
23
23
|
|
|
@@ -35,7 +35,7 @@ Each step can use either **functions** or **prompt objects** (LLM-based evaluati
|
|
|
35
35
|
|
|
36
36
|
You can mix and match approaches within a single scorer - for example, use a function for preprocessing data and an LLM for analyzing quality.
|
|
37
37
|
|
|
38
|
-
### Initializing a
|
|
38
|
+
### Initializing a scorer
|
|
39
39
|
|
|
40
40
|
Every scorer starts with the `createScorer` factory function, which requires a name and description, and optionally accepts a type specification and judge configuration.
|
|
41
41
|
|
|
@@ -60,7 +60,7 @@ const glutenCheckerScorer = createScorer({
|
|
|
60
60
|
|
|
61
61
|
The judge configuration is only needed if you plan to use prompt objects in any step. Individual steps can override this default configuration with their own judge settings.
|
|
62
62
|
|
|
63
|
-
#### Agent
|
|
63
|
+
#### Agent type for agent evaluation
|
|
64
64
|
|
|
65
65
|
For type safety and compatibility with both live agent scoring and trace scoring, use `type: 'agent'` when creating scorers for agent evaluation. This allows you to use the same scorer for an agent and also use it to score traces:
|
|
66
66
|
|
|
@@ -74,9 +74,9 @@ const myScorer = createScorer({
|
|
|
74
74
|
});
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
### Step-by-
|
|
77
|
+
### Step-by-step breakdown
|
|
78
78
|
|
|
79
|
-
####
|
|
79
|
+
#### Preprocess step (optional)
|
|
80
80
|
|
|
81
81
|
Prepares input/output data when you need to extract specific elements, filter content, or transform complex data structures.
|
|
82
82
|
|
|
@@ -118,7 +118,7 @@ const glutenCheckerScorer = createScorer(...)
|
|
|
118
118
|
|
|
119
119
|
**Data Flow:** Results are available to subsequent steps as `results.preprocessStepResult`
|
|
120
120
|
|
|
121
|
-
####
|
|
121
|
+
#### Analyze step (optional)
|
|
122
122
|
|
|
123
123
|
Performs core evaluation analysis, gathering insights that will inform the scoring decision.
|
|
124
124
|
|
|
@@ -168,7 +168,7 @@ const glutenCheckerScorer = createScorer({...})
|
|
|
168
168
|
|
|
169
169
|
**Data Flow:** Results are available to subsequent steps as `results.analyzeStepResult`
|
|
170
170
|
|
|
171
|
-
####
|
|
171
|
+
#### Generate score step (required)
|
|
172
172
|
|
|
173
173
|
Converts analysis results into a numerical score. This is the only required step in the pipeline.
|
|
174
174
|
|
|
@@ -191,7 +191,7 @@ const glutenCheckerScorer = createScorer({...})
|
|
|
191
191
|
|
|
192
192
|
**Data Flow:** The score is available to generateReason as the `score` parameter
|
|
193
193
|
|
|
194
|
-
####
|
|
194
|
+
#### Generate reason step (optional)
|
|
195
195
|
|
|
196
196
|
Generates human-readable explanations for the score, useful for debugging, transparency, or user feedback.
|
|
197
197
|
|
|
@@ -231,7 +231,7 @@ const glutenCheckerScorer = createScorer({...})
|
|
|
231
231
|
})
|
|
232
232
|
```
|
|
233
233
|
|
|
234
|
-
##
|
|
234
|
+
## Create a custom scorer
|
|
235
235
|
|
|
236
236
|
A custom scorer in Mastra uses `createScorer` with four core components:
|
|
237
237
|
|
|
@@ -332,7 +332,7 @@ export const glutenCheckerScorer = createScorer({
|
|
|
332
332
|
});
|
|
333
333
|
```
|
|
334
334
|
|
|
335
|
-
### Judge
|
|
335
|
+
### Judge configuration
|
|
336
336
|
|
|
337
337
|
Sets up the LLM model and defines its role as a domain expert.
|
|
338
338
|
|
|
@@ -343,7 +343,7 @@ judge: {
|
|
|
343
343
|
}
|
|
344
344
|
```
|
|
345
345
|
|
|
346
|
-
### Analysis
|
|
346
|
+
### Analysis step
|
|
347
347
|
|
|
348
348
|
Defines how the LLM should analyze the input and what structured output to return.
|
|
349
349
|
|
|
@@ -367,7 +367,7 @@ The analysis step uses a prompt object to:
|
|
|
367
367
|
- Define expected output structure with Zod schema (both boolean result and list of gluten sources)
|
|
368
368
|
- Generate dynamic prompts based on the input content
|
|
369
369
|
|
|
370
|
-
### Score
|
|
370
|
+
### Score generation
|
|
371
371
|
|
|
372
372
|
Converts the LLM's structured analysis into a numerical score.
|
|
373
373
|
|
|
@@ -379,7 +379,7 @@ Converts the LLM's structured analysis into a numerical score.
|
|
|
379
379
|
|
|
380
380
|
The score generation function takes the analysis results and applies business logic to produce a score. In this case, the LLM directly determines if the recipe is gluten-free, so we use that boolean result: 1 for gluten-free, 0 for contains gluten.
|
|
381
381
|
|
|
382
|
-
### Reason
|
|
382
|
+
### Reason generation
|
|
383
383
|
|
|
384
384
|
Provides human-readable explanations for the score using another LLM call.
|
|
385
385
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "
|
|
3
|
-
description: "
|
|
2
|
+
title: "Evals Overview | Evals"
|
|
3
|
+
description: "Overview of evals in Mastra, detailing their capabilities for evaluating AI outputs and measuring performance."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
#
|
|
6
|
+
# Evals Overview
|
|
7
7
|
|
|
8
8
|
:::info Scorers
|
|
9
9
|
This documentation refers to the legacy evals API. For the latest scorer features, see [Scorers](/docs/scorers/overview).
|
|
@@ -15,7 +15,7 @@ Evals are automated tests that evaluate Agents outputs using model-graded, rule-
|
|
|
15
15
|
|
|
16
16
|
Evals can be run in the cloud, capturing real-time results. But evals can also be part of your CI/CD pipeline, allowing you to test and monitor your agents over time.
|
|
17
17
|
|
|
18
|
-
## Types of
|
|
18
|
+
## Types of evals
|
|
19
19
|
|
|
20
20
|
There are different kinds of evals, each serving a specific purpose. Here are some common types:
|
|
21
21
|
|
|
@@ -31,7 +31,7 @@ To access Mastra's evals feature install the `@mastra/evals` package.
|
|
|
31
31
|
npm install @mastra/evals@latest
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
## Getting
|
|
34
|
+
## Getting started
|
|
35
35
|
|
|
36
36
|
Evals need to be added to an agent. Here's an example using the summarization, content similarity, and tone consistency metrics:
|
|
37
37
|
|
|
@@ -60,7 +60,7 @@ export const myAgent = new Agent({
|
|
|
60
60
|
|
|
61
61
|
You can view eval results in the Mastra dashboard when using `mastra dev`.
|
|
62
62
|
|
|
63
|
-
## Beyond
|
|
63
|
+
## Beyond automated testing
|
|
64
64
|
|
|
65
65
|
While automated evals are valuable, high-performing AI teams often combine them with:
|
|
66
66
|
|
|
@@ -68,11 +68,11 @@ While automated evals are valuable, high-performing AI teams often combine them
|
|
|
68
68
|
2. **Human Review**: Regular review of production data and traces
|
|
69
69
|
3. **Continuous Monitoring**: Track eval metrics over time to detect regressions
|
|
70
70
|
|
|
71
|
-
## Understanding
|
|
71
|
+
## Understanding eval results
|
|
72
72
|
|
|
73
73
|
Each eval metric measures a specific aspect of your agent's output. Here's how to interpret and improve your results:
|
|
74
74
|
|
|
75
|
-
### Understanding
|
|
75
|
+
### Understanding scores
|
|
76
76
|
|
|
77
77
|
For any metric:
|
|
78
78
|
|
|
@@ -81,7 +81,7 @@ For any metric:
|
|
|
81
81
|
3. Compare scores across different inputs and contexts
|
|
82
82
|
4. Track changes over time to spot trends
|
|
83
83
|
|
|
84
|
-
### Improving
|
|
84
|
+
### Improving results
|
|
85
85
|
|
|
86
86
|
When scores aren't meeting your targets:
|
|
87
87
|
|
|
@@ -90,7 +90,7 @@ When scores aren't meeting your targets:
|
|
|
90
90
|
3. Simplify your prompts - Break complex tasks into smaller steps
|
|
91
91
|
4. Add guardrails - Include specific rules for tricky cases
|
|
92
92
|
|
|
93
|
-
### Maintaining
|
|
93
|
+
### Maintaining quality
|
|
94
94
|
|
|
95
95
|
Once you're hitting your targets:
|
|
96
96
|
|
|
@@ -99,8 +99,8 @@ Once you're hitting your targets:
|
|
|
99
99
|
3. Test edge cases - Add examples that cover unusual scenarios
|
|
100
100
|
4. Fine-tune - Look for ways to improve efficiency
|
|
101
101
|
|
|
102
|
-
See [Textual Evals](/docs/scorers/evals-
|
|
102
|
+
See [Textual Evals](/docs/scorers/evals-legacy/textual-evals) for more info on what evals can do.
|
|
103
103
|
|
|
104
|
-
For more info on how to create your own evals, see the [Custom Evals](/docs/scorers/evals-
|
|
104
|
+
For more info on how to create your own evals, see the [Custom Evals](/docs/scorers/evals-legacy/custom-eval) guide.
|
|
105
105
|
|
|
106
|
-
For running evals in your CI pipeline, see the [Running in CI](/docs/scorers/evals-
|
|
106
|
+
For running evals in your CI pipeline, see the [Running in CI](/docs/scorers/evals-legacy/running-in-ci) guide.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Running Evals in CI | Scorers
|
|
2
|
+
title: "Running Evals in CI | Scorers"
|
|
3
3
|
description: "Learn how to run Mastra evals in your CI/CD pipeline to monitor agent quality over time."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -11,7 +11,7 @@ This documentation refers to the legacy evals API. For the latest scorer feature
|
|
|
11
11
|
|
|
12
12
|
Running evals in your CI pipeline helps bridge this gap by providing quantifiable metrics for measuring agent quality over time.
|
|
13
13
|
|
|
14
|
-
## Setting
|
|
14
|
+
## Setting up CI integration
|
|
15
15
|
|
|
16
16
|
We support any testing framework that supports ESM modules. For example, you can use [Vitest](https://vitest.dev/), [Jest](https://jestjs.io/) or [Mocha](https://mochajs.org/) to run evals in your CI/CD pipeline.
|
|
17
17
|
|
|
@@ -33,9 +33,9 @@ describe("My Agent", () => {
|
|
|
33
33
|
|
|
34
34
|
You will need to configure a testSetup and globalSetup script for your testing framework to capture the eval results. It allows us to show these results in your mastra dashboard.
|
|
35
35
|
|
|
36
|
-
## Framework
|
|
36
|
+
## Framework configuration
|
|
37
37
|
|
|
38
|
-
### Vitest
|
|
38
|
+
### Vitest setup
|
|
39
39
|
|
|
40
40
|
Add these files to your project to run evals in your CI/CD pipeline:
|
|
41
41
|
|
|
@@ -67,7 +67,7 @@ export default defineConfig({
|
|
|
67
67
|
});
|
|
68
68
|
```
|
|
69
69
|
|
|
70
|
-
## Storage
|
|
70
|
+
## Storage configuration
|
|
71
71
|
|
|
72
72
|
To store eval results in Mastra Storage and capture results in the Mastra dashboard:
|
|
73
73
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Textual Evals | Scorers
|
|
2
|
+
title: "Textual Evals | Scorers"
|
|
3
3
|
description: "Understand how Mastra uses LLM-as-judge methodology to evaluate text quality."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -13,9 +13,9 @@ Textual evals use an LLM-as-judge methodology to evaluate agent outputs. This ap
|
|
|
13
13
|
|
|
14
14
|
Each eval focuses on specific quality aspects and returns a score between 0 and 1, providing quantifiable metrics for non-deterministic AI outputs.
|
|
15
15
|
|
|
16
|
-
Mastra provides several eval metrics for assessing Agent outputs. Mastra is not limited to these metrics, and you can also [define your own evals](/docs/scorers/evals-
|
|
16
|
+
Mastra provides several eval metrics for assessing Agent outputs. Mastra is not limited to these metrics, and you can also [define your own evals](/docs/scorers/evals-legacy/custom-eval).
|
|
17
17
|
|
|
18
|
-
## Why
|
|
18
|
+
## Why use textual evals?
|
|
19
19
|
|
|
20
20
|
Textual evals help ensure your agent:
|
|
21
21
|
|
|
@@ -24,9 +24,9 @@ Textual evals help ensure your agent:
|
|
|
24
24
|
- Follows output requirements
|
|
25
25
|
- Maintains consistent quality over time
|
|
26
26
|
|
|
27
|
-
## Available
|
|
27
|
+
## Available metrics
|
|
28
28
|
|
|
29
|
-
### Accuracy and
|
|
29
|
+
### Accuracy and reliability
|
|
30
30
|
|
|
31
31
|
These metrics evaluate how correct, truthful, and complete your agent's answers are:
|
|
32
32
|
|
|
@@ -37,7 +37,7 @@ These metrics evaluate how correct, truthful, and complete your agent's answers
|
|
|
37
37
|
- [`answer-relevancy`](/reference/evals/answer-relevancy): Assesses how well responses address the original query
|
|
38
38
|
- [`textual-difference`](/reference/evals/textual-difference): Measures textual differences between strings
|
|
39
39
|
|
|
40
|
-
### Understanding
|
|
40
|
+
### Understanding context
|
|
41
41
|
|
|
42
42
|
These metrics evaluate how well your agent uses provided context:
|
|
43
43
|
|
|
@@ -46,7 +46,7 @@ These metrics evaluate how well your agent uses provided context:
|
|
|
46
46
|
- [`context-relevancy`](/reference/evals/context-relevancy): Measures use of appropriate context pieces
|
|
47
47
|
- [`contextual-recall`](/reference/evals/contextual-recall): Assesses completeness of context usage
|
|
48
48
|
|
|
49
|
-
### Output
|
|
49
|
+
### Output quality
|
|
50
50
|
|
|
51
51
|
These metrics evaluate adherence to format and style requirements:
|
|
52
52
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Scorers
|
|
2
|
+
title: "Scorers Overview | Scorers"
|
|
3
3
|
description: Overview of scorers in Mastra, detailing their capabilities for evaluating AI outputs and measuring performance.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Scorers
|
|
6
|
+
# Scorers Overview
|
|
7
7
|
|
|
8
8
|
While traditional software tests have clear pass/fail conditions, AI outputs are non-deterministic — they can vary with the same input. **Scorers** help bridge this gap by providing quantifiable metrics for measuring agent quality.
|
|
9
9
|
|
|
@@ -11,7 +11,7 @@ Scorers are automated tests that evaluate Agents outputs using model-graded, rul
|
|
|
11
11
|
|
|
12
12
|
Scorers can be run in the cloud, capturing real-time results. But scorers can also be part of your CI/CD pipeline, allowing you to test and monitor your agents over time.
|
|
13
13
|
|
|
14
|
-
## Types of
|
|
14
|
+
## Types of scorers
|
|
15
15
|
|
|
16
16
|
There are different kinds of scorers, each serving a specific purpose. Here are some common types:
|
|
17
17
|
|
|
@@ -33,7 +33,7 @@ npm install @mastra/evals@latest
|
|
|
33
33
|
|
|
34
34
|
### Adding scorers to agents
|
|
35
35
|
|
|
36
|
-
You can add built-in scorers to your agents to automatically evaluate their outputs. See the [full list of built-in scorers](/docs/scorers/
|
|
36
|
+
You can add built-in scorers to your agents to automatically evaluate their outputs. See the [full list of built-in scorers](/docs/scorers/built-in-scorers) for all available options.
|
|
37
37
|
|
|
38
38
|
```typescript title="src/mastra/agents/evaluated-agent.ts" showLineNumbers copy
|
|
39
39
|
import { Agent } from "@mastra/core/agent";
|
|
@@ -135,5 +135,5 @@ For more details, see [Studio](/docs/getting-started/studio) docs.
|
|
|
135
135
|
## Next steps
|
|
136
136
|
|
|
137
137
|
- Learn how to create your own scorers in the [Creating Custom Scorers](/docs/scorers/custom-scorers) guide
|
|
138
|
-
- Explore built-in scorers in the [Off-the-shelf Scorers](/docs/scorers/
|
|
138
|
+
- Explore built-in scorers in the [Off-the-shelf Scorers](/docs/scorers/built-in-scorers) section
|
|
139
139
|
- Test scorers with [Studio](/docs/getting-started/studio)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Mastra Client SDK | Server & DB
|
|
2
|
+
title: "Mastra Client SDK | Server & DB"
|
|
3
3
|
description: "Learn how to set up and use the Mastra Client SDK"
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -8,7 +8,7 @@ import TabItem from "@theme/TabItem";
|
|
|
8
8
|
|
|
9
9
|
# Mastra Client SDK
|
|
10
10
|
|
|
11
|
-
The Mastra Client SDK provides a simple and type-safe interface for interacting with your [Mastra Server](/docs/server-db/
|
|
11
|
+
The Mastra Client SDK provides a simple and type-safe interface for interacting with your [Mastra Server](/docs/server-db/mastra-server) from your client environment.
|
|
12
12
|
|
|
13
13
|
## Prerequisites
|
|
14
14
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "
|
|
2
|
+
title: "Mastra Server | Server & DB"
|
|
3
3
|
description: "Learn how to configure and deploy a production-ready Mastra server with custom settings for APIs, CORS, and more"
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
#
|
|
6
|
+
# Mastra Server
|
|
7
7
|
|
|
8
8
|
When deploying your Mastra application to production, it runs as an HTTP server that exposes your agents, workflows, and other functionality as API endpoints. This page covers how to configure and customize the server for a production environment.
|
|
9
9
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Middleware | Server & DB
|
|
2
|
+
title: "Middleware | Server & DB"
|
|
3
3
|
description: "Apply custom middleware functions to intercept requests."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -64,6 +64,35 @@ registerApiRoute("/my-custom-route", {
|
|
|
64
64
|
|
|
65
65
|
## Common examples
|
|
66
66
|
|
|
67
|
+
### Using `runtimeContext`
|
|
68
|
+
|
|
69
|
+
You can populate `runtimeContext` dynamically in server middleware by extracting information from the request. In this example, the `temperature-unit` is set based on the Cloudflare `CF-IPCountry` header to ensure responses match the user's locale.
|
|
70
|
+
|
|
71
|
+
```typescript title="src/mastra/index.ts" showLineNumbers copy
|
|
72
|
+
import { Mastra } from "@mastra/core/mastra";
|
|
73
|
+
import { RuntimeContext } from "@mastra/core/runtime-context";
|
|
74
|
+
import { testWeatherAgent } from "./agents/test-weather-agent";
|
|
75
|
+
|
|
76
|
+
export const mastra = new Mastra({
|
|
77
|
+
agents: { testWeatherAgent },
|
|
78
|
+
server: {
|
|
79
|
+
middleware: [
|
|
80
|
+
async (context, next) => {
|
|
81
|
+
const country = context.req.header("CF-IPCountry");
|
|
82
|
+
const runtimeContext = context.get("runtimeContext");
|
|
83
|
+
|
|
84
|
+
runtimeContext.set(
|
|
85
|
+
"temperature-unit",
|
|
86
|
+
country === "US" ? "fahrenheit" : "celsius",
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
await next();
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
67
96
|
### Authentication
|
|
68
97
|
|
|
69
98
|
```typescript copy
|
|
@@ -143,35 +172,6 @@ be inspected by middleware to tailor behavior:
|
|
|
143
172
|
- `x-mastra-client-type`: identifies the client SDK, e.g. `js` or `python`
|
|
144
173
|
- `x-studio`: request triggered from Studio
|
|
145
174
|
|
|
146
|
-
### Setting `runtimeContext`
|
|
147
|
-
|
|
148
|
-
You can populate `runtimeContext` dynamically in server middleware by extracting information from the request. In this example, the `temperature-unit` is set based on the Cloudflare `CF-IPCountry` header to ensure responses match the user's locale.
|
|
149
|
-
|
|
150
|
-
```typescript title="src/mastra/index.ts" showLineNumbers copy
|
|
151
|
-
import { Mastra } from "@mastra/core/mastra";
|
|
152
|
-
import { RuntimeContext } from "@mastra/core/runtime-context";
|
|
153
|
-
import { testWeatherAgent } from "./agents/test-weather-agent";
|
|
154
|
-
|
|
155
|
-
export const mastra = new Mastra({
|
|
156
|
-
agents: { testWeatherAgent },
|
|
157
|
-
server: {
|
|
158
|
-
middleware: [
|
|
159
|
-
async (context, next) => {
|
|
160
|
-
const country = context.req.header("CF-IPCountry");
|
|
161
|
-
const runtimeContext = context.get("runtimeContext");
|
|
162
|
-
|
|
163
|
-
runtimeContext.set(
|
|
164
|
-
"temperature-unit",
|
|
165
|
-
country === "US" ? "fahrenheit" : "celsius",
|
|
166
|
-
);
|
|
167
|
-
|
|
168
|
-
await next();
|
|
169
|
-
},
|
|
170
|
-
],
|
|
171
|
-
},
|
|
172
|
-
});
|
|
173
|
-
```
|
|
174
|
-
|
|
175
175
|
# Related
|
|
176
176
|
|
|
177
177
|
- [Runtime Context](./runtime-context)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Runtime Context | Server & DB
|
|
2
|
+
title: "Runtime Context | Server & DB"
|
|
3
3
|
description: Learn how to use Mastra's RuntimeContext to provide dynamic, request-specific configuration to agents.
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -188,6 +188,6 @@ export const weatherTool = createTool({
|
|
|
188
188
|
|
|
189
189
|
- [Runtime Context Example](/examples/agents/runtime-context)
|
|
190
190
|
- [Agent Runtime Context](/docs/agents/overview#using-runtimecontext)
|
|
191
|
+
- [Tool Runtime Context](/docs/server-db/runtime-context#accessing-values-with-tools)
|
|
191
192
|
- [Workflow Runtime Context](../workflows/overview#using-runtimecontext)
|
|
192
|
-
- [
|
|
193
|
-
- [Server Middleware Runtime Context](/docs/server-db/middleware)
|
|
193
|
+
- [Middleware Runtime Context](/docs/server-db/middleware#using-runtimecontext)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Streaming Events | Streaming
|
|
2
|
+
title: "Streaming Events | Streaming"
|
|
3
3
|
description: "Learn about the different types of streaming events in Mastra, including text deltas, tool calls, step events, and how to handle them in your applications."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Streaming Overview | Streaming
|
|
2
|
+
title: "Streaming Overview | Streaming"
|
|
3
3
|
description: "Streaming in Mastra enables real-time, incremental responses from both agents and workflows, providing immediate feedback as AI-generated content is produced."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Workflow streaming | Streaming
|
|
2
|
+
title: "Workflow streaming | Streaming"
|
|
3
3
|
description: "Learn how to use workflow streaming in Mastra, including handling workflow execution events, step streaming, and workflow integration with agents and tools."
|
|
4
4
|
---
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
title: "Speech-to-Speech Capabilities in Mastra | Voice
|
|
2
|
+
title: "Speech-to-Speech Capabilities in Mastra | Voice"
|
|
3
3
|
description: Overview of speech-to-speech capabilities in Mastra, including real-time interactions and event-driven architecture.
|
|
4
4
|
---
|
|
5
5
|
|