@mastra/mcp-docs-server 1.2.2-alpha.1 → 1.2.2-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.docs/docs/agents/background-tasks.md +5 -5
  2. package/.docs/docs/agents/code-mode.md +1 -1
  3. package/.docs/docs/agents/goals.md +1 -1
  4. package/.docs/docs/agents/processors.md +1 -1
  5. package/.docs/docs/agents/signal-providers.md +1 -1
  6. package/.docs/docs/agents/signals.md +1 -1
  7. package/.docs/docs/agents/supervisor-agents.md +15 -0
  8. package/.docs/docs/evals/built-in-scorers.md +17 -1
  9. package/.docs/docs/evals/gates-and-verdicts.md +140 -0
  10. package/.docs/docs/evals/overview.md +2 -0
  11. package/.docs/docs/evals/quick-checks.md +136 -0
  12. package/.docs/docs/harness/modes.md +21 -0
  13. package/.docs/docs/harness/overview.md +1 -1
  14. package/.docs/docs/harness/session.md +7 -3
  15. package/.docs/docs/harness/threads-and-state.md +2 -0
  16. package/.docs/docs/memory/observational-memory.md +1 -3
  17. package/.docs/guides/guide/signal-provider.md +1 -1
  18. package/.docs/models/gateways/netlify.md +1 -3
  19. package/.docs/models/gateways/openrouter.md +2 -1
  20. package/.docs/models/gateways/vercel.md +3 -1
  21. package/.docs/models/index.md +1 -1
  22. package/.docs/models/providers/baseten.md +2 -2
  23. package/.docs/models/providers/berget.md +4 -3
  24. package/.docs/models/providers/evroc.md +19 -17
  25. package/.docs/models/providers/huggingface.md +4 -1
  26. package/.docs/models/providers/llmgateway.md +184 -199
  27. package/.docs/models/providers/minimax-cn-coding-plan.md +1 -1
  28. package/.docs/models/providers/minimax-cn.md +1 -1
  29. package/.docs/models/providers/minimax-coding-plan.md +1 -1
  30. package/.docs/models/providers/minimax.md +1 -1
  31. package/.docs/models/providers/neuralwatt.md +2 -1
  32. package/.docs/models/providers/opencode-go.md +1 -1
  33. package/.docs/models/providers/ovhcloud.md +1 -2
  34. package/.docs/models/providers/stepfun.md +3 -2
  35. package/.docs/models/providers/togetherai.md +3 -2
  36. package/.docs/reference/agents/channels.md +4 -0
  37. package/.docs/reference/agents/durable-agent.md +30 -3
  38. package/.docs/reference/agents/generate.md +1 -1
  39. package/.docs/reference/ai-sdk/workflow-snapshot-to-stream.md +51 -0
  40. package/.docs/reference/cli/mastra.md +8 -0
  41. package/.docs/reference/evals/checks.md +210 -0
  42. package/.docs/reference/evals/run-evals.md +71 -1
  43. package/.docs/reference/harness/harness-class.md +90 -50
  44. package/.docs/reference/harness/session.md +35 -4
  45. package/.docs/reference/index.md +2 -0
  46. package/.docs/reference/memory/observational-memory.md +1 -1
  47. package/.docs/reference/memory/serialized-memory-config.md +1 -1
  48. package/.docs/reference/processors/processor-interface.md +2 -0
  49. package/.docs/reference/streaming/agents/stream.md +1 -1
  50. package/CHANGELOG.md +42 -0
  51. package/package.json +3 -3
@@ -1,6 +1,6 @@
1
1
  # ![StepFun logo](https://models.dev/logos/stepfun.svg)StepFun
2
2
 
3
- Access 4 StepFun models through Mastra's model router. Authentication is handled automatically using the `STEPFUN_API_KEY` environment variable.
3
+ Access 5 StepFun models through Mastra's model router. Authentication is handled automatically using the `STEPFUN_API_KEY` environment variable.
4
4
 
5
5
  Learn more in the [StepFun documentation](https://platform.stepfun.com/docs/zh/overview/concept).
6
6
 
@@ -38,6 +38,7 @@ for await (const chunk of stream) {
38
38
  | `stepfun/step-2-16k` | 16K | | | | | | $5 | $16 |
39
39
  | `stepfun/step-3.5-flash` | 256K | | | | | | $0.10 | $0.30 |
40
40
  | `stepfun/step-3.5-flash-2603` | 256K | | | | | | $0.10 | $0.30 |
41
+ | `stepfun/step-3.7-flash` | 256K | | | | | | $0.19 | $1 |
41
42
 
42
43
  ## Advanced configuration
43
44
 
@@ -67,7 +68,7 @@ const agent = new Agent({
67
68
  model: ({ requestContext }) => {
68
69
  const useAdvanced = requestContext.task === "complex";
69
70
  return useAdvanced
70
- ? "stepfun/step-3.5-flash-2603"
71
+ ? "stepfun/step-3.7-flash"
71
72
  : "stepfun/step-1-32k";
72
73
  }
73
74
  });
@@ -1,6 +1,6 @@
1
1
  # ![Together AI logo](https://models.dev/logos/togetherai.svg)Together AI
2
2
 
3
- Access 24 Together AI models through Mastra's model router. Authentication is handled automatically using the `TOGETHER_API_KEY` environment variable.
3
+ Access 25 Together AI models through Mastra's model router. Authentication is handled automatically using the `TOGETHER_API_KEY` environment variable.
4
4
 
5
5
  Learn more in the [Together AI documentation](https://docs.together.ai/docs/serverless-models).
6
6
 
@@ -56,6 +56,7 @@ for await (const chunk of stream) {
56
56
  | `togetherai/Qwen/Qwen3.7-Max` | 1.0M | | | | | | $1 | $4 |
57
57
  | `togetherai/zai-org/GLM-5` | 203K | | | | | | $1 | $3 |
58
58
  | `togetherai/zai-org/GLM-5.1` | 203K | | | | | | $1 | $4 |
59
+ | `togetherai/zai-org/GLM-5.2` | 262K | | | | | | $1 | $4 |
59
60
 
60
61
  ## Advanced configuration
61
62
 
@@ -84,7 +85,7 @@ const agent = new Agent({
84
85
  model: ({ requestContext }) => {
85
86
  const useAdvanced = requestContext.task === "complex";
86
87
  return useAdvanced
87
- ? "togetherai/zai-org/GLM-5.1"
88
+ ? "togetherai/zai-org/GLM-5.2"
88
89
  : "togetherai/LiquidAI/LFM2-24B-A2B";
89
90
  }
90
91
  });
@@ -47,6 +47,10 @@ export const supportAgent = new Agent({
47
47
 
48
48
  **resolveResourceId** (`(ctx: ResolveResourceIdContext) => string | Promise<string>`): Decide which \`resourceId\` owns resource-level memory for a channel thread, separately from who sent the message. Runs only when a new thread is created; reused threads keep their stored owner and never call the hook. Return \`ctx.defaultResourceId\` (\`${platform}:${message.author.userId}\`) to keep the built-in behavior.
49
49
 
50
+ **waitUntil** (`(promise: Promise<unknown>) => void`): Platform \`waitUntil\` function. Required on Vercel so background agent runs survive after the webhook returns 200. On Vercel pass \`waitUntil\` from \`@vercel/functions\`. Cloudflare Workers and Netlify Functions are detected automatically from the request context. AWS Lambda does not need \`waitUntil\` because it waits for the event loop to drain naturally.
51
+
52
+ **resolveWaitUntil** (`(c: Context) => ((promise: Promise<unknown>) => void) | undefined`): Resolver for runtimes where \`waitUntil\` lives on the Hono request context but is not covered by the built-in helper. Resolution order: bare \`waitUntil\` → \`resolveWaitUntil(c)\` → default (Cloudflare Workers, Netlify).
53
+
50
54
  ## Per-adapter options
51
55
 
52
56
  Wrap an adapter in a `ChannelAdapterConfig` object to set per-adapter options:
@@ -203,9 +203,13 @@ interface PrepareResult {
203
203
 
204
204
  **activeTools** (`string[]`): Restricts execution to the named subset of the agent's tools.
205
205
 
206
- **modelSettings** (`object`): Model-specific settings such as temperature.
206
+ **modelSettings** (`object`): Model-specific settings such as temperature. Credential-bearing headers (\`Authorization\`, \`X-Api-Key\`, and similar) are stripped from the serialized snapshot before it crosses process boundaries.
207
207
 
208
- **requireToolApproval** (`boolean`): Require approval for all tool calls, which suspends the run until resumed.
208
+ **stopWhen** (`AgentExecutionOptions['stopWhen']`): Predicate or composition that ends the agentic loop early. The closure rides on the in-process run registry; cross-process resumes degrade to \`maxSteps\` only.
209
+
210
+ **system** (`string | string[]`): Additional system message appended after the agent instructions and before user messages.
211
+
212
+ **requireToolApproval** (`boolean | ((args: { toolName: string; args: unknown; requestContext: RequestContext; workspace?: string }) => boolean | Promise<boolean>)`): Require approval for tool calls. Pass \`true\` or \`false\` to gate all or none, or a function for per-call policy. Function-form policies live on the in-process run registry; cross-process resumes fall back to a \`true\` shadow.
209
213
 
210
214
  **autoResumeSuspendedTools** (`boolean`): Automatically resume tools that suspended, instead of waiting for an external \`resume()\` call.
211
215
 
@@ -217,10 +221,26 @@ interface PrepareResult {
217
221
 
218
222
  **structuredOutput** (`object`): Structured output configuration.
219
223
 
220
- **untilIdle** (`boolean | { maxIdleMs?: number }`): When set, keeps the stream open across background-task continuations until the agent is idle. Pass \`true\` for the default 5-minute idle timeout, or \`{ maxIdleMs }\` to customise. Equivalent to the deprecated \`streamUntilIdle()\` method.
224
+ **untilIdle** (`boolean | { maxIdleMs?: number }`): When set, keeps the stream open across background-task continuations until the agent is idle. Pass \`true\` for the default 5-minute idle timeout, or \`{ maxIdleMs }\` to customise. Equivalent to the deprecated \`streamUntilIdle()\` method. Also supported on \`resume()\`.
225
+
226
+ **disableBackgroundTasks** (`boolean`): Disable background-task dispatch for this run. Background-eligible tools execute inline instead.
227
+
228
+ **tracingOptions** (`AgentExecutionOptions['tracingOptions']`): Tracing metadata, tags, trace ID, parent span ID, and \`requestContextKeys\` forwarded to the agent and model spans. Fully JSON-serializable.
229
+
230
+ **actor** (`AgentExecutionOptions['actor']`): Per-call actor signal forwarded to FGA checks and tool execution.
231
+
232
+ **transform** (`AgentExecutionOptions['transform']`): Per-invocation tool payload transform policy. The \`transformToolPayload\` closure lives on the in-process run registry; only the JSON-safe \`targets\` shadow is serialized.
233
+
234
+ **prepareStep** (`AgentExecutionOptions['prepareStep']`): Per-step preparation hook invoked as a \`PrepareStepProcessor\` at the start of every iteration. Closure-only — stored on the in-process run registry. Cross-process resumes lose the hook.
235
+
236
+ **isTaskComplete** (`AgentExecutionOptions['isTaskComplete']`): Per-call completion policy. Scorer instances and \`onComplete\` live on the in-process run registry; the JSON-safe primitives (\`strategy\`, \`timeout\`, \`parallel\`, \`suppressFeedback\`, \`scorerNames\`) are serialized for cross-process observability.
237
+
238
+ **delegation** (`AgentExecutionOptions['delegation']`): Sub-agent delegation hooks (\`onDelegationStart\`, \`onDelegationComplete\`, \`messageFilter\`). Callbacks are baked into the sub-agent tool wrappers at prepare time. Cross-process resumes lose the callbacks.
221
239
 
222
240
  **versions** (`object`): Version overrides for sub-agent delegation.
223
241
 
242
+ **abortSignal** (`AbortSignal`): External abort signal. Forwarded to the durable run's internal \`AbortController\`, so either source can cancel the run. Cross-process resumes cannot recover the signal — pass a fresh one to \`resume()\` if you need post-resume abortability.
243
+
224
244
  **onChunk** (`(chunk: ChunkType) => void | Promise<void>`): Called for each streamed chunk.
225
245
 
226
246
  **onStepFinish** (`(result: AgentStepFinishEventData) => void | Promise<void>`): Called when a step in the agentic loop finishes.
@@ -231,6 +251,10 @@ interface PrepareResult {
231
251
 
232
252
  **onSuspended** (`(data: AgentSuspendedEventData) => void | Promise<void>`): Called when the run suspends, for example for tool approval.
233
253
 
254
+ **onAbort** (`AgentExecutionOptions['onAbort']`): Called when the run is aborted via \`abortSignal\` or \`result.abort()\`.
255
+
256
+ **onIterationComplete** (`AgentExecutionOptions['onIterationComplete']`): Called after every agentic-loop iteration with the latest \`messageList\`, \`finishReason\`, and \`isFinal\` flag. Observation-only on durable agents: returning \`continue: false\` or feedback does not influence the loop.
257
+
234
258
  `resume()` and `observe()` accept the same lifecycle callbacks (`onChunk`, `onStepFinish`, `onFinish`, `onError`, `onSuspended`). `observe()` also accepts an `offset` to control where replay starts.
235
259
 
236
260
  ## DurableAgentStreamResult
@@ -245,6 +269,7 @@ interface DurableAgentStreamResult<OUTPUT = undefined> {
245
269
  threadId?: string
246
270
  resourceId?: string
247
271
  cleanup: () => void
272
+ abort: () => void
248
273
  }
249
274
  ```
250
275
 
@@ -260,6 +285,8 @@ interface DurableAgentStreamResult<OUTPUT = undefined> {
260
285
 
261
286
  **cleanup** (`() => void`): Unsubscribes from PubSub and clears registry entries for the run. Call it when done with the run.
262
287
 
288
+ **abort** (`() => void`): Aborts the run by flipping the internal \`AbortController\`. Surfaces as an \`AbortError\` inside the durable LLM-execution step and fires the \`onAbort\` callback. Safe to call after the run has finished — a no-op in that case.
289
+
263
290
  ## Related
264
291
 
265
292
  - [`createInngestAgent()`](https://mastra.ai/reference/agents/inngest-agent)
@@ -170,7 +170,7 @@ const response = await agent.generate('Help me organize my day', {
170
170
 
171
171
  **options.activeTools** (`Array<keyof ToolSet> | undefined`): Array of tool names that should be active during execution. If undefined, all available tools are active.
172
172
 
173
- **options.abortSignal** (`AbortSignal`): Signal object that allows you to abort the agent's execution. When the signal is aborted, all ongoing operations will be terminated.
173
+ **options.abortSignal** (`AbortSignal`): Signal object that allows you to abort the agent's execution. When the signal is aborted, all ongoing operations will be terminated, including any in-flight subagent runs the agent delegated to.
174
174
 
175
175
  **options.prepareStep** (`PrepareStepFunction`): Callback function called before each step of multi-step execution.
176
176
 
@@ -0,0 +1,51 @@
1
+ # workflowSnapshotToStream()
2
+
3
+ Converts a `WorkflowState` object (as returned by `workflow.getWorkflowRunById()`) into a `ReadableStream` of AI SDK UIMessage data parts. The stream contains the same `WorkflowDataPart` and `WorkflowStepDataPart` chunks that the live [`workflowRoute()`](https://mastra.ai/reference/ai-sdk/workflow-route) produces.
4
+
5
+ Use this when you want to display a completed or suspended workflow run using the same UI components that render live workflow streams.
6
+
7
+ ## Usage example
8
+
9
+ Next.js App Router endpoint that replays a past workflow run as a stream:
10
+
11
+ ```typescript
12
+ import { workflowSnapshotToStream } from '@mastra/ai-sdk'
13
+ import { createUIMessageStreamResponse } from 'ai'
14
+ import { mastra } from '@/src/mastra'
15
+
16
+ export async function GET(req: Request) {
17
+ const { searchParams } = new URL(req.url)
18
+ const runId = searchParams.get('runId')!
19
+
20
+ const workflowRun = await mastra.getWorkflow('myWorkflow').getWorkflowRunById(runId)
21
+
22
+ if (!workflowRun) {
23
+ return new Response('Not found', { status: 404 })
24
+ }
25
+
26
+ const stream = workflowSnapshotToStream(workflowRun)
27
+ return createUIMessageStreamResponse({ stream })
28
+ }
29
+ ```
30
+
31
+ ## Parameters
32
+
33
+ **workflowRun** (`WorkflowState`): The workflow run state object, as returned by \`getWorkflowRunById()\` or the workflow runs API. Contains \`runId\`, \`workflowName\`, \`status\`, and \`steps\`.
34
+
35
+ ## Returns
36
+
37
+ `ReadableStream` — A stream of AI SDK UIMessage data parts containing:
38
+
39
+ - A `start` marker
40
+ - A `WorkflowDataPart` with the overall workflow status and all step summaries
41
+ - A `WorkflowStepDataPart` for each step with its full output
42
+ - A `finish` marker
43
+
44
+ Pass this stream to [`createUIMessageStreamResponse()`](https://ai-sdk.dev/docs/reference/ai-sdk-ui/create-ui-message-stream-response) or `createUIMessageStream()` from the AI SDK.
45
+
46
+ ## Related
47
+
48
+ - [`workflowRoute()`](https://mastra.ai/reference/ai-sdk/workflow-route): Stream live workflow runs
49
+ - [`handleWorkflowStream()`](https://mastra.ai/reference/ai-sdk/handle-workflow-stream): Framework-agnostic handler for live workflow streaming
50
+ - [`toAISdkStream()`](https://mastra.ai/reference/ai-sdk/to-ai-sdk-stream): Convert live Mastra streams to AI SDK format
51
+ - [`toAISdkMessages()`](https://mastra.ai/reference/ai-sdk/to-ai-sdk-messages): Convert stored agent messages to AI SDK format
@@ -666,6 +666,14 @@ Target a specific Mastra server URL.
666
666
  mastra api --url https://example.com agent list
667
667
  ```
668
668
 
669
+ #### `--server-api-prefix <prefix>`
670
+
671
+ Set the API route prefix of the target server. Defaults to `/api`. Use this when the server is mounted under a custom prefix (for example a `@mastra/fastify` `MastraServer` with `prefix: "/api/mastra-studio"`), the same way `mastra studio` accepts `--server-api-prefix`. You can also set the `MASTRA_API_PREFIX` environment variable instead of passing the flag.
672
+
673
+ ```bash
674
+ mastra api --url https://example.com --server-api-prefix /api/mastra-studio agent list
675
+ ```
676
+
669
677
  #### `--header <"Key: Value">`
670
678
 
671
679
  Send a custom HTTP header. Repeat the flag to send multiple headers.
@@ -0,0 +1,210 @@
1
+ # Quick Checks
2
+
3
+ Quick Checks are zero-LLM, composable micro-scorers for common assertions. They plug into the existing `scorers: [...]` array anywhere scorers are used — in `runEvals`, live scoring, experiments, and Studio.
4
+
5
+ Internally they are standard `createScorer()` instances, so they have the same observability, storage, and pipeline integration as any other scorer.
6
+
7
+ ## Usage example
8
+
9
+ ```typescript
10
+ import { checks } from '@mastra/evals/checks'
11
+ import { runEvals } from '@mastra/core/evals'
12
+ import { myAgent } from '../agents'
13
+
14
+ const result = await runEvals({
15
+ data: [{ input: 'What is the weather in Brooklyn?' }],
16
+ target: myAgent,
17
+ scorers: [
18
+ checks.includes('sunny'),
19
+ checks.calledTool('get_weather'),
20
+ checks.toolOrder(['get_weather', 'summarize']),
21
+ checks.noToolErrors(),
22
+ ],
23
+ })
24
+
25
+ console.log(result.scores)
26
+ ```
27
+
28
+ ## Text checks
29
+
30
+ ### `checks.includes(expected, options?)`
31
+
32
+ Scores 1 if the agent's output text contains the expected substring, 0 otherwise.
33
+
34
+ ```typescript
35
+ checks.includes('sunny')
36
+ checks.includes('Sunny', { ignoreCase: false })
37
+ ```
38
+
39
+ **expected** (`string`): Substring to search for in the output.
40
+
41
+ **options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
42
+
43
+ Returns: `1` if found, `0` otherwise.
44
+
45
+ ### `checks.excludes(unwanted, options?)`
46
+
47
+ Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.
48
+
49
+ ```typescript
50
+ checks.excludes('error')
51
+ checks.excludes('Error', { ignoreCase: false })
52
+ ```
53
+
54
+ **unwanted** (`string`): Substring that must not appear in the output.
55
+
56
+ **options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
57
+
58
+ Returns: `1` if absent, `0` otherwise.
59
+
60
+ ### `checks.equals(expected, options?)`
61
+
62
+ Scores 1 if the output text exactly equals the expected string after optional normalization.
63
+
64
+ ```typescript
65
+ checks.equals('Hello, world!')
66
+ checks.equals('Hello', { ignoreCase: false })
67
+ ```
68
+
69
+ **expected** (`string`): Exact string the output must match.
70
+
71
+ **options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
72
+
73
+ Returns: `1` if equal, `0` otherwise.
74
+
75
+ ### `checks.matches(pattern, options?)`
76
+
77
+ Scores 1 if the output matches the given regular expression.
78
+
79
+ ```typescript
80
+ checks.matches(/\d+°[FC]/)
81
+ checks.matches(/^hello$/, { exact: true })
82
+ ```
83
+
84
+ **pattern** (`RegExp`): Regular expression to test against the output.
85
+
86
+ **options.exact** (`boolean`): Anchor the pattern to match the entire output (adds ^ and $). (Default: `false`)
87
+
88
+ Returns: `1` if matched, `0` otherwise.
89
+
90
+ ### `checks.similarity(expected, options?)`
91
+
92
+ Returns the string similarity score (0-1) between the output and an expected string using the Dice coefficient. When a `threshold` is set, returns binary 1/0 instead.
93
+
94
+ ```typescript
95
+ checks.similarity('Sunny, 72°F')
96
+ checks.similarity('Sunny, 72°F', { threshold: 0.7 })
97
+ ```
98
+
99
+ **expected** (`string`): Reference string to compare against.
100
+
101
+ **options.threshold** (`number`): Minimum similarity score (0-1) to return 1. When omitted, returns the raw similarity score.
102
+
103
+ **options.ignoreCase** (`boolean`): Case-insensitive comparison. (Default: `true`)
104
+
105
+ Returns: Raw similarity score (0-1), or binary `1`/`0` when `threshold` is set.
106
+
107
+ ## Tool call checks
108
+
109
+ ### `checks.calledTool(toolName, options?)`
110
+
111
+ Scores 1 if the agent called the specified tool at least the required number of times.
112
+
113
+ ```typescript
114
+ checks.calledTool('get_weather')
115
+ checks.calledTool('search', { times: 2 })
116
+ ```
117
+
118
+ **toolName** (`string`): Name of the tool to look for.
119
+
120
+ **options.times** (`number`): Minimum number of times the tool must be called. (Default: `1`)
121
+
122
+ Returns: `1` if called at least `times` times, `0` otherwise.
123
+
124
+ ### `checks.didNotCall(toolName)`
125
+
126
+ Scores 1 if the agent did NOT call the specified tool.
127
+
128
+ ```typescript
129
+ checks.didNotCall('delete_user')
130
+ ```
131
+
132
+ **toolName** (`string`): Name of the tool that must not appear.
133
+
134
+ Returns: `1` if the tool was not called, `0` otherwise.
135
+
136
+ ### `checks.toolOrder(expectedOrder)`
137
+
138
+ Scores 1 if the tools were called in the specified order. Uses relaxed matching — other tool calls between the expected tools are allowed.
139
+
140
+ ```typescript
141
+ checks.toolOrder(['search', 'summarize', 'respond'])
142
+ ```
143
+
144
+ **expectedOrder** (`string[]`): Tool names in the expected calling sequence. Must appear as a subsequence of the actual tool calls.
145
+
146
+ Returns: `1` if the expected order is satisfied, `0` otherwise.
147
+
148
+ ### `checks.maxToolCalls(max)`
149
+
150
+ Scores 1 if the agent used no more than `max` tool calls.
151
+
152
+ ```typescript
153
+ checks.maxToolCalls(5)
154
+ ```
155
+
156
+ **max** (`number`): Maximum number of tool calls allowed.
157
+
158
+ Returns: `1` if within the limit, `0` otherwise.
159
+
160
+ ### `checks.usedNoTools()`
161
+
162
+ Scores 1 if the agent made no tool calls at all.
163
+
164
+ ```typescript
165
+ checks.usedNoTools()
166
+ ```
167
+
168
+ Returns: `1` if no tools were called, `0` otherwise.
169
+
170
+ ### `checks.noToolErrors()`
171
+
172
+ Scores 1 if none of the tool invocations resulted in an error state. Detects both error results (`result.error` present) and incomplete tool calls (`state === 'call'`).
173
+
174
+ ```typescript
175
+ checks.noToolErrors()
176
+ ```
177
+
178
+ Returns: `1` if all tool calls succeeded, `0` otherwise.
179
+
180
+ ## Combining checks with other scorers
181
+
182
+ Checks compose with LLM-based and code-based scorers in the same `scorers` array:
183
+
184
+ ```typescript
185
+ import { checks } from '@mastra/evals/checks'
186
+ import { createAnswerRelevancyScorer } from '@mastra/evals/scorers/prebuilt'
187
+ import { runEvals } from '@mastra/core/evals'
188
+ import { myAgent } from '../agents'
189
+
190
+ const result = await runEvals({
191
+ data: [{ input: 'What is the weather in Brooklyn?' }],
192
+ target: myAgent,
193
+ scorers: [
194
+ // Zero-LLM checks
195
+ checks.includes('Brooklyn'),
196
+ checks.calledTool('get_weather'),
197
+ checks.noToolErrors(),
198
+ // LLM-based scorer
199
+ createAnswerRelevancyScorer({ model: 'openai/gpt-5-mini' }),
200
+ ],
201
+ })
202
+ ```
203
+
204
+ ## Related
205
+
206
+ - [Quick Checks overview](https://mastra.ai/docs/evals/quick-checks)
207
+ - [Built-in Scorers](https://mastra.ai/docs/evals/built-in-scorers)
208
+ - [`createScorer()` reference](https://mastra.ai/reference/evals/create-scorer)
209
+ - [`runEvals()` reference](https://mastra.ai/reference/evals/run-evals)
210
+ - [Custom Scorers](https://mastra.ai/docs/evals/custom-scorers)
@@ -29,13 +29,34 @@ console.log(`Average scores:`, result.scores)
29
29
  console.log(`Processed ${result.summary.totalItems} items`)
30
30
  ```
31
31
 
32
+ ### With gates and thresholds
33
+
34
+ ```typescript
35
+ import { runEvals } from '@mastra/core/evals'
36
+ import { checks } from '@mastra/evals/checks'
37
+ import { faithfulnessScorer } from './scorers'
38
+
39
+ const result = await runEvals({
40
+ target: myAgent,
41
+ data: [{ input: 'What is the weather in Brooklyn?' }],
42
+ gates: [checks.calledTool('get_weather'), checks.noToolErrors()],
43
+ scorers: [{ scorer: faithfulnessScorer, threshold: 0.7 }, checks.includes('Brooklyn')],
44
+ })
45
+
46
+ result.verdict // 'passed' | 'scored' | 'failed'
47
+ result.gateResults // [{ id, passed, score }]
48
+ result.thresholdResults // [{ id, passed, averageScore, threshold }]
49
+ ```
50
+
32
51
  ## Parameters
33
52
 
34
53
  **target** (`Agent | Workflow`): The agent or workflow to evaluate.
35
54
 
36
55
  **data** (`RunEvalsDataItem[]`): Array of test cases with input data and optional ground truth.
37
56
 
38
- **scorers** (`MastraScorer[] | AgentScorerConfig | WorkflowScorerConfig`): Scorers to use. A flat array applies all scorers to the raw output. For agents, an \`AgentScorerConfig\` object separates agent-level and trajectory scorers. For workflows, a \`WorkflowScorerConfig\` object specifies scorers for the workflow, individual steps, and trajectory.
57
+ **scorers** (`ScorerEntry[] | AgentScorerConfig | WorkflowScorerConfig`): Scorers to use. Each entry is either a bare \`MastraScorer\` or \`{ scorer, threshold }\` for threshold tracking. An \`AgentScorerConfig\` object separates agent-level and trajectory scorers. A \`WorkflowScorerConfig\` object specifies scorers for the workflow, individual steps, and trajectory.
58
+
59
+ **gates** (`MastraScorer[]`): Scorers that must score 1.0 for the run to pass. If any gate averages below 1.0 across data items, the verdict is \`failed\`. Gates run before regular scorers on each data item.
39
60
 
40
61
  **targetOptions** (`AgentExecutionOptions | WorkflowRunOptions`): Options forwarded to the target during execution. For agents: options passed to agent.generate() (e.g. maxSteps, modelSettings, instructions). For workflows: options passed to run.start() (e.g. perStep, outputOptions, initialState).
41
62
 
@@ -83,8 +104,55 @@ For workflows, use `WorkflowScorerConfig` to specify scorers at different levels
83
104
 
84
105
  **summary.totalItems** (`number`): Total number of test cases processed.
85
106
 
107
+ **verdict** (`'passed' | 'scored' | 'failed'`): Present when \`gates\` or threshold-bearing scorers are provided. \`passed\` = all gates and thresholds met. \`scored\` = gates passed but a threshold was missed. \`failed\` = at least one gate did not score 1.0.
108
+
109
+ **gateResults** (`GateResult[]`): Per-gate results averaged across all data items. Each entry has \`id\`, \`passed\` (boolean), and \`score\` (0–1).
110
+
111
+ **thresholdResults** (`ThresholdResult[]`): Per-threshold-scorer results averaged across all data items. Each entry has \`id\`, \`passed\`, \`averageScore\`, and \`threshold\`.
112
+
113
+ ## ScorerEntry
114
+
115
+ A scorer entry in the `scorers` array can be either a bare scorer or a scorer with a threshold:
116
+
117
+ **scorer** (`MastraScorer`): The scorer instance.
118
+
119
+ **threshold** (`number | { min?: number; max?: number }`): A number implies minimum threshold (score at or above passes). Use \`{ min, max }\` for range-based checks — e.g. \`{ max: 0.3 }\` for scorers like hallucination where a high score is bad. Both \`min\` and \`max\` must be between 0 and 1.
120
+
86
121
  ## Examples
87
122
 
123
+ ### Gates and verdict
124
+
125
+ Use `gates` for hard pass/fail requirements and `{ scorer, threshold }` for tracked quality metrics:
126
+
127
+ ```typescript
128
+ import { runEvals } from '@mastra/core/evals'
129
+ import { checks } from '@mastra/evals/checks'
130
+
131
+ const result = await runEvals({
132
+ target: weatherAgent,
133
+ data: [{ input: 'What is the weather in Brooklyn?' }],
134
+ gates: [checks.calledTool('get_weather'), checks.noToolErrors()],
135
+ scorers: [
136
+ { scorer: faithfulnessScorer, threshold: 0.7 }, // min threshold (number shorthand)
137
+ { scorer: hallucinationScorer, threshold: { max: 0.3 } }, // max threshold (high = bad)
138
+ { scorer: toneScorer, threshold: { min: 0.5, max: 0.9 } }, // range threshold
139
+ checks.includes('Brooklyn'), // bare scorer, no threshold
140
+ ],
141
+ })
142
+
143
+ if (result.verdict === 'failed') {
144
+ console.log(
145
+ 'Gate failures:',
146
+ result.gateResults?.filter(g => !g.passed),
147
+ )
148
+ } else if (result.verdict === 'scored') {
149
+ console.log(
150
+ 'Threshold misses:',
151
+ result.thresholdResults?.filter(t => !t.passed),
152
+ )
153
+ }
154
+ ```
155
+
88
156
  ### Agent Evaluation
89
157
 
90
158
  ```typescript
@@ -246,6 +314,8 @@ const result = await runEvals({
246
314
 
247
315
  ## Related
248
316
 
317
+ - [Gates and Verdicts](https://mastra.ai/docs/evals/gates-and-verdicts): Conceptual guide to severity semantics
318
+ - [Quick Checks](https://mastra.ai/reference/evals/checks): Zero-LLM composable micro-scorers
249
319
  - [createScorer()](https://mastra.ai/reference/evals/create-scorer): Create custom scorers for experiments
250
320
  - [MastraScorer](https://mastra.ai/reference/evals/mastra-scorer): Learn about scorer structure and methods
251
321
  - [Trajectory Accuracy](https://mastra.ai/reference/evals/trajectory-accuracy): Built-in trajectory evaluation scorers