@mastra/mcp-docs-server 1.2.2-alpha.1 → 1.2.2-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/docs/agents/background-tasks.md +5 -5
- package/.docs/docs/agents/code-mode.md +1 -1
- package/.docs/docs/agents/goals.md +1 -1
- package/.docs/docs/agents/processors.md +1 -1
- package/.docs/docs/agents/signal-providers.md +1 -1
- package/.docs/docs/agents/signals.md +1 -1
- package/.docs/docs/agents/supervisor-agents.md +15 -0
- package/.docs/docs/evals/built-in-scorers.md +17 -1
- package/.docs/docs/evals/gates-and-verdicts.md +140 -0
- package/.docs/docs/evals/overview.md +2 -0
- package/.docs/docs/evals/quick-checks.md +136 -0
- package/.docs/docs/harness/modes.md +21 -0
- package/.docs/docs/harness/overview.md +1 -1
- package/.docs/docs/harness/session.md +7 -3
- package/.docs/docs/harness/threads-and-state.md +2 -0
- package/.docs/docs/memory/observational-memory.md +1 -3
- package/.docs/guides/guide/signal-provider.md +1 -1
- package/.docs/models/gateways/netlify.md +1 -3
- package/.docs/models/gateways/openrouter.md +2 -1
- package/.docs/models/gateways/vercel.md +3 -1
- package/.docs/models/index.md +1 -1
- package/.docs/models/providers/baseten.md +2 -2
- package/.docs/models/providers/berget.md +4 -3
- package/.docs/models/providers/evroc.md +19 -17
- package/.docs/models/providers/huggingface.md +4 -1
- package/.docs/models/providers/llmgateway.md +184 -199
- package/.docs/models/providers/minimax-cn-coding-plan.md +1 -1
- package/.docs/models/providers/minimax-cn.md +1 -1
- package/.docs/models/providers/minimax-coding-plan.md +1 -1
- package/.docs/models/providers/minimax.md +1 -1
- package/.docs/models/providers/neuralwatt.md +2 -1
- package/.docs/models/providers/opencode-go.md +1 -1
- package/.docs/models/providers/ovhcloud.md +1 -2
- package/.docs/models/providers/stepfun.md +3 -2
- package/.docs/models/providers/togetherai.md +3 -2
- package/.docs/reference/agents/channels.md +4 -0
- package/.docs/reference/agents/durable-agent.md +30 -3
- package/.docs/reference/agents/generate.md +1 -1
- package/.docs/reference/ai-sdk/workflow-snapshot-to-stream.md +51 -0
- package/.docs/reference/cli/mastra.md +8 -0
- package/.docs/reference/evals/checks.md +210 -0
- package/.docs/reference/evals/run-evals.md +71 -1
- package/.docs/reference/harness/harness-class.md +90 -50
- package/.docs/reference/harness/session.md +35 -4
- package/.docs/reference/index.md +2 -0
- package/.docs/reference/memory/observational-memory.md +1 -1
- package/.docs/reference/memory/serialized-memory-config.md +1 -1
- package/.docs/reference/processors/processor-interface.md +2 -0
- package/.docs/reference/streaming/agents/stream.md +1 -1
- package/CHANGELOG.md +42 -0
- package/package.json +3 -3
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# StepFun
|
|
2
2
|
|
|
3
|
-
Access
|
|
3
|
+
Access 5 StepFun models through Mastra's model router. Authentication is handled automatically using the `STEPFUN_API_KEY` environment variable.
|
|
4
4
|
|
|
5
5
|
Learn more in the [StepFun documentation](https://platform.stepfun.com/docs/zh/overview/concept).
|
|
6
6
|
|
|
@@ -38,6 +38,7 @@ for await (const chunk of stream) {
|
|
|
38
38
|
| `stepfun/step-2-16k` | 16K | | | | | | $5 | $16 |
|
|
39
39
|
| `stepfun/step-3.5-flash` | 256K | | | | | | $0.10 | $0.30 |
|
|
40
40
|
| `stepfun/step-3.5-flash-2603` | 256K | | | | | | $0.10 | $0.30 |
|
|
41
|
+
| `stepfun/step-3.7-flash` | 256K | | | | | | $0.19 | $1 |
|
|
41
42
|
|
|
42
43
|
## Advanced configuration
|
|
43
44
|
|
|
@@ -67,7 +68,7 @@ const agent = new Agent({
|
|
|
67
68
|
model: ({ requestContext }) => {
|
|
68
69
|
const useAdvanced = requestContext.task === "complex";
|
|
69
70
|
return useAdvanced
|
|
70
|
-
? "stepfun/step-3.
|
|
71
|
+
? "stepfun/step-3.7-flash"
|
|
71
72
|
: "stepfun/step-1-32k";
|
|
72
73
|
}
|
|
73
74
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Together AI
|
|
2
2
|
|
|
3
|
-
Access
|
|
3
|
+
Access 25 Together AI models through Mastra's model router. Authentication is handled automatically using the `TOGETHER_API_KEY` environment variable.
|
|
4
4
|
|
|
5
5
|
Learn more in the [Together AI documentation](https://docs.together.ai/docs/serverless-models).
|
|
6
6
|
|
|
@@ -56,6 +56,7 @@ for await (const chunk of stream) {
|
|
|
56
56
|
| `togetherai/Qwen/Qwen3.7-Max` | 1.0M | | | | | | $1 | $4 |
|
|
57
57
|
| `togetherai/zai-org/GLM-5` | 203K | | | | | | $1 | $3 |
|
|
58
58
|
| `togetherai/zai-org/GLM-5.1` | 203K | | | | | | $1 | $4 |
|
|
59
|
+
| `togetherai/zai-org/GLM-5.2` | 262K | | | | | | $1 | $4 |
|
|
59
60
|
|
|
60
61
|
## Advanced configuration
|
|
61
62
|
|
|
@@ -84,7 +85,7 @@ const agent = new Agent({
|
|
|
84
85
|
model: ({ requestContext }) => {
|
|
85
86
|
const useAdvanced = requestContext.task === "complex";
|
|
86
87
|
return useAdvanced
|
|
87
|
-
? "togetherai/zai-org/GLM-5.
|
|
88
|
+
? "togetherai/zai-org/GLM-5.2"
|
|
88
89
|
: "togetherai/LiquidAI/LFM2-24B-A2B";
|
|
89
90
|
}
|
|
90
91
|
});
|
|
@@ -47,6 +47,10 @@ export const supportAgent = new Agent({
|
|
|
47
47
|
|
|
48
48
|
**resolveResourceId** (`(ctx: ResolveResourceIdContext) => string | Promise<string>`): Decide which \`resourceId\` owns resource-level memory for a channel thread, separately from who sent the message. Runs only when a new thread is created; reused threads keep their stored owner and never call the hook. Return \`ctx.defaultResourceId\` (\`${platform}:${message.author.userId}\`) to keep the built-in behavior.
|
|
49
49
|
|
|
50
|
+
**waitUntil** (`(promise: Promise<unknown>) => void`): Platform \`waitUntil\` function. Required on Vercel so background agent runs survive after the webhook returns 200. On Vercel pass \`waitUntil\` from \`@vercel/functions\`. Cloudflare Workers and Netlify Functions are detected automatically from the request context. AWS Lambda does not need \`waitUntil\` because it waits for the event loop to drain naturally.
|
|
51
|
+
|
|
52
|
+
**resolveWaitUntil** (`(c: Context) => ((promise: Promise<unknown>) => void) | undefined`): Resolver for runtimes where \`waitUntil\` lives on the Hono request context but is not covered by the built-in helper. Resolution order: bare \`waitUntil\` → \`resolveWaitUntil(c)\` → default (Cloudflare Workers, Netlify).
|
|
53
|
+
|
|
50
54
|
## Per-adapter options
|
|
51
55
|
|
|
52
56
|
Wrap an adapter in a `ChannelAdapterConfig` object to set per-adapter options:
|
|
@@ -203,9 +203,13 @@ interface PrepareResult {
|
|
|
203
203
|
|
|
204
204
|
**activeTools** (`string[]`): Restricts execution to the named subset of the agent's tools.
|
|
205
205
|
|
|
206
|
-
**modelSettings** (`object`): Model-specific settings such as temperature.
|
|
206
|
+
**modelSettings** (`object`): Model-specific settings such as temperature. Credential-bearing headers (\`Authorization\`, \`X-Api-Key\`, and similar) are stripped from the serialized snapshot before it crosses process boundaries.
|
|
207
207
|
|
|
208
|
-
**
|
|
208
|
+
**stopWhen** (`AgentExecutionOptions['stopWhen']`): Predicate or composition that ends the agentic loop early. The closure rides on the in-process run registry; cross-process resumes degrade to \`maxSteps\` only.
|
|
209
|
+
|
|
210
|
+
**system** (`string | string[]`): Additional system message appended after the agent instructions and before user messages.
|
|
211
|
+
|
|
212
|
+
**requireToolApproval** (`boolean | ((args: { toolName: string; args: unknown; requestContext: RequestContext; workspace?: string }) => boolean | Promise<boolean>)`): Require approval for tool calls. Pass \`true\` or \`false\` to gate all or none, or a function for per-call policy. Function-form policies live on the in-process run registry; cross-process resumes fall back to a \`true\` shadow.
|
|
209
213
|
|
|
210
214
|
**autoResumeSuspendedTools** (`boolean`): Automatically resume tools that suspended, instead of waiting for an external \`resume()\` call.
|
|
211
215
|
|
|
@@ -217,10 +221,26 @@ interface PrepareResult {
|
|
|
217
221
|
|
|
218
222
|
**structuredOutput** (`object`): Structured output configuration.
|
|
219
223
|
|
|
220
|
-
**untilIdle** (`boolean | { maxIdleMs?: number }`): When set, keeps the stream open across background-task continuations until the agent is idle. Pass \`true\` for the default 5-minute idle timeout, or \`{ maxIdleMs }\` to customise. Equivalent to the deprecated \`streamUntilIdle()\` method.
|
|
224
|
+
**untilIdle** (`boolean | { maxIdleMs?: number }`): When set, keeps the stream open across background-task continuations until the agent is idle. Pass \`true\` for the default 5-minute idle timeout, or \`{ maxIdleMs }\` to customise. Equivalent to the deprecated \`streamUntilIdle()\` method. Also supported on \`resume()\`.
|
|
225
|
+
|
|
226
|
+
**disableBackgroundTasks** (`boolean`): Disable background-task dispatch for this run. Background-eligible tools execute inline instead.
|
|
227
|
+
|
|
228
|
+
**tracingOptions** (`AgentExecutionOptions['tracingOptions']`): Tracing metadata, tags, trace ID, parent span ID, and \`requestContextKeys\` forwarded to the agent and model spans. Fully JSON-serializable.
|
|
229
|
+
|
|
230
|
+
**actor** (`AgentExecutionOptions['actor']`): Per-call actor signal forwarded to FGA checks and tool execution.
|
|
231
|
+
|
|
232
|
+
**transform** (`AgentExecutionOptions['transform']`): Per-invocation tool payload transform policy. The \`transformToolPayload\` closure lives on the in-process run registry; only the JSON-safe \`targets\` shadow is serialized.
|
|
233
|
+
|
|
234
|
+
**prepareStep** (`AgentExecutionOptions['prepareStep']`): Per-step preparation hook invoked as a \`PrepareStepProcessor\` at the start of every iteration. Closure-only — stored on the in-process run registry. Cross-process resumes lose the hook.
|
|
235
|
+
|
|
236
|
+
**isTaskComplete** (`AgentExecutionOptions['isTaskComplete']`): Per-call completion policy. Scorer instances and \`onComplete\` live on the in-process run registry; the JSON-safe primitives (\`strategy\`, \`timeout\`, \`parallel\`, \`suppressFeedback\`, \`scorerNames\`) are serialized for cross-process observability.
|
|
237
|
+
|
|
238
|
+
**delegation** (`AgentExecutionOptions['delegation']`): Sub-agent delegation hooks (\`onDelegationStart\`, \`onDelegationComplete\`, \`messageFilter\`). Callbacks are baked into the sub-agent tool wrappers at prepare time. Cross-process resumes lose the callbacks.
|
|
221
239
|
|
|
222
240
|
**versions** (`object`): Version overrides for sub-agent delegation.
|
|
223
241
|
|
|
242
|
+
**abortSignal** (`AbortSignal`): External abort signal. Forwarded to the durable run's internal \`AbortController\`, so either source can cancel the run. Cross-process resumes cannot recover the signal — pass a fresh one to \`resume()\` if you need post-resume abortability.
|
|
243
|
+
|
|
224
244
|
**onChunk** (`(chunk: ChunkType) => void | Promise<void>`): Called for each streamed chunk.
|
|
225
245
|
|
|
226
246
|
**onStepFinish** (`(result: AgentStepFinishEventData) => void | Promise<void>`): Called when a step in the agentic loop finishes.
|
|
@@ -231,6 +251,10 @@ interface PrepareResult {
|
|
|
231
251
|
|
|
232
252
|
**onSuspended** (`(data: AgentSuspendedEventData) => void | Promise<void>`): Called when the run suspends, for example for tool approval.
|
|
233
253
|
|
|
254
|
+
**onAbort** (`AgentExecutionOptions['onAbort']`): Called when the run is aborted via \`abortSignal\` or \`result.abort()\`.
|
|
255
|
+
|
|
256
|
+
**onIterationComplete** (`AgentExecutionOptions['onIterationComplete']`): Called after every agentic-loop iteration with the latest \`messageList\`, \`finishReason\`, and \`isFinal\` flag. Observation-only on durable agents: returning \`continue: false\` or feedback does not influence the loop.
|
|
257
|
+
|
|
234
258
|
`resume()` and `observe()` accept the same lifecycle callbacks (`onChunk`, `onStepFinish`, `onFinish`, `onError`, `onSuspended`). `observe()` also accepts an `offset` to control where replay starts.
|
|
235
259
|
|
|
236
260
|
## DurableAgentStreamResult
|
|
@@ -245,6 +269,7 @@ interface DurableAgentStreamResult<OUTPUT = undefined> {
|
|
|
245
269
|
threadId?: string
|
|
246
270
|
resourceId?: string
|
|
247
271
|
cleanup: () => void
|
|
272
|
+
abort: () => void
|
|
248
273
|
}
|
|
249
274
|
```
|
|
250
275
|
|
|
@@ -260,6 +285,8 @@ interface DurableAgentStreamResult<OUTPUT = undefined> {
|
|
|
260
285
|
|
|
261
286
|
**cleanup** (`() => void`): Unsubscribes from PubSub and clears registry entries for the run. Call it when done with the run.
|
|
262
287
|
|
|
288
|
+
**abort** (`() => void`): Aborts the run by flipping the internal \`AbortController\`. Surfaces as an \`AbortError\` inside the durable LLM-execution step and fires the \`onAbort\` callback. Safe to call after the run has finished — a no-op in that case.
|
|
289
|
+
|
|
263
290
|
## Related
|
|
264
291
|
|
|
265
292
|
- [`createInngestAgent()`](https://mastra.ai/reference/agents/inngest-agent)
|
|
@@ -170,7 +170,7 @@ const response = await agent.generate('Help me organize my day', {
|
|
|
170
170
|
|
|
171
171
|
**options.activeTools** (`Array<keyof ToolSet> | undefined`): Array of tool names that should be active during execution. If undefined, all available tools are active.
|
|
172
172
|
|
|
173
|
-
**options.abortSignal** (`AbortSignal`): Signal object that allows you to abort the agent's execution. When the signal is aborted, all ongoing operations will be terminated.
|
|
173
|
+
**options.abortSignal** (`AbortSignal`): Signal object that allows you to abort the agent's execution. When the signal is aborted, all ongoing operations will be terminated, including any in-flight subagent runs the agent delegated to.
|
|
174
174
|
|
|
175
175
|
**options.prepareStep** (`PrepareStepFunction`): Callback function called before each step of multi-step execution.
|
|
176
176
|
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# workflowSnapshotToStream()
|
|
2
|
+
|
|
3
|
+
Converts a `WorkflowState` object (as returned by `workflow.getWorkflowRunById()`) into a `ReadableStream` of AI SDK UIMessage data parts. The stream contains the same `WorkflowDataPart` and `WorkflowStepDataPart` chunks that the live [`workflowRoute()`](https://mastra.ai/reference/ai-sdk/workflow-route) produces.
|
|
4
|
+
|
|
5
|
+
Use this when you want to display a completed or suspended workflow run using the same UI components that render live workflow streams.
|
|
6
|
+
|
|
7
|
+
## Usage example
|
|
8
|
+
|
|
9
|
+
Next.js App Router endpoint that replays a past workflow run as a stream:
|
|
10
|
+
|
|
11
|
+
```typescript
|
|
12
|
+
import { workflowSnapshotToStream } from '@mastra/ai-sdk'
|
|
13
|
+
import { createUIMessageStreamResponse } from 'ai'
|
|
14
|
+
import { mastra } from '@/src/mastra'
|
|
15
|
+
|
|
16
|
+
export async function GET(req: Request) {
|
|
17
|
+
const { searchParams } = new URL(req.url)
|
|
18
|
+
const runId = searchParams.get('runId')!
|
|
19
|
+
|
|
20
|
+
const workflowRun = await mastra.getWorkflow('myWorkflow').getWorkflowRunById(runId)
|
|
21
|
+
|
|
22
|
+
if (!workflowRun) {
|
|
23
|
+
return new Response('Not found', { status: 404 })
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const stream = workflowSnapshotToStream(workflowRun)
|
|
27
|
+
return createUIMessageStreamResponse({ stream })
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Parameters
|
|
32
|
+
|
|
33
|
+
**workflowRun** (`WorkflowState`): The workflow run state object, as returned by \`getWorkflowRunById()\` or the workflow runs API. Contains \`runId\`, \`workflowName\`, \`status\`, and \`steps\`.
|
|
34
|
+
|
|
35
|
+
## Returns
|
|
36
|
+
|
|
37
|
+
`ReadableStream` — A stream of AI SDK UIMessage data parts containing:
|
|
38
|
+
|
|
39
|
+
- A `start` marker
|
|
40
|
+
- A `WorkflowDataPart` with the overall workflow status and all step summaries
|
|
41
|
+
- A `WorkflowStepDataPart` for each step with its full output
|
|
42
|
+
- A `finish` marker
|
|
43
|
+
|
|
44
|
+
Pass this stream to [`createUIMessageStreamResponse()`](https://ai-sdk.dev/docs/reference/ai-sdk-ui/create-ui-message-stream-response) or `createUIMessageStream()` from the AI SDK.
|
|
45
|
+
|
|
46
|
+
## Related
|
|
47
|
+
|
|
48
|
+
- [`workflowRoute()`](https://mastra.ai/reference/ai-sdk/workflow-route): Stream live workflow runs
|
|
49
|
+
- [`handleWorkflowStream()`](https://mastra.ai/reference/ai-sdk/handle-workflow-stream): Framework-agnostic handler for live workflow streaming
|
|
50
|
+
- [`toAISdkStream()`](https://mastra.ai/reference/ai-sdk/to-ai-sdk-stream): Convert live Mastra streams to AI SDK format
|
|
51
|
+
- [`toAISdkMessages()`](https://mastra.ai/reference/ai-sdk/to-ai-sdk-messages): Convert stored agent messages to AI SDK format
|
|
@@ -666,6 +666,14 @@ Target a specific Mastra server URL.
|
|
|
666
666
|
mastra api --url https://example.com agent list
|
|
667
667
|
```
|
|
668
668
|
|
|
669
|
+
#### `--server-api-prefix <prefix>`
|
|
670
|
+
|
|
671
|
+
Set the API route prefix of the target server. Defaults to `/api`. Use this when the server is mounted under a custom prefix (for example a `@mastra/fastify` `MastraServer` with `prefix: "/api/mastra-studio"`), the same way `mastra studio` accepts `--server-api-prefix`. You can also set the `MASTRA_API_PREFIX` environment variable instead of passing the flag.
|
|
672
|
+
|
|
673
|
+
```bash
|
|
674
|
+
mastra api --url https://example.com --server-api-prefix /api/mastra-studio agent list
|
|
675
|
+
```
|
|
676
|
+
|
|
669
677
|
#### `--header <"Key: Value">`
|
|
670
678
|
|
|
671
679
|
Send a custom HTTP header. Repeat the flag to send multiple headers.
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# Quick Checks
|
|
2
|
+
|
|
3
|
+
Quick Checks are zero-LLM, composable micro-scorers for common assertions. They plug into the existing `scorers: [...]` array anywhere scorers are used — in `runEvals`, live scoring, experiments, and Studio.
|
|
4
|
+
|
|
5
|
+
Internally they are standard `createScorer()` instances, so they have the same observability, storage, and pipeline integration as any other scorer.
|
|
6
|
+
|
|
7
|
+
## Usage example
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
import { checks } from '@mastra/evals/checks'
|
|
11
|
+
import { runEvals } from '@mastra/core/evals'
|
|
12
|
+
import { myAgent } from '../agents'
|
|
13
|
+
|
|
14
|
+
const result = await runEvals({
|
|
15
|
+
data: [{ input: 'What is the weather in Brooklyn?' }],
|
|
16
|
+
target: myAgent,
|
|
17
|
+
scorers: [
|
|
18
|
+
checks.includes('sunny'),
|
|
19
|
+
checks.calledTool('get_weather'),
|
|
20
|
+
checks.toolOrder(['get_weather', 'summarize']),
|
|
21
|
+
checks.noToolErrors(),
|
|
22
|
+
],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
console.log(result.scores)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Text checks
|
|
29
|
+
|
|
30
|
+
### `checks.includes(expected, options?)`
|
|
31
|
+
|
|
32
|
+
Scores 1 if the agent's output text contains the expected substring, 0 otherwise.
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
checks.includes('sunny')
|
|
36
|
+
checks.includes('Sunny', { ignoreCase: false })
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
**expected** (`string`): Substring to search for in the output.
|
|
40
|
+
|
|
41
|
+
**options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
|
|
42
|
+
|
|
43
|
+
Returns: `1` if found, `0` otherwise.
|
|
44
|
+
|
|
45
|
+
### `checks.excludes(unwanted, options?)`
|
|
46
|
+
|
|
47
|
+
Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.
|
|
48
|
+
|
|
49
|
+
```typescript
|
|
50
|
+
checks.excludes('error')
|
|
51
|
+
checks.excludes('Error', { ignoreCase: false })
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
**unwanted** (`string`): Substring that must not appear in the output.
|
|
55
|
+
|
|
56
|
+
**options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
|
|
57
|
+
|
|
58
|
+
Returns: `1` if absent, `0` otherwise.
|
|
59
|
+
|
|
60
|
+
### `checks.equals(expected, options?)`
|
|
61
|
+
|
|
62
|
+
Scores 1 if the output text exactly equals the expected string after optional normalization.
|
|
63
|
+
|
|
64
|
+
```typescript
|
|
65
|
+
checks.equals('Hello, world!')
|
|
66
|
+
checks.equals('Hello', { ignoreCase: false })
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
**expected** (`string`): Exact string the output must match.
|
|
70
|
+
|
|
71
|
+
**options.ignoreCase** (`boolean`): Case-insensitive matching. (Default: `true`)
|
|
72
|
+
|
|
73
|
+
Returns: `1` if equal, `0` otherwise.
|
|
74
|
+
|
|
75
|
+
### `checks.matches(pattern, options?)`
|
|
76
|
+
|
|
77
|
+
Scores 1 if the output matches the given regular expression.
|
|
78
|
+
|
|
79
|
+
```typescript
|
|
80
|
+
checks.matches(/\d+°[FC]/)
|
|
81
|
+
checks.matches(/^hello$/, { exact: true })
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**pattern** (`RegExp`): Regular expression to test against the output.
|
|
85
|
+
|
|
86
|
+
**options.exact** (`boolean`): Anchor the pattern to match the entire output (adds ^ and $). (Default: `false`)
|
|
87
|
+
|
|
88
|
+
Returns: `1` if matched, `0` otherwise.
|
|
89
|
+
|
|
90
|
+
### `checks.similarity(expected, options?)`
|
|
91
|
+
|
|
92
|
+
Returns the string similarity score (0-1) between the output and an expected string using the Dice coefficient. When a `threshold` is set, returns binary 1/0 instead.
|
|
93
|
+
|
|
94
|
+
```typescript
|
|
95
|
+
checks.similarity('Sunny, 72°F')
|
|
96
|
+
checks.similarity('Sunny, 72°F', { threshold: 0.7 })
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
**expected** (`string`): Reference string to compare against.
|
|
100
|
+
|
|
101
|
+
**options.threshold** (`number`): Minimum similarity score (0-1) to return 1. When omitted, returns the raw similarity score.
|
|
102
|
+
|
|
103
|
+
**options.ignoreCase** (`boolean`): Case-insensitive comparison. (Default: `true`)
|
|
104
|
+
|
|
105
|
+
Returns: Raw similarity score (0-1), or binary `1`/`0` when `threshold` is set.
|
|
106
|
+
|
|
107
|
+
## Tool call checks
|
|
108
|
+
|
|
109
|
+
### `checks.calledTool(toolName, options?)`
|
|
110
|
+
|
|
111
|
+
Scores 1 if the agent called the specified tool at least the required number of times.
|
|
112
|
+
|
|
113
|
+
```typescript
|
|
114
|
+
checks.calledTool('get_weather')
|
|
115
|
+
checks.calledTool('search', { times: 2 })
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
**toolName** (`string`): Name of the tool to look for.
|
|
119
|
+
|
|
120
|
+
**options.times** (`number`): Minimum number of times the tool must be called. (Default: `1`)
|
|
121
|
+
|
|
122
|
+
Returns: `1` if called at least `times` times, `0` otherwise.
|
|
123
|
+
|
|
124
|
+
### `checks.didNotCall(toolName)`
|
|
125
|
+
|
|
126
|
+
Scores 1 if the agent did NOT call the specified tool.
|
|
127
|
+
|
|
128
|
+
```typescript
|
|
129
|
+
checks.didNotCall('delete_user')
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**toolName** (`string`): Name of the tool that must not appear.
|
|
133
|
+
|
|
134
|
+
Returns: `1` if the tool was not called, `0` otherwise.
|
|
135
|
+
|
|
136
|
+
### `checks.toolOrder(expectedOrder)`
|
|
137
|
+
|
|
138
|
+
Scores 1 if the tools were called in the specified order. Uses relaxed matching — other tool calls between the expected tools are allowed.
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
checks.toolOrder(['search', 'summarize', 'respond'])
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
**expectedOrder** (`string[]`): Tool names in the expected calling sequence. Must appear as a subsequence of the actual tool calls.
|
|
145
|
+
|
|
146
|
+
Returns: `1` if the expected order is satisfied, `0` otherwise.
|
|
147
|
+
|
|
148
|
+
### `checks.maxToolCalls(max)`
|
|
149
|
+
|
|
150
|
+
Scores 1 if the agent used no more than `max` tool calls.
|
|
151
|
+
|
|
152
|
+
```typescript
|
|
153
|
+
checks.maxToolCalls(5)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
**max** (`number`): Maximum number of tool calls allowed.
|
|
157
|
+
|
|
158
|
+
Returns: `1` if within the limit, `0` otherwise.
|
|
159
|
+
|
|
160
|
+
### `checks.usedNoTools()`
|
|
161
|
+
|
|
162
|
+
Scores 1 if the agent made no tool calls at all.
|
|
163
|
+
|
|
164
|
+
```typescript
|
|
165
|
+
checks.usedNoTools()
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Returns: `1` if no tools were called, `0` otherwise.
|
|
169
|
+
|
|
170
|
+
### `checks.noToolErrors()`
|
|
171
|
+
|
|
172
|
+
Scores 1 if none of the tool invocations resulted in an error state. Detects both error results (`result.error` present) and incomplete tool calls (`state === 'call'`).
|
|
173
|
+
|
|
174
|
+
```typescript
|
|
175
|
+
checks.noToolErrors()
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Returns: `1` if all tool calls succeeded, `0` otherwise.
|
|
179
|
+
|
|
180
|
+
## Combining checks with other scorers
|
|
181
|
+
|
|
182
|
+
Checks compose with LLM-based and code-based scorers in the same `scorers` array:
|
|
183
|
+
|
|
184
|
+
```typescript
|
|
185
|
+
import { checks } from '@mastra/evals/checks'
|
|
186
|
+
import { createAnswerRelevancyScorer } from '@mastra/evals/scorers/prebuilt'
|
|
187
|
+
import { runEvals } from '@mastra/core/evals'
|
|
188
|
+
import { myAgent } from '../agents'
|
|
189
|
+
|
|
190
|
+
const result = await runEvals({
|
|
191
|
+
data: [{ input: 'What is the weather in Brooklyn?' }],
|
|
192
|
+
target: myAgent,
|
|
193
|
+
scorers: [
|
|
194
|
+
// Zero-LLM checks
|
|
195
|
+
checks.includes('Brooklyn'),
|
|
196
|
+
checks.calledTool('get_weather'),
|
|
197
|
+
checks.noToolErrors(),
|
|
198
|
+
// LLM-based scorer
|
|
199
|
+
createAnswerRelevancyScorer({ model: 'openai/gpt-5-mini' }),
|
|
200
|
+
],
|
|
201
|
+
})
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
## Related
|
|
205
|
+
|
|
206
|
+
- [Quick Checks overview](https://mastra.ai/docs/evals/quick-checks)
|
|
207
|
+
- [Built-in Scorers](https://mastra.ai/docs/evals/built-in-scorers)
|
|
208
|
+
- [`createScorer()` reference](https://mastra.ai/reference/evals/create-scorer)
|
|
209
|
+
- [`runEvals()` reference](https://mastra.ai/reference/evals/run-evals)
|
|
210
|
+
- [Custom Scorers](https://mastra.ai/docs/evals/custom-scorers)
|
|
@@ -29,13 +29,34 @@ console.log(`Average scores:`, result.scores)
|
|
|
29
29
|
console.log(`Processed ${result.summary.totalItems} items`)
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
+
### With gates and thresholds
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
import { runEvals } from '@mastra/core/evals'
|
|
36
|
+
import { checks } from '@mastra/evals/checks'
|
|
37
|
+
import { faithfulnessScorer } from './scorers'
|
|
38
|
+
|
|
39
|
+
const result = await runEvals({
|
|
40
|
+
target: myAgent,
|
|
41
|
+
data: [{ input: 'What is the weather in Brooklyn?' }],
|
|
42
|
+
gates: [checks.calledTool('get_weather'), checks.noToolErrors()],
|
|
43
|
+
scorers: [{ scorer: faithfulnessScorer, threshold: 0.7 }, checks.includes('Brooklyn')],
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
result.verdict // 'passed' | 'scored' | 'failed'
|
|
47
|
+
result.gateResults // [{ id, passed, score }]
|
|
48
|
+
result.thresholdResults // [{ id, passed, averageScore, threshold }]
|
|
49
|
+
```
|
|
50
|
+
|
|
32
51
|
## Parameters
|
|
33
52
|
|
|
34
53
|
**target** (`Agent | Workflow`): The agent or workflow to evaluate.
|
|
35
54
|
|
|
36
55
|
**data** (`RunEvalsDataItem[]`): Array of test cases with input data and optional ground truth.
|
|
37
56
|
|
|
38
|
-
**scorers** (`
|
|
57
|
+
**scorers** (`ScorerEntry[] | AgentScorerConfig | WorkflowScorerConfig`): Scorers to use. Each entry is either a bare \`MastraScorer\` or \`{ scorer, threshold }\` for threshold tracking. An \`AgentScorerConfig\` object separates agent-level and trajectory scorers. A \`WorkflowScorerConfig\` object specifies scorers for the workflow, individual steps, and trajectory.
|
|
58
|
+
|
|
59
|
+
**gates** (`MastraScorer[]`): Scorers that must score 1.0 for the run to pass. If any gate averages below 1.0 across data items, the verdict is \`failed\`. Gates run before regular scorers on each data item.
|
|
39
60
|
|
|
40
61
|
**targetOptions** (`AgentExecutionOptions | WorkflowRunOptions`): Options forwarded to the target during execution. For agents: options passed to agent.generate() (e.g. maxSteps, modelSettings, instructions). For workflows: options passed to run.start() (e.g. perStep, outputOptions, initialState).
|
|
41
62
|
|
|
@@ -83,8 +104,55 @@ For workflows, use `WorkflowScorerConfig` to specify scorers at different levels
|
|
|
83
104
|
|
|
84
105
|
**summary.totalItems** (`number`): Total number of test cases processed.
|
|
85
106
|
|
|
107
|
+
**verdict** (`'passed' | 'scored' | 'failed'`): Present when \`gates\` or threshold-bearing scorers are provided. \`passed\` = all gates and thresholds met. \`scored\` = gates passed but a threshold was missed. \`failed\` = at least one gate did not score 1.0.
|
|
108
|
+
|
|
109
|
+
**gateResults** (`GateResult[]`): Per-gate results averaged across all data items. Each entry has \`id\`, \`passed\` (boolean), and \`score\` (0–1).
|
|
110
|
+
|
|
111
|
+
**thresholdResults** (`ThresholdResult[]`): Per-threshold-scorer results averaged across all data items. Each entry has \`id\`, \`passed\`, \`averageScore\`, and \`threshold\`.
|
|
112
|
+
|
|
113
|
+
## ScorerEntry
|
|
114
|
+
|
|
115
|
+
A scorer entry in the `scorers` array can be either a bare scorer or a scorer with a threshold:
|
|
116
|
+
|
|
117
|
+
**scorer** (`MastraScorer`): The scorer instance.
|
|
118
|
+
|
|
119
|
+
**threshold** (`number | { min?: number; max?: number }`): A number implies minimum threshold (score at or above passes). Use \`{ min, max }\` for range-based checks — e.g. \`{ max: 0.3 }\` for scorers like hallucination where a high score is bad. Both \`min\` and \`max\` must be between 0 and 1.
|
|
120
|
+
|
|
86
121
|
## Examples
|
|
87
122
|
|
|
123
|
+
### Gates and verdict
|
|
124
|
+
|
|
125
|
+
Use `gates` for hard pass/fail requirements and `{ scorer, threshold }` for tracked quality metrics:
|
|
126
|
+
|
|
127
|
+
```typescript
|
|
128
|
+
import { runEvals } from '@mastra/core/evals'
|
|
129
|
+
import { checks } from '@mastra/evals/checks'
|
|
130
|
+
|
|
131
|
+
const result = await runEvals({
|
|
132
|
+
target: weatherAgent,
|
|
133
|
+
data: [{ input: 'What is the weather in Brooklyn?' }],
|
|
134
|
+
gates: [checks.calledTool('get_weather'), checks.noToolErrors()],
|
|
135
|
+
scorers: [
|
|
136
|
+
{ scorer: faithfulnessScorer, threshold: 0.7 }, // min threshold (number shorthand)
|
|
137
|
+
{ scorer: hallucinationScorer, threshold: { max: 0.3 } }, // max threshold (high = bad)
|
|
138
|
+
{ scorer: toneScorer, threshold: { min: 0.5, max: 0.9 } }, // range threshold
|
|
139
|
+
checks.includes('Brooklyn'), // bare scorer, no threshold
|
|
140
|
+
],
|
|
141
|
+
})
|
|
142
|
+
|
|
143
|
+
if (result.verdict === 'failed') {
|
|
144
|
+
console.log(
|
|
145
|
+
'Gate failures:',
|
|
146
|
+
result.gateResults?.filter(g => !g.passed),
|
|
147
|
+
)
|
|
148
|
+
} else if (result.verdict === 'scored') {
|
|
149
|
+
console.log(
|
|
150
|
+
'Threshold misses:',
|
|
151
|
+
result.thresholdResults?.filter(t => !t.passed),
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
88
156
|
### Agent Evaluation
|
|
89
157
|
|
|
90
158
|
```typescript
|
|
@@ -246,6 +314,8 @@ const result = await runEvals({
|
|
|
246
314
|
|
|
247
315
|
## Related
|
|
248
316
|
|
|
317
|
+
- [Gates and Verdicts](https://mastra.ai/docs/evals/gates-and-verdicts): Conceptual guide to severity semantics
|
|
318
|
+
- [Quick Checks](https://mastra.ai/reference/evals/checks): Zero-LLM composable micro-scorers
|
|
249
319
|
- [createScorer()](https://mastra.ai/reference/evals/create-scorer): Create custom scorers for experiments
|
|
250
320
|
- [MastraScorer](https://mastra.ai/reference/evals/mastra-scorer): Learn about scorer structure and methods
|
|
251
321
|
- [Trajectory Accuracy](https://mastra.ai/reference/evals/trajectory-accuracy): Built-in trajectory evaluation scorers
|