@mastra/mcp-docs-server 0.13.14-alpha.0 → 0.13.14-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/organized/changelogs/%40mastra%2Fclickhouse.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fclient-js.md +25 -25
- package/.docs/organized/changelogs/%40mastra%2Fcloudflare-d1.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fcloudflare.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fcore.md +30 -30
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-cloud.md +11 -0
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-cloudflare.md +25 -25
- package/.docs/organized/changelogs/%40mastra%2Fdeployer-netlify.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fdeployer.md +28 -28
- package/.docs/organized/changelogs/%40mastra%2Fevals.md +14 -14
- package/.docs/organized/changelogs/%40mastra%2Ffirecrawl.md +9 -9
- package/.docs/organized/changelogs/%40mastra%2Flance.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Flibsql.md +12 -12
- package/.docs/organized/changelogs/%40mastra%2Fmcp-docs-server.md +16 -16
- package/.docs/organized/changelogs/%40mastra%2Fmcp.md +13 -13
- package/.docs/organized/changelogs/%40mastra%2Fmemory.md +13 -13
- package/.docs/organized/changelogs/%40mastra%2Fmssql.md +9 -0
- package/.docs/organized/changelogs/%40mastra%2Fopensearch.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fpg.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fplayground-ui.md +15 -15
- package/.docs/organized/changelogs/%40mastra%2Fqdrant.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Frag.md +14 -14
- package/.docs/organized/changelogs/%40mastra%2Fschema-compat.md +8 -0
- package/.docs/organized/changelogs/%40mastra%2Fserver.md +21 -21
- package/.docs/organized/changelogs/%40mastra%2Fupstash.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvectorize.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-cloudflare.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-deepgram.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-elevenlabs.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-gladia.md +9 -0
- package/.docs/organized/changelogs/%40mastra%2Fvoice-google.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-murf.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fvoice-openai-realtime.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-openai.md +11 -11
- package/.docs/organized/changelogs/%40mastra%2Fvoice-playai.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-sarvam.md +10 -10
- package/.docs/organized/changelogs/%40mastra%2Fvoice-speechify.md +10 -10
- package/.docs/organized/changelogs/create-mastra.md +3 -3
- package/.docs/organized/changelogs/mastra.md +13 -13
- package/.docs/organized/code-examples/agent.md +2 -2
- package/.docs/organized/code-examples/heads-up-game.md +276 -0
- package/.docs/raw/frameworks/agentic-uis/ai-sdk.mdx +1 -1
- package/.docs/raw/reference/client-js/mastra-client.mdx +6 -0
- package/.docs/raw/reference/scorers/create-scorer.mdx +1 -1
- package/.docs/raw/reference/scorers/tool-call-accuracy.mdx +211 -0
- package/.docs/raw/scorers/off-the-shelf-scorers.mdx +1 -0
- package/.docs/raw/tools-mcp/mcp-overview.mdx +1 -1
- package/.docs/raw/workflows/control-flow.mdx +2 -0
- package/package.json +4 -4
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @mastra/voice-sarvam
|
|
2
2
|
|
|
3
|
+
## 0.10.9-alpha.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#6942](https://github.com/mastra-ai/mastra/pull/6942) [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01) Thanks [@wardpeet](https://github.com/wardpeet)! - Add zod as peerdeps for all packages
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`943a7f3`](https://github.com/mastra-ai/mastra/commit/943a7f3dbc6a8ab3f9b7bc7c8a1c5b319c3d7f56), [`be49354`](https://github.com/mastra-ai/mastra/commit/be493546dca540101923ec700feb31f9a13939f2), [`d591ab3`](https://github.com/mastra-ai/mastra/commit/d591ab3ecc985c1870c0db347f8d7a20f7360536), [`ba82abe`](https://github.com/mastra-ai/mastra/commit/ba82abe76e869316bb5a9c95e8ea3946f3436fae), [`727f7e5`](https://github.com/mastra-ai/mastra/commit/727f7e5086e62e0dfe3356fb6dcd8bcb420af246), [`82d9f64`](https://github.com/mastra-ai/mastra/commit/82d9f647fbe4f0177320e7c05073fce88599aa95), [`4189486`](https://github.com/mastra-ai/mastra/commit/4189486c6718fda78347bdf4ce4d3fc33b2236e1), [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01)]:
|
|
10
|
+
- @mastra/core@0.14.2-alpha.1
|
|
11
|
+
|
|
3
12
|
## 0.10.8
|
|
4
13
|
|
|
5
14
|
### Patch Changes
|
|
@@ -289,14 +298,5 @@
|
|
|
289
298
|
- Updated dependencies [344f453]
|
|
290
299
|
- Updated dependencies [0a3ae6d]
|
|
291
300
|
- Updated dependencies [95911be]
|
|
292
|
-
- Updated dependencies [5eb5a99]
|
|
293
|
-
- Updated dependencies [7e632c5]
|
|
294
|
-
- Updated dependencies [1e9fbfa]
|
|
295
|
-
- Updated dependencies [b2ae5aa]
|
|
296
|
-
- Updated dependencies [a7292b0]
|
|
297
|
-
- Updated dependencies [0dcb9f0]
|
|
298
|
-
- @mastra/core@0.10.0-alpha.1
|
|
299
|
-
|
|
300
|
-
## 0.1.14-alpha.0
|
|
301
301
|
|
|
302
|
-
...
|
|
302
|
+
... 750 more lines hidden. See full changelog in package directory.
|
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @mastra/voice-speechify
|
|
2
2
|
|
|
3
|
+
## 0.10.9-alpha.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#6942](https://github.com/mastra-ai/mastra/pull/6942) [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01) Thanks [@wardpeet](https://github.com/wardpeet)! - Add zod as peerdeps for all packages
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`943a7f3`](https://github.com/mastra-ai/mastra/commit/943a7f3dbc6a8ab3f9b7bc7c8a1c5b319c3d7f56), [`be49354`](https://github.com/mastra-ai/mastra/commit/be493546dca540101923ec700feb31f9a13939f2), [`d591ab3`](https://github.com/mastra-ai/mastra/commit/d591ab3ecc985c1870c0db347f8d7a20f7360536), [`ba82abe`](https://github.com/mastra-ai/mastra/commit/ba82abe76e869316bb5a9c95e8ea3946f3436fae), [`727f7e5`](https://github.com/mastra-ai/mastra/commit/727f7e5086e62e0dfe3356fb6dcd8bcb420af246), [`82d9f64`](https://github.com/mastra-ai/mastra/commit/82d9f647fbe4f0177320e7c05073fce88599aa95), [`4189486`](https://github.com/mastra-ai/mastra/commit/4189486c6718fda78347bdf4ce4d3fc33b2236e1), [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01)]:
|
|
10
|
+
- @mastra/core@0.14.2-alpha.1
|
|
11
|
+
|
|
3
12
|
## 0.10.8
|
|
4
13
|
|
|
5
14
|
### Patch Changes
|
|
@@ -289,14 +298,5 @@
|
|
|
289
298
|
- Updated dependencies [d0ee3c6]
|
|
290
299
|
- Updated dependencies [b2ae5aa]
|
|
291
300
|
- Updated dependencies [23f258c]
|
|
292
|
-
- Updated dependencies [a7292b0]
|
|
293
|
-
- Updated dependencies [0dcb9f0]
|
|
294
|
-
- Updated dependencies [2672a05]
|
|
295
|
-
- @mastra/core@0.10.0
|
|
296
|
-
|
|
297
|
-
## 0.2.0-alpha.1
|
|
298
|
-
|
|
299
|
-
### Minor Changes
|
|
300
|
-
|
|
301
301
|
|
|
302
|
-
...
|
|
302
|
+
... 1104 more lines hidden. See full changelog in package directory.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# create-mastra
|
|
2
2
|
|
|
3
|
+
## 0.10.24-alpha.1
|
|
4
|
+
|
|
3
5
|
## 0.10.24-alpha.0
|
|
4
6
|
|
|
5
7
|
### Patch Changes
|
|
@@ -296,7 +298,5 @@
|
|
|
296
298
|
- f78f399: Make AgentModelSettings shareable between cloud and playground
|
|
297
299
|
- 57929df: agent network display
|
|
298
300
|
|
|
299
|
-
## 0.10.8-alpha.0
|
|
300
|
-
|
|
301
301
|
|
|
302
|
-
...
|
|
302
|
+
... 1242 more lines hidden. See full changelog in package directory.
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# mastra
|
|
2
2
|
|
|
3
|
+
## 0.10.24-alpha.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#6942](https://github.com/mastra-ai/mastra/pull/6942) [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01) Thanks [@wardpeet](https://github.com/wardpeet)! - Add zod as peerdeps for all packages
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`2b38a60`](https://github.com/mastra-ai/mastra/commit/2b38a60da0c1153028d8241c7748b41c5fb81121), [`943a7f3`](https://github.com/mastra-ai/mastra/commit/943a7f3dbc6a8ab3f9b7bc7c8a1c5b319c3d7f56), [`681252d`](https://github.com/mastra-ai/mastra/commit/681252d20e57fcee6821377dea96cacab3bc230f), [`01be5d3`](https://github.com/mastra-ai/mastra/commit/01be5d358fad8faa101e5c69dfa54562c02cc0af), [`24d9ee3`](https://github.com/mastra-ai/mastra/commit/24d9ee3db1c09d15f27a5d0971b102abcfcf7dfd), [`be49354`](https://github.com/mastra-ai/mastra/commit/be493546dca540101923ec700feb31f9a13939f2), [`d591ab3`](https://github.com/mastra-ai/mastra/commit/d591ab3ecc985c1870c0db347f8d7a20f7360536), [`ba82abe`](https://github.com/mastra-ai/mastra/commit/ba82abe76e869316bb5a9c95e8ea3946f3436fae), [`727f7e5`](https://github.com/mastra-ai/mastra/commit/727f7e5086e62e0dfe3356fb6dcd8bcb420af246), [`82d9f64`](https://github.com/mastra-ai/mastra/commit/82d9f647fbe4f0177320e7c05073fce88599aa95), [`4189486`](https://github.com/mastra-ai/mastra/commit/4189486c6718fda78347bdf4ce4d3fc33b2236e1), [`ca8ec2f`](https://github.com/mastra-ai/mastra/commit/ca8ec2f61884b9dfec5fc0d5f4f29d281ad13c01)]:
|
|
10
|
+
- @mastra/deployer@0.14.2-alpha.1
|
|
11
|
+
- @mastra/core@0.14.2-alpha.1
|
|
12
|
+
- @mastra/mcp@0.10.13-alpha.1
|
|
13
|
+
|
|
3
14
|
## 0.10.24-alpha.0
|
|
4
15
|
|
|
5
16
|
### Patch Changes
|
|
@@ -287,16 +298,5 @@
|
|
|
287
298
|
- Updated dependencies [7b8172f]
|
|
288
299
|
- Updated dependencies [cb36de0]
|
|
289
300
|
- Updated dependencies [d0496e6]
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
- Updated dependencies [41a0a0e]
|
|
293
|
-
- Updated dependencies [2871020]
|
|
294
|
-
- Updated dependencies [9e792ef]
|
|
295
|
-
- Updated dependencies [94f4812]
|
|
296
|
-
- Updated dependencies [e202b82]
|
|
297
|
-
- Updated dependencies [e00f6a0]
|
|
298
|
-
- Updated dependencies [4a406ec]
|
|
299
|
-
- Updated dependencies [b0e43c1]
|
|
300
|
-
- Updated dependencies [5d377e5]
|
|
301
|
-
|
|
302
|
-
... 5064 more lines hidden. See full changelog in package directory.
|
|
301
|
+
|
|
302
|
+
... 5075 more lines hidden. See full changelog in package directory.
|
|
@@ -1113,10 +1113,10 @@ const step2 = createStep({
|
|
|
1113
1113
|
outputSchema: z.object({
|
|
1114
1114
|
result: z.string(),
|
|
1115
1115
|
}),
|
|
1116
|
-
execute: async ({ inputData, mastra,
|
|
1116
|
+
execute: async ({ inputData, mastra, tracingContext }) => {
|
|
1117
1117
|
const agent = mastra.getAgent('chefAgentResponses');
|
|
1118
1118
|
const response = await agent.generate(inputData.result, {
|
|
1119
|
-
|
|
1119
|
+
tracingContext,
|
|
1120
1120
|
});
|
|
1121
1121
|
return {
|
|
1122
1122
|
result: 'suh',
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
### package.json
|
|
2
|
+
```json
|
|
3
|
+
{
|
|
4
|
+
"name": "example-headsup",
|
|
5
|
+
"engines": {
|
|
6
|
+
"node": ">=20.9.0"
|
|
7
|
+
},
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"@ai-sdk/openai": "^1.3.23",
|
|
10
|
+
"@mastra/core": "^0.14.1",
|
|
11
|
+
"@mastra/libsql": "^0.13.4",
|
|
12
|
+
"@mastra/loggers": "^0.10.7",
|
|
13
|
+
"@mastra/memory": "^0.13.1",
|
|
14
|
+
"zod": "^3.25.76"
|
|
15
|
+
},
|
|
16
|
+
"devDependencies": {
|
|
17
|
+
"@types/node": "^24.3.0",
|
|
18
|
+
"mastra": "^0.10.23",
|
|
19
|
+
"typescript": "^5.9.2"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
### mastra/agents/example-famous-person-agent.ts
|
|
25
|
+
```typescript
|
|
26
|
+
import { openai } from '@ai-sdk/openai';
|
|
27
|
+
import { Agent } from '@mastra/core/agent';
|
|
28
|
+
import { Memory } from '@mastra/memory';
|
|
29
|
+
import { LibSQLVector } from '@mastra/libsql';
|
|
30
|
+
|
|
31
|
+
export const famousPersonAgent = new Agent({
|
|
32
|
+
name: 'Famous Person Generator',
|
|
33
|
+
instructions: `You are a famous person generator for a "Heads Up" guessing game.
|
|
34
|
+
|
|
35
|
+
Generate the name of a well-known famous person who:
|
|
36
|
+
- Is recognizable to most people
|
|
37
|
+
- Has distinctive characteristics that can be described with yes/no questions
|
|
38
|
+
- Is appropriate for all audiences
|
|
39
|
+
- Has a clear, unambiguous name
|
|
40
|
+
|
|
41
|
+
IMPORTANT: Use your memory to check what famous people you've already suggested and NEVER repeat a person you've already suggested.
|
|
42
|
+
|
|
43
|
+
Examples: Albert Einstein, Beyoncé, Leonardo da Vinci, Oprah Winfrey, Michael Jordan
|
|
44
|
+
|
|
45
|
+
Return only the person's name, nothing else.`,
|
|
46
|
+
model: openai('gpt-4o'),
|
|
47
|
+
memory: new Memory({
|
|
48
|
+
vector: new LibSQLVector({
|
|
49
|
+
connectionUrl: 'file:../mastra.db',
|
|
50
|
+
}),
|
|
51
|
+
embedder: openai.embedding('text-embedding-3-small'),
|
|
52
|
+
options: {
|
|
53
|
+
lastMessages: 5,
|
|
54
|
+
semanticRecall: {
|
|
55
|
+
topK: 10,
|
|
56
|
+
messageRange: 1,
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
}),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### mastra/agents/example-game-agent.ts
|
|
65
|
+
```typescript
|
|
66
|
+
import { openai } from '@ai-sdk/openai';
|
|
67
|
+
import { Agent } from '@mastra/core/agent';
|
|
68
|
+
|
|
69
|
+
export const gameAgent = new Agent({
|
|
70
|
+
name: 'Game Agent',
|
|
71
|
+
instructions: `You are a helpful game assistant for a "Heads Up" guessing game.
|
|
72
|
+
|
|
73
|
+
CRITICAL: You know the famous person's name but you must NEVER reveal it in any response.
|
|
74
|
+
|
|
75
|
+
When a user asks a question about the famous person:
|
|
76
|
+
- Answer truthfully based on the famous person provided
|
|
77
|
+
- Keep responses concise and friendly
|
|
78
|
+
- NEVER mention the person's name, even if it seems natural
|
|
79
|
+
- NEVER reveal gender, nationality, or other characteristics unless specifically asked about them
|
|
80
|
+
- Answer yes/no questions with clear "Yes" or "No" responses
|
|
81
|
+
- Be consistent - same question asked differently should get the same answer
|
|
82
|
+
- Ask for clarification if a question is unclear
|
|
83
|
+
- If multiple questions are asked at once, ask them to ask one at a time
|
|
84
|
+
|
|
85
|
+
When they make a guess:
|
|
86
|
+
- If correct: Congratulate them warmly
|
|
87
|
+
- If incorrect: Politely correct them and encourage them to try again
|
|
88
|
+
|
|
89
|
+
Encourage players to make a guess when they seem to have enough information.
|
|
90
|
+
|
|
91
|
+
You must return a JSON object with:
|
|
92
|
+
- response: Your response to the user
|
|
93
|
+
- gameWon: true if they guessed correctly, false otherwise`,
|
|
94
|
+
model: openai('gpt-4o'),
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### mastra/index.ts
|
|
100
|
+
```typescript
|
|
101
|
+
import { Mastra } from '@mastra/core/mastra';
|
|
102
|
+
import { PinoLogger } from '@mastra/loggers';
|
|
103
|
+
import { LibSQLStore } from '@mastra/libsql';
|
|
104
|
+
import { headsUpWorkflow } from './workflows/example-heads-up-workflow';
|
|
105
|
+
|
|
106
|
+
import { famousPersonAgent } from './agents/example-famous-person-agent';
|
|
107
|
+
import { gameAgent } from './agents/example-game-agent';
|
|
108
|
+
import { guessVerifierAgent } from './agents/example-guess-verifier-agent';
|
|
109
|
+
|
|
110
|
+
export const mastra = new Mastra({
|
|
111
|
+
workflows: { headsUpWorkflow },
|
|
112
|
+
agents: { famousPersonAgent, gameAgent, guessVerifierAgent },
|
|
113
|
+
storage: new LibSQLStore({
|
|
114
|
+
// stores telemetry, evals, ... into file storage for persistence across playground refreshes
|
|
115
|
+
url: 'file:../mastra.db',
|
|
116
|
+
}),
|
|
117
|
+
logger: new PinoLogger({
|
|
118
|
+
name: 'Mastra',
|
|
119
|
+
level: 'info',
|
|
120
|
+
}),
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### mastra/workflows/example-heads-up-workflow.ts
|
|
126
|
+
```typescript
|
|
127
|
+
import { createWorkflow, createStep } from '@mastra/core/workflows';
|
|
128
|
+
import { z } from 'zod';
|
|
129
|
+
import { Pool } from 'pg';
|
|
130
|
+
|
|
131
|
+
// PostgreSQL client
|
|
132
|
+
const pool = new Pool({
|
|
133
|
+
connectionString: process.env.DATABASE_URL!,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const startStep = createStep({
|
|
137
|
+
id: 'start-step',
|
|
138
|
+
description: "Generate a famous person's name",
|
|
139
|
+
inputSchema: z.object({
|
|
140
|
+
start: z.boolean(),
|
|
141
|
+
}),
|
|
142
|
+
outputSchema: z.object({
|
|
143
|
+
famousPerson: z.string(),
|
|
144
|
+
guessCount: z.number(),
|
|
145
|
+
}),
|
|
146
|
+
execute: async ({ mastra }) => {
|
|
147
|
+
const agent = mastra.getAgent('famousPersonAgent');
|
|
148
|
+
const response = await agent.generate("Generate a famous person's name", {
|
|
149
|
+
temperature: 1.2,
|
|
150
|
+
topP: 0.9,
|
|
151
|
+
memory: {
|
|
152
|
+
resource: 'heads-up-game',
|
|
153
|
+
thread: 'famous-person-generator',
|
|
154
|
+
},
|
|
155
|
+
});
|
|
156
|
+
const famousPerson = response.text.trim();
|
|
157
|
+
return { famousPerson, guessCount: 0 };
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
const gameStep = createStep({
|
|
162
|
+
id: 'game-step',
|
|
163
|
+
description: 'Handles the question-answer-continue loop',
|
|
164
|
+
inputSchema: z.object({
|
|
165
|
+
famousPerson: z.string(),
|
|
166
|
+
guessCount: z.number(),
|
|
167
|
+
}),
|
|
168
|
+
resumeSchema: z.object({
|
|
169
|
+
userMessage: z.string(),
|
|
170
|
+
}),
|
|
171
|
+
suspendSchema: z.object({
|
|
172
|
+
agentResponse: z.string(),
|
|
173
|
+
}),
|
|
174
|
+
outputSchema: z.object({
|
|
175
|
+
famousPerson: z.string(),
|
|
176
|
+
gameWon: z.boolean(),
|
|
177
|
+
agentResponse: z.string(),
|
|
178
|
+
guessCount: z.number(),
|
|
179
|
+
}),
|
|
180
|
+
execute: async ({ inputData, mastra, resumeData, suspend }) => {
|
|
181
|
+
let { famousPerson, guessCount } = inputData;
|
|
182
|
+
const { userMessage } = resumeData ?? {};
|
|
183
|
+
|
|
184
|
+
if (!userMessage) {
|
|
185
|
+
// First time - ask for a question
|
|
186
|
+
const message = "I'm thinking of a famous person. Ask me yes/no questions to figure out who it is!";
|
|
187
|
+
|
|
188
|
+
await suspend({
|
|
189
|
+
agentResponse: message,
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
return { famousPerson, gameWon: false, agentResponse: message, guessCount };
|
|
193
|
+
} else {
|
|
194
|
+
// Check if the user's message is a guess by using the guess verifier agent
|
|
195
|
+
const guessVerifier = mastra.getAgent('guessVerifierAgent');
|
|
196
|
+
const verificationResponse = await guessVerifier.generate(
|
|
197
|
+
[
|
|
198
|
+
{
|
|
199
|
+
role: 'user',
|
|
200
|
+
content: `Actual famous person: ${famousPerson}
|
|
201
|
+
User's guess: "${userMessage}"
|
|
202
|
+
Is this correct?`,
|
|
203
|
+
},
|
|
204
|
+
],
|
|
205
|
+
{
|
|
206
|
+
output: z.object({
|
|
207
|
+
isCorrect: z.boolean(),
|
|
208
|
+
}),
|
|
209
|
+
},
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
const gameWon = verificationResponse.object.isCorrect;
|
|
213
|
+
|
|
214
|
+
// Let the agent handle the user's message (question or guess)
|
|
215
|
+
const agent = mastra.getAgent('gameAgent');
|
|
216
|
+
const response = await agent.generate(`
|
|
217
|
+
The famous person is: ${famousPerson}
|
|
218
|
+
The user asked: "${userMessage}"
|
|
219
|
+
Is this a correct guess: ${gameWon}
|
|
220
|
+
Please respond appropriately.
|
|
221
|
+
`);
|
|
222
|
+
|
|
223
|
+
const agentResponse = response.text;
|
|
224
|
+
|
|
225
|
+
// Increment the guess count
|
|
226
|
+
guessCount++;
|
|
227
|
+
|
|
228
|
+
return { famousPerson, gameWon, agentResponse, guessCount };
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
const winStep = createStep({
|
|
234
|
+
id: 'win-step',
|
|
235
|
+
description: 'Handle game win logic',
|
|
236
|
+
inputSchema: z.object({
|
|
237
|
+
famousPerson: z.string(),
|
|
238
|
+
gameWon: z.boolean(),
|
|
239
|
+
agentResponse: z.string(),
|
|
240
|
+
guessCount: z.number(),
|
|
241
|
+
}),
|
|
242
|
+
outputSchema: z.object({
|
|
243
|
+
famousPerson: z.string(),
|
|
244
|
+
gameWon: z.boolean(),
|
|
245
|
+
guessCount: z.number(),
|
|
246
|
+
}),
|
|
247
|
+
execute: async ({ inputData }) => {
|
|
248
|
+
const { famousPerson, gameWon, guessCount } = inputData;
|
|
249
|
+
|
|
250
|
+
await pool.query('INSERT INTO heads_up_games (famous_person, game_won, guess_count) VALUES ($1, $2, $3)', [
|
|
251
|
+
famousPerson,
|
|
252
|
+
gameWon,
|
|
253
|
+
guessCount,
|
|
254
|
+
]);
|
|
255
|
+
|
|
256
|
+
return { famousPerson, gameWon, guessCount };
|
|
257
|
+
},
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
export const headsUpWorkflow = createWorkflow({
|
|
261
|
+
id: 'heads-up-workflow',
|
|
262
|
+
inputSchema: z.object({
|
|
263
|
+
start: z.boolean(),
|
|
264
|
+
}),
|
|
265
|
+
outputSchema: z.object({
|
|
266
|
+
famousPerson: z.string(),
|
|
267
|
+
gameWon: z.boolean(),
|
|
268
|
+
guessCount: z.number(),
|
|
269
|
+
}),
|
|
270
|
+
})
|
|
271
|
+
.then(startStep)
|
|
272
|
+
.dountil(gameStep, async ({ inputData: { gameWon } }) => gameWon)
|
|
273
|
+
.then(winStep)
|
|
274
|
+
.commit();
|
|
275
|
+
|
|
276
|
+
```
|
|
@@ -416,7 +416,7 @@ Mastra automatically handles AI SDK v4 data using its internal `MessageList` cla
|
|
|
416
416
|
For cases where you need to manually convert messages between AI SDK and Mastra formats, use the `convertMessages` utility:
|
|
417
417
|
|
|
418
418
|
```typescript
|
|
419
|
-
import { convertMessages } from '@mastra/core';
|
|
419
|
+
import { convertMessages } from '@mastra/core/agent';
|
|
420
420
|
|
|
421
421
|
// Convert AI SDK v4 messages to v5
|
|
422
422
|
const aiv5Messages = convertMessages(aiv4Messages).to('AIV5.UI');
|
|
@@ -54,6 +54,12 @@ export const mastraClient = new MastraClient({
|
|
|
54
54
|
description: "An object containing custom HTTP headers to include with every request.",
|
|
55
55
|
isOptional: true,
|
|
56
56
|
},
|
|
57
|
+
{
|
|
58
|
+
name: "credentials",
|
|
59
|
+
type: '"omit" | "same-origin" | "include"',
|
|
60
|
+
description: "Credentials mode for requests. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials for more info.",
|
|
61
|
+
isOptional: true,
|
|
62
|
+
},
|
|
57
63
|
]}
|
|
58
64
|
/>
|
|
59
65
|
|
|
@@ -79,7 +79,7 @@ This function returns a scorer builder that you can chain step methods onto. See
|
|
|
79
79
|
For better type inference and IntelliSense support, you can specify input/output types when creating scorers:
|
|
80
80
|
|
|
81
81
|
```typescript
|
|
82
|
-
import { createScorer, ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core';
|
|
82
|
+
import { createScorer, ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scorers';
|
|
83
83
|
|
|
84
84
|
// For agent evaluation with full type safety
|
|
85
85
|
const agentScorer = createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Reference: Tool Call Accuracy | Scorers | Mastra Docs"
|
|
3
|
+
description: Documentation for the Tool Call Accuracy Scorers in Mastra, which evaluate whether LLM outputs call the correct tools from available options.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Tool Call Accuracy Scorers
|
|
7
|
+
|
|
8
|
+
Mastra provides two tool call accuracy scorers for evaluating whether an LLM selects the correct tools from available options:
|
|
9
|
+
|
|
10
|
+
1. **Code-based scorer** - Deterministic evaluation using exact tool matching
|
|
11
|
+
2. **LLM-based scorer** - Semantic evaluation using AI to assess appropriateness
|
|
12
|
+
|
|
13
|
+
For usage examples, see the [Tool Call Accuracy Examples](/examples/scorers/tool-call-accuracy).
|
|
14
|
+
|
|
15
|
+
## Code-Based Tool Call Accuracy Scorer
|
|
16
|
+
|
|
17
|
+
The `createToolCallAccuracyScorerCode()` function from `@mastra/evals/scorers/code` provides deterministic binary scoring based on exact tool matching and supports both strict and lenient evaluation modes, as well as tool calling order validation.
|
|
18
|
+
|
|
19
|
+
### Parameters
|
|
20
|
+
|
|
21
|
+
<PropertiesTable
|
|
22
|
+
content={[
|
|
23
|
+
{
|
|
24
|
+
name: "expectedTool",
|
|
25
|
+
type: "string",
|
|
26
|
+
description: "The name of the tool that should be called for the given task. Ignored when expectedToolOrder is provided.",
|
|
27
|
+
required: false,
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
name: "strictMode",
|
|
31
|
+
type: "boolean",
|
|
32
|
+
description: "Controls evaluation strictness. For single tool mode: only exact single tool calls accepted. For order checking mode: tools must match exactly with no extra tools allowed.",
|
|
33
|
+
required: false,
|
|
34
|
+
default: "false",
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: "expectedToolOrder",
|
|
38
|
+
type: "string[]",
|
|
39
|
+
description: "Array of tool names in the expected calling order. When provided, enables order checking mode and ignores expectedTool parameter.",
|
|
40
|
+
required: false,
|
|
41
|
+
},
|
|
42
|
+
]}
|
|
43
|
+
/>
|
|
44
|
+
|
|
45
|
+
This function returns an instance of the MastraScorer class. See the [MastraScorer reference](./mastra-scorer) for details on the `.run()` method and its input/output.
|
|
46
|
+
|
|
47
|
+
### Evaluation Modes
|
|
48
|
+
|
|
49
|
+
The code-based scorer operates in two distinct modes:
|
|
50
|
+
|
|
51
|
+
#### Single Tool Mode
|
|
52
|
+
|
|
53
|
+
When `expectedToolOrder` is not provided, the scorer evaluates single tool selection:
|
|
54
|
+
|
|
55
|
+
- **Standard Mode (strictMode: false)**: Returns `1` if the expected tool is called, regardless of other tools
|
|
56
|
+
- **Strict Mode (strictMode: true)**: Returns `1` only if exactly one tool is called and it matches the expected tool
|
|
57
|
+
|
|
58
|
+
#### Order Checking Mode
|
|
59
|
+
|
|
60
|
+
When `expectedToolOrder` is provided, the scorer validates tool calling sequence:
|
|
61
|
+
|
|
62
|
+
- **Strict Order (strictMode: true)**: Tools must be called in exactly the specified order with no extra tools
|
|
63
|
+
- **Flexible Order (strictMode: false)**: Expected tools must appear in correct relative order (extra tools allowed)
|
|
64
|
+
|
|
65
|
+
### Examples
|
|
66
|
+
|
|
67
|
+
```typescript
|
|
68
|
+
import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/code';
|
|
69
|
+
|
|
70
|
+
// Single tool validation
|
|
71
|
+
const scorer = createToolCallAccuracyScorerCode({
|
|
72
|
+
expectedTool: 'weather-tool'
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
// Strict single tool (no other tools allowed)
|
|
76
|
+
const strictScorer = createToolCallAccuracyScorerCode({
|
|
77
|
+
expectedTool: 'calculator-tool',
|
|
78
|
+
strictMode: true
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// Tool order validation
|
|
82
|
+
const orderScorer = createToolCallAccuracyScorerCode({
|
|
83
|
+
expectedTool: 'search-tool', // ignored when order is specified
|
|
84
|
+
expectedToolOrder: ['search-tool', 'weather-tool'],
|
|
85
|
+
strictMode: true // exact match required
|
|
86
|
+
});
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## LLM-Based Tool Call Accuracy Scorer
|
|
90
|
+
|
|
91
|
+
The `createToolCallAccuracyScorerLLM()` function from `@mastra/evals/scorers/llm` uses an LLM to evaluate whether the tools called by an agent are appropriate for the given user request, providing semantic evaluation rather than exact matching.
|
|
92
|
+
|
|
93
|
+
### Parameters
|
|
94
|
+
|
|
95
|
+
<PropertiesTable
|
|
96
|
+
content={[
|
|
97
|
+
{
|
|
98
|
+
name: "model",
|
|
99
|
+
type: "MastraLanguageModel",
|
|
100
|
+
description: "The LLM model to use for evaluating tool appropriateness",
|
|
101
|
+
required: true,
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
name: "availableTools",
|
|
105
|
+
type: "Array<{name: string, description: string}>",
|
|
106
|
+
description: "List of available tools with their descriptions for context",
|
|
107
|
+
required: true,
|
|
108
|
+
},
|
|
109
|
+
]}
|
|
110
|
+
/>
|
|
111
|
+
|
|
112
|
+
### Features
|
|
113
|
+
|
|
114
|
+
The LLM-based scorer provides:
|
|
115
|
+
|
|
116
|
+
- **Semantic Evaluation**: Understands context and user intent
|
|
117
|
+
- **Appropriateness Assessment**: Distinguishes between "helpful" and "appropriate" tools
|
|
118
|
+
- **Clarification Handling**: Recognizes when agents appropriately ask for clarification
|
|
119
|
+
- **Missing Tool Detection**: Identifies tools that should have been called
|
|
120
|
+
- **Reasoning Generation**: Provides explanations for scoring decisions
|
|
121
|
+
|
|
122
|
+
### Evaluation Process
|
|
123
|
+
|
|
124
|
+
1. **Extract Tool Calls**: Identifies tools mentioned in agent output
|
|
125
|
+
2. **Analyze Appropriateness**: Evaluates each tool against user request
|
|
126
|
+
3. **Generate Score**: Calculates score based on appropriate vs total tool calls
|
|
127
|
+
4. **Generate Reasoning**: Provides human-readable explanation
|
|
128
|
+
|
|
129
|
+
### Examples
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
import { createToolCallAccuracyScorerLLM } from '@mastra/evals/scorers/llm';
|
|
133
|
+
import { openai } from '@ai-sdk/openai';
|
|
134
|
+
|
|
135
|
+
const llmScorer = createToolCallAccuracyScorerLLM({
|
|
136
|
+
model: openai('gpt-4o-mini'),
|
|
137
|
+
availableTools: [
|
|
138
|
+
{
|
|
139
|
+
name: 'weather-tool',
|
|
140
|
+
description: 'Get current weather information for any location'
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
name: 'search-tool',
|
|
144
|
+
description: 'Search the web for information'
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
name: 'calendar-tool',
|
|
148
|
+
description: 'Check calendar events and scheduling'
|
|
149
|
+
}
|
|
150
|
+
]
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
const result = await llmScorer.run(agentRun);
|
|
154
|
+
console.log(result.score); // 0.0 to 1.0
|
|
155
|
+
console.log(result.reason); // Explanation of the score
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Choosing Between Scorers
|
|
159
|
+
|
|
160
|
+
### Use the Code-Based Scorer When:
|
|
161
|
+
|
|
162
|
+
- You need **deterministic, reproducible** results
|
|
163
|
+
- You want to test **exact tool matching**
|
|
164
|
+
- You need to validate **specific tool sequences**
|
|
165
|
+
- Speed and cost are priorities (no LLM calls)
|
|
166
|
+
- You're running automated tests
|
|
167
|
+
|
|
168
|
+
### Use the LLM-Based Scorer When:
|
|
169
|
+
|
|
170
|
+
- You need **semantic understanding** of appropriateness
|
|
171
|
+
- Tool selection depends on **context and intent**
|
|
172
|
+
- You want to handle **edge cases** like clarification requests
|
|
173
|
+
- You need **explanations** for scoring decisions
|
|
174
|
+
- You're evaluating **production agent behavior**
|
|
175
|
+
|
|
176
|
+
## Scoring Details
|
|
177
|
+
|
|
178
|
+
### Code-Based Scoring
|
|
179
|
+
|
|
180
|
+
- **Binary scores**: Always returns 0 or 1
|
|
181
|
+
- **Deterministic**: Same input always produces same output
|
|
182
|
+
- **Fast**: No external API calls
|
|
183
|
+
|
|
184
|
+
### LLM-Based Scoring
|
|
185
|
+
|
|
186
|
+
- **Fractional scores**: Returns values between 0.0 and 1.0
|
|
187
|
+
- **Context-aware**: Considers user intent and appropriateness
|
|
188
|
+
- **Explanatory**: Provides reasoning for scores
|
|
189
|
+
|
|
190
|
+
## Use Cases
|
|
191
|
+
|
|
192
|
+
### Code-Based Scorer Use Cases
|
|
193
|
+
|
|
194
|
+
- **Unit Testing**: Verify specific tool selection behavior
|
|
195
|
+
- **Regression Testing**: Ensure tool selection doesn't change
|
|
196
|
+
- **Workflow Validation**: Check tool sequences in multi-step processes
|
|
197
|
+
- **CI/CD Pipelines**: Fast, deterministic validation
|
|
198
|
+
|
|
199
|
+
### LLM-Based Scorer Use Cases
|
|
200
|
+
|
|
201
|
+
- **Quality Assurance**: Evaluate production agent behavior
|
|
202
|
+
- **A/B Testing**: Compare different agent implementations
|
|
203
|
+
- **User Intent Alignment**: Ensure tools match user needs
|
|
204
|
+
- **Edge Case Handling**: Evaluate clarification and error scenarios
|
|
205
|
+
|
|
206
|
+
## Related
|
|
207
|
+
|
|
208
|
+
- [Answer Relevancy Scorer](./answer-relevancy)
|
|
209
|
+
- [Completeness Scorer](./completeness)
|
|
210
|
+
- [Faithfulness Scorer](./faithfulness)
|
|
211
|
+
- [Custom Scorers](/docs/scorers/custom-scorers)
|
|
@@ -19,6 +19,7 @@ These scorers evaluate how correct, truthful, and complete your agent's answers
|
|
|
19
19
|
- [`completeness`](/reference/scorers/completeness): Checks if responses include all necessary information (`0-1`, higher is better)
|
|
20
20
|
- [`content-similarity`](/reference/scorers/content-similarity): Measures textual similarity using character-level matching (`0-1`, higher is better)
|
|
21
21
|
- [`textual-difference`](/reference/scorers/textual-difference): Measures textual differences between strings (`0-1`, higher means more similar)
|
|
22
|
+
- [`tool-call-accuracy`](/reference/scorers/tool-call-accuracy): Evaluates whether the LLM selects the correct tool from available options (`0-1`, higher is better)
|
|
22
23
|
|
|
23
24
|
### Output Quality
|
|
24
25
|
|
|
@@ -341,7 +341,7 @@ When a tool includes an `outputSchema`, its `execute` function **must** return a
|
|
|
341
341
|
Here's an example of a tool with an `outputSchema`:
|
|
342
342
|
|
|
343
343
|
```typescript filename="src/tools/structured-tool.ts"
|
|
344
|
-
import { createTool } from '@mastra/core';
|
|
344
|
+
import { createTool } from '@mastra/core/tools';
|
|
345
345
|
import { z } from 'zod';
|
|
346
346
|
|
|
347
347
|
export const structuredTool = createTool({
|
|
@@ -55,6 +55,8 @@ This executes `step1` and `step2` concurrently, then continues to `step3` after
|
|
|
55
55
|
|
|
56
56
|
> See [Parallel Execution with Steps](/examples/workflows/parallel-steps) for more information.
|
|
57
57
|
|
|
58
|
+
> 📹 Watch: How to run steps in parallel and optimize your Mastra workflow → [YouTube (3 minutes)](https://youtu.be/pTSOSWbreE0)
|
|
59
|
+
|
|
58
60
|
## Conditional logic with `.branch()`
|
|
59
61
|
|
|
60
62
|
Execute steps conditionally using `.branch()`:
|