@mastra/core 1.18.0-alpha.2 → 1.18.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/dist/agent/agent-legacy.d.ts +3 -3
- package/dist/agent/agent-legacy.d.ts.map +1 -1
- package/dist/agent/agent.d.ts +12 -12
- package/dist/agent/agent.d.ts.map +1 -1
- package/dist/agent/index.cjs +8 -8
- package/dist/agent/index.js +1 -1
- package/dist/agent/types.d.ts +12 -12
- package/dist/agent/types.d.ts.map +1 -1
- package/dist/agent/workflows/prepare-stream/index.d.ts.map +1 -1
- package/dist/agent/workflows/prepare-stream/prepare-memory-step.d.ts +2 -1
- package/dist/agent/workflows/prepare-stream/prepare-memory-step.d.ts.map +1 -1
- package/dist/{chunk-VF3CL3JV.js → chunk-32NEFSTI.js} +153 -44
- package/dist/chunk-32NEFSTI.js.map +1 -0
- package/dist/{chunk-XRZNAZRP.cjs → chunk-3UM5VCQ5.cjs} +3 -3
- package/dist/{chunk-XRZNAZRP.cjs.map → chunk-3UM5VCQ5.cjs.map} +1 -1
- package/dist/{chunk-VAVK4KJI.cjs → chunk-4K2NKQGN.cjs} +316 -207
- package/dist/chunk-4K2NKQGN.cjs.map +1 -0
- package/dist/{chunk-WPDB2VAM.cjs → chunk-5IZ346TV.cjs} +7 -7
- package/dist/{chunk-WPDB2VAM.cjs.map → chunk-5IZ346TV.cjs.map} +1 -1
- package/dist/{chunk-R2V6BCYX.cjs → chunk-64565EZF.cjs} +9 -9
- package/dist/{chunk-R2V6BCYX.cjs.map → chunk-64565EZF.cjs.map} +1 -1
- package/dist/{chunk-6WVZQBYQ.cjs → chunk-6LHF5UDX.cjs} +9 -9
- package/dist/{chunk-6WVZQBYQ.cjs.map → chunk-6LHF5UDX.cjs.map} +1 -1
- package/dist/{chunk-L53NRDFU.js → chunk-7USX5P3I.js} +3 -3
- package/dist/{chunk-L53NRDFU.js.map → chunk-7USX5P3I.js.map} +1 -1
- package/dist/{chunk-FWDG2WUV.js → chunk-AHFHYQS6.js} +3 -3
- package/dist/{chunk-FWDG2WUV.js.map → chunk-AHFHYQS6.js.map} +1 -1
- package/dist/{chunk-G5HKDGNT.cjs → chunk-B6D4D2CY.cjs} +2 -2
- package/dist/{chunk-G5HKDGNT.cjs.map → chunk-B6D4D2CY.cjs.map} +1 -1
- package/dist/{chunk-URN2SKVG.js → chunk-BFRUEBG3.js} +4 -4
- package/dist/{chunk-URN2SKVG.js.map → chunk-BFRUEBG3.js.map} +1 -1
- package/dist/{chunk-CT4YYQI3.js → chunk-CX5I6VS4.js} +2 -2
- package/dist/{chunk-CT4YYQI3.js.map → chunk-CX5I6VS4.js.map} +1 -1
- package/dist/{chunk-JEWTRDRI.js → chunk-D5VCCQ4L.js} +5 -5
- package/dist/{chunk-JEWTRDRI.js.map → chunk-D5VCCQ4L.js.map} +1 -1
- package/dist/{chunk-WVTVBWOQ.js → chunk-FB6O7FNK.js} +42 -61
- package/dist/chunk-FB6O7FNK.js.map +1 -0
- package/dist/{chunk-LUONHWL6.cjs → chunk-GVHBM6VU.cjs} +6 -6
- package/dist/{chunk-LUONHWL6.cjs.map → chunk-GVHBM6VU.cjs.map} +1 -1
- package/dist/{chunk-ZOJZL62I.js → chunk-HDPQ3LEE.js} +4 -4
- package/dist/{chunk-ZOJZL62I.js.map → chunk-HDPQ3LEE.js.map} +1 -1
- package/dist/{chunk-IOCCTQYZ.js → chunk-IR4WRS3N.js} +38 -14
- package/dist/chunk-IR4WRS3N.js.map +1 -0
- package/dist/{chunk-E6XNM5ES.js → chunk-MP2VGF73.js} +3 -3
- package/dist/{chunk-E6XNM5ES.js.map → chunk-MP2VGF73.js.map} +1 -1
- package/dist/{chunk-DQYOFNZ7.cjs → chunk-OKKJRRSS.cjs} +39 -15
- package/dist/chunk-OKKJRRSS.cjs.map +1 -0
- package/dist/{chunk-H5O522FY.cjs → chunk-OVWOD5GI.cjs} +53 -72
- package/dist/chunk-OVWOD5GI.cjs.map +1 -0
- package/dist/{chunk-MBJ6IAKF.cjs → chunk-OXS5X7PW.cjs} +13 -13
- package/dist/{chunk-MBJ6IAKF.cjs.map → chunk-OXS5X7PW.cjs.map} +1 -1
- package/dist/{chunk-M5CZCWWW.js → chunk-OYAHOEQP.js} +4 -4
- package/dist/{chunk-M5CZCWWW.js.map → chunk-OYAHOEQP.js.map} +1 -1
- package/dist/{chunk-5AVTFQEP.cjs → chunk-PYNUWPGW.cjs} +19 -19
- package/dist/{chunk-5AVTFQEP.cjs.map → chunk-PYNUWPGW.cjs.map} +1 -1
- package/dist/{chunk-7IDACSBM.cjs → chunk-QXPUTX3W.cjs} +32 -32
- package/dist/chunk-QXPUTX3W.cjs.map +1 -0
- package/dist/{chunk-VMKNS3YO.js → chunk-SLHDZJ4B.js} +3 -3
- package/dist/chunk-SLHDZJ4B.js.map +1 -0
- package/dist/{chunk-YEKQKMCU.cjs → chunk-T5XU5TAV.cjs} +82 -82
- package/dist/{chunk-YEKQKMCU.cjs.map → chunk-T5XU5TAV.cjs.map} +1 -1
- package/dist/{chunk-YN3FI4AD.js → chunk-UORUVXKY.js} +5 -5
- package/dist/{chunk-YN3FI4AD.js.map → chunk-UORUVXKY.js.map} +1 -1
- package/dist/{chunk-T2CILERS.cjs → chunk-VVR3SFKH.cjs} +68 -68
- package/dist/{chunk-T2CILERS.cjs.map → chunk-VVR3SFKH.cjs.map} +1 -1
- package/dist/{chunk-EYM6DWKD.js → chunk-W2QYKEBN.js} +5 -5
- package/dist/{chunk-EYM6DWKD.js.map → chunk-W2QYKEBN.js.map} +1 -1
- package/dist/{chunk-PK3X5KIF.cjs → chunk-X36INADV.cjs} +23 -23
- package/dist/{chunk-PK3X5KIF.cjs.map → chunk-X36INADV.cjs.map} +1 -1
- package/dist/{chunk-45M6KUQZ.js → chunk-YBVRBNRY.js} +7 -7
- package/dist/{chunk-45M6KUQZ.js.map → chunk-YBVRBNRY.js.map} +1 -1
- package/dist/{chunk-QR2Z4WQ7.js → chunk-ZFH5KWHO.js} +3 -3
- package/dist/{chunk-QR2Z4WQ7.js.map → chunk-ZFH5KWHO.js.map} +1 -1
- package/dist/{chunk-4KGTZPYF.cjs → chunk-ZFZ6HQFT.cjs} +185 -185
- package/dist/{chunk-4KGTZPYF.cjs.map → chunk-ZFZ6HQFT.cjs.map} +1 -1
- package/dist/datasets/index.cjs +17 -17
- package/dist/datasets/index.js +2 -2
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +469 -469
- package/dist/docs/references/docs-memory-observational-memory.md +49 -4
- package/dist/docs/references/docs-server-mastra-client.md +17 -0
- package/dist/docs/references/reference-client-js-agents.md +11 -6
- package/dist/docs/references/reference-evals-scorer-utils.md +9 -5
- package/dist/docs/references/reference-evals-trajectory-accuracy.md +29 -15
- package/dist/docs/references/reference-memory-observational-memory.md +32 -6
- package/dist/evals/index.cjs +23 -23
- package/dist/evals/index.js +3 -3
- package/dist/evals/scoreTraces/index.cjs +7 -7
- package/dist/evals/scoreTraces/index.js +3 -3
- package/dist/evals/types.d.ts +58 -28
- package/dist/evals/types.d.ts.map +1 -1
- package/dist/harness/harness.d.ts.map +1 -1
- package/dist/harness/index.cjs +28 -8
- package/dist/harness/index.cjs.map +1 -1
- package/dist/harness/index.js +26 -6
- package/dist/harness/index.js.map +1 -1
- package/dist/index.cjs +2 -2
- package/dist/index.js +1 -1
- package/dist/llm/index.cjs +16 -16
- package/dist/llm/index.js +5 -5
- package/dist/llm/model/provider-types.generated.d.ts +14 -2
- package/dist/loop/index.cjs +14 -14
- package/dist/loop/index.js +1 -1
- package/dist/loop/workflows/agentic-execution/llm-execution-step.d.ts.map +1 -1
- package/dist/mastra/index.cjs +2 -2
- package/dist/mastra/index.d.ts +11 -0
- package/dist/mastra/index.d.ts.map +1 -1
- package/dist/mastra/index.js +1 -1
- package/dist/memory/index.cjs +14 -14
- package/dist/memory/index.js +1 -1
- package/dist/memory/memory.d.ts.map +1 -1
- package/dist/memory/types.d.ts +16 -2
- package/dist/memory/types.d.ts.map +1 -1
- package/dist/models-dev-MIC2CW43.cjs +12 -0
- package/dist/{models-dev-GLFQVCT4.cjs.map → models-dev-MIC2CW43.cjs.map} +1 -1
- package/dist/models-dev-VDZIOMAC.js +3 -0
- package/dist/{models-dev-WSLKK6FO.js.map → models-dev-VDZIOMAC.js.map} +1 -1
- package/dist/netlify-2CGMKEKT.js +3 -0
- package/dist/{netlify-SNTN56QS.js.map → netlify-2CGMKEKT.js.map} +1 -1
- package/dist/netlify-KZ4HQR5N.cjs +12 -0
- package/dist/{netlify-VFRYJYBK.cjs.map → netlify-KZ4HQR5N.cjs.map} +1 -1
- package/dist/observability/index.cjs +18 -18
- package/dist/observability/index.js +1 -1
- package/dist/observability/types/tracing.d.ts +3 -0
- package/dist/observability/types/tracing.d.ts.map +1 -1
- package/dist/processor-provider/index.cjs +10 -10
- package/dist/processor-provider/index.js +1 -1
- package/dist/processors/index.cjs +44 -44
- package/dist/processors/index.js +1 -1
- package/dist/processors/tool-result-reminder.d.ts +2 -0
- package/dist/processors/tool-result-reminder.d.ts.map +1 -1
- package/dist/provider-registry-HGWTTWCL.cjs +40 -0
- package/dist/{provider-registry-L6KDLMFM.cjs.map → provider-registry-HGWTTWCL.cjs.map} +1 -1
- package/dist/provider-registry-QRWUBN5N.js +3 -0
- package/dist/{provider-registry-LEXW2E53.js.map → provider-registry-QRWUBN5N.js.map} +1 -1
- package/dist/provider-registry.json +34 -10
- package/dist/relevance/index.cjs +3 -3
- package/dist/relevance/index.js +1 -1
- package/dist/storage/constants.cjs +56 -56
- package/dist/storage/constants.js +1 -1
- package/dist/storage/index.cjs +268 -268
- package/dist/storage/index.js +3 -3
- package/dist/stream/index.cjs +8 -8
- package/dist/stream/index.js +1 -1
- package/dist/test-utils/llm-mock.cjs +4 -4
- package/dist/test-utils/llm-mock.js +1 -1
- package/dist/tool-loop-agent/index.cjs +4 -4
- package/dist/tool-loop-agent/index.js +1 -1
- package/dist/utils.cjs +23 -23
- package/dist/utils.js +1 -1
- package/dist/vector/index.cjs +7 -7
- package/dist/vector/index.js +1 -1
- package/dist/workflows/evented/index.cjs +10 -10
- package/dist/workflows/evented/index.js +1 -1
- package/dist/workflows/index.cjs +24 -24
- package/dist/workflows/index.js +1 -1
- package/package.json +4 -3
- package/src/llm/model/provider-types.generated.d.ts +14 -2
- package/dist/chunk-7IDACSBM.cjs.map +0 -1
- package/dist/chunk-DQYOFNZ7.cjs.map +0 -1
- package/dist/chunk-H5O522FY.cjs.map +0 -1
- package/dist/chunk-IOCCTQYZ.js.map +0 -1
- package/dist/chunk-VAVK4KJI.cjs.map +0 -1
- package/dist/chunk-VF3CL3JV.js.map +0 -1
- package/dist/chunk-VMKNS3YO.js.map +0 -1
- package/dist/chunk-WVTVBWOQ.js.map +0 -1
- package/dist/models-dev-GLFQVCT4.cjs +0 -12
- package/dist/models-dev-WSLKK6FO.js +0 -3
- package/dist/netlify-SNTN56QS.js +0 -3
- package/dist/netlify-VFRYJYBK.cjs +0 -12
- package/dist/provider-registry-L6KDLMFM.cjs +0 -40
- package/dist/provider-registry-LEXW2E53.js +0 -3
|
@@ -95,27 +95,72 @@ The result is a three-tier system:
|
|
|
95
95
|
|
|
96
96
|
Normal OM compresses messages into observations, which is great for staying on task — but the original wording is gone. Retrieval mode fixes this by keeping each observation group linked to the raw messages that produced it. When the agent needs exact wording, tool output, or chronology that the summary compressed away, it can call a `recall` tool to page through the source messages.
|
|
97
97
|
|
|
98
|
+
#### Browsing only
|
|
99
|
+
|
|
100
|
+
Set `retrieval: true` to enable the recall tool for browsing raw messages. No vector store needed. By default, the recall tool can browse across all threads for the current resource.
|
|
101
|
+
|
|
98
102
|
```typescript
|
|
99
103
|
const memory = new Memory({
|
|
100
104
|
options: {
|
|
101
105
|
observationalMemory: {
|
|
102
106
|
model: 'google/gemini-2.5-flash',
|
|
103
|
-
scope: 'thread',
|
|
104
107
|
retrieval: true,
|
|
105
108
|
},
|
|
106
109
|
},
|
|
107
110
|
})
|
|
108
111
|
```
|
|
109
112
|
|
|
113
|
+
#### With semantic search
|
|
114
|
+
|
|
115
|
+
Set `retrieval: { vector: true }` to also enable semantic search. This reuses the vector store and embedder already configured on your Memory instance:
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
const memory = new Memory({
|
|
119
|
+
storage,
|
|
120
|
+
vector: myVectorStore,
|
|
121
|
+
embedder: myEmbedder,
|
|
122
|
+
options: {
|
|
123
|
+
observationalMemory: {
|
|
124
|
+
model: 'google/gemini-2.5-flash',
|
|
125
|
+
retrieval: { vector: true },
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
})
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
When vector search is configured, new observation groups are automatically indexed at buffer time and during synchronous observation (fire-and-forget, non-blocking). Semantic search returns observation-group matches with their raw source message ID ranges, so the recall tool can show the summarized memory alongside where it came from.
|
|
132
|
+
|
|
133
|
+
#### Restricting to the current thread
|
|
134
|
+
|
|
135
|
+
By default, the recall tool scope is `'resource'` — the agent can list threads, browse other threads, and search across all conversations. Set `scope: 'thread'` to restrict the agent to only the current thread:
|
|
136
|
+
|
|
137
|
+
```typescript
|
|
138
|
+
const memory = new Memory({
|
|
139
|
+
options: {
|
|
140
|
+
observationalMemory: {
|
|
141
|
+
model: 'google/gemini-2.5-flash',
|
|
142
|
+
retrieval: { vector: true, scope: 'thread' },
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
})
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
#### What retrieval enables
|
|
149
|
+
|
|
110
150
|
With retrieval mode enabled, OM:
|
|
111
151
|
|
|
112
152
|
- Stores a `range` (e.g. `startId:endId`) on each observation group pointing to the messages it was derived from
|
|
153
|
+
|
|
113
154
|
- Keeps range metadata visible in the agent's context so the agent knows which observations map to which messages
|
|
114
|
-
- Registers a `recall` tool the agent can call to page through the raw messages behind any range
|
|
115
155
|
|
|
116
|
-
|
|
156
|
+
- Registers a `recall` tool the agent can call to:
|
|
157
|
+
|
|
158
|
+
- Page through the raw messages behind any observation group range
|
|
159
|
+
- Search by semantic similarity (`mode: "search"` with a `query` string) — requires `vector: true`
|
|
160
|
+
- List all threads (`mode: "threads"`), browse other threads (`threadId`), and search across all threads (default `scope: 'resource'`)
|
|
161
|
+
- When `scope: 'thread'`: restrict browsing and search to the current thread only
|
|
117
162
|
|
|
118
|
-
See the [recall tool reference](https://mastra.ai/reference/memory/observational-memory) for the full API (detail levels, part indexing, pagination, and token limiting).
|
|
163
|
+
See the [recall tool reference](https://mastra.ai/reference/memory/observational-memory) for the full API (detail levels, part indexing, pagination, cross-thread browsing, and token limiting).
|
|
119
164
|
|
|
120
165
|
## Models
|
|
121
166
|
|
|
@@ -133,6 +133,23 @@ export const mastraClient = new MastraClient({
|
|
|
133
133
|
|
|
134
134
|
> **Info:** Visit [MastraClient](https://mastra.ai/reference/client-js/mastra-client) for more configuration options.
|
|
135
135
|
|
|
136
|
+
## Credentials and session cookies
|
|
137
|
+
|
|
138
|
+
**Authenticate Mastra API calls with session cookies** when your UI and Mastra API are not on the same origin—different host, subdomain, or port (for example Mastra Studio on one port and a custom server on another). Add **`credentials: 'include'`** to `MastraClient` so each request carries the cookies the user already has after sign-in. Skip this and you will often get **`401`** responses from Mastra even though login succeeded in the browser.
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
import { MastraClient } from '@mastra/client-js'
|
|
142
|
+
|
|
143
|
+
export const mastraClient = new MastraClient({
|
|
144
|
+
baseUrl: process.env.MASTRA_API_URL || 'http://localhost:4111',
|
|
145
|
+
credentials: 'include',
|
|
146
|
+
})
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
**Allow credentialed cross-origin requests on your server**—see [CORS: requests with credentials](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS#requests_with_credentials). You need a concrete `Access-Control-Allow-Origin` (not `*`) and `Access-Control-Allow-Credentials: true`, or the browser will block the call before it reaches Mastra.
|
|
150
|
+
|
|
151
|
+
**Using `@mastra/react`?** Wrap your app with `MastraReactProvider`, set `baseUrl` and `apiPrefix` to match your server, and rely on the default `credentials: 'include'`. Change `credentials` only when you deliberately want `same-origin` or `omit` behavior.
|
|
152
|
+
|
|
136
153
|
## Adding request cancelling
|
|
137
154
|
|
|
138
155
|
`MastraClient` supports request cancellation using the standard Node.js `AbortSignal` API. Useful for canceling in-flight requests, such as when users abort an operation or to clean up stale network calls.
|
|
@@ -308,7 +308,7 @@ response.processDataStream({
|
|
|
308
308
|
|
|
309
309
|
## Stored agents
|
|
310
310
|
|
|
311
|
-
Stored agents are agent configurations stored in a database that can be created, updated, and deleted at runtime. They reference primitives (tools, workflows, other agents,
|
|
311
|
+
Stored agents are agent configurations stored in a database that can be created, updated, and deleted at runtime. They reference primitives (tools, workflows, other agents, scorers) by key, which are resolved from the Mastra registry when the agent is instantiated. Memory is configured inline as a `SerializedMemoryConfig` object with options such as `lastMessages` and `semanticRecall`.
|
|
312
312
|
|
|
313
313
|
### `listStoredAgents()`
|
|
314
314
|
|
|
@@ -361,10 +361,15 @@ const agent = await mastraClient.createStoredAgent({
|
|
|
361
361
|
provider: 'openai',
|
|
362
362
|
name: 'gpt-5.4',
|
|
363
363
|
},
|
|
364
|
-
tools:
|
|
365
|
-
workflows:
|
|
366
|
-
agents:
|
|
367
|
-
memory:
|
|
364
|
+
tools: { calculator: {}, weather: {} },
|
|
365
|
+
workflows: { 'data-processing': {} },
|
|
366
|
+
agents: { 'subagent-1': {} },
|
|
367
|
+
memory: {
|
|
368
|
+
options: {
|
|
369
|
+
lastMessages: 20,
|
|
370
|
+
semanticRecall: false,
|
|
371
|
+
},
|
|
372
|
+
},
|
|
368
373
|
scorers: {
|
|
369
374
|
'quality-scorer': {
|
|
370
375
|
sampling: { type: 'ratio', rate: 0.1 },
|
|
@@ -415,7 +420,7 @@ const updated = await storedAgent.update({
|
|
|
415
420
|
```typescript
|
|
416
421
|
// Update just the tools
|
|
417
422
|
await storedAgent.update({
|
|
418
|
-
tools:
|
|
423
|
+
tools: { 'new-tool-1': {}, 'new-tool-2': {} },
|
|
419
424
|
})
|
|
420
425
|
|
|
421
426
|
// Update metadata
|
|
@@ -367,10 +367,11 @@ The `expected` parameter accepts either a `Trajectory` (actual trajectory) or `{
|
|
|
367
367
|
import { compareTrajectories } from '@mastra/evals/scorers/utils'
|
|
368
368
|
|
|
369
369
|
// Using ExpectedStep[] (recommended for expectations)
|
|
370
|
+
// Data fields (e.g. toolArgs) are auto-compared when present on expected steps
|
|
370
371
|
const result = compareTrajectories(
|
|
371
372
|
actualTrajectory,
|
|
372
373
|
{ steps: [{ name: 'search' }, { name: 'summarize', stepType: 'tool_call' }] },
|
|
373
|
-
{
|
|
374
|
+
{ allowRepeatedSteps: true },
|
|
374
375
|
)
|
|
375
376
|
// result.score — 0.0 to 1.0
|
|
376
377
|
// result.missingSteps — step names not found
|
|
@@ -412,7 +413,9 @@ const result = checkTrajectoryEfficiency(trajectory, {
|
|
|
412
413
|
})
|
|
413
414
|
// result.score — 1.0 if within all budgets, lower with penalties
|
|
414
415
|
// result.redundantCalls — duplicate tool+args combos
|
|
415
|
-
// result.
|
|
416
|
+
// result.overStepBudget — true if maxSteps exceeded
|
|
417
|
+
// result.overTokenBudget — true if maxTotalTokens exceeded
|
|
418
|
+
// result.overDurationBudget — true if maxTotalDurationMs exceeded
|
|
416
419
|
```
|
|
417
420
|
|
|
418
421
|
**Returns:** `TrajectoryEfficiencyResult`
|
|
@@ -428,8 +431,9 @@ const result = checkTrajectoryBlacklist(trajectory, {
|
|
|
428
431
|
blacklistedTools: ['deleteAll', 'admin-override'],
|
|
429
432
|
blacklistedSequences: [['escalate', 'admin-override']],
|
|
430
433
|
})
|
|
431
|
-
// result.
|
|
432
|
-
// result.
|
|
434
|
+
// result.score — 1.0 if no violations, 0.0 if any found
|
|
435
|
+
// result.violatedTools — blacklisted tools that were called
|
|
436
|
+
// result.violatedSequences — blacklisted sequences that were detected
|
|
433
437
|
```
|
|
434
438
|
|
|
435
439
|
**Returns:** `TrajectoryBlacklistResult`
|
|
@@ -442,7 +446,7 @@ Detects tool failure patterns including retries, fallbacks, and argument correct
|
|
|
442
446
|
import { analyzeToolFailures } from '@mastra/evals/scorers/utils'
|
|
443
447
|
|
|
444
448
|
const result = analyzeToolFailures(trajectory, {
|
|
445
|
-
maxRetriesPerTool:
|
|
449
|
+
maxRetriesPerTool: 2,
|
|
446
450
|
})
|
|
447
451
|
// result.score — 1.0 if no failure patterns, lower if patterns detected
|
|
448
452
|
// result.patterns — detected patterns (retry, fallback, arg_correction)
|
|
@@ -103,13 +103,15 @@ All step types share the base properties `name`, `durationMs`, `metadata`, and `
|
|
|
103
103
|
|
|
104
104
|
## Expected steps
|
|
105
105
|
|
|
106
|
-
When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a
|
|
106
|
+
When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a discriminated union that mirrors `TrajectoryStep` — when you specify a `stepType`, you get autocomplete for that variant's fields (e.g., `toolArgs` for `tool_call`, `modelId` for `model_generation`). All variant-specific fields are optional, so you only assert against what you care about.
|
|
107
|
+
|
|
108
|
+
Omit `stepType` entirely to match any step by name only.
|
|
107
109
|
|
|
108
110
|
**name** (`string`): Step name to match (tool name, agent ID, workflow step name, etc.).
|
|
109
111
|
|
|
110
|
-
**stepType** (`TrajectoryStepType`): Step type
|
|
112
|
+
**stepType** (`TrajectoryStepType`): Step type discriminant. When set, enables autocomplete for that variant's fields. If omitted, matches any step type with the given name.
|
|
111
113
|
|
|
112
|
-
**
|
|
114
|
+
**(variant fields)** (`varies`): Type-specific fields from the corresponding TrajectoryStep variant. For example, \`toolArgs\` and \`toolResult\` for \`tool\_call\`, \`modelId\` for \`model\_generation\`, \`output\` for \`workflow\_step\`. All optional — only specified fields are compared.
|
|
113
115
|
|
|
114
116
|
**children** (`TrajectoryExpectation`): Nested expectation config for this step's children. Overrides the parent config for evaluating children of this step.
|
|
115
117
|
|
|
@@ -120,11 +122,14 @@ const steps: ExpectedStep[] = [
|
|
|
120
122
|
// Match by name only (any step type)
|
|
121
123
|
{ name: 'search' },
|
|
122
124
|
|
|
123
|
-
// Match by name and step type
|
|
125
|
+
// Match by name and step type (autocomplete for tool_call fields)
|
|
124
126
|
{ name: 'search', stepType: 'tool_call' },
|
|
125
127
|
|
|
126
|
-
// Match with
|
|
127
|
-
{ name: 'search', stepType: 'tool_call',
|
|
128
|
+
// Match with specific toolArgs (auto-compared when present)
|
|
129
|
+
{ name: 'search', stepType: 'tool_call', toolArgs: { query: 'weather' } },
|
|
130
|
+
|
|
131
|
+
// Match a model generation step by model ID
|
|
132
|
+
{ name: 'gpt-4o', stepType: 'model_generation', modelId: 'gpt-4o' },
|
|
128
133
|
]
|
|
129
134
|
```
|
|
130
135
|
|
|
@@ -182,7 +187,7 @@ The `createTrajectoryAccuracyScorerCode()` function from `@mastra/evals/scorers/
|
|
|
182
187
|
|
|
183
188
|
### Parameters
|
|
184
189
|
|
|
185
|
-
**expectedTrajectory** (`
|
|
190
|
+
**expectedTrajectory** (`Trajectory | ExpectedStep[]`): Static expected trajectory to compare against. Accepts a full Trajectory or an array of ExpectedStep matchers. When omitted, the scorer reads expectedTrajectory from each dataset item at runtime.
|
|
186
191
|
|
|
187
192
|
**comparisonOptions** (`TrajectoryComparisonOptions`): Controls how the comparison is performed.
|
|
188
193
|
|
|
@@ -368,8 +373,8 @@ const scorer = createTrajectoryAccuracyScorerCode({
|
|
|
368
373
|
},
|
|
369
374
|
],
|
|
370
375
|
},
|
|
371
|
-
comparisonOptions: { compareStepData: true },
|
|
372
376
|
})
|
|
377
|
+
// Data fields like toolArgs are auto-compared when present on expected steps
|
|
373
378
|
```
|
|
374
379
|
|
|
375
380
|
## LLM-based trajectory accuracy scorer
|
|
@@ -380,7 +385,7 @@ The `createTrajectoryAccuracyScorerLLM()` function from `@mastra/evals/scorers/p
|
|
|
380
385
|
|
|
381
386
|
**model** (`MastraModelConfig`): The LLM model to use for evaluating trajectory quality.
|
|
382
387
|
|
|
383
|
-
**expectedTrajectory** (`
|
|
388
|
+
**expectedTrajectory** (`Trajectory | ExpectedStep[]`): Optional static expected trajectory to compare against. Accepts a full Trajectory or an array of ExpectedStep matchers. When omitted, the LLM evaluates the trajectory based on the task requirements alone. Can also come from dataset items at runtime.
|
|
384
389
|
|
|
385
390
|
### Features
|
|
386
391
|
|
|
@@ -461,7 +466,7 @@ The `createTrajectoryScorerCode()` function from `@mastra/evals/scorers/prebuilt
|
|
|
461
466
|
|
|
462
467
|
**defaults** (`TrajectoryExpectation`): Default expectations applied to all dataset items. Per-item expectedTrajectory values override these defaults.
|
|
463
468
|
|
|
464
|
-
**weights** (`
|
|
469
|
+
**weights** (`TrajectoryScoreWeights`): Custom weights for combining dimension scores. Weights are normalized to sum to 1.0.
|
|
465
470
|
|
|
466
471
|
### Scoring behavior
|
|
467
472
|
|
|
@@ -472,7 +477,7 @@ The unified scorer evaluates four dimensions:
|
|
|
472
477
|
3. **Blacklist** — Checks for forbidden tools or sequences. Any violation immediately results in a score of **0.0** regardless of other dimensions.
|
|
473
478
|
4. **Tool failures** — Detects retry patterns, fallback patterns, and argument correction patterns.
|
|
474
479
|
|
|
475
|
-
The final score is a weighted
|
|
480
|
+
The final score is a weighted combination of active dimensions, normalized by which dimensions are active. Default weights are accuracy 0.4, efficiency 0.3, tool failures 0.2, blacklist 0.1, but you can customize them via the `weights` option. Blacklist violations override everything to 0. When nested evaluations are present, the score is 70% top-level and 30% nested average.
|
|
476
481
|
|
|
477
482
|
### Unified scorer results
|
|
478
483
|
|
|
@@ -481,11 +486,13 @@ The final score is a weighted average of accuracy, efficiency, and tool failures
|
|
|
481
486
|
runId: string,
|
|
482
487
|
preprocessStepResult: {
|
|
483
488
|
accuracy?: TrajectoryComparisonResult,
|
|
484
|
-
efficiency
|
|
485
|
-
blacklist
|
|
486
|
-
toolFailures
|
|
489
|
+
efficiency?: TrajectoryEfficiencyResult,
|
|
490
|
+
blacklist?: TrajectoryBlacklistResult,
|
|
491
|
+
toolFailures?: ToolFailureAnalysisResult,
|
|
492
|
+
nested?: NestedEvaluationResult[],
|
|
487
493
|
},
|
|
488
|
-
score: number
|
|
494
|
+
score: number,
|
|
495
|
+
reason: string
|
|
489
496
|
}
|
|
490
497
|
```
|
|
491
498
|
|
|
@@ -542,6 +549,13 @@ const scorer = createTrajectoryScorerCode({
|
|
|
542
549
|
noRedundantCalls: true,
|
|
543
550
|
maxRetriesPerTool: 2,
|
|
544
551
|
},
|
|
552
|
+
// Customize how dimensions contribute to the final score
|
|
553
|
+
weights: {
|
|
554
|
+
accuracy: 0.5, // prioritize step accuracy
|
|
555
|
+
efficiency: 0.3,
|
|
556
|
+
toolFailures: 0.1,
|
|
557
|
+
blacklist: 0.1,
|
|
558
|
+
},
|
|
545
559
|
})
|
|
546
560
|
```
|
|
547
561
|
|
|
@@ -38,7 +38,7 @@ OM performs thresholding with fast local token estimation. Text uses `tokenx`, a
|
|
|
38
38
|
|
|
39
39
|
**shareTokenBudget** (`boolean`): Share the token budget between messages and observations. When enabled, the total budget is \`observation.messageTokens + reflection.observationTokens\`. Messages can use more space when observations are small, and vice versa. This maximizes context usage through flexible allocation. \`shareTokenBudget\` is not yet compatible with async buffering. You must set \`observation: { bufferTokens: false }\` when using this option (this is a temporary limitation). (Default: `false`)
|
|
40
40
|
|
|
41
|
-
**retrieval** (`boolean`): \*\*Experimental.\*\* Enable retrieval-mode observation groups as durable pointers to raw message history.
|
|
41
|
+
**retrieval** (`boolean | { vector?: boolean; scope?: 'thread' | 'resource' }`): \*\*Experimental.\*\* Enable retrieval-mode observation groups as durable pointers to raw message history. \`true\` enables cross-thread browsing by default. \`{ vector: true }\` also enables semantic search using Memory's vector store and embedder. \`{ scope: 'thread' }\` restricts the recall tool to the current thread only. Default scope is \`'resource'\`. (Default: `false`)
|
|
42
42
|
|
|
43
43
|
**observation** (`ObservationalMemoryObservationConfig`): Configuration for the observation step. Controls when the Observer agent runs and how it behaves.
|
|
44
44
|
|
|
@@ -578,21 +578,31 @@ The standalone `ObservationalMemory` class accepts all the same options as the `
|
|
|
578
578
|
|
|
579
579
|
## Recall tool
|
|
580
580
|
|
|
581
|
-
When `retrieval
|
|
581
|
+
When `retrieval` is set (any truthy value), a `recall` tool is registered so the agent can page through raw messages behind observation group ranges. By default (scope `'resource'`), the tool supports listing threads (`mode: "threads"`), browsing other threads (`threadId`), and cross-thread search. With `retrieval: { vector: true }`, semantic search is available (`mode: "search"`). Set `scope: 'thread'` to restrict the tool to the current thread only. The tool is automatically added to the agent's tool list — no manual registration is needed.
|
|
582
582
|
|
|
583
583
|
### Parameters
|
|
584
584
|
|
|
585
|
-
**
|
|
585
|
+
**mode** (`'messages' | 'threads' | 'search'`): What to retrieve. \`"messages"\` (default) pages through message history. \`"threads"\` lists all threads for the current user. \`"search"\` finds messages by semantic similarity across all threads (requires vector store and embedder). (Default: `'messages'`)
|
|
586
586
|
|
|
587
|
-
**
|
|
587
|
+
**query** (`string`): Search query for \`mode: "search"\`. Finds messages semantically similar to this text across all threads for the current user.
|
|
588
588
|
|
|
589
|
-
**
|
|
589
|
+
**cursor** (`string`): A message ID to anchor the recall query. Required for \`mode: "messages"\` when browsing the current thread. Extract the start or end ID from an observation group range (e.g. from \`\_range: \\\`startId:endId\\\`\_\`, use either \`startId\` or \`endId\`). If a range string is passed directly, the tool returns a hint explaining how to extract the correct ID. Can be omitted when \`threadId\` is provided to start reading from the beginning of that thread.
|
|
590
|
+
|
|
591
|
+
**threadId** (`string`): Browse a different thread by its ID. Use \`mode: "threads"\` first to discover thread IDs. When provided without a \`cursor\`, reading starts from the beginning of the thread.
|
|
592
|
+
|
|
593
|
+
**page** (`number`): Pagination offset. For messages: positive values page forward from cursor, negative values page backward. For threads: page number (0-indexed). \`0\` is treated as \`1\` for messages. (Default: `1`)
|
|
594
|
+
|
|
595
|
+
**limit** (`number`): Maximum number of items to return per page. (Default: `20`)
|
|
590
596
|
|
|
591
597
|
**detail** (`'low' | 'high'`): Controls how much content is shown per message part. \`'low'\` shows truncated text and tool names with positional indices (\`\[p0]\`, \`\[p1]\`). \`'high'\` shows full content including tool arguments and results, clamped to one part per call with continuation hints. (Default: `'low'`)
|
|
592
598
|
|
|
593
599
|
**partIndex** (`number`): Fetch a single message part at full detail by its positional index. Use this when a low-detail recall shows an interesting part at \`\[p1]\` — call again with \`partIndex: 1\` to see the full content without loading every part.
|
|
594
600
|
|
|
595
|
-
|
|
601
|
+
**before** (`string`): For \`mode: "threads"\` only. Filter to threads created before this date. Accepts ISO 8601 format (e.g. \`"2026-03-15"\`, \`"2026-03-10T00:00:00Z"\`).
|
|
602
|
+
|
|
603
|
+
**after** (`string`): For \`mode: "threads"\` only. Filter to threads created after this date. Accepts ISO 8601 format (e.g. \`"2026-03-01"\`, \`"2026-03-10T00:00:00Z"\`).
|
|
604
|
+
|
|
605
|
+
### Returns (messages mode)
|
|
596
606
|
|
|
597
607
|
**messages** (`string`): Formatted message content. Format depends on the \`detail\` level.
|
|
598
608
|
|
|
@@ -612,6 +622,22 @@ When `retrieval: true` is set with `scope: 'thread'`, OM registers a `recall` to
|
|
|
612
622
|
|
|
613
623
|
**tokenOffset** (`number`): Approximate number of tokens that were trimmed when \`truncated\` is true.
|
|
614
624
|
|
|
625
|
+
### Returns (threads mode)
|
|
626
|
+
|
|
627
|
+
**threads** (`string`): Formatted thread listing. Each thread shows its title, ID, and dates. The current thread is marked with \`← current\`.
|
|
628
|
+
|
|
629
|
+
**count** (`number`): Number of threads returned.
|
|
630
|
+
|
|
631
|
+
**page** (`number`): The page number returned.
|
|
632
|
+
|
|
633
|
+
**hasMore** (`boolean`): Whether more threads exist on the next page.
|
|
634
|
+
|
|
635
|
+
### Returns (search mode)
|
|
636
|
+
|
|
637
|
+
**results** (`string`): Formatted search results grouped by thread. Each result shows the thread title, thread ID, relevance score, message preview, and a cursor ID for browsing into that thread.
|
|
638
|
+
|
|
639
|
+
**count** (`number`): Number of matching messages found.
|
|
640
|
+
|
|
615
641
|
### ModelByInputTokens
|
|
616
642
|
|
|
617
643
|
`ModelByInputTokens` selects a model based on the input token count. It chooses the model for the smallest threshold that covers the actual input size.
|
package/dist/evals/index.cjs
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var
|
|
3
|
+
var chunkPYNUWPGW_cjs = require('../chunk-PYNUWPGW.cjs');
|
|
4
|
+
var chunk4K2NKQGN_cjs = require('../chunk-4K2NKQGN.cjs');
|
|
5
|
+
var chunkQXPUTX3W_cjs = require('../chunk-QXPUTX3W.cjs');
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
Object.defineProperty(exports, "runEvals", {
|
|
10
10
|
enumerable: true,
|
|
11
|
-
get: function () { return
|
|
11
|
+
get: function () { return chunkPYNUWPGW_cjs.runEvals; }
|
|
12
12
|
});
|
|
13
13
|
Object.defineProperty(exports, "MastraScorer", {
|
|
14
14
|
enumerable: true,
|
|
15
|
-
get: function () { return
|
|
15
|
+
get: function () { return chunk4K2NKQGN_cjs.MastraScorer; }
|
|
16
16
|
});
|
|
17
17
|
Object.defineProperty(exports, "createScorer", {
|
|
18
18
|
enumerable: true,
|
|
19
|
-
get: function () { return
|
|
19
|
+
get: function () { return chunk4K2NKQGN_cjs.createScorer; }
|
|
20
20
|
});
|
|
21
21
|
Object.defineProperty(exports, "extractTrajectory", {
|
|
22
22
|
enumerable: true,
|
|
23
|
-
get: function () { return
|
|
23
|
+
get: function () { return chunkQXPUTX3W_cjs.extractTrajectory; }
|
|
24
24
|
});
|
|
25
25
|
Object.defineProperty(exports, "extractTrajectoryFromTrace", {
|
|
26
26
|
enumerable: true,
|
|
27
|
-
get: function () { return
|
|
27
|
+
get: function () { return chunkQXPUTX3W_cjs.extractTrajectoryFromTrace; }
|
|
28
28
|
});
|
|
29
29
|
Object.defineProperty(exports, "extractWorkflowTrajectory", {
|
|
30
30
|
enumerable: true,
|
|
31
|
-
get: function () { return
|
|
31
|
+
get: function () { return chunkQXPUTX3W_cjs.extractWorkflowTrajectory; }
|
|
32
32
|
});
|
|
33
33
|
Object.defineProperty(exports, "listScoresResponseSchema", {
|
|
34
34
|
enumerable: true,
|
|
35
|
-
get: function () { return
|
|
35
|
+
get: function () { return chunkQXPUTX3W_cjs.listScoresResponseSchema; }
|
|
36
36
|
});
|
|
37
37
|
Object.defineProperty(exports, "saveScorePayloadSchema", {
|
|
38
38
|
enumerable: true,
|
|
39
|
-
get: function () { return
|
|
39
|
+
get: function () { return chunkQXPUTX3W_cjs.saveScorePayloadSchema; }
|
|
40
40
|
});
|
|
41
41
|
Object.defineProperty(exports, "scoreResultSchema", {
|
|
42
42
|
enumerable: true,
|
|
43
|
-
get: function () { return
|
|
43
|
+
get: function () { return chunkQXPUTX3W_cjs.scoreResultSchema; }
|
|
44
44
|
});
|
|
45
45
|
Object.defineProperty(exports, "scoreRowDataSchema", {
|
|
46
46
|
enumerable: true,
|
|
47
|
-
get: function () { return
|
|
47
|
+
get: function () { return chunkQXPUTX3W_cjs.scoreRowDataSchema; }
|
|
48
48
|
});
|
|
49
49
|
Object.defineProperty(exports, "scoringEntityTypeSchema", {
|
|
50
50
|
enumerable: true,
|
|
51
|
-
get: function () { return
|
|
51
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringEntityTypeSchema; }
|
|
52
52
|
});
|
|
53
53
|
Object.defineProperty(exports, "scoringExtractStepResultSchema", {
|
|
54
54
|
enumerable: true,
|
|
55
|
-
get: function () { return
|
|
55
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringExtractStepResultSchema; }
|
|
56
56
|
});
|
|
57
57
|
Object.defineProperty(exports, "scoringHookInputSchema", {
|
|
58
58
|
enumerable: true,
|
|
59
|
-
get: function () { return
|
|
59
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringHookInputSchema; }
|
|
60
60
|
});
|
|
61
61
|
Object.defineProperty(exports, "scoringInputSchema", {
|
|
62
62
|
enumerable: true,
|
|
63
|
-
get: function () { return
|
|
63
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringInputSchema; }
|
|
64
64
|
});
|
|
65
65
|
Object.defineProperty(exports, "scoringInputWithExtractStepResultAndAnalyzeStepResultSchema", {
|
|
66
66
|
enumerable: true,
|
|
67
|
-
get: function () { return
|
|
67
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringInputWithExtractStepResultAndAnalyzeStepResultSchema; }
|
|
68
68
|
});
|
|
69
69
|
Object.defineProperty(exports, "scoringInputWithExtractStepResultAndScoreAndReasonSchema", {
|
|
70
70
|
enumerable: true,
|
|
71
|
-
get: function () { return
|
|
71
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringInputWithExtractStepResultAndScoreAndReasonSchema; }
|
|
72
72
|
});
|
|
73
73
|
Object.defineProperty(exports, "scoringInputWithExtractStepResultSchema", {
|
|
74
74
|
enumerable: true,
|
|
75
|
-
get: function () { return
|
|
75
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringInputWithExtractStepResultSchema; }
|
|
76
76
|
});
|
|
77
77
|
Object.defineProperty(exports, "scoringPromptsSchema", {
|
|
78
78
|
enumerable: true,
|
|
79
|
-
get: function () { return
|
|
79
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringPromptsSchema; }
|
|
80
80
|
});
|
|
81
81
|
Object.defineProperty(exports, "scoringSourceSchema", {
|
|
82
82
|
enumerable: true,
|
|
83
|
-
get: function () { return
|
|
83
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringSourceSchema; }
|
|
84
84
|
});
|
|
85
85
|
Object.defineProperty(exports, "scoringValueSchema", {
|
|
86
86
|
enumerable: true,
|
|
87
|
-
get: function () { return
|
|
87
|
+
get: function () { return chunkQXPUTX3W_cjs.scoringValueSchema; }
|
|
88
88
|
});
|
|
89
89
|
//# sourceMappingURL=index.cjs.map
|
|
90
90
|
//# sourceMappingURL=index.cjs.map
|
package/dist/evals/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { runEvals } from '../chunk-
|
|
2
|
-
export { MastraScorer, createScorer } from '../chunk-
|
|
3
|
-
export { extractTrajectory, extractTrajectoryFromTrace, extractWorkflowTrajectory, listScoresResponseSchema, saveScorePayloadSchema, scoreResultSchema, scoreRowDataSchema, scoringEntityTypeSchema, scoringExtractStepResultSchema, scoringHookInputSchema, scoringInputSchema, scoringInputWithExtractStepResultAndAnalyzeStepResultSchema, scoringInputWithExtractStepResultAndScoreAndReasonSchema, scoringInputWithExtractStepResultSchema, scoringPromptsSchema, scoringSourceSchema, scoringValueSchema } from '../chunk-
|
|
1
|
+
export { runEvals } from '../chunk-D5VCCQ4L.js';
|
|
2
|
+
export { MastraScorer, createScorer } from '../chunk-32NEFSTI.js';
|
|
3
|
+
export { extractTrajectory, extractTrajectoryFromTrace, extractWorkflowTrajectory, listScoresResponseSchema, saveScorePayloadSchema, scoreResultSchema, scoreRowDataSchema, scoringEntityTypeSchema, scoringExtractStepResultSchema, scoringHookInputSchema, scoringInputSchema, scoringInputWithExtractStepResultAndAnalyzeStepResultSchema, scoringInputWithExtractStepResultAndScoreAndReasonSchema, scoringInputWithExtractStepResultSchema, scoringPromptsSchema, scoringSourceSchema, scoringValueSchema } from '../chunk-SLHDZJ4B.js';
|
|
4
4
|
//# sourceMappingURL=index.js.map
|
|
5
5
|
//# sourceMappingURL=index.js.map
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var
|
|
3
|
+
var chunkVVR3SFKH_cjs = require('../../chunk-VVR3SFKH.cjs');
|
|
4
|
+
var chunkQXPUTX3W_cjs = require('../../chunk-QXPUTX3W.cjs');
|
|
5
|
+
var chunkB6D4D2CY_cjs = require('../../chunk-B6D4D2CY.cjs');
|
|
6
6
|
var chunk4U7ZLI36_cjs = require('../../chunk-4U7ZLI36.cjs');
|
|
7
7
|
var pMap = require('p-map');
|
|
8
8
|
var v4 = require('zod/v4');
|
|
@@ -235,7 +235,7 @@ function transformTraceToScorerInputAndOutput(trace) {
|
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
// src/evals/scoreTraces/scoreTracesWorkflow.ts
|
|
238
|
-
var getTraceStep =
|
|
238
|
+
var getTraceStep = chunkVVR3SFKH_cjs.createStep({
|
|
239
239
|
id: "__process-trace-scoring",
|
|
240
240
|
inputSchema: v4.z.object({
|
|
241
241
|
targets: v4.z.array(
|
|
@@ -248,7 +248,7 @@ var getTraceStep = chunkT2CILERS_cjs.createStep({
|
|
|
248
248
|
}),
|
|
249
249
|
outputSchema: v4.z.any(),
|
|
250
250
|
execute: async ({ inputData, mastra, ...rest }) => {
|
|
251
|
-
const observabilityContext =
|
|
251
|
+
const observabilityContext = chunkB6D4D2CY_cjs.resolveObservabilityContext(rest);
|
|
252
252
|
const logger = mastra.getLogger();
|
|
253
253
|
if (!logger) {
|
|
254
254
|
console.warn(
|
|
@@ -383,7 +383,7 @@ async function validateAndSaveScore({ storage, scorerResult }) {
|
|
|
383
383
|
text: "Scores storage domain is not available"
|
|
384
384
|
});
|
|
385
385
|
}
|
|
386
|
-
const payloadToSave =
|
|
386
|
+
const payloadToSave = chunkQXPUTX3W_cjs.saveScorePayloadSchema.parse(scorerResult);
|
|
387
387
|
const result = await scoresStore.saveScore(payloadToSave);
|
|
388
388
|
return result.score;
|
|
389
389
|
}
|
|
@@ -446,7 +446,7 @@ async function attachScoreToSpan({
|
|
|
446
446
|
} catch {
|
|
447
447
|
}
|
|
448
448
|
}
|
|
449
|
-
var scoreTracesWorkflow =
|
|
449
|
+
var scoreTracesWorkflow = chunkVVR3SFKH_cjs.createWorkflow({
|
|
450
450
|
id: "__batch-scoring-traces",
|
|
451
451
|
inputSchema: v4.z.object({
|
|
452
452
|
targets: v4.z.array(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createStep, createWorkflow } from '../../chunk-
|
|
2
|
-
import { saveScorePayloadSchema } from '../../chunk-
|
|
3
|
-
import { resolveObservabilityContext } from '../../chunk-
|
|
1
|
+
import { createStep, createWorkflow } from '../../chunk-UORUVXKY.js';
|
|
2
|
+
import { saveScorePayloadSchema } from '../../chunk-SLHDZJ4B.js';
|
|
3
|
+
import { resolveObservabilityContext } from '../../chunk-CX5I6VS4.js';
|
|
4
4
|
import { MastraError } from '../../chunk-FJEVLHJT.js';
|
|
5
5
|
import pMap from 'p-map';
|
|
6
6
|
import { z } from 'zod/v4';
|