@mastra/evals 1.5.0-alpha.0 → 1.5.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# @mastra/evals
|
|
2
2
|
|
|
3
|
+
## 1.5.0-alpha.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- **Gates and verdict for `runEvals`** ([#18394](https://github.com/mastra-ai/mastra/pull/18394))
|
|
8
|
+
|
|
9
|
+
New optional `gates` field accepts scorers that must score 1.0 for the run to pass. Scorers can now use a `{ scorer, threshold }` form to set pass/fail thresholds. `threshold` accepts a number (minimum) or `{ min, max }` for range-based checks (e.g. hallucination where high = bad). The result includes `verdict`, `gateResults`, and `thresholdResults`. Fully backward compatible.
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { runEvals } from '@mastra/core/evals';
|
|
13
|
+
import { checks } from '@mastra/evals/checks';
|
|
14
|
+
|
|
15
|
+
const result = await runEvals({
|
|
16
|
+
data: [{ input: 'What is the weather?' }],
|
|
17
|
+
target: weatherAgent,
|
|
18
|
+
gates: [checks.calledTool('get_weather')],
|
|
19
|
+
scorers: [
|
|
20
|
+
{ scorer: faithfulnessScorer, threshold: 0.7 },
|
|
21
|
+
{ scorer: hallucinationScorer, threshold: { max: 0.3 } },
|
|
22
|
+
],
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
result.verdict; // 'passed' | 'scored' | 'failed'
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
- Updated dependencies [[`86623c1`](https://github.com/mastra-ai/mastra/commit/86623c1adf7d22de32cc916dda17f4155184db36), [`7c9dd77`](https://github.com/mastra-ai/mastra/commit/7c9dd77bd18cb8dc72797e25f1a0fbdc71a11347), [`9990965`](https://github.com/mastra-ai/mastra/commit/999096571635a83b42ef40841fd7028cfa630779), [`c0ffa3c`](https://github.com/mastra-ai/mastra/commit/c0ffa3c897ccd326de880df734740a7f0681a18f), [`0504bf5`](https://github.com/mastra-ai/mastra/commit/0504bf5e8cffc571a4b343326178de371e6f859b), [`5afe423`](https://github.com/mastra-ai/mastra/commit/5afe423e4badf040f1b0d4525183a856fcb8146e), [`86623c1`](https://github.com/mastra-ai/mastra/commit/86623c1adf7d22de32cc916dda17f4155184db36), [`8c9f1c0`](https://github.com/mastra-ai/mastra/commit/8c9f1c0361d89066f9bcd14a2f69e761b01766c8)]:
|
|
29
|
+
- @mastra/core@1.47.0-alpha.2
|
|
30
|
+
|
|
3
31
|
## 1.5.0-alpha.0
|
|
4
32
|
|
|
5
33
|
### Minor Changes
|
package/dist/docs/SKILL.md
CHANGED
|
@@ -136,6 +136,7 @@ Once registered, you can score traces interactively within Studio under the **Ob
|
|
|
136
136
|
## Next steps
|
|
137
137
|
|
|
138
138
|
- Use [Quick Checks](https://mastra.ai/docs/evals/quick-checks) for fast, deterministic assertions on text and tool usage
|
|
139
|
+
- Add [gates and verdicts](https://mastra.ai/docs/evals/gates-and-verdicts) to enforce hard requirements and quality thresholds
|
|
139
140
|
- Learn how to create your own scorers in the [Creating Custom Scorers](https://mastra.ai/docs/evals/custom-scorers) guide
|
|
140
141
|
- Explore built-in scorers in the [Built-in Scorers](https://mastra.ai/docs/evals/built-in-scorers) section
|
|
141
142
|
- Test scorers with [Studio](https://mastra.ai/docs/studio/overview)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/evals",
|
|
3
|
-
"version": "1.5.0-alpha.
|
|
3
|
+
"version": "1.5.0-alpha.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"files": [
|
|
@@ -91,9 +91,9 @@
|
|
|
91
91
|
"@internal/ai-sdk-v5": "0.0.55",
|
|
92
92
|
"@internal/lint": "0.0.108",
|
|
93
93
|
"@internal/llm-recorder": "0.0.44",
|
|
94
|
-
"@internal/test-utils": "0.0.44",
|
|
95
94
|
"@internal/types-builder": "0.0.83",
|
|
96
|
-
"@
|
|
95
|
+
"@internal/test-utils": "0.0.44",
|
|
96
|
+
"@mastra/core": "1.47.0-alpha.2"
|
|
97
97
|
},
|
|
98
98
|
"engines": {
|
|
99
99
|
"node": ">=22.13.0"
|