@assay-ai/vitest 0.2.1-beta → 1.3.1-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -67
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -1,35 +1,37 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
1
3
|
# @assay-ai/vitest
|
|
2
4
|
|
|
3
|
-
Vitest
|
|
5
|
+
*Custom Vitest matchers for LLM evaluation with Assay*
|
|
6
|
+
|
|
7
|
+
[](https://www.npmjs.com/package/@assay-ai/vitest)
|
|
8
|
+
[](https://www.npmjs.com/package/@assay-ai/vitest)
|
|
9
|
+
[](https://github.com/assay-ai/assay/blob/main/LICENSE)
|
|
4
10
|
|
|
5
|
-
[
|
|
6
|
-
|
|
11
|
+
[Documentation](https://assay.js.org) · [Metrics](https://assay.js.org/metrics/) · [API Reference](https://assay.js.org/api/)
|
|
12
|
+
|
|
13
|
+
</div>
|
|
7
14
|
|
|
8
15
|
## Installation
|
|
9
16
|
|
|
10
17
|
```bash
|
|
11
|
-
|
|
12
|
-
#
|
|
13
|
-
|
|
18
|
+
pnpm add -D @assay-ai/core @assay-ai/vitest # pnpm
|
|
19
|
+
npm install -D @assay-ai/core @assay-ai/vitest # npm
|
|
20
|
+
yarn add -D @assay-ai/core @assay-ai/vitest # Yarn
|
|
14
21
|
```
|
|
15
22
|
|
|
16
23
|
## Quick Start
|
|
17
24
|
|
|
18
|
-
### Custom Matchers
|
|
19
|
-
|
|
20
|
-
Register the Assay matchers once, then use them in any test file:
|
|
21
|
-
|
|
22
25
|
```typescript
|
|
23
|
-
// setup.ts or at the top of your test file
|
|
24
|
-
import { setupAssayMatchers } from "@assay-ai/vitest";
|
|
25
26
|
import { beforeAll, describe, expect, test } from "vitest";
|
|
27
|
+
import { setupAssayMatchers } from "@assay-ai/vitest";
|
|
26
28
|
|
|
27
29
|
beforeAll(() => {
|
|
28
30
|
setupAssayMatchers();
|
|
29
31
|
});
|
|
30
32
|
|
|
31
33
|
describe("Customer Support Chatbot", () => {
|
|
32
|
-
test("answers
|
|
34
|
+
test("answers are relevant", async () => {
|
|
33
35
|
await expect({
|
|
34
36
|
input: "What is your return policy?",
|
|
35
37
|
actualOutput: "You can return items within 30 days of purchase.",
|
|
@@ -51,58 +53,11 @@ describe("Customer Support Chatbot", () => {
|
|
|
51
53
|
});
|
|
52
54
|
```
|
|
53
55
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
| Matcher | Description |
|
|
57
|
-
|---------|-------------|
|
|
58
|
-
| `toBeRelevant(options?)` | Asserts the output is relevant to the input (Answer Relevancy) |
|
|
59
|
-
| `toBeFaithful(options?)` | Asserts the output is grounded in context (Faithfulness) |
|
|
60
|
-
| `toNotHallucinate(options?)` | Asserts the output doesn't contain hallucinations |
|
|
61
|
-
| `toPassMetric(metric)` | Asserts the test case passes a specific metric |
|
|
62
|
-
| `toPassAllMetrics(metrics)` | Asserts the test case passes all given metrics |
|
|
63
|
-
|
|
64
|
-
All matchers accept an optional `{ threshold?: number }` options object.
|
|
65
|
-
|
|
66
|
-
### Custom Metric Matcher
|
|
67
|
-
|
|
68
|
-
Use `toPassMetric` with any built-in or custom metric:
|
|
69
|
-
|
|
70
|
-
```typescript
|
|
71
|
-
import { GEval } from "@assay-ai/core";
|
|
72
|
-
|
|
73
|
-
const politeness = new GEval({
|
|
74
|
-
name: "Politeness",
|
|
75
|
-
criteria: "The response should be polite and professional.",
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
test("response is polite", async () => {
|
|
79
|
-
await expect({
|
|
80
|
-
input: "Help me with my order",
|
|
81
|
-
actualOutput: "I'd be happy to help! Could you share your order number?",
|
|
82
|
-
}).toPassMetric(politeness);
|
|
83
|
-
});
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
### Reporter
|
|
87
|
-
|
|
88
|
-
Assay includes a custom Vitest reporter that formats evaluation results:
|
|
89
|
-
|
|
90
|
-
```typescript
|
|
91
|
-
// vitest.config.ts
|
|
92
|
-
import { defineConfig } from "vitest/config";
|
|
93
|
-
|
|
94
|
-
export default defineConfig({
|
|
95
|
-
test: {
|
|
96
|
-
reporters: ["default", "@assay-ai/vitest/reporter"],
|
|
97
|
-
},
|
|
98
|
-
});
|
|
99
|
-
```
|
|
100
|
-
|
|
101
|
-
## Peer Dependencies
|
|
102
|
-
|
|
103
|
-
- `@assay-ai/core` >= 0.1.0
|
|
104
|
-
- `vitest` >= 2.0.0
|
|
105
|
-
|
|
106
|
-
## License
|
|
56
|
+
## Part of the [Assay](https://github.com/assay-ai/assay) monorepo
|
|
107
57
|
|
|
108
|
-
|
|
58
|
+
<p align="center">
|
|
59
|
+
<a href="https://assay.js.org"><img src="https://img.shields.io/badge/Documentation-6366f1?style=for-the-badge&logo=readthedocs&logoColor=white" alt="Documentation" /></a>
|
|
60
|
+
<a href="https://www.npmjs.com/package/@assay-ai/vitest"><img src="https://img.shields.io/badge/npm-cb3837?style=for-the-badge&logo=npm&logoColor=white" alt="npm" /></a>
|
|
61
|
+
<a href="https://github.com/assay-ai/assay"><img src="https://img.shields.io/badge/GitHub-181717?style=for-the-badge&logo=github&logoColor=white" alt="GitHub" /></a>
|
|
62
|
+
<a href="https://github.com/assay-ai/assay/issues"><img src="https://img.shields.io/badge/Issues-6366f1?style=for-the-badge&logo=github&logoColor=white" alt="Issues" /></a>
|
|
63
|
+
</p>
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@assay-ai/vitest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "1.3.1-beta",
|
|
4
4
|
"description": "Vitest integration for the Assay LLM evaluation framework",
|
|
5
|
+
"homepage": "https://assay.js.org",
|
|
5
6
|
"license": "MIT",
|
|
6
7
|
"type": "module",
|
|
7
8
|
"main": "./dist/index.cjs",
|
|
@@ -24,13 +25,13 @@
|
|
|
24
25
|
],
|
|
25
26
|
"peerDependencies": {
|
|
26
27
|
"vitest": ">=2.0.0",
|
|
27
|
-
"@assay-ai/core": "
|
|
28
|
+
"@assay-ai/core": "1.3.1-beta"
|
|
28
29
|
},
|
|
29
30
|
"devDependencies": {
|
|
30
31
|
"tsup": "^8.3.0",
|
|
31
32
|
"typescript": "^5.7.0",
|
|
32
33
|
"vitest": "^3.0.0",
|
|
33
|
-
"@assay-ai/core": "
|
|
34
|
+
"@assay-ai/core": "1.3.1-beta",
|
|
34
35
|
"@assay-ai/tsconfig": "0.0.0"
|
|
35
36
|
},
|
|
36
37
|
"repository": {
|