@assay-ai/jest 0.3.0-beta → 1.3.1-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -58
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -1,24 +1,27 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
1
3
|
# @assay-ai/jest
|
|
2
4
|
|
|
3
|
-
Jest
|
|
5
|
+
*Custom Jest matchers for LLM evaluation with Assay*
|
|
6
|
+
|
|
7
|
+
[](https://www.npmjs.com/package/@assay-ai/jest)
|
|
8
|
+
[](https://www.npmjs.com/package/@assay-ai/jest)
|
|
9
|
+
[](https://github.com/assay-ai/assay/blob/main/LICENSE)
|
|
10
|
+
|
|
11
|
+
[Documentation](https://assay.js.org) · [Metrics](https://assay.js.org/metrics/) · [API Reference](https://assay.js.org/api/)
|
|
4
12
|
|
|
5
|
-
|
|
6
|
-
[](https://opensource.org/licenses/MIT)
|
|
13
|
+
</div>
|
|
7
14
|
|
|
8
15
|
## Installation
|
|
9
16
|
|
|
10
17
|
```bash
|
|
11
|
-
|
|
12
|
-
#
|
|
13
|
-
|
|
18
|
+
pnpm add -D @assay-ai/core @assay-ai/jest # pnpm
|
|
19
|
+
npm install -D @assay-ai/core @assay-ai/jest # npm
|
|
20
|
+
yarn add -D @assay-ai/core @assay-ai/jest # Yarn
|
|
14
21
|
```
|
|
15
22
|
|
|
16
23
|
## Quick Start
|
|
17
24
|
|
|
18
|
-
### Setup
|
|
19
|
-
|
|
20
|
-
Register the Assay matchers in your Jest setup file:
|
|
21
|
-
|
|
22
25
|
```typescript
|
|
23
26
|
// jest.setup.ts
|
|
24
27
|
import { setupAssayMatchers } from "@assay-ai/jest";
|
|
@@ -26,23 +29,10 @@ import { setupAssayMatchers } from "@assay-ai/jest";
|
|
|
26
29
|
setupAssayMatchers();
|
|
27
30
|
```
|
|
28
31
|
|
|
29
|
-
Then configure Jest to use it:
|
|
30
|
-
|
|
31
|
-
```json
|
|
32
|
-
// jest.config.json
|
|
33
|
-
{
|
|
34
|
-
"setupFilesAfterSetup": ["./jest.setup.ts"]
|
|
35
|
-
}
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
### Writing Evaluations
|
|
39
|
-
|
|
40
32
|
```typescript
|
|
41
33
|
// chatbot.eval.ts
|
|
42
|
-
import { GEval } from "@assay-ai/core";
|
|
43
|
-
|
|
44
34
|
describe("Customer Support Chatbot", () => {
|
|
45
|
-
it("
|
|
35
|
+
it("answers are relevant", async () => {
|
|
46
36
|
await expect({
|
|
47
37
|
input: "What is your return policy?",
|
|
48
38
|
actualOutput: "You can return items within 30 days of purchase.",
|
|
@@ -52,47 +42,23 @@ describe("Customer Support Chatbot", () => {
|
|
|
52
42
|
}).toBeRelevant({ threshold: 0.8 });
|
|
53
43
|
});
|
|
54
44
|
|
|
55
|
-
it("
|
|
45
|
+
it("does not hallucinate", async () => {
|
|
56
46
|
await expect({
|
|
57
47
|
input: "What is your return policy?",
|
|
58
48
|
actualOutput: "You can return items within 30 days of purchase.",
|
|
59
|
-
|
|
49
|
+
context: [
|
|
60
50
|
"Our return policy allows returns within 30 days of purchase.",
|
|
61
51
|
],
|
|
62
|
-
}).
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
it("should pass custom metric", async () => {
|
|
66
|
-
const politeness = new GEval({
|
|
67
|
-
name: "Politeness",
|
|
68
|
-
criteria: "The response should be polite and professional.",
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
await expect({
|
|
72
|
-
input: "Help me with my order",
|
|
73
|
-
actualOutput: "I'd be happy to help! Could you share your order number?",
|
|
74
|
-
}).toPassMetric(politeness);
|
|
52
|
+
}).toNotHallucinate();
|
|
75
53
|
});
|
|
76
54
|
});
|
|
77
55
|
```
|
|
78
56
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
| Matcher | Description |
|
|
82
|
-
|---------|-------------|
|
|
83
|
-
| `toBeRelevant(options?)` | Asserts the output is relevant to the input (Answer Relevancy) |
|
|
84
|
-
| `toBeFaithful(options?)` | Asserts the output is grounded in context (Faithfulness) |
|
|
85
|
-
| `toNotHallucinate(options?)` | Asserts the output doesn't contain hallucinations |
|
|
86
|
-
| `toPassMetric(metric)` | Asserts the test case passes a specific metric |
|
|
87
|
-
| `toPassAllMetrics(metrics)` | Asserts the test case passes all given metrics |
|
|
88
|
-
|
|
89
|
-
All matchers accept an optional `{ threshold?: number }` options object.
|
|
90
|
-
|
|
91
|
-
## Peer Dependencies
|
|
92
|
-
|
|
93
|
-
- `@assay-ai/core` >= 0.1.0
|
|
94
|
-
- `jest` >= 29.0.0
|
|
95
|
-
|
|
96
|
-
## License
|
|
57
|
+
## Part of the [Assay](https://github.com/assay-ai/assay) monorepo
|
|
97
58
|
|
|
98
|
-
|
|
59
|
+
<p align="center">
|
|
60
|
+
<a href="https://assay.js.org"><img src="https://img.shields.io/badge/Documentation-6366f1?style=for-the-badge&logo=readthedocs&logoColor=white" alt="Documentation" /></a>
|
|
61
|
+
<a href="https://www.npmjs.com/package/@assay-ai/jest"><img src="https://img.shields.io/badge/npm-cb3837?style=for-the-badge&logo=npm&logoColor=white" alt="npm" /></a>
|
|
62
|
+
<a href="https://github.com/assay-ai/assay"><img src="https://img.shields.io/badge/GitHub-181717?style=for-the-badge&logo=github&logoColor=white" alt="GitHub" /></a>
|
|
63
|
+
<a href="https://github.com/assay-ai/assay/issues"><img src="https://img.shields.io/badge/Issues-6366f1?style=for-the-badge&logo=github&logoColor=white" alt="Issues" /></a>
|
|
64
|
+
</p>
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@assay-ai/jest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "1.3.1-beta",
|
|
4
4
|
"description": "Jest integration for the Assay LLM evaluation framework",
|
|
5
|
+
"homepage": "https://assay.js.org",
|
|
5
6
|
"license": "MIT",
|
|
6
7
|
"type": "module",
|
|
7
8
|
"main": "./dist/index.cjs",
|
|
@@ -24,13 +25,13 @@
|
|
|
24
25
|
],
|
|
25
26
|
"peerDependencies": {
|
|
26
27
|
"jest": ">=29.0.0",
|
|
27
|
-
"@assay-ai/core": "
|
|
28
|
+
"@assay-ai/core": "1.3.1-beta"
|
|
28
29
|
},
|
|
29
30
|
"devDependencies": {
|
|
30
31
|
"@types/jest": "^29.0.0",
|
|
31
32
|
"tsup": "^8.3.0",
|
|
32
33
|
"typescript": "^5.7.0",
|
|
33
|
-
"@assay-ai/core": "
|
|
34
|
+
"@assay-ai/core": "1.3.1-beta",
|
|
34
35
|
"@assay-ai/tsconfig": "0.0.0"
|
|
35
36
|
},
|
|
36
37
|
"repository": {
|