@assay-ai/jest 0.1.0-beta → 0.2.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -0
- package/package.json +3 -3
package/README.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# @assay-ai/jest
|
|
2
|
+
|
|
3
|
+
Jest integration for [Assay](https://github.com/assay-ai/assay) -- the TypeScript-native LLM evaluation framework.
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/@assay-ai/jest)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
npm install -D @assay-ai/core @assay-ai/jest
|
|
12
|
+
# or
|
|
13
|
+
pnpm add -D @assay-ai/core @assay-ai/jest
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Quick Start
|
|
17
|
+
|
|
18
|
+
### Setup
|
|
19
|
+
|
|
20
|
+
Register the Assay matchers in your Jest setup file:
|
|
21
|
+
|
|
22
|
+
```typescript
|
|
23
|
+
// jest.setup.ts
|
|
24
|
+
import { setupAssayMatchers } from "@assay-ai/jest";
|
|
25
|
+
|
|
26
|
+
setupAssayMatchers();
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Then configure Jest to use it:
|
|
30
|
+
|
|
31
|
+
```json
|
|
32
|
+
// jest.config.json
|
|
33
|
+
{
|
|
34
|
+
"setupFilesAfterSetup": ["./jest.setup.ts"]
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### Writing Evaluations
|
|
39
|
+
|
|
40
|
+
```typescript
|
|
41
|
+
// chatbot.eval.ts
|
|
42
|
+
import { GEval } from "@assay-ai/core";
|
|
43
|
+
|
|
44
|
+
describe("Customer Support Chatbot", () => {
|
|
45
|
+
it("should answer product questions accurately", async () => {
|
|
46
|
+
await expect({
|
|
47
|
+
input: "What is your return policy?",
|
|
48
|
+
actualOutput: "You can return items within 30 days of purchase.",
|
|
49
|
+
retrievalContext: [
|
|
50
|
+
"Our return policy allows returns within 30 days of purchase.",
|
|
51
|
+
],
|
|
52
|
+
}).toBeRelevant({ threshold: 0.8 });
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("should be faithful to context", async () => {
|
|
56
|
+
await expect({
|
|
57
|
+
input: "What is your return policy?",
|
|
58
|
+
actualOutput: "You can return items within 30 days of purchase.",
|
|
59
|
+
retrievalContext: [
|
|
60
|
+
"Our return policy allows returns within 30 days of purchase.",
|
|
61
|
+
],
|
|
62
|
+
}).toBeFaithful();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("should pass custom metric", async () => {
|
|
66
|
+
const politeness = new GEval({
|
|
67
|
+
name: "Politeness",
|
|
68
|
+
criteria: "The response should be polite and professional.",
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
await expect({
|
|
72
|
+
input: "Help me with my order",
|
|
73
|
+
actualOutput: "I'd be happy to help! Could you share your order number?",
|
|
74
|
+
}).toPassMetric(politeness);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Available Matchers
|
|
80
|
+
|
|
81
|
+
| Matcher | Description |
|
|
82
|
+
|---------|-------------|
|
|
83
|
+
| `toBeRelevant(options?)` | Asserts the output is relevant to the input (Answer Relevancy) |
|
|
84
|
+
| `toBeFaithful(options?)` | Asserts the output is grounded in context (Faithfulness) |
|
|
85
|
+
| `toNotHallucinate(options?)` | Asserts the output doesn't contain hallucinations |
|
|
86
|
+
| `toPassMetric(metric)` | Asserts the test case passes a specific metric |
|
|
87
|
+
| `toPassAllMetrics(metrics)` | Asserts the test case passes all given metrics |
|
|
88
|
+
|
|
89
|
+
All matchers accept an optional `{ threshold?: number }` options object.
|
|
90
|
+
|
|
91
|
+
## Peer Dependencies
|
|
92
|
+
|
|
93
|
+
- `@assay-ai/core` >= 0.1.0
|
|
94
|
+
- `jest` >= 29.0.0
|
|
95
|
+
|
|
96
|
+
## License
|
|
97
|
+
|
|
98
|
+
[MIT](https://github.com/assay-ai/assay/blob/main/LICENSE)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@assay-ai/jest",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0-beta",
|
|
4
4
|
"description": "Jest integration for the Assay LLM evaluation framework",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -24,13 +24,13 @@
|
|
|
24
24
|
],
|
|
25
25
|
"peerDependencies": {
|
|
26
26
|
"jest": ">=29.0.0",
|
|
27
|
-
"@assay-ai/core": "0.
|
|
27
|
+
"@assay-ai/core": "0.2.0-beta"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
30
|
"@types/jest": "^29.0.0",
|
|
31
31
|
"tsup": "^8.3.0",
|
|
32
32
|
"typescript": "^5.7.0",
|
|
33
|
-
"@assay-ai/core": "0.
|
|
33
|
+
"@assay-ai/core": "0.2.0-beta",
|
|
34
34
|
"@assay-ai/tsconfig": "0.0.0"
|
|
35
35
|
},
|
|
36
36
|
"repository": {
|