@rulvar/evals 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/index.js +6 -5
- package/package.json +3 -3
package/README.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# @rulvar/evals
|
|
2
|
+
|
|
3
|
+
The rulvar eval framework: eval cases with golden outputs, rubric and
|
|
4
|
+
judge graders that run through the engine itself, matrix sweeps across
|
|
5
|
+
models and configurations, and the canary fingerprint. Exports
|
|
6
|
+
`runEvalSuite`, `runEvalMatrix`, `goldenGrader`, `rubricGrader`,
|
|
7
|
+
`judgeGrader`, and `canaryFingerprint`.
|
|
8
|
+
|
|
9
|
+
Part of [rulvar](https://rulvar.com), an embeddable TypeScript engine
|
|
10
|
+
for durable, budget-bounded multi-agent LLM workflows, where a completed
|
|
11
|
+
LLM call is never paid for twice. Full documentation:
|
|
12
|
+
[docs.rulvar.com](https://docs.rulvar.com).
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pnpm add -D @rulvar/evals
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Documentation
|
|
21
|
+
|
|
22
|
+
- [Evals](https://docs.rulvar.com/guide/evals)
|
|
23
|
+
- [API reference](https://docs.rulvar.com/api/%40rulvar/evals/)
|
|
24
|
+
|
|
25
|
+
## License
|
|
26
|
+
|
|
27
|
+
[Apache-2.0](https://github.com/o-stepper/rulvar/blob/main/LICENSE)
|
package/dist/index.js
CHANGED
|
@@ -319,11 +319,12 @@ async function commitEvalMeasured(store, claims, options) {
|
|
|
319
319
|
//#region src/canary.ts
|
|
320
320
|
/**
|
|
321
321
|
* The canary fingerprint (M11-T04; OQ-06). The optional compensation for silent alias
|
|
322
|
-
* re-pointing that modelEpoch honestly cannot catch: a FIXED probe set
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
322
|
+
* re-pointing that modelEpoch honestly cannot catch: a FIXED probe set,
|
|
323
|
+
* run through the ordinary engine (journaled, budgeted,
|
|
324
|
+
* VCR-recordable), hashed over normalized outputs. Sampling parameters
|
|
325
|
+
* are not pinned; drift detection rests on the fixed prompts, the
|
|
326
|
+
* normalization, and exact fingerprint comparison. A fingerprint change
|
|
327
|
+
* flips the model's eval claims to stale in one command.
|
|
327
328
|
*
|
|
328
329
|
* The committed v1 design (closing OQ-06): the probe set is CALLER
|
|
329
330
|
* data (fixed, versioned alongside the store); normalization is NFC,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulvar/evals",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
"access": "public"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@rulvar/core": "1.
|
|
26
|
-
"@rulvar/testing": "1.
|
|
25
|
+
"@rulvar/core": "1.4.0",
|
|
26
|
+
"@rulvar/testing": "1.4.0"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
29
|
"@types/node": "^22.20.0",
|