@cogitator-ai/evals 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/assertions/custom.d.ts.map +1 -1
- package/dist/assertions/custom.js +11 -1
- package/dist/assertions/custom.js.map +1 -1
- package/dist/assertions/regression.d.ts.map +1 -1
- package/dist/assertions/regression.js +14 -1
- package/dist/assertions/regression.js.map +1 -1
- package/dist/datasets/csv-loader.js +2 -2
- package/dist/datasets/csv-loader.js.map +1 -1
- package/dist/datasets/dataset.js.map +1 -1
- package/dist/eval-builder.d.ts.map +1 -1
- package/dist/eval-builder.js +1 -4
- package/dist/eval-builder.js.map +1 -1
- package/dist/eval-suite.d.ts +2 -0
- package/dist/eval-suite.d.ts.map +1 -1
- package/dist/eval-suite.js +4 -3
- package/dist/eval-suite.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metrics/llm-judge.js +1 -1
- package/dist/metrics/llm-judge.js.map +1 -1
- package/dist/metrics/statistical.d.ts.map +1 -1
- package/dist/metrics/statistical.js +6 -6
- package/dist/metrics/statistical.js.map +1 -1
- package/dist/stats/percentiles.js +4 -4
- package/dist/stats/percentiles.js.map +1 -1
- package/package.json +1 -3
package/README.md
CHANGED
|
@@ -397,6 +397,7 @@ result.report('ci');
|
|
|
397
397
|
| `Dataset` | Immutable dataset with from/fromJsonl/fromCsv/filter/sample/shuffle |
|
|
398
398
|
| `loadJsonl` | Low-level JSONL file loader |
|
|
399
399
|
| `loadCsv` | Low-level CSV file loader |
|
|
400
|
+
| `isLLMMetric` | Type guard: checks if a `MetricFn` is an `LLMMetricFn` |
|
|
400
401
|
|
|
401
402
|
### Metrics
|
|
402
403
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7D,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,CACL,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KACrD,OAAO,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,WAAW,
|
|
1
|
+
{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7D,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,CACL,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KACrD,OAAO,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,WAAW,CAqBd"}
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
export function assertion(opts) {
|
|
2
2
|
return (aggregated, stats) => {
|
|
3
|
-
|
|
3
|
+
let passed;
|
|
4
|
+
try {
|
|
5
|
+
passed = opts.check(aggregated, stats);
|
|
6
|
+
}
|
|
7
|
+
catch (err) {
|
|
8
|
+
return {
|
|
9
|
+
name: opts.name,
|
|
10
|
+
passed: false,
|
|
11
|
+
message: `Custom assertion '${opts.name}' threw: ${err.message}`,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
return {
|
|
5
15
|
name: opts.name,
|
|
6
16
|
passed,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,SAAS,CAAC,IAOzB;IACC,OAAO,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QAC3B,
|
|
1
|
+
{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,SAAS,CAAC,IAOzB;IACC,OAAO,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QAC3B,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,qBAAqB,IAAI,CAAC,IAAI,YAAa,GAAa,CAAC,OAAO,EAAE;aAC5E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM;YACN,OAAO,EAAE,MAAM;gBACb,CAAC,CAAC,qBAAqB,IAAI,CAAC,IAAI,UAAU;gBAC1C,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,qBAAqB,IAAI,CAAC,IAAI,UAAU,CAAC;SAC/D,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAY3C,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAgE7F"}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
function isLowerBetter(name) {
|
|
3
|
-
|
|
3
|
+
const base = name.split('.')[0];
|
|
4
|
+
return (base.startsWith('latency') ||
|
|
5
|
+
base.startsWith('cost') ||
|
|
6
|
+
base.endsWith('Duration') ||
|
|
7
|
+
base.endsWith('Latency'));
|
|
4
8
|
}
|
|
5
9
|
export function noRegression(baselinePath, opts) {
|
|
6
10
|
return (aggregated, _stats) => {
|
|
@@ -17,10 +21,12 @@ export function noRegression(baselinePath, opts) {
|
|
|
17
21
|
};
|
|
18
22
|
}
|
|
19
23
|
const tolerance = opts?.tolerance ?? 0.05;
|
|
24
|
+
let validated = 0;
|
|
20
25
|
for (const [metric, baselineValue] of Object.entries(baseline)) {
|
|
21
26
|
const agg = aggregated[metric];
|
|
22
27
|
if (!agg)
|
|
23
28
|
continue;
|
|
29
|
+
validated++;
|
|
24
30
|
const actual = agg.mean;
|
|
25
31
|
const lowerBetter = isLowerBetter(metric);
|
|
26
32
|
if (lowerBetter) {
|
|
@@ -48,6 +54,13 @@ export function noRegression(baselinePath, opts) {
|
|
|
48
54
|
}
|
|
49
55
|
}
|
|
50
56
|
}
|
|
57
|
+
if (validated === 0) {
|
|
58
|
+
return {
|
|
59
|
+
name: 'noRegression',
|
|
60
|
+
passed: false,
|
|
61
|
+
message: 'No baseline metrics found in current results — cannot validate regression',
|
|
62
|
+
};
|
|
63
|
+
}
|
|
51
64
|
return {
|
|
52
65
|
name: 'noRegression',
|
|
53
66
|
passed: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAG9B,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAG9B,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,OAAO,CACL,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,IAA6B;IAC9E,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,IAAI,QAAgC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,iCAAiC,YAAY,EAAE;aACzD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC;QAC1C,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,SAAS,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC;YACxB,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YAE1C,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,2EAA2E;aACrF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,0CAA0C;SACpD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { EvalCaseSchema } from '../schema';
|
|
3
3
|
export async function loadCsv(path) {
|
|
4
4
|
let Papa;
|
|
@@ -8,7 +8,7 @@ export async function loadCsv(path) {
|
|
|
8
8
|
catch {
|
|
9
9
|
throw new Error('papaparse is required for CSV loading. Install it with: npm install papaparse');
|
|
10
10
|
}
|
|
11
|
-
const content =
|
|
11
|
+
const content = await readFile(path, 'utf-8');
|
|
12
12
|
const { data, meta } = Papa.parse(content, {
|
|
13
13
|
header: true,
|
|
14
14
|
skipEmptyLines: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,IAAI,IAAgC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAyB,OAAO,EAAE;QACjE,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAE/D,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YACtD,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QACnC,CAAC;QAED,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,MAAM,OAAO,GAA4B,EAAE,CAAC;QAE5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU;gBAAE,SAAS;YAEpD,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAClD,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAChD,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;QAEhE,OAAO,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,CAAS;QACd,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAe,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,
|
|
1
|
+
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAc;IAC7B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAKpC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA8BnB"}
|
package/dist/eval-builder.js
CHANGED
package/dist/eval-builder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAe;IACrB,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
package/dist/eval-suite.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { Dataset } from './datasets';
|
|
|
2
2
|
import type { EvalCase } from './schema';
|
|
3
3
|
import type { JudgeConfig } from './schema';
|
|
4
4
|
import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types';
|
|
5
|
+
import type { LLMMetricFn } from './metrics/llm-judge';
|
|
5
6
|
import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions';
|
|
6
7
|
import type { ReporterType, ReporterOptions } from './reporters';
|
|
7
8
|
export interface EvalTarget {
|
|
@@ -40,6 +41,7 @@ export interface EvalSuiteResult {
|
|
|
40
41
|
report: (type: ReporterType | ReporterType[], options?: ReporterOptions) => void;
|
|
41
42
|
saveBaseline: (path: string) => void;
|
|
42
43
|
}
|
|
44
|
+
export declare function isLLMMetric(m: MetricFn): m is LLMMetricFn;
|
|
43
45
|
export declare class EvalSuite {
|
|
44
46
|
private readonly dataset;
|
|
45
47
|
private readonly target;
|
package/dist/eval-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGnF,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IAoClC,OAAO,CAAC,cAAc;IAuBhB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;YA6FvB,WAAW;YAiBX,kBAAkB;YAyBlB,eAAe;YAMf,kBAAkB;YA+BlB,mBAAmB;IAKjC,OAAO,CAAC,eAAe;CAuBxB"}
|
package/dist/eval-suite.js
CHANGED
|
@@ -3,7 +3,7 @@ import { EvalSuiteConfigSchema } from './schema';
|
|
|
3
3
|
import { bindJudgeContext } from './metrics/llm-judge';
|
|
4
4
|
import { aggregate } from './stats';
|
|
5
5
|
import { report } from './reporters';
|
|
6
|
-
function isLLMMetric(m) {
|
|
6
|
+
export function isLLMMetric(m) {
|
|
7
7
|
return 'requiresJudge' in m && m.requiresJudge === true;
|
|
8
8
|
}
|
|
9
9
|
export class EvalSuite {
|
|
@@ -79,7 +79,7 @@ export class EvalSuite {
|
|
|
79
79
|
const i = nextIdx++;
|
|
80
80
|
const evalCase = cases[i];
|
|
81
81
|
active++;
|
|
82
|
-
this.executeCase(evalCase)
|
|
82
|
+
void this.executeCase(evalCase)
|
|
83
83
|
.then((caseResult) => this.evaluateCaseMetrics(caseResult).then((scores) => ({ ...caseResult, scores })))
|
|
84
84
|
.then((scored) => {
|
|
85
85
|
indexed.push({ idx: i, result: scored });
|
|
@@ -140,6 +140,7 @@ export class EvalSuite {
|
|
|
140
140
|
return suiteResult;
|
|
141
141
|
}
|
|
142
142
|
async executeCase(evalCase) {
|
|
143
|
+
const start = Date.now();
|
|
143
144
|
for (let attempt = 0; attempt <= this.retries; attempt++) {
|
|
144
145
|
try {
|
|
145
146
|
return await this.executeCaseAttempt(evalCase);
|
|
@@ -152,7 +153,7 @@ export class EvalSuite {
|
|
|
152
153
|
return {
|
|
153
154
|
case: evalCase,
|
|
154
155
|
output: '',
|
|
155
|
-
duration:
|
|
156
|
+
duration: Date.now() - start,
|
|
156
157
|
};
|
|
157
158
|
}
|
|
158
159
|
async executeCaseAttempt(evalCase) {
|
package/dist/eval-suite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAqCrC,
|
|
1
|
+
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAqCrC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACvC,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACjC,OAAO,gBAAgB,CAAC,CAAC,EAAE;oBACzB,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAqB,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;oBAC/E,WAAW,EAAE,IAAI,CAAC,KAAK;iBACxB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,KAAK,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC7E,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO;oBAAE,SAAS;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAAa,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;gBAC7C,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAC7B,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const VERSION = "0.1.0";
|
|
2
|
-
export { EvalSuite } from './eval-suite';
|
|
2
|
+
export { EvalSuite, isLLMMetric } from './eval-suite';
|
|
3
3
|
export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite';
|
|
4
4
|
export { EvalComparison } from './eval-comparison';
|
|
5
5
|
export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAE7F,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const VERSION = '0.1.0';
|
|
2
|
-
export { EvalSuite } from './eval-suite';
|
|
2
|
+
export { EvalSuite, isLLMMetric } from './eval-suite';
|
|
3
3
|
export { EvalComparison } from './eval-comparison';
|
|
4
4
|
export { EvalBuilder } from './eval-builder';
|
|
5
5
|
export { Dataset } from './datasets';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,GAAG,8BAA8B,CAAC;AAEnD,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,
|
|
1
|
+
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,GAAG,8BAA8B,CAAC;AAEnD,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;aACrE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAiB,GAAa,CAAC,OAAO,EAAE;aAClD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,iBAAiB,CACtB,cAAc,EACd,sUAAsU,CACvU,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,iBAAiB,CACtB,aAAa,EACb,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAYhF,wBAAgB,OAAO,IAAI,mBAAmB,CAmB7C;AAED,wBAAgB,IAAI,IAAI,mBAAmB,CA8B1C;AAED,wBAAgB,UAAU,IAAI,mBAAmB,
|
|
1
|
+
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAYhF,wBAAgB,OAAO,IAAI,mBAAmB,CAmB7C;AAED,wBAAgB,IAAI,IAAI,mBAAmB,CA8B1C;AAED,wBAAgB,UAAU,IAAI,mBAAmB,CAgChD"}
|
|
@@ -35,8 +35,8 @@ export function cost() {
|
|
|
35
35
|
}
|
|
36
36
|
const stats = aggregate(costs);
|
|
37
37
|
let total = 0;
|
|
38
|
-
for (
|
|
39
|
-
total +=
|
|
38
|
+
for (const c of costs) {
|
|
39
|
+
total += c;
|
|
40
40
|
}
|
|
41
41
|
return {
|
|
42
42
|
name: 'cost',
|
|
@@ -65,10 +65,10 @@ export function tokenUsage() {
|
|
|
65
65
|
const outputTokens = withUsage.map((r) => r.usage.outputTokens);
|
|
66
66
|
let totalInput = 0;
|
|
67
67
|
let totalOutput = 0;
|
|
68
|
-
for (
|
|
69
|
-
totalInput +=
|
|
70
|
-
|
|
71
|
-
|
|
68
|
+
for (const t of inputTokens)
|
|
69
|
+
totalInput += t;
|
|
70
|
+
for (const t of outputTokens)
|
|
71
|
+
totalOutput += t;
|
|
72
72
|
return {
|
|
73
73
|
name: 'tokenUsage',
|
|
74
74
|
score: 0,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAA8C;IAE9C,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QAEvE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;aAC3D,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,
|
|
1
|
+
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAA8C;IAE9C,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QAEvE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;aAC3D,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,KAAK;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEjD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;aACzF,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAEjE,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,WAAW;YAAE,UAAU,IAAI,CAAC,CAAC;QAC7C,KAAK,MAAM,CAAC,IAAI,YAAY;YAAE,WAAW,IAAI,CAAC,CAAC;QAE/C,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -2,8 +2,8 @@ export function mean(values) {
|
|
|
2
2
|
if (values.length === 0)
|
|
3
3
|
return 0;
|
|
4
4
|
let sum = 0;
|
|
5
|
-
for (
|
|
6
|
-
sum +=
|
|
5
|
+
for (const v of values) {
|
|
6
|
+
sum += v;
|
|
7
7
|
}
|
|
8
8
|
return sum / values.length;
|
|
9
9
|
}
|
|
@@ -12,8 +12,8 @@ export function stdDev(values) {
|
|
|
12
12
|
return 0;
|
|
13
13
|
const m = mean(values);
|
|
14
14
|
let sumSq = 0;
|
|
15
|
-
for (
|
|
16
|
-
const d =
|
|
15
|
+
for (const v of values) {
|
|
16
|
+
const d = v - m;
|
|
17
17
|
sumSq += d * d;
|
|
18
18
|
}
|
|
19
19
|
return Math.sqrt(sumSq / (values.length - 1));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"percentiles.js","sourceRoot":"","sources":["../../src/stats/percentiles.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,IAAI,CAAC,MAAgB;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,
|
|
1
|
+
{"version":3,"file":"percentiles.js","sourceRoot":"","sources":["../../src/stats/percentiles.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,IAAI,CAAC,MAAgB;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,GAAG,IAAI,CAAC,CAAC;IACX,CAAC;IACD,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAgB,EAAE,CAAS;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE9B,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,IAAI,GAAG,KAAK,CAAC;IAC9B,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAgB;IAUxC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACnF,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;QAClB,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;QAC/B,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;QACtB,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5B,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC;QAC7B,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC;KAC9B,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/evals",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Evaluation framework for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -23,8 +23,6 @@
|
|
|
23
23
|
"test:watch": "vitest"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"@cogitator-ai/types": "workspace:*",
|
|
27
|
-
"nanoid": "^5.0.4",
|
|
28
26
|
"zod": "^4.3.6"
|
|
29
27
|
},
|
|
30
28
|
"peerDependencies": {
|