@cogitator-ai/evals 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -10
- package/dist/assertions/direction.d.ts +6 -0
- package/dist/assertions/direction.d.ts.map +1 -0
- package/dist/assertions/direction.js +12 -0
- package/dist/assertions/direction.js.map +1 -0
- package/dist/assertions/index.d.ts +2 -0
- package/dist/assertions/index.d.ts.map +1 -1
- package/dist/assertions/index.js.map +1 -1
- package/dist/assertions/regression.d.ts.map +1 -1
- package/dist/assertions/regression.js +1 -7
- package/dist/assertions/regression.js.map +1 -1
- package/dist/assertions/threshold.d.ts.map +1 -1
- package/dist/assertions/threshold.js +1 -7
- package/dist/assertions/threshold.js.map +1 -1
- package/dist/eval-builder.d.ts +2 -2
- package/dist/eval-builder.d.ts.map +1 -1
- package/dist/eval-builder.js.map +1 -1
- package/dist/eval-comparison.d.ts +2 -2
- package/dist/eval-comparison.d.ts.map +1 -1
- package/dist/eval-suite.d.ts +7 -3
- package/dist/eval-suite.d.ts.map +1 -1
- package/dist/eval-suite.js +20 -14
- package/dist/eval-suite.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metrics/index.d.ts +1 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/llm-judge.d.ts +6 -1
- package/dist/metrics/llm-judge.d.ts.map +1 -1
- package/dist/metrics/llm-judge.js +53 -11
- package/dist/metrics/llm-judge.js.map +1 -1
- package/dist/metrics/statistical.d.ts +3 -0
- package/dist/metrics/statistical.d.ts.map +1 -1
- package/dist/metrics/statistical.js +21 -28
- package/dist/metrics/statistical.js.map +1 -1
- package/dist/metrics/types.d.ts +8 -1
- package/dist/metrics/types.d.ts.map +1 -1
- package/dist/reporters/index.d.ts +12 -0
- package/dist/reporters/index.d.ts.map +1 -1
- package/dist/reporters/index.js +20 -3
- package/dist/reporters/index.js.map +1 -1
- package/dist/schema.d.ts +12 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +7 -0
- package/dist/schema.js.map +1 -1
- package/dist/stats/mcnemar.js +1 -1
- package/dist/stats/mcnemar.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +19 -34
- package/dist/tools.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -9,8 +9,11 @@ pnpm add @cogitator-ai/evals
|
|
|
9
9
|
|
|
10
10
|
# Optional dependencies
|
|
11
11
|
pnpm add papaparse # CSV dataset loading
|
|
12
|
+
pnpm add @cogitator-ai/core # agent targets and LLM-as-judge metrics
|
|
12
13
|
```
|
|
13
14
|
|
|
15
|
+
Website docs: [Evals](https://cogitator.app/docs/evals), [Datasets](https://cogitator.app/docs/evals/datasets), [Metrics](https://cogitator.app/docs/evals/metrics), [Assertions](https://cogitator.app/docs/evals/assertions), [Comparison](https://cogitator.app/docs/evals/comparison), [Reporters](https://cogitator.app/docs/evals/reporters).
|
|
16
|
+
|
|
14
17
|
## Features
|
|
15
18
|
|
|
16
19
|
- **EvalSuite** — Run datasets against agents or plain functions with configurable concurrency, timeouts, and retries
|
|
@@ -59,7 +62,11 @@ result.report('console');
|
|
|
59
62
|
result.saveBaseline('./baseline.json');
|
|
60
63
|
```
|
|
61
64
|
|
|
62
|
-
A
|
|
65
|
+
A target is either `{ fn: (input) => Promise<string> }` or `{ agent, cogitator }`, which runs the agent with `cogitator.run(agent, { input })` and records its usage for the statistical metrics. Defaults: `concurrency` 5, `timeout` 30 000 ms (minimum 1 000), `retries` 0.
|
|
66
|
+
|
|
67
|
+
A case that throws or exceeds `timeout` is retried up to `retries` times. If every attempt fails, its result has an empty `output` and an `error` message describing the last failure.
|
|
68
|
+
|
|
69
|
+
`suite.run(options?: EvalRunOptions)` takes `{ maxCases?: number }`: with `maxCases` only the first N cases of the dataset run, and the results, aggregates, assertions and `stats.total` cover just those cases.
|
|
63
70
|
|
|
64
71
|
---
|
|
65
72
|
|
|
@@ -140,7 +147,7 @@ const metrics = [
|
|
|
140
147
|
|
|
141
148
|
### LLM-as-Judge
|
|
142
149
|
|
|
143
|
-
Metrics scored by an LLM judge (0.0 to 1.0).
|
|
150
|
+
Metrics scored by an LLM judge (0.0 to 1.0). They require a `judge` config on the suite and a Cogitator to run the judge agent: `judge.cogitator`, or the `cogitator` of an agent target. Without one, the suite throws when it is created.
|
|
144
151
|
|
|
145
152
|
| Metric | Evaluates |
|
|
146
153
|
| -------------- | --------------------------------------- |
|
|
@@ -164,10 +171,12 @@ const suite = new EvalSuite({
|
|
|
164
171
|
prompt: 'Rate how technically accurate the response is for a software engineering audience.',
|
|
165
172
|
}),
|
|
166
173
|
],
|
|
167
|
-
judge: { model: 'gpt-6.1-sol', temperature: 0 },
|
|
174
|
+
judge: { model: 'openai/gpt-6.1-sol', temperature: 0, cogitator: cog },
|
|
168
175
|
});
|
|
169
176
|
```
|
|
170
177
|
|
|
178
|
+
The `judge` option is a `JudgeConfigInput`: `{ model, temperature?, maxTokens?, cogitator? }`, with `temperature` defaulting to `0`.
|
|
179
|
+
|
|
171
180
|
### Statistical
|
|
172
181
|
|
|
173
182
|
Aggregate metrics computed across all results. These report percentile breakdowns (p50, p95, p99) rather than per-case scores.
|
|
@@ -177,7 +186,7 @@ import { latency, cost, tokenUsage } from '@cogitator-ai/evals';
|
|
|
177
186
|
|
|
178
187
|
const suite = new EvalSuite({
|
|
179
188
|
dataset,
|
|
180
|
-
target: { agent, cogitator },
|
|
189
|
+
target: { agent, cogitator: cog },
|
|
181
190
|
metrics: [exactMatch()],
|
|
182
191
|
statisticalMetrics: [latency(), cost(), tokenUsage()],
|
|
183
192
|
});
|
|
@@ -227,7 +236,7 @@ const assertions = [
|
|
|
227
236
|
];
|
|
228
237
|
```
|
|
229
238
|
|
|
230
|
-
Latency and
|
|
239
|
+
Latency, cost and tokenUsage metrics (and names ending in `Duration` or `Latency`) are lower-is-better: the assertion passes when the value is at or below the threshold. `noRegression` uses the same rule.
|
|
231
240
|
|
|
232
241
|
### noRegression
|
|
233
242
|
|
|
@@ -335,7 +344,7 @@ const suite = new EvalBuilder()
|
|
|
335
344
|
.withTarget({ fn: async (input) => myModel(input) })
|
|
336
345
|
.withMetrics([exactMatch(), contains(), faithfulness()])
|
|
337
346
|
.withStatisticalMetrics([latency()])
|
|
338
|
-
.withJudge({ model: 'gpt-6.1-sol', temperature: 0 })
|
|
347
|
+
.withJudge({ model: 'openai/gpt-6.1-sol', temperature: 0, cogitator: cog })
|
|
339
348
|
.withAssertions([threshold('exactMatch', 0.85), noRegression('./baseline.json')])
|
|
340
349
|
.withConcurrency(10)
|
|
341
350
|
.withTimeout(60_000)
|
|
@@ -447,10 +456,12 @@ result.report('ci');
|
|
|
447
456
|
|
|
448
457
|
### Agent Tools
|
|
449
458
|
|
|
450
|
-
| Export | Description
|
|
451
|
-
| ------------------- |
|
|
452
|
-
| `createRunEvalTool` | Creates a `run_eval` tool for agents |
|
|
453
|
-
| `evalTools` | Returns all eval tools as an array
|
|
459
|
+
| Export | Description |
|
|
460
|
+
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
461
|
+
| `createRunEvalTool` | Creates a `run_eval` tool for agents for a given suite; its optional `maxCases` runs `suite.run({ maxCases })` and the summary (`total`, `metrics`, `assertionsPassed`) covers those cases |
|
|
462
|
+
| `evalTools` | Returns all eval tools for a suite as an array |
|
|
463
|
+
|
|
464
|
+
Like the RAG tools, eval tools carry a Zod `parameters` schema and can be passed to `tool()` from `@cogitator-ai/core`.
|
|
454
465
|
|
|
455
466
|
---
|
|
456
467
|
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether smaller values of a metric are better, as for the built-in `latency`, `cost` and
|
|
3
|
+
* `tokenUsage` statistical metrics. A field path such as `latency.p95` follows its metric.
|
|
4
|
+
*/
|
|
5
|
+
export declare function isLowerBetter(metricPath: string): boolean;
|
|
6
|
+
//# sourceMappingURL=direction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"direction.d.ts","sourceRoot":"","sources":["../../src/assertions/direction.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAMzD"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
const LOWER_IS_BETTER_PREFIXES = ['latency', 'cost', 'tokenUsage'];
|
|
2
|
+
const LOWER_IS_BETTER_SUFFIXES = ['Duration', 'Latency'];
|
|
3
|
+
/**
|
|
4
|
+
* Whether smaller values of a metric are better, as for the built-in `latency`, `cost` and
|
|
5
|
+
* `tokenUsage` statistical metrics. A field path such as `latency.p95` follows its metric.
|
|
6
|
+
*/
|
|
7
|
+
export function isLowerBetter(metricPath) {
|
|
8
|
+
const base = metricPath.split('.')[0];
|
|
9
|
+
return (LOWER_IS_BETTER_PREFIXES.some((prefix) => base.startsWith(prefix)) ||
|
|
10
|
+
LOWER_IS_BETTER_SUFFIXES.some((suffix) => base.endsWith(suffix)));
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=direction.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"direction.js","sourceRoot":"","sources":["../../src/assertions/direction.ts"],"names":[],"mappings":"AAAA,MAAM,wBAAwB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;AACnE,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;AAEzD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,UAAkB;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,OAAO,CACL,wBAAwB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAClE,wBAAwB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CACjE,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,KACrD,eAAe,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AA2BA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAG3C,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAgE7F"}
|
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
|
-
|
|
3
|
-
const base = name.split('.')[0];
|
|
4
|
-
return (base.startsWith('latency') ||
|
|
5
|
-
base.startsWith('cost') ||
|
|
6
|
-
base.endsWith('Duration') ||
|
|
7
|
-
base.endsWith('Latency'));
|
|
8
|
-
}
|
|
2
|
+
import { isLowerBetter } from './direction.js';
|
|
9
3
|
export function noRegression(baselinePath, opts) {
|
|
10
4
|
return (aggregated, _stats) => {
|
|
11
5
|
let baseline;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,IAA6B;IAC9E,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,IAAI,QAAgC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,iCAAiC,YAAY,EAAE;aACzD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC;QAC1C,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,SAAS,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC;YACxB,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YAE1C,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,2EAA2E;aACrF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,0CAA0C;SACpD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"threshold.d.ts","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"threshold.d.ts","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,SAAS,CAAC;AAoB7D,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CA0BxE"}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const base = name.split('.')[0];
|
|
3
|
-
return (base.startsWith('latency') ||
|
|
4
|
-
base.startsWith('cost') ||
|
|
5
|
-
base.endsWith('Duration') ||
|
|
6
|
-
base.endsWith('Latency'));
|
|
7
|
-
}
|
|
1
|
+
import { isLowerBetter } from './direction.js';
|
|
8
2
|
function resolve(aggregated, path) {
|
|
9
3
|
const parts = path.split('.');
|
|
10
4
|
const metric = aggregated[parts[0]];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"threshold.js","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"threshold.js","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,SAAS,OAAO,CACd,UAA4C,EAC5C,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAEnE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAA2B,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,UAAkB,EAAE,KAAa;IACzD,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAEjE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,IAAI,EAAE,aAAa,UAAU,GAAG;gBAChC,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,WAAW,UAAU,mCAAmC;aAClE,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAC/D,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,aAAa,UAAU,GAAG;YAChC,MAAM;YACN,OAAO,EAAE,MAAM;gBACb,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,IAAI,SAAS,IAAI,KAAK,EAAE;gBACnD,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,cAAc,SAAS,IAAI,KAAK,EAAE;YAC/D,MAAM;YACN,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/eval-builder.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { EvalSuite } from './eval-suite.js';
|
|
|
2
2
|
import type { EvalTarget, EvalSuiteOptions } from './eval-suite.js';
|
|
3
3
|
import { Dataset } from './datasets/index.js';
|
|
4
4
|
import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
|
|
5
|
-
import type {
|
|
5
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
6
6
|
import type { AssertionFn } from './assertions/index.js';
|
|
7
7
|
export declare class EvalBuilder {
|
|
8
8
|
private _dataset?;
|
|
@@ -19,7 +19,7 @@ export declare class EvalBuilder {
|
|
|
19
19
|
withTarget(target: EvalTarget): this;
|
|
20
20
|
withMetrics(metrics: MetricFn[]): this;
|
|
21
21
|
withStatisticalMetrics(metrics: StatisticalMetricFn[]): this;
|
|
22
|
-
withJudge(config:
|
|
22
|
+
withJudge(config: JudgeConfigInput): this;
|
|
23
23
|
withAssertions(assertions: AssertionFn[]): this;
|
|
24
24
|
withConcurrency(n: number): this;
|
|
25
25
|
withTimeout(ms: number): this;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAmB;IAClC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAKzC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA8BnB"}
|
package/dist/eval-builder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,
|
|
1
|
+
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAoB;IAC1B,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAwB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EvalTarget, EvalSuiteResult } from './eval-suite.js';
|
|
2
2
|
import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
4
4
|
import { Dataset } from './datasets/index.js';
|
|
5
5
|
export interface EvalComparisonOptions {
|
|
6
6
|
dataset: Dataset;
|
|
@@ -10,7 +10,7 @@ export interface EvalComparisonOptions {
|
|
|
10
10
|
};
|
|
11
11
|
metrics?: MetricFn[];
|
|
12
12
|
statisticalMetrics?: StatisticalMetricFn[];
|
|
13
|
-
judge?:
|
|
13
|
+
judge?: JudgeConfigInput;
|
|
14
14
|
concurrency?: number;
|
|
15
15
|
timeout?: number;
|
|
16
16
|
retries?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAkED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IAIjC,GAAG,IAAI,OAAO,CAAC,gBAAgB,CAAC;CAoEvC"}
|
package/dist/eval-suite.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Dataset } from './datasets/index.js';
|
|
2
2
|
import type { EvalCase } from './schema.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
4
4
|
import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
|
|
5
5
|
import type { LLMMetricFn } from './metrics/llm-judge.js';
|
|
6
6
|
import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions/index.js';
|
|
@@ -20,13 +20,17 @@ export interface EvalSuiteOptions {
|
|
|
20
20
|
target: EvalTarget;
|
|
21
21
|
metrics?: MetricFn[];
|
|
22
22
|
statisticalMetrics?: StatisticalMetricFn[];
|
|
23
|
-
judge?:
|
|
23
|
+
judge?: JudgeConfigInput;
|
|
24
24
|
assertions?: AssertionFn[];
|
|
25
25
|
concurrency?: number;
|
|
26
26
|
timeout?: number;
|
|
27
27
|
retries?: number;
|
|
28
28
|
onProgress?: (progress: EvalProgress) => void;
|
|
29
29
|
}
|
|
30
|
+
export interface EvalRunOptions {
|
|
31
|
+
/** Run only the first `maxCases` cases of the dataset (a positive integer) */
|
|
32
|
+
maxCases?: number;
|
|
33
|
+
}
|
|
30
34
|
export interface EvalSuiteResult {
|
|
31
35
|
results: Array<EvalCaseResult & {
|
|
32
36
|
scores: MetricScore[];
|
|
@@ -54,7 +58,7 @@ export declare class EvalSuite {
|
|
|
54
58
|
private readonly onProgress?;
|
|
55
59
|
constructor(opts: EvalSuiteOptions);
|
|
56
60
|
private validateTarget;
|
|
57
|
-
run(): Promise<EvalSuiteResult>;
|
|
61
|
+
run(options?: EvalRunOptions): Promise<EvalSuiteResult>;
|
|
58
62
|
private executeCase;
|
|
59
63
|
private executeCaseAttempt;
|
|
60
64
|
private executeFnTarget;
|
package/dist/eval-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGnF,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IAyClC,OAAO,CAAC,cAAc;IAuBhB,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;YAsGnD,WAAW;YAmBX,kBAAkB;YAoBlB,eAAe;YAMf,kBAAkB;YA+BlB,mBAAmB;IAKjC,OAAO,CAAC,eAAe;CAuBxB"}
|
package/dist/eval-suite.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
|
-
import { EvalSuiteConfigSchema } from './schema.js';
|
|
3
|
-
import { bindJudgeContext } from './metrics/llm-judge.js';
|
|
2
|
+
import { EvalSuiteConfigSchema, JudgeConfigSchema } from './schema.js';
|
|
3
|
+
import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
|
|
4
4
|
import { aggregate } from './stats/index.js';
|
|
5
5
|
import { report } from './reporters/index.js';
|
|
6
6
|
export function isLLMMetric(m) {
|
|
@@ -36,15 +36,13 @@ export class EvalSuite {
|
|
|
36
36
|
if (hasLLMMetrics && !opts.judge) {
|
|
37
37
|
throw new Error('LLM metrics require a judge config');
|
|
38
38
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return m;
|
|
47
|
-
});
|
|
39
|
+
const judgeConfig = opts.judge ? JudgeConfigSchema.parse(opts.judge) : undefined;
|
|
40
|
+
const judgeCogitator = judgeConfig?.cogitator ?? this.target.cogitator;
|
|
41
|
+
if (hasLLMMetrics && !judgeCogitator) {
|
|
42
|
+
throw new Error('LLM metrics need a Cogitator to run the judge: pass judge.cogitator, or use an agent target');
|
|
43
|
+
}
|
|
44
|
+
const judge = judgeConfig && judgeCogitator ? judgeContextFor(judgeCogitator, judgeConfig) : null;
|
|
45
|
+
this.boundMetrics = rawMetrics.map((m) => isLLMMetric(m) && judge ? bindJudgeContext(m, judge) : m);
|
|
48
46
|
}
|
|
49
47
|
validateTarget() {
|
|
50
48
|
const { fn, agent, cogitator } = this.target;
|
|
@@ -64,9 +62,13 @@ export class EvalSuite {
|
|
|
64
62
|
throw new Error('Cogitator target requires agent instance');
|
|
65
63
|
}
|
|
66
64
|
}
|
|
67
|
-
async run() {
|
|
65
|
+
async run(options = {}) {
|
|
66
|
+
const { maxCases } = options;
|
|
67
|
+
if (maxCases !== undefined && (!Number.isInteger(maxCases) || maxCases < 1)) {
|
|
68
|
+
throw new Error(`maxCases must be a positive integer, got ${maxCases}`);
|
|
69
|
+
}
|
|
68
70
|
const suiteStart = Date.now();
|
|
69
|
-
const cases =
|
|
71
|
+
const cases = this.dataset.cases.slice(0, maxCases);
|
|
70
72
|
const total = cases.length;
|
|
71
73
|
let completed = 0;
|
|
72
74
|
const indexed = [];
|
|
@@ -105,7 +107,11 @@ export class EvalSuite {
|
|
|
105
107
|
const aggregated = this.aggregateScores(orderedResults);
|
|
106
108
|
for (const statMetric of this.statisticalMetrics) {
|
|
107
109
|
const score = statMetric(orderedResults);
|
|
108
|
-
aggregated[score.name] = {
|
|
110
|
+
aggregated[score.name] = {
|
|
111
|
+
name: score.name,
|
|
112
|
+
...aggregate(score.values ?? [score.score]),
|
|
113
|
+
...(score.metadata && { metadata: score.metadata }),
|
|
114
|
+
};
|
|
109
115
|
}
|
|
110
116
|
const totalCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
|
|
111
117
|
const suiteDuration = Date.now() - suiteStart;
|
package/dist/eval-suite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAIpE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0CrC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,KAAK,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;SAC1E,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,KAAgD,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,KAAK,GAAG,UAAU,CAChB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EAC5D,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare const VERSION = "0.1.0";
|
|
2
2
|
export { EvalSuite, isLLMMetric } from './eval-suite.js';
|
|
3
|
-
export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite.js';
|
|
3
|
+
export type { EvalTarget, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, } from './eval-suite.js';
|
|
4
4
|
export { EvalComparison } from './eval-comparison.js';
|
|
5
5
|
export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison.js';
|
|
6
6
|
export { EvalBuilder } from './eval-builder.js';
|
|
@@ -12,7 +12,7 @@ export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeCo
|
|
|
12
12
|
export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
|
|
13
13
|
export { latency, cost, tokenUsage } from './metrics/statistical.js';
|
|
14
14
|
export { metric } from './metrics/custom.js';
|
|
15
|
-
export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
|
|
15
|
+
export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
|
|
16
16
|
export type { CustomMetricConfig } from './metrics/custom.js';
|
|
17
17
|
export { threshold } from './assertions/index.js';
|
|
18
18
|
export { noRegression } from './assertions/index.js';
|
|
@@ -25,7 +25,7 @@ export type { TTestResult } from './stats/t-test.js';
|
|
|
25
25
|
export { mcnemarsTest } from './stats/mcnemar.js';
|
|
26
26
|
export type { McNemarResult } from './stats/mcnemar.js';
|
|
27
27
|
export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
|
|
28
|
-
export type { EvalCase, EvalSuiteConfig, JudgeConfig, EvalComparisonConfig } from './schema.js';
|
|
28
|
+
export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, EvalComparisonConfig, } from './schema.js';
|
|
29
29
|
export { createRunEvalTool, evalTools } from './tools.js';
|
|
30
30
|
export type { EvalTool } from './tools.js';
|
|
31
31
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAU1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/metrics/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn } from './types.js';
|
|
1
|
+
export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
|
|
2
2
|
export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
|
|
3
3
|
export { metric } from './custom.js';
|
|
4
4
|
export type { CustomMetricConfig } from './custom.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { JudgeConfig } from '../schema.js';
|
|
1
|
+
import type { JudgeCogitator, JudgeConfig } from '../schema.js';
|
|
2
2
|
import type { MetricFn } from './types.js';
|
|
3
3
|
export interface JudgeContext {
|
|
4
4
|
cogitator: {
|
|
@@ -15,6 +15,11 @@ export interface LLMMetricFn extends MetricFn {
|
|
|
15
15
|
readonly __judgeSystemPrompt: string;
|
|
16
16
|
readonly __judgeName: string;
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
|
|
20
|
+
* asking for a `{ score, reasoning }` JSON answer.
|
|
21
|
+
*/
|
|
22
|
+
export declare function judgeContextFor(cogitator: JudgeCogitator, judgeConfig: JudgeConfig): JudgeContext;
|
|
18
23
|
export declare function bindJudgeContext(metric: LLMMetricFn, context: JudgeContext): MetricFn;
|
|
19
24
|
export declare function faithfulness(): LLMMetricFn;
|
|
20
25
|
export declare function relevance(): LLMMetricFn;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAA+B,MAAM,SAAS,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC7E,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAiCD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAiCjG;AAoBD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CAmCrF;AAED,wBAAgB,YAAY,IAAI,WAAW,CAK1C;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,WAAW,IAAI,WAAW,CAKzC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
|
|
@@ -1,20 +1,62 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
1
2
|
const SCORE_REGEX = /\b(0(?:\.\d+)?|1(?:\.0+)?)\b/;
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
}
|
|
3
|
+
const JSON_OBJECT = /\{[\s\S]*\}/;
|
|
4
|
+
const JudgeVerdict = z.object({ score: z.number(), reasoning: z.string() });
|
|
5
|
+
const JUDGE_INSTRUCTIONS = 'You are an impartial evaluator. Score the response with the rubric in the request. Reply with JSON: {"score": <number from 0 to 1>, "reasoning": "<explanation>"}.';
|
|
6
|
+
function verdictOf(value) {
|
|
7
|
+
if (typeof value !== 'object' || value === null)
|
|
8
8
|
return null;
|
|
9
|
+
const { score, reasoning } = value;
|
|
10
|
+
if (typeof score !== 'number' || !Number.isFinite(score))
|
|
11
|
+
return null;
|
|
12
|
+
return { score, ...(typeof reasoning === 'string' && { reasoning }) };
|
|
13
|
+
}
|
|
14
|
+
function parseJson(text) {
|
|
15
|
+
if (text === undefined)
|
|
16
|
+
return { ok: false };
|
|
17
|
+
try {
|
|
18
|
+
return { ok: true, value: JSON.parse(text) };
|
|
9
19
|
}
|
|
10
20
|
catch {
|
|
11
|
-
|
|
12
|
-
if (match) {
|
|
13
|
-
return { score: parseFloat(match[1]) };
|
|
14
|
-
}
|
|
15
|
-
return null;
|
|
21
|
+
return { ok: false };
|
|
16
22
|
}
|
|
17
23
|
}
|
|
24
|
+
function parseJudgeOutput(raw) {
|
|
25
|
+
const json = [raw, JSON_OBJECT.exec(raw)?.[0]].map(parseJson).find((result) => result.ok);
|
|
26
|
+
if (json?.ok)
|
|
27
|
+
return verdictOf(json.value);
|
|
28
|
+
const match = SCORE_REGEX.exec(raw);
|
|
29
|
+
return match ? { score: parseFloat(match[1]) } : null;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
|
|
33
|
+
* asking for a `{ score, reasoning }` JSON answer.
|
|
34
|
+
*/
|
|
35
|
+
export function judgeContextFor(cogitator, judgeConfig) {
|
|
36
|
+
let judgeAgent;
|
|
37
|
+
const agent = () => (judgeAgent ??= import('@cogitator-ai/core').then(({ Agent }) => new Agent({
|
|
38
|
+
name: 'eval-judge',
|
|
39
|
+
model: judgeConfig.model,
|
|
40
|
+
instructions: JUDGE_INSTRUCTIONS,
|
|
41
|
+
temperature: judgeConfig.temperature,
|
|
42
|
+
...(judgeConfig.maxTokens !== undefined && { maxTokens: judgeConfig.maxTokens }),
|
|
43
|
+
maxIterations: 1,
|
|
44
|
+
responseFormat: { type: 'json_schema', schema: JudgeVerdict },
|
|
45
|
+
}), (error) => {
|
|
46
|
+
throw new Error(`The LLM judge needs @cogitator-ai/core: ${error instanceof Error ? error.message : String(error)}`);
|
|
47
|
+
}));
|
|
48
|
+
return {
|
|
49
|
+
judgeConfig,
|
|
50
|
+
cogitator: {
|
|
51
|
+
run: async ({ input }) => {
|
|
52
|
+
const result = await cogitator.run(await agent(), { input, useMemory: false });
|
|
53
|
+
return {
|
|
54
|
+
output: result.structured !== undefined ? JSON.stringify(result.structured) : result.output,
|
|
55
|
+
};
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
18
60
|
function createJudgeMetric(name, systemPrompt) {
|
|
19
61
|
const fn = (async (_result) => {
|
|
20
62
|
return { name, score: 0, details: 'unbound judge metric — call bindJudgeContext first' };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,IAAwB;IACzC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,EAAE,EAAE;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,OAAO;QACL,WAAW;QACX,SAAS,EAAE;YACT,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/E,OAAO;oBACL,MAAM,EACJ,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;iBACtF,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;aACrE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAiB,GAAa,CAAC,OAAO,EAAE;aAClD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,iBAAiB,CACtB,cAAc,EACd,sUAAsU,CACvU,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,iBAAiB,CACtB,aAAa,EACb,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { StatisticalMetricFn } from './types.js';
|
|
2
|
+
/** Run duration per case, in milliseconds. */
|
|
2
3
|
export declare function latency(): StatisticalMetricFn;
|
|
4
|
+
/** Cost per case, in USD, for cases that report usage. */
|
|
3
5
|
export declare function cost(): StatisticalMetricFn;
|
|
6
|
+
/** Total tokens per case, for cases that report usage. */
|
|
4
7
|
export declare function tokenUsage(): StatisticalMetricFn;
|
|
5
8
|
//# sourceMappingURL=statistical.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBrF,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD"}
|
|
@@ -4,13 +4,21 @@ function createStatisticalFn(name, fn) {
|
|
|
4
4
|
statFn.metricName = name;
|
|
5
5
|
return statFn;
|
|
6
6
|
}
|
|
7
|
+
function sum(values) {
|
|
8
|
+
let total = 0;
|
|
9
|
+
for (const value of values)
|
|
10
|
+
total += value;
|
|
11
|
+
return total;
|
|
12
|
+
}
|
|
13
|
+
/** Run duration per case, in milliseconds. */
|
|
7
14
|
export function latency() {
|
|
8
15
|
return createStatisticalFn('latency', (results) => {
|
|
9
16
|
const durations = results.map((r) => r.duration);
|
|
10
17
|
const stats = aggregate(durations);
|
|
11
18
|
return {
|
|
12
19
|
name: 'latency',
|
|
13
|
-
score:
|
|
20
|
+
score: stats.mean,
|
|
21
|
+
values: durations,
|
|
14
22
|
metadata: {
|
|
15
23
|
p50: stats.p50,
|
|
16
24
|
p95: stats.p95,
|
|
@@ -23,55 +31,40 @@ export function latency() {
|
|
|
23
31
|
};
|
|
24
32
|
});
|
|
25
33
|
}
|
|
34
|
+
/** Cost per case, in USD, for cases that report usage. */
|
|
26
35
|
export function cost() {
|
|
27
36
|
return createStatisticalFn('cost', (results) => {
|
|
28
37
|
const costs = results.filter((r) => r.usage).map((r) => r.usage.cost);
|
|
29
|
-
if (costs.length === 0) {
|
|
30
|
-
return {
|
|
31
|
-
name: 'cost',
|
|
32
|
-
score: 0,
|
|
33
|
-
metadata: { total: 0, mean: 0, median: 0, min: 0, max: 0 },
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
38
|
const stats = aggregate(costs);
|
|
37
|
-
let total = 0;
|
|
38
|
-
for (const c of costs) {
|
|
39
|
-
total += c;
|
|
40
|
-
}
|
|
41
39
|
return {
|
|
42
40
|
name: 'cost',
|
|
43
|
-
score:
|
|
41
|
+
score: stats.mean,
|
|
42
|
+
values: costs,
|
|
44
43
|
metadata: {
|
|
45
|
-
total,
|
|
44
|
+
total: sum(costs),
|
|
46
45
|
mean: stats.mean,
|
|
47
46
|
median: stats.median,
|
|
48
47
|
min: stats.min,
|
|
49
48
|
max: stats.max,
|
|
49
|
+
p95: stats.p95,
|
|
50
|
+
p99: stats.p99,
|
|
50
51
|
},
|
|
51
52
|
};
|
|
52
53
|
});
|
|
53
54
|
}
|
|
55
|
+
/** Total tokens per case, for cases that report usage. */
|
|
54
56
|
export function tokenUsage() {
|
|
55
57
|
return createStatisticalFn('tokenUsage', (results) => {
|
|
56
58
|
const withUsage = results.filter((r) => r.usage);
|
|
57
|
-
if (withUsage.length === 0) {
|
|
58
|
-
return {
|
|
59
|
-
name: 'tokenUsage',
|
|
60
|
-
score: 0,
|
|
61
|
-
metadata: { totalInput: 0, totalOutput: 0, totalTokens: 0, meanInput: 0, meanOutput: 0 },
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
59
|
const inputTokens = withUsage.map((r) => r.usage.inputTokens);
|
|
65
60
|
const outputTokens = withUsage.map((r) => r.usage.outputTokens);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
totalInput += t;
|
|
70
|
-
for (const t of outputTokens)
|
|
71
|
-
totalOutput += t;
|
|
61
|
+
const totals = withUsage.map((r) => r.usage.inputTokens + r.usage.outputTokens);
|
|
62
|
+
const totalInput = sum(inputTokens);
|
|
63
|
+
const totalOutput = sum(outputTokens);
|
|
72
64
|
return {
|
|
73
65
|
name: 'tokenUsage',
|
|
74
|
-
score:
|
|
66
|
+
score: mean(totals),
|
|
67
|
+
values: totals,
|
|
75
68
|
metadata: {
|
|
76
69
|
totalInput,
|
|
77
70
|
totalOutput,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,
|
|
1
|
+
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/metrics/types.d.ts
CHANGED
|
@@ -27,7 +27,14 @@ export interface EvalCaseResult {
|
|
|
27
27
|
export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {
|
|
28
28
|
metricName: string;
|
|
29
29
|
};
|
|
30
|
-
export
|
|
30
|
+
export interface StatisticalScore extends MetricScore {
|
|
31
|
+
/**
|
|
32
|
+
* One value per case; the suite aggregates them into mean, percentiles and so
|
|
33
|
+
* on. Without it the suite aggregates `score` alone.
|
|
34
|
+
*/
|
|
35
|
+
values?: number[];
|
|
36
|
+
}
|
|
37
|
+
export type StatisticalMetricFn = ((results: EvalCaseResult[]) => StatisticalScore) & {
|
|
31
38
|
metricName: string;
|
|
32
39
|
};
|
|
33
40
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC"}
|
|
@@ -8,6 +8,8 @@ export interface AggregatedMetric {
|
|
|
8
8
|
p50: number;
|
|
9
9
|
p95: number;
|
|
10
10
|
p99: number;
|
|
11
|
+
/** Extra figures a statistical metric reports, such as the total cost */
|
|
12
|
+
metadata?: Record<string, unknown>;
|
|
11
13
|
}
|
|
12
14
|
export interface AssertionResult {
|
|
13
15
|
name: string;
|
|
@@ -40,8 +42,18 @@ export interface EvalSuiteResult {
|
|
|
40
42
|
}
|
|
41
43
|
export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
|
|
42
44
|
export type ReporterOptions = {
|
|
45
|
+
/**
|
|
46
|
+
* Base path of file reports. Each file reporter writes it with its own
|
|
47
|
+
* extension, so `./reports/eval` gives `eval.json` and `eval.csv`.
|
|
48
|
+
*/
|
|
43
49
|
path?: string;
|
|
44
50
|
};
|
|
51
|
+
/** The file a reporter writes: the base path with the reporter's extension. */
|
|
52
|
+
export declare function reportPath(base: string | undefined, extension: 'json' | 'csv'): string;
|
|
53
|
+
/**
|
|
54
|
+
* Runs the reporters in the order given, except `ci`, which runs last because
|
|
55
|
+
* it ends the process when an assertion failed.
|
|
56
|
+
*/
|
|
45
57
|
export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
|
|
46
58
|
export { consoleReport } from './console.js';
|
|
47
59
|
export { jsonReport } from './json.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClE,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAGtF;AAOD;;;GAGG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAoBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
|
package/dist/reporters/index.js
CHANGED
|
@@ -1,19 +1,36 @@
|
|
|
1
|
+
import { mkdirSync } from 'node:fs';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
1
3
|
import { consoleReport } from './console.js';
|
|
2
4
|
import { jsonReport } from './json.js';
|
|
3
5
|
import { csvReport } from './csv.js';
|
|
4
6
|
import { ciReport } from './ci.js';
|
|
7
|
+
const DEFAULT_BASE = 'eval-report';
|
|
8
|
+
/** The file a reporter writes: the base path with the reporter's extension. */
|
|
9
|
+
export function reportPath(base, extension) {
|
|
10
|
+
const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv)$/i, '');
|
|
11
|
+
return `${stem}.${extension}`;
|
|
12
|
+
}
|
|
13
|
+
function prepare(path) {
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
return path;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Runs the reporters in the order given, except `ci`, which runs last because
|
|
19
|
+
* it ends the process when an assertion failed.
|
|
20
|
+
*/
|
|
5
21
|
export function report(result, type, options) {
|
|
6
22
|
const types = Array.isArray(type) ? type : [type];
|
|
7
|
-
|
|
23
|
+
const ordered = [...types.filter((t) => t !== 'ci'), ...types.filter((t) => t === 'ci')];
|
|
24
|
+
for (const t of ordered) {
|
|
8
25
|
switch (t) {
|
|
9
26
|
case 'console':
|
|
10
27
|
consoleReport(result);
|
|
11
28
|
break;
|
|
12
29
|
case 'json':
|
|
13
|
-
jsonReport(result, { path: options?.path
|
|
30
|
+
jsonReport(result, { path: prepare(reportPath(options?.path, 'json')) });
|
|
14
31
|
break;
|
|
15
32
|
case 'csv':
|
|
16
|
-
csvReport(result, { path: options?.path
|
|
33
|
+
csvReport(result, { path: prepare(reportPath(options?.path, 'csv')) });
|
|
17
34
|
break;
|
|
18
35
|
case 'ci':
|
|
19
36
|
ciReport(result);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AA6ChC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAyB;IAC5E,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
|
package/dist/schema.d.ts
CHANGED
|
@@ -10,10 +10,21 @@ export declare const EvalSuiteConfigSchema: z.ZodObject<{
|
|
|
10
10
|
timeout: z.ZodDefault<z.ZodNumber>;
|
|
11
11
|
retries: z.ZodDefault<z.ZodNumber>;
|
|
12
12
|
}, z.core.$strip>;
|
|
13
|
+
/** The part of a Cogitator runtime the judge needs: running an agent. */
|
|
14
|
+
export interface JudgeCogitator {
|
|
15
|
+
run(agent: unknown, options: {
|
|
16
|
+
input: string;
|
|
17
|
+
useMemory?: boolean;
|
|
18
|
+
}): Promise<{
|
|
19
|
+
output: string;
|
|
20
|
+
structured?: unknown;
|
|
21
|
+
}>;
|
|
22
|
+
}
|
|
13
23
|
export declare const JudgeConfigSchema: z.ZodObject<{
|
|
14
24
|
model: z.ZodString;
|
|
15
25
|
temperature: z.ZodDefault<z.ZodNumber>;
|
|
16
26
|
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
27
|
+
cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
|
|
17
28
|
}, z.core.$strip>;
|
|
18
29
|
export declare const EvalComparisonConfigSchema: z.ZodObject<{
|
|
19
30
|
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
@@ -25,5 +36,6 @@ export type EvalCaseInput = z.input<typeof EvalCaseSchema>;
|
|
|
25
36
|
export type EvalSuiteConfig = z.output<typeof EvalSuiteConfigSchema>;
|
|
26
37
|
export type EvalSuiteConfigInput = z.input<typeof EvalSuiteConfigSchema>;
|
|
27
38
|
export type JudgeConfig = z.output<typeof JudgeConfigSchema>;
|
|
39
|
+
export type JudgeConfigInput = z.input<typeof JudgeConfigSchema>;
|
|
28
40
|
export type EvalComparisonConfig = z.output<typeof EvalComparisonConfigSchema>;
|
|
29
41
|
//# sourceMappingURL=schema.d.ts.map
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,eAAO,MAAM,iBAAiB
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACtD;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
|
package/dist/schema.js
CHANGED
|
@@ -10,10 +10,17 @@ export const EvalSuiteConfigSchema = z.object({
|
|
|
10
10
|
timeout: z.number().int().min(1000).default(30000),
|
|
11
11
|
retries: z.number().int().min(0).max(10).default(0),
|
|
12
12
|
});
|
|
13
|
+
function isJudgeCogitator(value) {
|
|
14
|
+
return (typeof value === 'object' &&
|
|
15
|
+
value !== null &&
|
|
16
|
+
typeof value.run === 'function');
|
|
17
|
+
}
|
|
13
18
|
export const JudgeConfigSchema = z.object({
|
|
14
19
|
model: z.string(),
|
|
15
20
|
temperature: z.number().default(0),
|
|
16
21
|
maxTokens: z.number().int().positive().optional(),
|
|
22
|
+
/** Runtime that runs the judge; defaults to the `cogitator` of an agent target */
|
|
23
|
+
cogitator: z.custom(isJudgeCogitator).optional(),
|
|
17
24
|
});
|
|
18
25
|
export const EvalComparisonConfigSchema = z.object({
|
|
19
26
|
concurrency: z.number().int().min(1).default(5),
|
package/dist/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAUH,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CACjE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
|
package/dist/stats/mcnemar.js
CHANGED
|
@@ -22,7 +22,7 @@ export function mcnemarsTest(pairsACorrect_BIncorrect, pairsAIncorrect_BCorrect)
|
|
|
22
22
|
if (b + c === 0) {
|
|
23
23
|
return { chiSquare: 0, pValue: 1, significant: false };
|
|
24
24
|
}
|
|
25
|
-
const diff = Math.abs(b - c) - 1;
|
|
25
|
+
const diff = Math.max(0, Math.abs(b - c) - 1);
|
|
26
26
|
const chiSquare = (diff * diff) / (b + c);
|
|
27
27
|
const pValue = chiSquareSurvival(chiSquare);
|
|
28
28
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,SAAS;QACT,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;KAC3B,CAAC;AACJ,CAAC"}
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE/D,MAAM,WAAW,QAAQ,CAAC,OAAO,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,mBAAmB;;
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE/D,MAAM,WAAW,QAAQ,CAAC,OAAO,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,mBAAmB;;iBAOvB,CAAC;AAEH,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAkBzD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,CAc3E;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAErE"}
|
package/dist/tools.js
CHANGED
|
@@ -1,40 +1,25 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
const RunEvalParamsSchema = z.object({
|
|
3
|
-
maxCases: z
|
|
3
|
+
maxCases: z
|
|
4
|
+
.number()
|
|
5
|
+
.int()
|
|
6
|
+
.positive()
|
|
7
|
+
.optional()
|
|
8
|
+
.describe('Run only the first N cases of the dataset'),
|
|
4
9
|
});
|
|
5
|
-
function buildSummary(result
|
|
6
|
-
const capped = maxCases ? result.results.slice(0, maxCases) : result.results;
|
|
7
|
-
const total = capped.length;
|
|
8
|
-
const duration = maxCases
|
|
9
|
-
? capped.reduce((sum, r) => sum + r.duration, 0)
|
|
10
|
-
: result.stats.duration;
|
|
11
|
-
const cost = maxCases
|
|
12
|
-
? capped.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0)
|
|
13
|
-
: result.stats.cost;
|
|
10
|
+
function buildSummary(result) {
|
|
14
11
|
const metrics = {};
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
for (const r of capped) {
|
|
18
|
-
for (const s of r.scores) {
|
|
19
|
-
let arr = scoresByMetric.get(s.name);
|
|
20
|
-
if (!arr) {
|
|
21
|
-
arr = [];
|
|
22
|
-
scoresByMetric.set(s.name, arr);
|
|
23
|
-
}
|
|
24
|
-
arr.push(s.score);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
for (const [name, values] of scoresByMetric) {
|
|
28
|
-
metrics[name] = values.reduce((a, b) => a + b, 0) / values.length;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
for (const [name, agg] of Object.entries(result.aggregated)) {
|
|
33
|
-
metrics[name] = agg.mean;
|
|
34
|
-
}
|
|
12
|
+
for (const [name, agg] of Object.entries(result.aggregated)) {
|
|
13
|
+
metrics[name] = agg.mean;
|
|
35
14
|
}
|
|
36
|
-
|
|
37
|
-
|
|
15
|
+
return {
|
|
16
|
+
success: true,
|
|
17
|
+
total: result.stats.total,
|
|
18
|
+
duration: result.stats.duration,
|
|
19
|
+
cost: result.stats.cost,
|
|
20
|
+
metrics,
|
|
21
|
+
assertionsPassed: result.assertions.every((a) => a.passed),
|
|
22
|
+
};
|
|
38
23
|
}
|
|
39
24
|
export function createRunEvalTool(suite) {
|
|
40
25
|
return {
|
|
@@ -43,8 +28,8 @@ export function createRunEvalTool(suite) {
|
|
|
43
28
|
parameters: RunEvalParamsSchema,
|
|
44
29
|
execute: async ({ maxCases }) => {
|
|
45
30
|
try {
|
|
46
|
-
const result = await suite.run();
|
|
47
|
-
return buildSummary(result
|
|
31
|
+
const result = await suite.run({ maxCases });
|
|
32
|
+
return buildSummary(result);
|
|
48
33
|
}
|
|
49
34
|
catch (err) {
|
|
50
35
|
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,2CAA2C,CAAC;CACzD,CAAC,CAAC;AAIH,SAAS,YAAY,CAAC,MAAuB;IAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAa;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK;QACzB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;QAC/B,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI;QACvB,OAAO;QACP,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAgB;IAChD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,mEAAmE;QAChF,UAAU,EAAE,mBAAmB;QAC/B,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC7C,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAgB;IACxC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Evaluation framework for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"zod": "^4.6.5"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@cogitator-ai/core": ">=0.
|
|
24
|
+
"@cogitator-ai/core": ">=0.26.0"
|
|
25
25
|
},
|
|
26
26
|
"peerDependenciesMeta": {
|
|
27
27
|
"@cogitator-ai/core": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"papaparse": "^5.7.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
-
"@cogitator-ai/core": "0.
|
|
35
|
+
"@cogitator-ai/core": "0.26.0",
|
|
36
36
|
"@types/papaparse": "^5.3.15",
|
|
37
37
|
"typescript": "^6.0.3",
|
|
38
38
|
"vitest": "^5.0.3"
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"build": "tsc && tsx ../../scripts/fix-esm-imports.ts .",
|
|
51
51
|
"dev": "tsc --watch",
|
|
52
52
|
"clean": "rm -rf dist",
|
|
53
|
-
"typecheck": "tsc --noEmit",
|
|
53
|
+
"typecheck": "tsc --noEmit -p tsconfig.test.json",
|
|
54
54
|
"test": "vitest run",
|
|
55
55
|
"test:watch": "vitest"
|
|
56
56
|
}
|