@cogitator-ai/evals 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -10
- package/dist/assertions/direction.d.ts +6 -0
- package/dist/assertions/direction.d.ts.map +1 -0
- package/dist/assertions/direction.js +12 -0
- package/dist/assertions/direction.js.map +1 -0
- package/dist/assertions/regression.d.ts.map +1 -1
- package/dist/assertions/regression.js +1 -7
- package/dist/assertions/regression.js.map +1 -1
- package/dist/assertions/threshold.d.ts.map +1 -1
- package/dist/assertions/threshold.js +1 -7
- package/dist/assertions/threshold.js.map +1 -1
- package/dist/eval-builder.d.ts +2 -2
- package/dist/eval-builder.d.ts.map +1 -1
- package/dist/eval-builder.js.map +1 -1
- package/dist/eval-comparison.d.ts +2 -2
- package/dist/eval-comparison.d.ts.map +1 -1
- package/dist/eval-suite.d.ts +7 -3
- package/dist/eval-suite.d.ts.map +1 -1
- package/dist/eval-suite.js +10 -5
- package/dist/eval-suite.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/schema.d.ts +1 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/stats/mcnemar.js +1 -1
- package/dist/stats/mcnemar.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +19 -34
- package/dist/tools.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -9,8 +9,11 @@ pnpm add @cogitator-ai/evals
|
|
|
9
9
|
|
|
10
10
|
# Optional dependencies
|
|
11
11
|
pnpm add papaparse # CSV dataset loading
|
|
12
|
+
pnpm add @cogitator-ai/core # agent targets and LLM-as-judge metrics
|
|
12
13
|
```
|
|
13
14
|
|
|
15
|
+
Website docs: [Evals](https://cogitator.app/docs/evals), [Datasets](https://cogitator.app/docs/evals/datasets), [Metrics](https://cogitator.app/docs/evals/metrics), [Assertions](https://cogitator.app/docs/evals/assertions), [Comparison](https://cogitator.app/docs/evals/comparison), [Reporters](https://cogitator.app/docs/evals/reporters).
|
|
16
|
+
|
|
14
17
|
## Features
|
|
15
18
|
|
|
16
19
|
- **EvalSuite** — Run datasets against agents or plain functions with configurable concurrency, timeouts, and retries
|
|
@@ -59,7 +62,11 @@ result.report('console');
|
|
|
59
62
|
result.saveBaseline('./baseline.json');
|
|
60
63
|
```
|
|
61
64
|
|
|
62
|
-
A
|
|
65
|
+
A target is either `{ fn: (input) => Promise<string> }` or `{ agent, cogitator }`, which runs the agent with `cogitator.run(agent, { input })` and records its usage for the statistical metrics. Defaults: `concurrency` 5, `timeout` 30 000 ms (minimum 1 000), `retries` 0.
|
|
66
|
+
|
|
67
|
+
A case that throws or exceeds `timeout` is retried up to `retries` times. If every attempt fails, its result has an empty `output` and an `error` message describing the last failure.
|
|
68
|
+
|
|
69
|
+
`suite.run(options?: EvalRunOptions)` takes `{ maxCases?: number }`: with `maxCases` only the first N cases of the dataset run, and the results, aggregates, assertions and `stats.total` cover just those cases.
|
|
63
70
|
|
|
64
71
|
---
|
|
65
72
|
|
|
@@ -140,7 +147,7 @@ const metrics = [
|
|
|
140
147
|
|
|
141
148
|
### LLM-as-Judge
|
|
142
149
|
|
|
143
|
-
Metrics scored by an LLM judge (0.0 to 1.0).
|
|
150
|
+
Metrics scored by an LLM judge (0.0 to 1.0). They require a `judge` config on the suite and a Cogitator to run the judge agent: `judge.cogitator`, or the `cogitator` of an agent target. Without one, the suite throws when it is created.
|
|
144
151
|
|
|
145
152
|
| Metric | Evaluates |
|
|
146
153
|
| -------------- | --------------------------------------- |
|
|
@@ -164,10 +171,12 @@ const suite = new EvalSuite({
|
|
|
164
171
|
prompt: 'Rate how technically accurate the response is for a software engineering audience.',
|
|
165
172
|
}),
|
|
166
173
|
],
|
|
167
|
-
judge: { model: 'openai/gpt-6.1-sol', temperature: 0, cogitator },
|
|
174
|
+
judge: { model: 'openai/gpt-6.1-sol', temperature: 0, cogitator: cog },
|
|
168
175
|
});
|
|
169
176
|
```
|
|
170
177
|
|
|
178
|
+
The `judge` option is a `JudgeConfigInput`: `{ model, temperature?, maxTokens?, cogitator? }`, with `temperature` defaulting to `0`.
|
|
179
|
+
|
|
171
180
|
### Statistical
|
|
172
181
|
|
|
173
182
|
Aggregate metrics computed across all results. These report percentile breakdowns (p50, p95, p99) rather than per-case scores.
|
|
@@ -177,7 +186,7 @@ import { latency, cost, tokenUsage } from '@cogitator-ai/evals';
|
|
|
177
186
|
|
|
178
187
|
const suite = new EvalSuite({
|
|
179
188
|
dataset,
|
|
180
|
-
target: { agent, cogitator },
|
|
189
|
+
target: { agent, cogitator: cog },
|
|
181
190
|
metrics: [exactMatch()],
|
|
182
191
|
statisticalMetrics: [latency(), cost(), tokenUsage()],
|
|
183
192
|
});
|
|
@@ -227,7 +236,7 @@ const assertions = [
|
|
|
227
236
|
];
|
|
228
237
|
```
|
|
229
238
|
|
|
230
|
-
Latency and
|
|
239
|
+
Latency, cost and tokenUsage metrics (and names ending in `Duration` or `Latency`) are lower-is-better: the assertion passes when the value is at or below the threshold. `noRegression` uses the same rule.
|
|
231
240
|
|
|
232
241
|
### noRegression
|
|
233
242
|
|
|
@@ -335,7 +344,7 @@ const suite = new EvalBuilder()
|
|
|
335
344
|
.withTarget({ fn: async (input) => myModel(input) })
|
|
336
345
|
.withMetrics([exactMatch(), contains(), faithfulness()])
|
|
337
346
|
.withStatisticalMetrics([latency()])
|
|
338
|
-
.withJudge({ model: 'gpt-6.1-sol', temperature: 0 })
|
|
347
|
+
.withJudge({ model: 'openai/gpt-6.1-sol', temperature: 0, cogitator: cog })
|
|
339
348
|
.withAssertions([threshold('exactMatch', 0.85), noRegression('./baseline.json')])
|
|
340
349
|
.withConcurrency(10)
|
|
341
350
|
.withTimeout(60_000)
|
|
@@ -447,10 +456,12 @@ result.report('ci');
|
|
|
447
456
|
|
|
448
457
|
### Agent Tools
|
|
449
458
|
|
|
450
|
-
| Export | Description
|
|
451
|
-
| ------------------- |
|
|
452
|
-
| `createRunEvalTool` | Creates a `run_eval` tool for agents |
|
|
453
|
-
| `evalTools` | Returns all eval tools as an array
|
|
459
|
+
| Export | Description |
|
|
460
|
+
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
461
|
+
| `createRunEvalTool` | Creates a `run_eval` tool for agents for a given suite; its optional `maxCases` runs `suite.run({ maxCases })` and the summary (`total`, `metrics`, `assertionsPassed`) covers those cases |
|
|
462
|
+
| `evalTools` | Returns all eval tools for a suite as an array |
|
|
463
|
+
|
|
464
|
+
Like the RAG tools, eval tools carry a Zod `parameters` schema and can be passed to `tool()` from `@cogitator-ai/core`.
|
|
454
465
|
|
|
455
466
|
---
|
|
456
467
|
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether smaller values of a metric are better, as for the built-in `latency`, `cost` and
|
|
3
|
+
* `tokenUsage` statistical metrics. A field path such as `latency.p95` follows its metric.
|
|
4
|
+
*/
|
|
5
|
+
export declare function isLowerBetter(metricPath: string): boolean;
|
|
6
|
+
//# sourceMappingURL=direction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"direction.d.ts","sourceRoot":"","sources":["../../src/assertions/direction.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAMzD"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
const LOWER_IS_BETTER_PREFIXES = ['latency', 'cost', 'tokenUsage'];
|
|
2
|
+
const LOWER_IS_BETTER_SUFFIXES = ['Duration', 'Latency'];
|
|
3
|
+
/**
|
|
4
|
+
* Whether smaller values of a metric are better, as for the built-in `latency`, `cost` and
|
|
5
|
+
* `tokenUsage` statistical metrics. A field path such as `latency.p95` follows its metric.
|
|
6
|
+
*/
|
|
7
|
+
export function isLowerBetter(metricPath) {
|
|
8
|
+
const base = metricPath.split('.')[0];
|
|
9
|
+
return (LOWER_IS_BETTER_PREFIXES.some((prefix) => base.startsWith(prefix)) ||
|
|
10
|
+
LOWER_IS_BETTER_SUFFIXES.some((suffix) => base.endsWith(suffix)));
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=direction.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"direction.js","sourceRoot":"","sources":["../../src/assertions/direction.ts"],"names":[],"mappings":"AAAA,MAAM,wBAAwB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;AACnE,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;AAEzD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,UAAkB;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,OAAO,CACL,wBAAwB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAClE,wBAAwB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CACjE,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAG3C,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAgE7F"}
|
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
|
-
|
|
3
|
-
const base = name.split('.')[0];
|
|
4
|
-
return (base.startsWith('latency') ||
|
|
5
|
-
base.startsWith('cost') ||
|
|
6
|
-
base.endsWith('Duration') ||
|
|
7
|
-
base.endsWith('Latency'));
|
|
8
|
-
}
|
|
2
|
+
import { isLowerBetter } from './direction.js';
|
|
9
3
|
export function noRegression(baselinePath, opts) {
|
|
10
4
|
return (aggregated, _stats) => {
|
|
11
5
|
let baseline;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,IAA6B;IAC9E,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,IAAI,QAAgC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,iCAAiC,YAAY,EAAE;aACzD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC;QAC1C,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,SAAS,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC;YACxB,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YAE1C,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,2EAA2E;aACrF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,0CAA0C;SACpD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"threshold.d.ts","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"threshold.d.ts","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,SAAS,CAAC;AAoB7D,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CA0BxE"}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const base = name.split('.')[0];
|
|
3
|
-
return (base.startsWith('latency') ||
|
|
4
|
-
base.startsWith('cost') ||
|
|
5
|
-
base.endsWith('Duration') ||
|
|
6
|
-
base.endsWith('Latency'));
|
|
7
|
-
}
|
|
1
|
+
import { isLowerBetter } from './direction.js';
|
|
8
2
|
function resolve(aggregated, path) {
|
|
9
3
|
const parts = path.split('.');
|
|
10
4
|
const metric = aggregated[parts[0]];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"threshold.js","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"threshold.js","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,SAAS,OAAO,CACd,UAA4C,EAC5C,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAEnE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAA2B,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,UAAkB,EAAE,KAAa;IACzD,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAEjE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,IAAI,EAAE,aAAa,UAAU,GAAG;gBAChC,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,WAAW,UAAU,mCAAmC;aAClE,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAC/D,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,aAAa,UAAU,GAAG;YAChC,MAAM;YACN,OAAO,EAAE,MAAM;gBACb,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,IAAI,SAAS,IAAI,KAAK,EAAE;gBACnD,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,cAAc,SAAS,IAAI,KAAK,EAAE;YAC/D,MAAM;YACN,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/eval-builder.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { EvalSuite } from './eval-suite.js';
|
|
|
2
2
|
import type { EvalTarget, EvalSuiteOptions } from './eval-suite.js';
|
|
3
3
|
import { Dataset } from './datasets/index.js';
|
|
4
4
|
import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
|
|
5
|
-
import type {
|
|
5
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
6
6
|
import type { AssertionFn } from './assertions/index.js';
|
|
7
7
|
export declare class EvalBuilder {
|
|
8
8
|
private _dataset?;
|
|
@@ -19,7 +19,7 @@ export declare class EvalBuilder {
|
|
|
19
19
|
withTarget(target: EvalTarget): this;
|
|
20
20
|
withMetrics(metrics: MetricFn[]): this;
|
|
21
21
|
withStatisticalMetrics(metrics: StatisticalMetricFn[]): this;
|
|
22
|
-
withJudge(config:
|
|
22
|
+
withJudge(config: JudgeConfigInput): this;
|
|
23
23
|
withAssertions(assertions: AssertionFn[]): this;
|
|
24
24
|
withConcurrency(n: number): this;
|
|
25
25
|
withTimeout(ms: number): this;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAmB;IAClC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAKzC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA8BnB"}
|
package/dist/eval-builder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,
|
|
1
|
+
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAoB;IAC1B,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAwB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EvalTarget, EvalSuiteResult } from './eval-suite.js';
|
|
2
2
|
import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
4
4
|
import { Dataset } from './datasets/index.js';
|
|
5
5
|
export interface EvalComparisonOptions {
|
|
6
6
|
dataset: Dataset;
|
|
@@ -10,7 +10,7 @@ export interface EvalComparisonOptions {
|
|
|
10
10
|
};
|
|
11
11
|
metrics?: MetricFn[];
|
|
12
12
|
statisticalMetrics?: StatisticalMetricFn[];
|
|
13
|
-
judge?:
|
|
13
|
+
judge?: JudgeConfigInput;
|
|
14
14
|
concurrency?: number;
|
|
15
15
|
timeout?: number;
|
|
16
16
|
retries?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAkED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IAIjC,GAAG,IAAI,OAAO,CAAC,gBAAgB,CAAC;CAoEvC"}
|
package/dist/eval-suite.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Dataset } from './datasets/index.js';
|
|
2
2
|
import type { EvalCase } from './schema.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { JudgeConfigInput } from './schema.js';
|
|
4
4
|
import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
|
|
5
5
|
import type { LLMMetricFn } from './metrics/llm-judge.js';
|
|
6
6
|
import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions/index.js';
|
|
@@ -20,13 +20,17 @@ export interface EvalSuiteOptions {
|
|
|
20
20
|
target: EvalTarget;
|
|
21
21
|
metrics?: MetricFn[];
|
|
22
22
|
statisticalMetrics?: StatisticalMetricFn[];
|
|
23
|
-
judge?:
|
|
23
|
+
judge?: JudgeConfigInput;
|
|
24
24
|
assertions?: AssertionFn[];
|
|
25
25
|
concurrency?: number;
|
|
26
26
|
timeout?: number;
|
|
27
27
|
retries?: number;
|
|
28
28
|
onProgress?: (progress: EvalProgress) => void;
|
|
29
29
|
}
|
|
30
|
+
export interface EvalRunOptions {
|
|
31
|
+
/** Run only the first `maxCases` cases of the dataset (a positive integer) */
|
|
32
|
+
maxCases?: number;
|
|
33
|
+
}
|
|
30
34
|
export interface EvalSuiteResult {
|
|
31
35
|
results: Array<EvalCaseResult & {
|
|
32
36
|
scores: MetricScore[];
|
|
@@ -54,7 +58,7 @@ export declare class EvalSuite {
|
|
|
54
58
|
private readonly onProgress?;
|
|
55
59
|
constructor(opts: EvalSuiteOptions);
|
|
56
60
|
private validateTarget;
|
|
57
|
-
run(): Promise<EvalSuiteResult>;
|
|
61
|
+
run(options?: EvalRunOptions): Promise<EvalSuiteResult>;
|
|
58
62
|
private executeCase;
|
|
59
63
|
private executeCaseAttempt;
|
|
60
64
|
private executeFnTarget;
|
package/dist/eval-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGnF,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IAyClC,OAAO,CAAC,cAAc;IAuBhB,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;YAsGnD,WAAW;YAmBX,kBAAkB;YAoBlB,eAAe;YAMf,kBAAkB;YA+BlB,mBAAmB;IAKjC,OAAO,CAAC,eAAe;CAuBxB"}
|
package/dist/eval-suite.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
|
-
import { EvalSuiteConfigSchema } from './schema.js';
|
|
2
|
+
import { EvalSuiteConfigSchema, JudgeConfigSchema } from './schema.js';
|
|
3
3
|
import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
|
|
4
4
|
import { aggregate } from './stats/index.js';
|
|
5
5
|
import { report } from './reporters/index.js';
|
|
@@ -36,11 +36,12 @@ export class EvalSuite {
|
|
|
36
36
|
if (hasLLMMetrics && !opts.judge) {
|
|
37
37
|
throw new Error('LLM metrics require a judge config');
|
|
38
38
|
}
|
|
39
|
-
const
|
|
39
|
+
const judgeConfig = opts.judge ? JudgeConfigSchema.parse(opts.judge) : undefined;
|
|
40
|
+
const judgeCogitator = judgeConfig?.cogitator ?? this.target.cogitator;
|
|
40
41
|
if (hasLLMMetrics && !judgeCogitator) {
|
|
41
42
|
throw new Error('LLM metrics need a Cogitator to run the judge: pass judge.cogitator, or use an agent target');
|
|
42
43
|
}
|
|
43
|
-
const judge =
|
|
44
|
+
const judge = judgeConfig && judgeCogitator ? judgeContextFor(judgeCogitator, judgeConfig) : null;
|
|
44
45
|
this.boundMetrics = rawMetrics.map((m) => isLLMMetric(m) && judge ? bindJudgeContext(m, judge) : m);
|
|
45
46
|
}
|
|
46
47
|
validateTarget() {
|
|
@@ -61,9 +62,13 @@ export class EvalSuite {
|
|
|
61
62
|
throw new Error('Cogitator target requires agent instance');
|
|
62
63
|
}
|
|
63
64
|
}
|
|
64
|
-
async run() {
|
|
65
|
+
async run(options = {}) {
|
|
66
|
+
const { maxCases } = options;
|
|
67
|
+
if (maxCases !== undefined && (!Number.isInteger(maxCases) || maxCases < 1)) {
|
|
68
|
+
throw new Error(`maxCases must be a positive integer, got ${maxCases}`);
|
|
69
|
+
}
|
|
65
70
|
const suiteStart = Date.now();
|
|
66
|
-
const cases =
|
|
71
|
+
const cases = this.dataset.cases.slice(0, maxCases);
|
|
67
72
|
const total = cases.length;
|
|
68
73
|
let completed = 0;
|
|
69
74
|
const indexed = [];
|
package/dist/eval-suite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAIpE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0CrC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,KAAK,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;SAC1E,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,KAAgD,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,KAAK,GAAG,UAAU,CAChB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EAC5D,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare const VERSION = "0.1.0";
|
|
2
2
|
export { EvalSuite, isLLMMetric } from './eval-suite.js';
|
|
3
|
-
export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite.js';
|
|
3
|
+
export type { EvalTarget, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, } from './eval-suite.js';
|
|
4
4
|
export { EvalComparison } from './eval-comparison.js';
|
|
5
5
|
export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison.js';
|
|
6
6
|
export { EvalBuilder } from './eval-builder.js';
|
|
@@ -25,7 +25,7 @@ export type { TTestResult } from './stats/t-test.js';
|
|
|
25
25
|
export { mcnemarsTest } from './stats/mcnemar.js';
|
|
26
26
|
export type { McNemarResult } from './stats/mcnemar.js';
|
|
27
27
|
export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
|
|
28
|
-
export type { EvalCase, EvalSuiteConfig, JudgeConfig, EvalComparisonConfig } from './schema.js';
|
|
28
|
+
export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, EvalComparisonConfig, } from './schema.js';
|
|
29
29
|
export { createRunEvalTool, evalTools } from './tools.js';
|
|
30
30
|
export type { EvalTool } from './tools.js';
|
|
31
31
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAU1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/schema.d.ts
CHANGED
|
@@ -36,5 +36,6 @@ export type EvalCaseInput = z.input<typeof EvalCaseSchema>;
|
|
|
36
36
|
export type EvalSuiteConfig = z.output<typeof EvalSuiteConfigSchema>;
|
|
37
37
|
export type EvalSuiteConfigInput = z.input<typeof EvalSuiteConfigSchema>;
|
|
38
38
|
export type JudgeConfig = z.output<typeof JudgeConfigSchema>;
|
|
39
|
+
export type JudgeConfigInput = z.input<typeof JudgeConfigSchema>;
|
|
39
40
|
export type EvalComparisonConfig = z.output<typeof EvalComparisonConfigSchema>;
|
|
40
41
|
//# sourceMappingURL=schema.d.ts.map
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACtD;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACtD;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
|
package/dist/stats/mcnemar.js
CHANGED
|
@@ -22,7 +22,7 @@ export function mcnemarsTest(pairsACorrect_BIncorrect, pairsAIncorrect_BCorrect)
|
|
|
22
22
|
if (b + c === 0) {
|
|
23
23
|
return { chiSquare: 0, pValue: 1, significant: false };
|
|
24
24
|
}
|
|
25
|
-
const diff = Math.abs(b - c) - 1;
|
|
25
|
+
const diff = Math.max(0, Math.abs(b - c) - 1);
|
|
26
26
|
const chiSquare = (diff * diff) / (b + c);
|
|
27
27
|
const pValue = chiSquareSurvival(chiSquare);
|
|
28
28
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,SAAS;QACT,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;KAC3B,CAAC;AACJ,CAAC"}
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE/D,MAAM,WAAW,QAAQ,CAAC,OAAO,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,mBAAmB;;
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE/D,MAAM,WAAW,QAAQ,CAAC,OAAO,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,mBAAmB;;iBAOvB,CAAC;AAEH,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAkBzD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,CAc3E;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAErE"}
|
package/dist/tools.js
CHANGED
|
@@ -1,40 +1,25 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
const RunEvalParamsSchema = z.object({
|
|
3
|
-
maxCases: z
|
|
3
|
+
maxCases: z
|
|
4
|
+
.number()
|
|
5
|
+
.int()
|
|
6
|
+
.positive()
|
|
7
|
+
.optional()
|
|
8
|
+
.describe('Run only the first N cases of the dataset'),
|
|
4
9
|
});
|
|
5
|
-
function buildSummary(result
|
|
6
|
-
const capped = maxCases ? result.results.slice(0, maxCases) : result.results;
|
|
7
|
-
const total = capped.length;
|
|
8
|
-
const duration = maxCases
|
|
9
|
-
? capped.reduce((sum, r) => sum + r.duration, 0)
|
|
10
|
-
: result.stats.duration;
|
|
11
|
-
const cost = maxCases
|
|
12
|
-
? capped.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0)
|
|
13
|
-
: result.stats.cost;
|
|
10
|
+
function buildSummary(result) {
|
|
14
11
|
const metrics = {};
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
for (const r of capped) {
|
|
18
|
-
for (const s of r.scores) {
|
|
19
|
-
let arr = scoresByMetric.get(s.name);
|
|
20
|
-
if (!arr) {
|
|
21
|
-
arr = [];
|
|
22
|
-
scoresByMetric.set(s.name, arr);
|
|
23
|
-
}
|
|
24
|
-
arr.push(s.score);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
for (const [name, values] of scoresByMetric) {
|
|
28
|
-
metrics[name] = values.reduce((a, b) => a + b, 0) / values.length;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
for (const [name, agg] of Object.entries(result.aggregated)) {
|
|
33
|
-
metrics[name] = agg.mean;
|
|
34
|
-
}
|
|
12
|
+
for (const [name, agg] of Object.entries(result.aggregated)) {
|
|
13
|
+
metrics[name] = agg.mean;
|
|
35
14
|
}
|
|
36
|
-
|
|
37
|
-
|
|
15
|
+
return {
|
|
16
|
+
success: true,
|
|
17
|
+
total: result.stats.total,
|
|
18
|
+
duration: result.stats.duration,
|
|
19
|
+
cost: result.stats.cost,
|
|
20
|
+
metrics,
|
|
21
|
+
assertionsPassed: result.assertions.every((a) => a.passed),
|
|
22
|
+
};
|
|
38
23
|
}
|
|
39
24
|
export function createRunEvalTool(suite) {
|
|
40
25
|
return {
|
|
@@ -43,8 +28,8 @@ export function createRunEvalTool(suite) {
|
|
|
43
28
|
parameters: RunEvalParamsSchema,
|
|
44
29
|
execute: async ({ maxCases }) => {
|
|
45
30
|
try {
|
|
46
|
-
const result = await suite.run();
|
|
47
|
-
return buildSummary(result
|
|
31
|
+
const result = await suite.run({ maxCases });
|
|
32
|
+
return buildSummary(result);
|
|
48
33
|
}
|
|
49
34
|
catch (err) {
|
|
50
35
|
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,2CAA2C,CAAC;CACzD,CAAC,CAAC;AAIH,SAAS,YAAY,CAAC,MAAuB;IAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAa;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK;QACzB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;QAC/B,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI;QACvB,OAAO;QACP,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAgB;IAChD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,mEAAmE;QAChF,UAAU,EAAE,mBAAmB;QAC/B,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC7C,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAgB;IACxC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Evaluation framework for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"zod": "^4.6.5"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@cogitator-ai/core": ">=0.
|
|
24
|
+
"@cogitator-ai/core": ">=0.26.0"
|
|
25
25
|
},
|
|
26
26
|
"peerDependenciesMeta": {
|
|
27
27
|
"@cogitator-ai/core": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"papaparse": "^5.7.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
-
"@cogitator-ai/core": "0.
|
|
35
|
+
"@cogitator-ai/core": "0.26.0",
|
|
36
36
|
"@types/papaparse": "^5.3.15",
|
|
37
37
|
"typescript": "^6.0.3",
|
|
38
38
|
"vitest": "^5.0.3"
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"build": "tsc && tsx ../../scripts/fix-esm-imports.ts .",
|
|
51
51
|
"dev": "tsc --watch",
|
|
52
52
|
"clean": "rm -rf dist",
|
|
53
|
-
"typecheck": "tsc --noEmit",
|
|
53
|
+
"typecheck": "tsc --noEmit -p tsconfig.test.json",
|
|
54
54
|
"test": "vitest run",
|
|
55
55
|
"test:watch": "vitest"
|
|
56
56
|
}
|