@cogitator-ai/evals 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +202 -48
- package/dist/datasets/csv-loader.d.ts +5 -0
- package/dist/datasets/csv-loader.d.ts.map +1 -1
- package/dist/datasets/csv-loader.js +18 -0
- package/dist/datasets/csv-loader.js.map +1 -1
- package/dist/eval-builder.d.ts +3 -0
- package/dist/eval-builder.d.ts.map +1 -1
- package/dist/eval-builder.js +7 -0
- package/dist/eval-builder.js.map +1 -1
- package/dist/eval-comparison.d.ts +5 -0
- package/dist/eval-comparison.d.ts.map +1 -1
- package/dist/eval-comparison.js +20 -4
- package/dist/eval-comparison.js.map +1 -1
- package/dist/eval-suite.d.ts +48 -5
- package/dist/eval-suite.d.ts.map +1 -1
- package/dist/eval-suite.js +219 -57
- package/dist/eval-suite.js.map +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/matchers.d.ts +68 -0
- package/dist/matchers.d.ts.map +1 -0
- package/dist/matchers.js +112 -0
- package/dist/matchers.js.map +1 -0
- package/dist/metrics/custom.d.ts +19 -7
- package/dist/metrics/custom.d.ts.map +1 -1
- package/dist/metrics/custom.js +5 -0
- package/dist/metrics/custom.js.map +1 -1
- package/dist/metrics/deterministic.d.ts +1 -0
- package/dist/metrics/deterministic.d.ts.map +1 -1
- package/dist/metrics/deterministic.js +49 -4
- package/dist/metrics/deterministic.js.map +1 -1
- package/dist/metrics/index.d.ts +9 -5
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +5 -3
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/llm-judge.d.ts +20 -1
- package/dist/metrics/llm-judge.d.ts.map +1 -1
- package/dist/metrics/llm-judge.js +64 -10
- package/dist/metrics/llm-judge.js.map +1 -1
- package/dist/metrics/matching.d.ts +28 -0
- package/dist/metrics/matching.d.ts.map +1 -0
- package/dist/metrics/matching.js +124 -0
- package/dist/metrics/matching.js.map +1 -0
- package/dist/metrics/retrieval.d.ts +21 -0
- package/dist/metrics/retrieval.d.ts.map +1 -0
- package/dist/metrics/retrieval.js +108 -0
- package/dist/metrics/retrieval.js.map +1 -0
- package/dist/metrics/statistical.d.ts +28 -0
- package/dist/metrics/statistical.d.ts.map +1 -1
- package/dist/metrics/statistical.js +61 -0
- package/dist/metrics/statistical.js.map +1 -1
- package/dist/metrics/tool-calls.d.ts +43 -0
- package/dist/metrics/tool-calls.d.ts.map +1 -0
- package/dist/metrics/tool-calls.js +106 -0
- package/dist/metrics/tool-calls.js.map +1 -0
- package/dist/metrics/types.d.ts +47 -5
- package/dist/metrics/types.d.ts.map +1 -1
- package/dist/metrics/types.js.map +1 -1
- package/dist/reporters/console.d.ts.map +1 -1
- package/dist/reporters/console.js +6 -2
- package/dist/reporters/console.js.map +1 -1
- package/dist/reporters/format.d.ts +6 -0
- package/dist/reporters/format.d.ts.map +1 -1
- package/dist/reporters/format.js +4 -0
- package/dist/reporters/format.js.map +1 -1
- package/dist/reporters/github.d.ts.map +1 -1
- package/dist/reporters/github.js +3 -2
- package/dist/reporters/github.js.map +1 -1
- package/dist/reporters/index.d.ts +34 -21
- package/dist/reporters/index.d.ts.map +1 -1
- package/dist/reporters/index.js.map +1 -1
- package/dist/schema.d.ts +14 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +13 -0
- package/dist/schema.js.map +1 -1
- package/package.json +8 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AA8C3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,CAAC;YACD,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAChG,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}
|
package/dist/eval-suite.d.ts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { Dataset } from './datasets/index.js';
|
|
2
2
|
import type { EvalCase } from './schema.js';
|
|
3
3
|
import type { JudgeConfigInput } from './schema.js';
|
|
4
|
-
import type { MetricFn,
|
|
4
|
+
import type { MetricFn, Passage, ScoredCaseResult, StatisticalMetricFn } from './metrics/types.js';
|
|
5
5
|
import type { LLMMetricFn } from './metrics/llm-judge.js';
|
|
6
6
|
import type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
|
|
7
7
|
import type { ReporterType, ReporterOptions } from './reporters/index.js';
|
|
8
|
+
/** A turn of a conversation: what the user said and what the target answered */
|
|
9
|
+
export interface EvalTurn {
|
|
10
|
+
input: string;
|
|
11
|
+
output: string;
|
|
12
|
+
}
|
|
8
13
|
/** What a function target gets besides the input */
|
|
9
14
|
export interface EvalTargetContext {
|
|
10
15
|
/**
|
|
@@ -17,11 +22,27 @@ export interface EvalTargetContext {
|
|
|
17
22
|
case: EvalCase;
|
|
18
23
|
/** 0 for the first attempt, then 1, 2 and so on for retries */
|
|
19
24
|
attempt: number;
|
|
25
|
+
/** 0 for the case's `input`, then 1, 2 and so on for each of its further `turns` */
|
|
26
|
+
turn: number;
|
|
27
|
+
/** The turns of the case before this one, with the answers the target gave */
|
|
28
|
+
history: readonly EvalTurn[];
|
|
29
|
+
}
|
|
30
|
+
/** A tool result as the `retrieval` of a target reads it */
|
|
31
|
+
export interface RetrievalToolResult {
|
|
32
|
+
name: string;
|
|
33
|
+
result: unknown;
|
|
20
34
|
}
|
|
21
35
|
export interface EvalTarget {
|
|
22
36
|
agent?: unknown;
|
|
23
37
|
cogitator?: unknown;
|
|
38
|
+
/** Called once per turn of a case: with `input`, then with each of its further `turns` */
|
|
24
39
|
fn?: (input: string, context: EvalTargetContext) => Promise<string>;
|
|
40
|
+
/**
|
|
41
|
+
* Reads the passages a tool result of an agent target surfaced, into the case's `retrieved`,
|
|
42
|
+
* undefined for a result that holds none. Default: `ragSearchPassages`, which reads the
|
|
43
|
+
* results of the `rag_search` tool of `@cogitator-ai/rag`
|
|
44
|
+
*/
|
|
45
|
+
retrieval?: (toolResult: RetrievalToolResult) => Passage[] | undefined;
|
|
25
46
|
}
|
|
26
47
|
export interface EvalProgress {
|
|
27
48
|
completed: number;
|
|
@@ -38,7 +59,19 @@ export interface EvalSuiteOptions {
|
|
|
38
59
|
concurrency?: number;
|
|
39
60
|
timeout?: number;
|
|
40
61
|
retries?: number;
|
|
41
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* Runs of each case, a positive integer, 1 by default. Each trial is scored on its own, the
|
|
64
|
+
* case result is the first trial that answered (the first when all errored) with every trial in
|
|
65
|
+
* `trials`, and the aggregate of a metric
|
|
66
|
+
* takes the mean of a case's scored trials as the case's score. A case counts as errored in
|
|
67
|
+
* `stats.errors` when every trial errored, and `latency()`, `cost()` and `tokenUsage()` see
|
|
68
|
+
* every trial. Use `passAtK()` to score how often a case passes
|
|
69
|
+
*/
|
|
70
|
+
trials?: number;
|
|
71
|
+
/**
|
|
72
|
+
* Called after each case, once all its trials ran. A callback that throws is reported as a
|
|
73
|
+
* process warning
|
|
74
|
+
*/
|
|
42
75
|
onProgress?: (progress: EvalProgress) => void;
|
|
43
76
|
}
|
|
44
77
|
export interface EvalRunOptions {
|
|
@@ -53,9 +86,11 @@ export interface EvalRunOptions {
|
|
|
53
86
|
/** The run-wide figures of a finished suite run, every field filled in */
|
|
54
87
|
export type EvalSuiteStats = Required<EvalStats>;
|
|
55
88
|
export interface EvalSuiteResult {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
89
|
+
/**
|
|
90
|
+
* One result per case, in dataset order. With several trials it is the first trial that
|
|
91
|
+
* answered, or the first when every trial errored, with every trial in its `trials`
|
|
92
|
+
*/
|
|
93
|
+
results: ScoredCaseResult[];
|
|
59
94
|
aggregated: Record<string, AggregatedMetric>;
|
|
60
95
|
assertions: AssertionResult[];
|
|
61
96
|
stats: EvalSuiteStats;
|
|
@@ -73,9 +108,12 @@ export declare class EvalSuite {
|
|
|
73
108
|
private readonly concurrency;
|
|
74
109
|
private readonly timeout;
|
|
75
110
|
private readonly retries;
|
|
111
|
+
private readonly trials;
|
|
76
112
|
private readonly onProgress?;
|
|
77
113
|
constructor(opts: EvalSuiteOptions);
|
|
78
114
|
private validateTarget;
|
|
115
|
+
/** A `passAtK()` metric needs the metric it reads and at least `k` trials */
|
|
116
|
+
private validatePassAtK;
|
|
79
117
|
/** Two metrics with one name would be aggregated, compared and reported as one */
|
|
80
118
|
private validateMetricNames;
|
|
81
119
|
/** The cases a run with `maxCases` goes through, in order */
|
|
@@ -91,6 +129,10 @@ export declare class EvalSuite {
|
|
|
91
129
|
*/
|
|
92
130
|
private executeCaseAttempt;
|
|
93
131
|
private executeFnTarget;
|
|
132
|
+
/**
|
|
133
|
+
* Runs the agent on each turn of the case. A case with further `turns` runs them all on one
|
|
134
|
+
* thread of its own, so the Cogitator's memory carries the conversation from turn to turn
|
|
135
|
+
*/
|
|
94
136
|
private executeAgentTarget;
|
|
95
137
|
private evaluateCaseMetrics;
|
|
96
138
|
/**
|
|
@@ -101,6 +143,7 @@ export declare class EvalSuite {
|
|
|
101
143
|
* bad metric neither drops the case nor poisons the aggregate with NaN.
|
|
102
144
|
*/
|
|
103
145
|
private scoreMetric;
|
|
146
|
+
/** One score per case and metric: the mean over the case's trials the metric scored */
|
|
104
147
|
private aggregateScores;
|
|
105
148
|
}
|
|
106
149
|
//# sourceMappingURL=eval-suite.d.ts.map
|
package/dist/eval-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAMR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE/E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAyDD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA6ClC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAyGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YA4ElB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}
|
package/dist/eval-suite.js
CHANGED
|
@@ -1,9 +1,30 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
1
2
|
import { writeFileSync } from 'node:fs';
|
|
2
3
|
import { EvalSuiteConfigSchema, JudgeConfigSchema } from './schema.js';
|
|
3
4
|
import { nonFiniteScoreError } from './metrics/types.js';
|
|
5
|
+
import { isPassAtKMetric } from './metrics/statistical.js';
|
|
6
|
+
import { ragSearchPassages } from './metrics/retrieval.js';
|
|
4
7
|
import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
|
|
5
|
-
import { aggregate } from './stats/index.js';
|
|
8
|
+
import { aggregate, mean } from './stats/index.js';
|
|
6
9
|
import { report } from './reporters/index.js';
|
|
10
|
+
/**
|
|
11
|
+
* A turn of a multi-turn case failed: the error of the turn, with the turns run so far (the
|
|
12
|
+
* failed one last) and what they spent
|
|
13
|
+
*/
|
|
14
|
+
class TurnFailure extends Error {
|
|
15
|
+
turns;
|
|
16
|
+
usage;
|
|
17
|
+
constructor(cause, turns, usage) {
|
|
18
|
+
super(errorMessage(cause), { cause });
|
|
19
|
+
this.turns = turns;
|
|
20
|
+
this.usage = usage;
|
|
21
|
+
this.name = 'TurnFailure';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/** What the user says to the target, turn by turn: the case's `input`, then its `turns` */
|
|
25
|
+
function turnInputs(evalCase) {
|
|
26
|
+
return [evalCase.input, ...(evalCase.turns ?? [])];
|
|
27
|
+
}
|
|
7
28
|
export function isLLMMetric(m) {
|
|
8
29
|
return 'requiresJudge' in m && m.requiresJudge === true;
|
|
9
30
|
}
|
|
@@ -95,12 +116,14 @@ export class EvalSuite {
|
|
|
95
116
|
concurrency;
|
|
96
117
|
timeout;
|
|
97
118
|
retries;
|
|
119
|
+
trials;
|
|
98
120
|
onProgress;
|
|
99
121
|
constructor(opts) {
|
|
100
122
|
const config = EvalSuiteConfigSchema.parse({
|
|
101
123
|
concurrency: opts.concurrency,
|
|
102
124
|
timeout: opts.timeout,
|
|
103
125
|
retries: opts.retries,
|
|
126
|
+
trials: opts.trials,
|
|
104
127
|
});
|
|
105
128
|
this.dataset = opts.dataset;
|
|
106
129
|
this.target = opts.target;
|
|
@@ -109,10 +132,12 @@ export class EvalSuite {
|
|
|
109
132
|
this.concurrency = config.concurrency;
|
|
110
133
|
this.timeout = config.timeout;
|
|
111
134
|
this.retries = config.retries;
|
|
135
|
+
this.trials = config.trials;
|
|
112
136
|
this.onProgress = opts.onProgress;
|
|
113
137
|
this.validateTarget();
|
|
114
138
|
const rawMetrics = opts.metrics ?? [];
|
|
115
139
|
this.validateMetricNames(rawMetrics);
|
|
140
|
+
this.validatePassAtK(rawMetrics);
|
|
116
141
|
const hasLLMMetrics = rawMetrics.some(isLLMMetric);
|
|
117
142
|
if (hasLLMMetrics && !opts.judge) {
|
|
118
143
|
throw new Error('LLM metrics require a judge config');
|
|
@@ -142,6 +167,24 @@ export class EvalSuite {
|
|
|
142
167
|
if (hasCogitator && !hasAgent) {
|
|
143
168
|
throw new Error('Cogitator target requires agent instance');
|
|
144
169
|
}
|
|
170
|
+
if (hasFn && this.target.retrieval !== undefined) {
|
|
171
|
+
throw new Error('Target retrieval reads tool results, so it needs an agent target, not fn');
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/** A `passAtK()` metric needs the metric it reads and at least `k` trials */
|
|
175
|
+
validatePassAtK(metrics) {
|
|
176
|
+
const names = new Set(metrics.map((m) => m.metricName));
|
|
177
|
+
for (const statMetric of this.statisticalMetrics) {
|
|
178
|
+
if (!isPassAtKMetric(statMetric))
|
|
179
|
+
continue;
|
|
180
|
+
const { metric, k } = statMetric.passAtK;
|
|
181
|
+
if (!names.has(metric)) {
|
|
182
|
+
throw new Error(`${statMetric.metricName} reads the metric '${metric}', which the suite does not have: add it to metrics`);
|
|
183
|
+
}
|
|
184
|
+
if (k > this.trials) {
|
|
185
|
+
throw new Error(`${statMetric.metricName} needs at least ${k} trials per case, the suite runs ${this.trials}: set trials to ${k} or more`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
145
188
|
}
|
|
146
189
|
/** Two metrics with one name would be aggregated, compared and reported as one */
|
|
147
190
|
validateMetricNames(metrics) {
|
|
@@ -169,35 +212,43 @@ export class EvalSuite {
|
|
|
169
212
|
const suiteStart = Date.now();
|
|
170
213
|
const cases = this.cases(maxCases);
|
|
171
214
|
const total = cases.length;
|
|
172
|
-
const
|
|
173
|
-
|
|
215
|
+
const trials = this.trials;
|
|
216
|
+
const trialResults = cases.map(() => new Array(trials));
|
|
217
|
+
const trialsLeft = cases.map(() => trials);
|
|
218
|
+
const runs = total * trials;
|
|
219
|
+
let nextRun = 0;
|
|
174
220
|
let completed = 0;
|
|
175
221
|
const worker = async () => {
|
|
176
|
-
while (
|
|
177
|
-
const
|
|
222
|
+
while (nextRun < runs && !signal?.aborted) {
|
|
223
|
+
const run = nextRun++;
|
|
224
|
+
const i = Math.floor(run / trials);
|
|
178
225
|
const evalCase = cases[i];
|
|
179
|
-
|
|
226
|
+
trialResults[i][run % trials] = await this.runCase(evalCase, signal);
|
|
227
|
+
if (--trialsLeft[i] > 0)
|
|
228
|
+
continue;
|
|
180
229
|
completed++;
|
|
181
230
|
this.reportProgress({ completed, total, currentCase: evalCase });
|
|
182
231
|
}
|
|
183
232
|
};
|
|
184
|
-
await Promise.all(Array.from({ length: Math.min(this.concurrency,
|
|
233
|
+
await Promise.all(Array.from({ length: Math.min(this.concurrency, runs) }, worker));
|
|
185
234
|
signal?.throwIfAborted();
|
|
235
|
+
const orderedResults = trialResults.map((caseTrials) => trials === 1 ? caseTrials[0] : { ...leadTrial(caseTrials), trials: caseTrials });
|
|
236
|
+
const allTrials = orderedResults.flatMap(trialsOf);
|
|
186
237
|
const aggregated = this.aggregateScores(orderedResults);
|
|
187
238
|
for (const statMetric of this.statisticalMetrics) {
|
|
188
|
-
const score = statMetric(orderedResults);
|
|
239
|
+
const score = statMetric(isPassAtKMetric(statMetric) ? orderedResults : allTrials);
|
|
189
240
|
aggregated[score.name] = {
|
|
190
241
|
name: score.name,
|
|
191
242
|
...aggregate(score.values ?? [score.score]),
|
|
192
243
|
...(score.metadata && { metadata: score.metadata }),
|
|
193
244
|
};
|
|
194
245
|
}
|
|
195
|
-
const targetCost =
|
|
196
|
-
const judgeCost =
|
|
246
|
+
const targetCost = allTrials.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
|
|
247
|
+
const judgeCost = allTrials.reduce((sum, r) => sum + r.scores.reduce((s, score) => s + (score.usage?.cost ?? 0), 0), 0);
|
|
197
248
|
const stats = {
|
|
198
249
|
total,
|
|
199
250
|
errors: orderedResults.filter((r) => r.error !== undefined).length,
|
|
200
|
-
metricErrors:
|
|
251
|
+
metricErrors: allTrials.reduce((n, r) => n + r.scores.filter((s) => s.error !== undefined).length, 0),
|
|
201
252
|
duration: Date.now() - suiteStart,
|
|
202
253
|
cost: targetCost + judgeCost,
|
|
203
254
|
targetCost,
|
|
@@ -211,16 +262,7 @@ export class EvalSuite {
|
|
|
211
262
|
stats,
|
|
212
263
|
report: (type, options) => {
|
|
213
264
|
report({
|
|
214
|
-
results: orderedResults.map(
|
|
215
|
-
case: { input: r.case.input, expected: r.case.expected },
|
|
216
|
-
output: r.output,
|
|
217
|
-
duration: r.duration,
|
|
218
|
-
...(r.error !== undefined && { error: r.error }),
|
|
219
|
-
...(r.truncated && { truncated: true }),
|
|
220
|
-
...(r.iterationLimitReached && { iterationLimitReached: true }),
|
|
221
|
-
...(r.paused && { paused: true }),
|
|
222
|
-
scores: r.scores,
|
|
223
|
-
})),
|
|
265
|
+
results: orderedResults.map(reportedCase),
|
|
224
266
|
aggregated,
|
|
225
267
|
assertions: assertionResults,
|
|
226
268
|
stats,
|
|
@@ -267,6 +309,7 @@ export class EvalSuite {
|
|
|
267
309
|
async executeCase(evalCase, signal) {
|
|
268
310
|
const start = Date.now();
|
|
269
311
|
let lastError;
|
|
312
|
+
let lastTurns;
|
|
270
313
|
let usage;
|
|
271
314
|
let attempts = 0;
|
|
272
315
|
for (let attempt = 0; attempt <= this.retries && !signal?.aborted; attempt++) {
|
|
@@ -278,6 +321,7 @@ export class EvalSuite {
|
|
|
278
321
|
}
|
|
279
322
|
usage = addUsage(usage, outcome.usage);
|
|
280
323
|
lastError = outcome.error;
|
|
324
|
+
lastTurns = outcome.turns;
|
|
281
325
|
}
|
|
282
326
|
return {
|
|
283
327
|
case: evalCase,
|
|
@@ -286,6 +330,7 @@ export class EvalSuite {
|
|
|
286
330
|
error: errorMessage(signal?.aborted ? signal.reason : lastError),
|
|
287
331
|
attempts,
|
|
288
332
|
...(usage && { usage }),
|
|
333
|
+
...(lastTurns && { turns: lastTurns }),
|
|
289
334
|
};
|
|
290
335
|
}
|
|
291
336
|
/**
|
|
@@ -306,10 +351,12 @@ export class EvalSuite {
|
|
|
306
351
|
try {
|
|
307
352
|
const first = await Promise.race([settled, abort.aborted]);
|
|
308
353
|
if (first !== 'aborted' && (first.ok || !signal.aborted))
|
|
309
|
-
return first;
|
|
354
|
+
return failureDetails(first);
|
|
310
355
|
const late = await waitAtMost(settled, this.timeout);
|
|
311
|
-
const
|
|
312
|
-
|
|
356
|
+
const lateOutcome = late && failureDetails(late);
|
|
357
|
+
const usage = lateOutcome?.ok ? lateOutcome.result.usage : lateOutcome?.usage;
|
|
358
|
+
const turns = lateOutcome && !lateOutcome.ok ? lateOutcome.turns : undefined;
|
|
359
|
+
return { ok: false, error: signal.reason, ...(usage && { usage }), ...(turns && { turns }) };
|
|
313
360
|
}
|
|
314
361
|
finally {
|
|
315
362
|
clearTimeout(timer);
|
|
@@ -318,37 +365,105 @@ export class EvalSuite {
|
|
|
318
365
|
}
|
|
319
366
|
async executeFnTarget(evalCase, context) {
|
|
320
367
|
const start = Date.now();
|
|
321
|
-
const
|
|
322
|
-
|
|
368
|
+
const inputs = turnInputs(evalCase);
|
|
369
|
+
const multiTurn = inputs.length > 1;
|
|
370
|
+
const turns = [];
|
|
371
|
+
for (const [turn, input] of inputs.entries()) {
|
|
372
|
+
if (turn > 0 && context.signal.aborted) {
|
|
373
|
+
throw new TurnFailure(context.signal.reason, [...turns], undefined);
|
|
374
|
+
}
|
|
375
|
+
const history = turns.map(({ input, output }) => ({ input, output }));
|
|
376
|
+
try {
|
|
377
|
+
const output = await this.target.fn(input, { ...context, turn, history });
|
|
378
|
+
turns.push({ input, output });
|
|
379
|
+
}
|
|
380
|
+
catch (error) {
|
|
381
|
+
if (!multiTurn)
|
|
382
|
+
throw error;
|
|
383
|
+
throw new TurnFailure(error, [...turns, { input, output: '', error: errorMessage(error) }], undefined);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
return {
|
|
387
|
+
case: evalCase,
|
|
388
|
+
output: turns[turns.length - 1].output,
|
|
389
|
+
duration: Date.now() - start,
|
|
390
|
+
...(multiTurn && { turns }),
|
|
391
|
+
};
|
|
323
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* Runs the agent on each turn of the case. A case with further `turns` runs them all on one
|
|
395
|
+
* thread of its own, so the Cogitator's memory carries the conversation from turn to turn
|
|
396
|
+
*/
|
|
324
397
|
async executeAgentTarget(evalCase, signal) {
|
|
325
398
|
const start = Date.now();
|
|
326
399
|
const cogitator = this.target.cogitator;
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
const
|
|
333
|
-
const
|
|
400
|
+
const retrieval = this.target.retrieval ?? ragSearchPassages;
|
|
401
|
+
const inputs = turnInputs(evalCase);
|
|
402
|
+
const threadId = inputs.length > 1 ? `eval-${randomUUID()}` : undefined;
|
|
403
|
+
const turns = [];
|
|
404
|
+
const toolCalls = [];
|
|
405
|
+
const toolResults = [];
|
|
406
|
+
const retrieved = [];
|
|
407
|
+
let reportsToolCalls = false;
|
|
408
|
+
let usage;
|
|
409
|
+
const flags = {};
|
|
410
|
+
for (const input of inputs) {
|
|
411
|
+
if (turns.length > 0 && signal.aborted) {
|
|
412
|
+
throw new TurnFailure(signal.reason, [...turns], usage);
|
|
413
|
+
}
|
|
414
|
+
const turnResults = [];
|
|
415
|
+
let runResult;
|
|
416
|
+
try {
|
|
417
|
+
runResult = await cogitator.run(this.target.agent, {
|
|
418
|
+
input,
|
|
419
|
+
context: evalCase.context,
|
|
420
|
+
signal,
|
|
421
|
+
...(threadId !== undefined && { threadId }),
|
|
422
|
+
onToolResult: (toolResult) => turnResults.push(toolResultOf(toolResult)),
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
catch (error) {
|
|
426
|
+
if (threadId === undefined)
|
|
427
|
+
throw error;
|
|
428
|
+
throw new TurnFailure(error, [...turns, { input, output: '', error: errorMessage(error) }], usage);
|
|
429
|
+
}
|
|
430
|
+
const output = typeof runResult.output === 'string' ? runResult.output : '';
|
|
431
|
+
const turnCalls = Array.isArray(runResult.toolCalls)
|
|
432
|
+
? runResult.toolCalls
|
|
433
|
+
: undefined;
|
|
434
|
+
if (turnCalls)
|
|
435
|
+
reportsToolCalls = true;
|
|
436
|
+
toolCalls.push(...(turnCalls ?? []));
|
|
437
|
+
toolResults.push(...turnResults);
|
|
438
|
+
for (const toolResult of turnResults) {
|
|
439
|
+
if (toolResult.error !== undefined)
|
|
440
|
+
continue;
|
|
441
|
+
retrieved.push(...(retrieval({ name: toolResult.name, result: toolResult.result }) ?? []));
|
|
442
|
+
}
|
|
443
|
+
usage = addUsage(usage, targetUsageOf(runResult.usage));
|
|
444
|
+
if (runResult.truncated === true)
|
|
445
|
+
flags.truncated = true;
|
|
446
|
+
if (runResult.iterationLimitReached === true)
|
|
447
|
+
flags.iterationLimitReached = true;
|
|
448
|
+
if (runResult.status === 'paused')
|
|
449
|
+
flags.paused = true;
|
|
450
|
+
turns.push({
|
|
451
|
+
input,
|
|
452
|
+
output,
|
|
453
|
+
...(turnCalls && turnCalls.length > 0 && { toolCalls: turnCalls }),
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
return {
|
|
334
457
|
case: evalCase,
|
|
335
|
-
output:
|
|
336
|
-
duration,
|
|
458
|
+
output: turns[turns.length - 1].output,
|
|
459
|
+
duration: Date.now() - start,
|
|
460
|
+
...(usage && { usage }),
|
|
461
|
+
...(reportsToolCalls && { toolCalls }),
|
|
462
|
+
...(toolResults.length > 0 && { toolResults }),
|
|
463
|
+
...(retrieved.length > 0 && { retrieved }),
|
|
464
|
+
...(threadId !== undefined && { turns }),
|
|
465
|
+
...flags,
|
|
337
466
|
};
|
|
338
|
-
const usage = targetUsageOf(runResult.usage);
|
|
339
|
-
if (usage) {
|
|
340
|
-
result.usage = usage;
|
|
341
|
-
}
|
|
342
|
-
if (runResult.toolCalls) {
|
|
343
|
-
result.toolCalls = runResult.toolCalls;
|
|
344
|
-
}
|
|
345
|
-
if (runResult.truncated === true)
|
|
346
|
-
result.truncated = true;
|
|
347
|
-
if (runResult.iterationLimitReached === true)
|
|
348
|
-
result.iterationLimitReached = true;
|
|
349
|
-
if (runResult.status === 'paused')
|
|
350
|
-
result.paused = true;
|
|
351
|
-
return result;
|
|
352
467
|
}
|
|
353
468
|
async evaluateCaseMetrics(result) {
|
|
354
469
|
if (this.boundMetrics.length === 0)
|
|
@@ -386,23 +501,30 @@ export class EvalSuite {
|
|
|
386
501
|
return { name: metric.metricName, score: 0, details: `metric error: ${error}`, error };
|
|
387
502
|
}
|
|
388
503
|
}
|
|
504
|
+
/** One score per case and metric: the mean over the case's trials the metric scored */
|
|
389
505
|
aggregateScores(results) {
|
|
390
506
|
const scoresByMetric = new Map();
|
|
507
|
+
const push = (map, name, value) => {
|
|
508
|
+
const values = map.get(name);
|
|
509
|
+
if (values)
|
|
510
|
+
values.push(value);
|
|
511
|
+
else
|
|
512
|
+
map.set(name, [value]);
|
|
513
|
+
};
|
|
391
514
|
for (const r of results) {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
scoresByMetric.set(s.name, arr);
|
|
397
|
-
}
|
|
398
|
-
arr.push(s.score);
|
|
515
|
+
const caseScores = new Map();
|
|
516
|
+
for (const trial of trialsOf(r)) {
|
|
517
|
+
for (const s of trial.scores)
|
|
518
|
+
push(caseScores, s.name, s.score);
|
|
399
519
|
}
|
|
520
|
+
for (const [name, values] of caseScores)
|
|
521
|
+
push(scoresByMetric, name, mean(values));
|
|
400
522
|
}
|
|
401
523
|
const aggregated = {};
|
|
402
524
|
for (const [name, values] of scoresByMetric) {
|
|
403
525
|
aggregated[name] = { name, ...aggregate(values), count: values.length };
|
|
404
526
|
}
|
|
405
|
-
if (results.length > 0 && results.every((r) => r.error !== undefined)) {
|
|
527
|
+
if (results.length > 0 && results.flatMap(trialsOf).every((r) => r.error !== undefined)) {
|
|
406
528
|
for (const metric of this.boundMetrics) {
|
|
407
529
|
aggregated[metric.metricName] ??= unscored(metric.metricName);
|
|
408
530
|
}
|
|
@@ -410,4 +532,44 @@ export class EvalSuite {
|
|
|
410
532
|
return aggregated;
|
|
411
533
|
}
|
|
412
534
|
}
|
|
535
|
+
function trialsOf(result) {
|
|
536
|
+
return result.trials ?? [result];
|
|
537
|
+
}
|
|
538
|
+
function toolResultOf(toolResult) {
|
|
539
|
+
return {
|
|
540
|
+
callId: toolResult.callId,
|
|
541
|
+
name: toolResult.name,
|
|
542
|
+
...(toolResult.result !== undefined && { result: toolResult.result }),
|
|
543
|
+
...(toolResult.error !== undefined && { error: toolResult.error }),
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
/** The turns and usage a failed attempt leaves, when a turn of a multi-turn case failed */
|
|
547
|
+
function failureDetails(settled) {
|
|
548
|
+
if (settled.ok || !(settled.error instanceof TurnFailure))
|
|
549
|
+
return settled;
|
|
550
|
+
const { turns, usage } = settled.error;
|
|
551
|
+
return { ok: false, error: settled.error, turns, ...(usage && { usage }) };
|
|
552
|
+
}
|
|
553
|
+
/** A case result as the reporters get it, the agent's data and the trials kept */
|
|
554
|
+
/** The trial a case is reported by: the first that answered, the first when every trial errored */
|
|
555
|
+
function leadTrial(trials) {
|
|
556
|
+
return trials.find((trial) => trial.error === undefined) ?? trials[0];
|
|
557
|
+
}
|
|
558
|
+
function reportedCase(r) {
|
|
559
|
+
return {
|
|
560
|
+
case: { input: r.case.input, expected: r.case.expected },
|
|
561
|
+
output: r.output,
|
|
562
|
+
duration: r.duration,
|
|
563
|
+
...(r.error !== undefined && { error: r.error }),
|
|
564
|
+
...(r.truncated && { truncated: true }),
|
|
565
|
+
...(r.iterationLimitReached && { iterationLimitReached: true }),
|
|
566
|
+
...(r.paused && { paused: true }),
|
|
567
|
+
...(r.toolCalls && { toolCalls: r.toolCalls }),
|
|
568
|
+
...(r.toolResults && { toolResults: r.toolResults }),
|
|
569
|
+
...(r.retrieved && { retrieved: r.retrieved }),
|
|
570
|
+
...(r.turns && { turns: r.turns }),
|
|
571
|
+
scores: r.scores,
|
|
572
|
+
...(r.trials && { trials: r.trials.map(reportedCase) }),
|
|
573
|
+
};
|
|
574
|
+
}
|
|
413
575
|
//# sourceMappingURL=eval-suite.js.map
|