@cogitator-ai/evals 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/assertions/index.d.ts +2 -0
- package/dist/assertions/index.d.ts.map +1 -1
- package/dist/assertions/index.js.map +1 -1
- package/dist/eval-suite.d.ts.map +1 -1
- package/dist/eval-suite.js +12 -11
- package/dist/eval-suite.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metrics/index.d.ts +1 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/llm-judge.d.ts +6 -1
- package/dist/metrics/llm-judge.d.ts.map +1 -1
- package/dist/metrics/llm-judge.js +53 -11
- package/dist/metrics/llm-judge.js.map +1 -1
- package/dist/metrics/statistical.d.ts +3 -0
- package/dist/metrics/statistical.d.ts.map +1 -1
- package/dist/metrics/statistical.js +21 -28
- package/dist/metrics/statistical.js.map +1 -1
- package/dist/metrics/types.d.ts +8 -1
- package/dist/metrics/types.d.ts.map +1 -1
- package/dist/reporters/index.d.ts +12 -0
- package/dist/reporters/index.d.ts.map +1 -1
- package/dist/reporters/index.js +20 -3
- package/dist/reporters/index.js.map +1 -1
- package/dist/schema.d.ts +11 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +7 -0
- package/dist/schema.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -164,7 +164,7 @@ const suite = new EvalSuite({
|
|
|
164
164
|
prompt: 'Rate how technically accurate the response is for a software engineering audience.',
|
|
165
165
|
}),
|
|
166
166
|
],
|
|
167
|
-
judge: { model: 'gpt-6.1-sol', temperature: 0 },
|
|
167
|
+
judge: { model: 'openai/gpt-6.1-sol', temperature: 0, cogitator },
|
|
168
168
|
});
|
|
169
169
|
```
|
|
170
170
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,KACrD,eAAe,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AA2BA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
package/dist/eval-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGnF,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IAuClC,OAAO,CAAC,cAAc;IAuBhB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;YAiGvB,WAAW;YAmBX,kBAAkB;YAoBlB,eAAe;YAMf,kBAAkB;YA+BlB,mBAAmB;IAKjC,OAAO,CAAC,eAAe;CAuBxB"}
|
package/dist/eval-suite.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
2
|
import { EvalSuiteConfigSchema } from './schema.js';
|
|
3
|
-
import { bindJudgeContext } from './metrics/llm-judge.js';
|
|
3
|
+
import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
|
|
4
4
|
import { aggregate } from './stats/index.js';
|
|
5
5
|
import { report } from './reporters/index.js';
|
|
6
6
|
export function isLLMMetric(m) {
|
|
@@ -36,15 +36,12 @@ export class EvalSuite {
|
|
|
36
36
|
if (hasLLMMetrics && !opts.judge) {
|
|
37
37
|
throw new Error('LLM metrics require a judge config');
|
|
38
38
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
return m;
|
|
47
|
-
});
|
|
39
|
+
const judgeCogitator = opts.judge?.cogitator ?? this.target.cogitator;
|
|
40
|
+
if (hasLLMMetrics && !judgeCogitator) {
|
|
41
|
+
throw new Error('LLM metrics need a Cogitator to run the judge: pass judge.cogitator, or use an agent target');
|
|
42
|
+
}
|
|
43
|
+
const judge = opts.judge && judgeCogitator ? judgeContextFor(judgeCogitator, opts.judge) : null;
|
|
44
|
+
this.boundMetrics = rawMetrics.map((m) => isLLMMetric(m) && judge ? bindJudgeContext(m, judge) : m);
|
|
48
45
|
}
|
|
49
46
|
validateTarget() {
|
|
50
47
|
const { fn, agent, cogitator } = this.target;
|
|
@@ -105,7 +102,11 @@ export class EvalSuite {
|
|
|
105
102
|
const aggregated = this.aggregateScores(orderedResults);
|
|
106
103
|
for (const statMetric of this.statisticalMetrics) {
|
|
107
104
|
const score = statMetric(orderedResults);
|
|
108
|
-
aggregated[score.name] = {
|
|
105
|
+
aggregated[score.name] = {
|
|
106
|
+
name: score.name,
|
|
107
|
+
...aggregate(score.values ?? [score.score]),
|
|
108
|
+
...(score.metadata && { metadata: score.metadata }),
|
|
109
|
+
};
|
|
109
110
|
}
|
|
110
111
|
const totalCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
|
|
111
112
|
const suiteDuration = Date.now() - suiteStart;
|
package/dist/eval-suite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAqCrC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,cAAc,GAClB,IAAI,CAAC,KAAK,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QACjF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhG,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,KAAK,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;SAC1E,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,KAAgD,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,KAAK,GAAG,UAAU,CAChB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EAC5D,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeCo
|
|
|
12
12
|
export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
|
|
13
13
|
export { latency, cost, tokenUsage } from './metrics/statistical.js';
|
|
14
14
|
export { metric } from './metrics/custom.js';
|
|
15
|
-
export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
|
|
15
|
+
export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
|
|
16
16
|
export type { CustomMetricConfig } from './metrics/custom.js';
|
|
17
17
|
export { threshold } from './assertions/index.js';
|
|
18
18
|
export { noRegression } from './assertions/index.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAE7F,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAU1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
|
package/dist/metrics/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn } from './types.js';
|
|
1
|
+
export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
|
|
2
2
|
export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
|
|
3
3
|
export { metric } from './custom.js';
|
|
4
4
|
export type { CustomMetricConfig } from './custom.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { JudgeConfig } from '../schema.js';
|
|
1
|
+
import type { JudgeCogitator, JudgeConfig } from '../schema.js';
|
|
2
2
|
import type { MetricFn } from './types.js';
|
|
3
3
|
export interface JudgeContext {
|
|
4
4
|
cogitator: {
|
|
@@ -15,6 +15,11 @@ export interface LLMMetricFn extends MetricFn {
|
|
|
15
15
|
readonly __judgeSystemPrompt: string;
|
|
16
16
|
readonly __judgeName: string;
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
|
|
20
|
+
* asking for a `{ score, reasoning }` JSON answer.
|
|
21
|
+
*/
|
|
22
|
+
export declare function judgeContextFor(cogitator: JudgeCogitator, judgeConfig: JudgeConfig): JudgeContext;
|
|
18
23
|
export declare function bindJudgeContext(metric: LLMMetricFn, context: JudgeContext): MetricFn;
|
|
19
24
|
export declare function faithfulness(): LLMMetricFn;
|
|
20
25
|
export declare function relevance(): LLMMetricFn;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAA+B,MAAM,SAAS,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC7E,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAiCD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAiCjG;AAoBD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CAmCrF;AAED,wBAAgB,YAAY,IAAI,WAAW,CAK1C;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,WAAW,IAAI,WAAW,CAKzC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
|
|
@@ -1,20 +1,62 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
1
2
|
const SCORE_REGEX = /\b(0(?:\.\d+)?|1(?:\.0+)?)\b/;
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
}
|
|
3
|
+
const JSON_OBJECT = /\{[\s\S]*\}/;
|
|
4
|
+
const JudgeVerdict = z.object({ score: z.number(), reasoning: z.string() });
|
|
5
|
+
const JUDGE_INSTRUCTIONS = 'You are an impartial evaluator. Score the response with the rubric in the request. Reply with JSON: {"score": <number from 0 to 1>, "reasoning": "<explanation>"}.';
|
|
6
|
+
function verdictOf(value) {
|
|
7
|
+
if (typeof value !== 'object' || value === null)
|
|
8
8
|
return null;
|
|
9
|
+
const { score, reasoning } = value;
|
|
10
|
+
if (typeof score !== 'number' || !Number.isFinite(score))
|
|
11
|
+
return null;
|
|
12
|
+
return { score, ...(typeof reasoning === 'string' && { reasoning }) };
|
|
13
|
+
}
|
|
14
|
+
function parseJson(text) {
|
|
15
|
+
if (text === undefined)
|
|
16
|
+
return { ok: false };
|
|
17
|
+
try {
|
|
18
|
+
return { ok: true, value: JSON.parse(text) };
|
|
9
19
|
}
|
|
10
20
|
catch {
|
|
11
|
-
|
|
12
|
-
if (match) {
|
|
13
|
-
return { score: parseFloat(match[1]) };
|
|
14
|
-
}
|
|
15
|
-
return null;
|
|
21
|
+
return { ok: false };
|
|
16
22
|
}
|
|
17
23
|
}
|
|
24
|
+
function parseJudgeOutput(raw) {
|
|
25
|
+
const json = [raw, JSON_OBJECT.exec(raw)?.[0]].map(parseJson).find((result) => result.ok);
|
|
26
|
+
if (json?.ok)
|
|
27
|
+
return verdictOf(json.value);
|
|
28
|
+
const match = SCORE_REGEX.exec(raw);
|
|
29
|
+
return match ? { score: parseFloat(match[1]) } : null;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
|
|
33
|
+
* asking for a `{ score, reasoning }` JSON answer.
|
|
34
|
+
*/
|
|
35
|
+
export function judgeContextFor(cogitator, judgeConfig) {
|
|
36
|
+
let judgeAgent;
|
|
37
|
+
const agent = () => (judgeAgent ??= import('@cogitator-ai/core').then(({ Agent }) => new Agent({
|
|
38
|
+
name: 'eval-judge',
|
|
39
|
+
model: judgeConfig.model,
|
|
40
|
+
instructions: JUDGE_INSTRUCTIONS,
|
|
41
|
+
temperature: judgeConfig.temperature,
|
|
42
|
+
...(judgeConfig.maxTokens !== undefined && { maxTokens: judgeConfig.maxTokens }),
|
|
43
|
+
maxIterations: 1,
|
|
44
|
+
responseFormat: { type: 'json_schema', schema: JudgeVerdict },
|
|
45
|
+
}), (error) => {
|
|
46
|
+
throw new Error(`The LLM judge needs @cogitator-ai/core: ${error instanceof Error ? error.message : String(error)}`);
|
|
47
|
+
}));
|
|
48
|
+
return {
|
|
49
|
+
judgeConfig,
|
|
50
|
+
cogitator: {
|
|
51
|
+
run: async ({ input }) => {
|
|
52
|
+
const result = await cogitator.run(await agent(), { input, useMemory: false });
|
|
53
|
+
return {
|
|
54
|
+
output: result.structured !== undefined ? JSON.stringify(result.structured) : result.output,
|
|
55
|
+
};
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
18
60
|
function createJudgeMetric(name, systemPrompt) {
|
|
19
61
|
const fn = (async (_result) => {
|
|
20
62
|
return { name, score: 0, details: 'unbound judge metric — call bindJudgeContext first' };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,IAAwB;IACzC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,EAAE,EAAE;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,OAAO;QACL,WAAW;QACX,SAAS,EAAE;YACT,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/E,OAAO;oBACL,MAAM,EACJ,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;iBACtF,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;aACrE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAiB,GAAa,CAAC,OAAO,EAAE;aAClD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,iBAAiB,CACtB,cAAc,EACd,sUAAsU,CACvU,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,iBAAiB,CACtB,aAAa,EACb,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { StatisticalMetricFn } from './types.js';
|
|
2
|
+
/** Run duration per case, in milliseconds. */
|
|
2
3
|
export declare function latency(): StatisticalMetricFn;
|
|
4
|
+
/** Cost per case, in USD, for cases that report usage. */
|
|
3
5
|
export declare function cost(): StatisticalMetricFn;
|
|
6
|
+
/** Total tokens per case, for cases that report usage. */
|
|
4
7
|
export declare function tokenUsage(): StatisticalMetricFn;
|
|
5
8
|
//# sourceMappingURL=statistical.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBrF,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD"}
|
|
@@ -4,13 +4,21 @@ function createStatisticalFn(name, fn) {
|
|
|
4
4
|
statFn.metricName = name;
|
|
5
5
|
return statFn;
|
|
6
6
|
}
|
|
7
|
+
function sum(values) {
|
|
8
|
+
let total = 0;
|
|
9
|
+
for (const value of values)
|
|
10
|
+
total += value;
|
|
11
|
+
return total;
|
|
12
|
+
}
|
|
13
|
+
/** Run duration per case, in milliseconds. */
|
|
7
14
|
export function latency() {
|
|
8
15
|
return createStatisticalFn('latency', (results) => {
|
|
9
16
|
const durations = results.map((r) => r.duration);
|
|
10
17
|
const stats = aggregate(durations);
|
|
11
18
|
return {
|
|
12
19
|
name: 'latency',
|
|
13
|
-
score:
|
|
20
|
+
score: stats.mean,
|
|
21
|
+
values: durations,
|
|
14
22
|
metadata: {
|
|
15
23
|
p50: stats.p50,
|
|
16
24
|
p95: stats.p95,
|
|
@@ -23,55 +31,40 @@ export function latency() {
|
|
|
23
31
|
};
|
|
24
32
|
});
|
|
25
33
|
}
|
|
34
|
+
/** Cost per case, in USD, for cases that report usage. */
|
|
26
35
|
export function cost() {
|
|
27
36
|
return createStatisticalFn('cost', (results) => {
|
|
28
37
|
const costs = results.filter((r) => r.usage).map((r) => r.usage.cost);
|
|
29
|
-
if (costs.length === 0) {
|
|
30
|
-
return {
|
|
31
|
-
name: 'cost',
|
|
32
|
-
score: 0,
|
|
33
|
-
metadata: { total: 0, mean: 0, median: 0, min: 0, max: 0 },
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
38
|
const stats = aggregate(costs);
|
|
37
|
-
let total = 0;
|
|
38
|
-
for (const c of costs) {
|
|
39
|
-
total += c;
|
|
40
|
-
}
|
|
41
39
|
return {
|
|
42
40
|
name: 'cost',
|
|
43
|
-
score:
|
|
41
|
+
score: stats.mean,
|
|
42
|
+
values: costs,
|
|
44
43
|
metadata: {
|
|
45
|
-
total,
|
|
44
|
+
total: sum(costs),
|
|
46
45
|
mean: stats.mean,
|
|
47
46
|
median: stats.median,
|
|
48
47
|
min: stats.min,
|
|
49
48
|
max: stats.max,
|
|
49
|
+
p95: stats.p95,
|
|
50
|
+
p99: stats.p99,
|
|
50
51
|
},
|
|
51
52
|
};
|
|
52
53
|
});
|
|
53
54
|
}
|
|
55
|
+
/** Total tokens per case, for cases that report usage. */
|
|
54
56
|
export function tokenUsage() {
|
|
55
57
|
return createStatisticalFn('tokenUsage', (results) => {
|
|
56
58
|
const withUsage = results.filter((r) => r.usage);
|
|
57
|
-
if (withUsage.length === 0) {
|
|
58
|
-
return {
|
|
59
|
-
name: 'tokenUsage',
|
|
60
|
-
score: 0,
|
|
61
|
-
metadata: { totalInput: 0, totalOutput: 0, totalTokens: 0, meanInput: 0, meanOutput: 0 },
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
59
|
const inputTokens = withUsage.map((r) => r.usage.inputTokens);
|
|
65
60
|
const outputTokens = withUsage.map((r) => r.usage.outputTokens);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
totalInput += t;
|
|
70
|
-
for (const t of outputTokens)
|
|
71
|
-
totalOutput += t;
|
|
61
|
+
const totals = withUsage.map((r) => r.usage.inputTokens + r.usage.outputTokens);
|
|
62
|
+
const totalInput = sum(inputTokens);
|
|
63
|
+
const totalOutput = sum(outputTokens);
|
|
72
64
|
return {
|
|
73
65
|
name: 'tokenUsage',
|
|
74
|
-
score:
|
|
66
|
+
score: mean(totals),
|
|
67
|
+
values: totals,
|
|
75
68
|
metadata: {
|
|
76
69
|
totalInput,
|
|
77
70
|
totalOutput,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,
|
|
1
|
+
{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/metrics/types.d.ts
CHANGED
|
@@ -27,7 +27,14 @@ export interface EvalCaseResult {
|
|
|
27
27
|
export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {
|
|
28
28
|
metricName: string;
|
|
29
29
|
};
|
|
30
|
-
export
|
|
30
|
+
export interface StatisticalScore extends MetricScore {
|
|
31
|
+
/**
|
|
32
|
+
* One value per case; the suite aggregates them into mean, percentiles and so
|
|
33
|
+
* on. Without it the suite aggregates `score` alone.
|
|
34
|
+
*/
|
|
35
|
+
values?: number[];
|
|
36
|
+
}
|
|
37
|
+
export type StatisticalMetricFn = ((results: EvalCaseResult[]) => StatisticalScore) & {
|
|
31
38
|
metricName: string;
|
|
32
39
|
};
|
|
33
40
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC"}
|
|
@@ -8,6 +8,8 @@ export interface AggregatedMetric {
|
|
|
8
8
|
p50: number;
|
|
9
9
|
p95: number;
|
|
10
10
|
p99: number;
|
|
11
|
+
/** Extra figures a statistical metric reports, such as the total cost */
|
|
12
|
+
metadata?: Record<string, unknown>;
|
|
11
13
|
}
|
|
12
14
|
export interface AssertionResult {
|
|
13
15
|
name: string;
|
|
@@ -40,8 +42,18 @@ export interface EvalSuiteResult {
|
|
|
40
42
|
}
|
|
41
43
|
export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
|
|
42
44
|
export type ReporterOptions = {
|
|
45
|
+
/**
|
|
46
|
+
* Base path of file reports. Each file reporter writes it with its own
|
|
47
|
+
* extension, so `./reports/eval` gives `eval.json` and `eval.csv`.
|
|
48
|
+
*/
|
|
43
49
|
path?: string;
|
|
44
50
|
};
|
|
51
|
+
/** The file a reporter writes: the base path with the reporter's extension. */
|
|
52
|
+
export declare function reportPath(base: string | undefined, extension: 'json' | 'csv'): string;
|
|
53
|
+
/**
|
|
54
|
+
* Runs the reporters in the order given, except `ci`, which runs last because
|
|
55
|
+
* it ends the process when an assertion failed.
|
|
56
|
+
*/
|
|
45
57
|
export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
|
|
46
58
|
export { consoleReport } from './console.js';
|
|
47
59
|
export { jsonReport } from './json.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClE,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAGtF;AAOD;;;GAGG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAoBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
|
package/dist/reporters/index.js
CHANGED
|
@@ -1,19 +1,36 @@
|
|
|
1
|
+
import { mkdirSync } from 'node:fs';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
1
3
|
import { consoleReport } from './console.js';
|
|
2
4
|
import { jsonReport } from './json.js';
|
|
3
5
|
import { csvReport } from './csv.js';
|
|
4
6
|
import { ciReport } from './ci.js';
|
|
7
|
+
const DEFAULT_BASE = 'eval-report';
|
|
8
|
+
/** The file a reporter writes: the base path with the reporter's extension. */
|
|
9
|
+
export function reportPath(base, extension) {
|
|
10
|
+
const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv)$/i, '');
|
|
11
|
+
return `${stem}.${extension}`;
|
|
12
|
+
}
|
|
13
|
+
function prepare(path) {
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
return path;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Runs the reporters in the order given, except `ci`, which runs last because
|
|
19
|
+
* it ends the process when an assertion failed.
|
|
20
|
+
*/
|
|
5
21
|
export function report(result, type, options) {
|
|
6
22
|
const types = Array.isArray(type) ? type : [type];
|
|
7
|
-
|
|
23
|
+
const ordered = [...types.filter((t) => t !== 'ci'), ...types.filter((t) => t === 'ci')];
|
|
24
|
+
for (const t of ordered) {
|
|
8
25
|
switch (t) {
|
|
9
26
|
case 'console':
|
|
10
27
|
consoleReport(result);
|
|
11
28
|
break;
|
|
12
29
|
case 'json':
|
|
13
|
-
jsonReport(result, { path: options?.path
|
|
30
|
+
jsonReport(result, { path: prepare(reportPath(options?.path, 'json')) });
|
|
14
31
|
break;
|
|
15
32
|
case 'csv':
|
|
16
|
-
csvReport(result, { path: options?.path
|
|
33
|
+
csvReport(result, { path: prepare(reportPath(options?.path, 'csv')) });
|
|
17
34
|
break;
|
|
18
35
|
case 'ci':
|
|
19
36
|
ciReport(result);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AA6ChC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAyB;IAC5E,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
|
package/dist/schema.d.ts
CHANGED
|
@@ -10,10 +10,21 @@ export declare const EvalSuiteConfigSchema: z.ZodObject<{
|
|
|
10
10
|
timeout: z.ZodDefault<z.ZodNumber>;
|
|
11
11
|
retries: z.ZodDefault<z.ZodNumber>;
|
|
12
12
|
}, z.core.$strip>;
|
|
13
|
+
/** The part of a Cogitator runtime the judge needs: running an agent. */
|
|
14
|
+
export interface JudgeCogitator {
|
|
15
|
+
run(agent: unknown, options: {
|
|
16
|
+
input: string;
|
|
17
|
+
useMemory?: boolean;
|
|
18
|
+
}): Promise<{
|
|
19
|
+
output: string;
|
|
20
|
+
structured?: unknown;
|
|
21
|
+
}>;
|
|
22
|
+
}
|
|
13
23
|
export declare const JudgeConfigSchema: z.ZodObject<{
|
|
14
24
|
model: z.ZodString;
|
|
15
25
|
temperature: z.ZodDefault<z.ZodNumber>;
|
|
16
26
|
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
27
|
+
cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
|
|
17
28
|
}, z.core.$strip>;
|
|
18
29
|
export declare const EvalComparisonConfigSchema: z.ZodObject<{
|
|
19
30
|
concurrency: z.ZodDefault<z.ZodNumber>;
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,eAAO,MAAM,iBAAiB
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACtD;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
|
package/dist/schema.js
CHANGED
|
@@ -10,10 +10,17 @@ export const EvalSuiteConfigSchema = z.object({
|
|
|
10
10
|
timeout: z.number().int().min(1000).default(30000),
|
|
11
11
|
retries: z.number().int().min(0).max(10).default(0),
|
|
12
12
|
});
|
|
13
|
+
function isJudgeCogitator(value) {
|
|
14
|
+
return (typeof value === 'object' &&
|
|
15
|
+
value !== null &&
|
|
16
|
+
typeof value.run === 'function');
|
|
17
|
+
}
|
|
13
18
|
export const JudgeConfigSchema = z.object({
|
|
14
19
|
model: z.string(),
|
|
15
20
|
temperature: z.number().default(0),
|
|
16
21
|
maxTokens: z.number().int().positive().optional(),
|
|
22
|
+
/** Runtime that runs the judge; defaults to the `cogitator` of an agent target */
|
|
23
|
+
cogitator: z.custom(isJudgeCogitator).optional(),
|
|
17
24
|
});
|
|
18
25
|
export const EvalComparisonConfigSchema = z.object({
|
|
19
26
|
concurrency: z.number().int().min(1).default(5),
|
package/dist/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAUH,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CACjE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Evaluation framework for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"zod": "^4.6.5"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@cogitator-ai/core": ">=0.
|
|
24
|
+
"@cogitator-ai/core": ">=0.22.0"
|
|
25
25
|
},
|
|
26
26
|
"peerDependenciesMeta": {
|
|
27
27
|
"@cogitator-ai/core": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"papaparse": "^5.7.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
-
"@cogitator-ai/core": "0.
|
|
35
|
+
"@cogitator-ai/core": "0.22.0",
|
|
36
36
|
"@types/papaparse": "^5.3.15",
|
|
37
37
|
"typescript": "^6.0.3",
|
|
38
38
|
"vitest": "^5.0.3"
|