@zukhruf/evals 0.8.7 → 0.8.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -37
- package/dist/classification/all-class-matches.d.ts +1 -1
- package/dist/classification/all-class-matches.d.ts.map +1 -1
- package/dist/classification/all-class-matches.js +8 -4
- package/dist/classification/all-class-matches.js.map +1 -1
- package/dist/classification/average.d.ts +1 -2
- package/dist/classification/average.d.ts.map +1 -1
- package/dist/classification/binary-average.d.ts +2 -2
- package/dist/classification/binary-average.d.ts.map +1 -1
- package/dist/classification/binary-average.js +13 -12
- package/dist/classification/binary-average.js.map +1 -1
- package/dist/classification/class-counts.d.ts +5 -4
- package/dist/classification/class-counts.d.ts.map +1 -1
- package/dist/classification/class-counts.js +41 -33
- package/dist/classification/class-counts.js.map +1 -1
- package/dist/classification/label-comparer.d.ts +4 -0
- package/dist/classification/label-comparer.d.ts.map +1 -1
- package/dist/classification/label-comparer.js +10 -2
- package/dist/classification/label-comparer.js.map +1 -1
- package/dist/classification/label-comparison.d.ts +4 -2
- package/dist/classification/label-comparison.d.ts.map +1 -1
- package/dist/classification/label-comparison.js +11 -6
- package/dist/classification/label-comparison.js.map +1 -1
- package/dist/classification/macro-average.d.ts +1 -1
- package/dist/classification/macro-average.d.ts.map +1 -1
- package/dist/classification/macro-average.js +3 -3
- package/dist/classification/macro-average.js.map +1 -1
- package/dist/classification/micro-average.d.ts +1 -1
- package/dist/classification/micro-average.d.ts.map +1 -1
- package/dist/classification/micro-average.js +4 -4
- package/dist/classification/micro-average.js.map +1 -1
- package/dist/classification/per-class.d.ts +0 -6
- package/dist/classification/per-class.d.ts.map +1 -1
- package/dist/classification/per-class.js +4 -10
- package/dist/classification/per-class.js.map +1 -1
- package/dist/classification/weighted-average.d.ts +1 -1
- package/dist/classification/weighted-average.d.ts.map +1 -1
- package/dist/classification/weighted-average.js +3 -3
- package/dist/classification/weighted-average.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/failed-call.d.ts +13 -0
- package/dist/promptfoo/failed-call.d.ts.map +1 -0
- package/dist/promptfoo/failed-call.js +24 -0
- package/dist/promptfoo/failed-call.js.map +1 -0
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/scoring/grouped.d.ts +3 -3
- package/dist/scoring/grouped.d.ts.map +1 -1
- package/dist/scoring/grouped.js +7 -4
- package/dist/scoring/grouped.js.map +1 -1
- package/dist/scoring/paired-bootstrap.d.ts +24 -0
- package/dist/scoring/paired-bootstrap.d.ts.map +1 -0
- package/dist/scoring/paired-bootstrap.js +94 -0
- package/dist/scoring/paired-bootstrap.js.map +1 -0
- package/dist/scoring/run.d.ts +7 -0
- package/dist/scoring/run.d.ts.map +1 -0
- package/dist/scoring/run.js +19 -0
- package/dist/scoring/run.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +1 -1
- package/dist/scoring/scorer-set.d.ts.map +1 -1
- package/dist/scoring/scorer-set.js.map +1 -1
- package/dist/scoring/test-epochs.d.ts +11 -0
- package/dist/scoring/test-epochs.d.ts.map +1 -0
- package/dist/scoring/test-epochs.js +25 -0
- package/dist/scoring/test-epochs.js.map +1 -0
- package/package.json +5 -4
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { DifferenceInterval, IidBootstrap, } from '@zukhruf/statistics';
|
|
2
|
+
import { RatioOfSums } from "../aggregation/ratio-of-sums.js";
|
|
3
|
+
import { runTests } from "./run.js";
|
|
4
|
+
// The paired bootstrap of two runs on the same tests (Koehn 2004): each
|
|
5
|
+
// resample draws as many tests as there are, with replacement, and gives the
|
|
6
|
+
// ratio of sums of the name over the change minus the same over the baseline,
|
|
7
|
+
// on the same tests. The verdict reads the interval of these differences
|
|
8
|
+
// against 0, in the direction of the name (ADR 0004). The generator keeps its
|
|
9
|
+
// state between calls, so the order of the calls is the order of the draws.
|
|
10
|
+
export class PairedBootstrap {
|
|
11
|
+
#bootstrap;
|
|
12
|
+
#interval;
|
|
13
|
+
#nResamples;
|
|
14
|
+
constructor({ nResamples, randomIndex, interval, }) {
|
|
15
|
+
this.#bootstrap = new IidBootstrap({ nResamples, randomIndex });
|
|
16
|
+
this.#interval = interval;
|
|
17
|
+
this.#nResamples = nResamples;
|
|
18
|
+
}
|
|
19
|
+
verdict(baseline, change, name, { higherIsBetter }) {
|
|
20
|
+
const pairs = PairedBootstrap.#pairs(baseline, change, name);
|
|
21
|
+
const distribution = this.#bootstrap.distribution(pairs, PairedBootstrap.#difference);
|
|
22
|
+
const interval = this.#interval.of(distribution);
|
|
23
|
+
const difference = new DifferenceInterval(interval.low, interval.high, PairedBootstrap.#difference(pairs));
|
|
24
|
+
const direction = higherIsBetter ? 1 : -1;
|
|
25
|
+
const signs = pairs.map((pair) => Math.sign((pair.change.value - pair.baseline.value) * direction));
|
|
26
|
+
return {
|
|
27
|
+
name,
|
|
28
|
+
result: PairedBootstrap.#result(difference, higherIsBetter),
|
|
29
|
+
difference,
|
|
30
|
+
p: this.#p(distribution),
|
|
31
|
+
improvements: signs.filter((sign) => sign > 0).length,
|
|
32
|
+
regressions: signs.filter((sign) => sign < 0).length,
|
|
33
|
+
ties: signs.filter((sign) => sign === 0).length,
|
|
34
|
+
tests: pairs.length,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
// Two times the smaller share of the differences at or below 0 and at or
|
|
38
|
+
// above 0, at most 1, as mteb's pairedBootstrapTest gives it.
|
|
39
|
+
#p(distribution) {
|
|
40
|
+
const share = (side) => distribution.filter(side).length / this.#nResamples;
|
|
41
|
+
return Math.min(1, 2 *
|
|
42
|
+
Math.min(share((value) => value <= 0), share((value) => value >= 0)));
|
|
43
|
+
}
|
|
44
|
+
static #result(difference, higherIsBetter) {
|
|
45
|
+
const [better, worse] = higherIsBetter
|
|
46
|
+
? [difference.isAbove(0), difference.isBelow(0)]
|
|
47
|
+
: [difference.isBelow(0), difference.isAbove(0)];
|
|
48
|
+
if (better)
|
|
49
|
+
return 'better';
|
|
50
|
+
if (worse)
|
|
51
|
+
return 'worse';
|
|
52
|
+
return 'inconclusive';
|
|
53
|
+
}
|
|
54
|
+
static #difference(pairs) {
|
|
55
|
+
return (RatioOfSums.of(pairs.map((pair) => pair.change.times(pair.weight)))
|
|
56
|
+
.value -
|
|
57
|
+
RatioOfSums.of(pairs.map((pair) => pair.baseline.times(pair.weight)))
|
|
58
|
+
.value);
|
|
59
|
+
}
|
|
60
|
+
// The tests of the two runs, paired by test id in the order of the
|
|
61
|
+
// baseline's first adds. The pairing refuses a test id that one run does
|
|
62
|
+
// not have, as mteb's pairedBootstrapTest refuses (strike 2,
|
|
63
|
+
// docs/copied-code.md).
|
|
64
|
+
static #pairs(baseline, change, name) {
|
|
65
|
+
const baselineTests = runTests(baseline);
|
|
66
|
+
const changeTests = runTests(change);
|
|
67
|
+
const unpaired = [...baselineTests.keys(), ...changeTests.keys()].filter((id) => !(baselineTests.has(id) && changeTests.has(id)));
|
|
68
|
+
if (unpaired.length > 0)
|
|
69
|
+
throw new RangeError(`A paired bootstrap needs the same test ids in the baseline and the change. Not in both: ${unpaired.join(', ')}`);
|
|
70
|
+
const joined = [...baselineTests].flatMap(([id, test]) => {
|
|
71
|
+
const changeTest = changeTests.get(id);
|
|
72
|
+
return changeTest === undefined
|
|
73
|
+
? []
|
|
74
|
+
: [{ id, baseline: test, change: changeTest }];
|
|
75
|
+
});
|
|
76
|
+
const reweighted = joined.filter((test) => test.baseline.weight !== test.change.weight);
|
|
77
|
+
if (reweighted.length > 0)
|
|
78
|
+
throw new RangeError(`A test has the same weight in the baseline and the change. Other weights: ${reweighted
|
|
79
|
+
.map((test) => `${test.id} (${test.baseline.weight} and ${test.change.weight})`)
|
|
80
|
+
.join(', ')}`);
|
|
81
|
+
for (const [run, tests] of [
|
|
82
|
+
['baseline', baselineTests],
|
|
83
|
+
['change', changeTests],
|
|
84
|
+
])
|
|
85
|
+
if (![...tests.values()].some((test) => test.gives(name)))
|
|
86
|
+
throw new Error(`No scorecard of the ${run} run gave a ratio named "${name}"`);
|
|
87
|
+
return joined.map((test) => ({
|
|
88
|
+
baseline: test.baseline.mean(name),
|
|
89
|
+
change: test.change.mean(name),
|
|
90
|
+
weight: test.baseline.weight,
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=paired-bootstrap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paired-bootstrap.js","sourceRoot":"","sources":["../../src/scoring/paired-bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,kBAAkB,EAClB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAY,QAAQ,EAAE,MAAM,UAAU,CAAC;AAuB9C,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,4EAA4E;AAC5E,MAAM,OAAO,eAAe;IACjB,UAAU,CAAe;IACzB,SAAS,CAA2B;IACpC,WAAW,CAAS;IAE7B,YAAY,EACV,UAAU,EACV,WAAW,EACX,QAAQ,GAKT;QACC,IAAI,CAAC,UAAU,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,OAAO,CACL,QAAa,EACb,MAAW,EACX,IAAY,EACZ,EAAE,cAAc,EAAwC;QAExD,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAC/C,KAAK,EACL,eAAe,CAAC,WAAW,CAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,kBAAkB,CACvC,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,IAAI,EACb,eAAe,CAAC,WAAW,CAAC,KAAK,CAAC,CACnC,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,CACjE,CAAC;QACF,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE,cAAc,CAAC;YAC3D,UAAU;YACV,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC;YACxB,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM;YACrD,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM;YACpD,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM;YAC/C,KAAK,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,8DAA8D;IAC9D,EAAE,CAAC,YAA+B;QAChC,MAAM,KAAK,GAAG,CAAC,IAAgC,EAAE,EAAE,CACjD,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;QACtD,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,CAAC;YACC,IAAI,CAAC,GAAG,CACN,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,EAC5B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAC7B,CACJ,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,OAAO,CACZ,UAA8B,EAC9B,cAAuB;QAEvB,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,cAAc;YACpC,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,MAAM;YAAE,OAAO,QAAQ,CAAC;QAC5B,IAAI,KAAK;YAAE,OAAO,OAAO,CAAC;QAC1B,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,KAAsB;QACvC,OAAO,CACL,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;aAChE,KAAK;YACR,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;iBAClE,KAAK,CACT,CAAC;IACJ,CAAC;IAED,mEAAmE;IACnE,yEAAyE;IACzE,6DAA6D;IAC7D,wBAAwB;IACxB,MAAM,CAAC,MAAM,CAAC,QAAa,EAAE,MAAW,EAAE,IAAY;QACpD,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CACtE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACxD,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YACrB,MAAM,IAAI,UAAU,CAClB,2FAA2F,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjH,CAAC;QACJ,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;YACvD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,OAAO,UAAU,KAAK,SAAS;gBAC7B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAC9B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,CACtD,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YACvB,MAAM,IAAI,UAAU,CAClB,6EAA6E,UAAU;iBACpF,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CACnE;iBACA,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI;YACzB,CAAC,UAAU,EAAE,aAAa,CAAC;YAC3B,CAAC,QAAQ,EAAE,WAAW,CAAC;SACf;YACR,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,uBAAuB,GAAG,4BAA4B,IAAI,GAAG,CAC9D,CAAC;QACN,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Scorecard } from './scorecard.ts';
|
|
2
|
+
import { TestEpochs } from './test-epochs.ts';
|
|
3
|
+
export declare class Run {
|
|
4
|
+
add(testId: string, scorecard: Scorecard, weight: number): void;
|
|
5
|
+
}
|
|
6
|
+
export declare function runTests(run: Run): ReadonlyMap<string, TestEpochs>;
|
|
7
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,qBAAa,GAAG;IACd,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;CAWhE;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAElE"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { TestEpochs } from "./test-epochs.js";
|
|
2
|
+
// The tests of each run, in the order of their first add. PairedBootstrap of
|
|
3
|
+
// this package reads them through runTests; the package does not export it.
|
|
4
|
+
const testsOfRun = new WeakMap();
|
|
5
|
+
const testsOf = (run) => testsOfRun.getOrInsertComputed(run, () => new Map());
|
|
6
|
+
// The scorecards of one model on many tests. A second add of a test id is
|
|
7
|
+
// one more epoch of that test.
|
|
8
|
+
export class Run {
|
|
9
|
+
add(testId, scorecard, weight) {
|
|
10
|
+
const test = testsOf(this).getOrInsertComputed(testId, () => new TestEpochs(weight));
|
|
11
|
+
if (test.weight !== weight)
|
|
12
|
+
throw new RangeError(`Test "${testId}" has the weight ${test.weight}, not ${weight}: the weight belongs to the case`);
|
|
13
|
+
test.add(scorecard);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export function runTests(run) {
|
|
17
|
+
return testsOf(run);
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,6EAA6E;AAC7E,4EAA4E;AAC5E,MAAM,UAAU,GAAG,IAAI,OAAO,EAAgC,CAAC;AAC/D,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC3B,UAAU,CAAC,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;AAEvD,0EAA0E;AAC1E,+BAA+B;AAC/B,MAAM,OAAO,GAAG;IACd,GAAG,CAAC,MAAc,EAAE,SAAoB,EAAE,MAAc;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAC5C,MAAM,EACN,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAC7B,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YACxB,MAAM,IAAI,UAAU,CAClB,SAAS,MAAM,oBAAoB,IAAI,CAAC,MAAM,SAAS,MAAM,kCAAkC,CAChG,CAAC;QACJ,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,GAAQ;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC"}
|
|
@@ -5,7 +5,7 @@ import type { ScorecardScorer, Weighted } from './scorecard-scorer.ts';
|
|
|
5
5
|
import type { Scorecard } from './scorecard.ts';
|
|
6
6
|
export declare class ScorerSet<Case extends Weighted, Input> implements ScorecardScorer<Case> {
|
|
7
7
|
#private;
|
|
8
|
-
constructor(entries: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>, { comparer }: {
|
|
8
|
+
constructor(entries: Readonly<Record<string, Scorer<Input> | Grouped<Input, unknown>>>, { comparer }: {
|
|
9
9
|
readonly comparer: Comparer<Case, Input>;
|
|
10
10
|
});
|
|
11
11
|
score(stated: unknown, testCase: Case): Scorecard;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQvD,qBAAa,SAAS,CACpB,IAAI,SAAS,QAAQ,EACrB,KAAK,CACL,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQ9B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQvD,qBAAa,SAAS,CACpB,IAAI,SAAS,QAAQ,EACrB,KAAK,CACL,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQ9B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAC1E,EAAE,QAAQ,EAAE,EAAE;QAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;KAAE;IAM5D,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,SAAS;CAqClD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAG1C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAC9E,kEAAkE;AAClE,MAAM,OAAO,SAAS;IAIX,QAAQ,CAGZ;IACI,SAAS,CAAwB;IAE1C,YACE,
|
|
1
|
+
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAG1C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAC9E,kEAAkE;AAClE,MAAM,OAAO,SAAS;IAIX,QAAQ,CAGZ;IACI,SAAS,CAAwB;IAE1C,YACE,OAA0E,EAC1E,EAAE,QAAQ,EAAgD;QAE1D,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,MAAe,EAAE,QAAc;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK;wBACX,CAAC,CAAC,GAAG,KAAK,QAAQ;wBAClB,CAAC,CAAC,GAAG,KAAK,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EACJ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC;YACvB,MAAM;SACP,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorecard } from './scorecard.ts';
|
|
3
|
+
export declare class TestEpochs {
|
|
4
|
+
#private;
|
|
5
|
+
readonly weight: number;
|
|
6
|
+
constructor(weight: number);
|
|
7
|
+
add(scorecard: Scorecard): void;
|
|
8
|
+
gives(name: string): boolean;
|
|
9
|
+
mean(name: string): Ratio;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=test-epochs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-epochs.d.ts","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,qBAAa,UAAU;;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAGZ,MAAM,EAAE,MAAM;IAI1B,GAAG,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAI/B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAO5B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;CAS1B"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { RatioOfSums } from "../aggregation/ratio-of-sums.js";
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
// The epochs of one test of a run, with the weight of its case.
|
|
4
|
+
export class TestEpochs {
|
|
5
|
+
weight;
|
|
6
|
+
#scorecards = [];
|
|
7
|
+
constructor(weight) {
|
|
8
|
+
this.weight = weight;
|
|
9
|
+
}
|
|
10
|
+
add(scorecard) {
|
|
11
|
+
this.#scorecards.push(scorecard);
|
|
12
|
+
}
|
|
13
|
+
gives(name) {
|
|
14
|
+
return this.#scorecards.some((scorecard) => scorecard.ratios.has(name));
|
|
15
|
+
}
|
|
16
|
+
// The mean of the epochs: Ratio(sum of the parts / k, sum of the wholes / k)
|
|
17
|
+
// for k epochs (Miller 2024, §3.1). A scorecard without the name counts as
|
|
18
|
+
// Ratio(0, 0), as in Totals.
|
|
19
|
+
mean(name) {
|
|
20
|
+
const sum = RatioOfSums.of(this.#scorecards.map((scorecard) => scorecard.ratios.get(name) ?? new Ratio(0, 0))).ratio;
|
|
21
|
+
const k = this.#scorecards.length;
|
|
22
|
+
return new Ratio(sum.part / k, sum.whole / k);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=test-epochs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-epochs.js","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,gEAAgE;AAChE,MAAM,OAAO,UAAU;IACZ,MAAM,CAAS;IACf,WAAW,GAAgB,EAAE,CAAC;IAEvC,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,SAAoB;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,6BAA6B;IAC7B,IAAI,CAAC,IAAY;QACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,CACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAClB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAC7D,CACF,CAAC,KAAK,CAAC;QACR,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAClC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.9",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@zukhruf/python": "^0.8.
|
|
67
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.9",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.9"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"@cfworker/json-schema": "4.1.4",
|
|
@@ -80,7 +80,8 @@
|
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@cfworker/json-schema": "4.1.4",
|
|
83
|
-
"@zukhruf/
|
|
83
|
+
"@zukhruf/random": "*",
|
|
84
|
+
"@zukhruf/testing": "^0.8.9",
|
|
84
85
|
"munkres": "^2.1.1",
|
|
85
86
|
"promptfoo": "0.125.1",
|
|
86
87
|
"python-random": "1.0.0"
|