@zukhruf/evals 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/README.md +152 -79
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
- package/dist/aggregation/ratio-of-sums.js +7 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -1
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/average.d.ts +11 -0
- package/dist/classification/average.d.ts.map +1 -0
- package/dist/classification/average.js +2 -0
- package/dist/classification/average.js.map +1 -0
- package/dist/classification/binary-average.d.ts +9 -0
- package/dist/classification/binary-average.d.ts.map +1 -0
- package/dist/classification/binary-average.js +20 -0
- package/dist/classification/binary-average.js.map +1 -0
- package/dist/classification/brier-score.d.ts +13 -0
- package/dist/classification/brier-score.d.ts.map +1 -0
- package/dist/classification/brier-score.js +52 -0
- package/dist/classification/brier-score.js.map +1 -0
- package/dist/classification/class-counts.d.ts +39 -0
- package/dist/classification/class-counts.d.ts.map +1 -0
- package/dist/classification/class-counts.js +159 -0
- package/dist/classification/class-counts.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/confusion-matrix.d.ts +9 -0
- package/dist/classification/confusion-matrix.d.ts.map +1 -0
- package/dist/classification/confusion-matrix.js +18 -0
- package/dist/classification/confusion-matrix.js.map +1 -0
- package/dist/classification/index.d.ts +21 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +19 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/macro-average.d.ts +6 -0
- package/dist/classification/macro-average.d.ts.map +1 -0
- package/dist/classification/macro-average.js +15 -0
- package/dist/classification/macro-average.js.map +1 -0
- package/dist/classification/micro-average.d.ts +6 -0
- package/dist/classification/micro-average.d.ts.map +1 -0
- package/dist/classification/micro-average.js +16 -0
- package/dist/classification/micro-average.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/probability-comparer.d.ts +8 -0
- package/dist/classification/probability-comparer.d.ts.map +1 -0
- package/dist/classification/probability-comparer.js +35 -0
- package/dist/classification/probability-comparer.js.map +1 -0
- package/dist/classification/probability-comparison.d.ts +10 -0
- package/dist/classification/probability-comparison.d.ts.map +1 -0
- package/dist/classification/probability-comparison.js +11 -0
- package/dist/classification/probability-comparison.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/classification/weighted-average.d.ts +6 -0
- package/dist/classification/weighted-average.d.ts.map +1 -0
- package/dist/classification/weighted-average.js +20 -0
- package/dist/classification/weighted-average.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +6 -6
- package/dist/scorers/f1.d.ts.map +1 -1
- package/dist/scorers/f1.js +5 -9
- package/dist/scorers/f1.js.map +1 -1
- package/dist/scorers/penalized.d.ts.map +1 -1
- package/dist/scorers/penalized.js +2 -1
- package/dist/scorers/penalized.js.map +1 -1
- package/dist/scoring/bootstrap.js +3 -5
- package/dist/scoring/bootstrap.js.map +1 -1
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
- package/dist/structured/comparison/json-value.d.ts +1 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -1
- package/dist/structured/comparison/json-value.js +11 -2
- package/dist/structured/comparison/json-value.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts +5 -5
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +8 -4
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/cpython/python-text.d.ts +0 -1
- package/dist/structured/cpython/python-text.d.ts.map +1 -1
- package/dist/structured/cpython/python-text.js +1 -18
- package/dist/structured/cpython/python-text.js.map +1 -1
- package/dist/structured/index.d.ts +0 -2
- package/dist/structured/index.d.ts.map +1 -1
- package/dist/structured/index.js +0 -2
- package/dist/structured/index.js.map +1 -1
- package/dist/structured/scorers/token-f1.d.ts +3 -1
- package/dist/structured/scorers/token-f1.d.ts.map +1 -1
- package/dist/structured/scorers/token-f1.js +8 -5
- package/dist/structured/scorers/token-f1.js.map +1 -1
- package/dist/structured/scoring/per-field.d.ts +3 -6
- package/dist/structured/scoring/per-field.d.ts.map +1 -1
- package/dist/structured/scoring/per-field.js +8 -17
- package/dist/structured/scoring/per-field.js.map +1 -1
- package/package.json +18 -5
- package/dist/aggregation/f1-ratio.d.ts +0 -8
- package/dist/aggregation/f1-ratio.d.ts.map +0 -1
- package/dist/aggregation/f1-ratio.js +0 -24
- package/dist/aggregation/f1-ratio.js.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
- package/dist/structured/scoring/scorer-set.d.ts +0 -15
- package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
- package/dist/structured/scoring/scorer-set.js +0 -60
- package/dist/structured/scoring/scorer-set.js.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
-
import {
|
|
2
|
+
import { CountsForm, } from "../../aggregation/f-measure-form.js";
|
|
3
3
|
import { Ratio } from "../../aggregation/ratio.js";
|
|
4
4
|
import { BagOfWords } from "../comparison/bag-of-words.js";
|
|
5
5
|
// The sum of the token F1 of each expected leaf and the stated value at its
|
|
@@ -8,15 +8,18 @@ import { BagOfWords } from "../comparison/bag-of-words.js";
|
|
|
8
8
|
// a word gives whenNoWords (evaluate.py gives 1), and a path that the stated
|
|
9
9
|
// value does not have is read as absentAs (evaluate.py reads None). With
|
|
10
10
|
// nullIsMissing, a null leaf is no value: an expected null is not counted,
|
|
11
|
-
// and a stated null is read as absentAs.
|
|
11
|
+
// and a stated null is read as absentAs. The form is the arithmetic of each
|
|
12
|
+
// F1: evaluate.py's is PrecisionRecallForm.
|
|
12
13
|
export class TokenF1 {
|
|
13
14
|
#whenNoWords;
|
|
14
15
|
#absentAs;
|
|
15
16
|
#nullIsMissing;
|
|
16
|
-
|
|
17
|
+
#form;
|
|
18
|
+
constructor({ whenNoWords, absentAs, nullIsMissing = false, form = new CountsForm(), }) {
|
|
17
19
|
this.#whenNoWords = whenNoWords;
|
|
18
20
|
this.#absentAs = absentAs;
|
|
19
21
|
this.#nullIsMissing = nullIsMissing;
|
|
22
|
+
this.#form = form;
|
|
20
23
|
}
|
|
21
24
|
score(comparison) {
|
|
22
25
|
// CPython's sum, so the mean of the floats is that of evaluate.py.
|
|
@@ -32,13 +35,13 @@ export class TokenF1 {
|
|
|
32
35
|
}
|
|
33
36
|
return new Ratio(sum.value, expected);
|
|
34
37
|
}
|
|
35
|
-
// token_f1 of evaluate.py: the F1 of the shared words.
|
|
38
|
+
// token_f1 of evaluate.py: the F1 of the shared words, in the form.
|
|
36
39
|
#tokenF1(expected, stated) {
|
|
37
40
|
const expectedWords = BagOfWords.of(expected);
|
|
38
41
|
const statedWords = BagOfWords.of(stated);
|
|
39
42
|
if (expectedWords.size === 0 && statedWords.size === 0)
|
|
40
43
|
return this.#whenNoWords;
|
|
41
|
-
return
|
|
44
|
+
return this.#form.of(expectedWords.shared(statedWords), 1);
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
//# sourceMappingURL=token-f1.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token-f1.js","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,
|
|
1
|
+
{"version":3,"file":"token-f1.js","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EACL,UAAU,GAEX,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAK3D,4EAA4E;AAC5E,2EAA2E;AAC3E,8EAA8E;AAC9E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,4EAA4E;AAC5E,4CAA4C;AAC5C,MAAM,OAAO,OAAO;IACT,YAAY,CAAS;IACrB,SAAS,CAAY;IACrB,cAAc,CAAU;IACxB,KAAK,CAAe;IAE7B,YAAY,EACV,WAAW,EACX,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,IAAI,GAAG,IAAI,UAAU,EAAE,GAKV;QACb,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,mEAAmE;QACnE,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YAC1C,QAAQ,EAAE,CAAC;YACX,GAAG,CAAC,GAAG,CACL,IAAI,CAAC,QAAQ,CACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EACnB,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAC/D,CACF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,oEAAoE;IACpE,QAAQ,CAAC,QAAmB,EAAE,MAAiB;QAC7C,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YACpD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;CACF"}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
1
|
+
import { Grouped } from '../../scoring/grouped.ts';
|
|
3
2
|
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
-
export declare class PerField {
|
|
5
|
-
|
|
6
|
-
constructor(scorer: Scorer<Comparison>);
|
|
7
|
-
ratios(comparison: Comparison, label: string): [string, Ratio][];
|
|
3
|
+
export declare class PerField extends Grouped<Comparison> {
|
|
4
|
+
protected groups(comparison: Comparison): Iterable<[string, Comparison]>;
|
|
8
5
|
}
|
|
9
6
|
//# sourceMappingURL=per-field.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-field.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"per-field.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAQ9D,qBAAa,QAAS,SAAQ,OAAO,CAAC,UAAU,CAAC;IAC/C,SAAS,CAAE,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,QAAQ,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAI1E"}
|
|
@@ -1,23 +1,14 @@
|
|
|
1
|
+
import { Grouped } from "../../scoring/grouped.js";
|
|
1
2
|
// A scorer for each field of a comparison: in a ScorerSet, it gives the
|
|
2
3
|
// ratio of its scorer on the leaf pairs and item pairs of each field, under
|
|
3
4
|
// the label `${field} ${label}`, such as `$.offers[*].kind recall`. A field
|
|
4
|
-
// gives no ratio when the scorer counts nothing in it
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
this.#scorer = scorer;
|
|
12
|
-
}
|
|
13
|
-
ratios(comparison, label) {
|
|
14
|
-
return comparison
|
|
15
|
-
.fields()
|
|
16
|
-
.map((field) => [
|
|
17
|
-
`${field} ${label}`,
|
|
18
|
-
this.#scorer.score(comparison.only(field)),
|
|
19
|
-
])
|
|
20
|
-
.filter(([, ratio]) => ratio.whole !== 0);
|
|
5
|
+
// gives no ratio when the scorer counts nothing in it: an object or an array
|
|
6
|
+
// under a scorer of leaves, or a field whose leaves are all null under
|
|
7
|
+
// nullIsMissing.
|
|
8
|
+
export class PerField extends Grouped {
|
|
9
|
+
*groups(comparison) {
|
|
10
|
+
for (const field of comparison.fields())
|
|
11
|
+
yield [`${field}`, comparison.only(field)];
|
|
21
12
|
}
|
|
22
13
|
}
|
|
23
14
|
//# sourceMappingURL=per-field.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-field.js","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"per-field.js","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,wEAAwE;AACxE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,iBAAiB;AACjB,MAAM,OAAO,QAAS,SAAQ,OAAmB;IACrC,CAAC,MAAM,CAAC,UAAsB;QACtC,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE;YACrC,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
4
|
-
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios
|
|
3
|
+
"version": "0.8.5",
|
|
4
|
+
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
"scorers",
|
|
19
19
|
"structured-output",
|
|
20
20
|
"extraction",
|
|
21
|
+
"classification",
|
|
22
|
+
"scikit-learn",
|
|
21
23
|
"precision",
|
|
22
24
|
"recall",
|
|
23
25
|
"promptfoo"
|
|
@@ -31,6 +33,12 @@
|
|
|
31
33
|
"import": "./dist/index.js",
|
|
32
34
|
"default": "./dist/index.js"
|
|
33
35
|
},
|
|
36
|
+
"./classification": {
|
|
37
|
+
"@zukhruf/source": "./src/classification/index.ts",
|
|
38
|
+
"types": "./dist/classification/index.d.ts",
|
|
39
|
+
"import": "./dist/classification/index.js",
|
|
40
|
+
"default": "./dist/classification/index.js"
|
|
41
|
+
},
|
|
34
42
|
"./promptfoo": {
|
|
35
43
|
"@zukhruf/source": "./src/promptfoo/index.ts",
|
|
36
44
|
"types": "./dist/promptfoo/index.d.ts",
|
|
@@ -55,24 +63,29 @@
|
|
|
55
63
|
"access": "public"
|
|
56
64
|
},
|
|
57
65
|
"dependencies": {
|
|
58
|
-
"@zukhruf/python": "^0.8.
|
|
59
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.5",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.5"
|
|
60
68
|
},
|
|
61
69
|
"peerDependencies": {
|
|
62
70
|
"@cfworker/json-schema": "4.1.4",
|
|
71
|
+
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
63
72
|
"munkres": "^2.1.1"
|
|
64
73
|
},
|
|
65
74
|
"peerDependenciesMeta": {
|
|
66
75
|
"@cfworker/json-schema": {
|
|
67
76
|
"optional": true
|
|
68
77
|
},
|
|
78
|
+
"@stdlib/blas-ext-base-gsumpw": {
|
|
79
|
+
"optional": true
|
|
80
|
+
},
|
|
69
81
|
"munkres": {
|
|
70
82
|
"optional": true
|
|
71
83
|
}
|
|
72
84
|
},
|
|
73
85
|
"devDependencies": {
|
|
74
86
|
"@cfworker/json-schema": "4.1.4",
|
|
75
|
-
"@
|
|
87
|
+
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
88
|
+
"@zukhruf/testing": "^0.8.5",
|
|
76
89
|
"munkres": "^2.1.1",
|
|
77
90
|
"promptfoo": "0.125.1",
|
|
78
91
|
"python-random": "1.0.0"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"f1-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AASnC,qBAAa,OAAQ,SAAQ,KAAK;;gBAGpB,MAAM,EAAE,MAAM;IAK1B,IAAa,KAAK,IAAI,MAAM,CAM3B;CACF"}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { Ratio } from "./ratio.js";
|
|
2
|
-
// The ratio of an F1: its part is 2 × hits and its whole is stated +
|
|
3
|
-
// expected, so many F1 ratios add up to the micro F1. Its value is 0 when
|
|
4
|
-
// there are no hits, else 2·p·r/(p + r): precision and recall first, the
|
|
5
|
-
// arithmetic of SOB's evaluate.py (f1_from_counts, token_f1). That value can
|
|
6
|
-
// differ from part / whole in the last bit, such as for 1 hit of 1 expected
|
|
7
|
-
// and 5 stated (0.33333333333333337, not 1/3). It is the one exception to the
|
|
8
|
-
// rule of a Ratio (ADR 0011).
|
|
9
|
-
export class F1Ratio extends Ratio {
|
|
10
|
-
#counts;
|
|
11
|
-
constructor(counts) {
|
|
12
|
-
super(2 * counts.hits, counts.stated + counts.expected);
|
|
13
|
-
this.#counts = counts;
|
|
14
|
-
}
|
|
15
|
-
get value() {
|
|
16
|
-
const { hits, stated, expected } = this.#counts;
|
|
17
|
-
if (hits === 0)
|
|
18
|
-
return 0;
|
|
19
|
-
const precision = hits / stated;
|
|
20
|
-
const recall = hits / expected;
|
|
21
|
-
return (2 * precision * recall) / (precision + recall);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=f1-ratio.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"f1-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,qEAAqE;AACrE,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,8BAA8B;AAC9B,MAAM,OAAO,OAAQ,SAAQ,KAAK;IACvB,OAAO,CAAS;IAEzB,YAAY,MAAc;QACxB,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,IAAa,KAAK;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAChD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACzD,CAAC;CACF"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
2
|
-
import type { CaseReader } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
-
import { ComparisonSpec } from '../comparison/comparison-spec.ts';
|
|
4
|
-
import type { Comparison } from '../comparison/comparison.ts';
|
|
5
|
-
import type { Case } from '../scoring/case.ts';
|
|
6
|
-
export type VarsContext = {
|
|
7
|
-
readonly vars: Readonly<Record<string, unknown>>;
|
|
8
|
-
};
|
|
9
|
-
export declare class ScorerAssertion {
|
|
10
|
-
readonly metric: string | undefined;
|
|
11
|
-
readonly weight: number;
|
|
12
|
-
readonly threshold: number;
|
|
13
|
-
readonly value: (output: string, context: VarsContext) => number;
|
|
14
|
-
constructor(scorer: Scorer<Comparison>, { weight, threshold, metric, spec, cases, }: {
|
|
15
|
-
readonly weight: number;
|
|
16
|
-
readonly threshold: number;
|
|
17
|
-
readonly metric?: string;
|
|
18
|
-
readonly spec?: ComparisonSpec;
|
|
19
|
-
readonly cases?: CaseReader<Case>;
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=scorer-assertion.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-assertion.d.ts","sourceRoot":"","sources":["../../../src/structured/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAM/C,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAkBF,qBAAa,eAAe;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;gBAG/D,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAC1B,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,IAA2B,EAC3B,KAA0B,GAC3B,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;QAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;KACnC;CASJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-assertion.js","sourceRoot":"","sources":["../../../src/structured/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAEtD,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAS5D,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,8EAA8E;AAC9E,oDAAoD;AACpD,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,KAAK,CAAmD;IAEjE,YACE,MAA0B,EAC1B,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,IAAI,GAAG,IAAI,cAAc,EAAE,EAC3B,KAAK,GAAG,IAAI,YAAY,EAAE,GAO3B;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAChC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
2
|
-
import type { ScorecardScorer } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
-
import type { Scorecard } from '../../scoring/scorecard.ts';
|
|
4
|
-
import { ComparisonSpec } from '../comparison/comparison-spec.ts';
|
|
5
|
-
import type { Comparison } from '../comparison/comparison.ts';
|
|
6
|
-
import type { Case } from './case.ts';
|
|
7
|
-
import { PerField } from './per-field.ts';
|
|
8
|
-
export declare class ScorerSet implements ScorecardScorer<Case> {
|
|
9
|
-
#private;
|
|
10
|
-
constructor(entries: Readonly<Record<string, Scorer<Comparison> | PerField>>, { spec }?: {
|
|
11
|
-
readonly spec?: ComparisonSpec;
|
|
12
|
-
});
|
|
13
|
-
score(stated: unknown, testCase: Case): Scorecard;
|
|
14
|
-
}
|
|
15
|
-
//# sourceMappingURL=scorer-set.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAU1C,qBAAa,SAAU,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQnD,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,CAAC,EAChE,EAAE,IAA2B,EAAE,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAA;KAAO;IAM1E,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,SAAS;CAqClD"}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { RatioOfSums } from "../../aggregation/ratio-of-sums.js";
|
|
2
|
-
import { Ratio } from "../../aggregation/ratio.js";
|
|
3
|
-
import { Gate } from "../../scorers/gate.js";
|
|
4
|
-
import { ComparisonSpec } from "../comparison/comparison-spec.js";
|
|
5
|
-
import { statedData } from "../comparison/json-value.js";
|
|
6
|
-
import { TreeWalk } from "../comparison/tree-walk.js";
|
|
7
|
-
import { PerField } from "./per-field.js";
|
|
8
|
-
// Scores a stated value with labelled scorers, through a walk with the spec:
|
|
9
|
-
// the ratio of each scorer under its label, and of a PerField under the label
|
|
10
|
-
// of each field, so a ratio of sums of a label adds the parts and the wholes
|
|
11
|
-
// of the records. The record passes when each of its gates holds, and the
|
|
12
|
-
// reason is the explanations of the gates that fail. Each gate is a check.
|
|
13
|
-
// The score is the mean of the values of the scorers. A stated text is read
|
|
14
|
-
// as json.loads reads it, as a ScorerAssertion reads it; a stated JsonValue
|
|
15
|
-
// stays as it is.
|
|
16
|
-
export class ScorerSet {
|
|
17
|
-
#entries;
|
|
18
|
-
#walk;
|
|
19
|
-
constructor(entries, { spec = new ComparisonSpec() } = {}) {
|
|
20
|
-
this.#entries = Object.entries(entries);
|
|
21
|
-
this.#walk = new TreeWalk(spec);
|
|
22
|
-
}
|
|
23
|
-
score(stated, testCase) {
|
|
24
|
-
const comparison = this.#walk.compare(statedData(stated), testCase);
|
|
25
|
-
const ratios = [];
|
|
26
|
-
const mean = new RatioOfSums();
|
|
27
|
-
const checks = [];
|
|
28
|
-
for (const [label, entry] of this.#entries) {
|
|
29
|
-
if (entry instanceof PerField) {
|
|
30
|
-
ratios.push(...entry.ratios(comparison, label));
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
const ratio = entry.score(comparison);
|
|
34
|
-
ratios.push([label, ratio]);
|
|
35
|
-
mean.add(new Ratio(ratio.value, 1));
|
|
36
|
-
if (entry instanceof Gate) {
|
|
37
|
-
const holds = entry.holds(comparison);
|
|
38
|
-
checks.push({
|
|
39
|
-
name: label,
|
|
40
|
-
passed: holds,
|
|
41
|
-
score: ratio.value,
|
|
42
|
-
reason: holds
|
|
43
|
-
? `${label} holds`
|
|
44
|
-
: `${label}: ${entry.explain(comparison)}`,
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
const failed = checks.filter((check) => !check.passed);
|
|
49
|
-
return {
|
|
50
|
-
passed: failed.length === 0,
|
|
51
|
-
score: mean.value,
|
|
52
|
-
reason: failed.length === 0
|
|
53
|
-
? 'each gate holds'
|
|
54
|
-
: failed.map((check) => check.reason).join('; '),
|
|
55
|
-
ratios: new Map(ratios),
|
|
56
|
-
checks,
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
//# sourceMappingURL=scorer-set.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAI7C,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,6EAA6E;AAC7E,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,4EAA4E;AAC5E,4EAA4E;AAC5E,kBAAkB;AAClB,MAAM,OAAO,SAAS;IACX,QAAQ,CAGZ;IACI,KAAK,CAAW;IAEzB,YACE,OAAgE,EAChE,EAAE,IAAI,GAAG,IAAI,cAAc,EAAE,KAAyC,EAAE;QAExE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAe,EAAE,QAAc;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpE,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK;wBACX,CAAC,CAAC,GAAG,KAAK,QAAQ;wBAClB,CAAC,CAAC,GAAG,KAAK,KAAK,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EACJ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC;YACvB,MAAM;SACP,CAAC;IACJ,CAAC;CACF"}
|