@zukhruf/evals 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -10
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
- package/dist/aggregation/ratio-of-sums.js +7 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -1
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/index.d.ts +11 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +10 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +6 -6
- package/dist/scorers/f1.d.ts.map +1 -1
- package/dist/scorers/f1.js +5 -9
- package/dist/scorers/f1.js.map +1 -1
- package/dist/scorers/penalized.d.ts.map +1 -1
- package/dist/scorers/penalized.js +2 -1
- package/dist/scorers/penalized.js.map +1 -1
- package/dist/scoring/bootstrap.js +3 -5
- package/dist/scoring/bootstrap.js.map +1 -1
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
- package/dist/structured/comparison/json-value.d.ts +1 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -1
- package/dist/structured/comparison/json-value.js +11 -2
- package/dist/structured/comparison/json-value.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts +5 -5
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +8 -4
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/cpython/python-text.d.ts +0 -1
- package/dist/structured/cpython/python-text.d.ts.map +1 -1
- package/dist/structured/cpython/python-text.js +1 -18
- package/dist/structured/cpython/python-text.js.map +1 -1
- package/dist/structured/index.d.ts +0 -2
- package/dist/structured/index.d.ts.map +1 -1
- package/dist/structured/index.js +0 -2
- package/dist/structured/index.js.map +1 -1
- package/dist/structured/scorers/token-f1.d.ts +3 -1
- package/dist/structured/scorers/token-f1.d.ts.map +1 -1
- package/dist/structured/scorers/token-f1.js +8 -5
- package/dist/structured/scorers/token-f1.js.map +1 -1
- package/dist/structured/scoring/per-field.d.ts +3 -6
- package/dist/structured/scoring/per-field.d.ts.map +1 -1
- package/dist/structured/scoring/per-field.js +8 -17
- package/dist/structured/scoring/per-field.js.map +1 -1
- package/package.json +10 -4
- package/dist/aggregation/f1-ratio.d.ts +0 -8
- package/dist/aggregation/f1-ratio.d.ts.map +0 -1
- package/dist/aggregation/f1-ratio.js +0 -24
- package/dist/aggregation/f1-ratio.js.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
- package/dist/structured/scoring/scorer-set.d.ts +0 -15
- package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
- package/dist/structured/scoring/scorer-set.js +0 -60
- package/dist/structured/scoring/scorer-set.js.map +0 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Comparer } from '../../scoring/comparer.ts';
|
|
2
2
|
import type { Case } from '../scoring/case.ts';
|
|
3
|
-
import
|
|
3
|
+
import { ComparisonSpec } from './comparison-spec.ts';
|
|
4
4
|
import { Comparison } from './comparison.ts';
|
|
5
|
-
export declare class TreeWalk {
|
|
5
|
+
export declare class TreeWalk implements Comparer<Case, Comparison> {
|
|
6
6
|
#private;
|
|
7
|
-
constructor(spec
|
|
8
|
-
compare(stated:
|
|
7
|
+
constructor(spec?: ComparisonSpec);
|
|
8
|
+
compare(stated: unknown, testCase: Case): Comparison;
|
|
9
9
|
}
|
|
10
10
|
//# sourceMappingURL=tree-walk.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAuB7C,qBAAa,QAAS,YAAW,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;;gBAG7C,IAAI,iBAAuB;IAIvC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,UAAU;CAsNrD"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { ComparisonSpec } from "./comparison-spec.js";
|
|
1
2
|
import { Comparison } from "./comparison.js";
|
|
2
3
|
import { Field } from "./field.js";
|
|
4
|
+
import { statedData } from "./json-value.js";
|
|
3
5
|
import { Path } from "./path.js";
|
|
4
6
|
// Walks a stated value and the expected value of its case side by side, as
|
|
5
7
|
// the spec says, and gives their Comparison. Objects line up by key, and the
|
|
@@ -8,16 +10,18 @@ import { Path } from "./path.js";
|
|
|
8
10
|
// A value at the root is no leaf, so a stated value that is not an object or
|
|
9
11
|
// an array has no leaves. Where the two sides differ in kind, each side is
|
|
10
12
|
// walked alone. An excluded field adds no leaf pair and no item pairs, but its
|
|
11
|
-
// stated values still have their fields.
|
|
13
|
+
// stated values still have their fields. A stated text is read as
|
|
14
|
+
// json.loads reads it; a stated JsonValue stays as it is.
|
|
12
15
|
export class TreeWalk {
|
|
13
16
|
#spec;
|
|
14
|
-
constructor(spec) {
|
|
17
|
+
constructor(spec = new ComparisonSpec()) {
|
|
15
18
|
this.#spec = spec;
|
|
16
19
|
}
|
|
17
20
|
compare(stated, testCase) {
|
|
21
|
+
const data = statedData(stated);
|
|
18
22
|
const found = TreeWalk.#nothingFound();
|
|
19
|
-
this.#walk(
|
|
20
|
-
return new Comparison(testCase,
|
|
23
|
+
this.#walk(data, testCase.expected, new Path([]), new Path([]), new Field([]), found);
|
|
24
|
+
return new Comparison(testCase, data, found.leaves, found.statedPaths, found.arrays);
|
|
21
25
|
}
|
|
22
26
|
// The expected side goes first at each step, so the expected leaves come in
|
|
23
27
|
// the order of the expected value, as evaluate.py reads them.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAa,MAAM,WAAW,CAAC;AAS5C,2EAA2E;AAC3E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,kEAAkE;AAClE,0DAA0D;AAC1D,MAAM,OAAO,QAAQ;IACV,KAAK,CAAiB;IAE/B,YAAY,IAAI,GAAG,IAAI,cAAc,EAAE;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,MAAe,EAAE,QAAc;QACrC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CACR,IAAI,EACJ,QAAQ,CAAC,QAAQ,EACjB,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,KAAK,CAAC,EAAE,CAAC,EACb,KAAK,CACN,CAAC;QACF,OAAO,IAAI,UAAU,CACnB,QAAQ,EACR,IAAI,EACJ,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,MAAM,CACb,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,8DAA8D;IAC9D,KAAK,CACH,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtE,IAAI,MAAM,YAAY,GAAG,IAAI,QAAQ,YAAY,GAAG;YAClD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;aACpE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;;YACnE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3E,IAAI,IAAI;YAAE,OAAO;QACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,YAAY,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS;YACxD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAChB,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,UAAU;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK;qBACf,YAAY,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;aAChD,CAAC,CAAC;aACA,IAAI,YAAY,KAAK,SAAS;YACjC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;aAC3C,IAAI,UAAU,KAAK,SAAS;YAC/B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,CACL,MAA6B,EAC7B,QAA+B,EAC/B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,QAAQ;YACjC,IAAI,CAAC,KAAK,CACR,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EACf,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM;YAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBACpB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,iEAAiE;IACjE,MAAM,CACJ,MAA2B,EAC3B,QAA6B,EAC7B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAc;YAC3B,KAAK,EAAE,CAAC,UAAU,EAAE,YAAY,EAAE,EAAE,CAClC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC;SACvD,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK;aACrB,WAAW,CAAC,KAAK,CAAC;aAClB,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QACtC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;YAChB,KAAK;YACL,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAC1E,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,KAAK,CACR,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAC3D,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,EACtC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACpB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EACvB,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,uDAAuD;IACvD,MAAM,CACJ,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAClD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAChB,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;QACL,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC;YACjE,IAAI,CAAC,KAAK,CACR,SAAS,EACT,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/D,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;IACN,CAAC;IAED,0EAA0E;IAC1E,kDAAkD;IAClD,eAAe,CAAC,MAAgB,EAAE,QAAkB,EAAE,IAAW;QAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACtE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACpE,CAAC;IAED,0DAA0D;IAC1D,MAAM,CAAC,YAAY,CACjB,MAA4B,EAC5B,IAAU,EACV,KAAY;QAEZ,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,AAAD,EAAG,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YACtD,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,CAAC,SAAS,CACf,KAA2B;QAE3B,IAAI,KAAK,YAAY,GAAG;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;gBAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;aACvD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE;gBAC1C,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAA2B,EAAE,IAAU;QAClD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,aAAa;QAClB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrD,CAAC;CACF"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { type JsonData } from '@zukhruf/python';
|
|
2
|
-
export declare function compareStrings(left: string, right: string): number;
|
|
3
2
|
export declare function canonicalJson(value: JsonData): string;
|
|
4
3
|
export declare function pythonStr(value: bigint | number | string): string;
|
|
5
4
|
//# sourceMappingURL=python-text.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"python-text.d.ts","sourceRoot":"","sources":["../../../src/structured/cpython/python-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,QAAQ,
|
|
1
|
+
{"version":3,"file":"python-text.d.ts","sourceRoot":"","sources":["../../../src/structured/cpython/python-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,QAAQ,EAA6B,MAAM,iBAAiB,CAAC;AAkC3E,wBAAgB,aAAa,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAerD;AAID,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAIjE"}
|
|
@@ -1,21 +1,4 @@
|
|
|
1
|
-
import { floatRepr } from '@zukhruf/python';
|
|
2
|
-
// UTF-16 code units order the strings that have no character above U+FFFF
|
|
3
|
-
// as code points do.
|
|
4
|
-
const SURROGATE = /[\u{10000}-\u{10FFFF}]/u;
|
|
5
|
-
// Compares two strings as Python does: by Unicode code point. Copied from
|
|
6
|
-
// @zukhruf/mteb compareStrings (docs/copied-code.md).
|
|
7
|
-
export function compareStrings(left, right) {
|
|
8
|
-
if (!SURROGATE.test(left) && !SURROGATE.test(right))
|
|
9
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
10
|
-
const a = [...left];
|
|
11
|
-
const b = [...right];
|
|
12
|
-
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
|
13
|
-
const difference = (a[index]?.codePointAt(0) ?? 0) - (b[index]?.codePointAt(0) ?? 0);
|
|
14
|
-
if (difference !== 0)
|
|
15
|
-
return difference;
|
|
16
|
-
}
|
|
17
|
-
return a.length - b.length;
|
|
18
|
-
}
|
|
1
|
+
import { compareStrings, floatRepr } from '@zukhruf/python';
|
|
19
2
|
// The escapes of json.dumps with ensure_ascii=False (ESCAPE and ESCAPE_DCT,
|
|
20
3
|
// Lib/json/encoder.py): the quote, the backslash and the control characters.
|
|
21
4
|
const escapes = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"python-text.js","sourceRoot":"","sources":["../../../src/structured/cpython/python-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,
|
|
1
|
+
{"version":3,"file":"python-text.js","sourceRoot":"","sources":["../../../src/structured/cpython/python-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,cAAc,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE3E,4EAA4E;AAC5E,6EAA6E;AAC7E,MAAM,OAAO,GAAqC;IAChD,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;CACZ,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE;IAClC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,GAAG;YACD,OAAO,CAAC,IAAI,CAAC;gBACb,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,IAAI,GAAG,GAAG,CAAC;AACpB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE;IAClC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;IACzE,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEF,+EAA+E;AAC/E,yEAAyE;AACzE,YAAY;AACZ,MAAM,UAAU,aAAa,CAAC,KAAe;IAC3C,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IAChE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAC3E,IAAI,KAAK,YAAY,GAAG;QACtB,OAAO,IAAI,CAAC,GAAG,KAAK,CAAC;aAClB,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;aACjE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAClB,MAAM,IAAI,SAAS,CACjB,6DAA6D,CAC9D,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,kEAAkE;AAClE,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -31,9 +31,7 @@ export { ValidSchema } from './scorers/valid-schema.ts';
|
|
|
31
31
|
export { Case } from './scoring/case.ts';
|
|
32
32
|
export { ConfigCase } from './scoring/config-case.ts';
|
|
33
33
|
export { TextVarsCase } from './scoring/text-vars-case.ts';
|
|
34
|
-
export { ScorerSet } from './scoring/scorer-set.ts';
|
|
35
34
|
export { PerField } from './scoring/per-field.ts';
|
|
36
35
|
export { type Draft, JsonSchema, type Validation, } from './schema/json-schema.ts';
|
|
37
36
|
export type { SchemaTypeWalk } from './schema/schema-type-walk.ts';
|
|
38
|
-
export { ScorerAssertion } from './promptfoo/scorer-assertion.ts';
|
|
39
37
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC9D,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC9D,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EACL,KAAK,KAAK,EACV,UAAU,EACV,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC"}
|
package/dist/structured/index.js
CHANGED
|
@@ -22,8 +22,6 @@ export { ValidSchema } from "./scorers/valid-schema.js";
|
|
|
22
22
|
export { Case } from "./scoring/case.js";
|
|
23
23
|
export { ConfigCase } from "./scoring/config-case.js";
|
|
24
24
|
export { TextVarsCase } from "./scoring/text-vars-case.js";
|
|
25
|
-
export { ScorerSet } from "./scoring/scorer-set.js";
|
|
26
25
|
export { PerField } from "./scoring/per-field.js";
|
|
27
26
|
export { JsonSchema, } from "./schema/json-schema.js";
|
|
28
|
-
export { ScorerAssertion } from "./promptfoo/scorer-assertion.js";
|
|
29
27
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEpE,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AAEzF,OAAO,EACL,cAAc,GAEf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAMvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEpE,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AAEzF,OAAO,EACL,cAAc,GAEf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAMvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAEL,UAAU,GAEX,MAAM,yBAAyB,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type FMeasureForm } from '../../aggregation/f-measure-form.ts';
|
|
1
2
|
import { Ratio } from '../../aggregation/ratio.ts';
|
|
2
3
|
import type { Scorer } from '../../scorers/scorer.ts';
|
|
3
4
|
import type { Comparison } from '../comparison/comparison.ts';
|
|
@@ -5,9 +6,10 @@ import type { LeafValue } from '../comparison/leaf-pair.ts';
|
|
|
5
6
|
import type { LeafOptions } from './leaf-matches.ts';
|
|
6
7
|
export declare class TokenF1 implements Scorer<Comparison> {
|
|
7
8
|
#private;
|
|
8
|
-
constructor({ whenNoWords, absentAs, nullIsMissing, }: {
|
|
9
|
+
constructor({ whenNoWords, absentAs, nullIsMissing, form, }: {
|
|
9
10
|
readonly whenNoWords: number;
|
|
10
11
|
readonly absentAs: LeafValue;
|
|
12
|
+
readonly form?: FMeasureForm;
|
|
11
13
|
} & LeafOptions);
|
|
12
14
|
score(comparison: Comparison): Ratio;
|
|
13
15
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token-f1.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"token-f1.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUrD,qBAAa,OAAQ,YAAW,MAAM,CAAC,UAAU,CAAC;;gBAMpC,EACV,WAAW,EACX,QAAQ,EACR,aAAqB,EACrB,IAAuB,GACxB,EAAE;QACD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,WAAW;IAOf,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CA2BrC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
-
import {
|
|
2
|
+
import { CountsForm, } from "../../aggregation/f-measure-form.js";
|
|
3
3
|
import { Ratio } from "../../aggregation/ratio.js";
|
|
4
4
|
import { BagOfWords } from "../comparison/bag-of-words.js";
|
|
5
5
|
// The sum of the token F1 of each expected leaf and the stated value at its
|
|
@@ -8,15 +8,18 @@ import { BagOfWords } from "../comparison/bag-of-words.js";
|
|
|
8
8
|
// a word gives whenNoWords (evaluate.py gives 1), and a path that the stated
|
|
9
9
|
// value does not have is read as absentAs (evaluate.py reads None). With
|
|
10
10
|
// nullIsMissing, a null leaf is no value: an expected null is not counted,
|
|
11
|
-
// and a stated null is read as absentAs.
|
|
11
|
+
// and a stated null is read as absentAs. The form is the arithmetic of each
|
|
12
|
+
// F1: evaluate.py's is PrecisionRecallForm.
|
|
12
13
|
export class TokenF1 {
|
|
13
14
|
#whenNoWords;
|
|
14
15
|
#absentAs;
|
|
15
16
|
#nullIsMissing;
|
|
16
|
-
|
|
17
|
+
#form;
|
|
18
|
+
constructor({ whenNoWords, absentAs, nullIsMissing = false, form = new CountsForm(), }) {
|
|
17
19
|
this.#whenNoWords = whenNoWords;
|
|
18
20
|
this.#absentAs = absentAs;
|
|
19
21
|
this.#nullIsMissing = nullIsMissing;
|
|
22
|
+
this.#form = form;
|
|
20
23
|
}
|
|
21
24
|
score(comparison) {
|
|
22
25
|
// CPython's sum, so the mean of the floats is that of evaluate.py.
|
|
@@ -32,13 +35,13 @@ export class TokenF1 {
|
|
|
32
35
|
}
|
|
33
36
|
return new Ratio(sum.value, expected);
|
|
34
37
|
}
|
|
35
|
-
// token_f1 of evaluate.py: the F1 of the shared words.
|
|
38
|
+
// token_f1 of evaluate.py: the F1 of the shared words, in the form.
|
|
36
39
|
#tokenF1(expected, stated) {
|
|
37
40
|
const expectedWords = BagOfWords.of(expected);
|
|
38
41
|
const statedWords = BagOfWords.of(stated);
|
|
39
42
|
if (expectedWords.size === 0 && statedWords.size === 0)
|
|
40
43
|
return this.#whenNoWords;
|
|
41
|
-
return
|
|
44
|
+
return this.#form.of(expectedWords.shared(statedWords), 1);
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
//# sourceMappingURL=token-f1.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token-f1.js","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,
|
|
1
|
+
{"version":3,"file":"token-f1.js","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EACL,UAAU,GAEX,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAK3D,4EAA4E;AAC5E,2EAA2E;AAC3E,8EAA8E;AAC9E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,4EAA4E;AAC5E,4CAA4C;AAC5C,MAAM,OAAO,OAAO;IACT,YAAY,CAAS;IACrB,SAAS,CAAY;IACrB,cAAc,CAAU;IACxB,KAAK,CAAe;IAE7B,YAAY,EACV,WAAW,EACX,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,IAAI,GAAG,IAAI,UAAU,EAAE,GAKV;QACb,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,mEAAmE;QACnE,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YAC1C,QAAQ,EAAE,CAAC;YACX,GAAG,CAAC,GAAG,CACL,IAAI,CAAC,QAAQ,CACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EACnB,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAC/D,CACF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,oEAAoE;IACpE,QAAQ,CAAC,QAAmB,EAAE,MAAiB;QAC7C,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YACpD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;CACF"}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
1
|
+
import { Grouped } from '../../scoring/grouped.ts';
|
|
3
2
|
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
-
export declare class PerField {
|
|
5
|
-
|
|
6
|
-
constructor(scorer: Scorer<Comparison>);
|
|
7
|
-
ratios(comparison: Comparison, label: string): [string, Ratio][];
|
|
3
|
+
export declare class PerField extends Grouped<Comparison> {
|
|
4
|
+
protected groups(comparison: Comparison): Iterable<[string, Comparison]>;
|
|
8
5
|
}
|
|
9
6
|
//# sourceMappingURL=per-field.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-field.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"per-field.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAQ9D,qBAAa,QAAS,SAAQ,OAAO,CAAC,UAAU,CAAC;IAC/C,SAAS,CAAE,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,QAAQ,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAI1E"}
|
|
@@ -1,23 +1,14 @@
|
|
|
1
|
+
import { Grouped } from "../../scoring/grouped.js";
|
|
1
2
|
// A scorer for each field of a comparison: in a ScorerSet, it gives the
|
|
2
3
|
// ratio of its scorer on the leaf pairs and item pairs of each field, under
|
|
3
4
|
// the label `${field} ${label}`, such as `$.offers[*].kind recall`. A field
|
|
4
|
-
// gives no ratio when the scorer counts nothing in it
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
this.#scorer = scorer;
|
|
12
|
-
}
|
|
13
|
-
ratios(comparison, label) {
|
|
14
|
-
return comparison
|
|
15
|
-
.fields()
|
|
16
|
-
.map((field) => [
|
|
17
|
-
`${field} ${label}`,
|
|
18
|
-
this.#scorer.score(comparison.only(field)),
|
|
19
|
-
])
|
|
20
|
-
.filter(([, ratio]) => ratio.whole !== 0);
|
|
5
|
+
// gives no ratio when the scorer counts nothing in it: an object or an array
|
|
6
|
+
// under a scorer of leaves, or a field whose leaves are all null under
|
|
7
|
+
// nullIsMissing.
|
|
8
|
+
export class PerField extends Grouped {
|
|
9
|
+
*groups(comparison) {
|
|
10
|
+
for (const field of comparison.fields())
|
|
11
|
+
yield [`${field}`, comparison.only(field)];
|
|
21
12
|
}
|
|
22
13
|
}
|
|
23
14
|
//# sourceMappingURL=per-field.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-field.js","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"per-field.js","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,wEAAwE;AACxE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,iBAAiB;AACjB,MAAM,OAAO,QAAS,SAAQ,OAAmB;IACrC,CAAC,MAAM,CAAC,UAAsB;QACtC,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE;YACrC,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios, scorecards, totals and bootstrap intervals, ./structured for structured values, and ./promptfoo for promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -31,6 +31,12 @@
|
|
|
31
31
|
"import": "./dist/index.js",
|
|
32
32
|
"default": "./dist/index.js"
|
|
33
33
|
},
|
|
34
|
+
"./classification": {
|
|
35
|
+
"@zukhruf/source": "./src/classification/index.ts",
|
|
36
|
+
"types": "./dist/classification/index.d.ts",
|
|
37
|
+
"import": "./dist/classification/index.js",
|
|
38
|
+
"default": "./dist/classification/index.js"
|
|
39
|
+
},
|
|
34
40
|
"./promptfoo": {
|
|
35
41
|
"@zukhruf/source": "./src/promptfoo/index.ts",
|
|
36
42
|
"types": "./dist/promptfoo/index.d.ts",
|
|
@@ -55,8 +61,8 @@
|
|
|
55
61
|
"access": "public"
|
|
56
62
|
},
|
|
57
63
|
"dependencies": {
|
|
58
|
-
"@zukhruf/python": "^0.8.
|
|
59
|
-
"@zukhruf/statistics": "^0.8.
|
|
64
|
+
"@zukhruf/python": "^0.8.4",
|
|
65
|
+
"@zukhruf/statistics": "^0.8.4"
|
|
60
66
|
},
|
|
61
67
|
"peerDependencies": {
|
|
62
68
|
"@cfworker/json-schema": "4.1.4",
|
|
@@ -72,7 +78,7 @@
|
|
|
72
78
|
},
|
|
73
79
|
"devDependencies": {
|
|
74
80
|
"@cfworker/json-schema": "4.1.4",
|
|
75
|
-
"@zukhruf/testing": "^0.8.
|
|
81
|
+
"@zukhruf/testing": "^0.8.4",
|
|
76
82
|
"munkres": "^2.1.1",
|
|
77
83
|
"promptfoo": "0.125.1",
|
|
78
84
|
"python-random": "1.0.0"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"f1-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AASnC,qBAAa,OAAQ,SAAQ,KAAK;;gBAGpB,MAAM,EAAE,MAAM;IAK1B,IAAa,KAAK,IAAI,MAAM,CAM3B;CACF"}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { Ratio } from "./ratio.js";
|
|
2
|
-
// The ratio of an F1: its part is 2 × hits and its whole is stated +
|
|
3
|
-
// expected, so many F1 ratios add up to the micro F1. Its value is 0 when
|
|
4
|
-
// there are no hits, else 2·p·r/(p + r): precision and recall first, the
|
|
5
|
-
// arithmetic of SOB's evaluate.py (f1_from_counts, token_f1). That value can
|
|
6
|
-
// differ from part / whole in the last bit, such as for 1 hit of 1 expected
|
|
7
|
-
// and 5 stated (0.33333333333333337, not 1/3). It is the one exception to the
|
|
8
|
-
// rule of a Ratio (ADR 0011).
|
|
9
|
-
export class F1Ratio extends Ratio {
|
|
10
|
-
#counts;
|
|
11
|
-
constructor(counts) {
|
|
12
|
-
super(2 * counts.hits, counts.stated + counts.expected);
|
|
13
|
-
this.#counts = counts;
|
|
14
|
-
}
|
|
15
|
-
get value() {
|
|
16
|
-
const { hits, stated, expected } = this.#counts;
|
|
17
|
-
if (hits === 0)
|
|
18
|
-
return 0;
|
|
19
|
-
const precision = hits / stated;
|
|
20
|
-
const recall = hits / expected;
|
|
21
|
-
return (2 * precision * recall) / (precision + recall);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=f1-ratio.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"f1-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,qEAAqE;AACrE,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,8BAA8B;AAC9B,MAAM,OAAO,OAAQ,SAAQ,KAAK;IACvB,OAAO,CAAS;IAEzB,YAAY,MAAc;QACxB,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,IAAa,KAAK;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAChD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACzD,CAAC;CACF"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
2
|
-
import type { CaseReader } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
-
import { ComparisonSpec } from '../comparison/comparison-spec.ts';
|
|
4
|
-
import type { Comparison } from '../comparison/comparison.ts';
|
|
5
|
-
import type { Case } from '../scoring/case.ts';
|
|
6
|
-
export type VarsContext = {
|
|
7
|
-
readonly vars: Readonly<Record<string, unknown>>;
|
|
8
|
-
};
|
|
9
|
-
export declare class ScorerAssertion {
|
|
10
|
-
readonly metric: string | undefined;
|
|
11
|
-
readonly weight: number;
|
|
12
|
-
readonly threshold: number;
|
|
13
|
-
readonly value: (output: string, context: VarsContext) => number;
|
|
14
|
-
constructor(scorer: Scorer<Comparison>, { weight, threshold, metric, spec, cases, }: {
|
|
15
|
-
readonly weight: number;
|
|
16
|
-
readonly threshold: number;
|
|
17
|
-
readonly metric?: string;
|
|
18
|
-
readonly spec?: ComparisonSpec;
|
|
19
|
-
readonly cases?: CaseReader<Case>;
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=scorer-assertion.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-assertion.d.ts","sourceRoot":"","sources":["../../../src/structured/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAM/C,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAkBF,qBAAa,eAAe;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;gBAG/D,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAC1B,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,IAA2B,EAC3B,KAA0B,GAC3B,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;QAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;KACnC;CASJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-assertion.js","sourceRoot":"","sources":["../../../src/structured/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAEtD,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAS5D,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,8EAA8E;AAC9E,oDAAoD;AACpD,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,KAAK,CAAmD;IAEjE,YACE,MAA0B,EAC1B,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,IAAI,GAAG,IAAI,cAAc,EAAE,EAC3B,KAAK,GAAG,IAAI,YAAY,EAAE,GAO3B;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAChC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { Scorer } from '../../scorers/scorer.ts';
|
|
2
|
-
import type { ScorecardScorer } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
-
import type { Scorecard } from '../../scoring/scorecard.ts';
|
|
4
|
-
import { ComparisonSpec } from '../comparison/comparison-spec.ts';
|
|
5
|
-
import type { Comparison } from '../comparison/comparison.ts';
|
|
6
|
-
import type { Case } from './case.ts';
|
|
7
|
-
import { PerField } from './per-field.ts';
|
|
8
|
-
export declare class ScorerSet implements ScorecardScorer<Case> {
|
|
9
|
-
#private;
|
|
10
|
-
constructor(entries: Readonly<Record<string, Scorer<Comparison> | PerField>>, { spec }?: {
|
|
11
|
-
readonly spec?: ComparisonSpec;
|
|
12
|
-
});
|
|
13
|
-
score(stated: unknown, testCase: Case): Scorecard;
|
|
14
|
-
}
|
|
15
|
-
//# sourceMappingURL=scorer-set.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAU1C,qBAAa,SAAU,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQnD,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,CAAC,EAChE,EAAE,IAA2B,EAAE,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAA;KAAO;IAM1E,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,SAAS;CAqClD"}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { RatioOfSums } from "../../aggregation/ratio-of-sums.js";
|
|
2
|
-
import { Ratio } from "../../aggregation/ratio.js";
|
|
3
|
-
import { Gate } from "../../scorers/gate.js";
|
|
4
|
-
import { ComparisonSpec } from "../comparison/comparison-spec.js";
|
|
5
|
-
import { statedData } from "../comparison/json-value.js";
|
|
6
|
-
import { TreeWalk } from "../comparison/tree-walk.js";
|
|
7
|
-
import { PerField } from "./per-field.js";
|
|
8
|
-
// Scores a stated value with labelled scorers, through a walk with the spec:
|
|
9
|
-
// the ratio of each scorer under its label, and of a PerField under the label
|
|
10
|
-
// of each field, so a ratio of sums of a label adds the parts and the wholes
|
|
11
|
-
// of the records. The record passes when each of its gates holds, and the
|
|
12
|
-
// reason is the explanations of the gates that fail. Each gate is a check.
|
|
13
|
-
// The score is the mean of the values of the scorers. A stated text is read
|
|
14
|
-
// as json.loads reads it, as a ScorerAssertion reads it; a stated JsonValue
|
|
15
|
-
// stays as it is.
|
|
16
|
-
export class ScorerSet {
|
|
17
|
-
#entries;
|
|
18
|
-
#walk;
|
|
19
|
-
constructor(entries, { spec = new ComparisonSpec() } = {}) {
|
|
20
|
-
this.#entries = Object.entries(entries);
|
|
21
|
-
this.#walk = new TreeWalk(spec);
|
|
22
|
-
}
|
|
23
|
-
score(stated, testCase) {
|
|
24
|
-
const comparison = this.#walk.compare(statedData(stated), testCase);
|
|
25
|
-
const ratios = [];
|
|
26
|
-
const mean = new RatioOfSums();
|
|
27
|
-
const checks = [];
|
|
28
|
-
for (const [label, entry] of this.#entries) {
|
|
29
|
-
if (entry instanceof PerField) {
|
|
30
|
-
ratios.push(...entry.ratios(comparison, label));
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
const ratio = entry.score(comparison);
|
|
34
|
-
ratios.push([label, ratio]);
|
|
35
|
-
mean.add(new Ratio(ratio.value, 1));
|
|
36
|
-
if (entry instanceof Gate) {
|
|
37
|
-
const holds = entry.holds(comparison);
|
|
38
|
-
checks.push({
|
|
39
|
-
name: label,
|
|
40
|
-
passed: holds,
|
|
41
|
-
score: ratio.value,
|
|
42
|
-
reason: holds
|
|
43
|
-
? `${label} holds`
|
|
44
|
-
: `${label}: ${entry.explain(comparison)}`,
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
const failed = checks.filter((check) => !check.passed);
|
|
49
|
-
return {
|
|
50
|
-
passed: failed.length === 0,
|
|
51
|
-
score: mean.value,
|
|
52
|
-
reason: failed.length === 0
|
|
53
|
-
? 'each gate holds'
|
|
54
|
-
: failed.map((check) => check.reason).join('; '),
|
|
55
|
-
ratios: new Map(ratios),
|
|
56
|
-
checks,
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
//# sourceMappingURL=scorer-set.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAI7C,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,6EAA6E;AAC7E,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,4EAA4E;AAC5E,4EAA4E;AAC5E,kBAAkB;AAClB,MAAM,OAAO,SAAS;IACX,QAAQ,CAGZ;IACI,KAAK,CAAW;IAEzB,YACE,OAAgE,EAChE,EAAE,IAAI,GAAG,IAAI,cAAc,EAAE,KAAyC,EAAE;QAExE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAe,EAAE,QAAc;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpE,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK;wBACX,CAAC,CAAC,GAAG,KAAK,QAAQ;wBAClB,CAAC,CAAC,GAAG,KAAK,KAAK,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EACJ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC;YACvB,MAAM;SACP,CAAC;IACJ,CAAC;CACF"}
|