@zukhruf/evals 0.8.14 → 0.8.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,6 +3,9 @@ import type { Scorer } from '../../scorers/scorer.ts';
|
|
|
3
3
|
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
4
|
export declare class RequiredKeyRecall implements Scorer<Comparison> {
|
|
5
5
|
#private;
|
|
6
|
+
constructor({ statedItemsOnly, }?: {
|
|
7
|
+
readonly statedItemsOnly?: boolean;
|
|
8
|
+
});
|
|
6
9
|
score(comparison: Comparison): Ratio;
|
|
7
10
|
}
|
|
8
11
|
//# sourceMappingURL=required-key-recall.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"required-key-recall.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"required-key-recall.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAgB9D,qBAAa,iBAAkB,YAAW,MAAM,CAAC,UAAU,CAAC;;gBAG9C,EACV,eAAuB,GACxB,GAAE;QAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAO;IAI9C,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CAiErC"}
|
|
@@ -6,12 +6,31 @@ import { dictGet } from "../comparison/json-value.js";
|
|
|
6
6
|
// gives a ratio with the whole 0 (evaluate.py gives 1). It has no gate: evaluate.py scores it only for a
|
|
7
7
|
// parsed value and a valid schema (evaluate.py:296-300), and its caller
|
|
8
8
|
// composes that with Penalized.
|
|
9
|
+
//
|
|
10
|
+
// evaluate.py requires the keys of an item of an array also where the stated
|
|
11
|
+
// value has no item, so an empty array, which the schema allows, lacks them:
|
|
12
|
+
// a right answer to a case that expects no item scores below a wrong one that
|
|
13
|
+
// states an item. With statedItemsOnly the keys of an item are required only
|
|
14
|
+
// under an array where the stated value has an item, as the `items` of a JSON
|
|
15
|
+
// Schema asks nothing of an array with no item.
|
|
9
16
|
export class RequiredKeyRecall {
|
|
17
|
+
#statedItemsOnly;
|
|
18
|
+
constructor({ statedItemsOnly = false, } = {}) {
|
|
19
|
+
this.#statedItemsOnly = statedItemsOnly;
|
|
20
|
+
}
|
|
10
21
|
score(comparison) {
|
|
11
|
-
const required = new Set(RequiredKeyRecall.#requiredFields(comparison.schema.value, new Field([])).map((field) => field.id));
|
|
12
22
|
const stated = new Set(comparison.statedFields().map((field) => field.id));
|
|
23
|
+
const required = new Set(RequiredKeyRecall.#requiredFields(comparison.schema.value, new Field([]))
|
|
24
|
+
.filter((field) => !this.#statedItemsOnly ||
|
|
25
|
+
RequiredKeyRecall.#items(field).every((item) => stated.has(item.id)))
|
|
26
|
+
.map((field) => field.id));
|
|
13
27
|
return new Ratio(required.intersection(stated).size, required.size);
|
|
14
28
|
}
|
|
29
|
+
// The items that a field lies under: offers[*] and offers[*].tiers[*] for
|
|
30
|
+
// offers[*].tiers[*].name.
|
|
31
|
+
static #items(field) {
|
|
32
|
+
return field.steps.flatMap((step, at) => step === Field.item ? [new Field(field.steps.slice(0, at + 1))] : []);
|
|
33
|
+
}
|
|
15
34
|
// required_paths of evaluate.py: the required keys of each object schema,
|
|
16
35
|
// through `properties` and `items` only, as fields: evaluate.py writes an
|
|
17
36
|
// item as [], at any position.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"required-key-recall.js","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,uEAAuE;AACvE,4EAA4E;AAC5E,yGAAyG;AACzG,wEAAwE;AACxE,gCAAgC;AAChC,MAAM,OAAO,iBAAiB;
|
|
1
|
+
{"version":3,"file":"required-key-recall.js","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,uEAAuE;AACvE,4EAA4E;AAC5E,yGAAyG;AACzG,wEAAwE;AACxE,gCAAgC;AAChC,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAC9E,gDAAgD;AAChD,MAAM,OAAO,iBAAiB;IACnB,gBAAgB,CAAU;IAEnC,YAAY,EACV,eAAe,GAAG,KAAK,MACmB,EAAE;QAC5C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,iBAAiB,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC;aACtE,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,IAAI,CAAC,gBAAgB;YACtB,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CACpB,CACJ;aACA,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAC5B,CAAC;QACF,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,0EAA0E;IAC1E,2BAA2B;IAC3B,MAAM,CAAC,MAAM,CAAC,KAAY;QACxB,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CACtC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,+BAA+B;IAC/B,MAAM,CAAC,eAAe,CAAC,MAAgB,EAAE,KAAY;QACnD,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC5D,KAAK,MAAM,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAC1C,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAChC,EAAE,CAAC;gBACF,MAAM,KAAK,GACT,UAAU,YAAY,GAAG;oBACvB,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC;oBAC7C,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CACV,QAAQ,EACR,GAAG,iBAAiB,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CACtD,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,GAAG,iBAAiB,CAAC,eAAe,CAClC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,EACnC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACxB,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,MAAM,CAAC,QAAQ,CAAC,KAAe;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QACjD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.16",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@zukhruf/python": "^0.8.
|
|
67
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.16",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.16"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"@cfworker/json-schema": "4.1.4",
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@cfworker/json-schema": "4.1.4",
|
|
83
83
|
"@zukhruf/random": "*",
|
|
84
|
-
"@zukhruf/testing": "^0.8.
|
|
84
|
+
"@zukhruf/testing": "^0.8.16",
|
|
85
85
|
"munkres": "^2.1.1",
|
|
86
86
|
"promptfoo": "0.125.1",
|
|
87
87
|
"python-random": "1.0.0"
|