@zukhruf/evals 0.8.15 → 0.8.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,9 @@ import type { Scorer } from '../../scorers/scorer.ts';
3
3
  import type { Comparison } from '../comparison/comparison.ts';
4
4
  export declare class RequiredKeyRecall implements Scorer<Comparison> {
5
5
  #private;
6
+ constructor({ statedItemsOnly, }?: {
7
+ readonly statedItemsOnly?: boolean;
8
+ });
6
9
  score(comparison: Comparison): Ratio;
7
10
  }
8
11
  //# sourceMappingURL=required-key-recall.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"required-key-recall.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAS9D,qBAAa,iBAAkB,YAAW,MAAM,CAAC,UAAU,CAAC;;IAC1D,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CAoDrC"}
1
+ {"version":3,"file":"required-key-recall.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAgB9D,qBAAa,iBAAkB,YAAW,MAAM,CAAC,UAAU,CAAC;;gBAG9C,EACV,eAAuB,GACxB,GAAE;QAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAO;IAI9C,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CAiErC"}
@@ -6,12 +6,31 @@ import { dictGet } from "../comparison/json-value.js";
6
6
  // gives a ratio with the whole 0 (evaluate.py gives 1). It has no gate: evaluate.py scores it only for a
7
7
  // parsed value and a valid schema (evaluate.py:296-300), and its caller
8
8
  // composes that with Penalized.
9
+ //
10
+ // evaluate.py requires the keys of an item of an array also where the stated
11
+ // value has no item, so an empty array, which the schema allows, lacks them:
12
+ // a right answer to a case that expects no item scores below a wrong one that
13
+ // states an item. With statedItemsOnly the keys of an item are required only
14
+ // under an array where the stated value has an item, as the `items` of a JSON
15
+ // Schema asks nothing of an array with no item.
9
16
  export class RequiredKeyRecall {
17
+ #statedItemsOnly;
18
+ constructor({ statedItemsOnly = false, } = {}) {
19
+ this.#statedItemsOnly = statedItemsOnly;
20
+ }
10
21
  score(comparison) {
11
- const required = new Set(RequiredKeyRecall.#requiredFields(comparison.schema.value, new Field([])).map((field) => field.id));
12
22
  const stated = new Set(comparison.statedFields().map((field) => field.id));
23
+ const required = new Set(RequiredKeyRecall.#requiredFields(comparison.schema.value, new Field([]))
24
+ .filter((field) => !this.#statedItemsOnly ||
25
+ RequiredKeyRecall.#items(field).every((item) => stated.has(item.id)))
26
+ .map((field) => field.id));
13
27
  return new Ratio(required.intersection(stated).size, required.size);
14
28
  }
29
+ // The items that a field lies under: offers[*] and offers[*].tiers[*] for
30
+ // offers[*].tiers[*].name.
31
+ static #items(field) {
32
+ return field.steps.flatMap((step, at) => step === Field.item ? [new Field(field.steps.slice(0, at + 1))] : []);
33
+ }
15
34
  // required_paths of evaluate.py: the required keys of each object schema,
16
35
  // through `properties` and `items` only, as fields: evaluate.py writes an
17
36
  // item as [], at any position.
@@ -1 +1 @@
1
- {"version":3,"file":"required-key-recall.js","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,uEAAuE;AACvE,4EAA4E;AAC5E,yGAAyG;AACzG,wEAAwE;AACxE,gCAAgC;AAChC,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,UAAsB;QAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,iBAAiB,CAAC,eAAe,CAC/B,UAAU,CAAC,MAAM,CAAC,KAAK,EACvB,IAAI,KAAK,CAAC,EAAE,CAAC,CACd,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAC3B,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,+BAA+B;IAC/B,MAAM,CAAC,eAAe,CAAC,MAAgB,EAAE,KAAY;QACnD,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC5D,KAAK,MAAM,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAC1C,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAChC,EAAE,CAAC;gBACF,MAAM,KAAK,GACT,UAAU,YAAY,GAAG;oBACvB,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC;oBAC7C,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CACV,QAAQ,EACR,GAAG,iBAAiB,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CACtD,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,GAAG,iBAAiB,CAAC,eAAe,CAClC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,EACnC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACxB,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,MAAM,CAAC,QAAQ,CAAC,KAAe;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QACjD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
1
+ {"version":3,"file":"required-key-recall.js","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,uEAAuE;AACvE,4EAA4E;AAC5E,yGAAyG;AACzG,wEAAwE;AACxE,gCAAgC;AAChC,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAC9E,gDAAgD;AAChD,MAAM,OAAO,iBAAiB;IACnB,gBAAgB,CAAU;IAEnC,YAAY,EACV,eAAe,GAAG,KAAK,MACmB,EAAE;QAC5C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,iBAAiB,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC;aACtE,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,IAAI,CAAC,gBAAgB;YACtB,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CACpB,CACJ;aACA,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAC5B,CAAC;QACF,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,0EAA0E;IAC1E,2BAA2B;IAC3B,MAAM,CAAC,MAAM,CAAC,KAAY;QACxB,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CACtC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,+BAA+B;IAC/B,MAAM,CAAC,eAAe,CAAC,MAAgB,EAAE,KAAY;QACnD,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC5D,KAAK,MAAM,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAC1C,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAChC,EAAE,CAAC;gBACF,MAAM,KAAK,GACT,UAAU,YAAY,GAAG;oBACvB,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC;oBAC7C,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CACV,QAAQ,EACR,GAAG,iBAAiB,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CACtD,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,GAAG,iBAAiB,CAAC,eAAe,CAClC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,EACnC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACxB,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,MAAM,CAAC,QAAQ,CAAC,KAAe;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QACjD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zukhruf/evals",
3
- "version": "0.8.15",
3
+ "version": "0.8.16",
4
4
  "description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -63,8 +63,8 @@
63
63
  "access": "public"
64
64
  },
65
65
  "dependencies": {
66
- "@zukhruf/python": "^0.8.15",
67
- "@zukhruf/statistics": "^0.8.15"
66
+ "@zukhruf/python": "^0.8.16",
67
+ "@zukhruf/statistics": "^0.8.16"
68
68
  },
69
69
  "peerDependencies": {
70
70
  "@cfworker/json-schema": "4.1.4",
@@ -81,7 +81,7 @@
81
81
  "devDependencies": {
82
82
  "@cfworker/json-schema": "4.1.4",
83
83
  "@zukhruf/random": "*",
84
- "@zukhruf/testing": "^0.8.15",
84
+ "@zukhruf/testing": "^0.8.16",
85
85
  "munkres": "^2.1.1",
86
86
  "promptfoo": "0.125.1",
87
87
  "python-random": "1.0.0"