@zukhruf/evals 0.0.0-stage → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +82 -0
- package/README.md +528 -2
- package/dist/aggregation/counts.d.ts +11 -0
- package/dist/aggregation/counts.d.ts.map +1 -0
- package/dist/aggregation/counts.js +14 -0
- package/dist/aggregation/counts.js.map +1 -0
- package/dist/aggregation/f1-ratio.d.ts +8 -0
- package/dist/aggregation/f1-ratio.d.ts.map +1 -0
- package/dist/aggregation/f1-ratio.js +24 -0
- package/dist/aggregation/f1-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +8 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
- package/dist/aggregation/ratio-of-sums.js +21 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -0
- package/dist/aggregation/ratio.d.ts +8 -0
- package/dist/aggregation/ratio.d.ts.map +1 -0
- package/dist/aggregation/ratio.js +20 -0
- package/dist/aggregation/ratio.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/promptfoo/assertion-module.d.ts +23 -0
- package/dist/promptfoo/assertion-module.d.ts.map +1 -0
- package/dist/promptfoo/assertion-module.js +45 -0
- package/dist/promptfoo/assertion-module.js.map +1 -0
- package/dist/promptfoo/index.d.ts +3 -0
- package/dist/promptfoo/index.d.ts.map +1 -0
- package/dist/promptfoo/index.js +3 -0
- package/dist/promptfoo/index.js.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.js +31 -0
- package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
- package/dist/scorers/at-least.d.ts +9 -0
- package/dist/scorers/at-least.d.ts.map +1 -0
- package/dist/scorers/at-least.js +18 -0
- package/dist/scorers/at-least.js.map +1 -0
- package/dist/scorers/f1.d.ts +9 -0
- package/dist/scorers/f1.d.ts.map +1 -0
- package/dist/scorers/f1.js +12 -0
- package/dist/scorers/f1.js.map +1 -0
- package/dist/scorers/gate.d.ts +7 -0
- package/dist/scorers/gate.d.ts.map +1 -0
- package/dist/scorers/gate.js +10 -0
- package/dist/scorers/gate.js.map +1 -0
- package/dist/scorers/matches.d.ts +5 -0
- package/dist/scorers/matches.d.ts.map +1 -0
- package/dist/scorers/matches.js +2 -0
- package/dist/scorers/matches.js.map +1 -0
- package/dist/scorers/mean-of.d.ts +8 -0
- package/dist/scorers/mean-of.d.ts.map +1 -0
- package/dist/scorers/mean-of.js +15 -0
- package/dist/scorers/mean-of.js.map +1 -0
- package/dist/scorers/penalized.d.ts +9 -0
- package/dist/scorers/penalized.d.ts.map +1 -0
- package/dist/scorers/penalized.js +28 -0
- package/dist/scorers/penalized.js.map +1 -0
- package/dist/scorers/penalty.d.ts +7 -0
- package/dist/scorers/penalty.d.ts.map +1 -0
- package/dist/scorers/penalty.js +11 -0
- package/dist/scorers/penalty.js.map +1 -0
- package/dist/scorers/precision.d.ts +9 -0
- package/dist/scorers/precision.d.ts.map +1 -0
- package/dist/scorers/precision.js +13 -0
- package/dist/scorers/precision.js.map +1 -0
- package/dist/scorers/recall.d.ts +9 -0
- package/dist/scorers/recall.d.ts.map +1 -0
- package/dist/scorers/recall.js +13 -0
- package/dist/scorers/recall.js.map +1 -0
- package/dist/scorers/sample-averaged.d.ts +10 -0
- package/dist/scorers/sample-averaged.d.ts.map +1 -0
- package/dist/scorers/sample-averaged.js +17 -0
- package/dist/scorers/sample-averaged.js.map +1 -0
- package/dist/scorers/scorer.d.ts +5 -0
- package/dist/scorers/scorer.d.ts.map +1 -0
- package/dist/scorers/scorer.js +2 -0
- package/dist/scorers/scorer.js.map +1 -0
- package/dist/scorers/shortfall-penalty.d.ts +11 -0
- package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
- package/dist/scorers/shortfall-penalty.js +25 -0
- package/dist/scorers/shortfall-penalty.js.map +1 -0
- package/dist/scoring/bootstrap.d.ts +16 -0
- package/dist/scoring/bootstrap.d.ts.map +1 -0
- package/dist/scoring/bootstrap.js +31 -0
- package/dist/scoring/bootstrap.js.map +1 -0
- package/dist/scoring/scorecard-scorer.d.ts +11 -0
- package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
- package/dist/scoring/scorecard-scorer.js +2 -0
- package/dist/scoring/scorecard-scorer.js.map +1 -0
- package/dist/scoring/scorecard.d.ts +15 -0
- package/dist/scoring/scorecard.d.ts.map +1 -0
- package/dist/scoring/scorecard.js +2 -0
- package/dist/scoring/scorecard.js.map +1 -0
- package/dist/scoring/totals.d.ts +9 -0
- package/dist/scoring/totals.d.ts.map +1 -0
- package/dist/scoring/totals.js +23 -0
- package/dist/scoring/totals.js.map +1 -0
- package/dist/structured/comparison/alignment.d.ts +9 -0
- package/dist/structured/comparison/alignment.d.ts.map +1 -0
- package/dist/structured/comparison/alignment.js +2 -0
- package/dist/structured/comparison/alignment.js.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.js +25 -0
- package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
- package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
- package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-position.js +9 -0
- package/dist/structured/comparison/alignments/by-position.js.map +1 -0
- package/dist/structured/comparison/bag-of-words.d.ts +10 -0
- package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
- package/dist/structured/comparison/bag-of-words.js +48 -0
- package/dist/structured/comparison/bag-of-words.js.map +1 -0
- package/dist/structured/comparison/comparator.d.ts +5 -0
- package/dist/structured/comparison/comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparator.js +2 -0
- package/dist/structured/comparison/comparator.js.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.js +18 -0
- package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.js +13 -0
- package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
- package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized.js +3 -0
- package/dist/structured/comparison/comparators/normalized.js.map +1 -0
- package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
- package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/number-within.js +18 -0
- package/dist/structured/comparison/comparators/number-within.js.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.js +15 -0
- package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +9 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +20 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
- package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
- package/dist/structured/comparison/comparison-spec.d.ts +17 -0
- package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
- package/dist/structured/comparison/comparison-spec.js +36 -0
- package/dist/structured/comparison/comparison-spec.js.map +1 -0
- package/dist/structured/comparison/comparison.d.ts +25 -0
- package/dist/structured/comparison/comparison.d.ts.map +1 -0
- package/dist/structured/comparison/comparison.js +94 -0
- package/dist/structured/comparison/comparison.js.map +1 -0
- package/dist/structured/comparison/field.d.ts +10 -0
- package/dist/structured/comparison/field.d.ts.map +1 -0
- package/dist/structured/comparison/field.js +29 -0
- package/dist/structured/comparison/field.js.map +1 -0
- package/dist/structured/comparison/item-pairs.d.ts +8 -0
- package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
- package/dist/structured/comparison/item-pairs.js +2 -0
- package/dist/structured/comparison/item-pairs.js.map +1 -0
- package/dist/structured/comparison/json-path-query.d.ts +7 -0
- package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
- package/dist/structured/comparison/json-path-query.js +133 -0
- package/dist/structured/comparison/json-path-query.js.map +1 -0
- package/dist/structured/comparison/json-value.d.ts +10 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -0
- package/dist/structured/comparison/json-value.js +47 -0
- package/dist/structured/comparison/json-value.js.map +1 -0
- package/dist/structured/comparison/leaf-pair.d.ts +24 -0
- package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
- package/dist/structured/comparison/leaf-pair.js +2 -0
- package/dist/structured/comparison/leaf-pair.js.map +1 -0
- package/dist/structured/comparison/match.d.ts +5 -0
- package/dist/structured/comparison/match.d.ts.map +1 -0
- package/dist/structured/comparison/match.js +2 -0
- package/dist/structured/comparison/match.js.map +1 -0
- package/dist/structured/comparison/name-selector.d.ts +6 -0
- package/dist/structured/comparison/name-selector.d.ts.map +1 -0
- package/dist/structured/comparison/name-selector.js +41 -0
- package/dist/structured/comparison/name-selector.js.map +1 -0
- package/dist/structured/comparison/path.d.ts +10 -0
- package/dist/structured/comparison/path.d.ts.map +1 -0
- package/dist/structured/comparison/path.js +27 -0
- package/dist/structured/comparison/path.js.map +1 -0
- package/dist/structured/comparison/tree-walk.d.ts +10 -0
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
- package/dist/structured/comparison/tree-walk.js +136 -0
- package/dist/structured/comparison/tree-walk.js.map +1 -0
- package/dist/structured/cpython/python-text.d.ts +5 -0
- package/dist/structured/cpython/python-text.d.ts.map +1 -0
- package/dist/structured/cpython/python-text.js +79 -0
- package/dist/structured/cpython/python-text.js.map +1 -0
- package/dist/structured/index.d.ts +39 -0
- package/dist/structured/index.d.ts.map +1 -0
- package/dist/structured/index.js +29 -0
- package/dist/structured/index.js.map +1 -0
- package/dist/structured/promptfoo/scorer-assertion.d.ts +22 -0
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/structured/promptfoo/scorer-assertion.js +34 -0
- package/dist/structured/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/structured/schema/draft-07-schema.d.ts +3 -0
- package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-07-schema.js +243 -0
- package/dist/structured/schema/draft-07-schema.js.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.js +280 -0
- package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
- package/dist/structured/schema/is-record.d.ts +2 -0
- package/dist/structured/schema/is-record.d.ts.map +1 -0
- package/dist/structured/schema/is-record.js +3 -0
- package/dist/structured/schema/is-record.js.map +1 -0
- package/dist/structured/schema/json-schema.d.ts +20 -0
- package/dist/structured/schema/json-schema.d.ts.map +1 -0
- package/dist/structured/schema/json-schema.js +244 -0
- package/dist/structured/schema/json-schema.js.map +1 -0
- package/dist/structured/schema/schema-type-walk.d.ts +11 -0
- package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
- package/dist/structured/schema/schema-type-walk.js +196 -0
- package/dist/structured/schema/schema-type-walk.js.map +1 -0
- package/dist/structured/scorers/all-matched.d.ts +10 -0
- package/dist/structured/scorers/all-matched.d.ts.map +1 -0
- package/dist/structured/scorers/all-matched.js +36 -0
- package/dist/structured/scorers/all-matched.js.map +1 -0
- package/dist/structured/scorers/exact-match.d.ts +7 -0
- package/dist/structured/scorers/exact-match.d.ts.map +1 -0
- package/dist/structured/scorers/exact-match.js +13 -0
- package/dist/structured/scorers/exact-match.js.map +1 -0
- package/dist/structured/scorers/item-matches.d.ts +7 -0
- package/dist/structured/scorers/item-matches.d.ts.map +1 -0
- package/dist/structured/scorers/item-matches.js +17 -0
- package/dist/structured/scorers/item-matches.js.map +1 -0
- package/dist/structured/scorers/leaf-matches.d.ts +14 -0
- package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
- package/dist/structured/scorers/leaf-matches.js +28 -0
- package/dist/structured/scorers/leaf-matches.js.map +1 -0
- package/dist/structured/scorers/parse-success.d.ts +7 -0
- package/dist/structured/scorers/parse-success.d.ts.map +1 -0
- package/dist/structured/scorers/parse-success.js +11 -0
- package/dist/structured/scorers/parse-success.js.map +1 -0
- package/dist/structured/scorers/path-matches.d.ts +5 -0
- package/dist/structured/scorers/path-matches.d.ts.map +1 -0
- package/dist/structured/scorers/path-matches.js +9 -0
- package/dist/structured/scorers/path-matches.js.map +1 -0
- package/dist/structured/scorers/required-key-recall.d.ts +8 -0
- package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
- package/dist/structured/scorers/required-key-recall.js +50 -0
- package/dist/structured/scorers/required-key-recall.js.map +1 -0
- package/dist/structured/scorers/schema-compliance.d.ts +8 -0
- package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
- package/dist/structured/scorers/schema-compliance.js +29 -0
- package/dist/structured/scorers/schema-compliance.js.map +1 -0
- package/dist/structured/scorers/token-f1.d.ts +14 -0
- package/dist/structured/scorers/token-f1.d.ts.map +1 -0
- package/dist/structured/scorers/token-f1.js +44 -0
- package/dist/structured/scorers/token-f1.js.map +1 -0
- package/dist/structured/scorers/type-precision.d.ts +10 -0
- package/dist/structured/scorers/type-precision.d.ts.map +1 -0
- package/dist/structured/scorers/type-precision.js +30 -0
- package/dist/structured/scorers/type-precision.js.map +1 -0
- package/dist/structured/scorers/valid-schema.d.ts +7 -0
- package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
- package/dist/structured/scorers/valid-schema.js +11 -0
- package/dist/structured/scorers/valid-schema.js.map +1 -0
- package/dist/structured/scorers/value-matches.d.ts +6 -0
- package/dist/structured/scorers/value-matches.d.ts.map +1 -0
- package/dist/structured/scorers/value-matches.js +9 -0
- package/dist/structured/scorers/value-matches.js.map +1 -0
- package/dist/structured/scoring/case.d.ts +14 -0
- package/dist/structured/scoring/case.d.ts.map +1 -0
- package/dist/structured/scoring/case.js +14 -0
- package/dist/structured/scoring/case.js.map +1 -0
- package/dist/structured/scoring/config-case.d.ts +11 -0
- package/dist/structured/scoring/config-case.d.ts.map +1 -0
- package/dist/structured/scoring/config-case.js +22 -0
- package/dist/structured/scoring/config-case.js.map +1 -0
- package/dist/structured/scoring/per-field.d.ts +9 -0
- package/dist/structured/scoring/per-field.d.ts.map +1 -0
- package/dist/structured/scoring/per-field.js +23 -0
- package/dist/structured/scoring/per-field.js.map +1 -0
- package/dist/structured/scoring/scorer-set.d.ts +15 -0
- package/dist/structured/scoring/scorer-set.d.ts.map +1 -0
- package/dist/structured/scoring/scorer-set.js +60 -0
- package/dist/structured/scoring/scorer-set.js.map +1 -0
- package/dist/structured/scoring/text-vars-case.d.ts +12 -0
- package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
- package/dist/structured/scoring/text-vars-case.js +31 -0
- package/dist/structured/scoring/text-vars-case.js.map +1 -0
- package/package.json +78 -4
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Counts } from "../../aggregation/counts.js";
|
|
2
|
+
// The counts of the leaf pairs of one comparison: the stated leaves, the
|
|
3
|
+
// expected leaves, and the hits among the pairs with both sides. A subclass
|
|
4
|
+
// says only what a hit is (Template Method).
|
|
5
|
+
export class LeafMatches {
|
|
6
|
+
#nullIsMissing;
|
|
7
|
+
constructor({ nullIsMissing = false } = {}) {
|
|
8
|
+
this.#nullIsMissing = nullIsMissing;
|
|
9
|
+
}
|
|
10
|
+
counts(comparison) {
|
|
11
|
+
let hits = 0;
|
|
12
|
+
let stated = 0;
|
|
13
|
+
let expected = 0;
|
|
14
|
+
for (const pair of comparison.leaves({
|
|
15
|
+
nullIsMissing: this.#nullIsMissing,
|
|
16
|
+
})) {
|
|
17
|
+
if (pair.stated !== undefined)
|
|
18
|
+
stated++;
|
|
19
|
+
if (pair.expected === undefined)
|
|
20
|
+
continue;
|
|
21
|
+
expected++;
|
|
22
|
+
if (pair.stated !== undefined && this.hit(pair))
|
|
23
|
+
hits++;
|
|
24
|
+
}
|
|
25
|
+
return new Counts({ hits, stated, expected });
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=leaf-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leaf-matches.js","sourceRoot":"","sources":["../../../src/structured/scorers/leaf-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AASrD,yEAAyE;AACzE,4EAA4E;AAC5E,6CAA6C;AAC7C,MAAM,OAAgB,WAAW;IACtB,cAAc,CAAU;IAEjC,YAAY,EAAE,aAAa,GAAG,KAAK,KAAkB,EAAE;QACrD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAID,MAAM,CAAC,UAAsB;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;gBAAE,MAAM,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YAC1C,QAAQ,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,IAAI,EAAE,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAChD,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Gate } from '../../scorers/gate.ts';
|
|
2
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
3
|
+
export declare class ParseSuccess extends Gate<Comparison> {
|
|
4
|
+
holds({ parsed }: Comparison): boolean;
|
|
5
|
+
explain(): string;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=parse-success.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-success.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/parse-success.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,qBAAa,YAAa,SAAQ,IAAI,CAAC,UAAU,CAAC;IAChD,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,UAAU,GAAG,OAAO;IAItC,OAAO,IAAI,MAAM;CAGlB"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Gate } from "../../scorers/gate.js";
|
|
2
|
+
// 1 when the stated value is a JSON object or array, else 0.
|
|
3
|
+
export class ParseSuccess extends Gate {
|
|
4
|
+
holds({ parsed }) {
|
|
5
|
+
return parsed;
|
|
6
|
+
}
|
|
7
|
+
explain() {
|
|
8
|
+
return 'the stated value is not a JSON object or array';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=parse-success.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-success.js","sourceRoot":"","sources":["../../../src/structured/scorers/parse-success.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAG7C,6DAA6D;AAC7D,MAAM,OAAO,YAAa,SAAQ,IAAgB;IAChD,KAAK,CAAC,EAAE,MAAM,EAAc;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO;QACL,OAAO,gDAAgD,CAAC;IAC1D,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"path-matches.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/path-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,qBAAa,WAAY,SAAQ,WAAW;IAC1C,SAAS,CAAC,GAAG,IAAI,OAAO;CAGzB"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { LeafMatches } from "./leaf-matches.js";
|
|
2
|
+
// A hit is a leaf pair with a value on both sides, whatever the two values
|
|
3
|
+
// (presence).
|
|
4
|
+
export class PathMatches extends LeafMatches {
|
|
5
|
+
hit() {
|
|
6
|
+
return true;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=path-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"path-matches.js","sourceRoot":"","sources":["../../../src/structured/scorers/path-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,2EAA2E;AAC3E,cAAc;AACd,MAAM,OAAO,WAAY,SAAQ,WAAW;IAChC,GAAG;QACX,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Ratio } from '../../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../../scorers/scorer.ts';
|
|
3
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
+
export declare class RequiredKeyRecall implements Scorer<Comparison> {
|
|
5
|
+
#private;
|
|
6
|
+
score(comparison: Comparison): Ratio;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=required-key-recall.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"required-key-recall.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAS9D,qBAAa,iBAAkB,YAAW,MAAM,CAAC,UAAU,CAAC;;IAC1D,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CAoDrC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { Ratio } from "../../aggregation/ratio.js";
|
|
2
|
+
import { Field } from "../comparison/field.js";
|
|
3
|
+
import { dictGet } from "../comparison/json-value.js";
|
|
4
|
+
// The required paths of the schema that the stated value has, over the
|
|
5
|
+
// required paths (compute_required_key_recall). A schema that requires none
|
|
6
|
+
// gives a ratio with the whole 0 (evaluate.py gives 1). It has no gate: evaluate.py scores it only for a
|
|
7
|
+
// parsed value and a valid schema (evaluate.py:296-300), and its caller
|
|
8
|
+
// composes that with Penalized.
|
|
9
|
+
export class RequiredKeyRecall {
|
|
10
|
+
score(comparison) {
|
|
11
|
+
const required = new Set(RequiredKeyRecall.#requiredFields(comparison.schema.value, new Field([])).map((field) => field.id));
|
|
12
|
+
const stated = new Set(comparison.statedFields().map((field) => field.id));
|
|
13
|
+
return new Ratio(required.intersection(stated).size, required.size);
|
|
14
|
+
}
|
|
15
|
+
// required_paths of evaluate.py: the required keys of each object schema,
|
|
16
|
+
// through `properties` and `items` only, as fields: evaluate.py writes an
|
|
17
|
+
// item as [], at any position.
|
|
18
|
+
static #requiredFields(schema, field) {
|
|
19
|
+
if (!(schema instanceof Map))
|
|
20
|
+
return [];
|
|
21
|
+
const type = schema.get('type');
|
|
22
|
+
const results = [];
|
|
23
|
+
if (type === 'object') {
|
|
24
|
+
const properties = dictGet(schema, 'properties', new Map());
|
|
25
|
+
for (const key of RequiredKeyRecall.#iterate(dictGet(schema, 'required', []))) {
|
|
26
|
+
const child = properties instanceof Map
|
|
27
|
+
? dictGet(properties, String(key), new Map())
|
|
28
|
+
: new Map();
|
|
29
|
+
const keyField = field.child(String(key));
|
|
30
|
+
results.push(keyField, ...RequiredKeyRecall.#requiredFields(child, keyField));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
else if (type === 'array') {
|
|
34
|
+
results.push(...RequiredKeyRecall.#requiredFields(dictGet(schema, 'items', new Map()), field.child(Field.item)));
|
|
35
|
+
}
|
|
36
|
+
return results;
|
|
37
|
+
}
|
|
38
|
+
// A Python for loop over a list, the keys of a dict, or the characters of a
|
|
39
|
+
// str.
|
|
40
|
+
static #iterate(value) {
|
|
41
|
+
if (Array.isArray(value))
|
|
42
|
+
return value;
|
|
43
|
+
if (value instanceof Map)
|
|
44
|
+
return [...value.keys()];
|
|
45
|
+
if (typeof value === 'string')
|
|
46
|
+
return [...value];
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=required-key-recall.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"required-key-recall.js","sourceRoot":"","sources":["../../../src/structured/scorers/required-key-recall.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,uEAAuE;AACvE,4EAA4E;AAC5E,yGAAyG;AACzG,wEAAwE;AACxE,gCAAgC;AAChC,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,UAAsB;QAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,iBAAiB,CAAC,eAAe,CAC/B,UAAU,CAAC,MAAM,CAAC,KAAK,EACvB,IAAI,KAAK,CAAC,EAAE,CAAC,CACd,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAC3B,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,+BAA+B;IAC/B,MAAM,CAAC,eAAe,CAAC,MAAgB,EAAE,KAAY;QACnD,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC5D,KAAK,MAAM,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAC1C,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAChC,EAAE,CAAC;gBACF,MAAM,KAAK,GACT,UAAU,YAAY,GAAG;oBACvB,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC;oBAC7C,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CACV,QAAQ,EACR,GAAG,iBAAiB,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CACtD,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,GAAG,iBAAiB,CAAC,eAAe,CAClC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,EACnC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACxB,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,MAAM,CAAC,QAAQ,CAAC,KAAe;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QACjD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Gate } from '../../scorers/gate.ts';
|
|
2
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
3
|
+
export declare class SchemaCompliance extends Gate<Comparison> {
|
|
4
|
+
#private;
|
|
5
|
+
holds({ compliant }: Comparison): boolean;
|
|
6
|
+
explain(comparison: Comparison): string;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=schema-compliance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-compliance.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/schema-compliance.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAM9D,qBAAa,gBAAiB,SAAQ,IAAI,CAAC,UAAU,CAAC;;IAQpD,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,UAAU,GAAG,OAAO;IAIzC,OAAO,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM;CAaxC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Gate } from "../../scorers/gate.js";
|
|
2
|
+
import { ParseSuccess } from "./parse-success.js";
|
|
3
|
+
import { ValidSchema } from "./valid-schema.js";
|
|
4
|
+
// 1 when the stated value is parsed and agrees with a valid schema, else 0.
|
|
5
|
+
// Its explanation names the schema errors.
|
|
6
|
+
export class SchemaCompliance extends Gate {
|
|
7
|
+
static #parsed = new ParseSuccess();
|
|
8
|
+
static #validSchema = new ValidSchema();
|
|
9
|
+
// A union schema fails once in each branch, so the explanation shows the
|
|
10
|
+
// first errors only.
|
|
11
|
+
static #shown = 5;
|
|
12
|
+
holds({ compliant }) {
|
|
13
|
+
return compliant;
|
|
14
|
+
}
|
|
15
|
+
explain(comparison) {
|
|
16
|
+
for (const gate of [
|
|
17
|
+
SchemaCompliance.#parsed,
|
|
18
|
+
SchemaCompliance.#validSchema,
|
|
19
|
+
])
|
|
20
|
+
if (!gate.holds(comparison))
|
|
21
|
+
return gate.explain(comparison);
|
|
22
|
+
const { errors } = comparison.validation;
|
|
23
|
+
const more = errors.length > SchemaCompliance.#shown
|
|
24
|
+
? ` and ${errors.length - SchemaCompliance.#shown} more`
|
|
25
|
+
: '';
|
|
26
|
+
return `the record does not agree with the schema: ${errors.slice(0, SchemaCompliance.#shown).join(', ')}${more}`;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=schema-compliance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-compliance.js","sourceRoot":"","sources":["../../../src/structured/scorers/schema-compliance.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,4EAA4E;AAC5E,2CAA2C;AAC3C,MAAM,OAAO,gBAAiB,SAAQ,IAAgB;IACpD,MAAM,CAAU,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IAC7C,MAAM,CAAU,YAAY,GAAG,IAAI,WAAW,EAAE,CAAC;IAEjD,yEAAyE;IACzE,qBAAqB;IACrB,MAAM,CAAU,MAAM,GAAG,CAAC,CAAC;IAE3B,KAAK,CAAC,EAAE,SAAS,EAAc;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,CAAC,UAAsB;QAC5B,KAAK,MAAM,IAAI,IAAI;YACjB,gBAAgB,CAAC,OAAO;YACxB,gBAAgB,CAAC,YAAY;SAC9B;YACC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;gBAAE,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,UAAU,CAAC;QACzC,MAAM,IAAI,GACR,MAAM,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM;YACrC,CAAC,CAAC,QAAQ,MAAM,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,OAAO;YACxD,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,8CAA8C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACpH,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Ratio } from '../../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../../scorers/scorer.ts';
|
|
3
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
+
import type { LeafValue } from '../comparison/leaf-pair.ts';
|
|
5
|
+
import type { LeafOptions } from './leaf-matches.ts';
|
|
6
|
+
export declare class TokenF1 implements Scorer<Comparison> {
|
|
7
|
+
#private;
|
|
8
|
+
constructor({ whenNoWords, absentAs, nullIsMissing, }: {
|
|
9
|
+
readonly whenNoWords: number;
|
|
10
|
+
readonly absentAs: LeafValue;
|
|
11
|
+
} & LeafOptions);
|
|
12
|
+
score(comparison: Comparison): Ratio;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=token-f1.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-f1.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrD,qBAAa,OAAQ,YAAW,MAAM,CAAC,UAAU,CAAC;;gBAKpC,EACV,WAAW,EACX,QAAQ,EACR,aAAqB,GACtB,EAAE;QACD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC;KAC9B,GAAG,WAAW;IAMf,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CA2BrC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
+
import { F1Ratio } from "../../aggregation/f1-ratio.js";
|
|
3
|
+
import { Ratio } from "../../aggregation/ratio.js";
|
|
4
|
+
import { BagOfWords } from "../comparison/bag-of-words.js";
|
|
5
|
+
// The sum of the token F1 of each expected leaf and the stated value at its
|
|
6
|
+
// path, over the expected leaves (mean_token_f1 of evaluate.py). Words are
|
|
7
|
+
// read as SQuAD's normalize_answer reads them. A leaf where neither value has
|
|
8
|
+
// a word gives whenNoWords (evaluate.py gives 1), and a path that the stated
|
|
9
|
+
// value does not have is read as absentAs (evaluate.py reads None). With
|
|
10
|
+
// nullIsMissing, a null leaf is no value: an expected null is not counted,
|
|
11
|
+
// and a stated null is read as absentAs.
|
|
12
|
+
export class TokenF1 {
|
|
13
|
+
#whenNoWords;
|
|
14
|
+
#absentAs;
|
|
15
|
+
#nullIsMissing;
|
|
16
|
+
constructor({ whenNoWords, absentAs, nullIsMissing = false, }) {
|
|
17
|
+
this.#whenNoWords = whenNoWords;
|
|
18
|
+
this.#absentAs = absentAs;
|
|
19
|
+
this.#nullIsMissing = nullIsMissing;
|
|
20
|
+
}
|
|
21
|
+
score(comparison) {
|
|
22
|
+
// CPython's sum, so the mean of the floats is that of evaluate.py.
|
|
23
|
+
const sum = new NeumaierSum();
|
|
24
|
+
let expected = 0;
|
|
25
|
+
for (const pair of comparison.leaves({
|
|
26
|
+
nullIsMissing: this.#nullIsMissing,
|
|
27
|
+
})) {
|
|
28
|
+
if (pair.expected === undefined)
|
|
29
|
+
continue;
|
|
30
|
+
expected++;
|
|
31
|
+
sum.add(this.#tokenF1(pair.expected.value, pair.stated === undefined ? this.#absentAs : pair.stated.value));
|
|
32
|
+
}
|
|
33
|
+
return new Ratio(sum.value, expected);
|
|
34
|
+
}
|
|
35
|
+
// token_f1 of evaluate.py: the F1 of the shared words.
|
|
36
|
+
#tokenF1(expected, stated) {
|
|
37
|
+
const expectedWords = BagOfWords.of(expected);
|
|
38
|
+
const statedWords = BagOfWords.of(stated);
|
|
39
|
+
if (expectedWords.size === 0 && statedWords.size === 0)
|
|
40
|
+
return this.#whenNoWords;
|
|
41
|
+
return new F1Ratio(expectedWords.shared(statedWords)).value;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=token-f1.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-f1.js","sourceRoot":"","sources":["../../../src/structured/scorers/token-f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAK3D,4EAA4E;AAC5E,2EAA2E;AAC3E,8EAA8E;AAC9E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,yCAAyC;AACzC,MAAM,OAAO,OAAO;IACT,YAAY,CAAS;IACrB,SAAS,CAAY;IACrB,cAAc,CAAU;IAEjC,YAAY,EACV,WAAW,EACX,QAAQ,EACR,aAAa,GAAG,KAAK,GAIR;QACb,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,mEAAmE;QACnE,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YAC1C,QAAQ,EAAE,CAAC;YACX,GAAG,CAAC,GAAG,CACL,IAAI,CAAC,QAAQ,CACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EACnB,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAC/D,CACF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,uDAAuD;IACvD,QAAQ,CAAC,QAAmB,EAAE,MAAiB;QAC7C,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YACpD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9D,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Ratio } from '../../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../../scorers/scorer.ts';
|
|
3
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
+
import type { LeafOptions } from './leaf-matches.ts';
|
|
5
|
+
export declare class TypePrecision implements Scorer<Comparison> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor({ nullIsMissing }?: LeafOptions);
|
|
8
|
+
score(comparison: Comparison): Ratio;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=type-precision.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"type-precision.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/type-precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrD,qBAAa,aAAc,YAAW,MAAM,CAAC,UAAU,CAAC;;gBAG1C,EAAE,aAAqB,EAAE,GAAE,WAAgB;IAIvD,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK;CAarC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Ratio } from "../../aggregation/ratio.js";
|
|
2
|
+
// The stated leaves whose type the schema allows at their path, by the rules
|
|
3
|
+
// of the draft of the schema, over the stated leaves. With nullIsMissing, a
|
|
4
|
+
// stated null is no value, so it is not counted. A record that the schema
|
|
5
|
+
// accepts gives 1, so the value grades a record that the schema refuses. It
|
|
6
|
+
// has no gate: evaluate.py shows it only for a record that agrees with its
|
|
7
|
+
// schema (evaluate.py:291-295, 302-309), which makes it equal to
|
|
8
|
+
// SchemaCompliance, and its caller composes that with Penalized (ADR 0008).
|
|
9
|
+
export class TypePrecision {
|
|
10
|
+
#nullIsMissing;
|
|
11
|
+
constructor({ nullIsMissing = false } = {}) {
|
|
12
|
+
this.#nullIsMissing = nullIsMissing;
|
|
13
|
+
}
|
|
14
|
+
score(comparison) {
|
|
15
|
+
const { schema } = comparison;
|
|
16
|
+
let allowed = 0;
|
|
17
|
+
let stated = 0;
|
|
18
|
+
for (const pair of comparison.leaves({
|
|
19
|
+
nullIsMissing: this.#nullIsMissing,
|
|
20
|
+
})) {
|
|
21
|
+
if (pair.stated === undefined)
|
|
22
|
+
continue;
|
|
23
|
+
stated++;
|
|
24
|
+
if (schema.types.allows(pair.stated.path, pair.stated.value))
|
|
25
|
+
allowed++;
|
|
26
|
+
}
|
|
27
|
+
return new Ratio(allowed, stated);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=type-precision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"type-precision.js","sourceRoot":"","sources":["../../../src/structured/scorers/type-precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAKnD,6EAA6E;AAC7E,4EAA4E;AAC5E,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,iEAAiE;AACjE,4EAA4E;AAC5E,MAAM,OAAO,aAAa;IACf,cAAc,CAAU;IAEjC,YAAY,EAAE,aAAa,GAAG,KAAK,KAAkB,EAAE;QACrD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,UAAsB;QAC1B,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;gBAAE,SAAS;YACxC,MAAM,EAAE,CAAC;YACT,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;QAC1E,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Gate } from '../../scorers/gate.ts';
|
|
2
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
3
|
+
export declare class ValidSchema extends Gate<Comparison> {
|
|
4
|
+
holds({ schema }: Comparison): boolean;
|
|
5
|
+
explain({ schema }: Comparison): string;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=valid-schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"valid-schema.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/valid-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,qBAAa,WAAY,SAAQ,IAAI,CAAC,UAAU,CAAC;IAC/C,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,UAAU,GAAG,OAAO;IAItC,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,UAAU,GAAG,MAAM;CAGxC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Gate } from "../../scorers/gate.js";
|
|
2
|
+
// 1 when the case's JSON Schema is a valid schema of its draft, else 0.
|
|
3
|
+
export class ValidSchema extends Gate {
|
|
4
|
+
holds({ schema }) {
|
|
5
|
+
return schema.valid;
|
|
6
|
+
}
|
|
7
|
+
explain({ schema }) {
|
|
8
|
+
return `the JSON Schema of the case is not a valid schema of draft ${schema.draft}`;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=valid-schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"valid-schema.js","sourceRoot":"","sources":["../../../src/structured/scorers/valid-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAG7C,wEAAwE;AACxE,MAAM,OAAO,WAAY,SAAQ,IAAgB;IAC/C,KAAK,CAAC,EAAE,MAAM,EAAc;QAC1B,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,OAAO,CAAC,EAAE,MAAM,EAAc;QAC5B,OAAO,8DAA8D,MAAM,CAAC,KAAK,EAAE,CAAC;IACtF,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"value-matches.d.ts","sourceRoot":"","sources":["../../../src/structured/scorers/value-matches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,qBAAa,YAAa,SAAQ,WAAW;IAC3C,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO;CAGxC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { LeafMatches } from "./leaf-matches.js";
|
|
2
|
+
// A hit is a leaf pair whose two values agree, as the comparator of its field
|
|
3
|
+
// in the spec of the walk says.
|
|
4
|
+
export class ValueMatches extends LeafMatches {
|
|
5
|
+
hit(pair) {
|
|
6
|
+
return pair.agrees;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=value-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"value-matches.js","sourceRoot":"","sources":["../../../src/structured/scorers/value-matches.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,8EAA8E;AAC9E,gCAAgC;AAChC,MAAM,OAAO,YAAa,SAAQ,WAAW;IACjC,GAAG,CAAC,IAAe;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { JsonData } from '@zukhruf/python';
|
|
2
|
+
import type { Weighted } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
+
import type { JsonSchema } from '../schema/json-schema.ts';
|
|
4
|
+
export declare class Case implements Weighted {
|
|
5
|
+
readonly expected: JsonData;
|
|
6
|
+
readonly schema: JsonSchema | undefined;
|
|
7
|
+
readonly weight: number;
|
|
8
|
+
constructor({ expected, schema, weight, }: {
|
|
9
|
+
readonly expected: JsonData;
|
|
10
|
+
readonly schema?: JsonSchema;
|
|
11
|
+
readonly weight: number;
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAK3D,qBAAa,IAAK,YAAW,QAAQ;IACnC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,EACV,QAAQ,EACR,MAAM,EACN,MAAM,GACP,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;QAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB;CAKF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// What one test gives the scorers besides the stated value: the expected
|
|
2
|
+
// value, its JSON Schema when the scorers need one, and the weight of the
|
|
3
|
+
// test. A case reader reads it from the plain object of a runner.
|
|
4
|
+
export class Case {
|
|
5
|
+
expected;
|
|
6
|
+
schema;
|
|
7
|
+
weight;
|
|
8
|
+
constructor({ expected, schema, weight, }) {
|
|
9
|
+
this.expected = expected;
|
|
10
|
+
this.schema = schema;
|
|
11
|
+
this.weight = weight;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"case.js","sourceRoot":"","sources":["../../../src/structured/scoring/case.ts"],"names":[],"mappings":"AAKA,yEAAyE;AACzE,0EAA0E;AAC1E,kEAAkE;AAClE,MAAM,OAAO,IAAI;IACN,QAAQ,CAAW;IACnB,MAAM,CAAyB;IAC/B,MAAM,CAAS;IAExB,YAAY,EACV,QAAQ,EACR,MAAM,EACN,MAAM,GAKP;QACC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { CaseReader } from '../../scoring/scorecard-scorer.ts';
|
|
2
|
+
import type { JsonSchema } from '../schema/json-schema.ts';
|
|
3
|
+
import { Case } from './case.ts';
|
|
4
|
+
export declare class ConfigCase implements CaseReader<Case> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor({ schema }?: {
|
|
7
|
+
readonly schema?: JsonSchema;
|
|
8
|
+
});
|
|
9
|
+
from(config: Readonly<Record<string, unknown>>): Case;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=config-case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-case.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/config-case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAEpE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAMjC,qBAAa,UAAW,YAAW,UAAU,CAAC,IAAI,CAAC;;gBAGrC,EAAE,MAAM,EAAE,GAAE;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAA;KAAO;IAI7D,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI;CAWtD"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { JsonValue } from "../comparison/json-value.js";
|
|
2
|
+
import { Case } from "./case.js";
|
|
3
|
+
// Reads a case from the config of an assertion: config.expected, as a
|
|
4
|
+
// JavaScript value, so an integral float such as 7.0 is already 7. One
|
|
5
|
+
// JsonSchema, when the reader has one, serves every case. A case weighs 1
|
|
6
|
+
// (Strategy).
|
|
7
|
+
export class ConfigCase {
|
|
8
|
+
#schema;
|
|
9
|
+
constructor({ schema } = {}) {
|
|
10
|
+
this.#schema = schema;
|
|
11
|
+
}
|
|
12
|
+
from(config) {
|
|
13
|
+
if (!Object.hasOwn(config, 'expected'))
|
|
14
|
+
throw new Error('A config case needs config.expected: the expected value of the test');
|
|
15
|
+
return new Case({
|
|
16
|
+
expected: JsonValue.from(config['expected']).value,
|
|
17
|
+
...(this.#schema === undefined ? {} : { schema: this.#schema }),
|
|
18
|
+
weight: 1,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=config-case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-case.js","sourceRoot":"","sources":["../../../src/structured/scoring/config-case.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAExD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,sEAAsE;AACtE,uEAAuE;AACvE,0EAA0E;AAC1E,cAAc;AACd,MAAM,OAAO,UAAU;IACZ,OAAO,CAAyB;IAEzC,YAAY,EAAE,MAAM,KAAuC,EAAE;QAC3D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,IAAI,CAAC,MAAyC;QAC5C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,qEAAqE,CACtE,CAAC;QACJ,OAAO,IAAI,IAAI,CAAC;YACd,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK;YAClD,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,EAAE,CAAC;SACV,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Ratio } from '../../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../../scorers/scorer.ts';
|
|
3
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
4
|
+
export declare class PerField {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(scorer: Scorer<Comparison>);
|
|
7
|
+
ratios(comparison: Comparison, label: string): [string, Ratio][];
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=per-field.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"per-field.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAS9D,qBAAa,QAAQ;;gBAGP,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC;IAItC,MAAM,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;CASjE"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// A scorer for each field of a comparison: in a ScorerSet, it gives the
|
|
2
|
+
// ratio of its scorer on the leaf pairs and item pairs of each field, under
|
|
3
|
+
// the label `${field} ${label}`, such as `$.offers[*].kind recall`. A field
|
|
4
|
+
// gives no ratio when the scorer counts nothing in it (the whole 0): an
|
|
5
|
+
// object or an array under a scorer of leaves, or a field whose leaves are
|
|
6
|
+
// all null under nullIsMissing. A ratio with the whole 0 adds nothing to a
|
|
7
|
+
// ratio of sums, so the totals stay the same.
|
|
8
|
+
export class PerField {
|
|
9
|
+
#scorer;
|
|
10
|
+
constructor(scorer) {
|
|
11
|
+
this.#scorer = scorer;
|
|
12
|
+
}
|
|
13
|
+
ratios(comparison, label) {
|
|
14
|
+
return comparison
|
|
15
|
+
.fields()
|
|
16
|
+
.map((field) => [
|
|
17
|
+
`${field} ${label}`,
|
|
18
|
+
this.#scorer.score(comparison.only(field)),
|
|
19
|
+
])
|
|
20
|
+
.filter(([, ratio]) => ratio.whole !== 0);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=per-field.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"per-field.js","sourceRoot":"","sources":["../../../src/structured/scoring/per-field.ts"],"names":[],"mappings":"AAIA,wEAAwE;AACxE,4EAA4E;AAC5E,4EAA4E;AAC5E,wEAAwE;AACxE,2EAA2E;AAC3E,2EAA2E;AAC3E,8CAA8C;AAC9C,MAAM,OAAO,QAAQ;IACV,OAAO,CAAqB;IAErC,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,UAAsB,EAAE,KAAa;QAC1C,OAAO,UAAU;aACd,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC;YAC/B,GAAG,KAAK,IAAI,KAAK,EAAE;YACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SAC3C,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Scorer } from '../../scorers/scorer.ts';
|
|
2
|
+
import type { ScorecardScorer } from '../../scoring/scorecard-scorer.ts';
|
|
3
|
+
import type { Scorecard } from '../../scoring/scorecard.ts';
|
|
4
|
+
import { ComparisonSpec } from '../comparison/comparison-spec.ts';
|
|
5
|
+
import type { Comparison } from '../comparison/comparison.ts';
|
|
6
|
+
import type { Case } from './case.ts';
|
|
7
|
+
import { PerField } from './per-field.ts';
|
|
8
|
+
export declare class ScorerSet implements ScorecardScorer<Case> {
|
|
9
|
+
#private;
|
|
10
|
+
constructor(entries: Readonly<Record<string, Scorer<Comparison> | PerField>>, { spec }?: {
|
|
11
|
+
readonly spec?: ComparisonSpec;
|
|
12
|
+
});
|
|
13
|
+
score(stated: unknown, testCase: Case): Scorecard;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=scorer-set.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAU1C,qBAAa,SAAU,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQnD,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,CAAC,EAChE,EAAE,IAA2B,EAAE,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAA;KAAO;IAM1E,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,SAAS;CAqClD"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { RatioOfSums } from "../../aggregation/ratio-of-sums.js";
|
|
2
|
+
import { Ratio } from "../../aggregation/ratio.js";
|
|
3
|
+
import { Gate } from "../../scorers/gate.js";
|
|
4
|
+
import { ComparisonSpec } from "../comparison/comparison-spec.js";
|
|
5
|
+
import { statedData } from "../comparison/json-value.js";
|
|
6
|
+
import { TreeWalk } from "../comparison/tree-walk.js";
|
|
7
|
+
import { PerField } from "./per-field.js";
|
|
8
|
+
// Scores a stated value with labelled scorers, through a walk with the spec:
|
|
9
|
+
// the ratio of each scorer under its label, and of a PerField under the label
|
|
10
|
+
// of each field, so a ratio of sums of a label adds the parts and the wholes
|
|
11
|
+
// of the records. The record passes when each of its gates holds, and the
|
|
12
|
+
// reason is the explanations of the gates that fail. Each gate is a check.
|
|
13
|
+
// The score is the mean of the values of the scorers. A stated text is read
|
|
14
|
+
// as json.loads reads it, as a ScorerAssertion reads it; a stated JsonValue
|
|
15
|
+
// stays as it is.
|
|
16
|
+
export class ScorerSet {
|
|
17
|
+
#entries;
|
|
18
|
+
#walk;
|
|
19
|
+
constructor(entries, { spec = new ComparisonSpec() } = {}) {
|
|
20
|
+
this.#entries = Object.entries(entries);
|
|
21
|
+
this.#walk = new TreeWalk(spec);
|
|
22
|
+
}
|
|
23
|
+
score(stated, testCase) {
|
|
24
|
+
const comparison = this.#walk.compare(statedData(stated), testCase);
|
|
25
|
+
const ratios = [];
|
|
26
|
+
const mean = new RatioOfSums();
|
|
27
|
+
const checks = [];
|
|
28
|
+
for (const [label, entry] of this.#entries) {
|
|
29
|
+
if (entry instanceof PerField) {
|
|
30
|
+
ratios.push(...entry.ratios(comparison, label));
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
const ratio = entry.score(comparison);
|
|
34
|
+
ratios.push([label, ratio]);
|
|
35
|
+
mean.add(new Ratio(ratio.value, 1));
|
|
36
|
+
if (entry instanceof Gate) {
|
|
37
|
+
const holds = entry.holds(comparison);
|
|
38
|
+
checks.push({
|
|
39
|
+
name: label,
|
|
40
|
+
passed: holds,
|
|
41
|
+
score: ratio.value,
|
|
42
|
+
reason: holds
|
|
43
|
+
? `${label} holds`
|
|
44
|
+
: `${label}: ${entry.explain(comparison)}`,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const failed = checks.filter((check) => !check.passed);
|
|
49
|
+
return {
|
|
50
|
+
passed: failed.length === 0,
|
|
51
|
+
score: mean.value,
|
|
52
|
+
reason: failed.length === 0
|
|
53
|
+
? 'each gate holds'
|
|
54
|
+
: failed.map((check) => check.reason).join('; '),
|
|
55
|
+
ratios: new Map(ratios),
|
|
56
|
+
checks,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=scorer-set.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../../src/structured/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAI7C,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,6EAA6E;AAC7E,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,4EAA4E;AAC5E,4EAA4E;AAC5E,kBAAkB;AAClB,MAAM,OAAO,SAAS;IACX,QAAQ,CAGZ;IACI,KAAK,CAAW;IAEzB,YACE,OAAgE,EAChE,EAAE,IAAI,GAAG,IAAI,cAAc,EAAE,KAAyC,EAAE;QAExE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAe,EAAE,QAAc;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpE,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK;wBACX,CAAC,CAAC,GAAG,KAAK,QAAQ;wBAClB,CAAC,CAAC,GAAG,KAAK,KAAK,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EACJ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC;YACvB,MAAM;SACP,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { CaseReader } from '../../scoring/scorecard-scorer.ts';
|
|
2
|
+
import { JsonValue } from '../comparison/json-value.ts';
|
|
3
|
+
import { JsonSchema } from '../schema/json-schema.ts';
|
|
4
|
+
import { Case } from './case.ts';
|
|
5
|
+
export declare class TextVarsCase implements CaseReader<Case> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor({ schema, }?: {
|
|
8
|
+
readonly schema?: (value: JsonValue) => JsonSchema;
|
|
9
|
+
});
|
|
10
|
+
from(vars: Readonly<Record<string, unknown>>): Case;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=text-vars-case.d.ts.map
|