@zukhruf/evals 0.0.0-stage → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +82 -0
- package/README.md +528 -2
- package/dist/aggregation/counts.d.ts +11 -0
- package/dist/aggregation/counts.d.ts.map +1 -0
- package/dist/aggregation/counts.js +14 -0
- package/dist/aggregation/counts.js.map +1 -0
- package/dist/aggregation/f1-ratio.d.ts +8 -0
- package/dist/aggregation/f1-ratio.d.ts.map +1 -0
- package/dist/aggregation/f1-ratio.js +24 -0
- package/dist/aggregation/f1-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +8 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
- package/dist/aggregation/ratio-of-sums.js +21 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -0
- package/dist/aggregation/ratio.d.ts +8 -0
- package/dist/aggregation/ratio.d.ts.map +1 -0
- package/dist/aggregation/ratio.js +20 -0
- package/dist/aggregation/ratio.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/promptfoo/assertion-module.d.ts +23 -0
- package/dist/promptfoo/assertion-module.d.ts.map +1 -0
- package/dist/promptfoo/assertion-module.js +45 -0
- package/dist/promptfoo/assertion-module.js.map +1 -0
- package/dist/promptfoo/index.d.ts +3 -0
- package/dist/promptfoo/index.d.ts.map +1 -0
- package/dist/promptfoo/index.js +3 -0
- package/dist/promptfoo/index.js.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.js +31 -0
- package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
- package/dist/scorers/at-least.d.ts +9 -0
- package/dist/scorers/at-least.d.ts.map +1 -0
- package/dist/scorers/at-least.js +18 -0
- package/dist/scorers/at-least.js.map +1 -0
- package/dist/scorers/f1.d.ts +9 -0
- package/dist/scorers/f1.d.ts.map +1 -0
- package/dist/scorers/f1.js +12 -0
- package/dist/scorers/f1.js.map +1 -0
- package/dist/scorers/gate.d.ts +7 -0
- package/dist/scorers/gate.d.ts.map +1 -0
- package/dist/scorers/gate.js +10 -0
- package/dist/scorers/gate.js.map +1 -0
- package/dist/scorers/matches.d.ts +5 -0
- package/dist/scorers/matches.d.ts.map +1 -0
- package/dist/scorers/matches.js +2 -0
- package/dist/scorers/matches.js.map +1 -0
- package/dist/scorers/mean-of.d.ts +8 -0
- package/dist/scorers/mean-of.d.ts.map +1 -0
- package/dist/scorers/mean-of.js +15 -0
- package/dist/scorers/mean-of.js.map +1 -0
- package/dist/scorers/penalized.d.ts +9 -0
- package/dist/scorers/penalized.d.ts.map +1 -0
- package/dist/scorers/penalized.js +28 -0
- package/dist/scorers/penalized.js.map +1 -0
- package/dist/scorers/penalty.d.ts +7 -0
- package/dist/scorers/penalty.d.ts.map +1 -0
- package/dist/scorers/penalty.js +11 -0
- package/dist/scorers/penalty.js.map +1 -0
- package/dist/scorers/precision.d.ts +9 -0
- package/dist/scorers/precision.d.ts.map +1 -0
- package/dist/scorers/precision.js +13 -0
- package/dist/scorers/precision.js.map +1 -0
- package/dist/scorers/recall.d.ts +9 -0
- package/dist/scorers/recall.d.ts.map +1 -0
- package/dist/scorers/recall.js +13 -0
- package/dist/scorers/recall.js.map +1 -0
- package/dist/scorers/sample-averaged.d.ts +10 -0
- package/dist/scorers/sample-averaged.d.ts.map +1 -0
- package/dist/scorers/sample-averaged.js +17 -0
- package/dist/scorers/sample-averaged.js.map +1 -0
- package/dist/scorers/scorer.d.ts +5 -0
- package/dist/scorers/scorer.d.ts.map +1 -0
- package/dist/scorers/scorer.js +2 -0
- package/dist/scorers/scorer.js.map +1 -0
- package/dist/scorers/shortfall-penalty.d.ts +11 -0
- package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
- package/dist/scorers/shortfall-penalty.js +25 -0
- package/dist/scorers/shortfall-penalty.js.map +1 -0
- package/dist/scoring/bootstrap.d.ts +16 -0
- package/dist/scoring/bootstrap.d.ts.map +1 -0
- package/dist/scoring/bootstrap.js +31 -0
- package/dist/scoring/bootstrap.js.map +1 -0
- package/dist/scoring/scorecard-scorer.d.ts +11 -0
- package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
- package/dist/scoring/scorecard-scorer.js +2 -0
- package/dist/scoring/scorecard-scorer.js.map +1 -0
- package/dist/scoring/scorecard.d.ts +15 -0
- package/dist/scoring/scorecard.d.ts.map +1 -0
- package/dist/scoring/scorecard.js +2 -0
- package/dist/scoring/scorecard.js.map +1 -0
- package/dist/scoring/totals.d.ts +9 -0
- package/dist/scoring/totals.d.ts.map +1 -0
- package/dist/scoring/totals.js +23 -0
- package/dist/scoring/totals.js.map +1 -0
- package/dist/structured/comparison/alignment.d.ts +9 -0
- package/dist/structured/comparison/alignment.d.ts.map +1 -0
- package/dist/structured/comparison/alignment.js +2 -0
- package/dist/structured/comparison/alignment.js.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.js +25 -0
- package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
- package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
- package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-position.js +9 -0
- package/dist/structured/comparison/alignments/by-position.js.map +1 -0
- package/dist/structured/comparison/bag-of-words.d.ts +10 -0
- package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
- package/dist/structured/comparison/bag-of-words.js +48 -0
- package/dist/structured/comparison/bag-of-words.js.map +1 -0
- package/dist/structured/comparison/comparator.d.ts +5 -0
- package/dist/structured/comparison/comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparator.js +2 -0
- package/dist/structured/comparison/comparator.js.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.js +18 -0
- package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.js +13 -0
- package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
- package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized.js +3 -0
- package/dist/structured/comparison/comparators/normalized.js.map +1 -0
- package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
- package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/number-within.js +18 -0
- package/dist/structured/comparison/comparators/number-within.js.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.js +15 -0
- package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +9 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +20 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
- package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
- package/dist/structured/comparison/comparison-spec.d.ts +17 -0
- package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
- package/dist/structured/comparison/comparison-spec.js +36 -0
- package/dist/structured/comparison/comparison-spec.js.map +1 -0
- package/dist/structured/comparison/comparison.d.ts +25 -0
- package/dist/structured/comparison/comparison.d.ts.map +1 -0
- package/dist/structured/comparison/comparison.js +94 -0
- package/dist/structured/comparison/comparison.js.map +1 -0
- package/dist/structured/comparison/field.d.ts +10 -0
- package/dist/structured/comparison/field.d.ts.map +1 -0
- package/dist/structured/comparison/field.js +29 -0
- package/dist/structured/comparison/field.js.map +1 -0
- package/dist/structured/comparison/item-pairs.d.ts +8 -0
- package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
- package/dist/structured/comparison/item-pairs.js +2 -0
- package/dist/structured/comparison/item-pairs.js.map +1 -0
- package/dist/structured/comparison/json-path-query.d.ts +7 -0
- package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
- package/dist/structured/comparison/json-path-query.js +133 -0
- package/dist/structured/comparison/json-path-query.js.map +1 -0
- package/dist/structured/comparison/json-value.d.ts +10 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -0
- package/dist/structured/comparison/json-value.js +47 -0
- package/dist/structured/comparison/json-value.js.map +1 -0
- package/dist/structured/comparison/leaf-pair.d.ts +24 -0
- package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
- package/dist/structured/comparison/leaf-pair.js +2 -0
- package/dist/structured/comparison/leaf-pair.js.map +1 -0
- package/dist/structured/comparison/match.d.ts +5 -0
- package/dist/structured/comparison/match.d.ts.map +1 -0
- package/dist/structured/comparison/match.js +2 -0
- package/dist/structured/comparison/match.js.map +1 -0
- package/dist/structured/comparison/name-selector.d.ts +6 -0
- package/dist/structured/comparison/name-selector.d.ts.map +1 -0
- package/dist/structured/comparison/name-selector.js +41 -0
- package/dist/structured/comparison/name-selector.js.map +1 -0
- package/dist/structured/comparison/path.d.ts +10 -0
- package/dist/structured/comparison/path.d.ts.map +1 -0
- package/dist/structured/comparison/path.js +27 -0
- package/dist/structured/comparison/path.js.map +1 -0
- package/dist/structured/comparison/tree-walk.d.ts +10 -0
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
- package/dist/structured/comparison/tree-walk.js +136 -0
- package/dist/structured/comparison/tree-walk.js.map +1 -0
- package/dist/structured/cpython/python-text.d.ts +5 -0
- package/dist/structured/cpython/python-text.d.ts.map +1 -0
- package/dist/structured/cpython/python-text.js +79 -0
- package/dist/structured/cpython/python-text.js.map +1 -0
- package/dist/structured/index.d.ts +39 -0
- package/dist/structured/index.d.ts.map +1 -0
- package/dist/structured/index.js +29 -0
- package/dist/structured/index.js.map +1 -0
- package/dist/structured/promptfoo/scorer-assertion.d.ts +22 -0
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/structured/promptfoo/scorer-assertion.js +34 -0
- package/dist/structured/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/structured/schema/draft-07-schema.d.ts +3 -0
- package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-07-schema.js +243 -0
- package/dist/structured/schema/draft-07-schema.js.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.js +280 -0
- package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
- package/dist/structured/schema/is-record.d.ts +2 -0
- package/dist/structured/schema/is-record.d.ts.map +1 -0
- package/dist/structured/schema/is-record.js +3 -0
- package/dist/structured/schema/is-record.js.map +1 -0
- package/dist/structured/schema/json-schema.d.ts +20 -0
- package/dist/structured/schema/json-schema.d.ts.map +1 -0
- package/dist/structured/schema/json-schema.js +244 -0
- package/dist/structured/schema/json-schema.js.map +1 -0
- package/dist/structured/schema/schema-type-walk.d.ts +11 -0
- package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
- package/dist/structured/schema/schema-type-walk.js +196 -0
- package/dist/structured/schema/schema-type-walk.js.map +1 -0
- package/dist/structured/scorers/all-matched.d.ts +10 -0
- package/dist/structured/scorers/all-matched.d.ts.map +1 -0
- package/dist/structured/scorers/all-matched.js +36 -0
- package/dist/structured/scorers/all-matched.js.map +1 -0
- package/dist/structured/scorers/exact-match.d.ts +7 -0
- package/dist/structured/scorers/exact-match.d.ts.map +1 -0
- package/dist/structured/scorers/exact-match.js +13 -0
- package/dist/structured/scorers/exact-match.js.map +1 -0
- package/dist/structured/scorers/item-matches.d.ts +7 -0
- package/dist/structured/scorers/item-matches.d.ts.map +1 -0
- package/dist/structured/scorers/item-matches.js +17 -0
- package/dist/structured/scorers/item-matches.js.map +1 -0
- package/dist/structured/scorers/leaf-matches.d.ts +14 -0
- package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
- package/dist/structured/scorers/leaf-matches.js +28 -0
- package/dist/structured/scorers/leaf-matches.js.map +1 -0
- package/dist/structured/scorers/parse-success.d.ts +7 -0
- package/dist/structured/scorers/parse-success.d.ts.map +1 -0
- package/dist/structured/scorers/parse-success.js +11 -0
- package/dist/structured/scorers/parse-success.js.map +1 -0
- package/dist/structured/scorers/path-matches.d.ts +5 -0
- package/dist/structured/scorers/path-matches.d.ts.map +1 -0
- package/dist/structured/scorers/path-matches.js +9 -0
- package/dist/structured/scorers/path-matches.js.map +1 -0
- package/dist/structured/scorers/required-key-recall.d.ts +8 -0
- package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
- package/dist/structured/scorers/required-key-recall.js +50 -0
- package/dist/structured/scorers/required-key-recall.js.map +1 -0
- package/dist/structured/scorers/schema-compliance.d.ts +8 -0
- package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
- package/dist/structured/scorers/schema-compliance.js +29 -0
- package/dist/structured/scorers/schema-compliance.js.map +1 -0
- package/dist/structured/scorers/token-f1.d.ts +14 -0
- package/dist/structured/scorers/token-f1.d.ts.map +1 -0
- package/dist/structured/scorers/token-f1.js +44 -0
- package/dist/structured/scorers/token-f1.js.map +1 -0
- package/dist/structured/scorers/type-precision.d.ts +10 -0
- package/dist/structured/scorers/type-precision.d.ts.map +1 -0
- package/dist/structured/scorers/type-precision.js +30 -0
- package/dist/structured/scorers/type-precision.js.map +1 -0
- package/dist/structured/scorers/valid-schema.d.ts +7 -0
- package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
- package/dist/structured/scorers/valid-schema.js +11 -0
- package/dist/structured/scorers/valid-schema.js.map +1 -0
- package/dist/structured/scorers/value-matches.d.ts +6 -0
- package/dist/structured/scorers/value-matches.d.ts.map +1 -0
- package/dist/structured/scorers/value-matches.js +9 -0
- package/dist/structured/scorers/value-matches.js.map +1 -0
- package/dist/structured/scoring/case.d.ts +14 -0
- package/dist/structured/scoring/case.d.ts.map +1 -0
- package/dist/structured/scoring/case.js +14 -0
- package/dist/structured/scoring/case.js.map +1 -0
- package/dist/structured/scoring/config-case.d.ts +11 -0
- package/dist/structured/scoring/config-case.d.ts.map +1 -0
- package/dist/structured/scoring/config-case.js +22 -0
- package/dist/structured/scoring/config-case.js.map +1 -0
- package/dist/structured/scoring/per-field.d.ts +9 -0
- package/dist/structured/scoring/per-field.d.ts.map +1 -0
- package/dist/structured/scoring/per-field.js +23 -0
- package/dist/structured/scoring/per-field.js.map +1 -0
- package/dist/structured/scoring/scorer-set.d.ts +15 -0
- package/dist/structured/scoring/scorer-set.d.ts.map +1 -0
- package/dist/structured/scoring/scorer-set.js +60 -0
- package/dist/structured/scoring/scorer-set.js.map +1 -0
- package/dist/structured/scoring/text-vars-case.d.ts +12 -0
- package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
- package/dist/structured/scoring/text-vars-case.js +31 -0
- package/dist/structured/scoring/text-vars-case.js.map +1 -0
- package/package.json +78 -4
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AASnC,qBAAa,OAAQ,SAAQ,KAAK;;gBAGpB,MAAM,EAAE,MAAM;IAK1B,IAAa,KAAK,IAAI,MAAM,CAM3B;CACF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Ratio } from "./ratio.js";
|
|
2
|
+
// The ratio of an F1: its part is 2 × hits and its whole is stated +
|
|
3
|
+
// expected, so many F1 ratios add up to the micro F1. Its value is 0 when
|
|
4
|
+
// there are no hits, else 2·p·r/(p + r): precision and recall first, the
|
|
5
|
+
// arithmetic of SOB's evaluate.py (f1_from_counts, token_f1). That value can
|
|
6
|
+
// differ from part / whole in the last bit, such as for 1 hit of 1 expected
|
|
7
|
+
// and 5 stated (0.33333333333333337, not 1/3). It is the one exception to the
|
|
8
|
+
// rule of a Ratio (ADR 0011).
|
|
9
|
+
export class F1Ratio extends Ratio {
|
|
10
|
+
#counts;
|
|
11
|
+
constructor(counts) {
|
|
12
|
+
super(2 * counts.hits, counts.stated + counts.expected);
|
|
13
|
+
this.#counts = counts;
|
|
14
|
+
}
|
|
15
|
+
get value() {
|
|
16
|
+
const { hits, stated, expected } = this.#counts;
|
|
17
|
+
if (hits === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
const precision = hits / stated;
|
|
20
|
+
const recall = hits / expected;
|
|
21
|
+
return (2 * precision * recall) / (precision + recall);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=f1-ratio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,qEAAqE;AACrE,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,8BAA8B;AAC9B,MAAM,OAAO,OAAQ,SAAQ,KAAK;IACvB,OAAO,CAAS;IAEzB,YAAY,MAAc;QACxB,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,IAAa,KAAK;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAChD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACzD,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAItB,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
+
import { Ratio } from "./ratio.js";
|
|
3
|
+
// The sum of the parts of many ratios divided by the sum of their wholes.
|
|
4
|
+
// Each average of the package is one (ADR 0004): a micro average adds
|
|
5
|
+
// Ratio(right, expected), a mean adds Ratio(value, 1), and a weighted mean
|
|
6
|
+
// adds Ratio(value × weight, weight).
|
|
7
|
+
export class RatioOfSums {
|
|
8
|
+
#parts = new NeumaierSum();
|
|
9
|
+
#wholes = new NeumaierSum();
|
|
10
|
+
add(ratio) {
|
|
11
|
+
this.#parts.add(ratio.part);
|
|
12
|
+
this.#wholes.add(ratio.whole);
|
|
13
|
+
}
|
|
14
|
+
get ratio() {
|
|
15
|
+
return new Ratio(this.#parts.value, this.#wholes.value);
|
|
16
|
+
}
|
|
17
|
+
get value() {
|
|
18
|
+
return this.ratio.value;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=ratio-of-sums.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio.ts"],"names":[],"mappings":"AAEA,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAMvC,IAAI,KAAK,IAAI,MAAM,CAElB;IAID,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK;CAG7B"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// A part and a whole, such as 2 right values of 3 expected values. It keeps
|
|
2
|
+
// both, not only their quotient, so many ratios add up exactly (ADR 0004).
|
|
3
|
+
export class Ratio {
|
|
4
|
+
part;
|
|
5
|
+
whole;
|
|
6
|
+
constructor(part, whole) {
|
|
7
|
+
this.part = part;
|
|
8
|
+
this.whole = whole;
|
|
9
|
+
}
|
|
10
|
+
// 0 when the whole is 0, as the ratio() of SOB's evaluate.py gives.
|
|
11
|
+
get value() {
|
|
12
|
+
return this.whole === 0 ? 0 : this.part / this.whole;
|
|
13
|
+
}
|
|
14
|
+
// Ratio(value, 1).times(weight) is Ratio(value × weight, weight): the term
|
|
15
|
+
// of a weighted mean.
|
|
16
|
+
times(weight) {
|
|
17
|
+
return new Ratio(this.part * weight, this.whole * weight);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=ratio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ratio.js","sourceRoot":"","sources":["../../src/aggregation/ratio.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,2EAA2E;AAC3E,MAAM,OAAO,KAAK;IACP,IAAI,CAAS;IACb,KAAK,CAAS;IAEvB,YAAY,IAAY,EAAE,KAAa;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,oEAAoE;IACpE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;IACvD,CAAC;IAED,2EAA2E;IAC3E,sBAAsB;IACtB,KAAK,CAAC,MAAc;QAClB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;IAC5D,CAAC;CACF"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export { Ratio } from './aggregation/ratio.ts';
|
|
2
|
+
export { RatioOfSums } from './aggregation/ratio-of-sums.ts';
|
|
3
|
+
export { Counts } from './aggregation/counts.ts';
|
|
4
|
+
export { F1Ratio } from './aggregation/f1-ratio.ts';
|
|
5
|
+
export type { Check, Scorecard } from './scoring/scorecard.ts';
|
|
6
|
+
export type { CaseReader, ScorecardScorer, Weighted, } from './scoring/scorecard-scorer.ts';
|
|
7
|
+
export { Totals } from './scoring/totals.ts';
|
|
8
|
+
export { Bootstrap, type WeightedScorecard } from './scoring/bootstrap.ts';
|
|
9
|
+
export type { Scorer } from './scorers/scorer.ts';
|
|
10
|
+
export type { Matches } from './scorers/matches.ts';
|
|
11
|
+
export { Precision } from './scorers/precision.ts';
|
|
12
|
+
export { Recall } from './scorers/recall.ts';
|
|
13
|
+
export { F1 } from './scorers/f1.ts';
|
|
14
|
+
export { Penalty } from './scorers/penalty.ts';
|
|
15
|
+
export { Gate } from './scorers/gate.ts';
|
|
16
|
+
export { Penalized } from './scorers/penalized.ts';
|
|
17
|
+
export { AtLeast } from './scorers/at-least.ts';
|
|
18
|
+
export { ShortfallPenalty } from './scorers/shortfall-penalty.ts';
|
|
19
|
+
export { SampleAveraged } from './scorers/sample-averaged.ts';
|
|
20
|
+
export { MeanOf } from './scorers/mean-of.ts';
|
|
21
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { Ratio } from "./aggregation/ratio.js";
|
|
2
|
+
export { RatioOfSums } from "./aggregation/ratio-of-sums.js";
|
|
3
|
+
export { Counts } from "./aggregation/counts.js";
|
|
4
|
+
export { F1Ratio } from "./aggregation/f1-ratio.js";
|
|
5
|
+
export { Totals } from "./scoring/totals.js";
|
|
6
|
+
export { Bootstrap } from "./scoring/bootstrap.js";
|
|
7
|
+
export { Precision } from "./scorers/precision.js";
|
|
8
|
+
export { Recall } from "./scorers/recall.js";
|
|
9
|
+
export { F1 } from "./scorers/f1.js";
|
|
10
|
+
export { Penalty } from "./scorers/penalty.js";
|
|
11
|
+
export { Gate } from "./scorers/gate.js";
|
|
12
|
+
export { Penalized } from "./scorers/penalized.js";
|
|
13
|
+
export { AtLeast } from "./scorers/at-least.js";
|
|
14
|
+
export { ShortfallPenalty } from "./scorers/shortfall-penalty.js";
|
|
15
|
+
export { SampleAveraged } from "./scorers/sample-averaged.js";
|
|
16
|
+
export { MeanOf } from "./scorers/mean-of.js";
|
|
17
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AAOpD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export type VarsContext = {
|
|
2
|
+
readonly vars: Readonly<Record<string, unknown>>;
|
|
3
|
+
};
|
|
4
|
+
export type ModuleAssertion = {
|
|
5
|
+
readonly metric?: string | undefined;
|
|
6
|
+
readonly weight: number;
|
|
7
|
+
readonly threshold: number;
|
|
8
|
+
readonly value: (output: string, context: VarsContext) => number;
|
|
9
|
+
};
|
|
10
|
+
export type FileAssertion = {
|
|
11
|
+
readonly type: 'javascript';
|
|
12
|
+
readonly metric: string;
|
|
13
|
+
readonly weight: number;
|
|
14
|
+
readonly threshold: number;
|
|
15
|
+
readonly value: string;
|
|
16
|
+
};
|
|
17
|
+
export declare class AssertionModule<Name extends string> {
|
|
18
|
+
#private;
|
|
19
|
+
readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
|
|
20
|
+
constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
|
|
21
|
+
assert(...names: Name[]): FileAssertion[];
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=assertion-module.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion-module.d.ts","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;CAClE,CAAC;AAKF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAgBF,qBAAa,eAAe,CAAC,IAAI,SAAS,MAAM;;IAG9C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAKrE,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAcrD,MAAM,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,EAAE;CAY1C"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { fileURLToPath } from 'node:url';
|
|
2
|
+
// The module that promptfoo loads the function of each scorer assertion from
|
|
3
|
+
// (ADR 0013). promptfoo stores the config of an eval as JSON, and --resume
|
|
4
|
+
// and --retry-errors build the run from that stored config, so a function in
|
|
5
|
+
// the config is lost and its assertion scores 0. A config holds the file
|
|
6
|
+
// references that assert gives, and promptfoo loads each function from the
|
|
7
|
+
// default export of the module: importModule reads file://<module>:<name> on
|
|
8
|
+
// the default export when the module has one (promptfoo 0.125.1,
|
|
9
|
+
// dist/src/esm-B4NX_h4K.js:205-224). So the module gives functions as its
|
|
10
|
+
// default export, and url is the module's own import.meta.url.
|
|
11
|
+
//
|
|
12
|
+
// The reference holds the absolute path of the module. promptfoo 0.125.1
|
|
13
|
+
// splits a file reference at its first colon
|
|
14
|
+
// (dist/src/evaluator-BxCmvGtv.js:5683-5686), so on Windows the drive of the
|
|
15
|
+
// path ends the path: a defect of promptfoo.
|
|
16
|
+
export class AssertionModule {
|
|
17
|
+
// promptfoo reads a function by the name in its reference, so the names are
|
|
18
|
+
// keys, and assert holds the type of each name.
|
|
19
|
+
functions;
|
|
20
|
+
#path;
|
|
21
|
+
#assertions;
|
|
22
|
+
constructor(url, assertions) {
|
|
23
|
+
this.#path = fileURLToPath(url);
|
|
24
|
+
this.#assertions = assertions;
|
|
25
|
+
this.functions = Object.fromEntries(Object.entries(assertions).map(([name, assertion]) => [
|
|
26
|
+
name,
|
|
27
|
+
assertion.value,
|
|
28
|
+
]));
|
|
29
|
+
}
|
|
30
|
+
// One assertion for each name. The name is the metric, unless its scorer
|
|
31
|
+
// assertion gives one.
|
|
32
|
+
assert(...names) {
|
|
33
|
+
return names.map((name) => {
|
|
34
|
+
const { metric = name, weight, threshold } = this.#assertions[name];
|
|
35
|
+
return {
|
|
36
|
+
type: 'javascript',
|
|
37
|
+
metric,
|
|
38
|
+
weight,
|
|
39
|
+
threshold,
|
|
40
|
+
value: `file://${this.#path}:${name}`,
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=assertion-module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion-module.js","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA6BzC,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,6EAA6E;AAC7E,iEAAiE;AACjE,0EAA0E;AAC1E,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,6CAA6C;AAC7C,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,OAAO,eAAe;IAC1B,4EAA4E;IAC5E,gDAAgD;IACvC,SAAS,CAAqD;IAC9D,KAAK,CAAS;IACd,WAAW,CAA0C;IAE9D,YACE,GAAW,EACX,UAAmD;QAEnD,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CACjC,MAAM,CAAC,OAAO,CAAkB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC;YACrE,IAAI;YACJ,SAAS,CAAC,KAAK;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,uBAAuB;IACvB,MAAM,CAAC,GAAG,KAAa;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACpE,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM;gBACN,SAAS;gBACT,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE;aACtC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CaseReader, ScorecardScorer, Weighted } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
export type GradingResult = {
|
|
3
|
+
readonly pass: boolean;
|
|
4
|
+
readonly score: number;
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly namedScores?: Readonly<Record<string, number>>;
|
|
7
|
+
readonly namedScoreWeights?: Readonly<Record<string, number>>;
|
|
8
|
+
readonly componentResults?: readonly GradingResult[];
|
|
9
|
+
};
|
|
10
|
+
export type AssertionContext = {
|
|
11
|
+
readonly config?: Readonly<Record<string, unknown>>;
|
|
12
|
+
};
|
|
13
|
+
export declare class PromptfooAssertion<Case extends Weighted> {
|
|
14
|
+
#private;
|
|
15
|
+
constructor(scorer: ScorecardScorer<Case>, cases: CaseReader<Case>);
|
|
16
|
+
readonly grade: (output: unknown, context: AssertionContext) => GradingResult;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=promptfoo-assertion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAKxC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACtD,CAAC;AAIF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD,CAAC;AAOF,qBAAa,kBAAkB,CAAC,IAAI,SAAS,QAAQ;;gBAIvC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;IAKlE,QAAQ,CAAC,KAAK,GACZ,QAAQ,OAAO,EACf,SAAS,gBAAgB,KACxB,aAAa,CAsBd;CACH"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Adapts a scorer to a promptfoo javascript assertion (ADR 0004). Each ratio
|
|
2
|
+
// of the scorecard becomes a named score with its whole as the weight:
|
|
3
|
+
// promptfoo adds value × weight over the rows and divides by the weights
|
|
4
|
+
// (namedMetrics.ts, accumulateNamedMetric), which is the ratio of sums.
|
|
5
|
+
// promptfoo calls a function, so a module exports `grade`.
|
|
6
|
+
export class PromptfooAssertion {
|
|
7
|
+
#scorer;
|
|
8
|
+
#cases;
|
|
9
|
+
constructor(scorer, cases) {
|
|
10
|
+
this.#scorer = scorer;
|
|
11
|
+
this.#cases = cases;
|
|
12
|
+
}
|
|
13
|
+
grade = (output, context) => {
|
|
14
|
+
const testCase = this.#cases.from(context.config ?? {});
|
|
15
|
+
const scorecard = this.#scorer.score(output, testCase);
|
|
16
|
+
const ratios = [...scorecard.ratios].map(([name, ratio]) => [name, ratio.times(testCase.weight)]);
|
|
17
|
+
return {
|
|
18
|
+
pass: scorecard.passed,
|
|
19
|
+
score: scorecard.score,
|
|
20
|
+
reason: scorecard.reason,
|
|
21
|
+
namedScores: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.value])),
|
|
22
|
+
namedScoreWeights: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.whole])),
|
|
23
|
+
componentResults: scorecard.checks.map((check) => ({
|
|
24
|
+
pass: check.passed,
|
|
25
|
+
score: check.score,
|
|
26
|
+
reason: check.reason,
|
|
27
|
+
})),
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=promptfoo-assertion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAwBA,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,wEAAwE;AACxE,2DAA2D;AAC3D,MAAM,OAAO,kBAAkB;IACpB,OAAO,CAAwB;IAC/B,MAAM,CAAmB;IAElC,YAAY,MAA6B,EAAE,KAAuB;QAChE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAEQ,KAAK,GAAG,CACf,MAAe,EACf,OAAyB,EACV,EAAE;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CACtC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAU,CACjE,CAAC;QACF,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,MAAM,CAAC,WAAW,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CAAC;CACH"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Gate } from './gate.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare class AtLeast<Input> extends Gate<Input> {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(scorer: Scorer<Input>, threshold: number);
|
|
6
|
+
holds(input: Input): boolean;
|
|
7
|
+
explain(input: Input): string;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=at-least.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"at-least.d.ts","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,OAAO,CAAC,KAAK,CAAE,SAAQ,IAAI,CAAC,KAAK,CAAC;;gBAIjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM;IAMpD,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAI5B,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG9B"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Gate } from "./gate.js";
|
|
2
|
+
// A gate: it holds when the value of the scorer reaches the threshold.
|
|
3
|
+
export class AtLeast extends Gate {
|
|
4
|
+
#scorer;
|
|
5
|
+
#threshold;
|
|
6
|
+
constructor(scorer, threshold) {
|
|
7
|
+
super();
|
|
8
|
+
this.#scorer = scorer;
|
|
9
|
+
this.#threshold = threshold;
|
|
10
|
+
}
|
|
11
|
+
holds(input) {
|
|
12
|
+
return this.#scorer.score(input).value >= this.#threshold;
|
|
13
|
+
}
|
|
14
|
+
explain(input) {
|
|
15
|
+
return `the value ${this.#scorer.score(input).value} is below the threshold ${this.#threshold}`;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=at-least.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"at-least.js","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,uEAAuE;AACvE,MAAM,OAAO,OAAe,SAAQ,IAAW;IACpC,OAAO,CAAgB;IACvB,UAAU,CAAS;IAE5B,YAAY,MAAqB,EAAE,SAAiB;QAClD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,aAAa,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,2BAA2B,IAAI,CAAC,UAAU,EAAE,CAAC;IAClG,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { F1Ratio } from '../aggregation/f1-ratio.ts';
|
|
2
|
+
import type { Matches } from './matches.ts';
|
|
3
|
+
import type { Scorer } from './scorer.ts';
|
|
4
|
+
export declare class F1<Input> implements Scorer<Input> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(matches: Matches<Input>);
|
|
7
|
+
score(input: Input): F1Ratio;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=f1.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1.d.ts","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,EAAE,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGjC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;IAInC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;CAG7B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { F1Ratio } from "../aggregation/f1-ratio.js";
|
|
2
|
+
// The F1 of the hits: an F1Ratio of the counts.
|
|
3
|
+
export class F1 {
|
|
4
|
+
#matches;
|
|
5
|
+
constructor(matches) {
|
|
6
|
+
this.#matches = matches;
|
|
7
|
+
}
|
|
8
|
+
score(input) {
|
|
9
|
+
return new F1Ratio(this.#matches.counts(input));
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=f1.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1.js","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,gDAAgD;AAChD,MAAM,OAAO,EAAE;IACJ,QAAQ,CAAiB;IAElC,YAAY,OAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKvC,8BAAsB,IAAI,CAAC,KAAK,CAAE,SAAQ,OAAO,CAAC,KAAK,CAAC;IACtD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAGrC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG7B"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Penalty } from "./penalty.js";
|
|
2
|
+
// The penalty whose factor is 1 when the input holds what the gate checks,
|
|
3
|
+
// else 0. A subclass says what it checks and why an input fails it (Template
|
|
4
|
+
// Method), so a ScorerSet makes a check of each gate.
|
|
5
|
+
export class Gate extends Penalty {
|
|
6
|
+
factor(input) {
|
|
7
|
+
return this.holds(input) ? 1 : 0;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gate.js","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,2EAA2E;AAC3E,6EAA6E;AAC7E,sDAAsD;AACtD,MAAM,OAAgB,IAAY,SAAQ,OAAc;IAMtD,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"matches.d.ts","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIvD,MAAM,WAAW,OAAO,CAAC,KAAK;IAC5B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;CAC9B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"matches.js","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare class MeanOf<Input> implements Scorer<Input> {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(scorer: Scorer<Input>, ...scorers: readonly Scorer<Input>[]);
|
|
6
|
+
score(input: Input): Ratio;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=mean-of.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mean-of.d.ts","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,MAAM,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGrC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE;IAIvE,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAQ3B"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
// The mean of the values of some scorers for one record, over 1, with
|
|
4
|
+
// CPython's sum, such as a category of a benchmark. It takes one scorer or
|
|
5
|
+
// more, because the mean of no value has no value.
|
|
6
|
+
export class MeanOf {
|
|
7
|
+
#scorers;
|
|
8
|
+
constructor(scorer, ...scorers) {
|
|
9
|
+
this.#scorers = [scorer, ...scorers];
|
|
10
|
+
}
|
|
11
|
+
score(input) {
|
|
12
|
+
return new Ratio(NeumaierSum.mean(this.#scorers.map((scorer) => scorer.score(input).value)), 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=mean-of.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mean-of.js","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,sEAAsE;AACtE,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,OAAO,MAAM;IACR,QAAQ,CAA2B;IAE5C,YAAY,MAAqB,EAAE,GAAG,OAAiC;QACrE,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CACd,WAAW,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CACzD,EACD,CAAC,CACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Penalty } from './penalty.ts';
|
|
3
|
+
import type { Scorer } from './scorer.ts';
|
|
4
|
+
export declare class Penalized<Input> implements Scorer<Input> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(scorer: Scorer<Input>, ...penalties: readonly Penalty<Input>[]);
|
|
7
|
+
score(input: Input): Ratio;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=penalized.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalized.d.ts","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAQ1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAIxC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,EAAE;IAK1E,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAgB3B"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
2
|
+
// A scorer whose part is multiplied by the factor of each penalty, in the
|
|
3
|
+
// order of the penalties, for each record; the whole stays (Decorator). So a
|
|
4
|
+
// record that fails a gate keeps its expected values in the denominator of
|
|
5
|
+
// recall. A penalty on the means would give mean(a) × mean(b), so the
|
|
6
|
+
// penalty is per record (ADR 0004). When every factor is 1, the ratio of the
|
|
7
|
+
// scorer stays as it is, so an F1Ratio keeps its value (ADR 0012).
|
|
8
|
+
export class Penalized {
|
|
9
|
+
#scorer;
|
|
10
|
+
#penalties;
|
|
11
|
+
constructor(scorer, ...penalties) {
|
|
12
|
+
this.#scorer = scorer;
|
|
13
|
+
this.#penalties = penalties;
|
|
14
|
+
}
|
|
15
|
+
score(input) {
|
|
16
|
+
const ratio = this.#scorer.score(input);
|
|
17
|
+
const factors = this.#penalties.map((penalty) => {
|
|
18
|
+
const factor = penalty.factor(input);
|
|
19
|
+
if (!(factor >= 0 && factor <= 1))
|
|
20
|
+
throw new RangeError(`A penalty gave the factor ${factor}, which is not in [0, 1].`);
|
|
21
|
+
return factor;
|
|
22
|
+
});
|
|
23
|
+
if (factors.every((factor) => factor === 1))
|
|
24
|
+
return ratio;
|
|
25
|
+
return new Ratio(factors.reduce((part, factor) => part * factor, ratio.part), ratio.whole);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=penalized.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalized.js","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,SAAS;IACX,OAAO,CAAgB;IACvB,UAAU,CAA4B;IAE/C,YAAY,MAAqB,EAAE,GAAG,SAAoC;QACxE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,6BAA6B,MAAM,2BAA2B,CAC/D,CAAC;YACJ,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1D,OAAO,IAAI,KAAK,CACd,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAC3D,KAAK,CAAC,KAAK,CACZ,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare abstract class Penalty<Input> implements Scorer<Input> {
|
|
4
|
+
abstract factor(input: Input): number;
|
|
5
|
+
score(input: Input): Ratio;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=penalty.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalty.d.ts","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAM1C,8BAAsB,OAAO,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;IAC3D,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAErC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAG3B"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
2
|
+
// A factor in [0, 1] for one input, as BLEU's brevity penalty is: 1 is no
|
|
3
|
+
// penalty. Penalized multiplies the part of a scorer by it, for each record.
|
|
4
|
+
// A subclass gives the factor (Template Method). As a scorer, it gives the
|
|
5
|
+
// factor over 1.
|
|
6
|
+
export class Penalty {
|
|
7
|
+
score(input) {
|
|
8
|
+
return new Ratio(this.factor(input), 1);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=penalty.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalty.js","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,iBAAiB;AACjB,MAAM,OAAgB,OAAO;IAG3B,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Matches } from './matches.ts';
|
|
3
|
+
import type { Scorer } from './scorer.ts';
|
|
4
|
+
export declare class Precision<Input> implements Scorer<Input> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(matches: Matches<Input>);
|
|
7
|
+
score(input: Input): Ratio;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=precision.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"precision.d.ts","sourceRoot":"","sources":["../../src/scorers/precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGxC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;IAInC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAI3B"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
2
|
+
// The hits of the stated values: Ratio(hits, stated).
|
|
3
|
+
export class Precision {
|
|
4
|
+
#matches;
|
|
5
|
+
constructor(matches) {
|
|
6
|
+
this.#matches = matches;
|
|
7
|
+
}
|
|
8
|
+
score(input) {
|
|
9
|
+
const { hits, stated } = this.#matches.counts(input);
|
|
10
|
+
return new Ratio(hits, stated);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=precision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"precision.js","sourceRoot":"","sources":["../../src/scorers/precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,sDAAsD;AACtD,MAAM,OAAO,SAAS;IACX,QAAQ,CAAiB;IAElC,YAAY,OAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Matches } from './matches.ts';
|
|
3
|
+
import type { Scorer } from './scorer.ts';
|
|
4
|
+
export declare class Recall<Input> implements Scorer<Input> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(matches: Matches<Input>);
|
|
7
|
+
score(input: Input): Ratio;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=recall.d.ts.map
|