@zukhruf/evals 0.0.0-stage → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +82 -0
- package/README.md +540 -2
- package/dist/aggregation/counts.d.ts +11 -0
- package/dist/aggregation/counts.d.ts.map +1 -0
- package/dist/aggregation/counts.js +14 -0
- package/dist/aggregation/counts.js.map +1 -0
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +9 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
- package/dist/aggregation/ratio-of-sums.js +28 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -0
- package/dist/aggregation/ratio.d.ts +8 -0
- package/dist/aggregation/ratio.d.ts.map +1 -0
- package/dist/aggregation/ratio.js +20 -0
- package/dist/aggregation/ratio.js.map +1 -0
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/index.d.ts +11 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +10 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +20 -0
- package/dist/index.js.map +1 -0
- package/dist/promptfoo/assertion-module.d.ts +23 -0
- package/dist/promptfoo/assertion-module.d.ts.map +1 -0
- package/dist/promptfoo/assertion-module.js +45 -0
- package/dist/promptfoo/assertion-module.js.map +1 -0
- package/dist/promptfoo/index.d.ts +4 -0
- package/dist/promptfoo/index.d.ts.map +1 -0
- package/dist/promptfoo/index.js +4 -0
- package/dist/promptfoo/index.js.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
- package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
- package/dist/promptfoo/promptfoo-assertion.js +31 -0
- package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/promptfoo/scorer-assertion.js +28 -0
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/at-least.d.ts +9 -0
- package/dist/scorers/at-least.d.ts.map +1 -0
- package/dist/scorers/at-least.js +18 -0
- package/dist/scorers/at-least.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +9 -0
- package/dist/scorers/f1.d.ts.map +1 -0
- package/dist/scorers/f1.js +8 -0
- package/dist/scorers/f1.js.map +1 -0
- package/dist/scorers/gate.d.ts +7 -0
- package/dist/scorers/gate.d.ts.map +1 -0
- package/dist/scorers/gate.js +10 -0
- package/dist/scorers/gate.js.map +1 -0
- package/dist/scorers/matches.d.ts +5 -0
- package/dist/scorers/matches.d.ts.map +1 -0
- package/dist/scorers/matches.js +2 -0
- package/dist/scorers/matches.js.map +1 -0
- package/dist/scorers/mean-of.d.ts +8 -0
- package/dist/scorers/mean-of.d.ts.map +1 -0
- package/dist/scorers/mean-of.js +15 -0
- package/dist/scorers/mean-of.js.map +1 -0
- package/dist/scorers/penalized.d.ts +9 -0
- package/dist/scorers/penalized.d.ts.map +1 -0
- package/dist/scorers/penalized.js +29 -0
- package/dist/scorers/penalized.js.map +1 -0
- package/dist/scorers/penalty.d.ts +7 -0
- package/dist/scorers/penalty.d.ts.map +1 -0
- package/dist/scorers/penalty.js +11 -0
- package/dist/scorers/penalty.js.map +1 -0
- package/dist/scorers/precision.d.ts +9 -0
- package/dist/scorers/precision.d.ts.map +1 -0
- package/dist/scorers/precision.js +13 -0
- package/dist/scorers/precision.js.map +1 -0
- package/dist/scorers/recall.d.ts +9 -0
- package/dist/scorers/recall.d.ts.map +1 -0
- package/dist/scorers/recall.js +13 -0
- package/dist/scorers/recall.js.map +1 -0
- package/dist/scorers/sample-averaged.d.ts +10 -0
- package/dist/scorers/sample-averaged.d.ts.map +1 -0
- package/dist/scorers/sample-averaged.js +17 -0
- package/dist/scorers/sample-averaged.js.map +1 -0
- package/dist/scorers/scorer.d.ts +5 -0
- package/dist/scorers/scorer.d.ts.map +1 -0
- package/dist/scorers/scorer.js +2 -0
- package/dist/scorers/scorer.js.map +1 -0
- package/dist/scorers/shortfall-penalty.d.ts +11 -0
- package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
- package/dist/scorers/shortfall-penalty.js +25 -0
- package/dist/scorers/shortfall-penalty.js.map +1 -0
- package/dist/scoring/bootstrap.d.ts +16 -0
- package/dist/scoring/bootstrap.d.ts.map +1 -0
- package/dist/scoring/bootstrap.js +29 -0
- package/dist/scoring/bootstrap.js.map +1 -0
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorecard-scorer.d.ts +11 -0
- package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
- package/dist/scoring/scorecard-scorer.js +2 -0
- package/dist/scoring/scorecard-scorer.js.map +1 -0
- package/dist/scoring/scorecard.d.ts +15 -0
- package/dist/scoring/scorecard.d.ts.map +1 -0
- package/dist/scoring/scorecard.js +2 -0
- package/dist/scoring/scorecard.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/scoring/totals.d.ts +9 -0
- package/dist/scoring/totals.d.ts.map +1 -0
- package/dist/scoring/totals.js +23 -0
- package/dist/scoring/totals.js.map +1 -0
- package/dist/structured/comparison/alignment.d.ts +9 -0
- package/dist/structured/comparison/alignment.d.ts.map +1 -0
- package/dist/structured/comparison/alignment.js +2 -0
- package/dist/structured/comparison/alignment.js.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
- package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-best-match.js +25 -0
- package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
- package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
- package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
- package/dist/structured/comparison/alignments/by-position.js +9 -0
- package/dist/structured/comparison/alignments/by-position.js.map +1 -0
- package/dist/structured/comparison/bag-of-words.d.ts +10 -0
- package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
- package/dist/structured/comparison/bag-of-words.js +48 -0
- package/dist/structured/comparison/bag-of-words.js.map +1 -0
- package/dist/structured/comparison/comparator.d.ts +5 -0
- package/dist/structured/comparison/comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparator.js +2 -0
- package/dist/structured/comparison/comparator.js.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/json-equality.js +18 -0
- package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
- package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized-text.js +13 -0
- package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
- package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
- package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/normalized.js +3 -0
- package/dist/structured/comparison/comparators/normalized.js.map +1 -0
- package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
- package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/number-within.js +18 -0
- package/dist/structured/comparison/comparators/number-within.js.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
- package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/python-equality.js +15 -0
- package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +12 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +23 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
- package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
- package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
- package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
- package/dist/structured/comparison/comparison-spec.d.ts +17 -0
- package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
- package/dist/structured/comparison/comparison-spec.js +36 -0
- package/dist/structured/comparison/comparison-spec.js.map +1 -0
- package/dist/structured/comparison/comparison.d.ts +25 -0
- package/dist/structured/comparison/comparison.d.ts.map +1 -0
- package/dist/structured/comparison/comparison.js +94 -0
- package/dist/structured/comparison/comparison.js.map +1 -0
- package/dist/structured/comparison/field.d.ts +10 -0
- package/dist/structured/comparison/field.d.ts.map +1 -0
- package/dist/structured/comparison/field.js +29 -0
- package/dist/structured/comparison/field.js.map +1 -0
- package/dist/structured/comparison/item-pairs.d.ts +8 -0
- package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
- package/dist/structured/comparison/item-pairs.js +2 -0
- package/dist/structured/comparison/item-pairs.js.map +1 -0
- package/dist/structured/comparison/json-path-query.d.ts +7 -0
- package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
- package/dist/structured/comparison/json-path-query.js +133 -0
- package/dist/structured/comparison/json-path-query.js.map +1 -0
- package/dist/structured/comparison/json-value.d.ts +11 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -0
- package/dist/structured/comparison/json-value.js +56 -0
- package/dist/structured/comparison/json-value.js.map +1 -0
- package/dist/structured/comparison/leaf-pair.d.ts +24 -0
- package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
- package/dist/structured/comparison/leaf-pair.js +2 -0
- package/dist/structured/comparison/leaf-pair.js.map +1 -0
- package/dist/structured/comparison/match.d.ts +5 -0
- package/dist/structured/comparison/match.d.ts.map +1 -0
- package/dist/structured/comparison/match.js +2 -0
- package/dist/structured/comparison/match.js.map +1 -0
- package/dist/structured/comparison/name-selector.d.ts +6 -0
- package/dist/structured/comparison/name-selector.d.ts.map +1 -0
- package/dist/structured/comparison/name-selector.js +41 -0
- package/dist/structured/comparison/name-selector.js.map +1 -0
- package/dist/structured/comparison/path.d.ts +10 -0
- package/dist/structured/comparison/path.d.ts.map +1 -0
- package/dist/structured/comparison/path.js +27 -0
- package/dist/structured/comparison/path.js.map +1 -0
- package/dist/structured/comparison/tree-walk.d.ts +10 -0
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
- package/dist/structured/comparison/tree-walk.js +140 -0
- package/dist/structured/comparison/tree-walk.js.map +1 -0
- package/dist/structured/cpython/python-text.d.ts +4 -0
- package/dist/structured/cpython/python-text.d.ts.map +1 -0
- package/dist/structured/cpython/python-text.js +62 -0
- package/dist/structured/cpython/python-text.js.map +1 -0
- package/dist/structured/index.d.ts +37 -0
- package/dist/structured/index.d.ts.map +1 -0
- package/dist/structured/index.js +27 -0
- package/dist/structured/index.js.map +1 -0
- package/dist/structured/schema/draft-07-schema.d.ts +3 -0
- package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-07-schema.js +243 -0
- package/dist/structured/schema/draft-07-schema.js.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
- package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
- package/dist/structured/schema/draft-2020-12-schema.js +280 -0
- package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
- package/dist/structured/schema/is-record.d.ts +2 -0
- package/dist/structured/schema/is-record.d.ts.map +1 -0
- package/dist/structured/schema/is-record.js +3 -0
- package/dist/structured/schema/is-record.js.map +1 -0
- package/dist/structured/schema/json-schema.d.ts +20 -0
- package/dist/structured/schema/json-schema.d.ts.map +1 -0
- package/dist/structured/schema/json-schema.js +244 -0
- package/dist/structured/schema/json-schema.js.map +1 -0
- package/dist/structured/schema/schema-type-walk.d.ts +11 -0
- package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
- package/dist/structured/schema/schema-type-walk.js +196 -0
- package/dist/structured/schema/schema-type-walk.js.map +1 -0
- package/dist/structured/scorers/all-matched.d.ts +10 -0
- package/dist/structured/scorers/all-matched.d.ts.map +1 -0
- package/dist/structured/scorers/all-matched.js +36 -0
- package/dist/structured/scorers/all-matched.js.map +1 -0
- package/dist/structured/scorers/exact-match.d.ts +7 -0
- package/dist/structured/scorers/exact-match.d.ts.map +1 -0
- package/dist/structured/scorers/exact-match.js +13 -0
- package/dist/structured/scorers/exact-match.js.map +1 -0
- package/dist/structured/scorers/item-matches.d.ts +7 -0
- package/dist/structured/scorers/item-matches.d.ts.map +1 -0
- package/dist/structured/scorers/item-matches.js +17 -0
- package/dist/structured/scorers/item-matches.js.map +1 -0
- package/dist/structured/scorers/leaf-matches.d.ts +14 -0
- package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
- package/dist/structured/scorers/leaf-matches.js +28 -0
- package/dist/structured/scorers/leaf-matches.js.map +1 -0
- package/dist/structured/scorers/parse-success.d.ts +7 -0
- package/dist/structured/scorers/parse-success.d.ts.map +1 -0
- package/dist/structured/scorers/parse-success.js +11 -0
- package/dist/structured/scorers/parse-success.js.map +1 -0
- package/dist/structured/scorers/path-matches.d.ts +5 -0
- package/dist/structured/scorers/path-matches.d.ts.map +1 -0
- package/dist/structured/scorers/path-matches.js +9 -0
- package/dist/structured/scorers/path-matches.js.map +1 -0
- package/dist/structured/scorers/required-key-recall.d.ts +8 -0
- package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
- package/dist/structured/scorers/required-key-recall.js +50 -0
- package/dist/structured/scorers/required-key-recall.js.map +1 -0
- package/dist/structured/scorers/schema-compliance.d.ts +8 -0
- package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
- package/dist/structured/scorers/schema-compliance.js +29 -0
- package/dist/structured/scorers/schema-compliance.js.map +1 -0
- package/dist/structured/scorers/token-f1.d.ts +16 -0
- package/dist/structured/scorers/token-f1.d.ts.map +1 -0
- package/dist/structured/scorers/token-f1.js +47 -0
- package/dist/structured/scorers/token-f1.js.map +1 -0
- package/dist/structured/scorers/type-precision.d.ts +10 -0
- package/dist/structured/scorers/type-precision.d.ts.map +1 -0
- package/dist/structured/scorers/type-precision.js +30 -0
- package/dist/structured/scorers/type-precision.js.map +1 -0
- package/dist/structured/scorers/valid-schema.d.ts +7 -0
- package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
- package/dist/structured/scorers/valid-schema.js +11 -0
- package/dist/structured/scorers/valid-schema.js.map +1 -0
- package/dist/structured/scorers/value-matches.d.ts +6 -0
- package/dist/structured/scorers/value-matches.d.ts.map +1 -0
- package/dist/structured/scorers/value-matches.js +9 -0
- package/dist/structured/scorers/value-matches.js.map +1 -0
- package/dist/structured/scoring/case.d.ts +14 -0
- package/dist/structured/scoring/case.d.ts.map +1 -0
- package/dist/structured/scoring/case.js +14 -0
- package/dist/structured/scoring/case.js.map +1 -0
- package/dist/structured/scoring/config-case.d.ts +11 -0
- package/dist/structured/scoring/config-case.d.ts.map +1 -0
- package/dist/structured/scoring/config-case.js +22 -0
- package/dist/structured/scoring/config-case.js.map +1 -0
- package/dist/structured/scoring/per-field.d.ts +6 -0
- package/dist/structured/scoring/per-field.d.ts.map +1 -0
- package/dist/structured/scoring/per-field.js +14 -0
- package/dist/structured/scoring/per-field.js.map +1 -0
- package/dist/structured/scoring/text-vars-case.d.ts +12 -0
- package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
- package/dist/structured/scoring/text-vars-case.js +31 -0
- package/dist/structured/scoring/text-vars-case.js.map +1 -0
- package/package.json +84 -4
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Grouped } from "../scoring/grouped.js";
|
|
2
|
+
// A scorer for each class of a comparison: in a ScorerSet, it gives the ratio
|
|
3
|
+
// of its scorer on the comparison of each class, under the label
|
|
4
|
+
// `${class} ${label}`. Without labels, the classes are the labels of both
|
|
5
|
+
// sides of the answer; with labels, only those classes.
|
|
6
|
+
export class PerClass extends Grouped {
|
|
7
|
+
#labels;
|
|
8
|
+
constructor(scorer, { labels } = {}) {
|
|
9
|
+
super(scorer);
|
|
10
|
+
this.#labels = labels;
|
|
11
|
+
}
|
|
12
|
+
*groups(comparison) {
|
|
13
|
+
for (const label of this.#labels ?? comparison.labels())
|
|
14
|
+
yield [`${label}`, comparison.only(label)];
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=per-class.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"per-class.js","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIhD,8EAA8E;AAC9E,iEAAiE;AACjE,0EAA0E;AAC1E,wDAAwD;AACxD,MAAM,OAAO,QAAS,SAAQ,OAAwB;IAC3C,OAAO,CAA+B;IAE/C,YACE,MAA+B,EAC/B,EAAE,MAAM,KAA6C,EAAE;QAEvD,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAES,CAAC,MAAM,CACf,UAA2B;QAE3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE;YACrD,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-labels.d.ts","sourceRoot":"","sources":["../../src/classification/read-labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAW,MAAM,YAAY,CAAC;AAIjD,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,EAAE,CAMxE"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { isLabel } from "./label.js";
|
|
2
|
+
// The expected labels of a plain object of a runner: a label or a list of
|
|
3
|
+
// labels.
|
|
4
|
+
export function readLabels(value, name) {
|
|
5
|
+
if (isLabel(value))
|
|
6
|
+
return value;
|
|
7
|
+
if (Array.isArray(value) && value.every(isLabel))
|
|
8
|
+
return value;
|
|
9
|
+
throw new Error(`${name} is a label (a number or a string) or a list of labels, not ${JSON.stringify(value)}`);
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=read-labels.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-labels.js","sourceRoot":"","sources":["../../src/classification/read-labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,0EAA0E;AAC1E,UAAU;AACV,MAAM,UAAU,UAAU,CAAC,KAAc,EAAE,IAAY;IACrD,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,+DAA+D,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export { Ratio } from './aggregation/ratio.ts';
|
|
2
|
+
export { RatioOfSums } from './aggregation/ratio-of-sums.ts';
|
|
3
|
+
export { Counts } from './aggregation/counts.ts';
|
|
4
|
+
export { CountsForm, type FMeasureForm, PrecisionRecallForm, } from './aggregation/f-measure-form.ts';
|
|
5
|
+
export type { Check, Scorecard } from './scoring/scorecard.ts';
|
|
6
|
+
export type { CaseReader, ScorecardScorer, Weighted, } from './scoring/scorecard-scorer.ts';
|
|
7
|
+
export { Totals } from './scoring/totals.ts';
|
|
8
|
+
export { Bootstrap, type WeightedScorecard } from './scoring/bootstrap.ts';
|
|
9
|
+
export type { Scorer } from './scorers/scorer.ts';
|
|
10
|
+
export type { Matches } from './scorers/matches.ts';
|
|
11
|
+
export { Precision } from './scorers/precision.ts';
|
|
12
|
+
export { Recall } from './scorers/recall.ts';
|
|
13
|
+
export { FBeta } from './scorers/f-beta.ts';
|
|
14
|
+
export { F1 } from './scorers/f1.ts';
|
|
15
|
+
export { Penalty } from './scorers/penalty.ts';
|
|
16
|
+
export { Gate } from './scorers/gate.ts';
|
|
17
|
+
export { Penalized } from './scorers/penalized.ts';
|
|
18
|
+
export { AtLeast } from './scorers/at-least.ts';
|
|
19
|
+
export { ShortfallPenalty } from './scorers/shortfall-penalty.ts';
|
|
20
|
+
export { SampleAveraged } from './scorers/sample-averaged.ts';
|
|
21
|
+
export { MeanOf } from './scorers/mean-of.ts';
|
|
22
|
+
export type { Comparer } from './scoring/comparer.ts';
|
|
23
|
+
export { Grouped } from './scoring/grouped.ts';
|
|
24
|
+
export { ScorerSet } from './scoring/scorer-set.ts';
|
|
25
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AACzC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export { Ratio } from "./aggregation/ratio.js";
|
|
2
|
+
export { RatioOfSums } from "./aggregation/ratio-of-sums.js";
|
|
3
|
+
export { Counts } from "./aggregation/counts.js";
|
|
4
|
+
export { CountsForm, PrecisionRecallForm, } from "./aggregation/f-measure-form.js";
|
|
5
|
+
export { Totals } from "./scoring/totals.js";
|
|
6
|
+
export { Bootstrap } from "./scoring/bootstrap.js";
|
|
7
|
+
export { Precision } from "./scorers/precision.js";
|
|
8
|
+
export { Recall } from "./scorers/recall.js";
|
|
9
|
+
export { FBeta } from "./scorers/f-beta.js";
|
|
10
|
+
export { F1 } from "./scorers/f1.js";
|
|
11
|
+
export { Penalty } from "./scorers/penalty.js";
|
|
12
|
+
export { Gate } from "./scorers/gate.js";
|
|
13
|
+
export { Penalized } from "./scorers/penalized.js";
|
|
14
|
+
export { AtLeast } from "./scorers/at-least.js";
|
|
15
|
+
export { ShortfallPenalty } from "./scorers/shortfall-penalty.js";
|
|
16
|
+
export { SampleAveraged } from "./scorers/sample-averaged.js";
|
|
17
|
+
export { MeanOf } from "./scorers/mean-of.js";
|
|
18
|
+
export { Grouped } from "./scoring/grouped.js";
|
|
19
|
+
export { ScorerSet } from "./scoring/scorer-set.js";
|
|
20
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EAEV,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AAOzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export type VarsContext = {
|
|
2
|
+
readonly vars: Readonly<Record<string, unknown>>;
|
|
3
|
+
};
|
|
4
|
+
export type ModuleAssertion = {
|
|
5
|
+
readonly metric?: string | undefined;
|
|
6
|
+
readonly weight: number;
|
|
7
|
+
readonly threshold: number;
|
|
8
|
+
readonly value: (output: string, context: VarsContext) => number;
|
|
9
|
+
};
|
|
10
|
+
export type FileAssertion = {
|
|
11
|
+
readonly type: 'javascript';
|
|
12
|
+
readonly metric: string;
|
|
13
|
+
readonly weight: number;
|
|
14
|
+
readonly threshold: number;
|
|
15
|
+
readonly value: string;
|
|
16
|
+
};
|
|
17
|
+
export declare class AssertionModule<Name extends string> {
|
|
18
|
+
#private;
|
|
19
|
+
readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
|
|
20
|
+
constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
|
|
21
|
+
assert(...names: Name[]): FileAssertion[];
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=assertion-module.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion-module.d.ts","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;CAClE,CAAC;AAKF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAgBF,qBAAa,eAAe,CAAC,IAAI,SAAS,MAAM;;IAG9C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAKrE,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAcrD,MAAM,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,EAAE;CAY1C"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { fileURLToPath } from 'node:url';
|
|
2
|
+
// The module that promptfoo loads the function of each scorer assertion from
|
|
3
|
+
// (ADR 0013). promptfoo stores the config of an eval as JSON, and --resume
|
|
4
|
+
// and --retry-errors build the run from that stored config, so a function in
|
|
5
|
+
// the config is lost and its assertion scores 0. A config holds the file
|
|
6
|
+
// references that assert gives, and promptfoo loads each function from the
|
|
7
|
+
// default export of the module: importModule reads file://<module>:<name> on
|
|
8
|
+
// the default export when the module has one (promptfoo 0.125.1,
|
|
9
|
+
// dist/src/esm-B4NX_h4K.js:205-224). So the module gives functions as its
|
|
10
|
+
// default export, and url is the module's own import.meta.url.
|
|
11
|
+
//
|
|
12
|
+
// The reference holds the absolute path of the module. promptfoo 0.125.1
|
|
13
|
+
// splits a file reference at its first colon
|
|
14
|
+
// (dist/src/evaluator-BxCmvGtv.js:5683-5686), so on Windows the drive of the
|
|
15
|
+
// path ends the path: a defect of promptfoo.
|
|
16
|
+
export class AssertionModule {
|
|
17
|
+
// promptfoo reads a function by the name in its reference, so the names are
|
|
18
|
+
// keys, and assert holds the type of each name.
|
|
19
|
+
functions;
|
|
20
|
+
#path;
|
|
21
|
+
#assertions;
|
|
22
|
+
constructor(url, assertions) {
|
|
23
|
+
this.#path = fileURLToPath(url);
|
|
24
|
+
this.#assertions = assertions;
|
|
25
|
+
this.functions = Object.fromEntries(Object.entries(assertions).map(([name, assertion]) => [
|
|
26
|
+
name,
|
|
27
|
+
assertion.value,
|
|
28
|
+
]));
|
|
29
|
+
}
|
|
30
|
+
// One assertion for each name. The name is the metric, unless its scorer
|
|
31
|
+
// assertion gives one.
|
|
32
|
+
assert(...names) {
|
|
33
|
+
return names.map((name) => {
|
|
34
|
+
const { metric = name, weight, threshold } = this.#assertions[name];
|
|
35
|
+
return {
|
|
36
|
+
type: 'javascript',
|
|
37
|
+
metric,
|
|
38
|
+
weight,
|
|
39
|
+
threshold,
|
|
40
|
+
value: `file://${this.#path}:${name}`,
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=assertion-module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion-module.js","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA6BzC,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,6EAA6E;AAC7E,iEAAiE;AACjE,0EAA0E;AAC1E,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,6CAA6C;AAC7C,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,OAAO,eAAe;IAC1B,4EAA4E;IAC5E,gDAAgD;IACvC,SAAS,CAAqD;IAC9D,KAAK,CAAS;IACd,WAAW,CAA0C;IAE9D,YACE,GAAW,EACX,UAAmD;QAEnD,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CACjC,MAAM,CAAC,OAAO,CAAkB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC;YACrE,IAAI;YACJ,SAAS,CAAC,KAAK;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,uBAAuB;IACvB,MAAM,CAAC,GAAG,KAAa;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACpE,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM;gBACN,SAAS;gBACT,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE;aACtC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { AssertionModule, type FileAssertion, type ModuleAssertion, type VarsContext, } from './assertion-module.ts';
|
|
2
|
+
export { type AssertionContext, type GradingResult, PromptfooAssertion, } from './promptfoo-assertion.ts';
|
|
3
|
+
export { ScorerAssertion } from './scorer-assertion.ts';
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CaseReader, ScorecardScorer, Weighted } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
export type GradingResult = {
|
|
3
|
+
readonly pass: boolean;
|
|
4
|
+
readonly score: number;
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly namedScores?: Readonly<Record<string, number>>;
|
|
7
|
+
readonly namedScoreWeights?: Readonly<Record<string, number>>;
|
|
8
|
+
readonly componentResults?: readonly GradingResult[];
|
|
9
|
+
};
|
|
10
|
+
export type AssertionContext = {
|
|
11
|
+
readonly config?: Readonly<Record<string, unknown>>;
|
|
12
|
+
};
|
|
13
|
+
export declare class PromptfooAssertion<Case extends Weighted> {
|
|
14
|
+
#private;
|
|
15
|
+
constructor(scorer: ScorecardScorer<Case>, cases: CaseReader<Case>);
|
|
16
|
+
readonly grade: (output: unknown, context: AssertionContext) => GradingResult;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=promptfoo-assertion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAKxC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACtD,CAAC;AAIF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD,CAAC;AAOF,qBAAa,kBAAkB,CAAC,IAAI,SAAS,QAAQ;;gBAIvC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;IAKlE,QAAQ,CAAC,KAAK,GACZ,QAAQ,OAAO,EACf,SAAS,gBAAgB,KACxB,aAAa,CAsBd;CACH"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Adapts a scorer to a promptfoo javascript assertion (ADR 0004). Each ratio
|
|
2
|
+
// of the scorecard becomes a named score with its whole as the weight:
|
|
3
|
+
// promptfoo adds value × weight over the rows and divides by the weights
|
|
4
|
+
// (namedMetrics.ts, accumulateNamedMetric), which is the ratio of sums.
|
|
5
|
+
// promptfoo calls a function, so a module exports `grade`.
|
|
6
|
+
export class PromptfooAssertion {
|
|
7
|
+
#scorer;
|
|
8
|
+
#cases;
|
|
9
|
+
constructor(scorer, cases) {
|
|
10
|
+
this.#scorer = scorer;
|
|
11
|
+
this.#cases = cases;
|
|
12
|
+
}
|
|
13
|
+
grade = (output, context) => {
|
|
14
|
+
const testCase = this.#cases.from(context.config ?? {});
|
|
15
|
+
const scorecard = this.#scorer.score(output, testCase);
|
|
16
|
+
const ratios = [...scorecard.ratios].map(([name, ratio]) => [name, ratio.times(testCase.weight)]);
|
|
17
|
+
return {
|
|
18
|
+
pass: scorecard.passed,
|
|
19
|
+
score: scorecard.score,
|
|
20
|
+
reason: scorecard.reason,
|
|
21
|
+
namedScores: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.value])),
|
|
22
|
+
namedScoreWeights: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.whole])),
|
|
23
|
+
componentResults: scorecard.checks.map((check) => ({
|
|
24
|
+
pass: check.passed,
|
|
25
|
+
score: check.score,
|
|
26
|
+
reason: check.reason,
|
|
27
|
+
})),
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=promptfoo-assertion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAwBA,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,wEAAwE;AACxE,2DAA2D;AAC3D,MAAM,OAAO,kBAAkB;IACpB,OAAO,CAAwB;IAC/B,MAAM,CAAmB;IAElC,YAAY,MAA6B,EAAE,KAAuB;QAChE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAEQ,KAAK,GAAG,CACf,MAAe,EACf,OAAyB,EACV,EAAE;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CACtC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAU,CACjE,CAAC;QACF,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,MAAM,CAAC,WAAW,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CAAC;CACH"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Scorer } from '../scorers/scorer.ts';
|
|
2
|
+
import type { Comparer } from '../scoring/comparer.ts';
|
|
3
|
+
import type { CaseReader, Weighted } from '../scoring/scorecard-scorer.ts';
|
|
4
|
+
import type { VarsContext } from './assertion-module.ts';
|
|
5
|
+
export declare class ScorerAssertion<Case extends Weighted, Input> {
|
|
6
|
+
readonly metric: string | undefined;
|
|
7
|
+
readonly weight: number;
|
|
8
|
+
readonly threshold: number;
|
|
9
|
+
readonly value: (output: string, context: VarsContext) => number;
|
|
10
|
+
constructor(scorer: Scorer<Input>, { weight, threshold, metric, comparer, cases, }: {
|
|
11
|
+
readonly weight: number;
|
|
12
|
+
readonly threshold: number;
|
|
13
|
+
readonly metric?: string;
|
|
14
|
+
readonly comparer: Comparer<Case, Input>;
|
|
15
|
+
readonly cases: CaseReader<Case>;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=scorer-assertion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAiBzD,qBAAa,eAAe,CAAC,IAAI,SAAS,QAAQ,EAAE,KAAK;IACvD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;gBAG/D,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EACrB,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,QAAQ,EACR,KAAK,GACN,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;KAClC;CAQJ"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// One scorer with the rule of its promptfoo assertion (ADR 0007, ADR 0013):
|
|
2
|
+
// how much it counts in the score of the test (weight: the score is the sum
|
|
3
|
+
// of value × weight over the sum of the weights), when it passes (threshold:
|
|
4
|
+
// a number passes when it reaches it), and the metric it reports under when
|
|
5
|
+
// that is not its name in its assertion module (metric: promptfoo averages
|
|
6
|
+
// the values of one metric over the tests, which is a metric in the sense of
|
|
7
|
+
// ADR 0009). It is not a promptfoo assertion: promptfoo stores a config as
|
|
8
|
+
// JSON, so a config gets a file reference from its AssertionModule, and
|
|
9
|
+
// promptfoo loads value from the module. value gets the text of the harness,
|
|
10
|
+
// and gives the value of the ratio of the scorer: promptfoo takes the mean of
|
|
11
|
+
// the values of one metric, so a SampleAveraged scorer gives its samples
|
|
12
|
+
// average. The case comes from the vars of the test, as its case reader reads
|
|
13
|
+
// them: an assertion that defaultTest shares gets only the vars of each test.
|
|
14
|
+
// The comparer compares the answer with the case and gives the input of the
|
|
15
|
+
// scorer.
|
|
16
|
+
export class ScorerAssertion {
|
|
17
|
+
metric;
|
|
18
|
+
weight;
|
|
19
|
+
threshold;
|
|
20
|
+
value;
|
|
21
|
+
constructor(scorer, { weight, threshold, metric, comparer, cases, }) {
|
|
22
|
+
this.metric = metric;
|
|
23
|
+
this.weight = weight;
|
|
24
|
+
this.threshold = threshold;
|
|
25
|
+
this.value = (output, { vars }) => scorer.score(comparer.compare(output, cases.from(vars))).value;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=scorer-assertion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAKA,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,4EAA4E;AAC5E,UAAU;AACV,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,KAAK,CAAmD;IAEjE,YACE,MAAqB,EACrB,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,QAAQ,EACR,KAAK,GAON;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAChC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Gate } from './gate.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare class AtLeast<Input> extends Gate<Input> {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(scorer: Scorer<Input>, threshold: number);
|
|
6
|
+
holds(input: Input): boolean;
|
|
7
|
+
explain(input: Input): string;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=at-least.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"at-least.d.ts","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,OAAO,CAAC,KAAK,CAAE,SAAQ,IAAI,CAAC,KAAK,CAAC;;gBAIjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM;IAMpD,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAI5B,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG9B"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Gate } from "./gate.js";
|
|
2
|
+
// A gate: it holds when the value of the scorer reaches the threshold.
|
|
3
|
+
export class AtLeast extends Gate {
|
|
4
|
+
#scorer;
|
|
5
|
+
#threshold;
|
|
6
|
+
constructor(scorer, threshold) {
|
|
7
|
+
super();
|
|
8
|
+
this.#scorer = scorer;
|
|
9
|
+
this.#threshold = threshold;
|
|
10
|
+
}
|
|
11
|
+
holds(input) {
|
|
12
|
+
return this.#scorer.score(input).value >= this.#threshold;
|
|
13
|
+
}
|
|
14
|
+
explain(input) {
|
|
15
|
+
return `the value ${this.#scorer.score(input).value} is below the threshold ${this.#threshold}`;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=at-least.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"at-least.js","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,uEAAuE;AACvE,MAAM,OAAO,OAAe,SAAQ,IAAW;IACpC,OAAO,CAAgB;IACvB,UAAU,CAAS;IAE5B,YAAY,MAAqB,EAAE,SAAiB;QAClD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,aAAa,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,2BAA2B,IAAI,CAAC,UAAU,EAAE,CAAC;IAClG,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
2
|
+
import type { Ratio } from '../aggregation/ratio.ts';
|
|
3
|
+
import type { Matches } from './matches.ts';
|
|
4
|
+
import type { Scorer } from './scorer.ts';
|
|
5
|
+
export declare class FBeta<Input> implements Scorer<Input> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(matches: Matches<Input>, { beta, form, }: {
|
|
8
|
+
readonly beta: number;
|
|
9
|
+
readonly form?: FMeasureForm;
|
|
10
|
+
});
|
|
11
|
+
score(input: Input): Ratio;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=f-beta.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-beta.d.ts","sourceRoot":"","sources":["../../src/scorers/f-beta.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAE1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,KAAK,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAM9C,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EACvB,EACE,IAAI,EACJ,IAAuB,GACxB,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAA;KAAE;IAO5D,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAM3B"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
2
|
+
import { FMeasureRatio } from "../aggregation/f-measure-ratio.js";
|
|
3
|
+
// The F-beta of the hits, as scikit-learn's fbeta_score: β weighs recall β
|
|
4
|
+
// times as much as precision. The form is the arithmetic of the value, and
|
|
5
|
+
// CountsForm is the form of scikit-learn.
|
|
6
|
+
export class FBeta {
|
|
7
|
+
#matches;
|
|
8
|
+
#beta;
|
|
9
|
+
#form;
|
|
10
|
+
constructor(matches, { beta, form = new CountsForm(), }) {
|
|
11
|
+
this.#matches = matches;
|
|
12
|
+
this.#beta = beta;
|
|
13
|
+
this.#form = form;
|
|
14
|
+
}
|
|
15
|
+
score(input) {
|
|
16
|
+
return new FMeasureRatio(this.#matches.counts(input), {
|
|
17
|
+
beta: this.#beta,
|
|
18
|
+
form: this.#form,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=f-beta.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-beta.js","sourceRoot":"","sources":["../../src/scorers/f-beta.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAKlE,2EAA2E;AAC3E,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,OAAO,KAAK;IACP,QAAQ,CAAiB;IACzB,KAAK,CAAS;IACd,KAAK,CAAe;IAE7B,YACE,OAAuB,EACvB,EACE,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,GACiC;QAE1D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACpD,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,IAAI,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
2
|
+
import { FBeta } from './f-beta.ts';
|
|
3
|
+
import type { Matches } from './matches.ts';
|
|
4
|
+
export declare class F1<Input> extends FBeta<Input> {
|
|
5
|
+
constructor(matches: Matches<Input>, options?: {
|
|
6
|
+
readonly form?: FMeasureForm;
|
|
7
|
+
});
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=f1.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1.d.ts","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C,qBAAa,EAAE,CAAC,KAAK,CAAE,SAAQ,KAAK,CAAC,KAAK,CAAC;gBAEvC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EACvB,OAAO,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAA;KAAO;CAIjD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f1.js","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGpC,6CAA6C;AAC7C,MAAM,OAAO,EAAU,SAAQ,KAAY;IACzC,YACE,OAAuB,EACvB,UAA4C,EAAE;QAE9C,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKvC,8BAAsB,IAAI,CAAC,KAAK,CAAE,SAAQ,OAAO,CAAC,KAAK,CAAC;IACtD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAGrC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG7B"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Penalty } from "./penalty.js";
|
|
2
|
+
// The penalty whose factor is 1 when the input holds what the gate checks,
|
|
3
|
+
// else 0. A subclass says what it checks and why an input fails it (Template
|
|
4
|
+
// Method), so a ScorerSet makes a check of each gate.
|
|
5
|
+
export class Gate extends Penalty {
|
|
6
|
+
factor(input) {
|
|
7
|
+
return this.holds(input) ? 1 : 0;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gate.js","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,2EAA2E;AAC3E,6EAA6E;AAC7E,sDAAsD;AACtD,MAAM,OAAgB,IAAY,SAAQ,OAAc;IAMtD,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"matches.d.ts","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIvD,MAAM,WAAW,OAAO,CAAC,KAAK;IAC5B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;CAC9B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"matches.js","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare class MeanOf<Input> implements Scorer<Input> {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(scorer: Scorer<Input>, ...scorers: readonly Scorer<Input>[]);
|
|
6
|
+
score(input: Input): Ratio;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=mean-of.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mean-of.d.ts","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,MAAM,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGrC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE;IAIvE,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAQ3B"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { NeumaierSum } from '@zukhruf/python';
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
// The mean of the values of some scorers for one record, over 1, with
|
|
4
|
+
// CPython's sum, such as a category of a benchmark. It takes one scorer or
|
|
5
|
+
// more, because the mean of no value has no value.
|
|
6
|
+
export class MeanOf {
|
|
7
|
+
#scorers;
|
|
8
|
+
constructor(scorer, ...scorers) {
|
|
9
|
+
this.#scorers = [scorer, ...scorers];
|
|
10
|
+
}
|
|
11
|
+
score(input) {
|
|
12
|
+
return new Ratio(NeumaierSum.mean(this.#scorers.map((scorer) => scorer.score(input).value)), 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=mean-of.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mean-of.js","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,sEAAsE;AACtE,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,OAAO,MAAM;IACR,QAAQ,CAA2B;IAE5C,YAAY,MAAqB,EAAE,GAAG,OAAiC;QACrE,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CACd,WAAW,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CACzD,EACD,CAAC,CACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Penalty } from './penalty.ts';
|
|
3
|
+
import type { Scorer } from './scorer.ts';
|
|
4
|
+
export declare class Penalized<Input> implements Scorer<Input> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(scorer: Scorer<Input>, ...penalties: readonly Penalty<Input>[]);
|
|
7
|
+
score(input: Input): Ratio;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=penalized.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalized.d.ts","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAS1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAIxC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,EAAE;IAK1E,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAgB3B"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
2
|
+
// A scorer whose part is multiplied by the factor of each penalty, in the
|
|
3
|
+
// order of the penalties, for each record; the whole stays (Decorator). So a
|
|
4
|
+
// record that fails a gate keeps its expected values in the denominator of
|
|
5
|
+
// recall. A penalty on the means would give mean(a) × mean(b), so the
|
|
6
|
+
// penalty is per record (ADR 0004). When every factor is 1, the ratio of the
|
|
7
|
+
// scorer stays as it is, so an F-measure keeps the value of its form (ADR
|
|
8
|
+
// 0012).
|
|
9
|
+
export class Penalized {
|
|
10
|
+
#scorer;
|
|
11
|
+
#penalties;
|
|
12
|
+
constructor(scorer, ...penalties) {
|
|
13
|
+
this.#scorer = scorer;
|
|
14
|
+
this.#penalties = penalties;
|
|
15
|
+
}
|
|
16
|
+
score(input) {
|
|
17
|
+
const ratio = this.#scorer.score(input);
|
|
18
|
+
const factors = this.#penalties.map((penalty) => {
|
|
19
|
+
const factor = penalty.factor(input);
|
|
20
|
+
if (!(factor >= 0 && factor <= 1))
|
|
21
|
+
throw new RangeError(`A penalty gave the factor ${factor}, which is not in [0, 1].`);
|
|
22
|
+
return factor;
|
|
23
|
+
});
|
|
24
|
+
if (factors.every((factor) => factor === 1))
|
|
25
|
+
return ratio;
|
|
26
|
+
return new Ratio(factors.reduce((part, factor) => part * factor, ratio.part), ratio.whole);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=penalized.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalized.js","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,6EAA6E;AAC7E,0EAA0E;AAC1E,SAAS;AACT,MAAM,OAAO,SAAS;IACX,OAAO,CAAgB;IACvB,UAAU,CAA4B;IAE/C,YAAY,MAAqB,EAAE,GAAG,SAAoC;QACxE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,6BAA6B,MAAM,2BAA2B,CAC/D,CAAC;YACJ,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1D,OAAO,IAAI,KAAK,CACd,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAC3D,KAAK,CAAC,KAAK,CACZ,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from './scorer.ts';
|
|
3
|
+
export declare abstract class Penalty<Input> implements Scorer<Input> {
|
|
4
|
+
abstract factor(input: Input): number;
|
|
5
|
+
score(input: Input): Ratio;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=penalty.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalty.d.ts","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAM1C,8BAAsB,OAAO,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;IAC3D,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAErC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAG3B"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
2
|
+
// A factor in [0, 1] for one input, as BLEU's brevity penalty is: 1 is no
|
|
3
|
+
// penalty. Penalized multiplies the part of a scorer by it, for each record.
|
|
4
|
+
// A subclass gives the factor (Template Method). As a scorer, it gives the
|
|
5
|
+
// factor over 1.
|
|
6
|
+
export class Penalty {
|
|
7
|
+
score(input) {
|
|
8
|
+
return new Ratio(this.factor(input), 1);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=penalty.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"penalty.js","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,iBAAiB;AACjB,MAAM,OAAgB,OAAO;IAG3B,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;CACF"}
|