@zukhruf/evals 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/README.md +152 -79
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
- package/dist/aggregation/ratio-of-sums.js +7 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -1
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/average.d.ts +11 -0
- package/dist/classification/average.d.ts.map +1 -0
- package/dist/classification/average.js +2 -0
- package/dist/classification/average.js.map +1 -0
- package/dist/classification/binary-average.d.ts +9 -0
- package/dist/classification/binary-average.d.ts.map +1 -0
- package/dist/classification/binary-average.js +20 -0
- package/dist/classification/binary-average.js.map +1 -0
- package/dist/classification/brier-score.d.ts +13 -0
- package/dist/classification/brier-score.d.ts.map +1 -0
- package/dist/classification/brier-score.js +52 -0
- package/dist/classification/brier-score.js.map +1 -0
- package/dist/classification/class-counts.d.ts +39 -0
- package/dist/classification/class-counts.d.ts.map +1 -0
- package/dist/classification/class-counts.js +159 -0
- package/dist/classification/class-counts.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/confusion-matrix.d.ts +9 -0
- package/dist/classification/confusion-matrix.d.ts.map +1 -0
- package/dist/classification/confusion-matrix.js +18 -0
- package/dist/classification/confusion-matrix.js.map +1 -0
- package/dist/classification/index.d.ts +21 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +19 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/macro-average.d.ts +6 -0
- package/dist/classification/macro-average.d.ts.map +1 -0
- package/dist/classification/macro-average.js +15 -0
- package/dist/classification/macro-average.js.map +1 -0
- package/dist/classification/micro-average.d.ts +6 -0
- package/dist/classification/micro-average.d.ts.map +1 -0
- package/dist/classification/micro-average.js +16 -0
- package/dist/classification/micro-average.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/probability-comparer.d.ts +8 -0
- package/dist/classification/probability-comparer.d.ts.map +1 -0
- package/dist/classification/probability-comparer.js +35 -0
- package/dist/classification/probability-comparer.js.map +1 -0
- package/dist/classification/probability-comparison.d.ts +10 -0
- package/dist/classification/probability-comparison.d.ts.map +1 -0
- package/dist/classification/probability-comparison.js +11 -0
- package/dist/classification/probability-comparison.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/classification/weighted-average.d.ts +6 -0
- package/dist/classification/weighted-average.d.ts.map +1 -0
- package/dist/classification/weighted-average.js +20 -0
- package/dist/classification/weighted-average.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +6 -6
- package/dist/scorers/f1.d.ts.map +1 -1
- package/dist/scorers/f1.js +5 -9
- package/dist/scorers/f1.js.map +1 -1
- package/dist/scorers/penalized.d.ts.map +1 -1
- package/dist/scorers/penalized.js +2 -1
- package/dist/scorers/penalized.js.map +1 -1
- package/dist/scoring/bootstrap.js +3 -5
- package/dist/scoring/bootstrap.js.map +1 -1
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
- package/dist/structured/comparison/json-value.d.ts +1 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -1
- package/dist/structured/comparison/json-value.js +11 -2
- package/dist/structured/comparison/json-value.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts +5 -5
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +8 -4
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/cpython/python-text.d.ts +0 -1
- package/dist/structured/cpython/python-text.d.ts.map +1 -1
- package/dist/structured/cpython/python-text.js +1 -18
- package/dist/structured/cpython/python-text.js.map +1 -1
- package/dist/structured/index.d.ts +0 -2
- package/dist/structured/index.d.ts.map +1 -1
- package/dist/structured/index.js +0 -2
- package/dist/structured/index.js.map +1 -1
- package/dist/structured/scorers/token-f1.d.ts +3 -1
- package/dist/structured/scorers/token-f1.d.ts.map +1 -1
- package/dist/structured/scorers/token-f1.js +8 -5
- package/dist/structured/scorers/token-f1.js.map +1 -1
- package/dist/structured/scoring/per-field.d.ts +3 -6
- package/dist/structured/scoring/per-field.d.ts.map +1 -1
- package/dist/structured/scoring/per-field.js +8 -17
- package/dist/structured/scoring/per-field.js.map +1 -1
- package/package.json +18 -5
- package/dist/aggregation/f1-ratio.d.ts +0 -8
- package/dist/aggregation/f1-ratio.d.ts.map +0 -1
- package/dist/aggregation/f1-ratio.js +0 -24
- package/dist/aggregation/f1-ratio.js.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
- package/dist/structured/scoring/scorer-set.d.ts +0 -15
- package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
- package/dist/structured/scoring/scorer-set.js +0 -60
- package/dist/structured/scoring/scorer-set.js.map +0 -1
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Average, Classes } from './average.ts';
|
|
3
|
+
import type { Label } from './label.ts';
|
|
4
|
+
export declare class BinaryAverage implements Average {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(positiveLabel: Label);
|
|
7
|
+
of({ present, multilabel }: Classes, score: (counts: Counts) => number): number;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=binary-average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,EAChC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAYV"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// average='binary': the score of the positive class, for answers of two
|
|
3
|
+
// classes or fewer. It throws as scikit-learn raises: for a set of classes,
|
|
4
|
+
// for more than two classes, and for a positive label that is not one of two
|
|
5
|
+
// classes. A positive label that no answer has scores counts of 0.
|
|
6
|
+
export class BinaryAverage {
|
|
7
|
+
#positiveLabel;
|
|
8
|
+
constructor(positiveLabel) {
|
|
9
|
+
this.#positiveLabel = positiveLabel;
|
|
10
|
+
}
|
|
11
|
+
of({ present, multilabel }, score) {
|
|
12
|
+
if (multilabel || present.size > 2)
|
|
13
|
+
throw new RangeError(`The target is ${multilabel ? 'multilabel-indicator' : 'multiclass'}, but the average is binary: choose another average.`);
|
|
14
|
+
const counts = present.get(this.#positiveLabel);
|
|
15
|
+
if (counts === undefined && present.size === 2)
|
|
16
|
+
throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not one of the labels ${JSON.stringify([...present.keys()])}.`);
|
|
17
|
+
return score(counts ?? new Counts({ hits: 0, stated: 0, expected: 0 }));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=binary-average.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,wEAAwE;AACxE,4EAA4E;AAC5E,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAW,EAChC,KAAiC;QAEjC,IAAI,UAAU,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC;YAChC,MAAM,IAAI,UAAU,CAClB,iBAAiB,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,sDAAsD,CAC1H,CAAC;QACJ,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAC5C,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC7H,CAAC;QACJ,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../scorers/scorer.ts';
|
|
3
|
+
import { type Label } from './label.ts';
|
|
4
|
+
import type { ProbabilityComparison } from './probability-comparison.ts';
|
|
5
|
+
export declare class BrierScore implements Scorer<ProbabilityComparison> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor({ positiveLabel }?: {
|
|
8
|
+
readonly positiveLabel?: Label;
|
|
9
|
+
});
|
|
10
|
+
score(comparison: ProbabilityComparison): Ratio;
|
|
11
|
+
mean(comparisons: readonly ProbabilityComparison[], weights?: readonly number[]): number;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=brier-score.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AASzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAGlD,EAAE,aAAa,EAAE,GAAE;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,CAAA;KAAO;IAItE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAyCV"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
import { compareLabels } from "./label.js";
|
|
4
|
+
// scikit-learn 1.9.1's brier_score_loss. For one answer, the sum over the
|
|
5
|
+
// classes of (1 when the class is the expected label, else 0, minus the
|
|
6
|
+
// stated probability) squared, with numpy's pairwise sum over the classes in
|
|
7
|
+
// their sorted order; times 0.5 for two classes. With a positive label, the
|
|
8
|
+
// answer gives the probability p of that label only, and the two classes are
|
|
9
|
+
// 1 - p and p, as scikit-learn reads a one-dimensional y_proba. Its ratio is
|
|
10
|
+
// the score over 1, and mean gives the score of a whole run.
|
|
11
|
+
export class BrierScore {
|
|
12
|
+
#positiveLabel;
|
|
13
|
+
constructor({ positiveLabel } = {}) {
|
|
14
|
+
this.#positiveLabel = positiveLabel;
|
|
15
|
+
}
|
|
16
|
+
score(comparison) {
|
|
17
|
+
return new Ratio(this.#value(comparison), 1);
|
|
18
|
+
}
|
|
19
|
+
// The mean of the scores of the answers, with numpy's pairwise sum, each
|
|
20
|
+
// answer with its weight (1 when none is given).
|
|
21
|
+
mean(comparisons, weights) {
|
|
22
|
+
const scores = comparisons.map((comparison) => this.#value(comparison));
|
|
23
|
+
if (weights === undefined)
|
|
24
|
+
return gsumpw(scores.length, scores, 1) / scores.length;
|
|
25
|
+
if (weights.length !== scores.length)
|
|
26
|
+
throw new RangeError(`The answers and the weights have ${scores.length} and ${weights.length} values.`);
|
|
27
|
+
const weighted = scores.map((score, index) => score * (weights[index] ?? 0));
|
|
28
|
+
return (gsumpw(weighted.length, weighted, 1) / gsumpw(weights.length, weights, 1));
|
|
29
|
+
}
|
|
30
|
+
#value({ probabilities, expected }) {
|
|
31
|
+
if (this.#positiveLabel !== undefined) {
|
|
32
|
+
const p = probabilities.get(this.#positiveLabel);
|
|
33
|
+
if (p === undefined)
|
|
34
|
+
throw new RangeError(`The answer gives no probability of the positive label ${JSON.stringify(this.#positiveLabel)}.`);
|
|
35
|
+
const y = expected === this.#positiveLabel ? 1 : 0;
|
|
36
|
+
const negative = 1 - y - (1 - p);
|
|
37
|
+
const positive = y - p;
|
|
38
|
+
return 0.5 * (negative * negative + positive * positive);
|
|
39
|
+
}
|
|
40
|
+
if (!probabilities.has(expected))
|
|
41
|
+
throw new RangeError(`The answer gives no probability of the expected label ${JSON.stringify(expected)}.`);
|
|
42
|
+
const squares = [...probabilities]
|
|
43
|
+
.toSorted(([left], [right]) => compareLabels(left, right))
|
|
44
|
+
.map(([label, probability]) => {
|
|
45
|
+
const difference = (label === expected ? 1 : 0) - probability;
|
|
46
|
+
return difference * difference;
|
|
47
|
+
});
|
|
48
|
+
const sum = gsumpw(squares.length, squares, 1);
|
|
49
|
+
return squares.length < 3 ? 0.5 * sum : sum;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=brier-score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAE3C,YAAY,EAAE,aAAa,KAAyC,EAAE;QACpE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YACvB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,MAAM,CAAC,GAAG,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;CACF"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
2
|
+
import type { Average } from './average.ts';
|
|
3
|
+
import { ConfusionMatrix } from './confusion-matrix.ts';
|
|
4
|
+
import { LabelComparison } from './label-comparison.ts';
|
|
5
|
+
import { type Label } from './label.ts';
|
|
6
|
+
type ScoreOptions = {
|
|
7
|
+
readonly average: Average;
|
|
8
|
+
readonly labels?: readonly Label[];
|
|
9
|
+
readonly zeroDivision?: 0 | 1 | number;
|
|
10
|
+
};
|
|
11
|
+
export declare class ClassCounts {
|
|
12
|
+
#private;
|
|
13
|
+
constructor({ multilabel }: {
|
|
14
|
+
readonly multilabel: boolean;
|
|
15
|
+
});
|
|
16
|
+
static of({ expected, stated, weights, }: {
|
|
17
|
+
readonly expected: readonly Label[];
|
|
18
|
+
readonly stated: readonly Label[];
|
|
19
|
+
readonly weights?: readonly number[];
|
|
20
|
+
}): ClassCounts;
|
|
21
|
+
static ofIndicators({ expected, stated, weights, }: {
|
|
22
|
+
readonly expected: readonly (readonly number[])[];
|
|
23
|
+
readonly stated: readonly (readonly number[])[];
|
|
24
|
+
readonly weights?: readonly number[];
|
|
25
|
+
}): ClassCounts;
|
|
26
|
+
add(comparison: LabelComparison, weight: number): void;
|
|
27
|
+
labels(): Label[];
|
|
28
|
+
accuracy(): number;
|
|
29
|
+
precision(options: ScoreOptions): number;
|
|
30
|
+
recall(options: ScoreOptions): number;
|
|
31
|
+
fBeta({ beta, form, ...options }: ScoreOptions & {
|
|
32
|
+
readonly beta: number;
|
|
33
|
+
readonly form?: FMeasureForm;
|
|
34
|
+
}): number;
|
|
35
|
+
support(): ReadonlyMap<Label, number>;
|
|
36
|
+
confusionMatrix(): ConfusionMatrix;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
39
|
+
//# sourceMappingURL=class-counts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAQlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
import { Counts } from "../aggregation/counts.js";
|
|
3
|
+
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
|
+
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
5
|
+
import { LabelComparison } from "./label-comparison.js";
|
|
6
|
+
import { compareLabels } from "./label.js";
|
|
7
|
+
// The running counts of one class, added in the order of the answers.
|
|
8
|
+
class Tally {
|
|
9
|
+
hits = 0;
|
|
10
|
+
stated = 0;
|
|
11
|
+
expected = 0;
|
|
12
|
+
}
|
|
13
|
+
// The counts of each class over many answers, as scikit-learn's
|
|
14
|
+
// multilabel_confusion_matrix gives them: the true positives (hits), the
|
|
15
|
+
// predicted (stated) and the true (expected), each the sum of the weights of
|
|
16
|
+
// the answers in their order, as numpy's bincount adds them. Add each
|
|
17
|
+
// comparison, then read the metrics of a whole run (torchmetrics' update and
|
|
18
|
+
// compute). It also keeps the weight of each right answer for the accuracy,
|
|
19
|
+
// and the weight of each pair of labels for the confusion matrix. The counts
|
|
20
|
+
// are of one class for each answer (multiclass) or of a set of classes
|
|
21
|
+
// (multilabel), as torchmetrics has a multiclass and a multilabel class.
|
|
22
|
+
export class ClassCounts {
|
|
23
|
+
#multilabel;
|
|
24
|
+
#classes = new Map();
|
|
25
|
+
#rights = [];
|
|
26
|
+
#weights = [];
|
|
27
|
+
#pairs = new Map();
|
|
28
|
+
constructor({ multilabel }) {
|
|
29
|
+
this.#multilabel = multilabel;
|
|
30
|
+
}
|
|
31
|
+
// The counts of two arrays of labels, one label for each answer
|
|
32
|
+
// (multiclass), with the weight of each answer (1 when none is given).
|
|
33
|
+
static of({ expected, stated, weights, }) {
|
|
34
|
+
const counts = new ClassCounts({ multilabel: false });
|
|
35
|
+
for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
36
|
+
counts.add(new LabelComparison({
|
|
37
|
+
stated: new Set([statedLabel]),
|
|
38
|
+
expected: new Set([expectedLabel]),
|
|
39
|
+
}), weight);
|
|
40
|
+
return counts;
|
|
41
|
+
}
|
|
42
|
+
// The counts of two label indicator matrices (multilabel): a row of 0 and 1
|
|
43
|
+
// for each answer, a column for each label. Each column is a class, also
|
|
44
|
+
// when no answer has it.
|
|
45
|
+
static ofIndicators({ expected, stated, weights, }) {
|
|
46
|
+
const columns = expected[0]?.length ?? 0;
|
|
47
|
+
const counts = new ClassCounts({ multilabel: true });
|
|
48
|
+
for (let column = 0; column < columns; column++)
|
|
49
|
+
counts.#classes.set(column, new Tally());
|
|
50
|
+
const labels = (row) => {
|
|
51
|
+
if (row.length !== columns ||
|
|
52
|
+
row.some((value) => value !== 0 && value !== 1))
|
|
53
|
+
throw new RangeError(`A row of a label indicator matrix holds ${columns} values of 0 or 1, not ${JSON.stringify(row)}.`);
|
|
54
|
+
return new Set(row.flatMap((value, column) => (value === 1 ? [column] : [])));
|
|
55
|
+
};
|
|
56
|
+
for (const [expectedRow, statedRow, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
57
|
+
counts.add(new LabelComparison({
|
|
58
|
+
stated: labels(statedRow),
|
|
59
|
+
expected: labels(expectedRow),
|
|
60
|
+
}), weight);
|
|
61
|
+
return counts;
|
|
62
|
+
}
|
|
63
|
+
// The expected side, the stated side and the weight of each answer. An
|
|
64
|
+
// answer without a weight weighs 1.
|
|
65
|
+
static *#answers(expected, stated, weights) {
|
|
66
|
+
const unequal = new RangeError(`The expected labels, the stated labels and the weights have ${expected.length}, ${stated.length} and ${weights?.length ?? expected.length} answers.`);
|
|
67
|
+
if (stated.length !== expected.length ||
|
|
68
|
+
(weights !== undefined && weights.length !== expected.length))
|
|
69
|
+
throw unequal;
|
|
70
|
+
for (const [index, side] of expected.entries()) {
|
|
71
|
+
const other = stated[index];
|
|
72
|
+
if (other === undefined)
|
|
73
|
+
throw unequal;
|
|
74
|
+
yield [side, other, weights?.[index] ?? 1];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// Adds one answer with its weight. Multiclass counts refuse an answer with
|
|
78
|
+
// no label or more than one label on a side.
|
|
79
|
+
add(comparison, weight) {
|
|
80
|
+
const { stated, expected } = comparison;
|
|
81
|
+
if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
|
|
82
|
+
throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
|
|
83
|
+
for (const label of comparison.labels()) {
|
|
84
|
+
const tally = this.#classes.getOrInsertComputed(label, () => new Tally());
|
|
85
|
+
if (stated.has(label))
|
|
86
|
+
tally.stated += weight;
|
|
87
|
+
if (expected.has(label))
|
|
88
|
+
tally.expected += weight;
|
|
89
|
+
if (stated.has(label) && expected.has(label))
|
|
90
|
+
tally.hits += weight;
|
|
91
|
+
}
|
|
92
|
+
const right = stated.size === expected.size && stated.isSubsetOf(expected);
|
|
93
|
+
this.#rights.push(right ? weight : 0);
|
|
94
|
+
this.#weights.push(weight);
|
|
95
|
+
if (this.#multilabel)
|
|
96
|
+
return;
|
|
97
|
+
const [expectedLabel] = expected;
|
|
98
|
+
const [statedLabel] = stated;
|
|
99
|
+
if (expectedLabel === undefined || statedLabel === undefined)
|
|
100
|
+
return;
|
|
101
|
+
const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
|
|
102
|
+
row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
|
|
103
|
+
}
|
|
104
|
+
// The labels of the classes, in the order of numpy's unique().
|
|
105
|
+
labels() {
|
|
106
|
+
return [...this.#classes.keys()].toSorted(compareLabels);
|
|
107
|
+
}
|
|
108
|
+
// accuracy_score: the weight of the right answers over the weight of all
|
|
109
|
+
// answers, each with numpy's pairwise sum. An answer is right when its
|
|
110
|
+
// stated labels are its expected labels (the subset accuracy for a set of
|
|
111
|
+
// classes).
|
|
112
|
+
accuracy() {
|
|
113
|
+
return (gsumpw(this.#rights.length, this.#rights, 1) /
|
|
114
|
+
gsumpw(this.#weights.length, this.#weights, 1));
|
|
115
|
+
}
|
|
116
|
+
// precision_score: hits / stated of each class, then the average.
|
|
117
|
+
precision(options) {
|
|
118
|
+
return this.#score(options, ({ hits, stated }) => stated === 0 ? undefined : hits / stated);
|
|
119
|
+
}
|
|
120
|
+
// recall_score: hits / expected of each class, then the average.
|
|
121
|
+
recall(options) {
|
|
122
|
+
return this.#score(options, ({ hits, expected }) => expected === 0 ? undefined : hits / expected);
|
|
123
|
+
}
|
|
124
|
+
// fbeta_score: the F-beta of each class in the form (scikit-learn's
|
|
125
|
+
// CountsForm by default), then the average. Its denominator is
|
|
126
|
+
// β²·expected + stated.
|
|
127
|
+
fBeta({ beta, form = new CountsForm(), ...options }) {
|
|
128
|
+
return this.#score(options, (counts) => beta ** 2 * counts.expected + counts.stated === 0
|
|
129
|
+
? undefined
|
|
130
|
+
: form.of(counts, beta));
|
|
131
|
+
}
|
|
132
|
+
// The true count of each class, in the order of the labels.
|
|
133
|
+
support() {
|
|
134
|
+
return new Map(this.labels().map((label) => [label, this.#counts(label).expected]));
|
|
135
|
+
}
|
|
136
|
+
// confusion_matrix of the answers, with the labels in their order. A set
|
|
137
|
+
// of classes has no confusion matrix (scikit-learn raises), so it throws.
|
|
138
|
+
confusionMatrix() {
|
|
139
|
+
if (this.#multilabel)
|
|
140
|
+
throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
|
|
141
|
+
return new ConfusionMatrix(this.labels(), this.#pairs);
|
|
142
|
+
}
|
|
143
|
+
#score({ average, labels = this.labels(), zeroDivision = 0 }, score) {
|
|
144
|
+
return average.of({
|
|
145
|
+
selected: new Map(labels.map((label) => [label, this.#counts(label)])),
|
|
146
|
+
present: new Map(this.labels().map((label) => [label, this.#counts(label)])),
|
|
147
|
+
multilabel: this.#multilabel,
|
|
148
|
+
}, (counts) => score(counts) ?? zeroDivision);
|
|
149
|
+
}
|
|
150
|
+
#counts(label) {
|
|
151
|
+
const tally = this.#classes.get(label);
|
|
152
|
+
return new Counts({
|
|
153
|
+
hits: tally?.hits ?? 0,
|
|
154
|
+
stated: tally?.stated ?? 0,
|
|
155
|
+
expected: tally?.expected ?? 0,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=class-counts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAWvD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;CACd;AAED,gEAAgE;AAChE,yEAAyE;AACzE,6EAA6E;AAC7E,sEAAsE;AACtE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IACnC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EAAE,UAAU,EAAoC;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;aACnC,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;aAC9B,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6CAA6C;IAC7C,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,+DAA+D;IAC/D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,CACL,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAC/C,CAAC;IACJ,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAgB,EACnE,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,IAAI,GAAG,CACd,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAC3D;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YACtB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC1B,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Matches } from '../scorers/matches.ts';
|
|
3
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
import type { Label } from './label.ts';
|
|
5
|
+
export declare class ClassMatches implements Matches<LabelComparison> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(label: Label);
|
|
8
|
+
counts({ stated, expected }: LabelComparison): Counts;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=class-matches.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,YAAa,YAAW,OAAO,CAAC,eAAe,CAAC;;gBAG/C,KAAK,EAAE,KAAK;IAIxB,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAStD"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// The counts of one class in one answer: the hit is 1 when both sides have
|
|
3
|
+
// the label, the stated 1 when the answer has it, the expected 1 when the
|
|
4
|
+
// case has it. Pooled over the answers, they are scikit-learn's true
|
|
5
|
+
// positives, predicted and true of the class.
|
|
6
|
+
export class ClassMatches {
|
|
7
|
+
#label;
|
|
8
|
+
constructor(label) {
|
|
9
|
+
this.#label = label;
|
|
10
|
+
}
|
|
11
|
+
counts({ stated, expected }) {
|
|
12
|
+
const isStated = stated.has(this.#label) ? 1 : 0;
|
|
13
|
+
const isExpected = expected.has(this.#label) ? 1 : 0;
|
|
14
|
+
return new Counts({
|
|
15
|
+
hits: isStated * isExpected,
|
|
16
|
+
stated: isStated,
|
|
17
|
+
expected: isExpected,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=class-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-matches.js","sourceRoot":"","sources":["../../src/classification/class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAKlD,2EAA2E;AAC3E,0EAA0E;AAC1E,qEAAqE;AACrE,8CAA8C;AAC9C,MAAM,OAAO,YAAY;IACd,MAAM,CAAQ;IAEvB,YAAY,KAAY;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,QAAQ,GAAG,UAAU;YAC3B,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Label } from './label.ts';
|
|
2
|
+
export declare class ConfusionMatrix {
|
|
3
|
+
#private;
|
|
4
|
+
readonly labels: readonly Label[];
|
|
5
|
+
constructor(labels: readonly Label[], pairs: ReadonlyMap<Label, ReadonlyMap<Label, number>>);
|
|
6
|
+
count(expected: Label, stated: Label): number;
|
|
7
|
+
rows(): number[][];
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=confusion-matrix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"confusion-matrix.d.ts","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,eAAe;;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;gBAIhC,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAMvD,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,MAAM;IAI7C,IAAI,IAAI,MAAM,EAAE,EAAE;CAKnB"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// The weight of each pair of an expected label and a stated label, as
|
|
2
|
+
// scikit-learn's confusion_matrix gives it: a row for each expected label and
|
|
3
|
+
// a column for each stated label, in the order of the labels.
|
|
4
|
+
export class ConfusionMatrix {
|
|
5
|
+
labels;
|
|
6
|
+
#pairs;
|
|
7
|
+
constructor(labels, pairs) {
|
|
8
|
+
this.labels = labels;
|
|
9
|
+
this.#pairs = pairs;
|
|
10
|
+
}
|
|
11
|
+
count(expected, stated) {
|
|
12
|
+
return this.#pairs.get(expected)?.get(stated) ?? 0;
|
|
13
|
+
}
|
|
14
|
+
rows() {
|
|
15
|
+
return this.labels.map((expected) => this.labels.map((stated) => this.count(expected, stated)));
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=confusion-matrix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"confusion-matrix.js","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAEA,sEAAsE;AACtE,8EAA8E;AAC9E,8DAA8D;AAC9D,MAAM,OAAO,eAAe;IACjB,MAAM,CAAmB;IACzB,MAAM,CAAiD;IAEhE,YACE,MAAwB,EACxB,KAAqD;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAe,EAAE,MAAa;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAC1D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export { AllClassMatches } from './all-class-matches.ts';
|
|
2
|
+
export { ClassMatches } from './class-matches.ts';
|
|
3
|
+
export { LabelCase } from './label-case.ts';
|
|
4
|
+
export { LabelComparer } from './label-comparer.ts';
|
|
5
|
+
export { LabelComparison } from './label-comparison.ts';
|
|
6
|
+
export { LabelConfigCase } from './label-config-case.ts';
|
|
7
|
+
export { LabelVarsCase } from './label-vars-case.ts';
|
|
8
|
+
export { LabelsMatch } from './labels-match.ts';
|
|
9
|
+
export type { Label } from './label.ts';
|
|
10
|
+
export { PerClass } from './per-class.ts';
|
|
11
|
+
export type { Average, Classes } from './average.ts';
|
|
12
|
+
export { BinaryAverage } from './binary-average.ts';
|
|
13
|
+
export { ClassCounts } from './class-counts.ts';
|
|
14
|
+
export { ConfusionMatrix } from './confusion-matrix.ts';
|
|
15
|
+
export { MacroAverage } from './macro-average.ts';
|
|
16
|
+
export { MicroAverage } from './micro-average.ts';
|
|
17
|
+
export { WeightedAverage } from './weighted-average.ts';
|
|
18
|
+
export { BrierScore } from './brier-score.ts';
|
|
19
|
+
export { ProbabilityComparer } from './probability-comparer.ts';
|
|
20
|
+
export { ProbabilityComparison } from './probability-comparison.ts';
|
|
21
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export { AllClassMatches } from "./all-class-matches.js";
|
|
2
|
+
export { ClassMatches } from "./class-matches.js";
|
|
3
|
+
export { LabelCase } from "./label-case.js";
|
|
4
|
+
export { LabelComparer } from "./label-comparer.js";
|
|
5
|
+
export { LabelComparison } from "./label-comparison.js";
|
|
6
|
+
export { LabelConfigCase } from "./label-config-case.js";
|
|
7
|
+
export { LabelVarsCase } from "./label-vars-case.js";
|
|
8
|
+
export { LabelsMatch } from "./labels-match.js";
|
|
9
|
+
export { PerClass } from "./per-class.js";
|
|
10
|
+
export { BinaryAverage } from "./binary-average.js";
|
|
11
|
+
export { ClassCounts } from "./class-counts.js";
|
|
12
|
+
export { ConfusionMatrix } from "./confusion-matrix.js";
|
|
13
|
+
export { MacroAverage } from "./macro-average.js";
|
|
14
|
+
export { MicroAverage } from "./micro-average.js";
|
|
15
|
+
export { WeightedAverage } from "./weighted-average.js";
|
|
16
|
+
export { BrierScore } from "./brier-score.js";
|
|
17
|
+
export { ProbabilityComparer } from "./probability-comparer.js";
|
|
18
|
+
export { ProbabilityComparison } from "./probability-comparison.js";
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Weighted } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
import type { Label } from './label.ts';
|
|
3
|
+
export declare class LabelCase implements Weighted {
|
|
4
|
+
readonly expected: ReadonlySet<Label>;
|
|
5
|
+
readonly weight: number;
|
|
6
|
+
constructor({ expected, weight, }: {
|
|
7
|
+
readonly expected: Label | readonly Label[];
|
|
8
|
+
readonly weight: number;
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=label-case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-case.d.ts","sourceRoot":"","sources":["../../src/classification/label-case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,SAAU,YAAW,QAAQ;IACxC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,EACV,QAAQ,EACR,MAAM,GACP,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,SAAS,KAAK,EAAE,CAAC;QAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB;CAMF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// What one test gives the scorers of labels: its expected labels and its
|
|
2
|
+
// weight. One label is a set of one label, so a class (multiclass) and a set
|
|
3
|
+
// of classes (multilabel) are one model.
|
|
4
|
+
export class LabelCase {
|
|
5
|
+
expected;
|
|
6
|
+
weight;
|
|
7
|
+
constructor({ expected, weight, }) {
|
|
8
|
+
this.expected = new Set(typeof expected === 'object' ? expected : [expected]);
|
|
9
|
+
this.weight = weight;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=label-case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-case.js","sourceRoot":"","sources":["../../src/classification/label-case.ts"],"names":[],"mappings":"AAGA,yEAAyE;AACzE,6EAA6E;AAC7E,yCAAyC;AACzC,MAAM,OAAO,SAAS;IACX,QAAQ,CAAqB;IAC7B,MAAM,CAAS;IAExB,YAAY,EACV,QAAQ,EACR,MAAM,GAIP;QACC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CACrB,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrD,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Comparer } from '../scoring/comparer.ts';
|
|
2
|
+
import type { LabelCase } from './label-case.ts';
|
|
3
|
+
import { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
export declare class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
5
|
+
#private;
|
|
6
|
+
compare(stated: unknown, testCase: LabelCase): LabelComparison;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=label-comparer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAQxD,qBAAa,aAAc,YAAW,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;;IACxE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,eAAe;CAmC/D"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { LabelComparison } from "./label-comparison.js";
|
|
2
|
+
import { isLabel } from "./label.js";
|
|
3
|
+
// Reads the stated labels of an answer and gives its comparison with the case
|
|
4
|
+
// (Strategy). A label or a list of labels is read as it is. Text is read so:
|
|
5
|
+
// a JSON array of labels gives those labels; a number gives a number label
|
|
6
|
+
// only when each expected label of the case is a number; any other text,
|
|
7
|
+
// without its white space at the start and the end, is one string label.
|
|
8
|
+
export class LabelComparer {
|
|
9
|
+
compare(stated, testCase) {
|
|
10
|
+
return new LabelComparison({
|
|
11
|
+
stated: new Set(LabelComparer.#labels(stated, testCase)),
|
|
12
|
+
expected: testCase.expected,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
static #labels(stated, testCase) {
|
|
16
|
+
if (Array.isArray(stated) && stated.every(isLabel))
|
|
17
|
+
return stated;
|
|
18
|
+
if (typeof stated === 'number' && isLabel(stated))
|
|
19
|
+
return [stated];
|
|
20
|
+
if (typeof stated !== 'string')
|
|
21
|
+
throw new TypeError(`A stated answer is a label, a list of labels or text, not ${JSON.stringify(stated)}`);
|
|
22
|
+
const text = stated.trim();
|
|
23
|
+
const value = LabelComparer.#parse(text);
|
|
24
|
+
if (Array.isArray(value) && value.every(isLabel))
|
|
25
|
+
return value;
|
|
26
|
+
if (typeof value === 'number' &&
|
|
27
|
+
isLabel(value) &&
|
|
28
|
+
testCase.expected.size > 0 &&
|
|
29
|
+
[...testCase.expected].every((label) => typeof label === 'number'))
|
|
30
|
+
return [value];
|
|
31
|
+
return [text];
|
|
32
|
+
}
|
|
33
|
+
static #parse(text) {
|
|
34
|
+
try {
|
|
35
|
+
return JSON.parse(text);
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
if (error instanceof SyntaxError)
|
|
39
|
+
return undefined;
|
|
40
|
+
throw error;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=label-comparer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparer.js","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,OAAO,aAAa;IACxB,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACxD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,QAAmB;QACjD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,MAAM,CAAC;QAClE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,OAAO,MAAM,KAAK,QAAQ;YAC5B,MAAM,IAAI,SAAS,CACjB,6DAA6D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACtF,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/D,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,CAAC,KAAK,CAAC;YACd,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAC1B,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;YAElE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAY;QACxB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,WAAW;gBAAE,OAAO,SAAS,CAAC;YACnD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type Label } from './label.ts';
|
|
2
|
+
export declare class LabelComparison {
|
|
3
|
+
readonly stated: ReadonlySet<Label>;
|
|
4
|
+
readonly expected: ReadonlySet<Label>;
|
|
5
|
+
constructor({ stated, expected, }: {
|
|
6
|
+
readonly stated: ReadonlySet<Label>;
|
|
7
|
+
readonly expected: ReadonlySet<Label>;
|
|
8
|
+
});
|
|
9
|
+
labels(): Label[];
|
|
10
|
+
only(label: Label): LabelComparison;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=label-comparison.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAGvD,qBAAa,eAAe;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;gBAE1B,EACV,MAAM,EACN,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;KACvC;IAMD,MAAM,IAAI,KAAK,EAAE;IAKjB,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,eAAe;CAOpC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { compareLabels } from "./label.js";
|
|
2
|
+
// The stated labels of one answer and the expected labels of its case.
|
|
3
|
+
export class LabelComparison {
|
|
4
|
+
stated;
|
|
5
|
+
expected;
|
|
6
|
+
constructor({ stated, expected, }) {
|
|
7
|
+
this.stated = stated;
|
|
8
|
+
this.expected = expected;
|
|
9
|
+
}
|
|
10
|
+
// The labels of both sides, in the order of numpy's unique().
|
|
11
|
+
labels() {
|
|
12
|
+
return [...this.stated.union(this.expected)].toSorted(compareLabels);
|
|
13
|
+
}
|
|
14
|
+
// The comparison of one class: each side keeps the label when it has it.
|
|
15
|
+
only(label) {
|
|
16
|
+
const one = new Set([label]);
|
|
17
|
+
return new LabelComparison({
|
|
18
|
+
stated: this.stated.intersection(one),
|
|
19
|
+
expected: this.expected.intersection(one),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=label-comparison.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparison.js","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAEvD,uEAAuE;AACvE,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAEtC,YAAY,EACV,MAAM,EACN,QAAQ,GAIT;QACC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,8DAA8D;IAC9D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED,yEAAyE;IACzE,IAAI,CAAC,KAAY;QACf,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC;YACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC;SAC1C,CAAC,CAAC;IACL,CAAC;CACF"}
|