@zukhruf/evals 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/README.md +143 -72
- package/dist/classification/average.d.ts +11 -0
- package/dist/classification/average.d.ts.map +1 -0
- package/dist/classification/average.js +2 -0
- package/dist/classification/average.js.map +1 -0
- package/dist/classification/binary-average.d.ts +9 -0
- package/dist/classification/binary-average.d.ts.map +1 -0
- package/dist/classification/binary-average.js +20 -0
- package/dist/classification/binary-average.js.map +1 -0
- package/dist/classification/brier-form.d.ts +10 -0
- package/dist/classification/brier-form.d.ts.map +1 -0
- package/dist/classification/brier-form.js +18 -0
- package/dist/classification/brier-form.js.map +1 -0
- package/dist/classification/brier-score.d.ts +18 -0
- package/dist/classification/brier-score.d.ts.map +1 -0
- package/dist/classification/brier-score.js +53 -0
- package/dist/classification/brier-score.js.map +1 -0
- package/dist/classification/class-counts.d.ts +39 -0
- package/dist/classification/class-counts.d.ts.map +1 -0
- package/dist/classification/class-counts.js +159 -0
- package/dist/classification/class-counts.js.map +1 -0
- package/dist/classification/confusion-matrix.d.ts +9 -0
- package/dist/classification/confusion-matrix.d.ts.map +1 -0
- package/dist/classification/confusion-matrix.js +18 -0
- package/dist/classification/confusion-matrix.js.map +1 -0
- package/dist/classification/index.d.ts +11 -0
- package/dist/classification/index.d.ts.map +1 -1
- package/dist/classification/index.js +10 -0
- package/dist/classification/index.js.map +1 -1
- package/dist/classification/macro-average.d.ts +6 -0
- package/dist/classification/macro-average.d.ts.map +1 -0
- package/dist/classification/macro-average.js +15 -0
- package/dist/classification/macro-average.js.map +1 -0
- package/dist/classification/micro-average.d.ts +6 -0
- package/dist/classification/micro-average.d.ts.map +1 -0
- package/dist/classification/micro-average.js +16 -0
- package/dist/classification/micro-average.js.map +1 -0
- package/dist/classification/probability-comparer.d.ts +8 -0
- package/dist/classification/probability-comparer.d.ts.map +1 -0
- package/dist/classification/probability-comparer.js +35 -0
- package/dist/classification/probability-comparer.js.map +1 -0
- package/dist/classification/probability-comparison.d.ts +10 -0
- package/dist/classification/probability-comparison.d.ts.map +1 -0
- package/dist/classification/probability-comparison.js +11 -0
- package/dist/classification/probability-comparison.js.map +1 -0
- package/dist/classification/weighted-average.d.ts +6 -0
- package/dist/classification/weighted-average.d.ts.map +1 -0
- package/dist/classification/weighted-average.js +20 -0
- package/dist/classification/weighted-average.js.map +1 -0
- package/package.json +12 -5
package/NOTICE
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
@zukhruf/
|
|
1
|
+
@zukhruf/evals contains a TypeScript port of the scorer of the
|
|
2
2
|
Structured Output Benchmark (SOB): evaluate.py at commit da785a8 of
|
|
3
3
|
https://github.com/interfazeai/sob. That code has this licence:
|
|
4
4
|
|
|
@@ -24,7 +24,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
24
24
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
25
|
SOFTWARE.
|
|
26
26
|
|
|
27
|
-
@zukhruf/
|
|
27
|
+
@zukhruf/evals contains test cases from the JSON-Schema-Test-Suite:
|
|
28
28
|
tests/draft2020-12/optional/ecmascript-regex.json and
|
|
29
29
|
tests/draft7/optional/ecmascript-regex.json at commit
|
|
30
30
|
245a45abc90190c0bff04d97e6d2b541b4bb9f12 of
|
|
@@ -53,7 +53,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
53
53
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
54
54
|
THE SOFTWARE.
|
|
55
55
|
|
|
56
|
-
@zukhruf/
|
|
56
|
+
@zukhruf/evals contains the JSON Schema meta-schemas of draft 7 and
|
|
57
57
|
draft 2020-12 (src/schema/draft-07-schema.ts and
|
|
58
58
|
src/schema/draft-2020-12-schema.ts), copied from jsonschema-specifications
|
|
59
59
|
2025.9.1 (https://github.com/python-jsonschema/jsonschema-specifications). That
|
package/README.md
CHANGED
|
@@ -6,18 +6,18 @@ The answers of evals have different shapes: a structured record, a label, a rank
|
|
|
6
6
|
|
|
7
7
|
## The parts
|
|
8
8
|
|
|
9
|
-
| Import path
|
|
10
|
-
|
|
|
11
|
-
| `@zukhruf/evals`
|
|
12
|
-
| `@zukhruf/evals/promptfoo`
|
|
13
|
-
| `@zukhruf/evals/structured`
|
|
14
|
-
| `@zukhruf/evals/classification`
|
|
15
|
-
| `@zukhruf/evals/ranking` (later)
|
|
16
|
-
| `@zukhruf/evals/clustering` (later)
|
|
9
|
+
| Import path | Job |
|
|
10
|
+
| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
|
11
|
+
| `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
|
|
12
|
+
| `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
|
|
13
|
+
| `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
|
|
14
|
+
| `@zukhruf/evals/classification` | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does (ADR 0003). |
|
|
15
|
+
| `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
|
|
16
|
+
| `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
|
|
17
17
|
|
|
18
18
|
A subpath imports only the core. It never imports another subpath.
|
|
19
19
|
|
|
20
|
-
The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`.
|
|
20
|
+
The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. The families are `./structured` and `./classification`.
|
|
21
21
|
|
|
22
22
|
## Score one answer
|
|
23
23
|
|
|
@@ -41,6 +41,11 @@ new Precision(new ClassMatches('home')).score(comparison); // Ratio { part: 0, w
|
|
|
41
41
|
|
|
42
42
|
A `Ratio` is a part and a whole. Its value is the part divided by the whole, and 0 when the whole is 0. `Precision`, `Recall`, `F1` and `FBeta` read the counts of a `Matches`: the hits, the stated and the expected. Each family gives its own matches, and these scorers take any of them.
|
|
43
43
|
|
|
44
|
+
The ratio of `new FBeta(matches, { beta, form })` has the part (1 + β²) × hits and the whole β² × expected + stated, so many of them add up to the micro F-measure. `F1` is `FBeta` with β = 1. The form gives the value (ADR 0003):
|
|
45
|
+
|
|
46
|
+
- `new CountsForm()` is the default. It is scikit-learn's arithmetic, and its value is the part divided by the whole: 0 when the whole is 0. So 1 hit of 5 stated and 1 expected gives 1/3.
|
|
47
|
+
- `new PrecisionRecallForm()` is (1 + β²)·P·R / (β²·P + R): 0 when there are no hits. SOB's evaluate.py uses it. The same counts give 0.33333333333333337.
|
|
48
|
+
|
|
44
49
|
## Combine scorers
|
|
45
50
|
|
|
46
51
|
A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The factor 1 is no penalty. A penalty is also a scorer: its ratio is `Ratio(factor, 1)`. A gate is the penalty whose factor is 0 or 1. A gate says what it checks (`holds`) and why it fails (`explain`).
|
|
@@ -53,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
|
|
|
53
58
|
|
|
54
59
|
## Score with many scorers at once
|
|
55
60
|
|
|
56
|
-
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. The family
|
|
61
|
+
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
|
|
57
62
|
|
|
58
|
-
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core
|
|
63
|
+
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
|
|
59
64
|
|
|
60
65
|
```ts
|
|
61
66
|
import { Recall, ScorerSet } from '@zukhruf/evals';
|
|
@@ -81,6 +86,10 @@ const pageKinds = new ScorerSet(
|
|
|
81
86
|
- A plain scorer gives its counts, such as Ratio(2, 3). `Totals` then gives the micro average.
|
|
82
87
|
- A `SampleAveraged` scorer gives its value over 1. `Totals` then gives the samples average: the mean of the values of the answers.
|
|
83
88
|
|
|
89
|
+
`Totals` adds with CPython's compensated `sum()`. A sum of integer weights is exact, so its means and accuracies equal scikit-learn's for integer weights. A sum of float weights can differ from numpy's pairwise sum in the last bit. For float weights, use `ClassCounts` and `BrierScore.mean` of `./classification`.
|
|
90
|
+
|
|
91
|
+
`RatioOfSums.of(ratios)` adds a list of ratios in one call.
|
|
92
|
+
|
|
84
93
|
`Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
|
|
85
94
|
|
|
86
95
|
## Compare structured values
|
|
@@ -91,67 +100,92 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
|
|
|
91
100
|
|
|
92
101
|
## Score labels
|
|
93
102
|
|
|
94
|
-
|
|
103
|
+
`./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
|
|
95
104
|
|
|
96
|
-
|
|
105
|
+
It needs one optional peer: `npm install @zukhruf/evals @stdlib/blas-ext-base-gsumpw`. That package gives numpy's pairwise sum.
|
|
97
106
|
|
|
98
107
|
### For each answer
|
|
99
108
|
|
|
100
|
-
|
|
109
|
+
A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
|
|
110
|
+
|
|
111
|
+
`LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`. It reads a label or a list of labels as they are. It reads text in this order:
|
|
112
|
+
|
|
113
|
+
1. A JSON array of labels gives those labels.
|
|
114
|
+
2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
|
|
115
|
+
3. Other text is one string label, without its white space at the start and the end.
|
|
116
|
+
|
|
117
|
+
`comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
|
|
118
|
+
|
|
119
|
+
The matches of a comparison count one answer:
|
|
101
120
|
|
|
102
121
|
| Matches | Hits | Stated | Expected |
|
|
103
122
|
| ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
|
|
104
123
|
| `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
|
|
105
124
|
| `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
|
|
106
125
|
|
|
107
|
-
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `
|
|
126
|
+
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`. Without `labels`, the classes are the labels of both sides of the answer, in the order of numpy's `unique()` (numbers by value, strings by code point).
|
|
127
|
+
|
|
128
|
+
`Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
|
|
108
129
|
|
|
109
|
-
|
|
130
|
+
- `ClassMatches` gives the precision, recall and F1 of each class (`average=None`).
|
|
131
|
+
- `AllClassMatches` gives the micro average.
|
|
132
|
+
- `LabelsMatch` gives `accuracy_score` (for multilabel answers, the subset accuracy).
|
|
133
|
+
|
|
134
|
+
So a promptfoo run gives the micro precision, recall and F1, and the score of each class, in its column headers.
|
|
110
135
|
|
|
111
136
|
### For a whole run
|
|
112
137
|
|
|
113
|
-
A macro or weighted average is not a ratio of sums: it
|
|
138
|
+
A macro average or a weighted average is not a ratio of sums: it needs the score of each class first. `ClassCounts` keeps the counts of each class: the true positives (hits), the predicted (stated) and the true (expected). It adds the weight of each answer in the order of the answers, as numpy's `bincount` does. torchmetrics does the same with `update()` and `compute()`.
|
|
114
139
|
|
|
115
140
|
```ts
|
|
116
141
|
import {
|
|
117
142
|
ClassCounts,
|
|
118
|
-
|
|
143
|
+
LabelCase,
|
|
144
|
+
LabelComparer,
|
|
119
145
|
MacroAverage,
|
|
120
146
|
WeightedAverage,
|
|
121
147
|
} from '@zukhruf/evals/classification';
|
|
122
148
|
|
|
123
|
-
const counts =
|
|
124
|
-
|
|
149
|
+
const counts = ClassCounts.of({
|
|
150
|
+
expected: ['a', 'b', 'b'],
|
|
151
|
+
stated: ['a', 'a', 'b'],
|
|
152
|
+
});
|
|
125
153
|
|
|
126
154
|
counts.accuracy(); // accuracy_score
|
|
127
155
|
counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
|
|
128
156
|
counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
|
|
129
|
-
counts.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
});
|
|
134
|
-
|
|
157
|
+
counts.confusionMatrix().rows(); // confusion_matrix, labels in sorted order
|
|
158
|
+
|
|
159
|
+
// The same counts, one answer at a time:
|
|
160
|
+
const run = new ClassCounts({ multilabel: false });
|
|
161
|
+
const testCase = new LabelCase({ expected: 'a', weight: 1 });
|
|
162
|
+
run.add(new LabelComparer().compare('a', testCase), testCase.weight);
|
|
135
163
|
```
|
|
136
164
|
|
|
137
|
-
- `
|
|
138
|
-
- `
|
|
139
|
-
- `
|
|
140
|
-
- `
|
|
165
|
+
- `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
|
|
166
|
+
- `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
|
|
167
|
+
- `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
168
|
+
- `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` throws when scikit-learn raises: for multilabel counts, for more than two classes, and for a positive label that is not one of two classes. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
|
|
169
|
+
- `labels` selects the classes to score, in its order. By default it is every class, in sorted order.
|
|
170
|
+
- `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
|
|
171
|
+
- `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
|
|
172
|
+
- `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
|
|
141
173
|
|
|
142
174
|
### Probabilities
|
|
143
175
|
|
|
144
|
-
`
|
|
176
|
+
`ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
|
|
177
|
+
|
|
178
|
+
`BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. `form` is the arithmetic of the score of p (ADR 0003). `new TwoColumnForm()` is scikit-learn's and the default: 0.5 × the sum of the squares of the two classes. `new SquaredErrorForm()` is (p − y)², as numpy gives `np.mean((p - y) ** 2)`. The two forms can differ in the last bit. A form needs `positiveLabel`. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
|
|
145
179
|
|
|
146
180
|
## Use it with promptfoo
|
|
147
181
|
|
|
148
|
-
|
|
182
|
+
`ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels. [docs/structured.md](docs/structured.md) has a structured sample.
|
|
149
183
|
|
|
150
184
|
`PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
|
|
151
185
|
|
|
152
186
|
`ScorerAssertion` holds one scorer with its `weight` (how much it counts in the score of the test) and its `threshold` (when it passes). Both numbers are required. It reads the case from the vars of the test.
|
|
153
187
|
|
|
154
|
-
promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured
|
|
188
|
+
promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured ADR 0013 in docs/adr/structured).
|
|
155
189
|
|
|
156
190
|
```ts
|
|
157
191
|
// page-kinds/scorers.ts: the module that promptfoo loads
|
|
@@ -313,6 +347,7 @@ export interface Comparer<Case extends Weighted, Input> {
|
|
|
313
347
|
}
|
|
314
348
|
|
|
315
349
|
export type Check = {
|
|
350
|
+
readonly name: string;
|
|
316
351
|
readonly passed: boolean;
|
|
317
352
|
readonly score: number;
|
|
318
353
|
readonly reason: string;
|
|
@@ -421,7 +456,7 @@ export type ModuleAssertion = {
|
|
|
421
456
|
readonly threshold: number;
|
|
422
457
|
readonly value: (output: string, context: VarsContext) => number;
|
|
423
458
|
};
|
|
424
|
-
/** The module that promptfoo loads the function of each scorer assertion from (structured
|
|
459
|
+
/** The module that promptfoo loads the function of each scorer assertion from (structured ADR 0013 in docs/adr/structured). */
|
|
425
460
|
export class AssertionModule<Name extends string> {
|
|
426
461
|
constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
|
|
427
462
|
/** The default export of the module: the value function of each name. */
|
|
@@ -429,30 +464,36 @@ export class AssertionModule<Name extends string> {
|
|
|
429
464
|
assert(...names: Name[]): FileAssertion[];
|
|
430
465
|
}
|
|
431
466
|
|
|
432
|
-
// @zukhruf/evals/classification
|
|
467
|
+
// @zukhruf/evals/classification
|
|
433
468
|
|
|
434
|
-
/** A label:
|
|
469
|
+
/** A label: a number or a string, as scikit-learn reads a label. */
|
|
435
470
|
export type Label = number | string;
|
|
436
471
|
|
|
437
472
|
/** The expected labels of one test, and its weight. */
|
|
438
473
|
export class LabelCase implements Weighted {
|
|
439
|
-
constructor(expected: Label | readonly Label[]
|
|
474
|
+
constructor(options: { expected: Label | readonly Label[]; weight: number });
|
|
440
475
|
readonly expected: ReadonlySet<Label>;
|
|
441
476
|
readonly weight: number;
|
|
442
477
|
}
|
|
443
|
-
/** Reads `vars.expected` (a label or a list of labels) and `vars.weight
|
|
478
|
+
/** Reads `vars.expected` (a label or a list of labels) and `vars.weight` (1 when none). */
|
|
444
479
|
export class LabelVarsCase implements CaseReader<LabelCase> {}
|
|
445
|
-
/** Reads `config.expected
|
|
480
|
+
/** Reads `config.expected`, with the weight 1. */
|
|
446
481
|
export class LabelConfigCase implements CaseReader<LabelCase> {}
|
|
447
482
|
|
|
448
483
|
export class LabelComparison {
|
|
484
|
+
constructor(sides: {
|
|
485
|
+
stated: ReadonlySet<Label>;
|
|
486
|
+
expected: ReadonlySet<Label>;
|
|
487
|
+
});
|
|
449
488
|
readonly stated: ReadonlySet<Label>;
|
|
450
489
|
readonly expected: ReadonlySet<Label>;
|
|
490
|
+
/** The labels of both sides, in the order of numpy's unique(). */
|
|
491
|
+
labels(): Label[];
|
|
492
|
+
/** The comparison of one class. */
|
|
493
|
+
only(label: Label): LabelComparison;
|
|
451
494
|
}
|
|
452
|
-
/** Reads the stated labels
|
|
453
|
-
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
454
|
-
constructor(options?: { normalize?: (label: string) => string });
|
|
455
|
-
}
|
|
495
|
+
/** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
|
|
496
|
+
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {}
|
|
456
497
|
|
|
457
498
|
export class ClassMatches implements Matches<LabelComparison> {
|
|
458
499
|
constructor(label: Label);
|
|
@@ -463,12 +504,20 @@ export class LabelsMatch extends Gate<LabelComparison> {}
|
|
|
463
504
|
export class PerClass extends Grouped<LabelComparison> {
|
|
464
505
|
constructor(
|
|
465
506
|
scorer: Scorer<LabelComparison>,
|
|
466
|
-
options
|
|
507
|
+
options?: { labels?: readonly Label[] },
|
|
467
508
|
);
|
|
468
509
|
}
|
|
469
510
|
|
|
511
|
+
/** The classes that an average reads. */
|
|
512
|
+
export type Classes = {
|
|
513
|
+
readonly selected: ReadonlyMap<Label, Counts>;
|
|
514
|
+
readonly present: ReadonlyMap<Label, Counts>;
|
|
515
|
+
readonly multilabel: boolean;
|
|
516
|
+
};
|
|
470
517
|
/** How the scores of the classes become one number (Strategy). */
|
|
471
|
-
export interface Average {
|
|
518
|
+
export interface Average {
|
|
519
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
520
|
+
}
|
|
472
521
|
export class MicroAverage implements Average {}
|
|
473
522
|
export class MacroAverage implements Average {}
|
|
474
523
|
export class WeightedAverage implements Average {}
|
|
@@ -476,33 +525,33 @@ export class BinaryAverage implements Average {
|
|
|
476
525
|
constructor(positiveLabel: Label);
|
|
477
526
|
}
|
|
478
527
|
|
|
528
|
+
type ScoreOptions = {
|
|
529
|
+
average: Average;
|
|
530
|
+
labels?: readonly Label[];
|
|
531
|
+
/** 0 (the default), 1 or NaN. */
|
|
532
|
+
zeroDivision?: number;
|
|
533
|
+
};
|
|
479
534
|
/** The counts of each class over many answers: true positives, predicted and true. */
|
|
480
535
|
export class ClassCounts {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
expected: readonly
|
|
484
|
-
|
|
485
|
-
|
|
536
|
+
constructor(options: { multilabel: boolean });
|
|
537
|
+
static of(answers: {
|
|
538
|
+
expected: readonly Label[];
|
|
539
|
+
stated: readonly Label[];
|
|
540
|
+
weights?: readonly number[];
|
|
541
|
+
}): ClassCounts;
|
|
542
|
+
static ofIndicators(answers: {
|
|
543
|
+
expected: readonly (readonly number[])[];
|
|
544
|
+
stated: readonly (readonly number[])[];
|
|
545
|
+
weights?: readonly number[];
|
|
546
|
+
}): ClassCounts;
|
|
547
|
+
add(comparison: LabelComparison, weight: number): void;
|
|
486
548
|
labels(): Label[];
|
|
487
549
|
accuracy(): number;
|
|
488
|
-
precision(options:
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
labels?: readonly Label[];
|
|
492
|
-
}): number;
|
|
493
|
-
recall(options: {
|
|
494
|
-
average: Average;
|
|
495
|
-
zeroDivision?: number;
|
|
496
|
-
labels?: readonly Label[];
|
|
497
|
-
}): number;
|
|
498
|
-
fBeta(options: {
|
|
499
|
-
beta: number;
|
|
500
|
-
average: Average;
|
|
501
|
-
zeroDivision?: number;
|
|
502
|
-
labels?: readonly Label[];
|
|
503
|
-
form?: FMeasureForm;
|
|
504
|
-
}): number;
|
|
550
|
+
precision(options: ScoreOptions): number;
|
|
551
|
+
recall(options: ScoreOptions): number;
|
|
552
|
+
fBeta(options: ScoreOptions & { beta: number; form?: FMeasureForm }): number;
|
|
505
553
|
support(): ReadonlyMap<Label, number>;
|
|
554
|
+
/** Throws for multilabel counts. */
|
|
506
555
|
confusionMatrix(): ConfusionMatrix;
|
|
507
556
|
}
|
|
508
557
|
export class ConfusionMatrix {
|
|
@@ -511,12 +560,34 @@ export class ConfusionMatrix {
|
|
|
511
560
|
rows(): number[][];
|
|
512
561
|
}
|
|
513
562
|
|
|
514
|
-
/** The stated probability of each class of one answer. */
|
|
563
|
+
/** The stated probability of each class of one answer, and its expected label. */
|
|
515
564
|
export class ProbabilityComparison {
|
|
516
565
|
readonly probabilities: ReadonlyMap<Label, number>;
|
|
517
|
-
readonly expected:
|
|
566
|
+
readonly expected: Label;
|
|
567
|
+
}
|
|
568
|
+
/** Reads the probability of each label from a Map, an object or its JSON text. */
|
|
569
|
+
export class ProbabilityComparer implements Comparer<
|
|
570
|
+
LabelCase,
|
|
571
|
+
ProbabilityComparison
|
|
572
|
+
> {}
|
|
573
|
+
/** The arithmetic of the Brier score of the probability p of the positive label (ADR 0003). */
|
|
574
|
+
export interface BrierForm {
|
|
575
|
+
of(probability: number, expected: 0 | 1): number;
|
|
576
|
+
}
|
|
577
|
+
export class TwoColumnForm implements BrierForm {}
|
|
578
|
+
export class SquaredErrorForm implements BrierForm {}
|
|
579
|
+
export class BrierScore implements Scorer<ProbabilityComparison> {
|
|
580
|
+
constructor(
|
|
581
|
+
options?:
|
|
582
|
+
| { positiveLabel?: undefined; form?: undefined }
|
|
583
|
+
| { positiveLabel: Label; form?: BrierForm },
|
|
584
|
+
);
|
|
585
|
+
/** The score of a whole run, with numpy's pairwise sum. */
|
|
586
|
+
mean(
|
|
587
|
+
comparisons: readonly ProbabilityComparison[],
|
|
588
|
+
weights?: readonly number[],
|
|
589
|
+
): number;
|
|
518
590
|
}
|
|
519
|
-
export class BrierScore implements Scorer<ProbabilityComparison> {}
|
|
520
591
|
```
|
|
521
592
|
|
|
522
593
|
## Who uses each part
|
|
@@ -529,10 +600,10 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
|
|
|
529
600
|
| context-compiler, startup-credits offers | core, `./structured` (`ByBestMatch`), `./promptfoo` | context-compiler #3893, #3882 |
|
|
530
601
|
| context-compiler, startup-credits page kinds | `./classification` (accuracy, the precision and recall of each class, the confusion matrix) | #3882 slice 2 |
|
|
531
602
|
| `@zukhruf/mteb`, classification tasks | `./classification` (`ClassCounts`: accuracy, macro and weighted P, R and F1; label-indicator rows) | mutex #3890 |
|
|
532
|
-
| relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`)
|
|
603
|
+
| relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore` with `SquaredErrorForm`) | mutex #3891 |
|
|
533
604
|
| event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
|
|
534
605
|
|
|
535
|
-
The
|
|
606
|
+
The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
|
|
536
607
|
|
|
537
608
|
## Documentation
|
|
538
609
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Label } from './label.ts';
|
|
3
|
+
export type Classes = {
|
|
4
|
+
readonly selected: ReadonlyMap<Label, Counts>;
|
|
5
|
+
readonly present: ReadonlyMap<Label, Counts>;
|
|
6
|
+
readonly multilabel: boolean;
|
|
7
|
+
};
|
|
8
|
+
export interface Average {
|
|
9
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"average.js","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Average, Classes } from './average.ts';
|
|
3
|
+
import type { Label } from './label.ts';
|
|
4
|
+
export declare class BinaryAverage implements Average {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(positiveLabel: Label);
|
|
7
|
+
of({ present, multilabel }: Classes, score: (counts: Counts) => number): number;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=binary-average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,EAChC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAYV"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// average='binary': the score of the positive class, for answers of two
|
|
3
|
+
// classes or fewer. It throws as scikit-learn raises: for a set of classes,
|
|
4
|
+
// for more than two classes, and for a positive label that is not one of two
|
|
5
|
+
// classes. A positive label that no answer has scores counts of 0.
|
|
6
|
+
export class BinaryAverage {
|
|
7
|
+
#positiveLabel;
|
|
8
|
+
constructor(positiveLabel) {
|
|
9
|
+
this.#positiveLabel = positiveLabel;
|
|
10
|
+
}
|
|
11
|
+
of({ present, multilabel }, score) {
|
|
12
|
+
if (multilabel || present.size > 2)
|
|
13
|
+
throw new RangeError(`The target is ${multilabel ? 'multilabel-indicator' : 'multiclass'}, but the average is binary: choose another average.`);
|
|
14
|
+
const counts = present.get(this.#positiveLabel);
|
|
15
|
+
if (counts === undefined && present.size === 2)
|
|
16
|
+
throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not one of the labels ${JSON.stringify([...present.keys()])}.`);
|
|
17
|
+
return score(counts ?? new Counts({ hits: 0, stated: 0, expected: 0 }));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=binary-average.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,wEAAwE;AACxE,4EAA4E;AAC5E,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAW,EAChC,KAAiC;QAEjC,IAAI,UAAU,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC;YAChC,MAAM,IAAI,UAAU,CAClB,iBAAiB,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,sDAAsD,CAC1H,CAAC;QACJ,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAC5C,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC7H,CAAC;QACJ,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface BrierForm {
|
|
2
|
+
of(probability: number, expected: 0 | 1): number;
|
|
3
|
+
}
|
|
4
|
+
export declare class TwoColumnForm implements BrierForm {
|
|
5
|
+
of(probability: number, expected: 0 | 1): number;
|
|
6
|
+
}
|
|
7
|
+
export declare class SquaredErrorForm implements BrierForm {
|
|
8
|
+
of(probability: number, expected: 0 | 1): number;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=brier-form.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-form.d.ts","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,SAAS;IACxB,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CAClD;AAKD,qBAAa,aAAc,YAAW,SAAS;IAC7C,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAKjD;AAID,qBAAa,gBAAiB,YAAW,SAAS;IAChD,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAGjD"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// scikit-learn 1.9.1's brier_score_loss of a one-dimensional y_proba: the
|
|
2
|
+
// two classes are 1 - p and p, and the score is 0.5 times the sum of the
|
|
3
|
+
// squares of the two columns.
|
|
4
|
+
export class TwoColumnForm {
|
|
5
|
+
of(probability, expected) {
|
|
6
|
+
const negative = 1 - expected - (1 - probability);
|
|
7
|
+
const positive = expected - probability;
|
|
8
|
+
return 0.5 * (negative * negative + positive * positive);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
// (p - y)², the squared error of the probability (Brier 1950 for two
|
|
12
|
+
// classes), as numpy gives (p - y) ** 2.
|
|
13
|
+
export class SquaredErrorForm {
|
|
14
|
+
of(probability, expected) {
|
|
15
|
+
return (probability - expected) ** 2;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=brier-form.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-form.js","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAQA,0EAA0E;AAC1E,yEAAyE;AACzE,8BAA8B;AAC9B,MAAM,OAAO,aAAa;IACxB,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;QACxC,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;IAC3D,CAAC;CACF;AAED,qEAAqE;AACrE,yCAAyC;AACzC,MAAM,OAAO,gBAAgB;IAC3B,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorer } from '../scorers/scorer.ts';
|
|
3
|
+
import { type BrierForm } from './brier-form.ts';
|
|
4
|
+
import { type Label } from './label.ts';
|
|
5
|
+
import type { ProbabilityComparison } from './probability-comparison.ts';
|
|
6
|
+
export declare class BrierScore implements Scorer<ProbabilityComparison> {
|
|
7
|
+
#private;
|
|
8
|
+
constructor({ positiveLabel, form, }?: {
|
|
9
|
+
readonly positiveLabel?: undefined;
|
|
10
|
+
readonly form?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
readonly positiveLabel: Label;
|
|
13
|
+
readonly form?: BrierForm;
|
|
14
|
+
});
|
|
15
|
+
score(comparison: ProbabilityComparison): Ratio;
|
|
16
|
+
mean(comparisons: readonly ProbabilityComparison[], weights?: readonly number[]): number;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=brier-score.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAUzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAIlD,EACV,aAAa,EACb,IAA0B,GAC3B,GACG;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAE,GACjE;QAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAO;IAKrE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAsCV"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
import { TwoColumnForm } from "./brier-form.js";
|
|
4
|
+
import { compareLabels } from "./label.js";
|
|
5
|
+
// scikit-learn 1.9.1's brier_score_loss. For one answer, the sum over the
|
|
6
|
+
// classes of (1 when the class is the expected label, else 0, minus the
|
|
7
|
+
// stated probability) squared, with numpy's pairwise sum over the classes in
|
|
8
|
+
// their sorted order; times 0.5 for two classes. With a positive label, the
|
|
9
|
+
// answer gives the probability p of that label only, and the two classes are
|
|
10
|
+
// 1 - p and p, as scikit-learn reads a one-dimensional y_proba; the form
|
|
11
|
+
// (TwoColumnForm by default) gives the score of p. Its ratio is the score
|
|
12
|
+
// over 1, and mean gives the score of a whole run.
|
|
13
|
+
export class BrierScore {
|
|
14
|
+
#positiveLabel;
|
|
15
|
+
#form;
|
|
16
|
+
constructor({ positiveLabel, form = new TwoColumnForm(), } = {}) {
|
|
17
|
+
this.#positiveLabel = positiveLabel;
|
|
18
|
+
this.#form = form;
|
|
19
|
+
}
|
|
20
|
+
score(comparison) {
|
|
21
|
+
return new Ratio(this.#value(comparison), 1);
|
|
22
|
+
}
|
|
23
|
+
// The mean of the scores of the answers, with numpy's pairwise sum, each
|
|
24
|
+
// answer with its weight (1 when none is given).
|
|
25
|
+
mean(comparisons, weights) {
|
|
26
|
+
const scores = comparisons.map((comparison) => this.#value(comparison));
|
|
27
|
+
if (weights === undefined)
|
|
28
|
+
return gsumpw(scores.length, scores, 1) / scores.length;
|
|
29
|
+
if (weights.length !== scores.length)
|
|
30
|
+
throw new RangeError(`The answers and the weights have ${scores.length} and ${weights.length} values.`);
|
|
31
|
+
const weighted = scores.map((score, index) => score * (weights[index] ?? 0));
|
|
32
|
+
return (gsumpw(weighted.length, weighted, 1) / gsumpw(weights.length, weights, 1));
|
|
33
|
+
}
|
|
34
|
+
#value({ probabilities, expected }) {
|
|
35
|
+
if (this.#positiveLabel !== undefined) {
|
|
36
|
+
const p = probabilities.get(this.#positiveLabel);
|
|
37
|
+
if (p === undefined)
|
|
38
|
+
throw new RangeError(`The answer gives no probability of the positive label ${JSON.stringify(this.#positiveLabel)}.`);
|
|
39
|
+
return this.#form.of(p, expected === this.#positiveLabel ? 1 : 0);
|
|
40
|
+
}
|
|
41
|
+
if (!probabilities.has(expected))
|
|
42
|
+
throw new RangeError(`The answer gives no probability of the expected label ${JSON.stringify(expected)}.`);
|
|
43
|
+
const squares = [...probabilities]
|
|
44
|
+
.toSorted(([left], [right]) => compareLabels(left, right))
|
|
45
|
+
.map(([label, probability]) => {
|
|
46
|
+
const difference = (label === expected ? 1 : 0) - probability;
|
|
47
|
+
return difference * difference;
|
|
48
|
+
});
|
|
49
|
+
const sum = gsumpw(squares.length, squares, 1);
|
|
50
|
+
return squares.length < 3 ? 0.5 * sum : sum;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=brier-score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAkB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,yEAAyE;AACzE,0EAA0E;AAC1E,mDAAmD;AACnD,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAClC,KAAK,CAAY;IAE1B,YAAY,EACV,aAAa,EACb,IAAI,GAAG,IAAI,aAAa,EAAE,MAGuC,EAAE;QACnE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YACvB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;CACF"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
2
|
+
import type { Average } from './average.ts';
|
|
3
|
+
import { ConfusionMatrix } from './confusion-matrix.ts';
|
|
4
|
+
import { LabelComparison } from './label-comparison.ts';
|
|
5
|
+
import { type Label } from './label.ts';
|
|
6
|
+
type ScoreOptions = {
|
|
7
|
+
readonly average: Average;
|
|
8
|
+
readonly labels?: readonly Label[];
|
|
9
|
+
readonly zeroDivision?: 0 | 1 | number;
|
|
10
|
+
};
|
|
11
|
+
export declare class ClassCounts {
|
|
12
|
+
#private;
|
|
13
|
+
constructor({ multilabel }: {
|
|
14
|
+
readonly multilabel: boolean;
|
|
15
|
+
});
|
|
16
|
+
static of({ expected, stated, weights, }: {
|
|
17
|
+
readonly expected: readonly Label[];
|
|
18
|
+
readonly stated: readonly Label[];
|
|
19
|
+
readonly weights?: readonly number[];
|
|
20
|
+
}): ClassCounts;
|
|
21
|
+
static ofIndicators({ expected, stated, weights, }: {
|
|
22
|
+
readonly expected: readonly (readonly number[])[];
|
|
23
|
+
readonly stated: readonly (readonly number[])[];
|
|
24
|
+
readonly weights?: readonly number[];
|
|
25
|
+
}): ClassCounts;
|
|
26
|
+
add(comparison: LabelComparison, weight: number): void;
|
|
27
|
+
labels(): Label[];
|
|
28
|
+
accuracy(): number;
|
|
29
|
+
precision(options: ScoreOptions): number;
|
|
30
|
+
recall(options: ScoreOptions): number;
|
|
31
|
+
fBeta({ beta, form, ...options }: ScoreOptions & {
|
|
32
|
+
readonly beta: number;
|
|
33
|
+
readonly form?: FMeasureForm;
|
|
34
|
+
}): number;
|
|
35
|
+
support(): ReadonlyMap<Label, number>;
|
|
36
|
+
confusionMatrix(): ConfusionMatrix;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
39
|
+
//# sourceMappingURL=class-counts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAQlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
import { Counts } from "../aggregation/counts.js";
|
|
3
|
+
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
|
+
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
5
|
+
import { LabelComparison } from "./label-comparison.js";
|
|
6
|
+
import { compareLabels } from "./label.js";
|
|
7
|
+
// The running counts of one class, added in the order of the answers.
|
|
8
|
+
class Tally {
|
|
9
|
+
hits = 0;
|
|
10
|
+
stated = 0;
|
|
11
|
+
expected = 0;
|
|
12
|
+
}
|
|
13
|
+
// The counts of each class over many answers, as scikit-learn's
|
|
14
|
+
// multilabel_confusion_matrix gives them: the true positives (hits), the
|
|
15
|
+
// predicted (stated) and the true (expected), each the sum of the weights of
|
|
16
|
+
// the answers in their order, as numpy's bincount adds them. Add each
|
|
17
|
+
// comparison, then read the metrics of a whole run (torchmetrics' update and
|
|
18
|
+
// compute). It also keeps the weight of each right answer for the accuracy,
|
|
19
|
+
// and the weight of each pair of labels for the confusion matrix. The counts
|
|
20
|
+
// are of one class for each answer (multiclass) or of a set of classes
|
|
21
|
+
// (multilabel), as torchmetrics has a multiclass and a multilabel class.
|
|
22
|
+
export class ClassCounts {
|
|
23
|
+
#multilabel;
|
|
24
|
+
#classes = new Map();
|
|
25
|
+
#rights = [];
|
|
26
|
+
#weights = [];
|
|
27
|
+
#pairs = new Map();
|
|
28
|
+
constructor({ multilabel }) {
|
|
29
|
+
this.#multilabel = multilabel;
|
|
30
|
+
}
|
|
31
|
+
// The counts of two arrays of labels, one label for each answer
|
|
32
|
+
// (multiclass), with the weight of each answer (1 when none is given).
|
|
33
|
+
static of({ expected, stated, weights, }) {
|
|
34
|
+
const counts = new ClassCounts({ multilabel: false });
|
|
35
|
+
for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
36
|
+
counts.add(new LabelComparison({
|
|
37
|
+
stated: new Set([statedLabel]),
|
|
38
|
+
expected: new Set([expectedLabel]),
|
|
39
|
+
}), weight);
|
|
40
|
+
return counts;
|
|
41
|
+
}
|
|
42
|
+
// The counts of two label indicator matrices (multilabel): a row of 0 and 1
|
|
43
|
+
// for each answer, a column for each label. Each column is a class, also
|
|
44
|
+
// when no answer has it.
|
|
45
|
+
static ofIndicators({ expected, stated, weights, }) {
|
|
46
|
+
const columns = expected[0]?.length ?? 0;
|
|
47
|
+
const counts = new ClassCounts({ multilabel: true });
|
|
48
|
+
for (let column = 0; column < columns; column++)
|
|
49
|
+
counts.#classes.set(column, new Tally());
|
|
50
|
+
const labels = (row) => {
|
|
51
|
+
if (row.length !== columns ||
|
|
52
|
+
row.some((value) => value !== 0 && value !== 1))
|
|
53
|
+
throw new RangeError(`A row of a label indicator matrix holds ${columns} values of 0 or 1, not ${JSON.stringify(row)}.`);
|
|
54
|
+
return new Set(row.flatMap((value, column) => (value === 1 ? [column] : [])));
|
|
55
|
+
};
|
|
56
|
+
for (const [expectedRow, statedRow, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
57
|
+
counts.add(new LabelComparison({
|
|
58
|
+
stated: labels(statedRow),
|
|
59
|
+
expected: labels(expectedRow),
|
|
60
|
+
}), weight);
|
|
61
|
+
return counts;
|
|
62
|
+
}
|
|
63
|
+
// The expected side, the stated side and the weight of each answer. An
|
|
64
|
+
// answer without a weight weighs 1.
|
|
65
|
+
static *#answers(expected, stated, weights) {
|
|
66
|
+
const unequal = new RangeError(`The expected labels, the stated labels and the weights have ${expected.length}, ${stated.length} and ${weights?.length ?? expected.length} answers.`);
|
|
67
|
+
if (stated.length !== expected.length ||
|
|
68
|
+
(weights !== undefined && weights.length !== expected.length))
|
|
69
|
+
throw unequal;
|
|
70
|
+
for (const [index, side] of expected.entries()) {
|
|
71
|
+
const other = stated[index];
|
|
72
|
+
if (other === undefined)
|
|
73
|
+
throw unequal;
|
|
74
|
+
yield [side, other, weights?.[index] ?? 1];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// Adds one answer with its weight. Multiclass counts refuse an answer with
|
|
78
|
+
// no label or more than one label on a side.
|
|
79
|
+
add(comparison, weight) {
|
|
80
|
+
const { stated, expected } = comparison;
|
|
81
|
+
if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
|
|
82
|
+
throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
|
|
83
|
+
for (const label of comparison.labels()) {
|
|
84
|
+
const tally = this.#classes.getOrInsertComputed(label, () => new Tally());
|
|
85
|
+
if (stated.has(label))
|
|
86
|
+
tally.stated += weight;
|
|
87
|
+
if (expected.has(label))
|
|
88
|
+
tally.expected += weight;
|
|
89
|
+
if (stated.has(label) && expected.has(label))
|
|
90
|
+
tally.hits += weight;
|
|
91
|
+
}
|
|
92
|
+
const right = stated.size === expected.size && stated.isSubsetOf(expected);
|
|
93
|
+
this.#rights.push(right ? weight : 0);
|
|
94
|
+
this.#weights.push(weight);
|
|
95
|
+
if (this.#multilabel)
|
|
96
|
+
return;
|
|
97
|
+
const [expectedLabel] = expected;
|
|
98
|
+
const [statedLabel] = stated;
|
|
99
|
+
if (expectedLabel === undefined || statedLabel === undefined)
|
|
100
|
+
return;
|
|
101
|
+
const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
|
|
102
|
+
row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
|
|
103
|
+
}
|
|
104
|
+
// The labels of the classes, in the order of numpy's unique().
|
|
105
|
+
labels() {
|
|
106
|
+
return [...this.#classes.keys()].toSorted(compareLabels);
|
|
107
|
+
}
|
|
108
|
+
// accuracy_score: the weight of the right answers over the weight of all
|
|
109
|
+
// answers, each with numpy's pairwise sum. An answer is right when its
|
|
110
|
+
// stated labels are its expected labels (the subset accuracy for a set of
|
|
111
|
+
// classes).
|
|
112
|
+
accuracy() {
|
|
113
|
+
return (gsumpw(this.#rights.length, this.#rights, 1) /
|
|
114
|
+
gsumpw(this.#weights.length, this.#weights, 1));
|
|
115
|
+
}
|
|
116
|
+
// precision_score: hits / stated of each class, then the average.
|
|
117
|
+
precision(options) {
|
|
118
|
+
return this.#score(options, ({ hits, stated }) => stated === 0 ? undefined : hits / stated);
|
|
119
|
+
}
|
|
120
|
+
// recall_score: hits / expected of each class, then the average.
|
|
121
|
+
recall(options) {
|
|
122
|
+
return this.#score(options, ({ hits, expected }) => expected === 0 ? undefined : hits / expected);
|
|
123
|
+
}
|
|
124
|
+
// fbeta_score: the F-beta of each class in the form (scikit-learn's
|
|
125
|
+
// CountsForm by default), then the average. Its denominator is
|
|
126
|
+
// β²·expected + stated.
|
|
127
|
+
fBeta({ beta, form = new CountsForm(), ...options }) {
|
|
128
|
+
return this.#score(options, (counts) => beta ** 2 * counts.expected + counts.stated === 0
|
|
129
|
+
? undefined
|
|
130
|
+
: form.of(counts, beta));
|
|
131
|
+
}
|
|
132
|
+
// The true count of each class, in the order of the labels.
|
|
133
|
+
support() {
|
|
134
|
+
return new Map(this.labels().map((label) => [label, this.#counts(label).expected]));
|
|
135
|
+
}
|
|
136
|
+
// confusion_matrix of the answers, with the labels in their order. A set
|
|
137
|
+
// of classes has no confusion matrix (scikit-learn raises), so it throws.
|
|
138
|
+
confusionMatrix() {
|
|
139
|
+
if (this.#multilabel)
|
|
140
|
+
throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
|
|
141
|
+
return new ConfusionMatrix(this.labels(), this.#pairs);
|
|
142
|
+
}
|
|
143
|
+
#score({ average, labels = this.labels(), zeroDivision = 0 }, score) {
|
|
144
|
+
return average.of({
|
|
145
|
+
selected: new Map(labels.map((label) => [label, this.#counts(label)])),
|
|
146
|
+
present: new Map(this.labels().map((label) => [label, this.#counts(label)])),
|
|
147
|
+
multilabel: this.#multilabel,
|
|
148
|
+
}, (counts) => score(counts) ?? zeroDivision);
|
|
149
|
+
}
|
|
150
|
+
#counts(label) {
|
|
151
|
+
const tally = this.#classes.get(label);
|
|
152
|
+
return new Counts({
|
|
153
|
+
hits: tally?.hits ?? 0,
|
|
154
|
+
stated: tally?.stated ?? 0,
|
|
155
|
+
expected: tally?.expected ?? 0,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=class-counts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAWvD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;CACd;AAED,gEAAgE;AAChE,yEAAyE;AACzE,6EAA6E;AAC7E,sEAAsE;AACtE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IACnC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EAAE,UAAU,EAAoC;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;aACnC,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;aAC9B,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6CAA6C;IAC7C,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,+DAA+D;IAC/D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,CACL,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAC/C,CAAC;IACJ,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAgB,EACnE,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,IAAI,GAAG,CACd,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAC3D;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YACtB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC1B,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Label } from './label.ts';
|
|
2
|
+
export declare class ConfusionMatrix {
|
|
3
|
+
#private;
|
|
4
|
+
readonly labels: readonly Label[];
|
|
5
|
+
constructor(labels: readonly Label[], pairs: ReadonlyMap<Label, ReadonlyMap<Label, number>>);
|
|
6
|
+
count(expected: Label, stated: Label): number;
|
|
7
|
+
rows(): number[][];
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=confusion-matrix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"confusion-matrix.d.ts","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,eAAe;;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;gBAIhC,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAMvD,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,MAAM;IAI7C,IAAI,IAAI,MAAM,EAAE,EAAE;CAKnB"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// The weight of each pair of an expected label and a stated label, as
|
|
2
|
+
// scikit-learn's confusion_matrix gives it: a row for each expected label and
|
|
3
|
+
// a column for each stated label, in the order of the labels.
|
|
4
|
+
export class ConfusionMatrix {
|
|
5
|
+
labels;
|
|
6
|
+
#pairs;
|
|
7
|
+
constructor(labels, pairs) {
|
|
8
|
+
this.labels = labels;
|
|
9
|
+
this.#pairs = pairs;
|
|
10
|
+
}
|
|
11
|
+
count(expected, stated) {
|
|
12
|
+
return this.#pairs.get(expected)?.get(stated) ?? 0;
|
|
13
|
+
}
|
|
14
|
+
rows() {
|
|
15
|
+
return this.labels.map((expected) => this.labels.map((stated) => this.count(expected, stated)));
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=confusion-matrix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"confusion-matrix.js","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAEA,sEAAsE;AACtE,8EAA8E;AAC9E,8DAA8D;AAC9D,MAAM,OAAO,eAAe;IACjB,MAAM,CAAmB;IACzB,MAAM,CAAiD;IAEhE,YACE,MAAwB,EACxB,KAAqD;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAe,EAAE,MAAa;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAC1D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -8,4 +8,15 @@ export { LabelVarsCase } from './label-vars-case.ts';
|
|
|
8
8
|
export { LabelsMatch } from './labels-match.ts';
|
|
9
9
|
export type { Label } from './label.ts';
|
|
10
10
|
export { PerClass } from './per-class.ts';
|
|
11
|
+
export type { Average, Classes } from './average.ts';
|
|
12
|
+
export { BinaryAverage } from './binary-average.ts';
|
|
13
|
+
export { ClassCounts } from './class-counts.ts';
|
|
14
|
+
export { ConfusionMatrix } from './confusion-matrix.ts';
|
|
15
|
+
export { MacroAverage } from './macro-average.ts';
|
|
16
|
+
export { MicroAverage } from './micro-average.ts';
|
|
17
|
+
export { WeightedAverage } from './weighted-average.ts';
|
|
18
|
+
export { BrierScore } from './brier-score.ts';
|
|
19
|
+
export { type BrierForm, SquaredErrorForm, TwoColumnForm, } from './brier-form.ts';
|
|
20
|
+
export { ProbabilityComparer } from './probability-comparer.ts';
|
|
21
|
+
export { ProbabilityComparison } from './probability-comparison.ts';
|
|
11
22
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EACL,KAAK,SAAS,EACd,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -7,4 +7,14 @@ export { LabelConfigCase } from "./label-config-case.js";
|
|
|
7
7
|
export { LabelVarsCase } from "./label-vars-case.js";
|
|
8
8
|
export { LabelsMatch } from "./labels-match.js";
|
|
9
9
|
export { PerClass } from "./per-class.js";
|
|
10
|
+
export { BinaryAverage } from "./binary-average.js";
|
|
11
|
+
export { ClassCounts } from "./class-counts.js";
|
|
12
|
+
export { ConfusionMatrix } from "./confusion-matrix.js";
|
|
13
|
+
export { MacroAverage } from "./macro-average.js";
|
|
14
|
+
export { MicroAverage } from "./micro-average.js";
|
|
15
|
+
export { WeightedAverage } from "./weighted-average.js";
|
|
16
|
+
export { BrierScore } from "./brier-score.js";
|
|
17
|
+
export { SquaredErrorForm, TwoColumnForm, } from "./brier-form.js";
|
|
18
|
+
export { ProbabilityComparer } from "./probability-comparer.js";
|
|
19
|
+
export { ProbabilityComparison } from "./probability-comparison.js";
|
|
10
20
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Average, Classes } from './average.ts';
|
|
3
|
+
export declare class MacroAverage implements Average {
|
|
4
|
+
of({ selected }: Classes, score: (counts: Counts) => number): number;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=macro-average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"macro-average.d.ts","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAOrE"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
// average='macro': the mean of the scores of the selected classes, as
|
|
3
|
+
// numpy's nanmean gives it: a NaN score (zeroDivision NaN) adds 0 and is not
|
|
4
|
+
// counted, and the mean of no score is NaN.
|
|
5
|
+
export class MacroAverage {
|
|
6
|
+
of({ selected }, score) {
|
|
7
|
+
const scores = [...selected.values()].map(score);
|
|
8
|
+
const counted = scores.filter((value) => !Number.isNaN(value)).length;
|
|
9
|
+
if (counted === 0)
|
|
10
|
+
return Number.NaN;
|
|
11
|
+
const values = scores.map((value) => (Number.isNaN(value) ? 0 : value));
|
|
12
|
+
return gsumpw(values.length, values, 1) / counted;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=macro-average.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAKlD,sEAAsE;AACtE,6EAA6E;AAC7E,4CAA4C;AAC5C,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC;IACpD,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Average, Classes } from './average.ts';
|
|
3
|
+
export declare class MicroAverage implements Average {
|
|
4
|
+
of({ selected }: Classes, score: (counts: Counts) => number): number;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=micro-average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"micro-average.d.ts","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAYrE"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
import { Counts } from "../aggregation/counts.js";
|
|
3
|
+
// average='micro': the counts of the selected classes are added, with numpy's
|
|
4
|
+
// pairwise sum, and scored once.
|
|
5
|
+
export class MicroAverage {
|
|
6
|
+
of({ selected }, score) {
|
|
7
|
+
const classes = [...selected.values()];
|
|
8
|
+
const sum = (count) => gsumpw(classes.length, classes.map(count), 1);
|
|
9
|
+
return score(new Counts({
|
|
10
|
+
hits: sum(({ hits }) => hits),
|
|
11
|
+
stated: sum(({ stated }) => stated),
|
|
12
|
+
expected: sum(({ expected }) => expected),
|
|
13
|
+
}));
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=micro-average.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,8EAA8E;AAC9E,iCAAiC;AACjC,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,CAAC,KAAiC,EAAE,EAAE,CAChD,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,OAAO,KAAK,CACV,IAAI,MAAM,CAAC;YACT,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;YAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;YACnC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Comparer } from '../scoring/comparer.ts';
|
|
2
|
+
import type { LabelCase } from './label-case.ts';
|
|
3
|
+
import { ProbabilityComparison } from './probability-comparison.ts';
|
|
4
|
+
export declare class ProbabilityComparer implements Comparer<LabelCase, ProbabilityComparison> {
|
|
5
|
+
#private;
|
|
6
|
+
compare(stated: unknown, testCase: LabelCase): ProbabilityComparison;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=probability-comparer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probability-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/probability-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAOpE,qBAAa,mBAAoB,YAAW,QAAQ,CAClD,SAAS,EACT,qBAAqB,CACtB;;IACC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,qBAAqB;CAoCrE"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ProbabilityComparison } from "./probability-comparison.js";
|
|
2
|
+
// Reads the stated probability of each class of an answer and gives its
|
|
3
|
+
// comparison with the case (Strategy). The answer is a Map of labels, a plain
|
|
4
|
+
// object or the JSON text of an object, whose values are probabilities in
|
|
5
|
+
// [0, 1]. A key of an object is a number label when each expected label of
|
|
6
|
+
// the case is a number. The case has one expected label.
|
|
7
|
+
export class ProbabilityComparer {
|
|
8
|
+
compare(stated, testCase) {
|
|
9
|
+
const [expected, ...others] = testCase.expected;
|
|
10
|
+
if (expected === undefined || others.length > 0)
|
|
11
|
+
throw new RangeError(`A probability is scored against one expected label, not ${JSON.stringify([...testCase.expected])}.`);
|
|
12
|
+
const numbers = typeof expected === 'number';
|
|
13
|
+
const entries = ProbabilityComparer.#entries(typeof stated === 'string' ? JSON.parse(stated) : stated).map(([key, probability]) => {
|
|
14
|
+
if (typeof probability !== 'number' ||
|
|
15
|
+
!(probability >= 0 && probability <= 1))
|
|
16
|
+
throw new RangeError(`The probability of ${JSON.stringify(key)} is a number in [0, 1], not ${JSON.stringify(probability)}.`);
|
|
17
|
+
return [
|
|
18
|
+
numbers && typeof key === 'string' ? Number(key) : key,
|
|
19
|
+
probability,
|
|
20
|
+
];
|
|
21
|
+
});
|
|
22
|
+
return new ProbabilityComparison({
|
|
23
|
+
probabilities: new Map(entries),
|
|
24
|
+
expected,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
static #entries(value) {
|
|
28
|
+
if (value instanceof Map)
|
|
29
|
+
return [...value];
|
|
30
|
+
if (typeof value === 'object' && value !== null && !Array.isArray(value))
|
|
31
|
+
return Object.entries(value);
|
|
32
|
+
throw new TypeError(`A stated answer is the probability of each label, as an object, not ${JSON.stringify(value)}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=probability-comparer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probability-comparer.js","sourceRoot":"","sources":["../../src/classification/probability-comparer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,wEAAwE;AACxE,8EAA8E;AAC9E,0EAA0E;AAC1E,2EAA2E;AAC3E,yDAAyD;AACzD,MAAM,OAAO,mBAAmB;IAI9B,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAChD,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7C,MAAM,IAAI,UAAU,CAClB,2DAA2D,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,CACrG,CAAC;QACJ,MAAM,OAAO,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC;QAC7C,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAC1C,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CACzD,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,EAAmB,EAAE;YAC5C,IACE,OAAO,WAAW,KAAK,QAAQ;gBAC/B,CAAC,CAAC,WAAW,IAAI,CAAC,IAAI,WAAW,IAAI,CAAC,CAAC;gBAEvC,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,GAAG,CACvG,CAAC;YACJ,OAAO;gBACL,OAAO,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;gBACtD,WAAW;aACZ,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,qBAAqB,CAAC;YAC/B,aAAa,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC;YAC/B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,KAAc;QAC5B,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACtE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,SAAS,CACjB,uEAAuE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Label } from './label.ts';
|
|
2
|
+
export declare class ProbabilityComparison {
|
|
3
|
+
readonly probabilities: ReadonlyMap<Label, number>;
|
|
4
|
+
readonly expected: Label;
|
|
5
|
+
constructor({ probabilities, expected, }: {
|
|
6
|
+
readonly probabilities: ReadonlyMap<Label, number>;
|
|
7
|
+
readonly expected: Label;
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=probability-comparison.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probability-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/probability-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAIxC,qBAAa,qBAAqB;IAChC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;gBAEb,EACV,aAAa,EACb,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnD,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;KAC1B;CAIF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// The stated probability of each class of one answer, and the expected
|
|
2
|
+
// label of its case.
|
|
3
|
+
export class ProbabilityComparison {
|
|
4
|
+
probabilities;
|
|
5
|
+
expected;
|
|
6
|
+
constructor({ probabilities, expected, }) {
|
|
7
|
+
this.probabilities = probabilities;
|
|
8
|
+
this.expected = expected;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=probability-comparison.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probability-comparison.js","sourceRoot":"","sources":["../../src/classification/probability-comparison.ts"],"names":[],"mappings":"AAEA,uEAAuE;AACvE,qBAAqB;AACrB,MAAM,OAAO,qBAAqB;IACvB,aAAa,CAA6B;IAC1C,QAAQ,CAAQ;IAEzB,YAAY,EACV,aAAa,EACb,QAAQ,GAIT;QACC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Average, Classes } from './average.ts';
|
|
3
|
+
export declare class WeightedAverage implements Average {
|
|
4
|
+
of({ selected }: Classes, score: (counts: Counts) => number): number;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=weighted-average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAUrE"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import gsumpw from '@stdlib/blas-ext-base-gsumpw';
|
|
2
|
+
// average='weighted': the mean of the scores of the selected classes, each
|
|
3
|
+
// weighted by its true count, as scikit-learn's _nanaverage gives it: a NaN
|
|
4
|
+
// score is dropped with its weight, the mean of no score is NaN, and when the
|
|
5
|
+
// weights add up to 0 it is the plain mean.
|
|
6
|
+
export class WeightedAverage {
|
|
7
|
+
of({ selected }, score) {
|
|
8
|
+
const kept = [...selected.values()]
|
|
9
|
+
.map((counts) => ({ value: score(counts), weight: counts.expected }))
|
|
10
|
+
.filter(({ value }) => !Number.isNaN(value));
|
|
11
|
+
if (kept.length === 0)
|
|
12
|
+
return Number.NaN;
|
|
13
|
+
const sum = (values) => gsumpw(values.length, values, 1);
|
|
14
|
+
const weights = sum(kept.map(({ weight }) => weight));
|
|
15
|
+
if (weights === 0)
|
|
16
|
+
return sum(kept.map(({ value }) => value)) / kept.length;
|
|
17
|
+
return sum(kept.map(({ value, weight }) => value * weight)) / weights;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=weighted-average.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAKlD,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,OAAO,eAAe;IAC1B,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;aAChC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACpE,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACzC,MAAM,GAAG,GAAG,CAAC,MAAyB,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5E,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC;IACxE,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
4
|
-
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios
|
|
3
|
+
"version": "0.8.6",
|
|
4
|
+
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
"scorers",
|
|
19
19
|
"structured-output",
|
|
20
20
|
"extraction",
|
|
21
|
+
"classification",
|
|
22
|
+
"scikit-learn",
|
|
21
23
|
"precision",
|
|
22
24
|
"recall",
|
|
23
25
|
"promptfoo"
|
|
@@ -61,24 +63,29 @@
|
|
|
61
63
|
"access": "public"
|
|
62
64
|
},
|
|
63
65
|
"dependencies": {
|
|
64
|
-
"@zukhruf/python": "^0.8.
|
|
65
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.6",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.6"
|
|
66
68
|
},
|
|
67
69
|
"peerDependencies": {
|
|
68
70
|
"@cfworker/json-schema": "4.1.4",
|
|
71
|
+
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
69
72
|
"munkres": "^2.1.1"
|
|
70
73
|
},
|
|
71
74
|
"peerDependenciesMeta": {
|
|
72
75
|
"@cfworker/json-schema": {
|
|
73
76
|
"optional": true
|
|
74
77
|
},
|
|
78
|
+
"@stdlib/blas-ext-base-gsumpw": {
|
|
79
|
+
"optional": true
|
|
80
|
+
},
|
|
75
81
|
"munkres": {
|
|
76
82
|
"optional": true
|
|
77
83
|
}
|
|
78
84
|
},
|
|
79
85
|
"devDependencies": {
|
|
80
86
|
"@cfworker/json-schema": "4.1.4",
|
|
81
|
-
"@
|
|
87
|
+
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
88
|
+
"@zukhruf/testing": "^0.8.6",
|
|
82
89
|
"munkres": "^2.1.1",
|
|
83
90
|
"promptfoo": "0.125.1",
|
|
84
91
|
"python-random": "1.0.0"
|