@zukhruf/evals 0.8.4 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/NOTICE +3 -3
  2. package/README.md +132 -71
  3. package/dist/classification/average.d.ts +11 -0
  4. package/dist/classification/average.d.ts.map +1 -0
  5. package/dist/classification/average.js +2 -0
  6. package/dist/classification/average.js.map +1 -0
  7. package/dist/classification/binary-average.d.ts +9 -0
  8. package/dist/classification/binary-average.d.ts.map +1 -0
  9. package/dist/classification/binary-average.js +20 -0
  10. package/dist/classification/binary-average.js.map +1 -0
  11. package/dist/classification/brier-score.d.ts +13 -0
  12. package/dist/classification/brier-score.d.ts.map +1 -0
  13. package/dist/classification/brier-score.js +52 -0
  14. package/dist/classification/brier-score.js.map +1 -0
  15. package/dist/classification/class-counts.d.ts +39 -0
  16. package/dist/classification/class-counts.d.ts.map +1 -0
  17. package/dist/classification/class-counts.js +159 -0
  18. package/dist/classification/class-counts.js.map +1 -0
  19. package/dist/classification/confusion-matrix.d.ts +9 -0
  20. package/dist/classification/confusion-matrix.d.ts.map +1 -0
  21. package/dist/classification/confusion-matrix.js +18 -0
  22. package/dist/classification/confusion-matrix.js.map +1 -0
  23. package/dist/classification/index.d.ts +10 -0
  24. package/dist/classification/index.d.ts.map +1 -1
  25. package/dist/classification/index.js +9 -0
  26. package/dist/classification/index.js.map +1 -1
  27. package/dist/classification/macro-average.d.ts +6 -0
  28. package/dist/classification/macro-average.d.ts.map +1 -0
  29. package/dist/classification/macro-average.js +15 -0
  30. package/dist/classification/macro-average.js.map +1 -0
  31. package/dist/classification/micro-average.d.ts +6 -0
  32. package/dist/classification/micro-average.d.ts.map +1 -0
  33. package/dist/classification/micro-average.js +16 -0
  34. package/dist/classification/micro-average.js.map +1 -0
  35. package/dist/classification/probability-comparer.d.ts +8 -0
  36. package/dist/classification/probability-comparer.d.ts.map +1 -0
  37. package/dist/classification/probability-comparer.js +35 -0
  38. package/dist/classification/probability-comparer.js.map +1 -0
  39. package/dist/classification/probability-comparison.d.ts +10 -0
  40. package/dist/classification/probability-comparison.d.ts.map +1 -0
  41. package/dist/classification/probability-comparison.js +11 -0
  42. package/dist/classification/probability-comparison.js.map +1 -0
  43. package/dist/classification/weighted-average.d.ts +6 -0
  44. package/dist/classification/weighted-average.d.ts.map +1 -0
  45. package/dist/classification/weighted-average.js +20 -0
  46. package/dist/classification/weighted-average.js.map +1 -0
  47. package/package.json +12 -5
package/NOTICE CHANGED
@@ -1,4 +1,4 @@
1
- @zukhruf/structured-eval contains a TypeScript port of the scorer of the
1
+ @zukhruf/evals contains a TypeScript port of the scorer of the
2
2
  Structured Output Benchmark (SOB): evaluate.py at commit da785a8 of
3
3
  https://github.com/interfazeai/sob. That code has this licence:
4
4
 
@@ -24,7 +24,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
24
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
25
  SOFTWARE.
26
26
 
27
- @zukhruf/structured-eval contains test cases from the JSON-Schema-Test-Suite:
27
+ @zukhruf/evals contains test cases from the JSON-Schema-Test-Suite:
28
28
  tests/draft2020-12/optional/ecmascript-regex.json and
29
29
  tests/draft7/optional/ecmascript-regex.json at commit
30
30
  245a45abc90190c0bff04d97e6d2b541b4bb9f12 of
@@ -53,7 +53,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
53
53
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
54
54
  THE SOFTWARE.
55
55
 
56
- @zukhruf/structured-eval contains the JSON Schema meta-schemas of draft 7 and
56
+ @zukhruf/evals contains the JSON Schema meta-schemas of draft 7 and
57
57
  draft 2020-12 (src/schema/draft-07-schema.ts and
58
58
  src/schema/draft-2020-12-schema.ts), copied from jsonschema-specifications
59
59
  2025.9.1 (https://github.com/python-jsonschema/jsonschema-specifications). That
package/README.md CHANGED
@@ -6,18 +6,18 @@ The answers of evals have different shapes: a structured record, a label, a rank
6
6
 
7
7
  ## The parts
8
8
 
9
- | Import path | Job |
10
- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
11
- | `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
12
- | `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
13
- | `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
14
- | `@zukhruf/evals/classification` (next) | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does. Designed in ADR 0003; built in backlog #3889. |
15
- | `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
16
- | `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
9
+ | Import path | Job |
10
+ | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
11
+ | `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
12
+ | `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
13
+ | `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
14
+ | `@zukhruf/evals/classification` | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does (ADR 0003). |
15
+ | `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
16
+ | `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
17
17
 
18
18
  A subpath imports only the core. It never imports another subpath.
19
19
 
20
- The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. Today `./structured` is the one family. `./classification` (#3889) is the next family.
20
+ The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. The families are `./structured` and `./classification`.
21
21
 
22
22
  ## Score one answer
23
23
 
@@ -41,6 +41,11 @@ new Precision(new ClassMatches('home')).score(comparison); // Ratio { part: 0, w
41
41
 
42
42
  A `Ratio` is a part and a whole. Its value is the part divided by the whole, and 0 when the whole is 0. `Precision`, `Recall`, `F1` and `FBeta` read the counts of a `Matches`: the hits, the stated and the expected. Each family gives its own matches, and these scorers take any of them.
43
43
 
44
+ The ratio of `new FBeta(matches, { beta, form })` has the part (1 + β²) × hits and the whole β² × expected + stated, so many of them add up to the micro F-measure. `F1` is `FBeta` with β = 1. The form gives the value (ADR 0003):
45
+
46
+ - `new CountsForm()` is the default. It is scikit-learn's arithmetic, and its value is the part divided by the whole: 0 when the whole is 0. So 1 hit of 5 stated and 1 expected gives 1/3.
47
+ - `new PrecisionRecallForm()` is (1 + β²)·P·R / (β²·P + R): 0 when there are no hits. SOB's evaluate.py uses it. The same counts give 0.33333333333333337.
48
+
44
49
  ## Combine scorers
45
50
 
46
51
  A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The factor 1 is no penalty. A penalty is also a scorer: its ratio is `Ratio(factor, 1)`. A gate is the penalty whose factor is 0 or 1. A gate says what it checks (`holds`) and why it fails (`explain`).
@@ -53,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
53
58
 
54
59
  ## Score with many scorers at once
55
60
 
56
- A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. The family's comparer makes the input once, and each scorer reads it.
61
+ A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
57
62
 
58
- A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core, so a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
63
+ A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
59
64
 
60
65
  ```ts
61
66
  import { Recall, ScorerSet } from '@zukhruf/evals';
@@ -81,6 +86,10 @@ const pageKinds = new ScorerSet(
81
86
  - A plain scorer gives its counts, such as Ratio(2, 3). `Totals` then gives the micro average.
82
87
  - A `SampleAveraged` scorer gives its value over 1. `Totals` then gives the samples average: the mean of the values of the answers.
83
88
 
89
+ `Totals` adds with CPython's compensated `sum()`. A sum of integer weights is exact, so its means and accuracies equal scikit-learn's for integer weights. A sum of float weights can differ from numpy's pairwise sum in the last bit. For float weights, use `ClassCounts` and `BrierScore.mean` of `./classification`.
90
+
91
+ `RatioOfSums.of(ratios)` adds a list of ratios in one call.
92
+
84
93
  `Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
85
94
 
86
95
  ## Compare structured values
@@ -91,67 +100,92 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
91
100
 
92
101
  ## Score labels
93
102
 
94
- > **Designed, not in this release.** `./classification` is built in backlog #3889 (ADR 0003). This section is its API on paper.
103
+ `./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
95
104
 
96
- `./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves a class (multiclass) and a set of classes (multilabel). The arithmetic follows scikit-learn 1.9.1, bit for bit.
105
+ It needs one optional peer: `npm install @zukhruf/evals @stdlib/blas-ext-base-gsumpw`. That package gives numpy's pairwise sum.
97
106
 
98
107
  ### For each answer
99
108
 
100
- `LabelComparer` reads the stated labels from the answer and gives a `LabelComparison`. Its matches count one answer:
109
+ A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
110
+
111
+ `LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`. It reads a label or a list of labels as they are. It reads text in this order:
112
+
113
+ 1. A JSON array of labels gives those labels.
114
+ 2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
115
+ 3. Other text is one string label, without its white space at the start and the end.
116
+
117
+ `comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
118
+
119
+ The matches of a comparison count one answer:
101
120
 
102
121
  | Matches | Hits | Stated | Expected |
103
122
  | ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
104
123
  | `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
105
124
  | `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
106
125
 
107
- `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `Totals` of its ratios gives the accuracy (for a set of classes: the subset accuracy of scikit-learn). `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`.
126
+ `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`. Without `labels`, the classes are the labels of both sides of the answer, in the order of numpy's `unique()` (numbers by value, strings by code point).
127
+
128
+ `Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
108
129
 
109
- So a promptfoo run gives the micro precision, recall and F1, and the precision and recall of each class, in its column headers.
130
+ - `ClassMatches` gives the precision, recall and F1 of each class (`average=None`).
131
+ - `AllClassMatches` gives the micro average.
132
+ - `LabelsMatch` gives `accuracy_score` (for multilabel answers, the subset accuracy).
133
+
134
+ So a promptfoo run gives the micro precision, recall and F1, and the score of each class, in its column headers.
110
135
 
111
136
  ### For a whole run
112
137
 
113
- A macro or weighted average is not a ratio of sums: it takes the value of each class first. `ClassCounts` keeps the counts of each class (true positives, predicted, true), as scikit-learn's `multilabel_confusion_matrix` gives them. Add each comparison, then read the metrics:
138
+ A macro average or a weighted average is not a ratio of sums: it needs the score of each class first. `ClassCounts` keeps the counts of each class: the true positives (hits), the predicted (stated) and the true (expected). It adds the weight of each answer in the order of the answers, as numpy's `bincount` does. torchmetrics does the same with `update()` and `compute()`.
114
139
 
115
140
  ```ts
116
141
  import {
117
142
  ClassCounts,
118
- CountsForm,
143
+ LabelCase,
144
+ LabelComparer,
119
145
  MacroAverage,
120
146
  WeightedAverage,
121
147
  } from '@zukhruf/evals/classification';
122
148
 
123
- const counts = new ClassCounts();
124
- for (const comparison of comparisons) counts.add(comparison);
149
+ const counts = ClassCounts.of({
150
+ expected: ['a', 'b', 'b'],
151
+ stated: ['a', 'a', 'b'],
152
+ });
125
153
 
126
154
  counts.accuracy(); // accuracy_score
127
155
  counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
128
156
  counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
129
- counts.fBeta({
130
- beta: 0.5,
131
- average: new MacroAverage(),
132
- form: new CountsForm(),
133
- });
134
- counts.confusionMatrix(); // confusion_matrix, labels in sorted order
157
+ counts.confusionMatrix().rows(); // confusion_matrix, labels in sorted order
158
+
159
+ // The same counts, one answer at a time:
160
+ const run = new ClassCounts({ multilabel: false });
161
+ const testCase = new LabelCase({ expected: 'a', weight: 1 });
162
+ run.add(new LabelComparer().compare('a', testCase), testCase.weight);
135
163
  ```
136
164
 
137
- - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
138
- - `zeroDivision` is the value of a class score whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`.
139
- - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default; it also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
140
- - `ClassCounts.of(stated, expected)` builds the counts from two arrays of labels, as a batch.
165
+ - `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
166
+ - `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
167
+ - `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
168
+ - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` throws when scikit-learn raises: for multilabel counts, for more than two classes, and for a positive label that is not one of two classes. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
169
+ - `labels` selects the classes to score, in its order. By default it is every class, in sorted order.
170
+ - `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
171
+ - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
172
+ - `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
141
173
 
142
174
  ### Probabilities
143
175
 
144
- `BrierScore` compares the stated probability of each class with the expected labels, as scikit-learn's `brier_score_loss` does. Its ratio is the score of one answer over 1, so `Totals` gives the mean.
176
+ `ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
177
+
178
+ `BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
145
179
 
146
180
  ## Use it with promptfoo
147
181
 
148
- > `ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels, and `./classification` is not in this release (#3889). [docs/structured.md](docs/structured.md) has a structured sample of today.
182
+ `ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels. [docs/structured.md](docs/structured.md) has a structured sample.
149
183
 
150
184
  `PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
151
185
 
152
186
  `ScorerAssertion` holds one scorer with its `weight` (how much it counts in the score of the test) and its `threshold` (when it passes). Both numbers are required. It reads the case from the vars of the test.
153
187
 
154
- promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured-eval ADR 0013).
188
+ promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured ADR 0013 in docs/adr/structured).
155
189
 
156
190
  ```ts
157
191
  // page-kinds/scorers.ts: the module that promptfoo loads
@@ -313,6 +347,7 @@ export interface Comparer<Case extends Weighted, Input> {
313
347
  }
314
348
 
315
349
  export type Check = {
350
+ readonly name: string;
316
351
  readonly passed: boolean;
317
352
  readonly score: number;
318
353
  readonly reason: string;
@@ -421,7 +456,7 @@ export type ModuleAssertion = {
421
456
  readonly threshold: number;
422
457
  readonly value: (output: string, context: VarsContext) => number;
423
458
  };
424
- /** The module that promptfoo loads the function of each scorer assertion from (structured-eval ADR 0013). */
459
+ /** The module that promptfoo loads the function of each scorer assertion from (structured ADR 0013 in docs/adr/structured). */
425
460
  export class AssertionModule<Name extends string> {
426
461
  constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
427
462
  /** The default export of the module: the value function of each name. */
@@ -429,30 +464,36 @@ export class AssertionModule<Name extends string> {
429
464
  assert(...names: Name[]): FileAssertion[];
430
465
  }
431
466
 
432
- // @zukhruf/evals/classification (designed; built in #3889)
467
+ // @zukhruf/evals/classification
433
468
 
434
- /** A label: an int or a string, as scikit-learn reads a label. */
469
+ /** A label: a number or a string, as scikit-learn reads a label. */
435
470
  export type Label = number | string;
436
471
 
437
472
  /** The expected labels of one test, and its weight. */
438
473
  export class LabelCase implements Weighted {
439
- constructor(expected: Label | readonly Label[], weight?: number);
474
+ constructor(options: { expected: Label | readonly Label[]; weight: number });
440
475
  readonly expected: ReadonlySet<Label>;
441
476
  readonly weight: number;
442
477
  }
443
- /** Reads `vars.expected` (a label or a list of labels) and `vars.weight`. */
478
+ /** Reads `vars.expected` (a label or a list of labels) and `vars.weight` (1 when none). */
444
479
  export class LabelVarsCase implements CaseReader<LabelCase> {}
445
- /** Reads `config.expected`. */
480
+ /** Reads `config.expected`, with the weight 1. */
446
481
  export class LabelConfigCase implements CaseReader<LabelCase> {}
447
482
 
448
483
  export class LabelComparison {
484
+ constructor(sides: {
485
+ stated: ReadonlySet<Label>;
486
+ expected: ReadonlySet<Label>;
487
+ });
449
488
  readonly stated: ReadonlySet<Label>;
450
489
  readonly expected: ReadonlySet<Label>;
490
+ /** The labels of both sides, in the order of numpy's unique(). */
491
+ labels(): Label[];
492
+ /** The comparison of one class. */
493
+ only(label: Label): LabelComparison;
451
494
  }
452
- /** Reads the stated labels of an answer: a label, a list of labels, or their JSON text. */
453
- export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
454
- constructor(options?: { normalize?: (label: string) => string });
455
- }
495
+ /** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
496
+ export class LabelComparer implements Comparer<LabelCase, LabelComparison> {}
456
497
 
457
498
  export class ClassMatches implements Matches<LabelComparison> {
458
499
  constructor(label: Label);
@@ -463,12 +504,20 @@ export class LabelsMatch extends Gate<LabelComparison> {}
463
504
  export class PerClass extends Grouped<LabelComparison> {
464
505
  constructor(
465
506
  scorer: Scorer<LabelComparison>,
466
- options: { labels: readonly Label[] },
507
+ options?: { labels?: readonly Label[] },
467
508
  );
468
509
  }
469
510
 
511
+ /** The classes that an average reads. */
512
+ export type Classes = {
513
+ readonly selected: ReadonlyMap<Label, Counts>;
514
+ readonly present: ReadonlyMap<Label, Counts>;
515
+ readonly multilabel: boolean;
516
+ };
470
517
  /** How the scores of the classes become one number (Strategy). */
471
- export interface Average {}
518
+ export interface Average {
519
+ of(classes: Classes, score: (counts: Counts) => number): number;
520
+ }
472
521
  export class MicroAverage implements Average {}
473
522
  export class MacroAverage implements Average {}
474
523
  export class WeightedAverage implements Average {}
@@ -476,33 +525,33 @@ export class BinaryAverage implements Average {
476
525
  constructor(positiveLabel: Label);
477
526
  }
478
527
 
528
+ type ScoreOptions = {
529
+ average: Average;
530
+ labels?: readonly Label[];
531
+ /** 0 (the default), 1 or NaN. */
532
+ zeroDivision?: number;
533
+ };
479
534
  /** The counts of each class over many answers: true positives, predicted and true. */
480
535
  export class ClassCounts {
481
- static of(
482
- stated: readonly (Label | readonly Label[])[],
483
- expected: readonly (Label | readonly Label[])[],
484
- ): ClassCounts;
485
- add(comparison: LabelComparison): void;
536
+ constructor(options: { multilabel: boolean });
537
+ static of(answers: {
538
+ expected: readonly Label[];
539
+ stated: readonly Label[];
540
+ weights?: readonly number[];
541
+ }): ClassCounts;
542
+ static ofIndicators(answers: {
543
+ expected: readonly (readonly number[])[];
544
+ stated: readonly (readonly number[])[];
545
+ weights?: readonly number[];
546
+ }): ClassCounts;
547
+ add(comparison: LabelComparison, weight: number): void;
486
548
  labels(): Label[];
487
549
  accuracy(): number;
488
- precision(options: {
489
- average: Average;
490
- zeroDivision?: number;
491
- labels?: readonly Label[];
492
- }): number;
493
- recall(options: {
494
- average: Average;
495
- zeroDivision?: number;
496
- labels?: readonly Label[];
497
- }): number;
498
- fBeta(options: {
499
- beta: number;
500
- average: Average;
501
- zeroDivision?: number;
502
- labels?: readonly Label[];
503
- form?: FMeasureForm;
504
- }): number;
550
+ precision(options: ScoreOptions): number;
551
+ recall(options: ScoreOptions): number;
552
+ fBeta(options: ScoreOptions & { beta: number; form?: FMeasureForm }): number;
505
553
  support(): ReadonlyMap<Label, number>;
554
+ /** Throws for multilabel counts. */
506
555
  confusionMatrix(): ConfusionMatrix;
507
556
  }
508
557
  export class ConfusionMatrix {
@@ -511,12 +560,24 @@ export class ConfusionMatrix {
511
560
  rows(): number[][];
512
561
  }
513
562
 
514
- /** The stated probability of each class of one answer. */
563
+ /** The stated probability of each class of one answer, and its expected label. */
515
564
  export class ProbabilityComparison {
516
565
  readonly probabilities: ReadonlyMap<Label, number>;
517
- readonly expected: ReadonlySet<Label>;
566
+ readonly expected: Label;
567
+ }
568
+ /** Reads the probability of each label from a Map, an object or its JSON text. */
569
+ export class ProbabilityComparer implements Comparer<
570
+ LabelCase,
571
+ ProbabilityComparison
572
+ > {}
573
+ export class BrierScore implements Scorer<ProbabilityComparison> {
574
+ constructor(options?: { positiveLabel?: Label });
575
+ /** The score of a whole run, with numpy's pairwise sum. */
576
+ mean(
577
+ comparisons: readonly ProbabilityComparison[],
578
+ weights?: readonly number[],
579
+ ): number;
518
580
  }
519
- export class BrierScore implements Scorer<ProbabilityComparison> {}
520
581
  ```
521
582
 
522
583
  ## Who uses each part
@@ -532,7 +593,7 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
532
593
  | relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`) | mutex #3891 |
533
594
  | event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
534
595
 
535
- The build is in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (`./classification`).
596
+ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
536
597
 
537
598
  ## Documentation
538
599
 
@@ -0,0 +1,11 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ import type { Label } from './label.ts';
3
+ export type Classes = {
4
+ readonly selected: ReadonlyMap<Label, Counts>;
5
+ readonly present: ReadonlyMap<Label, Counts>;
6
+ readonly multilabel: boolean;
7
+ };
8
+ export interface Average {
9
+ of(classes: Classes, score: (counts: Counts) => number): number;
10
+ }
11
+ //# sourceMappingURL=average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"average.js","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":""}
@@ -0,0 +1,9 @@
1
+ import { Counts } from '../aggregation/counts.ts';
2
+ import type { Average, Classes } from './average.ts';
3
+ import type { Label } from './label.ts';
4
+ export declare class BinaryAverage implements Average {
5
+ #private;
6
+ constructor(positiveLabel: Label);
7
+ of({ present, multilabel }: Classes, score: (counts: Counts) => number): number;
8
+ }
9
+ //# sourceMappingURL=binary-average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,EAChC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAYV"}
@@ -0,0 +1,20 @@
1
+ import { Counts } from "../aggregation/counts.js";
2
+ // average='binary': the score of the positive class, for answers of two
3
+ // classes or fewer. It throws as scikit-learn raises: for a set of classes,
4
+ // for more than two classes, and for a positive label that is not one of two
5
+ // classes. A positive label that no answer has scores counts of 0.
6
+ export class BinaryAverage {
7
+ #positiveLabel;
8
+ constructor(positiveLabel) {
9
+ this.#positiveLabel = positiveLabel;
10
+ }
11
+ of({ present, multilabel }, score) {
12
+ if (multilabel || present.size > 2)
13
+ throw new RangeError(`The target is ${multilabel ? 'multilabel-indicator' : 'multiclass'}, but the average is binary: choose another average.`);
14
+ const counts = present.get(this.#positiveLabel);
15
+ if (counts === undefined && present.size === 2)
16
+ throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not one of the labels ${JSON.stringify([...present.keys()])}.`);
17
+ return score(counts ?? new Counts({ hits: 0, stated: 0, expected: 0 }));
18
+ }
19
+ }
20
+ //# sourceMappingURL=binary-average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,wEAAwE;AACxE,4EAA4E;AAC5E,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAW,EAChC,KAAiC;QAEjC,IAAI,UAAU,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC;YAChC,MAAM,IAAI,UAAU,CAClB,iBAAiB,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,sDAAsD,CAC1H,CAAC;QACJ,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAC5C,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC7H,CAAC;QACJ,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
@@ -0,0 +1,13 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Scorer } from '../scorers/scorer.ts';
3
+ import { type Label } from './label.ts';
4
+ import type { ProbabilityComparison } from './probability-comparison.ts';
5
+ export declare class BrierScore implements Scorer<ProbabilityComparison> {
6
+ #private;
7
+ constructor({ positiveLabel }?: {
8
+ readonly positiveLabel?: Label;
9
+ });
10
+ score(comparison: ProbabilityComparison): Ratio;
11
+ mean(comparisons: readonly ProbabilityComparison[], weights?: readonly number[]): number;
12
+ }
13
+ //# sourceMappingURL=brier-score.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AASzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAGlD,EAAE,aAAa,EAAE,GAAE;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,CAAA;KAAO;IAItE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAyCV"}
@@ -0,0 +1,52 @@
1
+ import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
+ import { Ratio } from "../aggregation/ratio.js";
3
+ import { compareLabels } from "./label.js";
4
+ // scikit-learn 1.9.1's brier_score_loss. For one answer, the sum over the
5
+ // classes of (1 when the class is the expected label, else 0, minus the
6
+ // stated probability) squared, with numpy's pairwise sum over the classes in
7
+ // their sorted order; times 0.5 for two classes. With a positive label, the
8
+ // answer gives the probability p of that label only, and the two classes are
9
+ // 1 - p and p, as scikit-learn reads a one-dimensional y_proba. Its ratio is
10
+ // the score over 1, and mean gives the score of a whole run.
11
+ export class BrierScore {
12
+ #positiveLabel;
13
+ constructor({ positiveLabel } = {}) {
14
+ this.#positiveLabel = positiveLabel;
15
+ }
16
+ score(comparison) {
17
+ return new Ratio(this.#value(comparison), 1);
18
+ }
19
+ // The mean of the scores of the answers, with numpy's pairwise sum, each
20
+ // answer with its weight (1 when none is given).
21
+ mean(comparisons, weights) {
22
+ const scores = comparisons.map((comparison) => this.#value(comparison));
23
+ if (weights === undefined)
24
+ return gsumpw(scores.length, scores, 1) / scores.length;
25
+ if (weights.length !== scores.length)
26
+ throw new RangeError(`The answers and the weights have ${scores.length} and ${weights.length} values.`);
27
+ const weighted = scores.map((score, index) => score * (weights[index] ?? 0));
28
+ return (gsumpw(weighted.length, weighted, 1) / gsumpw(weights.length, weights, 1));
29
+ }
30
+ #value({ probabilities, expected }) {
31
+ if (this.#positiveLabel !== undefined) {
32
+ const p = probabilities.get(this.#positiveLabel);
33
+ if (p === undefined)
34
+ throw new RangeError(`The answer gives no probability of the positive label ${JSON.stringify(this.#positiveLabel)}.`);
35
+ const y = expected === this.#positiveLabel ? 1 : 0;
36
+ const negative = 1 - y - (1 - p);
37
+ const positive = y - p;
38
+ return 0.5 * (negative * negative + positive * positive);
39
+ }
40
+ if (!probabilities.has(expected))
41
+ throw new RangeError(`The answer gives no probability of the expected label ${JSON.stringify(expected)}.`);
42
+ const squares = [...probabilities]
43
+ .toSorted(([left], [right]) => compareLabels(left, right))
44
+ .map(([label, probability]) => {
45
+ const difference = (label === expected ? 1 : 0) - probability;
46
+ return difference * difference;
47
+ });
48
+ const sum = gsumpw(squares.length, squares, 1);
49
+ return squares.length < 3 ? 0.5 * sum : sum;
50
+ }
51
+ }
52
+ //# sourceMappingURL=brier-score.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAE3C,YAAY,EAAE,aAAa,KAAyC,EAAE;QACpE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YACvB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,MAAM,CAAC,GAAG,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;CACF"}
@@ -0,0 +1,39 @@
1
+ import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
2
+ import type { Average } from './average.ts';
3
+ import { ConfusionMatrix } from './confusion-matrix.ts';
4
+ import { LabelComparison } from './label-comparison.ts';
5
+ import { type Label } from './label.ts';
6
+ type ScoreOptions = {
7
+ readonly average: Average;
8
+ readonly labels?: readonly Label[];
9
+ readonly zeroDivision?: 0 | 1 | number;
10
+ };
11
+ export declare class ClassCounts {
12
+ #private;
13
+ constructor({ multilabel }: {
14
+ readonly multilabel: boolean;
15
+ });
16
+ static of({ expected, stated, weights, }: {
17
+ readonly expected: readonly Label[];
18
+ readonly stated: readonly Label[];
19
+ readonly weights?: readonly number[];
20
+ }): ClassCounts;
21
+ static ofIndicators({ expected, stated, weights, }: {
22
+ readonly expected: readonly (readonly number[])[];
23
+ readonly stated: readonly (readonly number[])[];
24
+ readonly weights?: readonly number[];
25
+ }): ClassCounts;
26
+ add(comparison: LabelComparison, weight: number): void;
27
+ labels(): Label[];
28
+ accuracy(): number;
29
+ precision(options: ScoreOptions): number;
30
+ recall(options: ScoreOptions): number;
31
+ fBeta({ beta, form, ...options }: ScoreOptions & {
32
+ readonly beta: number;
33
+ readonly form?: FMeasureForm;
34
+ }): number;
35
+ support(): ReadonlyMap<Label, number>;
36
+ confusionMatrix(): ConfusionMatrix;
37
+ }
38
+ export {};
39
+ //# sourceMappingURL=class-counts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAQlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
@@ -0,0 +1,159 @@
1
+ import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
+ import { Counts } from "../aggregation/counts.js";
3
+ import { CountsForm, } from "../aggregation/f-measure-form.js";
4
+ import { ConfusionMatrix } from "./confusion-matrix.js";
5
+ import { LabelComparison } from "./label-comparison.js";
6
+ import { compareLabels } from "./label.js";
7
+ // The running counts of one class, added in the order of the answers.
8
+ class Tally {
9
+ hits = 0;
10
+ stated = 0;
11
+ expected = 0;
12
+ }
13
+ // The counts of each class over many answers, as scikit-learn's
14
+ // multilabel_confusion_matrix gives them: the true positives (hits), the
15
+ // predicted (stated) and the true (expected), each the sum of the weights of
16
+ // the answers in their order, as numpy's bincount adds them. Add each
17
+ // comparison, then read the metrics of a whole run (torchmetrics' update and
18
+ // compute). It also keeps the weight of each right answer for the accuracy,
19
+ // and the weight of each pair of labels for the confusion matrix. The counts
20
+ // are of one class for each answer (multiclass) or of a set of classes
21
+ // (multilabel), as torchmetrics has a multiclass and a multilabel class.
22
+ export class ClassCounts {
23
+ #multilabel;
24
+ #classes = new Map();
25
+ #rights = [];
26
+ #weights = [];
27
+ #pairs = new Map();
28
+ constructor({ multilabel }) {
29
+ this.#multilabel = multilabel;
30
+ }
31
+ // The counts of two arrays of labels, one label for each answer
32
+ // (multiclass), with the weight of each answer (1 when none is given).
33
+ static of({ expected, stated, weights, }) {
34
+ const counts = new ClassCounts({ multilabel: false });
35
+ for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
36
+ counts.add(new LabelComparison({
37
+ stated: new Set([statedLabel]),
38
+ expected: new Set([expectedLabel]),
39
+ }), weight);
40
+ return counts;
41
+ }
42
+ // The counts of two label indicator matrices (multilabel): a row of 0 and 1
43
+ // for each answer, a column for each label. Each column is a class, also
44
+ // when no answer has it.
45
+ static ofIndicators({ expected, stated, weights, }) {
46
+ const columns = expected[0]?.length ?? 0;
47
+ const counts = new ClassCounts({ multilabel: true });
48
+ for (let column = 0; column < columns; column++)
49
+ counts.#classes.set(column, new Tally());
50
+ const labels = (row) => {
51
+ if (row.length !== columns ||
52
+ row.some((value) => value !== 0 && value !== 1))
53
+ throw new RangeError(`A row of a label indicator matrix holds ${columns} values of 0 or 1, not ${JSON.stringify(row)}.`);
54
+ return new Set(row.flatMap((value, column) => (value === 1 ? [column] : [])));
55
+ };
56
+ for (const [expectedRow, statedRow, weight] of ClassCounts.#answers(expected, stated, weights))
57
+ counts.add(new LabelComparison({
58
+ stated: labels(statedRow),
59
+ expected: labels(expectedRow),
60
+ }), weight);
61
+ return counts;
62
+ }
63
+ // The expected side, the stated side and the weight of each answer. An
64
+ // answer without a weight weighs 1.
65
+ static *#answers(expected, stated, weights) {
66
+ const unequal = new RangeError(`The expected labels, the stated labels and the weights have ${expected.length}, ${stated.length} and ${weights?.length ?? expected.length} answers.`);
67
+ if (stated.length !== expected.length ||
68
+ (weights !== undefined && weights.length !== expected.length))
69
+ throw unequal;
70
+ for (const [index, side] of expected.entries()) {
71
+ const other = stated[index];
72
+ if (other === undefined)
73
+ throw unequal;
74
+ yield [side, other, weights?.[index] ?? 1];
75
+ }
76
+ }
77
+ // Adds one answer with its weight. Multiclass counts refuse an answer with
78
+ // no label or more than one label on a side.
79
+ add(comparison, weight) {
80
+ const { stated, expected } = comparison;
81
+ if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
82
+ throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
83
+ for (const label of comparison.labels()) {
84
+ const tally = this.#classes.getOrInsertComputed(label, () => new Tally());
85
+ if (stated.has(label))
86
+ tally.stated += weight;
87
+ if (expected.has(label))
88
+ tally.expected += weight;
89
+ if (stated.has(label) && expected.has(label))
90
+ tally.hits += weight;
91
+ }
92
+ const right = stated.size === expected.size && stated.isSubsetOf(expected);
93
+ this.#rights.push(right ? weight : 0);
94
+ this.#weights.push(weight);
95
+ if (this.#multilabel)
96
+ return;
97
+ const [expectedLabel] = expected;
98
+ const [statedLabel] = stated;
99
+ if (expectedLabel === undefined || statedLabel === undefined)
100
+ return;
101
+ const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
102
+ row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
103
+ }
104
+ // The labels of the classes, in the order of numpy's unique().
105
+ labels() {
106
+ return [...this.#classes.keys()].toSorted(compareLabels);
107
+ }
108
+ // accuracy_score: the weight of the right answers over the weight of all
109
+ // answers, each with numpy's pairwise sum. An answer is right when its
110
+ // stated labels are its expected labels (the subset accuracy for a set of
111
+ // classes).
112
+ accuracy() {
113
+ return (gsumpw(this.#rights.length, this.#rights, 1) /
114
+ gsumpw(this.#weights.length, this.#weights, 1));
115
+ }
116
+ // precision_score: hits / stated of each class, then the average.
117
+ precision(options) {
118
+ return this.#score(options, ({ hits, stated }) => stated === 0 ? undefined : hits / stated);
119
+ }
120
+ // recall_score: hits / expected of each class, then the average.
121
+ recall(options) {
122
+ return this.#score(options, ({ hits, expected }) => expected === 0 ? undefined : hits / expected);
123
+ }
124
+ // fbeta_score: the F-beta of each class in the form (scikit-learn's
125
+ // CountsForm by default), then the average. Its denominator is
126
+ // β²·expected + stated.
127
+ fBeta({ beta, form = new CountsForm(), ...options }) {
128
+ return this.#score(options, (counts) => beta ** 2 * counts.expected + counts.stated === 0
129
+ ? undefined
130
+ : form.of(counts, beta));
131
+ }
132
+ // The true count of each class, in the order of the labels.
133
+ support() {
134
+ return new Map(this.labels().map((label) => [label, this.#counts(label).expected]));
135
+ }
136
+ // confusion_matrix of the answers, with the labels in their order. A set
137
+ // of classes has no confusion matrix (scikit-learn raises), so it throws.
138
+ confusionMatrix() {
139
+ if (this.#multilabel)
140
+ throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
141
+ return new ConfusionMatrix(this.labels(), this.#pairs);
142
+ }
143
+ #score({ average, labels = this.labels(), zeroDivision = 0 }, score) {
144
+ return average.of({
145
+ selected: new Map(labels.map((label) => [label, this.#counts(label)])),
146
+ present: new Map(this.labels().map((label) => [label, this.#counts(label)])),
147
+ multilabel: this.#multilabel,
148
+ }, (counts) => score(counts) ?? zeroDivision);
149
+ }
150
+ #counts(label) {
151
+ const tally = this.#classes.get(label);
152
+ return new Counts({
153
+ hits: tally?.hits ?? 0,
154
+ stated: tally?.stated ?? 0,
155
+ expected: tally?.expected ?? 0,
156
+ });
157
+ }
158
+ }
159
+ //# sourceMappingURL=class-counts.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAWvD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;CACd;AAED,gEAAgE;AAChE,yEAAyE;AACzE,6EAA6E;AAC7E,sEAAsE;AACtE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IACnC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EAAE,UAAU,EAAoC;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;aACnC,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;aAC9B,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6CAA6C;IAC7C,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,+DAA+D;IAC/D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,CACL,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAC/C,CAAC;IACJ,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAgB,EACnE,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,IAAI,GAAG,CACd,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAC3D;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YACtB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC1B,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import type { Label } from './label.ts';
2
+ export declare class ConfusionMatrix {
3
+ #private;
4
+ readonly labels: readonly Label[];
5
+ constructor(labels: readonly Label[], pairs: ReadonlyMap<Label, ReadonlyMap<Label, number>>);
6
+ count(expected: Label, stated: Label): number;
7
+ rows(): number[][];
8
+ }
9
+ //# sourceMappingURL=confusion-matrix.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"confusion-matrix.d.ts","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,eAAe;;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;gBAIhC,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAMvD,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,MAAM;IAI7C,IAAI,IAAI,MAAM,EAAE,EAAE;CAKnB"}
@@ -0,0 +1,18 @@
1
+ // The weight of each pair of an expected label and a stated label, as
2
+ // scikit-learn's confusion_matrix gives it: a row for each expected label and
3
+ // a column for each stated label, in the order of the labels.
4
+ export class ConfusionMatrix {
5
+ labels;
6
+ #pairs;
7
+ constructor(labels, pairs) {
8
+ this.labels = labels;
9
+ this.#pairs = pairs;
10
+ }
11
+ count(expected, stated) {
12
+ return this.#pairs.get(expected)?.get(stated) ?? 0;
13
+ }
14
+ rows() {
15
+ return this.labels.map((expected) => this.labels.map((stated) => this.count(expected, stated)));
16
+ }
17
+ }
18
+ //# sourceMappingURL=confusion-matrix.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"confusion-matrix.js","sourceRoot":"","sources":["../../src/classification/confusion-matrix.ts"],"names":[],"mappings":"AAEA,sEAAsE;AACtE,8EAA8E;AAC9E,8DAA8D;AAC9D,MAAM,OAAO,eAAe;IACjB,MAAM,CAAmB;IACzB,MAAM,CAAiD;IAEhE,YACE,MAAwB,EACxB,KAAqD;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAe,EAAE,MAAa;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAC1D,CAAC;IACJ,CAAC;CACF"}
@@ -8,4 +8,14 @@ export { LabelVarsCase } from './label-vars-case.ts';
8
8
  export { LabelsMatch } from './labels-match.ts';
9
9
  export type { Label } from './label.ts';
10
10
  export { PerClass } from './per-class.ts';
11
+ export type { Average, Classes } from './average.ts';
12
+ export { BinaryAverage } from './binary-average.ts';
13
+ export { ClassCounts } from './class-counts.ts';
14
+ export { ConfusionMatrix } from './confusion-matrix.ts';
15
+ export { MacroAverage } from './macro-average.ts';
16
+ export { MicroAverage } from './micro-average.ts';
17
+ export { WeightedAverage } from './weighted-average.ts';
18
+ export { BrierScore } from './brier-score.ts';
19
+ export { ProbabilityComparer } from './probability-comparer.ts';
20
+ export { ProbabilityComparison } from './probability-comparison.ts';
11
21
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
@@ -7,4 +7,13 @@ export { LabelConfigCase } from "./label-config-case.js";
7
7
  export { LabelVarsCase } from "./label-vars-case.js";
8
8
  export { LabelsMatch } from "./labels-match.js";
9
9
  export { PerClass } from "./per-class.js";
10
+ export { BinaryAverage } from "./binary-average.js";
11
+ export { ClassCounts } from "./class-counts.js";
12
+ export { ConfusionMatrix } from "./confusion-matrix.js";
13
+ export { MacroAverage } from "./macro-average.js";
14
+ export { MicroAverage } from "./micro-average.js";
15
+ export { WeightedAverage } from "./weighted-average.js";
16
+ export { BrierScore } from "./brier-score.js";
17
+ export { ProbabilityComparer } from "./probability-comparer.js";
18
+ export { ProbabilityComparison } from "./probability-comparison.js";
10
19
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ import type { Average, Classes } from './average.ts';
3
+ export declare class MacroAverage implements Average {
4
+ of({ selected }: Classes, score: (counts: Counts) => number): number;
5
+ }
6
+ //# sourceMappingURL=macro-average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"macro-average.d.ts","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAOrE"}
@@ -0,0 +1,15 @@
1
+ import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
+ // average='macro': the mean of the scores of the selected classes, as
3
+ // numpy's nanmean gives it: a NaN score (zeroDivision NaN) adds 0 and is not
4
+ // counted, and the mean of no score is NaN.
5
+ export class MacroAverage {
6
+ of({ selected }, score) {
7
+ const scores = [...selected.values()].map(score);
8
+ const counted = scores.filter((value) => !Number.isNaN(value)).length;
9
+ if (counted === 0)
10
+ return Number.NaN;
11
+ const values = scores.map((value) => (Number.isNaN(value) ? 0 : value));
12
+ return gsumpw(values.length, values, 1) / counted;
13
+ }
14
+ }
15
+ //# sourceMappingURL=macro-average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAKlD,sEAAsE;AACtE,6EAA6E;AAC7E,4CAA4C;AAC5C,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC;IACpD,CAAC;CACF"}
@@ -0,0 +1,6 @@
1
+ import { Counts } from '../aggregation/counts.ts';
2
+ import type { Average, Classes } from './average.ts';
3
+ export declare class MicroAverage implements Average {
4
+ of({ selected }: Classes, score: (counts: Counts) => number): number;
5
+ }
6
+ //# sourceMappingURL=micro-average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"micro-average.d.ts","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAYrE"}
@@ -0,0 +1,16 @@
1
+ import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
+ import { Counts } from "../aggregation/counts.js";
3
+ // average='micro': the counts of the selected classes are added, with numpy's
4
+ // pairwise sum, and scored once.
5
+ export class MicroAverage {
6
+ of({ selected }, score) {
7
+ const classes = [...selected.values()];
8
+ const sum = (count) => gsumpw(classes.length, classes.map(count), 1);
9
+ return score(new Counts({
10
+ hits: sum(({ hits }) => hits),
11
+ stated: sum(({ stated }) => stated),
12
+ expected: sum(({ expected }) => expected),
13
+ }));
14
+ }
15
+ }
16
+ //# sourceMappingURL=micro-average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,8EAA8E;AAC9E,iCAAiC;AACjC,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,CAAC,KAAiC,EAAE,EAAE,CAChD,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,OAAO,KAAK,CACV,IAAI,MAAM,CAAC;YACT,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;YAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;YACnC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,8 @@
1
+ import type { Comparer } from '../scoring/comparer.ts';
2
+ import type { LabelCase } from './label-case.ts';
3
+ import { ProbabilityComparison } from './probability-comparison.ts';
4
+ export declare class ProbabilityComparer implements Comparer<LabelCase, ProbabilityComparison> {
5
+ #private;
6
+ compare(stated: unknown, testCase: LabelCase): ProbabilityComparison;
7
+ }
8
+ //# sourceMappingURL=probability-comparer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"probability-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/probability-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAOpE,qBAAa,mBAAoB,YAAW,QAAQ,CAClD,SAAS,EACT,qBAAqB,CACtB;;IACC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,qBAAqB;CAoCrE"}
@@ -0,0 +1,35 @@
1
+ import { ProbabilityComparison } from "./probability-comparison.js";
2
+ // Reads the stated probability of each class of an answer and gives its
3
+ // comparison with the case (Strategy). The answer is a Map of labels, a plain
4
+ // object or the JSON text of an object, whose values are probabilities in
5
+ // [0, 1]. A key of an object is a number label when each expected label of
6
+ // the case is a number. The case has one expected label.
7
+ export class ProbabilityComparer {
8
+ compare(stated, testCase) {
9
+ const [expected, ...others] = testCase.expected;
10
+ if (expected === undefined || others.length > 0)
11
+ throw new RangeError(`A probability is scored against one expected label, not ${JSON.stringify([...testCase.expected])}.`);
12
+ const numbers = typeof expected === 'number';
13
+ const entries = ProbabilityComparer.#entries(typeof stated === 'string' ? JSON.parse(stated) : stated).map(([key, probability]) => {
14
+ if (typeof probability !== 'number' ||
15
+ !(probability >= 0 && probability <= 1))
16
+ throw new RangeError(`The probability of ${JSON.stringify(key)} is a number in [0, 1], not ${JSON.stringify(probability)}.`);
17
+ return [
18
+ numbers && typeof key === 'string' ? Number(key) : key,
19
+ probability,
20
+ ];
21
+ });
22
+ return new ProbabilityComparison({
23
+ probabilities: new Map(entries),
24
+ expected,
25
+ });
26
+ }
27
+ static #entries(value) {
28
+ if (value instanceof Map)
29
+ return [...value];
30
+ if (typeof value === 'object' && value !== null && !Array.isArray(value))
31
+ return Object.entries(value);
32
+ throw new TypeError(`A stated answer is the probability of each label, as an object, not ${JSON.stringify(value)}`);
33
+ }
34
+ }
35
+ //# sourceMappingURL=probability-comparer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"probability-comparer.js","sourceRoot":"","sources":["../../src/classification/probability-comparer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,wEAAwE;AACxE,8EAA8E;AAC9E,0EAA0E;AAC1E,2EAA2E;AAC3E,yDAAyD;AACzD,MAAM,OAAO,mBAAmB;IAI9B,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAChD,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7C,MAAM,IAAI,UAAU,CAClB,2DAA2D,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,CACrG,CAAC;QACJ,MAAM,OAAO,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC;QAC7C,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAC1C,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CACzD,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,EAAmB,EAAE;YAC5C,IACE,OAAO,WAAW,KAAK,QAAQ;gBAC/B,CAAC,CAAC,WAAW,IAAI,CAAC,IAAI,WAAW,IAAI,CAAC,CAAC;gBAEvC,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,GAAG,CACvG,CAAC;YACJ,OAAO;gBACL,OAAO,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;gBACtD,WAAW;aACZ,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,qBAAqB,CAAC;YAC/B,aAAa,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC;YAC/B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,KAAc;QAC5B,IAAI,KAAK,YAAY,GAAG;YAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACtE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,SAAS,CACjB,uEAAuE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,10 @@
1
+ import type { Label } from './label.ts';
2
+ export declare class ProbabilityComparison {
3
+ readonly probabilities: ReadonlyMap<Label, number>;
4
+ readonly expected: Label;
5
+ constructor({ probabilities, expected, }: {
6
+ readonly probabilities: ReadonlyMap<Label, number>;
7
+ readonly expected: Label;
8
+ });
9
+ }
10
+ //# sourceMappingURL=probability-comparison.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"probability-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/probability-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAIxC,qBAAa,qBAAqB;IAChC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;gBAEb,EACV,aAAa,EACb,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnD,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;KAC1B;CAIF"}
@@ -0,0 +1,11 @@
1
+ // The stated probability of each class of one answer, and the expected
2
+ // label of its case.
3
+ export class ProbabilityComparison {
4
+ probabilities;
5
+ expected;
6
+ constructor({ probabilities, expected, }) {
7
+ this.probabilities = probabilities;
8
+ this.expected = expected;
9
+ }
10
+ }
11
+ //# sourceMappingURL=probability-comparison.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"probability-comparison.js","sourceRoot":"","sources":["../../src/classification/probability-comparison.ts"],"names":[],"mappings":"AAEA,uEAAuE;AACvE,qBAAqB;AACrB,MAAM,OAAO,qBAAqB;IACvB,aAAa,CAA6B;IAC1C,QAAQ,CAAQ;IAEzB,YAAY,EACV,aAAa,EACb,QAAQ,GAIT;QACC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF"}
@@ -0,0 +1,6 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ import type { Average, Classes } from './average.ts';
3
+ export declare class WeightedAverage implements Average {
4
+ of({ selected }: Classes, score: (counts: Counts) => number): number;
5
+ }
6
+ //# sourceMappingURL=weighted-average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAUrE"}
@@ -0,0 +1,20 @@
1
+ import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
+ // average='weighted': the mean of the scores of the selected classes, each
3
+ // weighted by its true count, as scikit-learn's _nanaverage gives it: a NaN
4
+ // score is dropped with its weight, the mean of no score is NaN, and when the
5
+ // weights add up to 0 it is the plain mean.
6
+ export class WeightedAverage {
7
+ of({ selected }, score) {
8
+ const kept = [...selected.values()]
9
+ .map((counts) => ({ value: score(counts), weight: counts.expected }))
10
+ .filter(({ value }) => !Number.isNaN(value));
11
+ if (kept.length === 0)
12
+ return Number.NaN;
13
+ const sum = (values) => gsumpw(values.length, values, 1);
14
+ const weights = sum(kept.map(({ weight }) => weight));
15
+ if (weights === 0)
16
+ return sum(kept.map(({ value }) => value)) / kept.length;
17
+ return sum(kept.map(({ value, weight }) => value * weight)) / weights;
18
+ }
19
+ }
20
+ //# sourceMappingURL=weighted-average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAKlD,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,OAAO,eAAe;IAC1B,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;aAChC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACpE,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACzC,MAAM,GAAG,GAAG,CAAC,MAAyB,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5E,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC;IACxE,CAAC;CACF"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@zukhruf/evals",
3
- "version": "0.8.4",
4
- "description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios, scorecards, totals and bootstrap intervals, ./structured for structured values, and ./promptfoo for promptfoo.",
3
+ "version": "0.8.5",
4
+ "description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
5
5
  "license": "MIT",
6
6
  "repository": {
7
7
  "type": "git",
@@ -18,6 +18,8 @@
18
18
  "scorers",
19
19
  "structured-output",
20
20
  "extraction",
21
+ "classification",
22
+ "scikit-learn",
21
23
  "precision",
22
24
  "recall",
23
25
  "promptfoo"
@@ -61,24 +63,29 @@
61
63
  "access": "public"
62
64
  },
63
65
  "dependencies": {
64
- "@zukhruf/python": "^0.8.4",
65
- "@zukhruf/statistics": "^0.8.4"
66
+ "@zukhruf/python": "^0.8.5",
67
+ "@zukhruf/statistics": "^0.8.5"
66
68
  },
67
69
  "peerDependencies": {
68
70
  "@cfworker/json-schema": "4.1.4",
71
+ "@stdlib/blas-ext-base-gsumpw": "^0.3.1",
69
72
  "munkres": "^2.1.1"
70
73
  },
71
74
  "peerDependenciesMeta": {
72
75
  "@cfworker/json-schema": {
73
76
  "optional": true
74
77
  },
78
+ "@stdlib/blas-ext-base-gsumpw": {
79
+ "optional": true
80
+ },
75
81
  "munkres": {
76
82
  "optional": true
77
83
  }
78
84
  },
79
85
  "devDependencies": {
80
86
  "@cfworker/json-schema": "4.1.4",
81
- "@zukhruf/testing": "^0.8.4",
87
+ "@stdlib/blas-ext-base-gsumpw": "^0.3.1",
88
+ "@zukhruf/testing": "^0.8.5",
82
89
  "munkres": "^2.1.1",
83
90
  "promptfoo": "0.125.1",
84
91
  "python-random": "1.0.0"