@zukhruf/evals 0.8.7 → 0.8.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +106 -37
  2. package/dist/classification/all-class-matches.d.ts +1 -1
  3. package/dist/classification/all-class-matches.d.ts.map +1 -1
  4. package/dist/classification/all-class-matches.js +8 -4
  5. package/dist/classification/all-class-matches.js.map +1 -1
  6. package/dist/classification/average.d.ts +1 -2
  7. package/dist/classification/average.d.ts.map +1 -1
  8. package/dist/classification/binary-average.d.ts +2 -2
  9. package/dist/classification/binary-average.d.ts.map +1 -1
  10. package/dist/classification/binary-average.js +13 -12
  11. package/dist/classification/binary-average.js.map +1 -1
  12. package/dist/classification/class-counts.d.ts +5 -4
  13. package/dist/classification/class-counts.d.ts.map +1 -1
  14. package/dist/classification/class-counts.js +41 -33
  15. package/dist/classification/class-counts.js.map +1 -1
  16. package/dist/classification/label-comparer.d.ts +4 -0
  17. package/dist/classification/label-comparer.d.ts.map +1 -1
  18. package/dist/classification/label-comparer.js +10 -2
  19. package/dist/classification/label-comparer.js.map +1 -1
  20. package/dist/classification/label-comparison.d.ts +4 -2
  21. package/dist/classification/label-comparison.d.ts.map +1 -1
  22. package/dist/classification/label-comparison.js +11 -6
  23. package/dist/classification/label-comparison.js.map +1 -1
  24. package/dist/classification/macro-average.d.ts +1 -1
  25. package/dist/classification/macro-average.d.ts.map +1 -1
  26. package/dist/classification/macro-average.js +3 -3
  27. package/dist/classification/macro-average.js.map +1 -1
  28. package/dist/classification/micro-average.d.ts +1 -1
  29. package/dist/classification/micro-average.d.ts.map +1 -1
  30. package/dist/classification/micro-average.js +4 -4
  31. package/dist/classification/micro-average.js.map +1 -1
  32. package/dist/classification/per-class.d.ts +0 -6
  33. package/dist/classification/per-class.d.ts.map +1 -1
  34. package/dist/classification/per-class.js +4 -10
  35. package/dist/classification/per-class.js.map +1 -1
  36. package/dist/classification/weighted-average.d.ts +1 -1
  37. package/dist/classification/weighted-average.d.ts.map +1 -1
  38. package/dist/classification/weighted-average.js +3 -3
  39. package/dist/classification/weighted-average.js.map +1 -1
  40. package/dist/index.d.ts +2 -0
  41. package/dist/index.d.ts.map +1 -1
  42. package/dist/index.js +2 -0
  43. package/dist/index.js.map +1 -1
  44. package/dist/promptfoo/failed-call.d.ts +13 -0
  45. package/dist/promptfoo/failed-call.d.ts.map +1 -0
  46. package/dist/promptfoo/failed-call.js +24 -0
  47. package/dist/promptfoo/failed-call.js.map +1 -0
  48. package/dist/promptfoo/index.d.ts +1 -0
  49. package/dist/promptfoo/index.d.ts.map +1 -1
  50. package/dist/promptfoo/index.js +1 -0
  51. package/dist/promptfoo/index.js.map +1 -1
  52. package/dist/scoring/grouped.d.ts +3 -3
  53. package/dist/scoring/grouped.d.ts.map +1 -1
  54. package/dist/scoring/grouped.js +7 -4
  55. package/dist/scoring/grouped.js.map +1 -1
  56. package/dist/scoring/paired-bootstrap.d.ts +24 -0
  57. package/dist/scoring/paired-bootstrap.d.ts.map +1 -0
  58. package/dist/scoring/paired-bootstrap.js +94 -0
  59. package/dist/scoring/paired-bootstrap.js.map +1 -0
  60. package/dist/scoring/run.d.ts +7 -0
  61. package/dist/scoring/run.d.ts.map +1 -0
  62. package/dist/scoring/run.js +19 -0
  63. package/dist/scoring/run.js.map +1 -0
  64. package/dist/scoring/scorer-set.d.ts +1 -1
  65. package/dist/scoring/scorer-set.d.ts.map +1 -1
  66. package/dist/scoring/scorer-set.js.map +1 -1
  67. package/dist/scoring/test-epochs.d.ts +11 -0
  68. package/dist/scoring/test-epochs.d.ts.map +1 -0
  69. package/dist/scoring/test-epochs.js +25 -0
  70. package/dist/scoring/test-epochs.js.map +1 -0
  71. package/package.json +5 -4
package/README.md CHANGED
@@ -31,7 +31,7 @@ import {
31
31
  LabelVarsCase,
32
32
  } from '@zukhruf/evals/classification';
33
33
 
34
- const comparer = new LabelComparer();
34
+ const comparer = new LabelComparer({ labels: ['program', 'home', 'list'] });
35
35
  const testCase = new LabelVarsCase().from({ expected: 'program' });
36
36
  const comparison = comparer.compare('home', testCase);
37
37
 
@@ -58,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
58
58
 
59
59
  ## Score with many scorers at once
60
60
 
61
- A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
61
+ A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer({ labels }) })`. It makes the input once, and each scorer reads it.
62
62
 
63
- A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
63
+ A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way). A group can also be a projection of the input, of another type: `Grouped<Input, Group>` takes a `Scorer<Group>`, such as a count of the stated items without a pair, for each of their kinds. A grouped scorer is a breakdown, so its ratios stay out of the score of the record, which is the mean of the plain scorers.
64
64
 
65
65
  ```ts
66
66
  import { Recall, ScorerSet } from '@zukhruf/evals';
@@ -75,7 +75,7 @@ const pageKinds = new ScorerSet(
75
75
  accuracy: new LabelsMatch(),
76
76
  recall: new Recall(new AllClassMatches()),
77
77
  },
78
- { comparer: new LabelComparer() },
78
+ { comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }) },
79
79
  );
80
80
  ```
81
81
 
@@ -94,7 +94,7 @@ const pageKinds = new ScorerSet(
94
94
 
95
95
  ## Compare two runs
96
96
 
97
- Status: design for review (backlog #3870, [ADR 0004](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)). The released package does not have `Run` or `PairedBootstrap` yet.
97
+ The package has `Run` and `PairedBootstrap` (backlog #3870, [ADR 0004](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)).
98
98
 
99
99
  A caller runs one harness two times on the same tests: once with the baseline model and once with a change, such as another model. Two means side by side do not say if the change is better. `PairedBootstrap` gives one verdict for each name: better, worse or inconclusive, with the difference, its interval and a p-value.
100
100
 
@@ -146,7 +146,7 @@ const brier = paired.verdict(baseline, change, 'brier', {
146
146
  A run holds the scorecards of one model. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
147
147
 
148
148
  ```ts
149
- // @zukhruf/evals: two runs side by side (design for review)
149
+ // @zukhruf/evals: two runs side by side
150
150
  import type {
151
151
  ConfidenceIntervalMethod,
152
152
  DifferenceInterval,
@@ -224,22 +224,22 @@ It needs no peer: `npm install @zukhruf/evals`. Its sums are `sum` and `mean` of
224
224
 
225
225
  A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
226
226
 
227
- `LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`. It reads a label or a list of labels as they are. It reads text in this order:
227
+ `new LabelComparer({ labels })` reads the stated labels of an answer and gives a `LabelComparison`. `labels` is the class set (see [The class set](#the-class-set)), and each comparison holds it. The comparer reads a label or a list of labels as they are. It reads text in this order:
228
228
 
229
229
  1. A JSON array of labels gives those labels.
230
230
  2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
231
231
  3. Other text is one string label, without its white space at the start and the end.
232
232
 
233
- `comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
233
+ `comparison.labels()` gives the class set, in the order that the caller gives. `comparison.only(label)` gives the comparison of one class: its class set is `[label]`, and each side keeps the label when it has it.
234
234
 
235
235
  The matches of a comparison count one answer:
236
236
 
237
237
  | Matches | Hits | Stated | Expected |
238
238
  | ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
239
239
  | `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
240
- | `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
240
+ | `new AllClassMatches()` | the classes that both sides have | the stated classes | the expected classes |
241
241
 
242
- `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`. Without `labels`, the classes are the labels of both sides of the answer, in the order of numpy's `unique()` (numbers by value, strings by code point).
242
+ `AllClassMatches` counts only the classes of the class set of the comparison. `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer)` gives one ratio for each class of the class set, under the label `${class} ${label}`, in the order of the class set.
243
243
 
244
244
  `Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
245
245
 
@@ -263,6 +263,7 @@ import {
263
263
  } from '@zukhruf/evals/classification';
264
264
 
265
265
  const counts = ClassCounts.of({
266
+ labels: ['a', 'b'],
266
267
  expected: ['a', 'b', 'b'],
267
268
  stated: ['a', 'a', 'b'],
268
269
  });
@@ -270,23 +271,38 @@ const counts = ClassCounts.of({
270
271
  counts.accuracy(); // accuracy_score
271
272
  counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
272
273
  counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
273
- counts.confusionMatrix().rows(); // confusion_matrix, labels in sorted order
274
+ counts.confusionMatrix().rows(); // confusion_matrix, the classes in the order of labels
274
275
 
275
276
  // The same counts, one answer at a time:
276
- const run = new ClassCounts({ multilabel: false });
277
+ const run = new ClassCounts({ labels: ['a', 'b'], multilabel: false });
277
278
  const testCase = new LabelCase({ expected: 'a', weight: 1 });
278
- run.add(new LabelComparer().compare('a', testCase), testCase.weight);
279
+ const comparer = new LabelComparer({ labels: ['a', 'b'] });
280
+ run.add(comparer.compare('a', testCase), testCase.weight);
279
281
  ```
280
282
 
281
- - `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
283
+ - `ClassCounts.of({ labels, expected, stated, weights })` builds multiclass counts from two arrays of labels. `labels` is the class set. `weights` is optional, and an answer without a weight weighs 1.
282
284
  - `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
283
- - `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
284
- - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` throws when scikit-learn raises: for multilabel counts, for more than two classes, and for a positive label that is not one of two classes. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
285
- - `labels` selects the classes to score, in its order. By default it is every class, in sorted order.
285
+ - `new ClassCounts({ labels, multilabel })` starts empty, and `add(comparison, weight)` adds one answer to the classes of the class set of the counts. Multiclass counts refuse an answer with no label or more than one label on a side.
286
+ - `labels()`, `support()` and `confusionMatrix()` give the classes of the class set, in its order.
287
+ - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` reads the class set. It throws for multilabel counts, for a class set of more than two classes, and for a positive label outside the class set. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
286
288
  - `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
287
289
  - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
288
290
  - `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
289
291
 
292
+ ### The class set
293
+
294
+ [ADR 0006](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md) gives this rule. A harness can state any text, such as the answer null of a failed call. If the answers give the classes, each such text adds a class, and the macro averages change. So the caller gives the class set, scikit-learn's `labels`, to the comparer and to the counts. The answers never add a class.
295
+
296
+ ```ts
297
+ import { ClassCounts, LabelComparer } from '@zukhruf/evals/classification';
298
+
299
+ const pageKinds = ['program', 'home', 'list', 'investor', 'other'];
300
+ const comparer = new LabelComparer({ labels: pageKinds });
301
+ const counts = new ClassCounts({ labels: pageKinds, multilabel: false });
302
+ ```
303
+
304
+ A label outside the class set counts in no class, as scikit-learn gives it when the caller gives `labels`. A stated label outside it is a miss of the expected class, and it is not a predicted count of any class. `LabelsMatch` still reads it as a wrong answer. `PerClass` and `AllClassMatches` read the class set of the comparison, so they take no `labels`. `ClassCounts.ofIndicators` takes its classes from the columns. To port a scikit-learn call with `labels=None`, give the sorted labels of both sides.
305
+
290
306
  ### Probabilities
291
307
 
292
308
  `ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
@@ -314,7 +330,10 @@ import {
314
330
  } from '@zukhruf/evals/classification';
315
331
  import { AssertionModule, ScorerAssertion } from '@zukhruf/evals/promptfoo';
316
332
 
317
- const labels = { comparer: new LabelComparer(), cases: new LabelVarsCase() };
333
+ const labels = {
334
+ comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }),
335
+ cases: new LabelVarsCase(),
336
+ };
318
337
 
319
338
  export const scorers = new AssertionModule(import.meta.url, {
320
339
  accuracy: new ScorerAssertion(new LabelsMatch(), {
@@ -344,6 +363,51 @@ export default {
344
363
 
345
364
  The name is the promptfoo `metric`, unless the scorer assertion gives its own `metric`. A name that the module does not hold is a compile error. promptfoo shows the mean of each `metric` over the tests. A `ScorerAssertion` gives one value for one test, so a micro average needs `PromptfooAssertion`, whose ratios keep their wholes.
346
365
 
366
+ ### A failed call
367
+
368
+ [ADR 0005](docs/adr/0005-a-failed-call-is-the-answer-null.md) gives the reasons (backlog #3970).
369
+
370
+ A failed call is a model call that failed for its one test, also after its retries. promptfoo does not grade a result whose provider gives `error` or no output. That result is an error result: it has no named scores, so its case adds nothing to the wholes, and the recall goes up. A failed call is an answer of the harness: the answer null, the text `'null'`. promptfoo grades it, and each family reads it as no answer. `FailedCall` is a coinage of this package. A consumer uses one of two forms.
371
+
372
+ When the provider is your own, return `new FailedCall(reason).response(metadata)`. promptfoo grades the result, and `promptfoo retry <evalId>` does not run it again. The reason is in `metadata.failed`.
373
+
374
+ ```ts
375
+ import { FailedCall } from '@zukhruf/evals/promptfoo';
376
+
377
+ const [result] = await rule.compute([row.input], options?.abortSignal);
378
+ if (result !== undefined && 'error' in result)
379
+ return new FailedCall(result.error).response(metadata);
380
+ ```
381
+
382
+ When the provider is a built-in one, such as `openai:chat`, an `afterEach` extension hook grades each error result on `FailedCall.answer`. The result stays an error result, so `promptfoo retry <evalId>` and `--retry-errors` run it again.
383
+
384
+ ```ts
385
+ import {
386
+ type AfterEachExtensionHookContext,
387
+ ResultFailureReason,
388
+ assertions,
389
+ } from 'promptfoo';
390
+
391
+ import { FailedCall } from '@zukhruf/evals/promptfoo';
392
+
393
+ export async function afterEach({
394
+ test,
395
+ result,
396
+ }: AfterEachExtensionHookContext): Promise<void> {
397
+ if (result.failureReason !== ResultFailureReason.ERROR) return;
398
+ const graded = await assertions.runAssertions({
399
+ test,
400
+ providerResponse: { output: FailedCall.answer },
401
+ vars: test.vars ?? {},
402
+ });
403
+ Object.assign(result.namedScores, graded.namedScores);
404
+ }
405
+ ```
406
+
407
+ A hook cannot write `namedScoreWeights`, so promptfoo gives each score of the hook the weight 1. Use the hook only with scorer assertions whose metrics weigh 1, as in SOB. A `PromptfooAssertion` gives each ratio its whole as the weight, so it needs the first form.
408
+
409
+ A failure of the run, such as a 401, is not a failed call: each call would fail. Throw it. The result then stays an error result, and `promptfoo retry <evalId>` runs it again.
410
+
347
411
  ## What the package does not do
348
412
 
349
413
  - It does not run a harness, load a dataset, store results or show them. promptfoo does that.
@@ -480,10 +544,10 @@ export interface ScorecardScorer<Case extends Weighted> {
480
544
  score(stated: unknown, testCase: Case): Scorecard;
481
545
  }
482
546
  /** Gives one ratio for each group of an input, such as each field or each class. */
483
- export abstract class Grouped<Input> {
484
- constructor(scorer: Scorer<Input>);
485
- /** The groups of an input (Template Method). */
486
- protected abstract groups(input: Input): Iterable<[string, Input]>;
547
+ export abstract class Grouped<Input, Group = Input> {
548
+ constructor(scorer: Scorer<Group>);
549
+ /** The groups of an input (Template Method): a part of it, or a projection of another type. */
550
+ protected abstract groups(input: Input): Iterable<[string, Group]>;
487
551
  /** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
488
552
  ratios(input: Input, label: string): [string, Ratio][];
489
553
  }
@@ -492,7 +556,7 @@ export class ScorerSet<
492
556
  Input,
493
557
  > implements ScorecardScorer<Case> {
494
558
  constructor(
495
- scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>,
559
+ scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input, unknown>>>,
496
560
  options: { comparer: Comparer<Case, Input> },
497
561
  );
498
562
  }
@@ -600,34 +664,36 @@ export class LabelComparison {
600
664
  constructor(sides: {
601
665
  stated: ReadonlySet<Label>;
602
666
  expected: ReadonlySet<Label>;
667
+ labels: readonly Label[];
603
668
  });
604
669
  readonly stated: ReadonlySet<Label>;
605
670
  readonly expected: ReadonlySet<Label>;
606
- /** The labels of both sides, in the order of numpy's unique(). */
671
+ /** The class set, in the order that the caller gives. */
607
672
  labels(): Label[];
608
- /** The comparison of one class. */
673
+ /** The comparison of one class: its class set is [label]. */
609
674
  only(label: Label): LabelComparison;
610
675
  }
611
676
  /** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
612
- export class LabelComparer implements Comparer<LabelCase, LabelComparison> {}
677
+ export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
678
+ /** labels: the class set. */
679
+ constructor(options: { labels: readonly Label[] });
680
+ }
613
681
 
614
682
  export class ClassMatches implements Matches<LabelComparison> {
615
683
  constructor(label: Label);
616
684
  }
685
+ /** Counts only the classes of the class set of the comparison. */
617
686
  export class AllClassMatches implements Matches<LabelComparison> {}
618
687
  /** Holds when the stated labels are the expected labels. */
619
688
  export class LabelsMatch extends Gate<LabelComparison> {}
689
+ /** A ratio for each class of the class set of the comparison. */
620
690
  export class PerClass extends Grouped<LabelComparison> {
621
- constructor(
622
- scorer: Scorer<LabelComparison>,
623
- options?: { labels?: readonly Label[] },
624
- );
691
+ constructor(scorer: Scorer<LabelComparison>);
625
692
  }
626
693
 
627
- /** The classes that an average reads. */
694
+ /** The classes that an average reads: the counts of each class of the class set. */
628
695
  export type Classes = {
629
- readonly selected: ReadonlyMap<Label, Counts>;
630
- readonly present: ReadonlyMap<Label, Counts>;
696
+ readonly counts: ReadonlyMap<Label, Counts>;
631
697
  readonly multilabel: boolean;
632
698
  };
633
699
  /** How the scores of the classes become one number (Strategy). */
@@ -643,14 +709,14 @@ export class BinaryAverage implements Average {
643
709
 
644
710
  type ScoreOptions = {
645
711
  average: Average;
646
- labels?: readonly Label[];
647
712
  /** 0 (the default), 1 or NaN. */
648
713
  zeroDivision?: number;
649
714
  };
650
- /** The counts of each class over many answers: true positives, predicted and true. */
715
+ /** The counts of each class of the class set over many answers: true positives, predicted and true. */
651
716
  export class ClassCounts {
652
- constructor(options: { multilabel: boolean });
717
+ constructor(options: { labels: readonly Label[]; multilabel: boolean });
653
718
  static of(answers: {
719
+ labels: readonly Label[];
654
720
  expected: readonly Label[];
655
721
  stated: readonly Label[];
656
722
  weights?: readonly number[];
@@ -661,6 +727,7 @@ export class ClassCounts {
661
727
  weights?: readonly number[];
662
728
  }): ClassCounts;
663
729
  add(comparison: LabelComparison, weight: number): void;
730
+ /** The class set, in the order that the caller gives. */
664
731
  labels(): Label[];
665
732
  accuracy(): number;
666
733
  precision(options: ScoreOptions): number;
@@ -726,4 +793,6 @@ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the
726
793
  - [ADR 0001: one package, with a subpath for each shape of answer](docs/adr/0001-one-package-with-a-subpath-for-each-shape-of-answer.md)
727
794
  - [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
728
795
  - [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
729
- - [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md) (design for review)
796
+ - [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)
797
+ - [ADR 0005: a failed call is the answer null](docs/adr/0005-a-failed-call-is-the-answer-null.md)
798
+ - [ADR 0006: classification counts only the classes that the caller gives](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md)
@@ -2,6 +2,6 @@ import { Counts } from '../aggregation/counts.ts';
2
2
  import type { Matches } from '../scorers/matches.ts';
3
3
  import type { LabelComparison } from './label-comparison.ts';
4
4
  export declare class AllClassMatches implements Matches<LabelComparison> {
5
- counts({ stated, expected }: LabelComparison): Counts;
5
+ counts(comparison: LabelComparison): Counts;
6
6
  }
7
7
  //# sourceMappingURL=all-class-matches.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAOtD"}
1
+ {"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAM7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM;CAU5C"}
@@ -1,9 +1,13 @@
1
1
  import { Counts } from "../aggregation/counts.js";
2
- // The counts of every class in one answer: the hits are the labels that both
3
- // sides have, the stated and the expected are the labels of each side.
4
- // Pooled over the answers, they are scikit-learn's micro counts.
2
+ // The counts of the classes of the class set in one answer: the hits are the
3
+ // classes that both sides have, the stated and the expected are the classes
4
+ // of each side. A label outside the class set counts in no class. Pooled over
5
+ // the answers, they are scikit-learn's micro counts.
5
6
  export class AllClassMatches {
6
- counts({ stated, expected }) {
7
+ counts(comparison) {
8
+ const classes = new Set(comparison.labels());
9
+ const stated = comparison.stated.intersection(classes);
10
+ const expected = comparison.expected.intersection(classes);
7
11
  return new Counts({
8
12
  hits: stated.intersection(expected).size,
9
13
  stated: stated.size,
@@ -1 +1 @@
1
- {"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,qDAAqD;AACrD,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,UAA2B;QAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3D,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -1,8 +1,7 @@
1
1
  import type { Counts } from '../aggregation/counts.ts';
2
2
  import type { Label } from './label.ts';
3
3
  export type Classes = {
4
- readonly selected: ReadonlyMap<Label, Counts>;
5
- readonly present: ReadonlyMap<Label, Counts>;
4
+ readonly counts: ReadonlyMap<Label, Counts>;
6
5
  readonly multilabel: boolean;
7
6
  };
8
7
  export interface Average {
@@ -1 +1 @@
1
- {"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
1
+ {"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
@@ -1,9 +1,9 @@
1
- import { Counts } from '../aggregation/counts.ts';
1
+ import type { Counts } from '../aggregation/counts.ts';
2
2
  import type { Average, Classes } from './average.ts';
3
3
  import type { Label } from './label.ts';
4
4
  export declare class BinaryAverage implements Average {
5
5
  #private;
6
6
  constructor(positiveLabel: Label);
7
- of({ present, multilabel }: Classes, score: (counts: Counts) => number): number;
7
+ of({ counts, multilabel }: Classes, score: (counts: Counts) => number): number;
8
8
  }
9
9
  //# sourceMappingURL=binary-average.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,EAChC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAYV"}
1
+ {"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,OAAO,EAC/B,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAgBV"}
@@ -1,20 +1,21 @@
1
- import { Counts } from "../aggregation/counts.js";
2
- // average='binary': the score of the positive class, for answers of two
3
- // classes or fewer. It throws as scikit-learn raises: for a set of classes,
4
- // for more than two classes, and for a positive label that is not one of two
5
- // classes. A positive label that no answer has scores counts of 0.
1
+ // average='binary': the score of the positive class, for a class set of two
2
+ // classes or fewer. It throws as scikit-learn raises: for a set of classes
3
+ // (multilabel), for a class set of more than two classes, and for a positive
4
+ // label outside the class set.
6
5
  export class BinaryAverage {
7
6
  #positiveLabel;
8
7
  constructor(positiveLabel) {
9
8
  this.#positiveLabel = positiveLabel;
10
9
  }
11
- of({ present, multilabel }, score) {
12
- if (multilabel || present.size > 2)
13
- throw new RangeError(`The target is ${multilabel ? 'multilabel-indicator' : 'multiclass'}, but the average is binary: choose another average.`);
14
- const counts = present.get(this.#positiveLabel);
15
- if (counts === undefined && present.size === 2)
16
- throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not one of the labels ${JSON.stringify([...present.keys()])}.`);
17
- return score(counts ?? new Counts({ hits: 0, stated: 0, expected: 0 }));
10
+ of({ counts, multilabel }, score) {
11
+ if (multilabel)
12
+ throw new RangeError('The target is multilabel-indicator, but the average is binary: choose another average.');
13
+ if (counts.size > 2)
14
+ throw new RangeError(`The class set has ${counts.size} classes, but the average is binary: choose another average.`);
15
+ const positive = counts.get(this.#positiveLabel);
16
+ if (positive === undefined)
17
+ throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not in the class set ${JSON.stringify([...counts.keys()])}.`);
18
+ return score(positive);
18
19
  }
19
20
  }
20
21
  //# sourceMappingURL=binary-average.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,wEAAwE;AACxE,4EAA4E;AAC5E,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,OAAO,EAAE,UAAU,EAAW,EAChC,KAAiC;QAEjC,IAAI,UAAU,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC;YAChC,MAAM,IAAI,UAAU,CAClB,iBAAiB,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,sDAAsD,CAC1H,CAAC;QACJ,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAC5C,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC7H,CAAC;QACJ,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
1
+ {"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAIA,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,+BAA+B;AAC/B,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAW,EAC/B,KAAiC;QAEjC,IAAI,UAAU;YACZ,MAAM,IAAI,UAAU,CAClB,wFAAwF,CACzF,CAAC;QACJ,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;YACjB,MAAM,IAAI,UAAU,CAClB,qBAAqB,MAAM,CAAC,IAAI,8DAA8D,CAC/F,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjD,IAAI,QAAQ,KAAK,SAAS;YACxB,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC3H,CAAC;QACJ,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;CACF"}
@@ -2,18 +2,19 @@ import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
2
2
  import type { Average } from './average.ts';
3
3
  import { ConfusionMatrix } from './confusion-matrix.ts';
4
4
  import { LabelComparison } from './label-comparison.ts';
5
- import { type Label } from './label.ts';
5
+ import type { Label } from './label.ts';
6
6
  type ScoreOptions = {
7
7
  readonly average: Average;
8
- readonly labels?: readonly Label[];
9
8
  readonly zeroDivision?: 0 | 1 | number;
10
9
  };
11
10
  export declare class ClassCounts {
12
11
  #private;
13
- constructor({ multilabel }: {
12
+ constructor({ labels, multilabel, }: {
13
+ readonly labels: readonly Label[];
14
14
  readonly multilabel: boolean;
15
15
  });
16
- static of({ expected, stated, weights, }: {
16
+ static of({ labels, expected, stated, weights, }: {
17
+ readonly labels: readonly Label[];
17
18
  readonly expected: readonly Label[];
18
19
  readonly stated: readonly Label[];
19
20
  readonly weights?: readonly number[];
@@ -1 +1 @@
1
- {"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
1
+ {"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAIxC,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AA2BF,qBAAa,WAAW;;gBAOV,EACV,MAAM,EACN,UAAU,GACX,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;KAC9B;IAOD,MAAM,CAAC,EAAE,CAAC,EACR,MAAM,EACN,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAqBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAyDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IA+BtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAwBnC"}
@@ -3,39 +3,48 @@ import { Counts } from "../aggregation/counts.js";
3
3
  import { CountsForm, } from "../aggregation/f-measure-form.js";
4
4
  import { ConfusionMatrix } from "./confusion-matrix.js";
5
5
  import { LabelComparison } from "./label-comparison.js";
6
- import { compareLabels } from "./label.js";
7
6
  // The running counts of one class, added in the order of the answers.
8
7
  class Tally {
9
8
  hits = 0;
10
9
  stated = 0;
11
10
  expected = 0;
11
+ counts() {
12
+ return new Counts(this);
13
+ }
12
14
  }
13
- // The counts of each class over many answers, as scikit-learn's
14
- // multilabel_confusion_matrix gives them: the true positives (hits), the
15
- // predicted (stated) and the true (expected), each the sum of the weights of
16
- // the answers in their order, as numpy's bincount adds them. Add each
15
+ // The counts of each class of the class set over many answers, as
16
+ // scikit-learn's multilabel_confusion_matrix gives them with its labels: the
17
+ // true positives (hits), the predicted (stated) and the true (expected), each
18
+ // the sum of the weights of the answers in their order, as numpy's bincount
19
+ // adds them. Add each
17
20
  // comparison, then read the metrics of a whole run (torchmetrics' update and
18
21
  // compute). It also keeps the weight of each right answer for the accuracy,
19
22
  // and the weight of each pair of labels for the confusion matrix. The counts
20
23
  // are of one class for each answer (multiclass) or of a set of classes
21
24
  // (multilabel), as torchmetrics has a multiclass and a multilabel class.
25
+ // The class set is the classes of the task, in the order that the caller
26
+ // gives (scikit-learn's labels). A label outside it counts in no class, on
27
+ // either side, and the confusion matrix has no pair with it. The accuracy
28
+ // still reads it: the answer is wrong.
22
29
  export class ClassCounts {
23
30
  #multilabel;
24
- #classes = new Map();
31
+ #classes;
25
32
  #rights = [];
26
33
  #weights = [];
27
34
  #pairs = new Map();
28
- constructor({ multilabel }) {
35
+ constructor({ labels, multilabel, }) {
29
36
  this.#multilabel = multilabel;
37
+ this.#classes = new Map(labels.map((label) => [label, new Tally()]));
30
38
  }
31
39
  // The counts of two arrays of labels, one label for each answer
32
40
  // (multiclass), with the weight of each answer (1 when none is given).
33
- static of({ expected, stated, weights, }) {
34
- const counts = new ClassCounts({ multilabel: false });
41
+ static of({ labels, expected, stated, weights, }) {
42
+ const counts = new ClassCounts({ labels, multilabel: false });
35
43
  for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
36
44
  counts.add(new LabelComparison({
37
45
  stated: new Set([statedLabel]),
38
46
  expected: new Set([expectedLabel]),
47
+ labels,
39
48
  }), weight);
40
49
  return counts;
41
50
  }
@@ -44,9 +53,8 @@ export class ClassCounts {
44
53
  // when no answer has it.
45
54
  static ofIndicators({ expected, stated, weights, }) {
46
55
  const columns = expected[0]?.length ?? 0;
47
- const counts = new ClassCounts({ multilabel: true });
48
- for (let column = 0; column < columns; column++)
49
- counts.#classes.set(column, new Tally());
56
+ const classes = Array.from({ length: columns }, (_, column) => column);
57
+ const counts = new ClassCounts({ labels: classes, multilabel: true });
50
58
  const labels = (row) => {
51
59
  if (row.length !== columns ||
52
60
  row.some((value) => value !== 0 && value !== 1))
@@ -57,6 +65,7 @@ export class ClassCounts {
57
65
  counts.add(new LabelComparison({
58
66
  stated: labels(statedRow),
59
67
  expected: labels(expectedRow),
68
+ labels: classes,
60
69
  }), weight);
61
70
  return counts;
62
71
  }
@@ -74,14 +83,17 @@ export class ClassCounts {
74
83
  yield [side, other, weights?.[index] ?? 1];
75
84
  }
76
85
  }
77
- // Adds one answer with its weight. Multiclass counts refuse an answer with
78
- // no label or more than one label on a side.
86
+ // Adds one answer with its weight, to the classes of the class set of the
87
+ // counts. Multiclass counts refuse an answer with no label or more than one
88
+ // label on a side.
79
89
  add(comparison, weight) {
80
90
  const { stated, expected } = comparison;
81
91
  if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
82
92
  throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
83
- for (const label of comparison.labels()) {
84
- const tally = this.#classes.getOrInsertComputed(label, () => new Tally());
93
+ for (const label of stated.union(expected)) {
94
+ const tally = this.#classes.get(label);
95
+ if (tally === undefined)
96
+ continue;
85
97
  if (stated.has(label))
86
98
  tally.stated += weight;
87
99
  if (expected.has(label))
@@ -96,14 +108,17 @@ export class ClassCounts {
96
108
  return;
97
109
  const [expectedLabel] = expected;
98
110
  const [statedLabel] = stated;
99
- if (expectedLabel === undefined || statedLabel === undefined)
111
+ if (expectedLabel === undefined ||
112
+ statedLabel === undefined ||
113
+ !this.#classes.has(expectedLabel) ||
114
+ !this.#classes.has(statedLabel))
100
115
  return;
101
116
  const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
102
117
  row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
103
118
  }
104
- // The labels of the classes, in the order of numpy's unique().
119
+ // The class set, in the order that the caller gives.
105
120
  labels() {
106
- return [...this.#classes.keys()].toSorted(compareLabels);
121
+ return [...this.#classes.keys()];
107
122
  }
108
123
  // accuracy_score: the weight of the right answers over the weight of all
109
124
  // answers, each with numpy's pairwise sum. An answer is right when its
@@ -128,31 +143,24 @@ export class ClassCounts {
128
143
  ? undefined
129
144
  : form.of(counts, beta));
130
145
  }
131
- // The true count of each class, in the order of the labels.
146
+ // The true count of each class of the class set, in its order.
132
147
  support() {
133
- return new Map(this.labels().map((label) => [label, this.#counts(label).expected]));
148
+ return new Map(this.#classes.entries().map(([label, { expected }]) => [label, expected]));
134
149
  }
135
- // confusion_matrix of the answers, with the labels in their order. A set
150
+ // confusion_matrix of the answers, with the class set in its order. A set
136
151
  // of classes has no confusion matrix (scikit-learn raises), so it throws.
137
152
  confusionMatrix() {
138
153
  if (this.#multilabel)
139
154
  throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
140
155
  return new ConfusionMatrix(this.labels(), this.#pairs);
141
156
  }
142
- #score({ average, labels = this.labels(), zeroDivision = 0 }, score) {
157
+ #score({ average, zeroDivision = 0 }, score) {
143
158
  return average.of({
144
- selected: new Map(labels.map((label) => [label, this.#counts(label)])),
145
- present: new Map(this.labels().map((label) => [label, this.#counts(label)])),
159
+ counts: new Map(this.#classes
160
+ .entries()
161
+ .map(([label, tally]) => [label, tally.counts()])),
146
162
  multilabel: this.#multilabel,
147
163
  }, (counts) => score(counts) ?? zeroDivision);
148
164
  }
149
- #counts(label) {
150
- const tally = this.#classes.get(label);
151
- return new Counts({
152
- hits: tally?.hits ?? 0,
153
- stated: tally?.stated ?? 0,
154
- expected: tally?.expected ?? 0,
155
- });
156
- }
157
165
  }
158
166
  //# sourceMappingURL=class-counts.js.map