@zukhruf/evals 0.8.7 → 0.8.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -37
- package/dist/classification/all-class-matches.d.ts +1 -1
- package/dist/classification/all-class-matches.d.ts.map +1 -1
- package/dist/classification/all-class-matches.js +8 -4
- package/dist/classification/all-class-matches.js.map +1 -1
- package/dist/classification/average.d.ts +1 -2
- package/dist/classification/average.d.ts.map +1 -1
- package/dist/classification/binary-average.d.ts +2 -2
- package/dist/classification/binary-average.d.ts.map +1 -1
- package/dist/classification/binary-average.js +13 -12
- package/dist/classification/binary-average.js.map +1 -1
- package/dist/classification/class-counts.d.ts +5 -4
- package/dist/classification/class-counts.d.ts.map +1 -1
- package/dist/classification/class-counts.js +41 -33
- package/dist/classification/class-counts.js.map +1 -1
- package/dist/classification/label-comparer.d.ts +4 -0
- package/dist/classification/label-comparer.d.ts.map +1 -1
- package/dist/classification/label-comparer.js +10 -2
- package/dist/classification/label-comparer.js.map +1 -1
- package/dist/classification/label-comparison.d.ts +4 -2
- package/dist/classification/label-comparison.d.ts.map +1 -1
- package/dist/classification/label-comparison.js +11 -6
- package/dist/classification/label-comparison.js.map +1 -1
- package/dist/classification/macro-average.d.ts +1 -1
- package/dist/classification/macro-average.d.ts.map +1 -1
- package/dist/classification/macro-average.js +3 -3
- package/dist/classification/macro-average.js.map +1 -1
- package/dist/classification/micro-average.d.ts +1 -1
- package/dist/classification/micro-average.d.ts.map +1 -1
- package/dist/classification/micro-average.js +4 -4
- package/dist/classification/micro-average.js.map +1 -1
- package/dist/classification/per-class.d.ts +0 -6
- package/dist/classification/per-class.d.ts.map +1 -1
- package/dist/classification/per-class.js +4 -10
- package/dist/classification/per-class.js.map +1 -1
- package/dist/classification/weighted-average.d.ts +1 -1
- package/dist/classification/weighted-average.d.ts.map +1 -1
- package/dist/classification/weighted-average.js +3 -3
- package/dist/classification/weighted-average.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/failed-call.d.ts +13 -0
- package/dist/promptfoo/failed-call.d.ts.map +1 -0
- package/dist/promptfoo/failed-call.js +24 -0
- package/dist/promptfoo/failed-call.js.map +1 -0
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/scoring/grouped.d.ts +3 -3
- package/dist/scoring/grouped.d.ts.map +1 -1
- package/dist/scoring/grouped.js +7 -4
- package/dist/scoring/grouped.js.map +1 -1
- package/dist/scoring/paired-bootstrap.d.ts +24 -0
- package/dist/scoring/paired-bootstrap.d.ts.map +1 -0
- package/dist/scoring/paired-bootstrap.js +94 -0
- package/dist/scoring/paired-bootstrap.js.map +1 -0
- package/dist/scoring/run.d.ts +7 -0
- package/dist/scoring/run.d.ts.map +1 -0
- package/dist/scoring/run.js +19 -0
- package/dist/scoring/run.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +1 -1
- package/dist/scoring/scorer-set.d.ts.map +1 -1
- package/dist/scoring/scorer-set.js.map +1 -1
- package/dist/scoring/test-epochs.d.ts +11 -0
- package/dist/scoring/test-epochs.d.ts.map +1 -0
- package/dist/scoring/test-epochs.js +25 -0
- package/dist/scoring/test-epochs.js.map +1 -0
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
LabelVarsCase,
|
|
32
32
|
} from '@zukhruf/evals/classification';
|
|
33
33
|
|
|
34
|
-
const comparer = new LabelComparer();
|
|
34
|
+
const comparer = new LabelComparer({ labels: ['program', 'home', 'list'] });
|
|
35
35
|
const testCase = new LabelVarsCase().from({ expected: 'program' });
|
|
36
36
|
const comparison = comparer.compare('home', testCase);
|
|
37
37
|
|
|
@@ -58,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
|
|
|
58
58
|
|
|
59
59
|
## Score with many scorers at once
|
|
60
60
|
|
|
61
|
-
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
|
|
61
|
+
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer({ labels }) })`. It makes the input once, and each scorer reads it.
|
|
62
62
|
|
|
63
|
-
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
|
|
63
|
+
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way). A group can also be a projection of the input, of another type: `Grouped<Input, Group>` takes a `Scorer<Group>`, such as a count of the stated items without a pair, for each of their kinds. A grouped scorer is a breakdown, so its ratios stay out of the score of the record, which is the mean of the plain scorers.
|
|
64
64
|
|
|
65
65
|
```ts
|
|
66
66
|
import { Recall, ScorerSet } from '@zukhruf/evals';
|
|
@@ -75,7 +75,7 @@ const pageKinds = new ScorerSet(
|
|
|
75
75
|
accuracy: new LabelsMatch(),
|
|
76
76
|
recall: new Recall(new AllClassMatches()),
|
|
77
77
|
},
|
|
78
|
-
{ comparer: new LabelComparer() },
|
|
78
|
+
{ comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }) },
|
|
79
79
|
);
|
|
80
80
|
```
|
|
81
81
|
|
|
@@ -94,7 +94,7 @@ const pageKinds = new ScorerSet(
|
|
|
94
94
|
|
|
95
95
|
## Compare two runs
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
The package has `Run` and `PairedBootstrap` (backlog #3870, [ADR 0004](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)).
|
|
98
98
|
|
|
99
99
|
A caller runs one harness two times on the same tests: once with the baseline model and once with a change, such as another model. Two means side by side do not say if the change is better. `PairedBootstrap` gives one verdict for each name: better, worse or inconclusive, with the difference, its interval and a p-value.
|
|
100
100
|
|
|
@@ -146,7 +146,7 @@ const brier = paired.verdict(baseline, change, 'brier', {
|
|
|
146
146
|
A run holds the scorecards of one model. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
|
|
147
147
|
|
|
148
148
|
```ts
|
|
149
|
-
// @zukhruf/evals: two runs side by side
|
|
149
|
+
// @zukhruf/evals: two runs side by side
|
|
150
150
|
import type {
|
|
151
151
|
ConfidenceIntervalMethod,
|
|
152
152
|
DifferenceInterval,
|
|
@@ -224,22 +224,22 @@ It needs no peer: `npm install @zukhruf/evals`. Its sums are `sum` and `mean` of
|
|
|
224
224
|
|
|
225
225
|
A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
|
|
226
226
|
|
|
227
|
-
`LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`.
|
|
227
|
+
`new LabelComparer({ labels })` reads the stated labels of an answer and gives a `LabelComparison`. `labels` is the class set (see [The class set](#the-class-set)), and each comparison holds it. The comparer reads a label or a list of labels as they are. It reads text in this order:
|
|
228
228
|
|
|
229
229
|
1. A JSON array of labels gives those labels.
|
|
230
230
|
2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
|
|
231
231
|
3. Other text is one string label, without its white space at the start and the end.
|
|
232
232
|
|
|
233
|
-
`comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
|
|
233
|
+
`comparison.labels()` gives the class set, in the order that the caller gives. `comparison.only(label)` gives the comparison of one class: its class set is `[label]`, and each side keeps the label when it has it.
|
|
234
234
|
|
|
235
235
|
The matches of a comparison count one answer:
|
|
236
236
|
|
|
237
237
|
| Matches | Hits | Stated | Expected |
|
|
238
238
|
| ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
|
|
239
239
|
| `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
|
|
240
|
-
| `new AllClassMatches()` | the
|
|
240
|
+
| `new AllClassMatches()` | the classes that both sides have | the stated classes | the expected classes |
|
|
241
241
|
|
|
242
|
-
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer
|
|
242
|
+
`AllClassMatches` counts only the classes of the class set of the comparison. `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer)` gives one ratio for each class of the class set, under the label `${class} ${label}`, in the order of the class set.
|
|
243
243
|
|
|
244
244
|
`Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
|
|
245
245
|
|
|
@@ -263,6 +263,7 @@ import {
|
|
|
263
263
|
} from '@zukhruf/evals/classification';
|
|
264
264
|
|
|
265
265
|
const counts = ClassCounts.of({
|
|
266
|
+
labels: ['a', 'b'],
|
|
266
267
|
expected: ['a', 'b', 'b'],
|
|
267
268
|
stated: ['a', 'a', 'b'],
|
|
268
269
|
});
|
|
@@ -270,23 +271,38 @@ const counts = ClassCounts.of({
|
|
|
270
271
|
counts.accuracy(); // accuracy_score
|
|
271
272
|
counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
|
|
272
273
|
counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
|
|
273
|
-
counts.confusionMatrix().rows(); // confusion_matrix,
|
|
274
|
+
counts.confusionMatrix().rows(); // confusion_matrix, the classes in the order of labels
|
|
274
275
|
|
|
275
276
|
// The same counts, one answer at a time:
|
|
276
|
-
const run = new ClassCounts({ multilabel: false });
|
|
277
|
+
const run = new ClassCounts({ labels: ['a', 'b'], multilabel: false });
|
|
277
278
|
const testCase = new LabelCase({ expected: 'a', weight: 1 });
|
|
278
|
-
|
|
279
|
+
const comparer = new LabelComparer({ labels: ['a', 'b'] });
|
|
280
|
+
run.add(comparer.compare('a', testCase), testCase.weight);
|
|
279
281
|
```
|
|
280
282
|
|
|
281
|
-
- `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
|
|
283
|
+
- `ClassCounts.of({ labels, expected, stated, weights })` builds multiclass counts from two arrays of labels. `labels` is the class set. `weights` is optional, and an answer without a weight weighs 1.
|
|
282
284
|
- `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
|
|
283
|
-
- `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
284
|
-
- `
|
|
285
|
-
- `
|
|
285
|
+
- `new ClassCounts({ labels, multilabel })` starts empty, and `add(comparison, weight)` adds one answer to the classes of the class set of the counts. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
286
|
+
- `labels()`, `support()` and `confusionMatrix()` give the classes of the class set, in its order.
|
|
287
|
+
- `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` reads the class set. It throws for multilabel counts, for a class set of more than two classes, and for a positive label outside the class set. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
|
|
286
288
|
- `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
|
|
287
289
|
- `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
|
|
288
290
|
- `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
|
|
289
291
|
|
|
292
|
+
### The class set
|
|
293
|
+
|
|
294
|
+
[ADR 0006](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md) gives this rule. A harness can state any text, such as the answer null of a failed call. If the answers give the classes, each such text adds a class, and the macro averages change. So the caller gives the class set, scikit-learn's `labels`, to the comparer and to the counts. The answers never add a class.
|
|
295
|
+
|
|
296
|
+
```ts
|
|
297
|
+
import { ClassCounts, LabelComparer } from '@zukhruf/evals/classification';
|
|
298
|
+
|
|
299
|
+
const pageKinds = ['program', 'home', 'list', 'investor', 'other'];
|
|
300
|
+
const comparer = new LabelComparer({ labels: pageKinds });
|
|
301
|
+
const counts = new ClassCounts({ labels: pageKinds, multilabel: false });
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
A label outside the class set counts in no class, as scikit-learn gives it when the caller gives `labels`. A stated label outside it is a miss of the expected class, and it is not a predicted count of any class. `LabelsMatch` still reads it as a wrong answer. `PerClass` and `AllClassMatches` read the class set of the comparison, so they take no `labels`. `ClassCounts.ofIndicators` takes its classes from the columns. To port a scikit-learn call with `labels=None`, give the sorted labels of both sides.
|
|
305
|
+
|
|
290
306
|
### Probabilities
|
|
291
307
|
|
|
292
308
|
`ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
|
|
@@ -314,7 +330,10 @@ import {
|
|
|
314
330
|
} from '@zukhruf/evals/classification';
|
|
315
331
|
import { AssertionModule, ScorerAssertion } from '@zukhruf/evals/promptfoo';
|
|
316
332
|
|
|
317
|
-
const labels = {
|
|
333
|
+
const labels = {
|
|
334
|
+
comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }),
|
|
335
|
+
cases: new LabelVarsCase(),
|
|
336
|
+
};
|
|
318
337
|
|
|
319
338
|
export const scorers = new AssertionModule(import.meta.url, {
|
|
320
339
|
accuracy: new ScorerAssertion(new LabelsMatch(), {
|
|
@@ -344,6 +363,51 @@ export default {
|
|
|
344
363
|
|
|
345
364
|
The name is the promptfoo `metric`, unless the scorer assertion gives its own `metric`. A name that the module does not hold is a compile error. promptfoo shows the mean of each `metric` over the tests. A `ScorerAssertion` gives one value for one test, so a micro average needs `PromptfooAssertion`, whose ratios keep their wholes.
|
|
346
365
|
|
|
366
|
+
### A failed call
|
|
367
|
+
|
|
368
|
+
[ADR 0005](docs/adr/0005-a-failed-call-is-the-answer-null.md) gives the reasons (backlog #3970).
|
|
369
|
+
|
|
370
|
+
A failed call is a model call that failed for its one test, also after its retries. promptfoo does not grade a result whose provider gives `error` or no output. That result is an error result: it has no named scores, so its case adds nothing to the wholes, and the recall goes up. A failed call is an answer of the harness: the answer null, the text `'null'`. promptfoo grades it, and each family reads it as no answer. `FailedCall` is a coinage of this package. A consumer uses one of two forms.
|
|
371
|
+
|
|
372
|
+
When the provider is your own, return `new FailedCall(reason).response(metadata)`. promptfoo grades the result, and `promptfoo retry <evalId>` does not run it again. The reason is in `metadata.failed`.
|
|
373
|
+
|
|
374
|
+
```ts
|
|
375
|
+
import { FailedCall } from '@zukhruf/evals/promptfoo';
|
|
376
|
+
|
|
377
|
+
const [result] = await rule.compute([row.input], options?.abortSignal);
|
|
378
|
+
if (result !== undefined && 'error' in result)
|
|
379
|
+
return new FailedCall(result.error).response(metadata);
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
When the provider is a built-in one, such as `openai:chat`, an `afterEach` extension hook grades each error result on `FailedCall.answer`. The result stays an error result, so `promptfoo retry <evalId>` and `--retry-errors` run it again.
|
|
383
|
+
|
|
384
|
+
```ts
|
|
385
|
+
import {
|
|
386
|
+
type AfterEachExtensionHookContext,
|
|
387
|
+
ResultFailureReason,
|
|
388
|
+
assertions,
|
|
389
|
+
} from 'promptfoo';
|
|
390
|
+
|
|
391
|
+
import { FailedCall } from '@zukhruf/evals/promptfoo';
|
|
392
|
+
|
|
393
|
+
export async function afterEach({
|
|
394
|
+
test,
|
|
395
|
+
result,
|
|
396
|
+
}: AfterEachExtensionHookContext): Promise<void> {
|
|
397
|
+
if (result.failureReason !== ResultFailureReason.ERROR) return;
|
|
398
|
+
const graded = await assertions.runAssertions({
|
|
399
|
+
test,
|
|
400
|
+
providerResponse: { output: FailedCall.answer },
|
|
401
|
+
vars: test.vars ?? {},
|
|
402
|
+
});
|
|
403
|
+
Object.assign(result.namedScores, graded.namedScores);
|
|
404
|
+
}
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
A hook cannot write `namedScoreWeights`, so promptfoo gives each score of the hook the weight 1. Use the hook only with scorer assertions whose metrics weigh 1, as in SOB. A `PromptfooAssertion` gives each ratio its whole as the weight, so it needs the first form.
|
|
408
|
+
|
|
409
|
+
A failure of the run, such as a 401, is not a failed call: each call would fail. Throw it. The result then stays an error result, and `promptfoo retry <evalId>` runs it again.
|
|
410
|
+
|
|
347
411
|
## What the package does not do
|
|
348
412
|
|
|
349
413
|
- It does not run a harness, load a dataset, store results or show them. promptfoo does that.
|
|
@@ -480,10 +544,10 @@ export interface ScorecardScorer<Case extends Weighted> {
|
|
|
480
544
|
score(stated: unknown, testCase: Case): Scorecard;
|
|
481
545
|
}
|
|
482
546
|
/** Gives one ratio for each group of an input, such as each field or each class. */
|
|
483
|
-
export abstract class Grouped<Input> {
|
|
484
|
-
constructor(scorer: Scorer<
|
|
485
|
-
/** The groups of an input (Template Method). */
|
|
486
|
-
protected abstract groups(input: Input): Iterable<[string,
|
|
547
|
+
export abstract class Grouped<Input, Group = Input> {
|
|
548
|
+
constructor(scorer: Scorer<Group>);
|
|
549
|
+
/** The groups of an input (Template Method): a part of it, or a projection of another type. */
|
|
550
|
+
protected abstract groups(input: Input): Iterable<[string, Group]>;
|
|
487
551
|
/** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
|
|
488
552
|
ratios(input: Input, label: string): [string, Ratio][];
|
|
489
553
|
}
|
|
@@ -492,7 +556,7 @@ export class ScorerSet<
|
|
|
492
556
|
Input,
|
|
493
557
|
> implements ScorecardScorer<Case> {
|
|
494
558
|
constructor(
|
|
495
|
-
scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>,
|
|
559
|
+
scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input, unknown>>>,
|
|
496
560
|
options: { comparer: Comparer<Case, Input> },
|
|
497
561
|
);
|
|
498
562
|
}
|
|
@@ -600,34 +664,36 @@ export class LabelComparison {
|
|
|
600
664
|
constructor(sides: {
|
|
601
665
|
stated: ReadonlySet<Label>;
|
|
602
666
|
expected: ReadonlySet<Label>;
|
|
667
|
+
labels: readonly Label[];
|
|
603
668
|
});
|
|
604
669
|
readonly stated: ReadonlySet<Label>;
|
|
605
670
|
readonly expected: ReadonlySet<Label>;
|
|
606
|
-
/** The
|
|
671
|
+
/** The class set, in the order that the caller gives. */
|
|
607
672
|
labels(): Label[];
|
|
608
|
-
/** The comparison of one class. */
|
|
673
|
+
/** The comparison of one class: its class set is [label]. */
|
|
609
674
|
only(label: Label): LabelComparison;
|
|
610
675
|
}
|
|
611
676
|
/** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
|
|
612
|
-
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
677
|
+
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
678
|
+
/** labels: the class set. */
|
|
679
|
+
constructor(options: { labels: readonly Label[] });
|
|
680
|
+
}
|
|
613
681
|
|
|
614
682
|
export class ClassMatches implements Matches<LabelComparison> {
|
|
615
683
|
constructor(label: Label);
|
|
616
684
|
}
|
|
685
|
+
/** Counts only the classes of the class set of the comparison. */
|
|
617
686
|
export class AllClassMatches implements Matches<LabelComparison> {}
|
|
618
687
|
/** Holds when the stated labels are the expected labels. */
|
|
619
688
|
export class LabelsMatch extends Gate<LabelComparison> {}
|
|
689
|
+
/** A ratio for each class of the class set of the comparison. */
|
|
620
690
|
export class PerClass extends Grouped<LabelComparison> {
|
|
621
|
-
constructor(
|
|
622
|
-
scorer: Scorer<LabelComparison>,
|
|
623
|
-
options?: { labels?: readonly Label[] },
|
|
624
|
-
);
|
|
691
|
+
constructor(scorer: Scorer<LabelComparison>);
|
|
625
692
|
}
|
|
626
693
|
|
|
627
|
-
/** The classes that an average reads. */
|
|
694
|
+
/** The classes that an average reads: the counts of each class of the class set. */
|
|
628
695
|
export type Classes = {
|
|
629
|
-
readonly
|
|
630
|
-
readonly present: ReadonlyMap<Label, Counts>;
|
|
696
|
+
readonly counts: ReadonlyMap<Label, Counts>;
|
|
631
697
|
readonly multilabel: boolean;
|
|
632
698
|
};
|
|
633
699
|
/** How the scores of the classes become one number (Strategy). */
|
|
@@ -643,14 +709,14 @@ export class BinaryAverage implements Average {
|
|
|
643
709
|
|
|
644
710
|
type ScoreOptions = {
|
|
645
711
|
average: Average;
|
|
646
|
-
labels?: readonly Label[];
|
|
647
712
|
/** 0 (the default), 1 or NaN. */
|
|
648
713
|
zeroDivision?: number;
|
|
649
714
|
};
|
|
650
|
-
/** The counts of each class over many answers: true positives, predicted and true. */
|
|
715
|
+
/** The counts of each class of the class set over many answers: true positives, predicted and true. */
|
|
651
716
|
export class ClassCounts {
|
|
652
|
-
constructor(options: { multilabel: boolean });
|
|
717
|
+
constructor(options: { labels: readonly Label[]; multilabel: boolean });
|
|
653
718
|
static of(answers: {
|
|
719
|
+
labels: readonly Label[];
|
|
654
720
|
expected: readonly Label[];
|
|
655
721
|
stated: readonly Label[];
|
|
656
722
|
weights?: readonly number[];
|
|
@@ -661,6 +727,7 @@ export class ClassCounts {
|
|
|
661
727
|
weights?: readonly number[];
|
|
662
728
|
}): ClassCounts;
|
|
663
729
|
add(comparison: LabelComparison, weight: number): void;
|
|
730
|
+
/** The class set, in the order that the caller gives. */
|
|
664
731
|
labels(): Label[];
|
|
665
732
|
accuracy(): number;
|
|
666
733
|
precision(options: ScoreOptions): number;
|
|
@@ -726,4 +793,6 @@ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the
|
|
|
726
793
|
- [ADR 0001: one package, with a subpath for each shape of answer](docs/adr/0001-one-package-with-a-subpath-for-each-shape-of-answer.md)
|
|
727
794
|
- [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
|
|
728
795
|
- [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
|
|
729
|
-
- [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)
|
|
796
|
+
- [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)
|
|
797
|
+
- [ADR 0005: a failed call is the answer null](docs/adr/0005-a-failed-call-is-the-answer-null.md)
|
|
798
|
+
- [ADR 0006: classification counts only the classes that the caller gives](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md)
|
|
@@ -2,6 +2,6 @@ import { Counts } from '../aggregation/counts.ts';
|
|
|
2
2
|
import type { Matches } from '../scorers/matches.ts';
|
|
3
3
|
import type { LabelComparison } from './label-comparison.ts';
|
|
4
4
|
export declare class AllClassMatches implements Matches<LabelComparison> {
|
|
5
|
-
counts(
|
|
5
|
+
counts(comparison: LabelComparison): Counts;
|
|
6
6
|
}
|
|
7
7
|
//# sourceMappingURL=all-class-matches.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAM7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM;CAU5C"}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { Counts } from "../aggregation/counts.js";
|
|
2
|
-
// The counts of
|
|
3
|
-
// sides have, the stated and the expected are the
|
|
4
|
-
//
|
|
2
|
+
// The counts of the classes of the class set in one answer: the hits are the
|
|
3
|
+
// classes that both sides have, the stated and the expected are the classes
|
|
4
|
+
// of each side. A label outside the class set counts in no class. Pooled over
|
|
5
|
+
// the answers, they are scikit-learn's micro counts.
|
|
5
6
|
export class AllClassMatches {
|
|
6
|
-
counts(
|
|
7
|
+
counts(comparison) {
|
|
8
|
+
const classes = new Set(comparison.labels());
|
|
9
|
+
const stated = comparison.stated.intersection(classes);
|
|
10
|
+
const expected = comparison.expected.intersection(classes);
|
|
7
11
|
return new Counts({
|
|
8
12
|
hits: stated.intersection(expected).size,
|
|
9
13
|
stated: stated.size,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,
|
|
1
|
+
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,qDAAqD;AACrD,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,UAA2B;QAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3D,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Label } from './label.ts';
|
|
3
3
|
export type Classes = {
|
|
4
|
-
readonly
|
|
5
|
-
readonly present: ReadonlyMap<Label, Counts>;
|
|
4
|
+
readonly counts: ReadonlyMap<Label, Counts>;
|
|
6
5
|
readonly multilabel: boolean;
|
|
7
6
|
};
|
|
8
7
|
export interface Average {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { Counts } from '../aggregation/counts.ts';
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Average, Classes } from './average.ts';
|
|
3
3
|
import type { Label } from './label.ts';
|
|
4
4
|
export declare class BinaryAverage implements Average {
|
|
5
5
|
#private;
|
|
6
6
|
constructor(positiveLabel: Label);
|
|
7
|
-
of({
|
|
7
|
+
of({ counts, multilabel }: Classes, score: (counts: Counts) => number): number;
|
|
8
8
|
}
|
|
9
9
|
//# sourceMappingURL=binary-average.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,OAAO,EAC/B,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAgBV"}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// classes. A positive label that no answer has scores counts of 0.
|
|
1
|
+
// average='binary': the score of the positive class, for a class set of two
|
|
2
|
+
// classes or fewer. It throws as scikit-learn raises: for a set of classes
|
|
3
|
+
// (multilabel), for a class set of more than two classes, and for a positive
|
|
4
|
+
// label outside the class set.
|
|
6
5
|
export class BinaryAverage {
|
|
7
6
|
#positiveLabel;
|
|
8
7
|
constructor(positiveLabel) {
|
|
9
8
|
this.#positiveLabel = positiveLabel;
|
|
10
9
|
}
|
|
11
|
-
of({
|
|
12
|
-
if (multilabel
|
|
13
|
-
throw new RangeError(
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
10
|
+
of({ counts, multilabel }, score) {
|
|
11
|
+
if (multilabel)
|
|
12
|
+
throw new RangeError('The target is multilabel-indicator, but the average is binary: choose another average.');
|
|
13
|
+
if (counts.size > 2)
|
|
14
|
+
throw new RangeError(`The class set has ${counts.size} classes, but the average is binary: choose another average.`);
|
|
15
|
+
const positive = counts.get(this.#positiveLabel);
|
|
16
|
+
if (positive === undefined)
|
|
17
|
+
throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not in the class set ${JSON.stringify([...counts.keys()])}.`);
|
|
18
|
+
return score(positive);
|
|
18
19
|
}
|
|
19
20
|
}
|
|
20
21
|
//# sourceMappingURL=binary-average.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAIA,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,+BAA+B;AAC/B,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAW,EAC/B,KAAiC;QAEjC,IAAI,UAAU;YACZ,MAAM,IAAI,UAAU,CAClB,wFAAwF,CACzF,CAAC;QACJ,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;YACjB,MAAM,IAAI,UAAU,CAClB,qBAAqB,MAAM,CAAC,IAAI,8DAA8D,CAC/F,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjD,IAAI,QAAQ,KAAK,SAAS;YACxB,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC3H,CAAC;QACJ,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -2,18 +2,19 @@ import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
|
2
2
|
import type { Average } from './average.ts';
|
|
3
3
|
import { ConfusionMatrix } from './confusion-matrix.ts';
|
|
4
4
|
import { LabelComparison } from './label-comparison.ts';
|
|
5
|
-
import {
|
|
5
|
+
import type { Label } from './label.ts';
|
|
6
6
|
type ScoreOptions = {
|
|
7
7
|
readonly average: Average;
|
|
8
|
-
readonly labels?: readonly Label[];
|
|
9
8
|
readonly zeroDivision?: 0 | 1 | number;
|
|
10
9
|
};
|
|
11
10
|
export declare class ClassCounts {
|
|
12
11
|
#private;
|
|
13
|
-
constructor({ multilabel }: {
|
|
12
|
+
constructor({ labels, multilabel, }: {
|
|
13
|
+
readonly labels: readonly Label[];
|
|
14
14
|
readonly multilabel: boolean;
|
|
15
15
|
});
|
|
16
|
-
static of({ expected, stated, weights, }: {
|
|
16
|
+
static of({ labels, expected, stated, weights, }: {
|
|
17
|
+
readonly labels: readonly Label[];
|
|
17
18
|
readonly expected: readonly Label[];
|
|
18
19
|
readonly stated: readonly Label[];
|
|
19
20
|
readonly weights?: readonly number[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAIxC,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AA2BF,qBAAa,WAAW;;gBAOV,EACV,MAAM,EACN,UAAU,GACX,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;KAC9B;IAOD,MAAM,CAAC,EAAE,CAAC,EACR,MAAM,EACN,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAqBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAyDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IA+BtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAwBnC"}
|
|
@@ -3,39 +3,48 @@ import { Counts } from "../aggregation/counts.js";
|
|
|
3
3
|
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
4
|
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
5
5
|
import { LabelComparison } from "./label-comparison.js";
|
|
6
|
-
import { compareLabels } from "./label.js";
|
|
7
6
|
// The running counts of one class, added in the order of the answers.
|
|
8
7
|
class Tally {
|
|
9
8
|
hits = 0;
|
|
10
9
|
stated = 0;
|
|
11
10
|
expected = 0;
|
|
11
|
+
counts() {
|
|
12
|
+
return new Counts(this);
|
|
13
|
+
}
|
|
12
14
|
}
|
|
13
|
-
// The counts of each class over many answers, as
|
|
14
|
-
// multilabel_confusion_matrix gives them
|
|
15
|
-
// predicted (stated) and the true (expected), each
|
|
16
|
-
// the answers in their order, as numpy's bincount
|
|
15
|
+
// The counts of each class of the class set over many answers, as
|
|
16
|
+
// scikit-learn's multilabel_confusion_matrix gives them with its labels: the
|
|
17
|
+
// true positives (hits), the predicted (stated) and the true (expected), each
|
|
18
|
+
// the sum of the weights of the answers in their order, as numpy's bincount
|
|
19
|
+
// adds them. Add each
|
|
17
20
|
// comparison, then read the metrics of a whole run (torchmetrics' update and
|
|
18
21
|
// compute). It also keeps the weight of each right answer for the accuracy,
|
|
19
22
|
// and the weight of each pair of labels for the confusion matrix. The counts
|
|
20
23
|
// are of one class for each answer (multiclass) or of a set of classes
|
|
21
24
|
// (multilabel), as torchmetrics has a multiclass and a multilabel class.
|
|
25
|
+
// The class set is the classes of the task, in the order that the caller
|
|
26
|
+
// gives (scikit-learn's labels). A label outside it counts in no class, on
|
|
27
|
+
// either side, and the confusion matrix has no pair with it. The accuracy
|
|
28
|
+
// still reads it: the answer is wrong.
|
|
22
29
|
export class ClassCounts {
|
|
23
30
|
#multilabel;
|
|
24
|
-
#classes
|
|
31
|
+
#classes;
|
|
25
32
|
#rights = [];
|
|
26
33
|
#weights = [];
|
|
27
34
|
#pairs = new Map();
|
|
28
|
-
constructor({ multilabel }) {
|
|
35
|
+
constructor({ labels, multilabel, }) {
|
|
29
36
|
this.#multilabel = multilabel;
|
|
37
|
+
this.#classes = new Map(labels.map((label) => [label, new Tally()]));
|
|
30
38
|
}
|
|
31
39
|
// The counts of two arrays of labels, one label for each answer
|
|
32
40
|
// (multiclass), with the weight of each answer (1 when none is given).
|
|
33
|
-
static of({ expected, stated, weights, }) {
|
|
34
|
-
const counts = new ClassCounts({ multilabel: false });
|
|
41
|
+
static of({ labels, expected, stated, weights, }) {
|
|
42
|
+
const counts = new ClassCounts({ labels, multilabel: false });
|
|
35
43
|
for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
36
44
|
counts.add(new LabelComparison({
|
|
37
45
|
stated: new Set([statedLabel]),
|
|
38
46
|
expected: new Set([expectedLabel]),
|
|
47
|
+
labels,
|
|
39
48
|
}), weight);
|
|
40
49
|
return counts;
|
|
41
50
|
}
|
|
@@ -44,9 +53,8 @@ export class ClassCounts {
|
|
|
44
53
|
// when no answer has it.
|
|
45
54
|
static ofIndicators({ expected, stated, weights, }) {
|
|
46
55
|
const columns = expected[0]?.length ?? 0;
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
counts.#classes.set(column, new Tally());
|
|
56
|
+
const classes = Array.from({ length: columns }, (_, column) => column);
|
|
57
|
+
const counts = new ClassCounts({ labels: classes, multilabel: true });
|
|
50
58
|
const labels = (row) => {
|
|
51
59
|
if (row.length !== columns ||
|
|
52
60
|
row.some((value) => value !== 0 && value !== 1))
|
|
@@ -57,6 +65,7 @@ export class ClassCounts {
|
|
|
57
65
|
counts.add(new LabelComparison({
|
|
58
66
|
stated: labels(statedRow),
|
|
59
67
|
expected: labels(expectedRow),
|
|
68
|
+
labels: classes,
|
|
60
69
|
}), weight);
|
|
61
70
|
return counts;
|
|
62
71
|
}
|
|
@@ -74,14 +83,17 @@ export class ClassCounts {
|
|
|
74
83
|
yield [side, other, weights?.[index] ?? 1];
|
|
75
84
|
}
|
|
76
85
|
}
|
|
77
|
-
// Adds one answer with its weight
|
|
78
|
-
// no label or more than one
|
|
86
|
+
// Adds one answer with its weight, to the classes of the class set of the
|
|
87
|
+
// counts. Multiclass counts refuse an answer with no label or more than one
|
|
88
|
+
// label on a side.
|
|
79
89
|
add(comparison, weight) {
|
|
80
90
|
const { stated, expected } = comparison;
|
|
81
91
|
if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
|
|
82
92
|
throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
|
|
83
|
-
for (const label of
|
|
84
|
-
const tally = this.#classes.
|
|
93
|
+
for (const label of stated.union(expected)) {
|
|
94
|
+
const tally = this.#classes.get(label);
|
|
95
|
+
if (tally === undefined)
|
|
96
|
+
continue;
|
|
85
97
|
if (stated.has(label))
|
|
86
98
|
tally.stated += weight;
|
|
87
99
|
if (expected.has(label))
|
|
@@ -96,14 +108,17 @@ export class ClassCounts {
|
|
|
96
108
|
return;
|
|
97
109
|
const [expectedLabel] = expected;
|
|
98
110
|
const [statedLabel] = stated;
|
|
99
|
-
if (expectedLabel === undefined ||
|
|
111
|
+
if (expectedLabel === undefined ||
|
|
112
|
+
statedLabel === undefined ||
|
|
113
|
+
!this.#classes.has(expectedLabel) ||
|
|
114
|
+
!this.#classes.has(statedLabel))
|
|
100
115
|
return;
|
|
101
116
|
const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
|
|
102
117
|
row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
|
|
103
118
|
}
|
|
104
|
-
// The
|
|
119
|
+
// The class set, in the order that the caller gives.
|
|
105
120
|
labels() {
|
|
106
|
-
return [...this.#classes.keys()]
|
|
121
|
+
return [...this.#classes.keys()];
|
|
107
122
|
}
|
|
108
123
|
// accuracy_score: the weight of the right answers over the weight of all
|
|
109
124
|
// answers, each with numpy's pairwise sum. An answer is right when its
|
|
@@ -128,31 +143,24 @@ export class ClassCounts {
|
|
|
128
143
|
? undefined
|
|
129
144
|
: form.of(counts, beta));
|
|
130
145
|
}
|
|
131
|
-
// The true count of each class
|
|
146
|
+
// The true count of each class of the class set, in its order.
|
|
132
147
|
support() {
|
|
133
|
-
return new Map(this.
|
|
148
|
+
return new Map(this.#classes.entries().map(([label, { expected }]) => [label, expected]));
|
|
134
149
|
}
|
|
135
|
-
// confusion_matrix of the answers, with the
|
|
150
|
+
// confusion_matrix of the answers, with the class set in its order. A set
|
|
136
151
|
// of classes has no confusion matrix (scikit-learn raises), so it throws.
|
|
137
152
|
confusionMatrix() {
|
|
138
153
|
if (this.#multilabel)
|
|
139
154
|
throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
|
|
140
155
|
return new ConfusionMatrix(this.labels(), this.#pairs);
|
|
141
156
|
}
|
|
142
|
-
#score({ average,
|
|
157
|
+
#score({ average, zeroDivision = 0 }, score) {
|
|
143
158
|
return average.of({
|
|
144
|
-
|
|
145
|
-
|
|
159
|
+
counts: new Map(this.#classes
|
|
160
|
+
.entries()
|
|
161
|
+
.map(([label, tally]) => [label, tally.counts()])),
|
|
146
162
|
multilabel: this.#multilabel,
|
|
147
163
|
}, (counts) => score(counts) ?? zeroDivision);
|
|
148
164
|
}
|
|
149
|
-
#counts(label) {
|
|
150
|
-
const tally = this.#classes.get(label);
|
|
151
|
-
return new Counts({
|
|
152
|
-
hits: tally?.hits ?? 0,
|
|
153
|
-
stated: tally?.stated ?? 0,
|
|
154
|
-
expected: tally?.expected ?? 0,
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
165
|
}
|
|
158
166
|
//# sourceMappingURL=class-counts.js.map
|