@zukhruf/evals 0.8.8 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -44
- package/dist/classification/all-class-matches.d.ts +1 -1
- package/dist/classification/all-class-matches.d.ts.map +1 -1
- package/dist/classification/all-class-matches.js +8 -4
- package/dist/classification/all-class-matches.js.map +1 -1
- package/dist/classification/average.d.ts +1 -2
- package/dist/classification/average.d.ts.map +1 -1
- package/dist/classification/binary-average.d.ts +2 -2
- package/dist/classification/binary-average.d.ts.map +1 -1
- package/dist/classification/binary-average.js +13 -12
- package/dist/classification/binary-average.js.map +1 -1
- package/dist/classification/class-counts.d.ts +5 -4
- package/dist/classification/class-counts.d.ts.map +1 -1
- package/dist/classification/class-counts.js +41 -33
- package/dist/classification/class-counts.js.map +1 -1
- package/dist/classification/label-comparer.d.ts +4 -0
- package/dist/classification/label-comparer.d.ts.map +1 -1
- package/dist/classification/label-comparer.js +10 -2
- package/dist/classification/label-comparer.js.map +1 -1
- package/dist/classification/label-comparison.d.ts +4 -2
- package/dist/classification/label-comparison.d.ts.map +1 -1
- package/dist/classification/label-comparison.js +11 -6
- package/dist/classification/label-comparison.js.map +1 -1
- package/dist/classification/macro-average.d.ts +1 -1
- package/dist/classification/macro-average.d.ts.map +1 -1
- package/dist/classification/macro-average.js +4 -9
- package/dist/classification/macro-average.js.map +1 -1
- package/dist/classification/micro-average.d.ts +1 -1
- package/dist/classification/micro-average.d.ts.map +1 -1
- package/dist/classification/micro-average.js +4 -4
- package/dist/classification/micro-average.js.map +1 -1
- package/dist/classification/per-class.d.ts +0 -6
- package/dist/classification/per-class.d.ts.map +1 -1
- package/dist/classification/per-class.js +4 -10
- package/dist/classification/per-class.js.map +1 -1
- package/dist/classification/weighted-average.d.ts +1 -1
- package/dist/classification/weighted-average.d.ts.map +1 -1
- package/dist/classification/weighted-average.js +3 -3
- package/dist/classification/weighted-average.js.map +1 -1
- package/dist/promptfoo/failed-call.d.ts +13 -0
- package/dist/promptfoo/failed-call.d.ts.map +1 -0
- package/dist/promptfoo/failed-call.js +24 -0
- package/dist/promptfoo/failed-call.js.map +1 -0
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/scoring/grouped.d.ts +3 -3
- package/dist/scoring/grouped.d.ts.map +1 -1
- package/dist/scoring/grouped.js +7 -4
- package/dist/scoring/grouped.js.map +1 -1
- package/dist/scoring/scorer-set.d.ts +1 -1
- package/dist/scoring/scorer-set.d.ts.map +1 -1
- package/dist/scoring/scorer-set.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
LabelVarsCase,
|
|
32
32
|
} from '@zukhruf/evals/classification';
|
|
33
33
|
|
|
34
|
-
const comparer = new LabelComparer();
|
|
34
|
+
const comparer = new LabelComparer({ labels: ['program', 'home', 'list'] });
|
|
35
35
|
const testCase = new LabelVarsCase().from({ expected: 'program' });
|
|
36
36
|
const comparison = comparer.compare('home', testCase);
|
|
37
37
|
|
|
@@ -58,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
|
|
|
58
58
|
|
|
59
59
|
## Score with many scorers at once
|
|
60
60
|
|
|
61
|
-
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
|
|
61
|
+
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer({ labels }) })`. It makes the input once, and each scorer reads it.
|
|
62
62
|
|
|
63
|
-
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
|
|
63
|
+
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way). A group can also be a projection of the input, of another type: `Grouped<Input, Group>` takes a `Scorer<Group>`, such as a count of the stated items without a pair, for each of their kinds. A grouped scorer is a breakdown, so its ratios stay out of the score of the record, which is the mean of the plain scorers.
|
|
64
64
|
|
|
65
65
|
```ts
|
|
66
66
|
import { Recall, ScorerSet } from '@zukhruf/evals';
|
|
@@ -75,7 +75,7 @@ const pageKinds = new ScorerSet(
|
|
|
75
75
|
accuracy: new LabelsMatch(),
|
|
76
76
|
recall: new Recall(new AllClassMatches()),
|
|
77
77
|
},
|
|
78
|
-
{ comparer: new LabelComparer() },
|
|
78
|
+
{ comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }) },
|
|
79
79
|
);
|
|
80
80
|
```
|
|
81
81
|
|
|
@@ -224,22 +224,22 @@ It needs no peer: `npm install @zukhruf/evals`. Its sums are `sum` and `mean` of
|
|
|
224
224
|
|
|
225
225
|
A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
|
|
226
226
|
|
|
227
|
-
`LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`.
|
|
227
|
+
`new LabelComparer({ labels })` reads the stated labels of an answer and gives a `LabelComparison`. `labels` is the class set (see [The class set](#the-class-set)), and each comparison holds it. The comparer reads a label or a list of labels as they are. It reads text in this order:
|
|
228
228
|
|
|
229
229
|
1. A JSON array of labels gives those labels.
|
|
230
230
|
2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
|
|
231
231
|
3. Other text is one string label, without its white space at the start and the end.
|
|
232
232
|
|
|
233
|
-
`comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
|
|
233
|
+
`comparison.labels()` gives the class set, in the order that the caller gives. `comparison.only(label)` gives the comparison of one class: its class set is `[label]`, and each side keeps the label when it has it.
|
|
234
234
|
|
|
235
235
|
The matches of a comparison count one answer:
|
|
236
236
|
|
|
237
237
|
| Matches | Hits | Stated | Expected |
|
|
238
238
|
| ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
|
|
239
239
|
| `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
|
|
240
|
-
| `new AllClassMatches()` | the
|
|
240
|
+
| `new AllClassMatches()` | the classes that both sides have | the stated classes | the expected classes |
|
|
241
241
|
|
|
242
|
-
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer
|
|
242
|
+
`AllClassMatches` counts only the classes of the class set of the comparison. `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer)` gives one ratio for each class of the class set, under the label `${class} ${label}`, in the order of the class set.
|
|
243
243
|
|
|
244
244
|
`Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
|
|
245
245
|
|
|
@@ -263,6 +263,7 @@ import {
|
|
|
263
263
|
} from '@zukhruf/evals/classification';
|
|
264
264
|
|
|
265
265
|
const counts = ClassCounts.of({
|
|
266
|
+
labels: ['a', 'b'],
|
|
266
267
|
expected: ['a', 'b', 'b'],
|
|
267
268
|
stated: ['a', 'a', 'b'],
|
|
268
269
|
});
|
|
@@ -270,31 +271,29 @@ const counts = ClassCounts.of({
|
|
|
270
271
|
counts.accuracy(); // accuracy_score
|
|
271
272
|
counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
|
|
272
273
|
counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
|
|
273
|
-
counts.confusionMatrix().rows(); // confusion_matrix,
|
|
274
|
+
counts.confusionMatrix().rows(); // confusion_matrix, the classes in the order of labels
|
|
274
275
|
|
|
275
276
|
// The same counts, one answer at a time:
|
|
276
|
-
const run = new ClassCounts({ multilabel: false });
|
|
277
|
+
const run = new ClassCounts({ labels: ['a', 'b'], multilabel: false });
|
|
277
278
|
const testCase = new LabelCase({ expected: 'a', weight: 1 });
|
|
278
|
-
|
|
279
|
+
const comparer = new LabelComparer({ labels: ['a', 'b'] });
|
|
280
|
+
run.add(comparer.compare('a', testCase), testCase.weight);
|
|
279
281
|
```
|
|
280
282
|
|
|
281
|
-
- `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
|
|
283
|
+
- `ClassCounts.of({ labels, expected, stated, weights })` builds multiclass counts from two arrays of labels. `labels` is the class set. `weights` is optional, and an answer without a weight weighs 1.
|
|
282
284
|
- `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
|
|
283
|
-
- `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
284
|
-
- `
|
|
285
|
-
- `
|
|
285
|
+
- `new ClassCounts({ labels, multilabel })` starts empty, and `add(comparison, weight)` adds one answer to the classes of the class set of the counts. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
286
|
+
- `labels()`, `support()` and `confusionMatrix()` give the classes of the class set, in its order.
|
|
287
|
+
- `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` reads the class set. It throws for multilabel counts, for a class set of more than two classes, and for a positive label outside the class set. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
|
|
286
288
|
- `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
|
|
287
289
|
- `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
|
|
288
290
|
- `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
|
|
289
291
|
|
|
290
292
|
### The class set
|
|
291
293
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
A harness can state any text, such as the answer null of a failed call. If the answers give the classes, each such text adds a class, and the macro averages change. So the caller gives the class set, scikit-learn's `labels`, to the comparer and to the counts. The answers never add a class.
|
|
294
|
+
[ADR 0006](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md) gives this rule. A harness can state any text, such as the answer null of a failed call. If the answers give the classes, each such text adds a class, and the macro averages change. So the caller gives the class set, scikit-learn's `labels`, to the comparer and to the counts. The answers never add a class.
|
|
295
295
|
|
|
296
296
|
```ts
|
|
297
|
-
// Proposal: no code exists yet.
|
|
298
297
|
import { ClassCounts, LabelComparer } from '@zukhruf/evals/classification';
|
|
299
298
|
|
|
300
299
|
const pageKinds = ['program', 'home', 'list', 'investor', 'other'];
|
|
@@ -302,7 +301,7 @@ const comparer = new LabelComparer({ labels: pageKinds });
|
|
|
302
301
|
const counts = new ClassCounts({ labels: pageKinds, multilabel: false });
|
|
303
302
|
```
|
|
304
303
|
|
|
305
|
-
A label outside the class set counts in no class, as scikit-learn gives it when the caller gives `labels`. A stated label outside it is a miss of the expected class, and it is not a predicted count of any class. `LabelsMatch` still reads it as a wrong answer. `PerClass` and `AllClassMatches` read the class set of the comparison, so they take no `labels`. `ClassCounts.ofIndicators` takes its classes from the columns
|
|
304
|
+
A label outside the class set counts in no class, as scikit-learn gives it when the caller gives `labels`. A stated label outside it is a miss of the expected class, and it is not a predicted count of any class. `LabelsMatch` still reads it as a wrong answer. `PerClass` and `AllClassMatches` read the class set of the comparison, so they take no `labels`. `ClassCounts.ofIndicators` takes its classes from the columns. To port a scikit-learn call with `labels=None`, give the sorted labels of both sides.
|
|
306
305
|
|
|
307
306
|
### Probabilities
|
|
308
307
|
|
|
@@ -331,7 +330,10 @@ import {
|
|
|
331
330
|
} from '@zukhruf/evals/classification';
|
|
332
331
|
import { AssertionModule, ScorerAssertion } from '@zukhruf/evals/promptfoo';
|
|
333
332
|
|
|
334
|
-
const labels = {
|
|
333
|
+
const labels = {
|
|
334
|
+
comparer: new LabelComparer({ labels: ['program', 'home', 'list'] }),
|
|
335
|
+
cases: new LabelVarsCase(),
|
|
336
|
+
};
|
|
335
337
|
|
|
336
338
|
export const scorers = new AssertionModule(import.meta.url, {
|
|
337
339
|
accuracy: new ScorerAssertion(new LabelsMatch(), {
|
|
@@ -363,14 +365,13 @@ The name is the promptfoo `metric`, unless the scorer assertion gives its own `m
|
|
|
363
365
|
|
|
364
366
|
### A failed call
|
|
365
367
|
|
|
366
|
-
|
|
368
|
+
[ADR 0005](docs/adr/0005-a-failed-call-is-the-answer-null.md) gives the reasons (backlog #3970).
|
|
367
369
|
|
|
368
370
|
A failed call is a model call that failed for its one test, also after its retries. promptfoo does not grade a result whose provider gives `error` or no output. That result is an error result: it has no named scores, so its case adds nothing to the wholes, and the recall goes up. A failed call is an answer of the harness: the answer null, the text `'null'`. promptfoo grades it, and each family reads it as no answer. `FailedCall` is a coinage of this package. A consumer uses one of two forms.
|
|
369
371
|
|
|
370
372
|
When the provider is your own, return `new FailedCall(reason).response(metadata)`. promptfoo grades the result, and `promptfoo retry <evalId>` does not run it again. The reason is in `metadata.failed`.
|
|
371
373
|
|
|
372
374
|
```ts
|
|
373
|
-
// Proposal: no code exists yet.
|
|
374
375
|
import { FailedCall } from '@zukhruf/evals/promptfoo';
|
|
375
376
|
|
|
376
377
|
const [result] = await rule.compute([row.input], options?.abortSignal);
|
|
@@ -381,7 +382,6 @@ if (result !== undefined && 'error' in result)
|
|
|
381
382
|
When the provider is a built-in one, such as `openai:chat`, an `afterEach` extension hook grades each error result on `FailedCall.answer`. The result stays an error result, so `promptfoo retry <evalId>` and `--retry-errors` run it again.
|
|
382
383
|
|
|
383
384
|
```ts
|
|
384
|
-
// Proposal: no code exists yet.
|
|
385
385
|
import {
|
|
386
386
|
type AfterEachExtensionHookContext,
|
|
387
387
|
ResultFailureReason,
|
|
@@ -544,10 +544,10 @@ export interface ScorecardScorer<Case extends Weighted> {
|
|
|
544
544
|
score(stated: unknown, testCase: Case): Scorecard;
|
|
545
545
|
}
|
|
546
546
|
/** Gives one ratio for each group of an input, such as each field or each class. */
|
|
547
|
-
export abstract class Grouped<Input> {
|
|
548
|
-
constructor(scorer: Scorer<
|
|
549
|
-
/** The groups of an input (Template Method). */
|
|
550
|
-
protected abstract groups(input: Input): Iterable<[string,
|
|
547
|
+
export abstract class Grouped<Input, Group = Input> {
|
|
548
|
+
constructor(scorer: Scorer<Group>);
|
|
549
|
+
/** The groups of an input (Template Method): a part of it, or a projection of another type. */
|
|
550
|
+
protected abstract groups(input: Input): Iterable<[string, Group]>;
|
|
551
551
|
/** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
|
|
552
552
|
ratios(input: Input, label: string): [string, Ratio][];
|
|
553
553
|
}
|
|
@@ -556,7 +556,7 @@ export class ScorerSet<
|
|
|
556
556
|
Input,
|
|
557
557
|
> implements ScorecardScorer<Case> {
|
|
558
558
|
constructor(
|
|
559
|
-
scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>,
|
|
559
|
+
scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input, unknown>>>,
|
|
560
560
|
options: { comparer: Comparer<Case, Input> },
|
|
561
561
|
);
|
|
562
562
|
}
|
|
@@ -664,34 +664,36 @@ export class LabelComparison {
|
|
|
664
664
|
constructor(sides: {
|
|
665
665
|
stated: ReadonlySet<Label>;
|
|
666
666
|
expected: ReadonlySet<Label>;
|
|
667
|
+
labels: readonly Label[];
|
|
667
668
|
});
|
|
668
669
|
readonly stated: ReadonlySet<Label>;
|
|
669
670
|
readonly expected: ReadonlySet<Label>;
|
|
670
|
-
/** The
|
|
671
|
+
/** The class set, in the order that the caller gives. */
|
|
671
672
|
labels(): Label[];
|
|
672
|
-
/** The comparison of one class. */
|
|
673
|
+
/** The comparison of one class: its class set is [label]. */
|
|
673
674
|
only(label: Label): LabelComparison;
|
|
674
675
|
}
|
|
675
676
|
/** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
|
|
676
|
-
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
677
|
+
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
678
|
+
/** labels: the class set. */
|
|
679
|
+
constructor(options: { labels: readonly Label[] });
|
|
680
|
+
}
|
|
677
681
|
|
|
678
682
|
export class ClassMatches implements Matches<LabelComparison> {
|
|
679
683
|
constructor(label: Label);
|
|
680
684
|
}
|
|
685
|
+
/** Counts only the classes of the class set of the comparison. */
|
|
681
686
|
export class AllClassMatches implements Matches<LabelComparison> {}
|
|
682
687
|
/** Holds when the stated labels are the expected labels. */
|
|
683
688
|
export class LabelsMatch extends Gate<LabelComparison> {}
|
|
689
|
+
/** A ratio for each class of the class set of the comparison. */
|
|
684
690
|
export class PerClass extends Grouped<LabelComparison> {
|
|
685
|
-
constructor(
|
|
686
|
-
scorer: Scorer<LabelComparison>,
|
|
687
|
-
options?: { labels?: readonly Label[] },
|
|
688
|
-
);
|
|
691
|
+
constructor(scorer: Scorer<LabelComparison>);
|
|
689
692
|
}
|
|
690
693
|
|
|
691
|
-
/** The classes that an average reads. */
|
|
694
|
+
/** The classes that an average reads: the counts of each class of the class set. */
|
|
692
695
|
export type Classes = {
|
|
693
|
-
readonly
|
|
694
|
-
readonly present: ReadonlyMap<Label, Counts>;
|
|
696
|
+
readonly counts: ReadonlyMap<Label, Counts>;
|
|
695
697
|
readonly multilabel: boolean;
|
|
696
698
|
};
|
|
697
699
|
/** How the scores of the classes become one number (Strategy). */
|
|
@@ -707,14 +709,14 @@ export class BinaryAverage implements Average {
|
|
|
707
709
|
|
|
708
710
|
type ScoreOptions = {
|
|
709
711
|
average: Average;
|
|
710
|
-
labels?: readonly Label[];
|
|
711
712
|
/** 0 (the default), 1 or NaN. */
|
|
712
713
|
zeroDivision?: number;
|
|
713
714
|
};
|
|
714
|
-
/** The counts of each class over many answers: true positives, predicted and true. */
|
|
715
|
+
/** The counts of each class of the class set over many answers: true positives, predicted and true. */
|
|
715
716
|
export class ClassCounts {
|
|
716
|
-
constructor(options: { multilabel: boolean });
|
|
717
|
+
constructor(options: { labels: readonly Label[]; multilabel: boolean });
|
|
717
718
|
static of(answers: {
|
|
719
|
+
labels: readonly Label[];
|
|
718
720
|
expected: readonly Label[];
|
|
719
721
|
stated: readonly Label[];
|
|
720
722
|
weights?: readonly number[];
|
|
@@ -725,6 +727,7 @@ export class ClassCounts {
|
|
|
725
727
|
weights?: readonly number[];
|
|
726
728
|
}): ClassCounts;
|
|
727
729
|
add(comparison: LabelComparison, weight: number): void;
|
|
730
|
+
/** The class set, in the order that the caller gives. */
|
|
728
731
|
labels(): Label[];
|
|
729
732
|
accuracy(): number;
|
|
730
733
|
precision(options: ScoreOptions): number;
|
|
@@ -791,5 +794,5 @@ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the
|
|
|
791
794
|
- [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
|
|
792
795
|
- [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
|
|
793
796
|
- [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)
|
|
794
|
-
- [ADR 0005: a failed call is the answer null](docs/adr/0005-a-failed-call-is-the-answer-null.md)
|
|
795
|
-
- [ADR 0006: classification counts only the classes that the caller gives](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md)
|
|
797
|
+
- [ADR 0005: a failed call is the answer null](docs/adr/0005-a-failed-call-is-the-answer-null.md)
|
|
798
|
+
- [ADR 0006: classification counts only the classes that the caller gives](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md)
|
|
@@ -2,6 +2,6 @@ import { Counts } from '../aggregation/counts.ts';
|
|
|
2
2
|
import type { Matches } from '../scorers/matches.ts';
|
|
3
3
|
import type { LabelComparison } from './label-comparison.ts';
|
|
4
4
|
export declare class AllClassMatches implements Matches<LabelComparison> {
|
|
5
|
-
counts(
|
|
5
|
+
counts(comparison: LabelComparison): Counts;
|
|
6
6
|
}
|
|
7
7
|
//# sourceMappingURL=all-class-matches.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAM7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM;CAU5C"}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { Counts } from "../aggregation/counts.js";
|
|
2
|
-
// The counts of
|
|
3
|
-
// sides have, the stated and the expected are the
|
|
4
|
-
//
|
|
2
|
+
// The counts of the classes of the class set in one answer: the hits are the
|
|
3
|
+
// classes that both sides have, the stated and the expected are the classes
|
|
4
|
+
// of each side. A label outside the class set counts in no class. Pooled over
|
|
5
|
+
// the answers, they are scikit-learn's micro counts.
|
|
5
6
|
export class AllClassMatches {
|
|
6
|
-
counts(
|
|
7
|
+
counts(comparison) {
|
|
8
|
+
const classes = new Set(comparison.labels());
|
|
9
|
+
const stated = comparison.stated.intersection(classes);
|
|
10
|
+
const expected = comparison.expected.intersection(classes);
|
|
7
11
|
return new Counts({
|
|
8
12
|
hits: stated.intersection(expected).size,
|
|
9
13
|
stated: stated.size,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,
|
|
1
|
+
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,qDAAqD;AACrD,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,UAA2B;QAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3D,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Label } from './label.ts';
|
|
3
3
|
export type Classes = {
|
|
4
|
-
readonly
|
|
5
|
-
readonly present: ReadonlyMap<Label, Counts>;
|
|
4
|
+
readonly counts: ReadonlyMap<Label, Counts>;
|
|
6
5
|
readonly multilabel: boolean;
|
|
7
6
|
};
|
|
8
7
|
export interface Average {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { Counts } from '../aggregation/counts.ts';
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Average, Classes } from './average.ts';
|
|
3
3
|
import type { Label } from './label.ts';
|
|
4
4
|
export declare class BinaryAverage implements Average {
|
|
5
5
|
#private;
|
|
6
6
|
constructor(positiveLabel: Label);
|
|
7
|
-
of({
|
|
7
|
+
of({ counts, multilabel }: Classes, score: (counts: Counts) => number): number;
|
|
8
8
|
}
|
|
9
9
|
//# sourceMappingURL=binary-average.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"binary-average.d.ts","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,aAAc,YAAW,OAAO;;gBAG/B,aAAa,EAAE,KAAK;IAIhC,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,OAAO,EAC/B,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAChC,MAAM;CAgBV"}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// classes. A positive label that no answer has scores counts of 0.
|
|
1
|
+
// average='binary': the score of the positive class, for a class set of two
|
|
2
|
+
// classes or fewer. It throws as scikit-learn raises: for a set of classes
|
|
3
|
+
// (multilabel), for a class set of more than two classes, and for a positive
|
|
4
|
+
// label outside the class set.
|
|
6
5
|
export class BinaryAverage {
|
|
7
6
|
#positiveLabel;
|
|
8
7
|
constructor(positiveLabel) {
|
|
9
8
|
this.#positiveLabel = positiveLabel;
|
|
10
9
|
}
|
|
11
|
-
of({
|
|
12
|
-
if (multilabel
|
|
13
|
-
throw new RangeError(
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
10
|
+
of({ counts, multilabel }, score) {
|
|
11
|
+
if (multilabel)
|
|
12
|
+
throw new RangeError('The target is multilabel-indicator, but the average is binary: choose another average.');
|
|
13
|
+
if (counts.size > 2)
|
|
14
|
+
throw new RangeError(`The class set has ${counts.size} classes, but the average is binary: choose another average.`);
|
|
15
|
+
const positive = counts.get(this.#positiveLabel);
|
|
16
|
+
if (positive === undefined)
|
|
17
|
+
throw new RangeError(`The positive label ${JSON.stringify(this.#positiveLabel)} is not in the class set ${JSON.stringify([...counts.keys()])}.`);
|
|
18
|
+
return score(positive);
|
|
18
19
|
}
|
|
19
20
|
}
|
|
20
21
|
//# sourceMappingURL=binary-average.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"binary-average.js","sourceRoot":"","sources":["../../src/classification/binary-average.ts"],"names":[],"mappings":"AAIA,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,+BAA+B;AAC/B,MAAM,OAAO,aAAa;IACf,cAAc,CAAQ;IAE/B,YAAY,aAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,EAAE,CACA,EAAE,MAAM,EAAE,UAAU,EAAW,EAC/B,KAAiC;QAEjC,IAAI,UAAU;YACZ,MAAM,IAAI,UAAU,CAClB,wFAAwF,CACzF,CAAC;QACJ,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;YACjB,MAAM,IAAI,UAAU,CAClB,qBAAqB,MAAM,CAAC,IAAI,8DAA8D,CAC/F,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjD,IAAI,QAAQ,KAAK,SAAS;YACxB,MAAM,IAAI,UAAU,CAClB,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC3H,CAAC;QACJ,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -2,18 +2,19 @@ import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
|
|
|
2
2
|
import type { Average } from './average.ts';
|
|
3
3
|
import { ConfusionMatrix } from './confusion-matrix.ts';
|
|
4
4
|
import { LabelComparison } from './label-comparison.ts';
|
|
5
|
-
import {
|
|
5
|
+
import type { Label } from './label.ts';
|
|
6
6
|
type ScoreOptions = {
|
|
7
7
|
readonly average: Average;
|
|
8
|
-
readonly labels?: readonly Label[];
|
|
9
8
|
readonly zeroDivision?: 0 | 1 | number;
|
|
10
9
|
};
|
|
11
10
|
export declare class ClassCounts {
|
|
12
11
|
#private;
|
|
13
|
-
constructor({ multilabel }: {
|
|
12
|
+
constructor({ labels, multilabel, }: {
|
|
13
|
+
readonly labels: readonly Label[];
|
|
14
14
|
readonly multilabel: boolean;
|
|
15
15
|
});
|
|
16
|
-
static of({ expected, stated, weights, }: {
|
|
16
|
+
static of({ labels, expected, stated, weights, }: {
|
|
17
|
+
readonly labels: readonly Label[];
|
|
17
18
|
readonly expected: readonly Label[];
|
|
18
19
|
readonly stated: readonly Label[];
|
|
19
20
|
readonly weights?: readonly number[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAIxC,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AA2BF,qBAAa,WAAW;;gBAOV,EACV,MAAM,EACN,UAAU,GACX,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;KAC9B;IAOD,MAAM,CAAC,EAAE,CAAC,EACR,MAAM,EACN,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAqBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAyDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IA+BtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAwBnC"}
|
|
@@ -3,39 +3,48 @@ import { Counts } from "../aggregation/counts.js";
|
|
|
3
3
|
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
4
|
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
5
5
|
import { LabelComparison } from "./label-comparison.js";
|
|
6
|
-
import { compareLabels } from "./label.js";
|
|
7
6
|
// The running counts of one class, added in the order of the answers.
|
|
8
7
|
class Tally {
|
|
9
8
|
hits = 0;
|
|
10
9
|
stated = 0;
|
|
11
10
|
expected = 0;
|
|
11
|
+
counts() {
|
|
12
|
+
return new Counts(this);
|
|
13
|
+
}
|
|
12
14
|
}
|
|
13
|
-
// The counts of each class over many answers, as
|
|
14
|
-
// multilabel_confusion_matrix gives them
|
|
15
|
-
// predicted (stated) and the true (expected), each
|
|
16
|
-
// the answers in their order, as numpy's bincount
|
|
15
|
+
// The counts of each class of the class set over many answers, as
|
|
16
|
+
// scikit-learn's multilabel_confusion_matrix gives them with its labels: the
|
|
17
|
+
// true positives (hits), the predicted (stated) and the true (expected), each
|
|
18
|
+
// the sum of the weights of the answers in their order, as numpy's bincount
|
|
19
|
+
// adds them. Add each
|
|
17
20
|
// comparison, then read the metrics of a whole run (torchmetrics' update and
|
|
18
21
|
// compute). It also keeps the weight of each right answer for the accuracy,
|
|
19
22
|
// and the weight of each pair of labels for the confusion matrix. The counts
|
|
20
23
|
// are of one class for each answer (multiclass) or of a set of classes
|
|
21
24
|
// (multilabel), as torchmetrics has a multiclass and a multilabel class.
|
|
25
|
+
// The class set is the classes of the task, in the order that the caller
|
|
26
|
+
// gives (scikit-learn's labels). A label outside it counts in no class, on
|
|
27
|
+
// either side, and the confusion matrix has no pair with it. The accuracy
|
|
28
|
+
// still reads it: the answer is wrong.
|
|
22
29
|
export class ClassCounts {
|
|
23
30
|
#multilabel;
|
|
24
|
-
#classes
|
|
31
|
+
#classes;
|
|
25
32
|
#rights = [];
|
|
26
33
|
#weights = [];
|
|
27
34
|
#pairs = new Map();
|
|
28
|
-
constructor({ multilabel }) {
|
|
35
|
+
constructor({ labels, multilabel, }) {
|
|
29
36
|
this.#multilabel = multilabel;
|
|
37
|
+
this.#classes = new Map(labels.map((label) => [label, new Tally()]));
|
|
30
38
|
}
|
|
31
39
|
// The counts of two arrays of labels, one label for each answer
|
|
32
40
|
// (multiclass), with the weight of each answer (1 when none is given).
|
|
33
|
-
static of({ expected, stated, weights, }) {
|
|
34
|
-
const counts = new ClassCounts({ multilabel: false });
|
|
41
|
+
static of({ labels, expected, stated, weights, }) {
|
|
42
|
+
const counts = new ClassCounts({ labels, multilabel: false });
|
|
35
43
|
for (const [expectedLabel, statedLabel, weight] of ClassCounts.#answers(expected, stated, weights))
|
|
36
44
|
counts.add(new LabelComparison({
|
|
37
45
|
stated: new Set([statedLabel]),
|
|
38
46
|
expected: new Set([expectedLabel]),
|
|
47
|
+
labels,
|
|
39
48
|
}), weight);
|
|
40
49
|
return counts;
|
|
41
50
|
}
|
|
@@ -44,9 +53,8 @@ export class ClassCounts {
|
|
|
44
53
|
// when no answer has it.
|
|
45
54
|
static ofIndicators({ expected, stated, weights, }) {
|
|
46
55
|
const columns = expected[0]?.length ?? 0;
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
counts.#classes.set(column, new Tally());
|
|
56
|
+
const classes = Array.from({ length: columns }, (_, column) => column);
|
|
57
|
+
const counts = new ClassCounts({ labels: classes, multilabel: true });
|
|
50
58
|
const labels = (row) => {
|
|
51
59
|
if (row.length !== columns ||
|
|
52
60
|
row.some((value) => value !== 0 && value !== 1))
|
|
@@ -57,6 +65,7 @@ export class ClassCounts {
|
|
|
57
65
|
counts.add(new LabelComparison({
|
|
58
66
|
stated: labels(statedRow),
|
|
59
67
|
expected: labels(expectedRow),
|
|
68
|
+
labels: classes,
|
|
60
69
|
}), weight);
|
|
61
70
|
return counts;
|
|
62
71
|
}
|
|
@@ -74,14 +83,17 @@ export class ClassCounts {
|
|
|
74
83
|
yield [side, other, weights?.[index] ?? 1];
|
|
75
84
|
}
|
|
76
85
|
}
|
|
77
|
-
// Adds one answer with its weight
|
|
78
|
-
// no label or more than one
|
|
86
|
+
// Adds one answer with its weight, to the classes of the class set of the
|
|
87
|
+
// counts. Multiclass counts refuse an answer with no label or more than one
|
|
88
|
+
// label on a side.
|
|
79
89
|
add(comparison, weight) {
|
|
80
90
|
const { stated, expected } = comparison;
|
|
81
91
|
if (!this.#multilabel && (stated.size !== 1 || expected.size !== 1))
|
|
82
92
|
throw new RangeError(`Multiclass counts take one stated and one expected label for each answer, not ${JSON.stringify([...stated])} and ${JSON.stringify([...expected])}.`);
|
|
83
|
-
for (const label of
|
|
84
|
-
const tally = this.#classes.
|
|
93
|
+
for (const label of stated.union(expected)) {
|
|
94
|
+
const tally = this.#classes.get(label);
|
|
95
|
+
if (tally === undefined)
|
|
96
|
+
continue;
|
|
85
97
|
if (stated.has(label))
|
|
86
98
|
tally.stated += weight;
|
|
87
99
|
if (expected.has(label))
|
|
@@ -96,14 +108,17 @@ export class ClassCounts {
|
|
|
96
108
|
return;
|
|
97
109
|
const [expectedLabel] = expected;
|
|
98
110
|
const [statedLabel] = stated;
|
|
99
|
-
if (expectedLabel === undefined ||
|
|
111
|
+
if (expectedLabel === undefined ||
|
|
112
|
+
statedLabel === undefined ||
|
|
113
|
+
!this.#classes.has(expectedLabel) ||
|
|
114
|
+
!this.#classes.has(statedLabel))
|
|
100
115
|
return;
|
|
101
116
|
const row = this.#pairs.getOrInsertComputed(expectedLabel, () => new Map());
|
|
102
117
|
row.set(statedLabel, (row.get(statedLabel) ?? 0) + weight);
|
|
103
118
|
}
|
|
104
|
-
// The
|
|
119
|
+
// The class set, in the order that the caller gives.
|
|
105
120
|
labels() {
|
|
106
|
-
return [...this.#classes.keys()]
|
|
121
|
+
return [...this.#classes.keys()];
|
|
107
122
|
}
|
|
108
123
|
// accuracy_score: the weight of the right answers over the weight of all
|
|
109
124
|
// answers, each with numpy's pairwise sum. An answer is right when its
|
|
@@ -128,31 +143,24 @@ export class ClassCounts {
|
|
|
128
143
|
? undefined
|
|
129
144
|
: form.of(counts, beta));
|
|
130
145
|
}
|
|
131
|
-
// The true count of each class
|
|
146
|
+
// The true count of each class of the class set, in its order.
|
|
132
147
|
support() {
|
|
133
|
-
return new Map(this.
|
|
148
|
+
return new Map(this.#classes.entries().map(([label, { expected }]) => [label, expected]));
|
|
134
149
|
}
|
|
135
|
-
// confusion_matrix of the answers, with the
|
|
150
|
+
// confusion_matrix of the answers, with the class set in its order. A set
|
|
136
151
|
// of classes has no confusion matrix (scikit-learn raises), so it throws.
|
|
137
152
|
confusionMatrix() {
|
|
138
153
|
if (this.#multilabel)
|
|
139
154
|
throw new RangeError('Answers with a set of classes (multilabel) have no confusion matrix.');
|
|
140
155
|
return new ConfusionMatrix(this.labels(), this.#pairs);
|
|
141
156
|
}
|
|
142
|
-
#score({ average,
|
|
157
|
+
#score({ average, zeroDivision = 0 }, score) {
|
|
143
158
|
return average.of({
|
|
144
|
-
|
|
145
|
-
|
|
159
|
+
counts: new Map(this.#classes
|
|
160
|
+
.entries()
|
|
161
|
+
.map(([label, tally]) => [label, tally.counts()])),
|
|
146
162
|
multilabel: this.#multilabel,
|
|
147
163
|
}, (counts) => score(counts) ?? zeroDivision);
|
|
148
164
|
}
|
|
149
|
-
#counts(label) {
|
|
150
|
-
const tally = this.#classes.get(label);
|
|
151
|
-
return new Counts({
|
|
152
|
-
hits: tally?.hits ?? 0,
|
|
153
|
-
stated: tally?.stated ?? 0,
|
|
154
|
-
expected: tally?.expected ?? 0,
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
165
|
}
|
|
158
166
|
//# sourceMappingURL=class-counts.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAUxD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;IAEb,MAAM;QACJ,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,kEAAkE;AAClE,6EAA6E;AAC7E,8EAA8E;AAC9E,4EAA4E;AAC5E,sBAAsB;AACtB,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,yEAAyE;AACzE,2EAA2E;AAC3E,0EAA0E;AAC1E,uCAAuC;AACvC,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,CAA4B;IACpC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EACV,MAAM,EACN,UAAU,GAIX;QACC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,MAAM,EACN,QAAQ,EACR,MAAM,EACN,OAAO,GAMR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;gBAClC,MAAM;aACP,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;gBAC7B,MAAM,EAAE,OAAO;aAChB,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,4EAA4E;IAC5E,mBAAmB;IACnB,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YAClC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IACE,aAAa,KAAK,SAAS;YAC3B,WAAW,KAAK,SAAS;YACzB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC;YACjC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;YAE/B,OAAO;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,qDAAqD;IACrD,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,+DAA+D;IAC/D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,YAAY,GAAG,CAAC,EAAgB,EAC3C,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,MAAM,EAAE,IAAI,GAAG,CACb,IAAI,CAAC,QAAQ;iBACV,OAAO,EAAE;iBACT,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CACpD;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import type { Comparer } from '../scoring/comparer.ts';
|
|
2
2
|
import type { LabelCase } from './label-case.ts';
|
|
3
3
|
import { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
import { type Label } from './label.ts';
|
|
4
5
|
export declare class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
5
6
|
#private;
|
|
7
|
+
constructor({ labels }: {
|
|
8
|
+
readonly labels: readonly Label[];
|
|
9
|
+
});
|
|
6
10
|
compare(stated: unknown, testCase: LabelCase): LabelComparison;
|
|
7
11
|
}
|
|
8
12
|
//# sourceMappingURL=label-comparer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"label-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"label-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAW,MAAM,YAAY,CAAC;AAUjD,qBAAa,aAAc,YAAW,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;;gBAG5D,EAAE,MAAM,EAAE,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAA;KAAE;IAI7D,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,eAAe;CAoC/D"}
|
|
@@ -5,14 +5,22 @@ import { isLabel } from "./label.js";
|
|
|
5
5
|
// a JSON array of labels gives those labels; a number gives a number label
|
|
6
6
|
// only when each expected label of the case is a number; any other text,
|
|
7
7
|
// without its white space at the start and the end, is one string label.
|
|
8
|
+
// Its comparisons hold the class set: the classes of the task, in the order
|
|
9
|
+
// that the caller gives (scikit-learn's labels). The answers never add a
|
|
10
|
+
// class.
|
|
8
11
|
export class LabelComparer {
|
|
12
|
+
#labels;
|
|
13
|
+
constructor({ labels }) {
|
|
14
|
+
this.#labels = labels;
|
|
15
|
+
}
|
|
9
16
|
compare(stated, testCase) {
|
|
10
17
|
return new LabelComparison({
|
|
11
|
-
stated: new Set(LabelComparer.#
|
|
18
|
+
stated: new Set(LabelComparer.#stated(stated, testCase)),
|
|
12
19
|
expected: testCase.expected,
|
|
20
|
+
labels: this.#labels,
|
|
13
21
|
});
|
|
14
22
|
}
|
|
15
|
-
static #
|
|
23
|
+
static #stated(stated, testCase) {
|
|
16
24
|
if (Array.isArray(stated) && stated.every(isLabel))
|
|
17
25
|
return stated;
|
|
18
26
|
if (typeof stated === 'number' && isLabel(stated))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"label-comparer.js","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,OAAO,aAAa;IACxB,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACxD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;
|
|
1
|
+
{"version":3,"file":"label-comparer.js","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,yEAAyE;AACzE,yEAAyE;AACzE,4EAA4E;AAC5E,yEAAyE;AACzE,SAAS;AACT,MAAM,OAAO,aAAa;IACf,OAAO,CAAmB;IAEnC,YAAY,EAAE,MAAM,EAAyC;QAC3D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACxD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,QAAmB;QACjD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,MAAM,CAAC;QAClE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,OAAO,MAAM,KAAK,QAAQ;YAC5B,MAAM,IAAI,SAAS,CACjB,6DAA6D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACtF,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/D,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,CAAC,KAAK,CAAC;YACd,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAC1B,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;YAElE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAY;QACxB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,WAAW;gBAAE,OAAO,SAAS,CAAC;YACnD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Label } from './label.ts';
|
|
2
2
|
export declare class LabelComparison {
|
|
3
|
+
#private;
|
|
3
4
|
readonly stated: ReadonlySet<Label>;
|
|
4
5
|
readonly expected: ReadonlySet<Label>;
|
|
5
|
-
constructor({ stated, expected, }: {
|
|
6
|
+
constructor({ stated, expected, labels, }: {
|
|
6
7
|
readonly stated: ReadonlySet<Label>;
|
|
7
8
|
readonly expected: ReadonlySet<Label>;
|
|
9
|
+
readonly labels: readonly Label[];
|
|
8
10
|
});
|
|
9
11
|
labels(): Label[];
|
|
10
12
|
only(label: Label): LabelComparison;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"label-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"label-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,eAAe;;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;gBAG1B,EACV,MAAM,EACN,QAAQ,EACR,MAAM,GACP,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QACtC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;KACnC;IAOD,MAAM,IAAI,KAAK,EAAE;IAMjB,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,eAAe;CAQpC"}
|
|
@@ -1,22 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
1
|
+
// The stated labels of one answer, the expected labels of its case, and the
|
|
2
|
+
// class set: the classes of the task, in the order that the caller gives
|
|
3
|
+
// (scikit-learn's labels). A label outside the class set counts in no class.
|
|
3
4
|
export class LabelComparison {
|
|
4
5
|
stated;
|
|
5
6
|
expected;
|
|
6
|
-
|
|
7
|
+
#labels;
|
|
8
|
+
constructor({ stated, expected, labels, }) {
|
|
7
9
|
this.stated = stated;
|
|
8
10
|
this.expected = expected;
|
|
11
|
+
this.#labels = labels;
|
|
9
12
|
}
|
|
10
|
-
// The
|
|
13
|
+
// The class set, in the order that the caller gives.
|
|
11
14
|
labels() {
|
|
12
|
-
return [...this
|
|
15
|
+
return [...this.#labels];
|
|
13
16
|
}
|
|
14
|
-
// The comparison of one class:
|
|
17
|
+
// The comparison of one class: its class set is [label], and each side
|
|
18
|
+
// keeps the label when it has it.
|
|
15
19
|
only(label) {
|
|
16
20
|
const one = new Set([label]);
|
|
17
21
|
return new LabelComparison({
|
|
18
22
|
stated: this.stated.intersection(one),
|
|
19
23
|
expected: this.expected.intersection(one),
|
|
24
|
+
labels: [label],
|
|
20
25
|
});
|
|
21
26
|
}
|
|
22
27
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"label-comparison.js","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"label-comparison.js","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAEA,4EAA4E;AAC5E,yEAAyE;AACzE,6EAA6E;AAC7E,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAC7B,OAAO,CAAmB;IAEnC,YAAY,EACV,MAAM,EACN,QAAQ,EACR,MAAM,GAKP;QACC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,qDAAqD;IACrD,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,uEAAuE;IACvE,kCAAkC;IAClC,IAAI,CAAC,KAAY;QACf,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC;YACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC;YACzC,MAAM,EAAE,CAAC,KAAK,CAAC;SAChB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Average, Classes } from './average.ts';
|
|
3
3
|
export declare class MacroAverage implements Average {
|
|
4
|
-
of(
|
|
4
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=macro-average.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"macro-average.d.ts","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"macro-average.d.ts","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAGhE"}
|
|
@@ -1,15 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
// average='macro': the mean of the scores of the
|
|
1
|
+
import { nanmean } from '@zukhruf/statistics';
|
|
2
|
+
// average='macro': the mean of the scores of the classes, as
|
|
3
3
|
// numpy's nanmean gives it: a NaN score (zeroDivision NaN) adds 0 and is not
|
|
4
4
|
// counted, and the mean of no score is NaN.
|
|
5
5
|
export class MacroAverage {
|
|
6
|
-
of(
|
|
7
|
-
|
|
8
|
-
const counted = scores.filter((value) => !Number.isNaN(value)).length;
|
|
9
|
-
if (counted === 0)
|
|
10
|
-
return Number.NaN;
|
|
11
|
-
const values = scores.map((value) => (Number.isNaN(value) ? 0 : value));
|
|
12
|
-
return sum(values) / counted;
|
|
6
|
+
of(classes, score) {
|
|
7
|
+
return nanmean([...classes.counts.values()].map(score));
|
|
13
8
|
}
|
|
14
9
|
}
|
|
15
10
|
//# sourceMappingURL=macro-average.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAK9C,6DAA6D;AAC7D,6EAA6E;AAC7E,4CAA4C;AAC5C,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,OAAgB,EAAE,KAAiC;QACpD,OAAO,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1D,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Average, Classes } from './average.ts';
|
|
3
3
|
export declare class MicroAverage implements Average {
|
|
4
|
-
of(
|
|
4
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=micro-average.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"micro-average.d.ts","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"micro-average.d.ts","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIrD,qBAAa,YAAa,YAAW,OAAO;IAC1C,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAWhE"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Counts } from "../aggregation/counts.js";
|
|
3
|
-
// average='micro': the counts of the
|
|
3
|
+
// average='micro': the counts of the classes are added, with numpy's
|
|
4
4
|
// pairwise sum, and scored once.
|
|
5
5
|
export class MicroAverage {
|
|
6
|
-
of(
|
|
7
|
-
const
|
|
8
|
-
const total = (count) => sum(
|
|
6
|
+
of(classes, score) {
|
|
7
|
+
const each = [...classes.counts.values()];
|
|
8
|
+
const total = (count) => sum(each.map(count));
|
|
9
9
|
return score(new Counts({
|
|
10
10
|
hits: total(({ hits }) => hits),
|
|
11
11
|
stated: total(({ stated }) => stated),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,
|
|
1
|
+
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,qEAAqE;AACrE,iCAAiC;AACjC,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,OAAgB,EAAE,KAAiC;QACpD,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,CAAC,KAAiC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1E,OAAO,KAAK,CACV,IAAI,MAAM,CAAC;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;YAC/B,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;YACrC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC5C,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
import type { Scorer } from '../scorers/scorer.ts';
|
|
2
1
|
import { Grouped } from '../scoring/grouped.ts';
|
|
3
2
|
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
-
import type { Label } from './label.ts';
|
|
5
3
|
export declare class PerClass extends Grouped<LabelComparison> {
|
|
6
|
-
#private;
|
|
7
|
-
constructor(scorer: Scorer<LabelComparison>, { labels }?: {
|
|
8
|
-
readonly labels?: readonly Label[];
|
|
9
|
-
});
|
|
10
4
|
protected groups(comparison: LabelComparison): Iterable<[string, LabelComparison]>;
|
|
11
5
|
}
|
|
12
6
|
//# sourceMappingURL=per-class.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-class.d.ts","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"per-class.d.ts","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,QAAS,SAAQ,OAAO,CAAC,eAAe,CAAC;IACpD,SAAS,CAAE,MAAM,CACf,UAAU,EAAE,eAAe,GAC1B,QAAQ,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CAIvC"}
|
|
@@ -1,16 +1,10 @@
|
|
|
1
1
|
import { Grouped } from "../scoring/grouped.js";
|
|
2
|
-
// A scorer for each class of a comparison: in a ScorerSet,
|
|
3
|
-
// of its scorer on the comparison of each class, under the
|
|
4
|
-
// `${class} ${label}
|
|
5
|
-
// sides of the answer; with labels, only those classes.
|
|
2
|
+
// A scorer for each class of the class set of a comparison: in a ScorerSet,
|
|
3
|
+
// it gives the ratio of its scorer on the comparison of each class, under the
|
|
4
|
+
// label `${class} ${label}`, in the order of the class set.
|
|
6
5
|
export class PerClass extends Grouped {
|
|
7
|
-
#labels;
|
|
8
|
-
constructor(scorer, { labels } = {}) {
|
|
9
|
-
super(scorer);
|
|
10
|
-
this.#labels = labels;
|
|
11
|
-
}
|
|
12
6
|
*groups(comparison) {
|
|
13
|
-
for (const label of
|
|
7
|
+
for (const label of comparison.labels())
|
|
14
8
|
yield [`${label}`, comparison.only(label)];
|
|
15
9
|
}
|
|
16
10
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"per-class.js","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"per-class.js","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAGhD,4EAA4E;AAC5E,8EAA8E;AAC9E,4DAA4D;AAC5D,MAAM,OAAO,QAAS,SAAQ,OAAwB;IAC1C,CAAC,MAAM,CACf,UAA2B;QAE3B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE;YACrC,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Counts } from '../aggregation/counts.ts';
|
|
2
2
|
import type { Average, Classes } from './average.ts';
|
|
3
3
|
export declare class WeightedAverage implements Average {
|
|
4
|
-
of(
|
|
4
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=weighted-average.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CAShE"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { sum } from '@zukhruf/statistics';
|
|
2
|
-
// average='weighted': the mean of the scores of the
|
|
2
|
+
// average='weighted': the mean of the scores of the classes, each
|
|
3
3
|
// weighted by its true count, as scikit-learn's _nanaverage gives it: a NaN
|
|
4
4
|
// score is dropped with its weight, the mean of no score is NaN, and when the
|
|
5
5
|
// weights add up to 0 it is the plain mean.
|
|
6
6
|
export class WeightedAverage {
|
|
7
|
-
of(
|
|
8
|
-
const kept = [...
|
|
7
|
+
of(classes, score) {
|
|
8
|
+
const kept = [...classes.counts.values()]
|
|
9
9
|
.map((counts) => ({ value: score(counts), weight: counts.expected }))
|
|
10
10
|
.filter(({ value }) => !Number.isNaN(value));
|
|
11
11
|
if (kept.length === 0)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,
|
|
1
|
+
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,kEAAkE;AAClE,4EAA4E;AAC5E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,OAAO,eAAe;IAC1B,EAAE,CAAC,OAAgB,EAAE,KAAiC;QACpD,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;aACtC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACpE,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5E,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC;IACxE,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare class FailedCall {
|
|
2
|
+
static readonly answer = "null";
|
|
3
|
+
readonly reason: string;
|
|
4
|
+
constructor(reason: string);
|
|
5
|
+
response(metadata?: Readonly<Record<string, unknown>>): FailedCallResponse;
|
|
6
|
+
}
|
|
7
|
+
export type FailedCallResponse = {
|
|
8
|
+
readonly output: typeof FailedCall.answer;
|
|
9
|
+
readonly metadata: Readonly<Record<string, unknown>> & {
|
|
10
|
+
readonly failed: string;
|
|
11
|
+
};
|
|
12
|
+
};
|
|
13
|
+
//# sourceMappingURL=failed-call.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failed-call.d.ts","sourceRoot":"","sources":["../../src/promptfoo/failed-call.ts"],"names":[],"mappings":"AAQA,qBAAa,UAAU;IACrB,MAAM,CAAC,QAAQ,CAAC,MAAM,UAAU;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,MAAM,EAAE,MAAM;IAM1B,QAAQ,CACN,QAAQ,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAM,GAC/C,kBAAkB;CAMtB;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,MAAM,EAAE,OAAO,UAAU,CAAC,MAAM,CAAC;IAC1C,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG;QACrD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// What a provider returns for a model call that failed for its one test,
|
|
2
|
+
// also after its retries (ADR 0005): the answer null. promptfoo 0.125.1
|
|
3
|
+
// grades no result whose provider gave an error or no output
|
|
4
|
+
// (dist/src/evaluator-BxCmvGtv.js:7940-7952), so that case would add nothing
|
|
5
|
+
// to the wholes and the recall would go up. promptfoo grades the answer null,
|
|
6
|
+
// and each family reads it as no answer. A failure of the run, such as a 401,
|
|
7
|
+
// is thrown instead, so the result stays an error result that promptfoo
|
|
8
|
+
// retry runs again.
|
|
9
|
+
export class FailedCall {
|
|
10
|
+
static answer = 'null';
|
|
11
|
+
reason;
|
|
12
|
+
constructor(reason) {
|
|
13
|
+
this.reason = reason;
|
|
14
|
+
}
|
|
15
|
+
// promptfoo's ProviderResponse of the failed call: the answer null, and the
|
|
16
|
+
// metadata of the provider with the reason under `failed`.
|
|
17
|
+
response(metadata = {}) {
|
|
18
|
+
return {
|
|
19
|
+
output: FailedCall.answer,
|
|
20
|
+
metadata: { ...metadata, failed: this.reason },
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=failed-call.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failed-call.js","sourceRoot":"","sources":["../../src/promptfoo/failed-call.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,wEAAwE;AACxE,6DAA6D;AAC7D,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,wEAAwE;AACxE,oBAAoB;AACpB,MAAM,OAAO,UAAU;IACrB,MAAM,CAAU,MAAM,GAAG,MAAM,CAAC;IACvB,MAAM,CAAS;IAExB,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,4EAA4E;IAC5E,2DAA2D;IAC3D,QAAQ,CACN,WAA8C,EAAE;QAEhD,OAAO;YACL,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;SAC/C,CAAC;IACJ,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { AssertionModule, type FileAssertion, type ModuleAssertion, type VarsContext, } from './assertion-module.ts';
|
|
2
2
|
export { type AssertionContext, type GradingResult, PromptfooAssertion, } from './promptfoo-assertion.ts';
|
|
3
3
|
export { ScorerAssertion } from './scorer-assertion.ts';
|
|
4
|
+
export { FailedCall, type FailedCallResponse } from './failed-call.ts';
|
|
4
5
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/promptfoo/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAA2B,MAAM,kBAAkB,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Ratio } from '../aggregation/ratio.ts';
|
|
2
2
|
import type { Scorer } from '../scorers/scorer.ts';
|
|
3
|
-
export declare abstract class Grouped<Input> {
|
|
3
|
+
export declare abstract class Grouped<Input, Group = Input> {
|
|
4
4
|
#private;
|
|
5
|
-
constructor(scorer: Scorer<
|
|
6
|
-
protected abstract groups(input: Input): Iterable<[string,
|
|
5
|
+
constructor(scorer: Scorer<Group>);
|
|
6
|
+
protected abstract groups(input: Input): Iterable<[string, Group]>;
|
|
7
7
|
ratios(input: Input, label: string): [string, Ratio][];
|
|
8
8
|
}
|
|
9
9
|
//# sourceMappingURL=grouped.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grouped.d.ts","sourceRoot":"","sources":["../../src/scoring/grouped.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"grouped.d.ts","sourceRoot":"","sources":["../../src/scoring/grouped.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAUnD,8BAAsB,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;;gBAGpC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;IAIjC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,QAAQ,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAElE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;CAQvD"}
|
package/dist/scoring/grouped.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
// A scorer for each group of an input: in a ScorerSet, it gives the ratio of
|
|
2
|
-
// its scorer on each group, under the label `${group} ${label}
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
2
|
+
// its scorer on each group, under the label `${group} ${label}`, and stays
|
|
3
|
+
// out of the score of the record, as a breakdown is not a score of its own.
|
|
4
|
+
// A subclass gives the groups (Template Method): a part of the input, such as
|
|
5
|
+
// one class, or a projection of it, such as the stated items of one kind. A
|
|
6
|
+
// group gives no ratio when the scorer counts nothing in it (the whole 0): a
|
|
7
|
+
// ratio with the whole 0 adds nothing to a ratio of sums, so the totals stay
|
|
8
|
+
// the same.
|
|
6
9
|
export class Grouped {
|
|
7
10
|
#scorer;
|
|
8
11
|
constructor(scorer) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grouped.js","sourceRoot":"","sources":["../../src/scoring/grouped.ts"],"names":[],"mappings":"AAGA,6EAA6E;AAC7E,4EAA4E;AAC5E,
|
|
1
|
+
{"version":3,"file":"grouped.js","sourceRoot":"","sources":["../../src/scoring/grouped.ts"],"names":[],"mappings":"AAGA,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,YAAY;AACZ,MAAM,OAAgB,OAAO;IAClB,OAAO,CAAgB;IAEhC,YAAY,MAAqB;QAC/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAID,MAAM,CAAC,KAAY,EAAE,KAAa;QAChC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAmB,EAAE,CAAC;YACvC,GAAG,KAAK,IAAI,KAAK,EAAE;YACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;SACzB,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF"}
|
|
@@ -5,7 +5,7 @@ import type { ScorecardScorer, Weighted } from './scorecard-scorer.ts';
|
|
|
5
5
|
import type { Scorecard } from './scorecard.ts';
|
|
6
6
|
export declare class ScorerSet<Case extends Weighted, Input> implements ScorecardScorer<Case> {
|
|
7
7
|
#private;
|
|
8
|
-
constructor(entries: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>, { comparer }: {
|
|
8
|
+
constructor(entries: Readonly<Record<string, Scorer<Input> | Grouped<Input, unknown>>>, { comparer }: {
|
|
9
9
|
readonly comparer: Comparer<Case, Input>;
|
|
10
10
|
});
|
|
11
11
|
score(stated: unknown, testCase: Case): Scorecard;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQvD,qBAAa,SAAS,CACpB,IAAI,SAAS,QAAQ,EACrB,KAAK,CACL,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQ9B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"scorer-set.d.ts","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQvD,qBAAa,SAAS,CACpB,IAAI,SAAS,QAAQ,EACrB,KAAK,CACL,YAAW,eAAe,CAAC,IAAI,CAAC;;gBAQ9B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAC1E,EAAE,QAAQ,EAAE,EAAE;QAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;KAAE;IAM5D,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,SAAS;CAqClD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAG1C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAC9E,kEAAkE;AAClE,MAAM,OAAO,SAAS;IAIX,QAAQ,CAGZ;IACI,SAAS,CAAwB;IAE1C,YACE,
|
|
1
|
+
{"version":3,"file":"scorer-set.js","sourceRoot":"","sources":["../../src/scoring/scorer-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAG1C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAC9E,kEAAkE;AAClE,MAAM,OAAO,SAAS;IAIX,QAAQ,CAGZ;IACI,SAAS,CAAwB;IAE1C,YACE,OAA0E,EAC1E,EAAE,QAAQ,EAAgD;QAE1D,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,MAAe,EAAE,QAAc;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,KAAK,YAAY,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK;wBACX,CAAC,CAAC,GAAG,KAAK,QAAQ;wBAClB,CAAC,CAAC,GAAG,KAAK,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EACJ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC;YACvB,MAAM;SACP,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.10",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@zukhruf/python": "^0.8.
|
|
67
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.10",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.10"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"@cfworker/json-schema": "4.1.4",
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@cfworker/json-schema": "4.1.4",
|
|
83
83
|
"@zukhruf/random": "*",
|
|
84
|
-
"@zukhruf/testing": "^0.8.
|
|
84
|
+
"@zukhruf/testing": "^0.8.10",
|
|
85
85
|
"munkres": "^2.1.1",
|
|
86
86
|
"promptfoo": "0.125.1",
|
|
87
87
|
"python-random": "1.0.0"
|