@zukhruf/evals 0.8.5 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -4
- package/dist/classification/brier-form.d.ts +10 -0
- package/dist/classification/brier-form.d.ts.map +1 -0
- package/dist/classification/brier-form.js +18 -0
- package/dist/classification/brier-form.js.map +1 -0
- package/dist/classification/brier-score.d.ts +7 -2
- package/dist/classification/brier-score.d.ts.map +1 -1
- package/dist/classification/brier-score.js +13 -12
- package/dist/classification/brier-score.js.map +1 -1
- package/dist/classification/class-counts.d.ts.map +1 -1
- package/dist/classification/class-counts.js +2 -3
- package/dist/classification/class-counts.js.map +1 -1
- package/dist/classification/index.d.ts +1 -0
- package/dist/classification/index.d.ts.map +1 -1
- package/dist/classification/index.js +1 -0
- package/dist/classification/index.js.map +1 -1
- package/dist/classification/macro-average.js +2 -2
- package/dist/classification/macro-average.js.map +1 -1
- package/dist/classification/micro-average.js +5 -5
- package/dist/classification/micro-average.js.map +1 -1
- package/dist/classification/weighted-average.d.ts.map +1 -1
- package/dist/classification/weighted-average.js +1 -2
- package/dist/classification/weighted-average.js.map +1 -1
- package/dist/structured/comparison/item-pairs.d.ts +19 -5
- package/dist/structured/comparison/item-pairs.d.ts.map +1 -1
- package/dist/structured/comparison/item-pairs.js +25 -1
- package/dist/structured/comparison/item-pairs.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +36 -20
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/index.d.ts +1 -1
- package/dist/structured/index.d.ts.map +1 -1
- package/package.json +4 -9
package/README.md
CHANGED
|
@@ -92,6 +92,122 @@ const pageKinds = new ScorerSet(
|
|
|
92
92
|
|
|
93
93
|
`Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
|
|
94
94
|
|
|
95
|
+
## Compare two runs
|
|
96
|
+
|
|
97
|
+
Status: design for review (backlog #3870, [ADR 0004](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)). The released package does not have `Run` or `PairedBootstrap` yet.
|
|
98
|
+
|
|
99
|
+
A caller runs one harness two times on the same tests: once with the baseline model and once with a change, such as another model. Two means side by side do not say if the change is better. `PairedBootstrap` gives one verdict for each name: better, worse or inconclusive, with the difference, its interval and a p-value.
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
import { PairedBootstrap, Run } from '@zukhruf/evals';
|
|
103
|
+
import { defaultRng } from '@zukhruf/random';
|
|
104
|
+
import { Linear, PercentileInterval } from '@zukhruf/statistics';
|
|
105
|
+
|
|
106
|
+
const baseline = new Run();
|
|
107
|
+
const change = new Run();
|
|
108
|
+
for (const answer of baselineAnswers)
|
|
109
|
+
baseline.add(
|
|
110
|
+
answer.testId,
|
|
111
|
+
scorer.score(answer.stated, answer.case),
|
|
112
|
+
answer.case.weight,
|
|
113
|
+
);
|
|
114
|
+
for (const answer of changeAnswers)
|
|
115
|
+
change.add(
|
|
116
|
+
answer.testId,
|
|
117
|
+
scorer.score(answer.stated, answer.case),
|
|
118
|
+
answer.case.weight,
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
const generator = defaultRng(31957);
|
|
122
|
+
const paired = new PairedBootstrap({
|
|
123
|
+
nResamples: 2000,
|
|
124
|
+
randomIndex: (n) => generator.integers(0, n, 1)[0] ?? Number.NaN,
|
|
125
|
+
interval: new PercentileInterval({
|
|
126
|
+
low: 0.025,
|
|
127
|
+
high: 0.975,
|
|
128
|
+
method: new Linear(),
|
|
129
|
+
}),
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
const recall = paired.verdict(baseline, change, 'recall', {
|
|
133
|
+
higherIsBetter: true,
|
|
134
|
+
});
|
|
135
|
+
recall.result; // 'better', 'worse' or 'inconclusive'
|
|
136
|
+
recall.difference.observed; // Totals of the change minus Totals of the baseline
|
|
137
|
+
recall.difference.low; // the 2.5% quantile of the resample differences
|
|
138
|
+
recall.p; // the two-sided p-value of the same resamples
|
|
139
|
+
recall.improvements; // the tests whose value is higher in the change
|
|
140
|
+
|
|
141
|
+
const brier = paired.verdict(baseline, change, 'brier', {
|
|
142
|
+
higherIsBetter: false,
|
|
143
|
+
});
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
A run holds the scorecards of one model. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
// @zukhruf/evals: two runs side by side (design for review)
|
|
150
|
+
import type {
|
|
151
|
+
ConfidenceIntervalMethod,
|
|
152
|
+
DifferenceInterval,
|
|
153
|
+
} from '@zukhruf/statistics';
|
|
154
|
+
|
|
155
|
+
/** The scorecards of one model on many tests. A test can have many epochs. */
|
|
156
|
+
export class Run {
|
|
157
|
+
/** Adds one epoch of a test. Another weight for a test id that the run has is a RangeError. */
|
|
158
|
+
add(testId: string, scorecard: Scorecard, weight: number): void;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export type Verdict = {
|
|
162
|
+
readonly name: string;
|
|
163
|
+
readonly result: 'better' | 'worse' | 'inconclusive';
|
|
164
|
+
/** change − baseline: its observed value and its interval. */
|
|
165
|
+
readonly difference: DifferenceInterval;
|
|
166
|
+
/** The two-sided percentile-bootstrap p-value, with no correction. */
|
|
167
|
+
readonly p: number;
|
|
168
|
+
readonly improvements: number;
|
|
169
|
+
readonly regressions: number;
|
|
170
|
+
readonly ties: number;
|
|
171
|
+
readonly tests: number;
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
/** The paired bootstrap of two runs on the same tests (Koehn 2004). */
|
|
175
|
+
export class PairedBootstrap {
|
|
176
|
+
constructor(options: {
|
|
177
|
+
nResamples: number;
|
|
178
|
+
randomIndex: (n: number) => number;
|
|
179
|
+
interval: ConfidenceIntervalMethod;
|
|
180
|
+
});
|
|
181
|
+
/** A test id in one run only, or a test with two weights, is a RangeError. A name that one run does not give is an Error. */
|
|
182
|
+
verdict(
|
|
183
|
+
baseline: Run,
|
|
184
|
+
change: Run,
|
|
185
|
+
name: string,
|
|
186
|
+
options: { higherIsBetter: boolean },
|
|
187
|
+
): Verdict;
|
|
188
|
+
}
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
| Call | What it does underneath |
|
|
192
|
+
| ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
193
|
+
| `run.add(testId, scorecard, weight)` | Keeps the scorecard as one epoch of the test. The weight belongs to the case, so another weight for the same test id throws. |
|
|
194
|
+
| `paired.verdict(baseline, change, …)`: join | Pairs the tests by test id, in the order in which the baseline run first got them. A test id in one run only throws, with each such id, as mteb's `pairedBootstrapTest` does. A test whose weight differs between the two runs throws. |
|
|
195
|
+
| reduce | Takes the mean of the epochs of each test, name by name: Ratio(sum of the parts / k, sum of the wholes / k) for k epochs (Miller 2024, §3.1). A scorecard without the name counts as Ratio(0, 0), as in `Totals`. |
|
|
196
|
+
| resample | `IidBootstrap` of `@zukhruf/statistics` draws the pairs of tests `nResamples` times. Each resample takes one `randomIndex` for each test. |
|
|
197
|
+
| statistic | On the same resampled tests: the ratio of sums of the name over the change, minus the same over the baseline, with each weight (Koehn 2004). `difference.observed` is the same statistic on all the tests. |
|
|
198
|
+
| interval and verdict | The `ConfidenceIntervalMethod` gives the interval. With `higherIsBetter: true`, `isAbove(0)` is better and `isBelow(0)` is worse. With `false`, the two change places. Else the verdict is inconclusive. |
|
|
199
|
+
| p | Two times the smaller share of the resample differences that are at or below 0 and at or above 0, at most 1, as mteb's `pairedBootstrapTest` gives it. The caller applies a correction, such as Holm. |
|
|
200
|
+
| counts | Counts the tests by the sign of (change value − baseline value), with the direction: improvements, regressions and ties. |
|
|
201
|
+
|
|
202
|
+
`Bootstrap` and `PairedBootstrap` use different statistics. `Bootstrap` takes the mean of the values of the scorecards: the samples average. For a plain scorer, `Totals` gives the micro average, so the interval of `Bootstrap` is not the interval of the number of `Totals`. `PairedBootstrap` uses the statistic of `Totals`, so its interval is the interval of its observed difference (ADR 0004, "Consequences").
|
|
203
|
+
|
|
204
|
+
The design does not resample clustered tests, such as many questions about one page, and it reduces the epochs only by the mean.
|
|
205
|
+
|
|
206
|
+
| Who uses it | For | Backlog |
|
|
207
|
+
| --------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------- |
|
|
208
|
+
| context-compiler, startup-credits page kinds | jev-1.14 against another model on the 121 pages, with the runs of each model as epochs | context-compiler #3882 slice 2, mutex #3870 |
|
|
209
|
+
| offers-evals and context-compiler, startup-credits offers | a run against the baseline run of its harness, for recall, precision, kind and amount | mutex #3870, context-compiler #3882 |
|
|
210
|
+
|
|
95
211
|
## Compare structured values
|
|
96
212
|
|
|
97
213
|
`./structured` compares a structured value with the value that a person labelled: one walk pairs the leaves of the two values, the alignment of each array is a Strategy (by position, or by best match with the Hungarian algorithm), and each scorer gives a ratio for one record. SOB's evaluate.py is a configuration of the same scorers. [docs/structured.md](docs/structured.md) shows the whole API: `ComparisonSpec`, `TreeWalk`, `Comparison`, `PathMatches`, `ValueMatches`, `ItemMatches`, `TokenF1`, `TypePrecision`, `RequiredKeyRecall`, the gates, `PerField`, `JsonSchema`, `ConfigCase` and `TextVarsCase`. Its decisions are the ADRs in [docs/adr/structured](docs/adr/structured), from the time when it was `@zukhruf/structured-eval`.
|
|
@@ -102,7 +218,7 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
|
|
|
102
218
|
|
|
103
219
|
`./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
|
|
104
220
|
|
|
105
|
-
It needs
|
|
221
|
+
It needs no peer: `npm install @zukhruf/evals`. Its sums are `sum` and `mean` of `@zukhruf/statistics`, which give `numpy.sum` and `numpy.mean` bit for bit.
|
|
106
222
|
|
|
107
223
|
### For each answer
|
|
108
224
|
|
|
@@ -175,7 +291,7 @@ run.add(new LabelComparer().compare('a', testCase), testCase.weight);
|
|
|
175
291
|
|
|
176
292
|
`ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
|
|
177
293
|
|
|
178
|
-
`BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
|
|
294
|
+
`BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. `form` is the arithmetic of the score of p (ADR 0003). `new TwoColumnForm()` is scikit-learn's and the default: 0.5 × the sum of the squares of the two classes. `new SquaredErrorForm()` is (p − y)², as numpy gives `np.mean((p - y) ** 2)`. The two forms can differ in the last bit. A form needs `positiveLabel`. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
|
|
179
295
|
|
|
180
296
|
## Use it with promptfoo
|
|
181
297
|
|
|
@@ -570,8 +686,18 @@ export class ProbabilityComparer implements Comparer<
|
|
|
570
686
|
LabelCase,
|
|
571
687
|
ProbabilityComparison
|
|
572
688
|
> {}
|
|
689
|
+
/** The arithmetic of the Brier score of the probability p of the positive label (ADR 0003). */
|
|
690
|
+
export interface BrierForm {
|
|
691
|
+
of(probability: number, expected: 0 | 1): number;
|
|
692
|
+
}
|
|
693
|
+
export class TwoColumnForm implements BrierForm {}
|
|
694
|
+
export class SquaredErrorForm implements BrierForm {}
|
|
573
695
|
export class BrierScore implements Scorer<ProbabilityComparison> {
|
|
574
|
-
constructor(
|
|
696
|
+
constructor(
|
|
697
|
+
options?:
|
|
698
|
+
| { positiveLabel?: undefined; form?: undefined }
|
|
699
|
+
| { positiveLabel: Label; form?: BrierForm },
|
|
700
|
+
);
|
|
575
701
|
/** The score of a whole run, with numpy's pairwise sum. */
|
|
576
702
|
mean(
|
|
577
703
|
comparisons: readonly ProbabilityComparison[],
|
|
@@ -590,7 +716,7 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
|
|
|
590
716
|
| context-compiler, startup-credits offers | core, `./structured` (`ByBestMatch`), `./promptfoo` | context-compiler #3893, #3882 |
|
|
591
717
|
| context-compiler, startup-credits page kinds | `./classification` (accuracy, the precision and recall of each class, the confusion matrix) | #3882 slice 2 |
|
|
592
718
|
| `@zukhruf/mteb`, classification tasks | `./classification` (`ClassCounts`: accuracy, macro and weighted P, R and F1; label-indicator rows) | mutex #3890 |
|
|
593
|
-
| relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`)
|
|
719
|
+
| relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore` with `SquaredErrorForm`) | mutex #3891 |
|
|
594
720
|
| event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
|
|
595
721
|
|
|
596
722
|
The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
|
|
@@ -600,3 +726,4 @@ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the
|
|
|
600
726
|
- [ADR 0001: one package, with a subpath for each shape of answer](docs/adr/0001-one-package-with-a-subpath-for-each-shape-of-answer.md)
|
|
601
727
|
- [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
|
|
602
728
|
- [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
|
|
729
|
+
- [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md) (design for review)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface BrierForm {
|
|
2
|
+
of(probability: number, expected: 0 | 1): number;
|
|
3
|
+
}
|
|
4
|
+
export declare class TwoColumnForm implements BrierForm {
|
|
5
|
+
of(probability: number, expected: 0 | 1): number;
|
|
6
|
+
}
|
|
7
|
+
export declare class SquaredErrorForm implements BrierForm {
|
|
8
|
+
of(probability: number, expected: 0 | 1): number;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=brier-form.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-form.d.ts","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,SAAS;IACxB,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CAClD;AAKD,qBAAa,aAAc,YAAW,SAAS;IAC7C,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAKjD;AAID,qBAAa,gBAAiB,YAAW,SAAS;IAChD,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAGjD"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// scikit-learn 1.9.1's brier_score_loss of a one-dimensional y_proba: the
|
|
2
|
+
// two classes are 1 - p and p, and the score is 0.5 times the sum of the
|
|
3
|
+
// squares of the two columns.
|
|
4
|
+
export class TwoColumnForm {
|
|
5
|
+
of(probability, expected) {
|
|
6
|
+
const negative = 1 - expected - (1 - probability);
|
|
7
|
+
const positive = expected - probability;
|
|
8
|
+
return 0.5 * (negative * negative + positive * positive);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
// (p - y)², the squared error of the probability (Brier 1950 for two
|
|
12
|
+
// classes), as numpy gives (p - y) ** 2.
|
|
13
|
+
export class SquaredErrorForm {
|
|
14
|
+
of(probability, expected) {
|
|
15
|
+
return (probability - expected) ** 2;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=brier-form.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brier-form.js","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAQA,0EAA0E;AAC1E,yEAAyE;AACzE,8BAA8B;AAC9B,MAAM,OAAO,aAAa;IACxB,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;QACxC,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;IAC3D,CAAC;CACF;AAED,qEAAqE;AACrE,yCAAyC;AACzC,MAAM,OAAO,gBAAgB;IAC3B,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
2
|
import type { Scorer } from '../scorers/scorer.ts';
|
|
3
|
+
import { type BrierForm } from './brier-form.ts';
|
|
3
4
|
import { type Label } from './label.ts';
|
|
4
5
|
import type { ProbabilityComparison } from './probability-comparison.ts';
|
|
5
6
|
export declare class BrierScore implements Scorer<ProbabilityComparison> {
|
|
6
7
|
#private;
|
|
7
|
-
constructor({ positiveLabel }?: {
|
|
8
|
-
readonly positiveLabel?:
|
|
8
|
+
constructor({ positiveLabel, form, }?: {
|
|
9
|
+
readonly positiveLabel?: undefined;
|
|
10
|
+
readonly form?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
readonly positiveLabel: Label;
|
|
13
|
+
readonly form?: BrierForm;
|
|
9
14
|
});
|
|
10
15
|
score(comparison: ProbabilityComparison): Ratio;
|
|
11
16
|
mean(comparisons: readonly ProbabilityComparison[], weights?: readonly number[]): number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAUzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAIlD,EACV,aAAa,EACb,IAA0B,GAC3B,GACG;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAE,GACjE;QAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAO;IAKrE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAmCV"}
|
|
@@ -1,17 +1,21 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { mean, sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
import { TwoColumnForm } from "./brier-form.js";
|
|
3
4
|
import { compareLabels } from "./label.js";
|
|
4
5
|
// scikit-learn 1.9.1's brier_score_loss. For one answer, the sum over the
|
|
5
6
|
// classes of (1 when the class is the expected label, else 0, minus the
|
|
6
7
|
// stated probability) squared, with numpy's pairwise sum over the classes in
|
|
7
8
|
// their sorted order; times 0.5 for two classes. With a positive label, the
|
|
8
9
|
// answer gives the probability p of that label only, and the two classes are
|
|
9
|
-
// 1 - p and p, as scikit-learn reads a one-dimensional y_proba
|
|
10
|
-
//
|
|
10
|
+
// 1 - p and p, as scikit-learn reads a one-dimensional y_proba; the form
|
|
11
|
+
// (TwoColumnForm by default) gives the score of p. Its ratio is the score
|
|
12
|
+
// over 1, and mean gives the score of a whole run.
|
|
11
13
|
export class BrierScore {
|
|
12
14
|
#positiveLabel;
|
|
13
|
-
|
|
15
|
+
#form;
|
|
16
|
+
constructor({ positiveLabel, form = new TwoColumnForm(), } = {}) {
|
|
14
17
|
this.#positiveLabel = positiveLabel;
|
|
18
|
+
this.#form = form;
|
|
15
19
|
}
|
|
16
20
|
score(comparison) {
|
|
17
21
|
return new Ratio(this.#value(comparison), 1);
|
|
@@ -21,21 +25,18 @@ export class BrierScore {
|
|
|
21
25
|
mean(comparisons, weights) {
|
|
22
26
|
const scores = comparisons.map((comparison) => this.#value(comparison));
|
|
23
27
|
if (weights === undefined)
|
|
24
|
-
return
|
|
28
|
+
return mean(scores);
|
|
25
29
|
if (weights.length !== scores.length)
|
|
26
30
|
throw new RangeError(`The answers and the weights have ${scores.length} and ${weights.length} values.`);
|
|
27
31
|
const weighted = scores.map((score, index) => score * (weights[index] ?? 0));
|
|
28
|
-
return (
|
|
32
|
+
return sum(weighted) / sum(weights);
|
|
29
33
|
}
|
|
30
34
|
#value({ probabilities, expected }) {
|
|
31
35
|
if (this.#positiveLabel !== undefined) {
|
|
32
36
|
const p = probabilities.get(this.#positiveLabel);
|
|
33
37
|
if (p === undefined)
|
|
34
38
|
throw new RangeError(`The answer gives no probability of the positive label ${JSON.stringify(this.#positiveLabel)}.`);
|
|
35
|
-
|
|
36
|
-
const negative = 1 - y - (1 - p);
|
|
37
|
-
const positive = y - p;
|
|
38
|
-
return 0.5 * (negative * negative + positive * positive);
|
|
39
|
+
return this.#form.of(p, expected === this.#positiveLabel ? 1 : 0);
|
|
39
40
|
}
|
|
40
41
|
if (!probabilities.has(expected))
|
|
41
42
|
throw new RangeError(`The answer gives no probability of the expected label ${JSON.stringify(expected)}.`);
|
|
@@ -45,8 +46,8 @@ export class BrierScore {
|
|
|
45
46
|
const difference = (label === expected ? 1 : 0) - probability;
|
|
46
47
|
return difference * difference;
|
|
47
48
|
});
|
|
48
|
-
const
|
|
49
|
-
return squares.length < 3 ? 0.5 *
|
|
49
|
+
const total = sum(squares);
|
|
50
|
+
return squares.length < 3 ? 0.5 * total : total;
|
|
50
51
|
}
|
|
51
52
|
}
|
|
52
53
|
//# sourceMappingURL=brier-score.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAkB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,yEAAyE;AACzE,0EAA0E;AAC1E,mDAAmD;AACnD,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAClC,KAAK,CAAY;IAE1B,YAAY,EACV,aAAa,EACb,IAAI,GAAG,IAAI,aAAa,EAAE,MAGuC,EAAE;QACnE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Counts } from "../aggregation/counts.js";
|
|
3
3
|
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
4
|
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
@@ -110,8 +110,7 @@ export class ClassCounts {
|
|
|
110
110
|
// stated labels are its expected labels (the subset accuracy for a set of
|
|
111
111
|
// classes).
|
|
112
112
|
accuracy() {
|
|
113
|
-
return (
|
|
114
|
-
gsumpw(this.#weights.length, this.#weights, 1));
|
|
113
|
+
return sum(this.#rights) / sum(this.#weights);
|
|
115
114
|
}
|
|
116
115
|
// precision_score: hits / stated of each class, then the average.
|
|
117
116
|
precision(options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAWvD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;CACd;AAED,gEAAgE;AAChE,yEAAyE;AACzE,6EAA6E;AAC7E,sEAAsE;AACtE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IACnC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EAAE,UAAU,EAAoC;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;aACnC,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;aAC9B,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6CAA6C;IAC7C,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,+DAA+D;IAC/D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAgB,EACnE,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,IAAI,GAAG,CACd,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAC3D;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YACtB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC1B,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -16,6 +16,7 @@ export { MacroAverage } from './macro-average.ts';
|
|
|
16
16
|
export { MicroAverage } from './micro-average.ts';
|
|
17
17
|
export { WeightedAverage } from './weighted-average.ts';
|
|
18
18
|
export { BrierScore } from './brier-score.ts';
|
|
19
|
+
export { type BrierForm, SquaredErrorForm, TwoColumnForm, } from './brier-form.ts';
|
|
19
20
|
export { ProbabilityComparer } from './probability-comparer.ts';
|
|
20
21
|
export { ProbabilityComparison } from './probability-comparison.ts';
|
|
21
22
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EACL,KAAK,SAAS,EACd,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -14,6 +14,7 @@ export { MacroAverage } from "./macro-average.js";
|
|
|
14
14
|
export { MicroAverage } from "./micro-average.js";
|
|
15
15
|
export { WeightedAverage } from "./weighted-average.js";
|
|
16
16
|
export { BrierScore } from "./brier-score.js";
|
|
17
|
+
export { SquaredErrorForm, TwoColumnForm, } from "./brier-form.js";
|
|
17
18
|
export { ProbabilityComparer } from "./probability-comparer.js";
|
|
18
19
|
export { ProbabilityComparison } from "./probability-comparison.js";
|
|
19
20
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
// average='macro': the mean of the scores of the selected classes, as
|
|
3
3
|
// numpy's nanmean gives it: a NaN score (zeroDivision NaN) adds 0 and is not
|
|
4
4
|
// counted, and the mean of no score is NaN.
|
|
@@ -9,7 +9,7 @@ export class MacroAverage {
|
|
|
9
9
|
if (counted === 0)
|
|
10
10
|
return Number.NaN;
|
|
11
11
|
const values = scores.map((value) => (Number.isNaN(value) ? 0 : value));
|
|
12
|
-
return
|
|
12
|
+
return sum(values) / counted;
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
15
|
//# sourceMappingURL=macro-average.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,sEAAsE;AACtE,6EAA6E;AAC7E,4CAA4C;AAC5C,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Counts } from "../aggregation/counts.js";
|
|
3
3
|
// average='micro': the counts of the selected classes are added, with numpy's
|
|
4
4
|
// pairwise sum, and scored once.
|
|
5
5
|
export class MicroAverage {
|
|
6
6
|
of({ selected }, score) {
|
|
7
7
|
const classes = [...selected.values()];
|
|
8
|
-
const
|
|
8
|
+
const total = (count) => sum(classes.map(count));
|
|
9
9
|
return score(new Counts({
|
|
10
|
-
hits:
|
|
11
|
-
stated:
|
|
12
|
-
expected:
|
|
10
|
+
hits: total(({ hits }) => hits),
|
|
11
|
+
stated: total(({ stated }) => stated),
|
|
12
|
+
expected: total(({ expected }) => expected),
|
|
13
13
|
}));
|
|
14
14
|
}
|
|
15
15
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,8EAA8E;AAC9E,iCAAiC;AACjC,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,CAAC,KAAiC,EAAE,EAAE,CAClD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1B,OAAO,KAAK,CACV,IAAI,MAAM,CAAC;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;YAC/B,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;YACrC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC5C,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;
|
|
1
|
+
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CASrE"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
// average='weighted': the mean of the scores of the selected classes, each
|
|
3
3
|
// weighted by its true count, as scikit-learn's _nanaverage gives it: a NaN
|
|
4
4
|
// score is dropped with its weight, the mean of no score is NaN, and when the
|
|
@@ -10,7 +10,6 @@ export class WeightedAverage {
|
|
|
10
10
|
.filter(({ value }) => !Number.isNaN(value));
|
|
11
11
|
if (kept.length === 0)
|
|
12
12
|
return Number.NaN;
|
|
13
|
-
const sum = (values) => gsumpw(values.length, values, 1);
|
|
14
13
|
const weights = sum(kept.map(({ weight }) => weight));
|
|
15
14
|
if (weights === 0)
|
|
16
15
|
return sum(kept.map(({ value }) => value)) / kept.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,OAAO,eAAe;IAC1B,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;aAChC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACpE,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5E,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC;IACxE,CAAC;CACF"}
|
|
@@ -1,8 +1,22 @@
|
|
|
1
|
+
import type { JsonData } from '@zukhruf/python';
|
|
1
2
|
import type { Field } from './field.ts';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
readonly
|
|
5
|
-
readonly
|
|
6
|
-
|
|
3
|
+
import type { Path } from './path.ts';
|
|
4
|
+
export type Item = {
|
|
5
|
+
readonly path: Path;
|
|
6
|
+
readonly value: JsonData;
|
|
7
|
+
};
|
|
8
|
+
export type Pair = {
|
|
9
|
+
readonly stated: Item;
|
|
10
|
+
readonly expected: Item;
|
|
7
11
|
};
|
|
12
|
+
export declare class ItemPairs {
|
|
13
|
+
readonly field: Field;
|
|
14
|
+
readonly pairs: readonly Pair[];
|
|
15
|
+
readonly unpairedStated: readonly Item[];
|
|
16
|
+
readonly unpairedExpected: readonly Item[];
|
|
17
|
+
constructor(field: Field, pairs: readonly Pair[], unpairedStated: readonly Item[], unpairedExpected: readonly Item[]);
|
|
18
|
+
get expected(): number;
|
|
19
|
+
get stated(): number;
|
|
20
|
+
get paired(): number;
|
|
21
|
+
}
|
|
8
22
|
//# sourceMappingURL=item-pairs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-pairs.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"item-pairs.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,MAAM,IAAI,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAA;CAAE,CAAC;AAGrE,MAAM,MAAM,IAAI,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;CAAE,CAAC;AAMtE,qBAAa,SAAS;IACpB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,SAAS,IAAI,EAAE,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,IAAI,EAAE,CAAC;gBAGzC,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,cAAc,EAAE,SAAS,IAAI,EAAE,EAC/B,gBAAgB,EAAE,SAAS,IAAI,EAAE;IAQnC,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;CACF"}
|
|
@@ -1,2 +1,26 @@
|
|
|
1
|
-
|
|
1
|
+
// The items of one array of one comparison: the pairs that its alignment
|
|
2
|
+
// made, in the order of the expected items, and the items of each side
|
|
3
|
+
// without a pair, in the order of that side. A side that has no array at the
|
|
4
|
+
// path has no items, so each item of the other side has no pair.
|
|
5
|
+
export class ItemPairs {
|
|
6
|
+
field;
|
|
7
|
+
pairs;
|
|
8
|
+
unpairedStated;
|
|
9
|
+
unpairedExpected;
|
|
10
|
+
constructor(field, pairs, unpairedStated, unpairedExpected) {
|
|
11
|
+
this.field = field;
|
|
12
|
+
this.pairs = pairs;
|
|
13
|
+
this.unpairedStated = unpairedStated;
|
|
14
|
+
this.unpairedExpected = unpairedExpected;
|
|
15
|
+
}
|
|
16
|
+
get expected() {
|
|
17
|
+
return this.pairs.length + this.unpairedExpected.length;
|
|
18
|
+
}
|
|
19
|
+
get stated() {
|
|
20
|
+
return this.pairs.length + this.unpairedStated.length;
|
|
21
|
+
}
|
|
22
|
+
get paired() {
|
|
23
|
+
return this.pairs.length;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
2
26
|
//# sourceMappingURL=item-pairs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-pairs.js","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"item-pairs.js","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAWA,yEAAyE;AACzE,uEAAuE;AACvE,6EAA6E;AAC7E,iEAAiE;AACjE,MAAM,OAAO,SAAS;IACX,KAAK,CAAQ;IACb,KAAK,CAAkB;IACvB,cAAc,CAAkB;IAChC,gBAAgB,CAAkB;IAE3C,YACE,KAAY,EACZ,KAAsB,EACtB,cAA+B,EAC/B,gBAAiC;QAEjC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC3C,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;IAC1D,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;IACxD,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAuB7C,qBAAa,QAAS,YAAW,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;;gBAG7C,IAAI,iBAAuB;IAIvC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,UAAU;
|
|
1
|
+
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAuB7C,qBAAa,QAAS,YAAW,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;;gBAG7C,IAAI,iBAAuB;IAIvC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,UAAU;CA0OrD"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ComparisonSpec } from "./comparison-spec.js";
|
|
2
2
|
import { Comparison } from "./comparison.js";
|
|
3
3
|
import { Field } from "./field.js";
|
|
4
|
+
import { ItemPairs } from "./item-pairs.js";
|
|
4
5
|
import { statedData } from "./json-value.js";
|
|
5
6
|
import { Path } from "./path.js";
|
|
6
7
|
// Walks a stated value and the expected value of its case side by side, as
|
|
@@ -70,35 +71,41 @@ export class TreeWalk {
|
|
|
70
71
|
const agreement = {
|
|
71
72
|
count: (statedItem, expectedItem) => this.#agreeingLeaves(statedItem, expectedItem, item),
|
|
72
73
|
};
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
74
|
+
const expectedOf = new Map(this.#spec.alignmentOf(field).pairs(stated, expected, agreement));
|
|
75
|
+
const statedOf = new Map();
|
|
76
|
+
const unpairedStated = [];
|
|
77
|
+
for (const [index, value] of stated.entries()) {
|
|
78
|
+
const statedItem = { path: statedPath.child(index), value };
|
|
79
|
+
const expectedIndex = expectedOf.get(index);
|
|
80
|
+
if (expectedIndex === undefined)
|
|
81
|
+
unpairedStated.push(statedItem);
|
|
82
|
+
else
|
|
83
|
+
statedOf.set(expectedIndex, statedItem);
|
|
84
|
+
}
|
|
85
|
+
const pairs = [];
|
|
86
|
+
const unpairedExpected = [];
|
|
87
|
+
for (const [index, value] of expected.entries()) {
|
|
88
|
+
const expectedItem = { path: expectedPath.child(index), value };
|
|
89
|
+
const statedItem = statedOf.get(index);
|
|
90
|
+
if (statedItem === undefined)
|
|
91
|
+
unpairedExpected.push(expectedItem);
|
|
92
|
+
else
|
|
93
|
+
pairs.push({ stated: statedItem, expected: expectedItem });
|
|
94
|
+
}
|
|
95
|
+
found.arrays.push(new ItemPairs(field, pairs, unpairedStated, unpairedExpected));
|
|
83
96
|
for (const [index, value] of expected.entries()) {
|
|
84
|
-
const
|
|
85
|
-
this.#walk(
|
|
97
|
+
const statedItem = statedOf.get(index);
|
|
98
|
+
this.#walk(statedItem?.value, value, statedItem?.path ?? statedPath.child(index), expectedPath.child(index), item, found);
|
|
86
99
|
}
|
|
87
|
-
const paired = new Set(statedOf.values());
|
|
88
100
|
for (const [index, value] of stated.entries())
|
|
89
|
-
if (!
|
|
101
|
+
if (!expectedOf.has(index))
|
|
90
102
|
this.#walk(value, undefined, statedPath.child(index), expectedPath.child(index), item, found);
|
|
91
103
|
}
|
|
92
104
|
// Two values of different kinds: each side is walked alone. An array on
|
|
93
105
|
// one side has its items, and the other side has none.
|
|
94
106
|
#apart(stated, expected, statedPath, expectedPath, field, found) {
|
|
95
107
|
if (Array.isArray(stated) || Array.isArray(expected))
|
|
96
|
-
found.arrays.push(
|
|
97
|
-
field,
|
|
98
|
-
expected: Array.isArray(expected) ? expected.length : 0,
|
|
99
|
-
stated: Array.isArray(stated) ? stated.length : 0,
|
|
100
|
-
paired: 0,
|
|
101
|
-
});
|
|
108
|
+
found.arrays.push(new ItemPairs(field, [], TreeWalk.#items(stated, statedPath), TreeWalk.#items(expected, expectedPath)));
|
|
102
109
|
for (const [step, fieldStep, value] of TreeWalk.#children(expected))
|
|
103
110
|
this.#walk(undefined, value, statedPath.child(step), expectedPath.child(step), field.child(fieldStep), found);
|
|
104
111
|
for (const [step, fieldStep, value] of TreeWalk.#children(stated))
|
|
@@ -128,6 +135,15 @@ export class TreeWalk {
|
|
|
128
135
|
for (const [index, child] of value.entries())
|
|
129
136
|
yield [index, Field.item, child];
|
|
130
137
|
}
|
|
138
|
+
// The items of a value that is an array, else none.
|
|
139
|
+
static #items(value, path) {
|
|
140
|
+
if (!Array.isArray(value))
|
|
141
|
+
return [];
|
|
142
|
+
return value.map((item, index) => ({
|
|
143
|
+
path: path.child(index),
|
|
144
|
+
value: item,
|
|
145
|
+
}));
|
|
146
|
+
}
|
|
131
147
|
static #leaf(value, path) {
|
|
132
148
|
if (value === undefined || value instanceof Map || Array.isArray(value))
|
|
133
149
|
return undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;AACnD,OAAO,EAAa,SAAS,EAAa,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAa,MAAM,WAAW,CAAC;AAS5C,2EAA2E;AAC3E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,kEAAkE;AAClE,0DAA0D;AAC1D,MAAM,OAAO,QAAQ;IACV,KAAK,CAAiB;IAE/B,YAAY,IAAI,GAAG,IAAI,cAAc,EAAE;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,MAAe,EAAE,QAAc;QACrC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CACR,IAAI,EACJ,QAAQ,CAAC,QAAQ,EACjB,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,KAAK,CAAC,EAAE,CAAC,EACb,KAAK,CACN,CAAC;QACF,OAAO,IAAI,UAAU,CACnB,QAAQ,EACR,IAAI,EACJ,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,MAAM,CACb,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,8DAA8D;IAC9D,KAAK,CACH,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtE,IAAI,MAAM,YAAY,GAAG,IAAI,QAAQ,YAAY,GAAG;YAClD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;aACpE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;;YACnE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3E,IAAI,IAAI;YAAE,OAAO;QACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,YAAY,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS;YACxD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAChB,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,UAAU;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK;qBACf,YAAY,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;aAChD,CAAC,CAAC;aACA,IAAI,YAAY,KAAK,SAAS;YACjC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;aAC3C,IAAI,UAAU,KAAK,SAAS;YAC/B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,CACL,MAA6B,EAC7B,QAA+B,EAC/B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,QAAQ;YACjC,IAAI,CAAC,KAAK,CACR,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EACf,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM;YAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBACpB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,iEAAiE;IACjE,MAAM,CACJ,MAA2B,EAC3B,QAA6B,EAC7B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAc;YAC3B,KAAK,EAAE,CAAC,UAAU,EAAE,YAAY,EAAE,EAAE,CAClC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC;SACvD,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CACjE,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;QACzC,MAAM,cAAc,GAAW,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC;YAC5D,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,aAAa,KAAK,SAAS;gBAAE,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;;gBAC5D,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,MAAM,gBAAgB,GAAW,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC;YAChE,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,SAAS;gBAAE,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;;gBAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,IAAI,CACf,IAAI,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAC9D,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,CACR,UAAU,EAAE,KAAK,EACjB,KAAK,EACL,UAAU,EAAE,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAC3C,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;YAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EACvB,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,uDAAuD;IACvD,MAAM,CACJ,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAClD,KAAK,CAAC,MAAM,CAAC,IAAI,CACf,IAAI,SAAS,CACX,KAAK,EACL,EAAE,EACF,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACnC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CACxC,CACF,CAAC;QACJ,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC;YACjE,IAAI,CAAC,KAAK,CACR,SAAS,EACT,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/D,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;IACN,CAAC;IAED,0EAA0E;IAC1E,kDAAkD;IAClD,eAAe,CAAC,MAAgB,EAAE,QAAkB,EAAE,IAAW;QAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACtE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACpE,CAAC;IAED,0DAA0D;IAC1D,MAAM,CAAC,YAAY,CACjB,MAA4B,EAC5B,IAAU,EACV,KAAY;QAEZ,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,AAAD,EAAG,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YACtD,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,CAAC,SAAS,CACf,KAA2B;QAE3B,IAAI,KAAK,YAAY,GAAG;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;gBAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;aACvD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE;gBAC1C,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,oDAAoD;IACpD,MAAM,CAAC,MAAM,CAAC,KAA2B,EAAE,IAAU;QACnD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YACvB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAA2B,EAAE,IAAU;QAClD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,aAAa;QAClB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrD,CAAC;CACF"}
|
|
@@ -12,7 +12,7 @@ export { ComparisonSpec, type ComparisonSpecOptions, } from './comparison/compar
|
|
|
12
12
|
export { JsonValue } from './comparison/json-value.ts';
|
|
13
13
|
export type { Comparison } from './comparison/comparison.ts';
|
|
14
14
|
export type { Leaf, LeafPair, LeafValue } from './comparison/leaf-pair.ts';
|
|
15
|
-
export type { ItemPairs } from './comparison/item-pairs.ts';
|
|
15
|
+
export type { Item, ItemPairs, Pair } from './comparison/item-pairs.ts';
|
|
16
16
|
export type { Field, FieldStep } from './comparison/field.ts';
|
|
17
17
|
export type { Path, Step } from './comparison/path.ts';
|
|
18
18
|
export { TreeWalk } from './comparison/tree-walk.ts';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AACxE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC9D,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EACL,KAAK,KAAK,EACV,UAAU,EACV,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.7",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -63,29 +63,24 @@
|
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@zukhruf/python": "^0.8.
|
|
67
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.7",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.7"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"@cfworker/json-schema": "4.1.4",
|
|
71
|
-
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
72
71
|
"munkres": "^2.1.1"
|
|
73
72
|
},
|
|
74
73
|
"peerDependenciesMeta": {
|
|
75
74
|
"@cfworker/json-schema": {
|
|
76
75
|
"optional": true
|
|
77
76
|
},
|
|
78
|
-
"@stdlib/blas-ext-base-gsumpw": {
|
|
79
|
-
"optional": true
|
|
80
|
-
},
|
|
81
77
|
"munkres": {
|
|
82
78
|
"optional": true
|
|
83
79
|
}
|
|
84
80
|
},
|
|
85
81
|
"devDependencies": {
|
|
86
82
|
"@cfworker/json-schema": "4.1.4",
|
|
87
|
-
"@
|
|
88
|
-
"@zukhruf/testing": "^0.8.5",
|
|
83
|
+
"@zukhruf/testing": "^0.8.7",
|
|
89
84
|
"munkres": "^2.1.1",
|
|
90
85
|
"promptfoo": "0.125.1",
|
|
91
86
|
"python-random": "1.0.0"
|