@zukhruf/evals 0.8.6 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +184 -1
- package/dist/classification/brier-score.d.ts.map +1 -1
- package/dist/classification/brier-score.js +5 -5
- package/dist/classification/brier-score.js.map +1 -1
- package/dist/classification/class-counts.d.ts.map +1 -1
- package/dist/classification/class-counts.js +2 -3
- package/dist/classification/class-counts.js.map +1 -1
- package/dist/classification/macro-average.js +2 -2
- package/dist/classification/macro-average.js.map +1 -1
- package/dist/classification/micro-average.js +5 -5
- package/dist/classification/micro-average.js.map +1 -1
- package/dist/classification/weighted-average.d.ts.map +1 -1
- package/dist/classification/weighted-average.js +1 -2
- package/dist/classification/weighted-average.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/scoring/paired-bootstrap.d.ts +24 -0
- package/dist/scoring/paired-bootstrap.d.ts.map +1 -0
- package/dist/scoring/paired-bootstrap.js +94 -0
- package/dist/scoring/paired-bootstrap.js.map +1 -0
- package/dist/scoring/run.d.ts +7 -0
- package/dist/scoring/run.d.ts.map +1 -0
- package/dist/scoring/run.js +19 -0
- package/dist/scoring/run.js.map +1 -0
- package/dist/scoring/test-epochs.d.ts +11 -0
- package/dist/scoring/test-epochs.d.ts.map +1 -0
- package/dist/scoring/test-epochs.js +25 -0
- package/dist/scoring/test-epochs.js.map +1 -0
- package/dist/structured/comparison/item-pairs.d.ts +19 -5
- package/dist/structured/comparison/item-pairs.d.ts.map +1 -1
- package/dist/structured/comparison/item-pairs.js +25 -1
- package/dist/structured/comparison/item-pairs.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +36 -20
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/index.d.ts +1 -1
- package/dist/structured/index.d.ts.map +1 -1
- package/package.json +5 -9
package/README.md
CHANGED
|
@@ -92,6 +92,122 @@ const pageKinds = new ScorerSet(
|
|
|
92
92
|
|
|
93
93
|
`Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
|
|
94
94
|
|
|
95
|
+
## Compare two runs
|
|
96
|
+
|
|
97
|
+
The package has `Run` and `PairedBootstrap` (backlog #3870, [ADR 0004](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)).
|
|
98
|
+
|
|
99
|
+
A caller runs one harness two times on the same tests: once with the baseline model and once with a change, such as another model. Two means side by side do not say if the change is better. `PairedBootstrap` gives one verdict for each name: better, worse or inconclusive, with the difference, its interval and a p-value.
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
import { PairedBootstrap, Run } from '@zukhruf/evals';
|
|
103
|
+
import { defaultRng } from '@zukhruf/random';
|
|
104
|
+
import { Linear, PercentileInterval } from '@zukhruf/statistics';
|
|
105
|
+
|
|
106
|
+
const baseline = new Run();
|
|
107
|
+
const change = new Run();
|
|
108
|
+
for (const answer of baselineAnswers)
|
|
109
|
+
baseline.add(
|
|
110
|
+
answer.testId,
|
|
111
|
+
scorer.score(answer.stated, answer.case),
|
|
112
|
+
answer.case.weight,
|
|
113
|
+
);
|
|
114
|
+
for (const answer of changeAnswers)
|
|
115
|
+
change.add(
|
|
116
|
+
answer.testId,
|
|
117
|
+
scorer.score(answer.stated, answer.case),
|
|
118
|
+
answer.case.weight,
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
const generator = defaultRng(31957);
|
|
122
|
+
const paired = new PairedBootstrap({
|
|
123
|
+
nResamples: 2000,
|
|
124
|
+
randomIndex: (n) => generator.integers(0, n, 1)[0] ?? Number.NaN,
|
|
125
|
+
interval: new PercentileInterval({
|
|
126
|
+
low: 0.025,
|
|
127
|
+
high: 0.975,
|
|
128
|
+
method: new Linear(),
|
|
129
|
+
}),
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
const recall = paired.verdict(baseline, change, 'recall', {
|
|
133
|
+
higherIsBetter: true,
|
|
134
|
+
});
|
|
135
|
+
recall.result; // 'better', 'worse' or 'inconclusive'
|
|
136
|
+
recall.difference.observed; // Totals of the change minus Totals of the baseline
|
|
137
|
+
recall.difference.low; // the 2.5% quantile of the resample differences
|
|
138
|
+
recall.p; // the two-sided p-value of the same resamples
|
|
139
|
+
recall.improvements; // the tests whose value is higher in the change
|
|
140
|
+
|
|
141
|
+
const brier = paired.verdict(baseline, change, 'brier', {
|
|
142
|
+
higherIsBetter: false,
|
|
143
|
+
});
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
A run holds the scorecards of one model. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
// @zukhruf/evals: two runs side by side
|
|
150
|
+
import type {
|
|
151
|
+
ConfidenceIntervalMethod,
|
|
152
|
+
DifferenceInterval,
|
|
153
|
+
} from '@zukhruf/statistics';
|
|
154
|
+
|
|
155
|
+
/** The scorecards of one model on many tests. A test can have many epochs. */
|
|
156
|
+
export class Run {
|
|
157
|
+
/** Adds one epoch of a test. Another weight for a test id that the run has is a RangeError. */
|
|
158
|
+
add(testId: string, scorecard: Scorecard, weight: number): void;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export type Verdict = {
|
|
162
|
+
readonly name: string;
|
|
163
|
+
readonly result: 'better' | 'worse' | 'inconclusive';
|
|
164
|
+
/** change − baseline: its observed value and its interval. */
|
|
165
|
+
readonly difference: DifferenceInterval;
|
|
166
|
+
/** The two-sided percentile-bootstrap p-value, with no correction. */
|
|
167
|
+
readonly p: number;
|
|
168
|
+
readonly improvements: number;
|
|
169
|
+
readonly regressions: number;
|
|
170
|
+
readonly ties: number;
|
|
171
|
+
readonly tests: number;
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
/** The paired bootstrap of two runs on the same tests (Koehn 2004). */
|
|
175
|
+
export class PairedBootstrap {
|
|
176
|
+
constructor(options: {
|
|
177
|
+
nResamples: number;
|
|
178
|
+
randomIndex: (n: number) => number;
|
|
179
|
+
interval: ConfidenceIntervalMethod;
|
|
180
|
+
});
|
|
181
|
+
/** A test id in one run only, or a test with two weights, is a RangeError. A name that one run does not give is an Error. */
|
|
182
|
+
verdict(
|
|
183
|
+
baseline: Run,
|
|
184
|
+
change: Run,
|
|
185
|
+
name: string,
|
|
186
|
+
options: { higherIsBetter: boolean },
|
|
187
|
+
): Verdict;
|
|
188
|
+
}
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
| Call | What it does underneath |
|
|
192
|
+
| ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
193
|
+
| `run.add(testId, scorecard, weight)` | Keeps the scorecard as one epoch of the test. The weight belongs to the case, so another weight for the same test id throws. |
|
|
194
|
+
| `paired.verdict(baseline, change, …)`: join | Pairs the tests by test id, in the order in which the baseline run first got them. A test id in one run only throws, with each such id, as mteb's `pairedBootstrapTest` does. A test whose weight differs between the two runs throws. |
|
|
195
|
+
| reduce | Takes the mean of the epochs of each test, name by name: Ratio(sum of the parts / k, sum of the wholes / k) for k epochs (Miller 2024, §3.1). A scorecard without the name counts as Ratio(0, 0), as in `Totals`. |
|
|
196
|
+
| resample | `IidBootstrap` of `@zukhruf/statistics` draws the pairs of tests `nResamples` times. Each resample takes one `randomIndex` for each test. |
|
|
197
|
+
| statistic | On the same resampled tests: the ratio of sums of the name over the change, minus the same over the baseline, with each weight (Koehn 2004). `difference.observed` is the same statistic on all the tests. |
|
|
198
|
+
| interval and verdict | The `ConfidenceIntervalMethod` gives the interval. With `higherIsBetter: true`, `isAbove(0)` is better and `isBelow(0)` is worse. With `false`, the two change places. Else the verdict is inconclusive. |
|
|
199
|
+
| p | Two times the smaller share of the resample differences that are at or below 0 and at or above 0, at most 1, as mteb's `pairedBootstrapTest` gives it. The caller applies a correction, such as Holm. |
|
|
200
|
+
| counts | Counts the tests by the sign of (change value − baseline value), with the direction: improvements, regressions and ties. |
|
|
201
|
+
|
|
202
|
+
`Bootstrap` and `PairedBootstrap` use different statistics. `Bootstrap` takes the mean of the values of the scorecards: the samples average. For a plain scorer, `Totals` gives the micro average, so the interval of `Bootstrap` is not the interval of the number of `Totals`. `PairedBootstrap` uses the statistic of `Totals`, so its interval is the interval of its observed difference (ADR 0004, "Consequences").
|
|
203
|
+
|
|
204
|
+
The design does not resample clustered tests, such as many questions about one page, and it reduces the epochs only by the mean.
|
|
205
|
+
|
|
206
|
+
| Who uses it | For | Backlog |
|
|
207
|
+
| --------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------- |
|
|
208
|
+
| context-compiler, startup-credits page kinds | jev-1.14 against another model on the 121 pages, with the runs of each model as epochs | context-compiler #3882 slice 2, mutex #3870 |
|
|
209
|
+
| offers-evals and context-compiler, startup-credits offers | a run against the baseline run of its harness, for recall, precision, kind and amount | mutex #3870, context-compiler #3882 |
|
|
210
|
+
|
|
95
211
|
## Compare structured values
|
|
96
212
|
|
|
97
213
|
`./structured` compares a structured value with the value that a person labelled: one walk pairs the leaves of the two values, the alignment of each array is a Strategy (by position, or by best match with the Hungarian algorithm), and each scorer gives a ratio for one record. SOB's evaluate.py is a configuration of the same scorers. [docs/structured.md](docs/structured.md) shows the whole API: `ComparisonSpec`, `TreeWalk`, `Comparison`, `PathMatches`, `ValueMatches`, `ItemMatches`, `TokenF1`, `TypePrecision`, `RequiredKeyRecall`, the gates, `PerField`, `JsonSchema`, `ConfigCase` and `TextVarsCase`. Its decisions are the ADRs in [docs/adr/structured](docs/adr/structured), from the time when it was `@zukhruf/structured-eval`.
|
|
@@ -102,7 +218,7 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
|
|
|
102
218
|
|
|
103
219
|
`./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
|
|
104
220
|
|
|
105
|
-
It needs
|
|
221
|
+
It needs no peer: `npm install @zukhruf/evals`. Its sums are `sum` and `mean` of `@zukhruf/statistics`, which give `numpy.sum` and `numpy.mean` bit for bit.
|
|
106
222
|
|
|
107
223
|
### For each answer
|
|
108
224
|
|
|
@@ -171,6 +287,23 @@ run.add(new LabelComparer().compare('a', testCase), testCase.weight);
|
|
|
171
287
|
- `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
|
|
172
288
|
- `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
|
|
173
289
|
|
|
290
|
+
### The class set
|
|
291
|
+
|
|
292
|
+
Status: design for review (backlog #3970, [ADR 0006](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md)). The released package takes the classes from the answers, as scikit-learn's `labels=None` does.
|
|
293
|
+
|
|
294
|
+
A harness can state any text, such as the answer null of a failed call. If the answers give the classes, each such text adds a class, and the macro averages change. So the caller gives the class set, scikit-learn's `labels`, to the comparer and to the counts. The answers never add a class.
|
|
295
|
+
|
|
296
|
+
```ts
|
|
297
|
+
// Proposal: no code exists yet.
|
|
298
|
+
import { ClassCounts, LabelComparer } from '@zukhruf/evals/classification';
|
|
299
|
+
|
|
300
|
+
const pageKinds = ['program', 'home', 'list', 'investor', 'other'];
|
|
301
|
+
const comparer = new LabelComparer({ labels: pageKinds });
|
|
302
|
+
const counts = new ClassCounts({ labels: pageKinds, multilabel: false });
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
A label outside the class set counts in no class, as scikit-learn gives it when the caller gives `labels`. A stated label outside it is a miss of the expected class, and it is not a predicted count of any class. `LabelsMatch` still reads it as a wrong answer. `PerClass` and `AllClassMatches` read the class set of the comparison, so they take no `labels`. `ClassCounts.ofIndicators` takes its classes from the columns, as today. To port a scikit-learn call with `labels=None`, give the sorted labels of both sides.
|
|
306
|
+
|
|
174
307
|
### Probabilities
|
|
175
308
|
|
|
176
309
|
`ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
|
|
@@ -228,6 +361,53 @@ export default {
|
|
|
228
361
|
|
|
229
362
|
The name is the promptfoo `metric`, unless the scorer assertion gives its own `metric`. A name that the module does not hold is a compile error. promptfoo shows the mean of each `metric` over the tests. A `ScorerAssertion` gives one value for one test, so a micro average needs `PromptfooAssertion`, whose ratios keep their wholes.
|
|
230
363
|
|
|
364
|
+
### A failed call
|
|
365
|
+
|
|
366
|
+
Status: design for review (backlog #3970, [ADR 0005](docs/adr/0005-a-failed-call-is-the-answer-null.md)). The released package does not have `FailedCall` yet.
|
|
367
|
+
|
|
368
|
+
A failed call is a model call that failed for its one test, also after its retries. promptfoo does not grade a result whose provider gives `error` or no output. That result is an error result: it has no named scores, so its case adds nothing to the wholes, and the recall goes up. A failed call is an answer of the harness: the answer null, the text `'null'`. promptfoo grades it, and each family reads it as no answer. `FailedCall` is a coinage of this package. A consumer uses one of two forms.
|
|
369
|
+
|
|
370
|
+
When the provider is your own, return `new FailedCall(reason).response(metadata)`. promptfoo grades the result, and `promptfoo retry <evalId>` does not run it again. The reason is in `metadata.failed`.
|
|
371
|
+
|
|
372
|
+
```ts
|
|
373
|
+
// Proposal: no code exists yet.
|
|
374
|
+
import { FailedCall } from '@zukhruf/evals/promptfoo';
|
|
375
|
+
|
|
376
|
+
const [result] = await rule.compute([row.input], options?.abortSignal);
|
|
377
|
+
if (result !== undefined && 'error' in result)
|
|
378
|
+
return new FailedCall(result.error).response(metadata);
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
When the provider is a built-in one, such as `openai:chat`, an `afterEach` extension hook grades each error result on `FailedCall.answer`. The result stays an error result, so `promptfoo retry <evalId>` and `--retry-errors` run it again.
|
|
382
|
+
|
|
383
|
+
```ts
|
|
384
|
+
// Proposal: no code exists yet.
|
|
385
|
+
import {
|
|
386
|
+
type AfterEachExtensionHookContext,
|
|
387
|
+
ResultFailureReason,
|
|
388
|
+
assertions,
|
|
389
|
+
} from 'promptfoo';
|
|
390
|
+
|
|
391
|
+
import { FailedCall } from '@zukhruf/evals/promptfoo';
|
|
392
|
+
|
|
393
|
+
export async function afterEach({
|
|
394
|
+
test,
|
|
395
|
+
result,
|
|
396
|
+
}: AfterEachExtensionHookContext): Promise<void> {
|
|
397
|
+
if (result.failureReason !== ResultFailureReason.ERROR) return;
|
|
398
|
+
const graded = await assertions.runAssertions({
|
|
399
|
+
test,
|
|
400
|
+
providerResponse: { output: FailedCall.answer },
|
|
401
|
+
vars: test.vars ?? {},
|
|
402
|
+
});
|
|
403
|
+
Object.assign(result.namedScores, graded.namedScores);
|
|
404
|
+
}
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
A hook cannot write `namedScoreWeights`, so promptfoo gives each score of the hook the weight 1. Use the hook only with scorer assertions whose metrics weigh 1, as in SOB. A `PromptfooAssertion` gives each ratio its whole as the weight, so it needs the first form.
|
|
408
|
+
|
|
409
|
+
A failure of the run, such as a 401, is not a failed call: each call would fail. Throw it. The result then stays an error result, and `promptfoo retry <evalId>` runs it again.
|
|
410
|
+
|
|
231
411
|
## What the package does not do
|
|
232
412
|
|
|
233
413
|
- It does not run a harness, load a dataset, store results or show them. promptfoo does that.
|
|
@@ -610,3 +790,6 @@ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the
|
|
|
610
790
|
- [ADR 0001: one package, with a subpath for each shape of answer](docs/adr/0001-one-package-with-a-subpath-for-each-shape-of-answer.md)
|
|
611
791
|
- [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
|
|
612
792
|
- [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
|
|
793
|
+
- [ADR 0004: two runs are compared by a paired bootstrap over their tests](docs/adr/0004-two-runs-are-compared-by-a-paired-bootstrap-over-their-tests.md)
|
|
794
|
+
- [ADR 0005: a failed call is the answer null](docs/adr/0005-a-failed-call-is-the-answer-null.md) (design for review)
|
|
795
|
+
- [ADR 0006: classification counts only the classes that the caller gives](docs/adr/0006-classification-counts-only-the-classes-that-the-caller-gives.md) (design for review)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAUzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAIlD,EACV,aAAa,EACb,IAA0B,GAC3B,GACG;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAE,GACjE;QAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAO;IAKrE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;
|
|
1
|
+
{"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAUzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAIlD,EACV,aAAa,EACb,IAA0B,GAC3B,GACG;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAE,GACjE;QAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAO;IAKrE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAmCV"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { mean, sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Ratio } from "../aggregation/ratio.js";
|
|
3
3
|
import { TwoColumnForm } from "./brier-form.js";
|
|
4
4
|
import { compareLabels } from "./label.js";
|
|
@@ -25,11 +25,11 @@ export class BrierScore {
|
|
|
25
25
|
mean(comparisons, weights) {
|
|
26
26
|
const scores = comparisons.map((comparison) => this.#value(comparison));
|
|
27
27
|
if (weights === undefined)
|
|
28
|
-
return
|
|
28
|
+
return mean(scores);
|
|
29
29
|
if (weights.length !== scores.length)
|
|
30
30
|
throw new RangeError(`The answers and the weights have ${scores.length} and ${weights.length} values.`);
|
|
31
31
|
const weighted = scores.map((score, index) => score * (weights[index] ?? 0));
|
|
32
|
-
return (
|
|
32
|
+
return sum(weighted) / sum(weights);
|
|
33
33
|
}
|
|
34
34
|
#value({ probabilities, expected }) {
|
|
35
35
|
if (this.#positiveLabel !== undefined) {
|
|
@@ -46,8 +46,8 @@ export class BrierScore {
|
|
|
46
46
|
const difference = (label === expected ? 1 : 0) - probability;
|
|
47
47
|
return difference * difference;
|
|
48
48
|
});
|
|
49
|
-
const
|
|
50
|
-
return squares.length < 3 ? 0.5 *
|
|
49
|
+
const total = sum(squares);
|
|
50
|
+
return squares.length < 3 ? 0.5 * total : total;
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
//# sourceMappingURL=brier-score.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAkB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,yEAAyE;AACzE,0EAA0E;AAC1E,mDAAmD;AACnD,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAClC,KAAK,CAAY;IAE1B,YAAY,EACV,aAAa,EACb,IAAI,GAAG,IAAI,aAAa,EAAE,MAGuC,EAAE;QACnE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;
|
|
1
|
+
{"version":3,"file":"class-counts.d.ts","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAKvD,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CACxC,CAAC;AAkBF,qBAAa,WAAW;;gBAOV,EAAE,UAAU,EAAE,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAM5D,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAoBf,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACtC,GAAG,WAAW;IAwDf,GAAG,CAAC,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAwBtD,MAAM,IAAI,KAAK,EAAE;IAQjB,QAAQ,IAAI,MAAM;IAKlB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAOxC,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IASrC,KAAK,CAAC,EACJ,IAAI,EACJ,IAAuB,EACvB,GAAG,OAAO,EACX,EAAE,YAAY,GAAG;QAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;KAC9B,GAAG,MAAM;IASV,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;IAQrC,eAAe,IAAI,eAAe;CAgCnC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Counts } from "../aggregation/counts.js";
|
|
3
3
|
import { CountsForm, } from "../aggregation/f-measure-form.js";
|
|
4
4
|
import { ConfusionMatrix } from "./confusion-matrix.js";
|
|
@@ -110,8 +110,7 @@ export class ClassCounts {
|
|
|
110
110
|
// stated labels are its expected labels (the subset accuracy for a set of
|
|
111
111
|
// classes).
|
|
112
112
|
accuracy() {
|
|
113
|
-
return (
|
|
114
|
-
gsumpw(this.#weights.length, this.#weights, 1));
|
|
113
|
+
return sum(this.#rights) / sum(this.#weights);
|
|
115
114
|
}
|
|
116
115
|
// precision_score: hits / stated of each class, then the average.
|
|
117
116
|
precision(options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"class-counts.js","sourceRoot":"","sources":["../../src/classification/class-counts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAWvD,sEAAsE;AACtE,MAAM,KAAK;IACT,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,QAAQ,GAAG,CAAC,CAAC;CACd;AAED,gEAAgE;AAChE,yEAAyE;AACzE,6EAA6E;AAC7E,sEAAsE;AACtE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,OAAO,WAAW;IACb,WAAW,CAAU;IACrB,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IACnC,OAAO,GAAa,EAAE,CAAC;IACvB,QAAQ,GAAa,EAAE,CAAC;IACxB,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEvD,YAAY,EAAE,UAAU,EAAoC;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,gEAAgE;IAChE,uEAAuE;IACvE,MAAM,CAAC,EAAE,CAAC,EACR,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACrE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;gBAC9B,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;aACnC,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,yBAAyB;IACzB,MAAM,CAAC,YAAY,CAAC,EAClB,QAAQ,EACR,MAAM,EACN,OAAO,GAKR;QACC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,GAAsB,EAAE,EAAE;YACxC,IACE,GAAG,CAAC,MAAM,KAAK,OAAO;gBACtB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAE/C,MAAM,IAAI,UAAU,CAClB,2CAA2C,OAAO,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnG,CAAC;YACJ,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CACjE,QAAQ,EACR,MAAM,EACN,OAAO,CACR;YACC,MAAM,CAAC,GAAG,CACR,IAAI,eAAe,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;gBACzB,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;aAC9B,CAAC,EACF,MAAM,CACP,CAAC;QACJ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,oCAAoC;IACpC,MAAM,CAAC,CAAC,QAAQ,CACd,QAAyB,EACzB,MAAuB,EACvB,OAAsC;QAEtC,MAAM,OAAO,GAAG,IAAI,UAAU,CAC5B,+DAA+D,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,CACtJ,CAAC;QACF,IACE,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;YACjC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC;YAE7D,MAAM,OAAO,CAAC;QAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6CAA6C;IAC7C,GAAG,CAAC,UAA2B,EAAE,MAAc;QAC7C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,UAAU,CAClB,iFAAiF,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CACrJ,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;YAClD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC;QACjC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,+DAA+D;IAC/D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,YAAY;IACZ,QAAQ;QACN,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,kEAAkE;IAClE,SAAS,CAAC,OAAqB;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CACzC,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,OAAqB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAC7C,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,+DAA+D;IAC/D,wBAAwB;IACxB,KAAK,CAAC,EACJ,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,EACvB,GAAG,OAAO,EAIX;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,eAAe;QACb,IAAI,IAAI,CAAC,WAAW;YAClB,MAAM,IAAI,UAAU,CAClB,sEAAsE,CACvE,CAAC;QACJ,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CACJ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,CAAC,EAAgB,EACnE,KAA6C;QAE7C,OAAO,OAAO,CAAC,EAAE,CACf;YACE,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,IAAI,GAAG,CACd,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAC3D;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,YAAY,CAC1C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YACtB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC1B,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
// average='macro': the mean of the scores of the selected classes, as
|
|
3
3
|
// numpy's nanmean gives it: a NaN score (zeroDivision NaN) adds 0 and is not
|
|
4
4
|
// counted, and the mean of no score is NaN.
|
|
@@ -9,7 +9,7 @@ export class MacroAverage {
|
|
|
9
9
|
if (counted === 0)
|
|
10
10
|
return Number.NaN;
|
|
11
11
|
const values = scores.map((value) => (Number.isNaN(value) ? 0 : value));
|
|
12
|
-
return
|
|
12
|
+
return sum(values) / counted;
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
15
|
//# sourceMappingURL=macro-average.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"macro-average.js","sourceRoot":"","sources":["../../src/classification/macro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,sEAAsE;AACtE,6EAA6E;AAC7E,4CAA4C;AAC5C,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
import { Counts } from "../aggregation/counts.js";
|
|
3
3
|
// average='micro': the counts of the selected classes are added, with numpy's
|
|
4
4
|
// pairwise sum, and scored once.
|
|
5
5
|
export class MicroAverage {
|
|
6
6
|
of({ selected }, score) {
|
|
7
7
|
const classes = [...selected.values()];
|
|
8
|
-
const
|
|
8
|
+
const total = (count) => sum(classes.map(count));
|
|
9
9
|
return score(new Counts({
|
|
10
|
-
hits:
|
|
11
|
-
stated:
|
|
12
|
-
expected:
|
|
10
|
+
hits: total(({ hits }) => hits),
|
|
11
|
+
stated: total(({ stated }) => stated),
|
|
12
|
+
expected: total(({ expected }) => expected),
|
|
13
13
|
}));
|
|
14
14
|
}
|
|
15
15
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"micro-average.js","sourceRoot":"","sources":["../../src/classification/micro-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAGlD,8EAA8E;AAC9E,iCAAiC;AACjC,MAAM,OAAO,YAAY;IACvB,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,CAAC,KAAiC,EAAE,EAAE,CAClD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1B,OAAO,KAAK,CACV,IAAI,MAAM,CAAC;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;YAC/B,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;YACrC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC5C,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;
|
|
1
|
+
{"version":3,"file":"weighted-average.d.ts","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAMrD,qBAAa,eAAgB,YAAW,OAAO;IAC7C,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM;CASrE"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { sum } from '@zukhruf/statistics';
|
|
2
2
|
// average='weighted': the mean of the scores of the selected classes, each
|
|
3
3
|
// weighted by its true count, as scikit-learn's _nanaverage gives it: a NaN
|
|
4
4
|
// score is dropped with its weight, the mean of no score is NaN, and when the
|
|
@@ -10,7 +10,6 @@ export class WeightedAverage {
|
|
|
10
10
|
.filter(({ value }) => !Number.isNaN(value));
|
|
11
11
|
if (kept.length === 0)
|
|
12
12
|
return Number.NaN;
|
|
13
|
-
const sum = (values) => gsumpw(values.length, values, 1);
|
|
14
13
|
const weights = sum(kept.map(({ weight }) => weight));
|
|
15
14
|
if (weights === 0)
|
|
16
15
|
return sum(kept.map(({ value }) => value)) / kept.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"weighted-average.js","sourceRoot":"","sources":["../../src/classification/weighted-average.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAK1C,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,OAAO,eAAe;IAC1B,EAAE,CAAC,EAAE,QAAQ,EAAW,EAAE,KAAiC;QACzD,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;aAChC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACpE,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5E,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC;IACxE,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export type { Check, Scorecard } from './scoring/scorecard.ts';
|
|
|
6
6
|
export type { CaseReader, ScorecardScorer, Weighted, } from './scoring/scorecard-scorer.ts';
|
|
7
7
|
export { Totals } from './scoring/totals.ts';
|
|
8
8
|
export { Bootstrap, type WeightedScorecard } from './scoring/bootstrap.ts';
|
|
9
|
+
export { Run } from './scoring/run.ts';
|
|
10
|
+
export { PairedBootstrap, type Verdict } from './scoring/paired-bootstrap.ts';
|
|
9
11
|
export type { Scorer } from './scorers/scorer.ts';
|
|
10
12
|
export type { Matches } from './scorers/matches.ts';
|
|
11
13
|
export { Precision } from './scorers/precision.ts';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AACzC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AACzC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,KAAK,OAAO,EAAE,MAAM,+BAA+B,CAAC;AAC9E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -4,6 +4,8 @@ export { Counts } from "./aggregation/counts.js";
|
|
|
4
4
|
export { CountsForm, PrecisionRecallForm, } from "./aggregation/f-measure-form.js";
|
|
5
5
|
export { Totals } from "./scoring/totals.js";
|
|
6
6
|
export { Bootstrap } from "./scoring/bootstrap.js";
|
|
7
|
+
export { Run } from "./scoring/run.js";
|
|
8
|
+
export { PairedBootstrap } from "./scoring/paired-bootstrap.js";
|
|
7
9
|
export { Precision } from "./scorers/precision.js";
|
|
8
10
|
export { Recall } from "./scorers/recall.js";
|
|
9
11
|
export { FBeta } from "./scorers/f-beta.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EAEV,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AAOzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EAEV,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AAOzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,eAAe,EAAgB,MAAM,+BAA+B,CAAC;AAG9E,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type ConfidenceIntervalMethod, DifferenceInterval } from '@zukhruf/statistics';
|
|
2
|
+
import { type Run } from './run.ts';
|
|
3
|
+
export type Verdict = {
|
|
4
|
+
readonly name: string;
|
|
5
|
+
readonly result: 'better' | 'worse' | 'inconclusive';
|
|
6
|
+
readonly difference: DifferenceInterval;
|
|
7
|
+
readonly p: number;
|
|
8
|
+
readonly improvements: number;
|
|
9
|
+
readonly regressions: number;
|
|
10
|
+
readonly ties: number;
|
|
11
|
+
readonly tests: number;
|
|
12
|
+
};
|
|
13
|
+
export declare class PairedBootstrap {
|
|
14
|
+
#private;
|
|
15
|
+
constructor({ nResamples, randomIndex, interval, }: {
|
|
16
|
+
readonly nResamples: number;
|
|
17
|
+
readonly randomIndex: (n: number) => number;
|
|
18
|
+
readonly interval: ConfidenceIntervalMethod;
|
|
19
|
+
});
|
|
20
|
+
verdict(baseline: Run, change: Run, name: string, { higherIsBetter }: {
|
|
21
|
+
readonly higherIsBetter: boolean;
|
|
22
|
+
}): Verdict;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=paired-bootstrap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paired-bootstrap.d.ts","sourceRoot":"","sources":["../../src/scoring/paired-bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,wBAAwB,EAC7B,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAE,KAAK,GAAG,EAAY,MAAM,UAAU,CAAC;AAE9C,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,GAAG,cAAc,CAAC;IAErD,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;IAExC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAgBF,qBAAa,eAAe;;gBAKd,EACV,UAAU,EACV,WAAW,EACX,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;QAC5C,QAAQ,CAAC,QAAQ,EAAE,wBAAwB,CAAC;KAC7C;IAMD,OAAO,CACL,QAAQ,EAAE,GAAG,EACb,MAAM,EAAE,GAAG,EACX,IAAI,EAAE,MAAM,EACZ,EAAE,cAAc,EAAE,EAAE;QAAE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;KAAE,GACvD,OAAO;CA8GX"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { DifferenceInterval, IidBootstrap, } from '@zukhruf/statistics';
|
|
2
|
+
import { RatioOfSums } from "../aggregation/ratio-of-sums.js";
|
|
3
|
+
import { runTests } from "./run.js";
|
|
4
|
+
// The paired bootstrap of two runs on the same tests (Koehn 2004): each
|
|
5
|
+
// resample draws as many tests as there are, with replacement, and gives the
|
|
6
|
+
// ratio of sums of the name over the change minus the same over the baseline,
|
|
7
|
+
// on the same tests. The verdict reads the interval of these differences
|
|
8
|
+
// against 0, in the direction of the name (ADR 0004). The generator keeps its
|
|
9
|
+
// state between calls, so the order of the calls is the order of the draws.
|
|
10
|
+
export class PairedBootstrap {
|
|
11
|
+
#bootstrap;
|
|
12
|
+
#interval;
|
|
13
|
+
#nResamples;
|
|
14
|
+
constructor({ nResamples, randomIndex, interval, }) {
|
|
15
|
+
this.#bootstrap = new IidBootstrap({ nResamples, randomIndex });
|
|
16
|
+
this.#interval = interval;
|
|
17
|
+
this.#nResamples = nResamples;
|
|
18
|
+
}
|
|
19
|
+
verdict(baseline, change, name, { higherIsBetter }) {
|
|
20
|
+
const pairs = PairedBootstrap.#pairs(baseline, change, name);
|
|
21
|
+
const distribution = this.#bootstrap.distribution(pairs, PairedBootstrap.#difference);
|
|
22
|
+
const interval = this.#interval.of(distribution);
|
|
23
|
+
const difference = new DifferenceInterval(interval.low, interval.high, PairedBootstrap.#difference(pairs));
|
|
24
|
+
const direction = higherIsBetter ? 1 : -1;
|
|
25
|
+
const signs = pairs.map((pair) => Math.sign((pair.change.value - pair.baseline.value) * direction));
|
|
26
|
+
return {
|
|
27
|
+
name,
|
|
28
|
+
result: PairedBootstrap.#result(difference, higherIsBetter),
|
|
29
|
+
difference,
|
|
30
|
+
p: this.#p(distribution),
|
|
31
|
+
improvements: signs.filter((sign) => sign > 0).length,
|
|
32
|
+
regressions: signs.filter((sign) => sign < 0).length,
|
|
33
|
+
ties: signs.filter((sign) => sign === 0).length,
|
|
34
|
+
tests: pairs.length,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
// Two times the smaller share of the differences at or below 0 and at or
|
|
38
|
+
// above 0, at most 1, as mteb's pairedBootstrapTest gives it.
|
|
39
|
+
#p(distribution) {
|
|
40
|
+
const share = (side) => distribution.filter(side).length / this.#nResamples;
|
|
41
|
+
return Math.min(1, 2 *
|
|
42
|
+
Math.min(share((value) => value <= 0), share((value) => value >= 0)));
|
|
43
|
+
}
|
|
44
|
+
static #result(difference, higherIsBetter) {
|
|
45
|
+
const [better, worse] = higherIsBetter
|
|
46
|
+
? [difference.isAbove(0), difference.isBelow(0)]
|
|
47
|
+
: [difference.isBelow(0), difference.isAbove(0)];
|
|
48
|
+
if (better)
|
|
49
|
+
return 'better';
|
|
50
|
+
if (worse)
|
|
51
|
+
return 'worse';
|
|
52
|
+
return 'inconclusive';
|
|
53
|
+
}
|
|
54
|
+
static #difference(pairs) {
|
|
55
|
+
return (RatioOfSums.of(pairs.map((pair) => pair.change.times(pair.weight)))
|
|
56
|
+
.value -
|
|
57
|
+
RatioOfSums.of(pairs.map((pair) => pair.baseline.times(pair.weight)))
|
|
58
|
+
.value);
|
|
59
|
+
}
|
|
60
|
+
// The tests of the two runs, paired by test id in the order of the
|
|
61
|
+
// baseline's first adds. The pairing refuses a test id that one run does
|
|
62
|
+
// not have, as mteb's pairedBootstrapTest refuses (strike 2,
|
|
63
|
+
// docs/copied-code.md).
|
|
64
|
+
static #pairs(baseline, change, name) {
|
|
65
|
+
const baselineTests = runTests(baseline);
|
|
66
|
+
const changeTests = runTests(change);
|
|
67
|
+
const unpaired = [...baselineTests.keys(), ...changeTests.keys()].filter((id) => !(baselineTests.has(id) && changeTests.has(id)));
|
|
68
|
+
if (unpaired.length > 0)
|
|
69
|
+
throw new RangeError(`A paired bootstrap needs the same test ids in the baseline and the change. Not in both: ${unpaired.join(', ')}`);
|
|
70
|
+
const joined = [...baselineTests].flatMap(([id, test]) => {
|
|
71
|
+
const changeTest = changeTests.get(id);
|
|
72
|
+
return changeTest === undefined
|
|
73
|
+
? []
|
|
74
|
+
: [{ id, baseline: test, change: changeTest }];
|
|
75
|
+
});
|
|
76
|
+
const reweighted = joined.filter((test) => test.baseline.weight !== test.change.weight);
|
|
77
|
+
if (reweighted.length > 0)
|
|
78
|
+
throw new RangeError(`A test has the same weight in the baseline and the change. Other weights: ${reweighted
|
|
79
|
+
.map((test) => `${test.id} (${test.baseline.weight} and ${test.change.weight})`)
|
|
80
|
+
.join(', ')}`);
|
|
81
|
+
for (const [run, tests] of [
|
|
82
|
+
['baseline', baselineTests],
|
|
83
|
+
['change', changeTests],
|
|
84
|
+
])
|
|
85
|
+
if (![...tests.values()].some((test) => test.gives(name)))
|
|
86
|
+
throw new Error(`No scorecard of the ${run} run gave a ratio named "${name}"`);
|
|
87
|
+
return joined.map((test) => ({
|
|
88
|
+
baseline: test.baseline.mean(name),
|
|
89
|
+
change: test.change.mean(name),
|
|
90
|
+
weight: test.baseline.weight,
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=paired-bootstrap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paired-bootstrap.js","sourceRoot":"","sources":["../../src/scoring/paired-bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,kBAAkB,EAClB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAY,QAAQ,EAAE,MAAM,UAAU,CAAC;AAuB9C,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,4EAA4E;AAC5E,MAAM,OAAO,eAAe;IACjB,UAAU,CAAe;IACzB,SAAS,CAA2B;IACpC,WAAW,CAAS;IAE7B,YAAY,EACV,UAAU,EACV,WAAW,EACX,QAAQ,GAKT;QACC,IAAI,CAAC,UAAU,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,OAAO,CACL,QAAa,EACb,MAAW,EACX,IAAY,EACZ,EAAE,cAAc,EAAwC;QAExD,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAC/C,KAAK,EACL,eAAe,CAAC,WAAW,CAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,kBAAkB,CACvC,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,IAAI,EACb,eAAe,CAAC,WAAW,CAAC,KAAK,CAAC,CACnC,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,CACjE,CAAC;QACF,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE,cAAc,CAAC;YAC3D,UAAU;YACV,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC;YACxB,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM;YACrD,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM;YACpD,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM;YAC/C,KAAK,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,8DAA8D;IAC9D,EAAE,CAAC,YAA+B;QAChC,MAAM,KAAK,GAAG,CAAC,IAAgC,EAAE,EAAE,CACjD,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;QACtD,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,CAAC;YACC,IAAI,CAAC,GAAG,CACN,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,EAC5B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAC7B,CACJ,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,OAAO,CACZ,UAA8B,EAC9B,cAAuB;QAEvB,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,cAAc;YACpC,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,MAAM;YAAE,OAAO,QAAQ,CAAC;QAC5B,IAAI,KAAK;YAAE,OAAO,OAAO,CAAC;QAC1B,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,KAAsB;QACvC,OAAO,CACL,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;aAChE,KAAK;YACR,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;iBAClE,KAAK,CACT,CAAC;IACJ,CAAC;IAED,mEAAmE;IACnE,yEAAyE;IACzE,6DAA6D;IAC7D,wBAAwB;IACxB,MAAM,CAAC,MAAM,CAAC,QAAa,EAAE,MAAW,EAAE,IAAY;QACpD,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CACtE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACxD,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YACrB,MAAM,IAAI,UAAU,CAClB,2FAA2F,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjH,CAAC;QACJ,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;YACvD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,OAAO,UAAU,KAAK,SAAS;gBAC7B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAC9B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,CACtD,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YACvB,MAAM,IAAI,UAAU,CAClB,6EAA6E,UAAU;iBACpF,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CACnE;iBACA,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI;YACzB,CAAC,UAAU,EAAE,aAAa,CAAC;YAC3B,CAAC,QAAQ,EAAE,WAAW,CAAC;SACf;YACR,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,uBAAuB,GAAG,4BAA4B,IAAI,GAAG,CAC9D,CAAC;QACN,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Scorecard } from './scorecard.ts';
|
|
2
|
+
import { TestEpochs } from './test-epochs.ts';
|
|
3
|
+
export declare class Run {
|
|
4
|
+
add(testId: string, scorecard: Scorecard, weight: number): void;
|
|
5
|
+
}
|
|
6
|
+
export declare function runTests(run: Run): ReadonlyMap<string, TestEpochs>;
|
|
7
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,qBAAa,GAAG;IACd,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;CAWhE;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAElE"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { TestEpochs } from "./test-epochs.js";
|
|
2
|
+
// The tests of each run, in the order of their first add. PairedBootstrap of
|
|
3
|
+
// this package reads them through runTests; the package does not export it.
|
|
4
|
+
const testsOfRun = new WeakMap();
|
|
5
|
+
const testsOf = (run) => testsOfRun.getOrInsertComputed(run, () => new Map());
|
|
6
|
+
// The scorecards of one model on many tests. A second add of a test id is
|
|
7
|
+
// one more epoch of that test.
|
|
8
|
+
export class Run {
|
|
9
|
+
add(testId, scorecard, weight) {
|
|
10
|
+
const test = testsOf(this).getOrInsertComputed(testId, () => new TestEpochs(weight));
|
|
11
|
+
if (test.weight !== weight)
|
|
12
|
+
throw new RangeError(`Test "${testId}" has the weight ${test.weight}, not ${weight}: the weight belongs to the case`);
|
|
13
|
+
test.add(scorecard);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export function runTests(run) {
|
|
17
|
+
return testsOf(run);
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,6EAA6E;AAC7E,4EAA4E;AAC5E,MAAM,UAAU,GAAG,IAAI,OAAO,EAAgC,CAAC;AAC/D,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC3B,UAAU,CAAC,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;AAEvD,0EAA0E;AAC1E,+BAA+B;AAC/B,MAAM,OAAO,GAAG;IACd,GAAG,CAAC,MAAc,EAAE,SAAoB,EAAE,MAAc;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAC5C,MAAM,EACN,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAC7B,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YACxB,MAAM,IAAI,UAAU,CAClB,SAAS,MAAM,oBAAoB,IAAI,CAAC,MAAM,SAAS,MAAM,kCAAkC,CAChG,CAAC;QACJ,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,GAAQ;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Ratio } from '../aggregation/ratio.ts';
|
|
2
|
+
import type { Scorecard } from './scorecard.ts';
|
|
3
|
+
export declare class TestEpochs {
|
|
4
|
+
#private;
|
|
5
|
+
readonly weight: number;
|
|
6
|
+
constructor(weight: number);
|
|
7
|
+
add(scorecard: Scorecard): void;
|
|
8
|
+
gives(name: string): boolean;
|
|
9
|
+
mean(name: string): Ratio;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=test-epochs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-epochs.d.ts","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,qBAAa,UAAU;;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAGZ,MAAM,EAAE,MAAM;IAI1B,GAAG,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAI/B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAO5B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;CAS1B"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { RatioOfSums } from "../aggregation/ratio-of-sums.js";
|
|
2
|
+
import { Ratio } from "../aggregation/ratio.js";
|
|
3
|
+
// The epochs of one test of a run, with the weight of its case.
|
|
4
|
+
export class TestEpochs {
|
|
5
|
+
weight;
|
|
6
|
+
#scorecards = [];
|
|
7
|
+
constructor(weight) {
|
|
8
|
+
this.weight = weight;
|
|
9
|
+
}
|
|
10
|
+
add(scorecard) {
|
|
11
|
+
this.#scorecards.push(scorecard);
|
|
12
|
+
}
|
|
13
|
+
gives(name) {
|
|
14
|
+
return this.#scorecards.some((scorecard) => scorecard.ratios.has(name));
|
|
15
|
+
}
|
|
16
|
+
// The mean of the epochs: Ratio(sum of the parts / k, sum of the wholes / k)
|
|
17
|
+
// for k epochs (Miller 2024, §3.1). A scorecard without the name counts as
|
|
18
|
+
// Ratio(0, 0), as in Totals.
|
|
19
|
+
mean(name) {
|
|
20
|
+
const sum = RatioOfSums.of(this.#scorecards.map((scorecard) => scorecard.ratios.get(name) ?? new Ratio(0, 0))).ratio;
|
|
21
|
+
const k = this.#scorecards.length;
|
|
22
|
+
return new Ratio(sum.part / k, sum.whole / k);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=test-epochs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-epochs.js","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,gEAAgE;AAChE,MAAM,OAAO,UAAU;IACZ,MAAM,CAAS;IACf,WAAW,GAAgB,EAAE,CAAC;IAEvC,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,SAAoB;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,6BAA6B;IAC7B,IAAI,CAAC,IAAY;QACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,CACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAClB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAC7D,CACF,CAAC,KAAK,CAAC;QACR,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAClC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;CACF"}
|
|
@@ -1,8 +1,22 @@
|
|
|
1
|
+
import type { JsonData } from '@zukhruf/python';
|
|
1
2
|
import type { Field } from './field.ts';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
readonly
|
|
5
|
-
readonly
|
|
6
|
-
|
|
3
|
+
import type { Path } from './path.ts';
|
|
4
|
+
export type Item = {
|
|
5
|
+
readonly path: Path;
|
|
6
|
+
readonly value: JsonData;
|
|
7
|
+
};
|
|
8
|
+
export type Pair = {
|
|
9
|
+
readonly stated: Item;
|
|
10
|
+
readonly expected: Item;
|
|
7
11
|
};
|
|
12
|
+
export declare class ItemPairs {
|
|
13
|
+
readonly field: Field;
|
|
14
|
+
readonly pairs: readonly Pair[];
|
|
15
|
+
readonly unpairedStated: readonly Item[];
|
|
16
|
+
readonly unpairedExpected: readonly Item[];
|
|
17
|
+
constructor(field: Field, pairs: readonly Pair[], unpairedStated: readonly Item[], unpairedExpected: readonly Item[]);
|
|
18
|
+
get expected(): number;
|
|
19
|
+
get stated(): number;
|
|
20
|
+
get paired(): number;
|
|
21
|
+
}
|
|
8
22
|
//# sourceMappingURL=item-pairs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-pairs.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"item-pairs.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,MAAM,IAAI,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAA;CAAE,CAAC;AAGrE,MAAM,MAAM,IAAI,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;CAAE,CAAC;AAMtE,qBAAa,SAAS;IACpB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,SAAS,IAAI,EAAE,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,IAAI,EAAE,CAAC;gBAGzC,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,cAAc,EAAE,SAAS,IAAI,EAAE,EAC/B,gBAAgB,EAAE,SAAS,IAAI,EAAE;IAQnC,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;CACF"}
|
|
@@ -1,2 +1,26 @@
|
|
|
1
|
-
|
|
1
|
+
// The items of one array of one comparison: the pairs that its alignment
|
|
2
|
+
// made, in the order of the expected items, and the items of each side
|
|
3
|
+
// without a pair, in the order of that side. A side that has no array at the
|
|
4
|
+
// path has no items, so each item of the other side has no pair.
|
|
5
|
+
export class ItemPairs {
|
|
6
|
+
field;
|
|
7
|
+
pairs;
|
|
8
|
+
unpairedStated;
|
|
9
|
+
unpairedExpected;
|
|
10
|
+
constructor(field, pairs, unpairedStated, unpairedExpected) {
|
|
11
|
+
this.field = field;
|
|
12
|
+
this.pairs = pairs;
|
|
13
|
+
this.unpairedStated = unpairedStated;
|
|
14
|
+
this.unpairedExpected = unpairedExpected;
|
|
15
|
+
}
|
|
16
|
+
get expected() {
|
|
17
|
+
return this.pairs.length + this.unpairedExpected.length;
|
|
18
|
+
}
|
|
19
|
+
get stated() {
|
|
20
|
+
return this.pairs.length + this.unpairedStated.length;
|
|
21
|
+
}
|
|
22
|
+
get paired() {
|
|
23
|
+
return this.pairs.length;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
2
26
|
//# sourceMappingURL=item-pairs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-pairs.js","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"item-pairs.js","sourceRoot":"","sources":["../../../src/structured/comparison/item-pairs.ts"],"names":[],"mappings":"AAWA,yEAAyE;AACzE,uEAAuE;AACvE,6EAA6E;AAC7E,iEAAiE;AACjE,MAAM,OAAO,SAAS;IACX,KAAK,CAAQ;IACb,KAAK,CAAkB;IACvB,cAAc,CAAkB;IAChC,gBAAgB,CAAkB;IAE3C,YACE,KAAY,EACZ,KAAsB,EACtB,cAA+B,EAC/B,gBAAiC;QAEjC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC3C,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;IAC1D,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;IACxD,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAuB7C,qBAAa,QAAS,YAAW,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;;gBAG7C,IAAI,iBAAuB;IAIvC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,UAAU;
|
|
1
|
+
{"version":3,"file":"tree-walk.d.ts","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAuB7C,qBAAa,QAAS,YAAW,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;;gBAG7C,IAAI,iBAAuB;IAIvC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,UAAU;CA0OrD"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ComparisonSpec } from "./comparison-spec.js";
|
|
2
2
|
import { Comparison } from "./comparison.js";
|
|
3
3
|
import { Field } from "./field.js";
|
|
4
|
+
import { ItemPairs } from "./item-pairs.js";
|
|
4
5
|
import { statedData } from "./json-value.js";
|
|
5
6
|
import { Path } from "./path.js";
|
|
6
7
|
// Walks a stated value and the expected value of its case side by side, as
|
|
@@ -70,35 +71,41 @@ export class TreeWalk {
|
|
|
70
71
|
const agreement = {
|
|
71
72
|
count: (statedItem, expectedItem) => this.#agreeingLeaves(statedItem, expectedItem, item),
|
|
72
73
|
};
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
74
|
+
const expectedOf = new Map(this.#spec.alignmentOf(field).pairs(stated, expected, agreement));
|
|
75
|
+
const statedOf = new Map();
|
|
76
|
+
const unpairedStated = [];
|
|
77
|
+
for (const [index, value] of stated.entries()) {
|
|
78
|
+
const statedItem = { path: statedPath.child(index), value };
|
|
79
|
+
const expectedIndex = expectedOf.get(index);
|
|
80
|
+
if (expectedIndex === undefined)
|
|
81
|
+
unpairedStated.push(statedItem);
|
|
82
|
+
else
|
|
83
|
+
statedOf.set(expectedIndex, statedItem);
|
|
84
|
+
}
|
|
85
|
+
const pairs = [];
|
|
86
|
+
const unpairedExpected = [];
|
|
87
|
+
for (const [index, value] of expected.entries()) {
|
|
88
|
+
const expectedItem = { path: expectedPath.child(index), value };
|
|
89
|
+
const statedItem = statedOf.get(index);
|
|
90
|
+
if (statedItem === undefined)
|
|
91
|
+
unpairedExpected.push(expectedItem);
|
|
92
|
+
else
|
|
93
|
+
pairs.push({ stated: statedItem, expected: expectedItem });
|
|
94
|
+
}
|
|
95
|
+
found.arrays.push(new ItemPairs(field, pairs, unpairedStated, unpairedExpected));
|
|
83
96
|
for (const [index, value] of expected.entries()) {
|
|
84
|
-
const
|
|
85
|
-
this.#walk(
|
|
97
|
+
const statedItem = statedOf.get(index);
|
|
98
|
+
this.#walk(statedItem?.value, value, statedItem?.path ?? statedPath.child(index), expectedPath.child(index), item, found);
|
|
86
99
|
}
|
|
87
|
-
const paired = new Set(statedOf.values());
|
|
88
100
|
for (const [index, value] of stated.entries())
|
|
89
|
-
if (!
|
|
101
|
+
if (!expectedOf.has(index))
|
|
90
102
|
this.#walk(value, undefined, statedPath.child(index), expectedPath.child(index), item, found);
|
|
91
103
|
}
|
|
92
104
|
// Two values of different kinds: each side is walked alone. An array on
|
|
93
105
|
// one side has its items, and the other side has none.
|
|
94
106
|
#apart(stated, expected, statedPath, expectedPath, field, found) {
|
|
95
107
|
if (Array.isArray(stated) || Array.isArray(expected))
|
|
96
|
-
found.arrays.push(
|
|
97
|
-
field,
|
|
98
|
-
expected: Array.isArray(expected) ? expected.length : 0,
|
|
99
|
-
stated: Array.isArray(stated) ? stated.length : 0,
|
|
100
|
-
paired: 0,
|
|
101
|
-
});
|
|
108
|
+
found.arrays.push(new ItemPairs(field, [], TreeWalk.#items(stated, statedPath), TreeWalk.#items(expected, expectedPath)));
|
|
102
109
|
for (const [step, fieldStep, value] of TreeWalk.#children(expected))
|
|
103
110
|
this.#walk(undefined, value, statedPath.child(step), expectedPath.child(step), field.child(fieldStep), found);
|
|
104
111
|
for (const [step, fieldStep, value] of TreeWalk.#children(stated))
|
|
@@ -128,6 +135,15 @@ export class TreeWalk {
|
|
|
128
135
|
for (const [index, child] of value.entries())
|
|
129
136
|
yield [index, Field.item, child];
|
|
130
137
|
}
|
|
138
|
+
// The items of a value that is an array, else none.
|
|
139
|
+
static #items(value, path) {
|
|
140
|
+
if (!Array.isArray(value))
|
|
141
|
+
return [];
|
|
142
|
+
return value.map((item, index) => ({
|
|
143
|
+
path: path.child(index),
|
|
144
|
+
value: item,
|
|
145
|
+
}));
|
|
146
|
+
}
|
|
131
147
|
static #leaf(value, path) {
|
|
132
148
|
if (value === undefined || value instanceof Map || Array.isArray(value))
|
|
133
149
|
return undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"tree-walk.js","sourceRoot":"","sources":["../../../src/structured/comparison/tree-walk.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAkB,MAAM,YAAY,CAAC;AACnD,OAAO,EAAa,SAAS,EAAa,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAa,MAAM,WAAW,CAAC;AAS5C,2EAA2E;AAC3E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,kEAAkE;AAClE,0DAA0D;AAC1D,MAAM,OAAO,QAAQ;IACV,KAAK,CAAiB;IAE/B,YAAY,IAAI,GAAG,IAAI,cAAc,EAAE;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,MAAe,EAAE,QAAc;QACrC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CACR,IAAI,EACJ,QAAQ,CAAC,QAAQ,EACjB,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,IAAI,CAAC,EAAE,CAAC,EACZ,IAAI,KAAK,CAAC,EAAE,CAAC,EACb,KAAK,CACN,CAAC;QACF,OAAO,IAAI,UAAU,CACnB,QAAQ,EACR,IAAI,EACJ,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,MAAM,CACb,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,8DAA8D;IAC9D,KAAK,CACH,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtE,IAAI,MAAM,YAAY,GAAG,IAAI,QAAQ,YAAY,GAAG;YAClD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;aACpE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;;YACnE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3E,IAAI,IAAI;YAAE,OAAO;QACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,YAAY,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS;YACxD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAChB,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,UAAU;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK;qBACf,YAAY,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;aAChD,CAAC,CAAC;aACA,IAAI,YAAY,KAAK,SAAS;YACjC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;aAC3C,IAAI,UAAU,KAAK,SAAS;YAC/B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,CACL,MAA6B,EAC7B,QAA+B,EAC/B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,QAAQ;YACjC,IAAI,CAAC,KAAK,CACR,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EACf,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM;YAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBACpB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EACrB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAChB,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,iEAAiE;IACjE,MAAM,CACJ,MAA2B,EAC3B,QAA6B,EAC7B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAc;YAC3B,KAAK,EAAE,CAAC,UAAU,EAAE,YAAY,EAAE,EAAE,CAClC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC;SACvD,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CACjE,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;QACzC,MAAM,cAAc,GAAW,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC;YAC5D,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,aAAa,KAAK,SAAS;gBAAE,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;;gBAC5D,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,MAAM,gBAAgB,GAAW,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC;YAChE,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,SAAS;gBAAE,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;;gBAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,IAAI,CACf,IAAI,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAC9D,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,CACR,UAAU,EAAE,KAAK,EACjB,KAAK,EACL,UAAU,EAAE,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAC3C,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;YAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EACvB,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EACzB,IAAI,EACJ,KAAK,CACN,CAAC;IACR,CAAC;IAED,wEAAwE;IACxE,uDAAuD;IACvD,MAAM,CACJ,MAA4B,EAC5B,QAA8B,EAC9B,UAAgB,EAChB,YAAkB,EAClB,KAAY,EACZ,KAAY;QAEZ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAClD,KAAK,CAAC,MAAM,CAAC,IAAI,CACf,IAAI,SAAS,CACX,KAAK,EACL,EAAE,EACF,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACnC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CACxC,CACF,CAAC;QACJ,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC;YACjE,IAAI,CAAC,KAAK,CACR,SAAS,EACT,KAAK,EACL,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;QACJ,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/D,IAAI,CAAC,KAAK,CACR,KAAK,EACL,SAAS,EACT,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EACtB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EACtB,KAAK,CACN,CAAC;IACN,CAAC;IAED,0EAA0E;IAC1E,kDAAkD;IAClD,eAAe,CAAC,MAAgB,EAAE,QAAkB,EAAE,IAAW;QAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACtE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACpE,CAAC;IAED,0DAA0D;IAC1D,MAAM,CAAC,YAAY,CACjB,MAA4B,EAC5B,IAAU,EACV,KAAY;QAEZ,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,AAAD,EAAG,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YACtD,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,CAAC,SAAS,CACf,KAA2B;QAE3B,IAAI,KAAK,YAAY,GAAG;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;gBAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;aACvD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE;gBAC1C,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,oDAAoD;IACpD,MAAM,CAAC,MAAM,CAAC,KAA2B,EAAE,IAAU;QACnD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YACvB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAA2B,EAAE,IAAU;QAClD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,aAAa;QAClB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrD,CAAC;CACF"}
|
|
@@ -12,7 +12,7 @@ export { ComparisonSpec, type ComparisonSpecOptions, } from './comparison/compar
|
|
|
12
12
|
export { JsonValue } from './comparison/json-value.ts';
|
|
13
13
|
export type { Comparison } from './comparison/comparison.ts';
|
|
14
14
|
export type { Leaf, LeafPair, LeafValue } from './comparison/leaf-pair.ts';
|
|
15
|
-
export type { ItemPairs } from './comparison/item-pairs.ts';
|
|
15
|
+
export type { Item, ItemPairs, Pair } from './comparison/item-pairs.ts';
|
|
16
16
|
export type { Field, FieldStep } from './comparison/field.ts';
|
|
17
17
|
export type { Path, Step } from './comparison/path.ts';
|
|
18
18
|
export { TreeWalk } from './comparison/tree-walk.ts';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/structured/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,6CAA6C,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,2CAA2C,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oDAAoD,CAAC;AACzF,YAAY,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AACxE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC9D,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EACL,KAAK,KAAK,EACV,UAAU,EACV,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/evals",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.8",
|
|
4
4
|
"description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -63,29 +63,25 @@
|
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@zukhruf/python": "^0.8.
|
|
67
|
-
"@zukhruf/statistics": "^0.8.
|
|
66
|
+
"@zukhruf/python": "^0.8.8",
|
|
67
|
+
"@zukhruf/statistics": "^0.8.8"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"@cfworker/json-schema": "4.1.4",
|
|
71
|
-
"@stdlib/blas-ext-base-gsumpw": "^0.3.1",
|
|
72
71
|
"munkres": "^2.1.1"
|
|
73
72
|
},
|
|
74
73
|
"peerDependenciesMeta": {
|
|
75
74
|
"@cfworker/json-schema": {
|
|
76
75
|
"optional": true
|
|
77
76
|
},
|
|
78
|
-
"@stdlib/blas-ext-base-gsumpw": {
|
|
79
|
-
"optional": true
|
|
80
|
-
},
|
|
81
77
|
"munkres": {
|
|
82
78
|
"optional": true
|
|
83
79
|
}
|
|
84
80
|
},
|
|
85
81
|
"devDependencies": {
|
|
86
82
|
"@cfworker/json-schema": "4.1.4",
|
|
87
|
-
"@
|
|
88
|
-
"@zukhruf/testing": "^0.8.
|
|
83
|
+
"@zukhruf/random": "*",
|
|
84
|
+
"@zukhruf/testing": "^0.8.8",
|
|
89
85
|
"munkres": "^2.1.1",
|
|
90
86
|
"promptfoo": "0.125.1",
|
|
91
87
|
"python-random": "1.0.0"
|