@zukhruf/evals 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -10
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
- package/dist/aggregation/ratio-of-sums.js +7 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -1
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/index.d.ts +11 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +10 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +6 -6
- package/dist/scorers/f1.d.ts.map +1 -1
- package/dist/scorers/f1.js +5 -9
- package/dist/scorers/f1.js.map +1 -1
- package/dist/scorers/penalized.d.ts.map +1 -1
- package/dist/scorers/penalized.js +2 -1
- package/dist/scorers/penalized.js.map +1 -1
- package/dist/scoring/bootstrap.js +3 -5
- package/dist/scoring/bootstrap.js.map +1 -1
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
- package/dist/structured/comparison/json-value.d.ts +1 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -1
- package/dist/structured/comparison/json-value.js +11 -2
- package/dist/structured/comparison/json-value.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts +5 -5
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +8 -4
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/cpython/python-text.d.ts +0 -1
- package/dist/structured/cpython/python-text.d.ts.map +1 -1
- package/dist/structured/cpython/python-text.js +1 -18
- package/dist/structured/cpython/python-text.js.map +1 -1
- package/dist/structured/index.d.ts +0 -2
- package/dist/structured/index.d.ts.map +1 -1
- package/dist/structured/index.js +0 -2
- package/dist/structured/index.js.map +1 -1
- package/dist/structured/scorers/token-f1.d.ts +3 -1
- package/dist/structured/scorers/token-f1.d.ts.map +1 -1
- package/dist/structured/scorers/token-f1.js +8 -5
- package/dist/structured/scorers/token-f1.js.map +1 -1
- package/dist/structured/scoring/per-field.d.ts +3 -6
- package/dist/structured/scoring/per-field.d.ts.map +1 -1
- package/dist/structured/scoring/per-field.js +8 -17
- package/dist/structured/scoring/per-field.js.map +1 -1
- package/package.json +10 -4
- package/dist/aggregation/f1-ratio.d.ts +0 -8
- package/dist/aggregation/f1-ratio.d.ts.map +0 -1
- package/dist/aggregation/f1-ratio.js +0 -24
- package/dist/aggregation/f1-ratio.js.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
- package/dist/structured/scoring/scorer-set.d.ts +0 -15
- package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
- package/dist/structured/scoring/scorer-set.js +0 -60
- package/dist/structured/scoring/scorer-set.js.map +0 -1
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ The answers of evals have different shapes: a structured record, a label, a rank
|
|
|
17
17
|
|
|
18
18
|
A subpath imports only the core. It never imports another subpath.
|
|
19
19
|
|
|
20
|
-
The
|
|
20
|
+
The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. Today `./structured` is the one family. `./classification` (#3889) is the next family.
|
|
21
21
|
|
|
22
22
|
## Score one answer
|
|
23
23
|
|
|
@@ -145,7 +145,7 @@ counts.confusionMatrix(); // confusion_matrix, labels in sorted order
|
|
|
145
145
|
|
|
146
146
|
## Use it with promptfoo
|
|
147
147
|
|
|
148
|
-
>
|
|
148
|
+
> `ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels, and `./classification` is not in this release (#3889). [docs/structured.md](docs/structured.md) has a structured sample of today.
|
|
149
149
|
|
|
150
150
|
`PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
|
|
151
151
|
|
|
@@ -330,7 +330,11 @@ export interface ScorecardScorer<Case extends Weighted> {
|
|
|
330
330
|
}
|
|
331
331
|
/** Gives one ratio for each group of an input, such as each field or each class. */
|
|
332
332
|
export abstract class Grouped<Input> {
|
|
333
|
-
|
|
333
|
+
constructor(scorer: Scorer<Input>);
|
|
334
|
+
/** The groups of an input (Template Method). */
|
|
335
|
+
protected abstract groups(input: Input): Iterable<[string, Input]>;
|
|
336
|
+
/** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
|
|
337
|
+
ratios(input: Input, label: string): [string, Ratio][];
|
|
334
338
|
}
|
|
335
339
|
export class ScorerSet<
|
|
336
340
|
Case extends Weighted,
|
|
@@ -344,6 +348,8 @@ export class ScorerSet<
|
|
|
344
348
|
|
|
345
349
|
/** Adds ratios: the sum of the parts divided by the sum of the wholes. */
|
|
346
350
|
export class RatioOfSums {
|
|
351
|
+
/** The ratio of sums of a list of ratios, in one call. */
|
|
352
|
+
static of(ratios: Iterable<Ratio>): RatioOfSums;
|
|
347
353
|
add(ratio: Ratio): void;
|
|
348
354
|
get ratio(): Ratio;
|
|
349
355
|
get value(): number;
|
|
@@ -404,16 +410,22 @@ export type FileAssertion = {
|
|
|
404
410
|
readonly threshold: number;
|
|
405
411
|
readonly value: string; // file://<module>:<name>
|
|
406
412
|
};
|
|
413
|
+
/** The vars of the test that promptfoo gives a javascript assertion of a shared defaultTest. */
|
|
414
|
+
export type VarsContext = {
|
|
415
|
+
readonly vars: Readonly<Record<string, unknown>>;
|
|
416
|
+
};
|
|
417
|
+
/** What an assertion module needs of an assertion. A ScorerAssertion is one. */
|
|
418
|
+
export type ModuleAssertion = {
|
|
419
|
+
readonly metric?: string | undefined;
|
|
420
|
+
readonly weight: number;
|
|
421
|
+
readonly threshold: number;
|
|
422
|
+
readonly value: (output: string, context: VarsContext) => number;
|
|
423
|
+
};
|
|
407
424
|
/** The module that promptfoo loads the function of each scorer assertion from (structured-eval ADR 0013). */
|
|
408
425
|
export class AssertionModule<Name extends string> {
|
|
409
|
-
constructor(
|
|
410
|
-
url: string,
|
|
411
|
-
assertions: Readonly<Record<Name, ScorerAssertion<Weighted, unknown>>>,
|
|
412
|
-
);
|
|
426
|
+
constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
|
|
413
427
|
/** The default export of the module: the value function of each name. */
|
|
414
|
-
readonly functions: Readonly<
|
|
415
|
-
Record<string, ScorerAssertion<Weighted, unknown>['value']>
|
|
416
|
-
>;
|
|
428
|
+
readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
|
|
417
429
|
assert(...names: Name[]): FileAssertion[];
|
|
418
430
|
}
|
|
419
431
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Counts } from './counts.ts';
|
|
2
|
+
export interface FMeasureForm {
|
|
3
|
+
of(counts: Counts, beta: number): number;
|
|
4
|
+
}
|
|
5
|
+
export declare class CountsForm implements FMeasureForm {
|
|
6
|
+
of({ hits, stated, expected }: Counts, beta: number): number;
|
|
7
|
+
}
|
|
8
|
+
export declare class PrecisionRecallForm implements FMeasureForm {
|
|
9
|
+
of({ hits, stated, expected }: Counts, beta: number): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=f-measure-form.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-form.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1C;AAMD,qBAAa,UAAW,YAAW,YAAY;IAC7C,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAI7D;AAKD,qBAAa,mBAAoB,YAAW,YAAY;IACtD,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAO7D"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Ratio } from "./ratio.js";
|
|
2
|
+
// (1 + β²)·hits / (β²·expected + stated), in the order of the operations of
|
|
3
|
+
// scikit-learn 1.9.1's fbeta_score (sklearn/metrics/_classification.py). It
|
|
4
|
+
// is the part of an F-measure ratio divided by its whole, and 0 when the
|
|
5
|
+
// whole is 0.
|
|
6
|
+
export class CountsForm {
|
|
7
|
+
of({ hits, stated, expected }, beta) {
|
|
8
|
+
const beta2 = beta ** 2;
|
|
9
|
+
return new Ratio((1 + beta2) * hits, beta2 * expected + stated).value;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
// (1 + β²)·P·R / (β²·P + R), precision and recall first (van Rijsbergen
|
|
13
|
+
// 1979), and 0 when there are no hits. With β = 1 it is 2·p·r/(p + r), the
|
|
14
|
+
// arithmetic of SOB's evaluate.py (f1_from_counts, token_f1).
|
|
15
|
+
export class PrecisionRecallForm {
|
|
16
|
+
of({ hits, stated, expected }, beta) {
|
|
17
|
+
if (hits === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
const beta2 = beta ** 2;
|
|
20
|
+
const precision = hits / stated;
|
|
21
|
+
const recall = hits / expected;
|
|
22
|
+
return ((1 + beta2) * precision * recall) / (beta2 * precision + recall);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=f-measure-form.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-form.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAQnC,4EAA4E;AAC5E,4EAA4E;AAC5E,yEAAyE;AACzE,cAAc;AACd,MAAM,OAAO,UAAU;IACrB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;IACxE,CAAC;CACF;AAED,wEAAwE;AACxE,2EAA2E;AAC3E,8DAA8D;AAC9D,MAAM,OAAO,mBAAmB;IAC9B,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Counts } from './counts.ts';
|
|
2
|
+
import type { FMeasureForm } from './f-measure-form.ts';
|
|
3
|
+
import { Ratio } from './ratio.ts';
|
|
4
|
+
export declare class FMeasureRatio extends Ratio {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(counts: Counts, { beta, form }: {
|
|
7
|
+
readonly beta: number;
|
|
8
|
+
readonly form: FMeasureForm;
|
|
9
|
+
});
|
|
10
|
+
get value(): number;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=f-measure-ratio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,aAAc,SAAQ,KAAK;;gBAMpC,MAAM,EAAE,MAAM,EACd,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;KAAE;IASxE,IAAa,KAAK,IAAI,MAAM,CAE3B;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Ratio } from "./ratio.js";
|
|
2
|
+
// The ratio of an F-measure: its part is (1 + β²)·hits and its whole is
|
|
3
|
+
// β²·expected + stated, so many of them add up to the micro F-measure. Its
|
|
4
|
+
// value is the value of its form, which can differ from part / whole in the
|
|
5
|
+
// last bit (ADR 0003).
|
|
6
|
+
export class FMeasureRatio extends Ratio {
|
|
7
|
+
#counts;
|
|
8
|
+
#beta;
|
|
9
|
+
#form;
|
|
10
|
+
constructor(counts, { beta, form }) {
|
|
11
|
+
const beta2 = beta ** 2;
|
|
12
|
+
super((1 + beta2) * counts.hits, beta2 * counts.expected + counts.stated);
|
|
13
|
+
this.#counts = counts;
|
|
14
|
+
this.#beta = beta;
|
|
15
|
+
this.#form = form;
|
|
16
|
+
}
|
|
17
|
+
get value() {
|
|
18
|
+
return this.#form.of(this.#counts, this.#beta);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=f-measure-ratio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,wEAAwE;AACxE,2EAA2E;AAC3E,4EAA4E;AAC5E,uBAAuB;AACvB,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,KAAK,CAAe;IAE7B,YACE,MAAc,EACd,EAAE,IAAI,EAAE,IAAI,EAA0D;QAEtE,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAa,KAAK;QAChB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAKtB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW;IAM/C,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
|
|
@@ -7,6 +7,13 @@ import { Ratio } from "./ratio.js";
|
|
|
7
7
|
export class RatioOfSums {
|
|
8
8
|
#parts = new NeumaierSum();
|
|
9
9
|
#wholes = new NeumaierSum();
|
|
10
|
+
// The ratio of sums of the ratios, in their order.
|
|
11
|
+
static of(ratios) {
|
|
12
|
+
const sum = new RatioOfSums();
|
|
13
|
+
for (const ratio of ratios)
|
|
14
|
+
sum.add(ratio);
|
|
15
|
+
return sum;
|
|
16
|
+
}
|
|
10
17
|
add(ratio) {
|
|
11
18
|
this.#parts.add(ratio.part);
|
|
12
19
|
this.#wholes.add(ratio.whole);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,mDAAmD;IACnD,MAAM,CAAC,EAAE,CAAC,MAAuB;QAC/B,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Matches } from '../scorers/matches.ts';
|
|
3
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
export declare class AllClassMatches implements Matches<LabelComparison> {
|
|
5
|
+
counts({ stated, expected }: LabelComparison): Counts;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=all-class-matches.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAOtD"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// The counts of every class in one answer: the hits are the labels that both
|
|
3
|
+
// sides have, the stated and the expected are the labels of each side.
|
|
4
|
+
// Pooled over the answers, they are scikit-learn's micro counts.
|
|
5
|
+
export class AllClassMatches {
|
|
6
|
+
counts({ stated, expected }) {
|
|
7
|
+
return new Counts({
|
|
8
|
+
hits: stated.intersection(expected).size,
|
|
9
|
+
stated: stated.size,
|
|
10
|
+
expected: expected.size,
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=all-class-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Matches } from '../scorers/matches.ts';
|
|
3
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
import type { Label } from './label.ts';
|
|
5
|
+
export declare class ClassMatches implements Matches<LabelComparison> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(label: Label);
|
|
8
|
+
counts({ stated, expected }: LabelComparison): Counts;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=class-matches.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,YAAa,YAAW,OAAO,CAAC,eAAe,CAAC;;gBAG/C,KAAK,EAAE,KAAK;IAIxB,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAStD"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// The counts of one class in one answer: the hit is 1 when both sides have
|
|
3
|
+
// the label, the stated 1 when the answer has it, the expected 1 when the
|
|
4
|
+
// case has it. Pooled over the answers, they are scikit-learn's true
|
|
5
|
+
// positives, predicted and true of the class.
|
|
6
|
+
export class ClassMatches {
|
|
7
|
+
#label;
|
|
8
|
+
constructor(label) {
|
|
9
|
+
this.#label = label;
|
|
10
|
+
}
|
|
11
|
+
counts({ stated, expected }) {
|
|
12
|
+
const isStated = stated.has(this.#label) ? 1 : 0;
|
|
13
|
+
const isExpected = expected.has(this.#label) ? 1 : 0;
|
|
14
|
+
return new Counts({
|
|
15
|
+
hits: isStated * isExpected,
|
|
16
|
+
stated: isStated,
|
|
17
|
+
expected: isExpected,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=class-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"class-matches.js","sourceRoot":"","sources":["../../src/classification/class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAKlD,2EAA2E;AAC3E,0EAA0E;AAC1E,qEAAqE;AACrE,8CAA8C;AAC9C,MAAM,OAAO,YAAY;IACd,MAAM,CAAQ;IAEvB,YAAY,KAAY;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,QAAQ,GAAG,UAAU;YAC3B,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { AllClassMatches } from './all-class-matches.ts';
|
|
2
|
+
export { ClassMatches } from './class-matches.ts';
|
|
3
|
+
export { LabelCase } from './label-case.ts';
|
|
4
|
+
export { LabelComparer } from './label-comparer.ts';
|
|
5
|
+
export { LabelComparison } from './label-comparison.ts';
|
|
6
|
+
export { LabelConfigCase } from './label-config-case.ts';
|
|
7
|
+
export { LabelVarsCase } from './label-vars-case.ts';
|
|
8
|
+
export { LabelsMatch } from './labels-match.ts';
|
|
9
|
+
export type { Label } from './label.ts';
|
|
10
|
+
export { PerClass } from './per-class.ts';
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { AllClassMatches } from "./all-class-matches.js";
|
|
2
|
+
export { ClassMatches } from "./class-matches.js";
|
|
3
|
+
export { LabelCase } from "./label-case.js";
|
|
4
|
+
export { LabelComparer } from "./label-comparer.js";
|
|
5
|
+
export { LabelComparison } from "./label-comparison.js";
|
|
6
|
+
export { LabelConfigCase } from "./label-config-case.js";
|
|
7
|
+
export { LabelVarsCase } from "./label-vars-case.js";
|
|
8
|
+
export { LabelsMatch } from "./labels-match.js";
|
|
9
|
+
export { PerClass } from "./per-class.js";
|
|
10
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Weighted } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
import type { Label } from './label.ts';
|
|
3
|
+
export declare class LabelCase implements Weighted {
|
|
4
|
+
readonly expected: ReadonlySet<Label>;
|
|
5
|
+
readonly weight: number;
|
|
6
|
+
constructor({ expected, weight, }: {
|
|
7
|
+
readonly expected: Label | readonly Label[];
|
|
8
|
+
readonly weight: number;
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=label-case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-case.d.ts","sourceRoot":"","sources":["../../src/classification/label-case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,qBAAa,SAAU,YAAW,QAAQ;IACxC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,EACV,QAAQ,EACR,MAAM,GACP,EAAE;QACD,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,SAAS,KAAK,EAAE,CAAC;QAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB;CAMF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// What one test gives the scorers of labels: its expected labels and its
|
|
2
|
+
// weight. One label is a set of one label, so a class (multiclass) and a set
|
|
3
|
+
// of classes (multilabel) are one model.
|
|
4
|
+
export class LabelCase {
|
|
5
|
+
expected;
|
|
6
|
+
weight;
|
|
7
|
+
constructor({ expected, weight, }) {
|
|
8
|
+
this.expected = new Set(typeof expected === 'object' ? expected : [expected]);
|
|
9
|
+
this.weight = weight;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=label-case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-case.js","sourceRoot":"","sources":["../../src/classification/label-case.ts"],"names":[],"mappings":"AAGA,yEAAyE;AACzE,6EAA6E;AAC7E,yCAAyC;AACzC,MAAM,OAAO,SAAS;IACX,QAAQ,CAAqB;IAC7B,MAAM,CAAS;IAExB,YAAY,EACV,QAAQ,EACR,MAAM,GAIP;QACC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CACrB,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrD,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Comparer } from '../scoring/comparer.ts';
|
|
2
|
+
import type { LabelCase } from './label-case.ts';
|
|
3
|
+
import { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
export declare class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
5
|
+
#private;
|
|
6
|
+
compare(stated: unknown, testCase: LabelCase): LabelComparison;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=label-comparer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparer.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAQxD,qBAAa,aAAc,YAAW,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;;IACxE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,eAAe;CAmC/D"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { LabelComparison } from "./label-comparison.js";
|
|
2
|
+
import { isLabel } from "./label.js";
|
|
3
|
+
// Reads the stated labels of an answer and gives its comparison with the case
|
|
4
|
+
// (Strategy). A label or a list of labels is read as it is. Text is read so:
|
|
5
|
+
// a JSON array of labels gives those labels; a number gives a number label
|
|
6
|
+
// only when each expected label of the case is a number; any other text,
|
|
7
|
+
// without its white space at the start and the end, is one string label.
|
|
8
|
+
export class LabelComparer {
|
|
9
|
+
compare(stated, testCase) {
|
|
10
|
+
return new LabelComparison({
|
|
11
|
+
stated: new Set(LabelComparer.#labels(stated, testCase)),
|
|
12
|
+
expected: testCase.expected,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
static #labels(stated, testCase) {
|
|
16
|
+
if (Array.isArray(stated) && stated.every(isLabel))
|
|
17
|
+
return stated;
|
|
18
|
+
if (typeof stated === 'number' && isLabel(stated))
|
|
19
|
+
return [stated];
|
|
20
|
+
if (typeof stated !== 'string')
|
|
21
|
+
throw new TypeError(`A stated answer is a label, a list of labels or text, not ${JSON.stringify(stated)}`);
|
|
22
|
+
const text = stated.trim();
|
|
23
|
+
const value = LabelComparer.#parse(text);
|
|
24
|
+
if (Array.isArray(value) && value.every(isLabel))
|
|
25
|
+
return value;
|
|
26
|
+
if (typeof value === 'number' &&
|
|
27
|
+
isLabel(value) &&
|
|
28
|
+
testCase.expected.size > 0 &&
|
|
29
|
+
[...testCase.expected].every((label) => typeof label === 'number'))
|
|
30
|
+
return [value];
|
|
31
|
+
return [text];
|
|
32
|
+
}
|
|
33
|
+
static #parse(text) {
|
|
34
|
+
try {
|
|
35
|
+
return JSON.parse(text);
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
if (error instanceof SyntaxError)
|
|
39
|
+
return undefined;
|
|
40
|
+
throw error;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=label-comparer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparer.js","sourceRoot":"","sources":["../../src/classification/label-comparer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,OAAO,aAAa;IACxB,OAAO,CAAC,MAAe,EAAE,QAAmB;QAC1C,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACxD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,QAAmB;QACjD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,MAAM,CAAC;QAClE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,OAAO,MAAM,KAAK,QAAQ;YAC5B,MAAM,IAAI,SAAS,CACjB,6DAA6D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACtF,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/D,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,CAAC,KAAK,CAAC;YACd,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAC1B,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;YAElE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAY;QACxB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,WAAW;gBAAE,OAAO,SAAS,CAAC;YACnD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type Label } from './label.ts';
|
|
2
|
+
export declare class LabelComparison {
|
|
3
|
+
readonly stated: ReadonlySet<Label>;
|
|
4
|
+
readonly expected: ReadonlySet<Label>;
|
|
5
|
+
constructor({ stated, expected, }: {
|
|
6
|
+
readonly stated: ReadonlySet<Label>;
|
|
7
|
+
readonly expected: ReadonlySet<Label>;
|
|
8
|
+
});
|
|
9
|
+
labels(): Label[];
|
|
10
|
+
only(label: Label): LabelComparison;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=label-comparison.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparison.d.ts","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AAGvD,qBAAa,eAAe;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;gBAE1B,EACV,MAAM,EACN,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;KACvC;IAMD,MAAM,IAAI,KAAK,EAAE;IAKjB,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,eAAe;CAOpC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { compareLabels } from "./label.js";
|
|
2
|
+
// The stated labels of one answer and the expected labels of its case.
|
|
3
|
+
export class LabelComparison {
|
|
4
|
+
stated;
|
|
5
|
+
expected;
|
|
6
|
+
constructor({ stated, expected, }) {
|
|
7
|
+
this.stated = stated;
|
|
8
|
+
this.expected = expected;
|
|
9
|
+
}
|
|
10
|
+
// The labels of both sides, in the order of numpy's unique().
|
|
11
|
+
labels() {
|
|
12
|
+
return [...this.stated.union(this.expected)].toSorted(compareLabels);
|
|
13
|
+
}
|
|
14
|
+
// The comparison of one class: each side keeps the label when it has it.
|
|
15
|
+
only(label) {
|
|
16
|
+
const one = new Set([label]);
|
|
17
|
+
return new LabelComparison({
|
|
18
|
+
stated: this.stated.intersection(one),
|
|
19
|
+
expected: this.expected.intersection(one),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=label-comparison.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-comparison.js","sourceRoot":"","sources":["../../src/classification/label-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAEvD,uEAAuE;AACvE,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAEtC,YAAY,EACV,MAAM,EACN,QAAQ,GAIT;QACC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,8DAA8D;IAC9D,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED,yEAAyE;IACzE,IAAI,CAAC,KAAY;QACf,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,OAAO,IAAI,eAAe,CAAC;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC;YACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC;SAC1C,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { CaseReader } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
import { LabelCase } from './label-case.ts';
|
|
3
|
+
export declare class LabelConfigCase implements CaseReader<LabelCase> {
|
|
4
|
+
from(config: Readonly<Record<string, unknown>>): LabelCase;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=label-config-case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-config-case.d.ts","sourceRoot":"","sources":["../../src/classification/label-config-case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAK5C,qBAAa,eAAgB,YAAW,UAAU,CAAC,SAAS,CAAC;IAC3D,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS;CAM3D"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { LabelCase } from "./label-case.js";
|
|
2
|
+
import { readLabels } from "./read-labels.js";
|
|
3
|
+
// Reads a case from the config of an assertion: config.expected is a label
|
|
4
|
+
// or a list of labels. A case weighs 1 (Strategy).
|
|
5
|
+
export class LabelConfigCase {
|
|
6
|
+
from(config) {
|
|
7
|
+
return new LabelCase({
|
|
8
|
+
expected: readLabels(config['expected'], 'config.expected'),
|
|
9
|
+
weight: 1,
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=label-config-case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-config-case.js","sourceRoot":"","sources":["../../src/classification/label-config-case.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,OAAO,eAAe;IAC1B,IAAI,CAAC,MAAyC;QAC5C,OAAO,IAAI,SAAS,CAAC;YACnB,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC;YAC3D,MAAM,EAAE,CAAC;SACV,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { CaseReader } from '../scoring/scorecard-scorer.ts';
|
|
2
|
+
import { LabelCase } from './label-case.ts';
|
|
3
|
+
export declare class LabelVarsCase implements CaseReader<LabelCase> {
|
|
4
|
+
from(vars: Readonly<Record<string, unknown>>): LabelCase;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=label-vars-case.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-vars-case.d.ts","sourceRoot":"","sources":["../../src/classification/label-vars-case.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAM5C,qBAAa,aAAc,YAAW,UAAU,CAAC,SAAS,CAAC;IACzD,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS;CASzD"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { LabelCase } from "./label-case.js";
|
|
2
|
+
import { readLabels } from "./read-labels.js";
|
|
3
|
+
// Reads a case from the vars of a test (Strategy): vars.expected is a label
|
|
4
|
+
// or a list of labels, and vars.weight is a number. A test without a weight
|
|
5
|
+
// weighs 1.
|
|
6
|
+
export class LabelVarsCase {
|
|
7
|
+
from(vars) {
|
|
8
|
+
const { expected, weight } = vars;
|
|
9
|
+
if (weight !== undefined && typeof weight !== 'number')
|
|
10
|
+
throw new Error(`vars.weight is a number, not ${JSON.stringify(weight)}`);
|
|
11
|
+
return new LabelCase({
|
|
12
|
+
expected: readLabels(expected, 'vars.expected'),
|
|
13
|
+
weight: weight ?? 1,
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=label-vars-case.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label-vars-case.js","sourceRoot":"","sources":["../../src/classification/label-vars-case.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,4EAA4E;AAC5E,4EAA4E;AAC5E,YAAY;AACZ,MAAM,OAAO,aAAa;IACxB,IAAI,CAAC,IAAuC;QAC1C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAClC,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,MAAM,KAAK,QAAQ;YACpD,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5E,OAAO,IAAI,SAAS,CAAC;YACnB,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,eAAe,CAAC;YAC/C,MAAM,EAAE,MAAM,IAAI,CAAC;SACpB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label.d.ts","sourceRoot":"","sources":["../../src/classification/label.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;AAGpC,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,KAAK,CAKtD;AAID,wBAAgB,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,CAQ/D"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { compareStrings } from '@zukhruf/python';
|
|
2
|
+
// Is the value a label: a finite number or a string.
|
|
3
|
+
export function isLabel(value) {
|
|
4
|
+
return (typeof value === 'string' ||
|
|
5
|
+
(typeof value === 'number' && Number.isFinite(value)));
|
|
6
|
+
}
|
|
7
|
+
// The order of numpy's unique() of labels: numbers by value, strings by code
|
|
8
|
+
// point. scikit-learn refuses a mix of numbers and strings.
|
|
9
|
+
export function compareLabels(left, right) {
|
|
10
|
+
if (typeof left === 'number' && typeof right === 'number')
|
|
11
|
+
return left - right;
|
|
12
|
+
if (typeof left === 'string' && typeof right === 'string')
|
|
13
|
+
return compareStrings(left, right);
|
|
14
|
+
throw new TypeError(`The labels ${JSON.stringify(left)} and ${JSON.stringify(right)} mix a number and a string, which scikit-learn refuses.`);
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=label.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"label.js","sourceRoot":"","sources":["../../src/classification/label.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAKjD,qDAAqD;AACrD,MAAM,UAAU,OAAO,CAAC,KAAc;IACpC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACtD,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,4DAA4D;AAC5D,MAAM,UAAU,aAAa,CAAC,IAAW,EAAE,KAAY;IACrD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ;QACvD,OAAO,IAAI,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ;QACvD,OAAO,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACrC,MAAM,IAAI,SAAS,CACjB,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,yDAAyD,CACzH,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Gate } from '../scorers/gate.ts';
|
|
2
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
3
|
+
export declare class LabelsMatch extends Gate<LabelComparison> {
|
|
4
|
+
holds({ stated, expected }: LabelComparison): boolean;
|
|
5
|
+
explain(comparison: LabelComparison): string;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=labels-match.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"labels-match.d.ts","sourceRoot":"","sources":["../../src/classification/labels-match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,WAAY,SAAQ,IAAI,CAAC,eAAe,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,OAAO;IAIrD,OAAO,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM;CAG7C"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Gate } from "../scorers/gate.js";
|
|
2
|
+
// A gate that holds when the stated labels are the expected labels. Pooled
|
|
3
|
+
// over the answers, it is scikit-learn's accuracy_score (the subset accuracy
|
|
4
|
+
// for a set of classes).
|
|
5
|
+
export class LabelsMatch extends Gate {
|
|
6
|
+
holds({ stated, expected }) {
|
|
7
|
+
return stated.size === expected.size && stated.isSubsetOf(expected);
|
|
8
|
+
}
|
|
9
|
+
explain(comparison) {
|
|
10
|
+
return `the stated labels ${JSON.stringify([...comparison.stated])} are not the expected labels ${JSON.stringify([...comparison.expected])}`;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=labels-match.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"labels-match.js","sourceRoot":"","sources":["../../src/classification/labels-match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAG1C,2EAA2E;AAC3E,6EAA6E;AAC7E,yBAAyB;AACzB,MAAM,OAAO,WAAY,SAAQ,IAAqB;IACpD,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QACzC,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,CAAC,UAA2B;QACjC,OAAO,qBAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;IAC/I,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Scorer } from '../scorers/scorer.ts';
|
|
2
|
+
import { Grouped } from '../scoring/grouped.ts';
|
|
3
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
import type { Label } from './label.ts';
|
|
5
|
+
export declare class PerClass extends Grouped<LabelComparison> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(scorer: Scorer<LabelComparison>, { labels }?: {
|
|
8
|
+
readonly labels?: readonly Label[];
|
|
9
|
+
});
|
|
10
|
+
protected groups(comparison: LabelComparison): Iterable<[string, LabelComparison]>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=per-class.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"per-class.d.ts","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,qBAAa,QAAS,SAAQ,OAAO,CAAC,eAAe,CAAC;;gBAIlD,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,EAC/B,EAAE,MAAM,EAAE,GAAE;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,KAAK,EAAE,CAAA;KAAO;IAMzD,SAAS,CAAE,MAAM,CACf,UAAU,EAAE,eAAe,GAC1B,QAAQ,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CAIvC"}
|