@zukhruf/evals 0.8.5 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -175,7 +175,7 @@ run.add(new LabelComparer().compare('a', testCase), testCase.weight);
175
175
 
176
176
  `ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
177
177
 
178
- `BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
178
+ `BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. `form` is the arithmetic of the score of p (ADR 0003). `new TwoColumnForm()` is scikit-learn's and the default: 0.5 × the sum of the squares of the two classes. `new SquaredErrorForm()` is (p − y)², as numpy gives `np.mean((p - y) ** 2)`. The two forms can differ in the last bit. A form needs `positiveLabel`. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
179
179
 
180
180
  ## Use it with promptfoo
181
181
 
@@ -570,8 +570,18 @@ export class ProbabilityComparer implements Comparer<
570
570
  LabelCase,
571
571
  ProbabilityComparison
572
572
  > {}
573
+ /** The arithmetic of the Brier score of the probability p of the positive label (ADR 0003). */
574
+ export interface BrierForm {
575
+ of(probability: number, expected: 0 | 1): number;
576
+ }
577
+ export class TwoColumnForm implements BrierForm {}
578
+ export class SquaredErrorForm implements BrierForm {}
573
579
  export class BrierScore implements Scorer<ProbabilityComparison> {
574
- constructor(options?: { positiveLabel?: Label });
580
+ constructor(
581
+ options?:
582
+ | { positiveLabel?: undefined; form?: undefined }
583
+ | { positiveLabel: Label; form?: BrierForm },
584
+ );
575
585
  /** The score of a whole run, with numpy's pairwise sum. */
576
586
  mean(
577
587
  comparisons: readonly ProbabilityComparison[],
@@ -590,7 +600,7 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
590
600
  | context-compiler, startup-credits offers | core, `./structured` (`ByBestMatch`), `./promptfoo` | context-compiler #3893, #3882 |
591
601
  | context-compiler, startup-credits page kinds | `./classification` (accuracy, the precision and recall of each class, the confusion matrix) | #3882 slice 2 |
592
602
  | `@zukhruf/mteb`, classification tasks | `./classification` (`ClassCounts`: accuracy, macro and weighted P, R and F1; label-indicator rows) | mutex #3890 |
593
- | relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`) | mutex #3891 |
603
+ | relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore` with `SquaredErrorForm`) | mutex #3891 |
594
604
  | event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
595
605
 
596
606
  The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
@@ -0,0 +1,10 @@
1
+ export interface BrierForm {
2
+ of(probability: number, expected: 0 | 1): number;
3
+ }
4
+ export declare class TwoColumnForm implements BrierForm {
5
+ of(probability: number, expected: 0 | 1): number;
6
+ }
7
+ export declare class SquaredErrorForm implements BrierForm {
8
+ of(probability: number, expected: 0 | 1): number;
9
+ }
10
+ //# sourceMappingURL=brier-form.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brier-form.d.ts","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,SAAS;IACxB,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;CAClD;AAKD,qBAAa,aAAc,YAAW,SAAS;IAC7C,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAKjD;AAID,qBAAa,gBAAiB,YAAW,SAAS;IAChD,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM;CAGjD"}
@@ -0,0 +1,18 @@
1
+ // scikit-learn 1.9.1's brier_score_loss of a one-dimensional y_proba: the
2
+ // two classes are 1 - p and p, and the score is 0.5 times the sum of the
3
+ // squares of the two columns.
4
+ export class TwoColumnForm {
5
+ of(probability, expected) {
6
+ const negative = 1 - expected - (1 - probability);
7
+ const positive = expected - probability;
8
+ return 0.5 * (negative * negative + positive * positive);
9
+ }
10
+ }
11
+ // (p - y)², the squared error of the probability (Brier 1950 for two
12
+ // classes), as numpy gives (p - y) ** 2.
13
+ export class SquaredErrorForm {
14
+ of(probability, expected) {
15
+ return (probability - expected) ** 2;
16
+ }
17
+ }
18
+ //# sourceMappingURL=brier-form.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brier-form.js","sourceRoot":"","sources":["../../src/classification/brier-form.ts"],"names":[],"mappings":"AAQA,0EAA0E;AAC1E,yEAAyE;AACzE,8BAA8B;AAC9B,MAAM,OAAO,aAAa;IACxB,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;QACxC,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;IAC3D,CAAC;CACF;AAED,qEAAqE;AACrE,yCAAyC;AACzC,MAAM,OAAO,gBAAgB;IAC3B,EAAE,CAAC,WAAmB,EAAE,QAAe;QACrC,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;CACF"}
@@ -1,11 +1,16 @@
1
1
  import { Ratio } from '../aggregation/ratio.ts';
2
2
  import type { Scorer } from '../scorers/scorer.ts';
3
+ import { type BrierForm } from './brier-form.ts';
3
4
  import { type Label } from './label.ts';
4
5
  import type { ProbabilityComparison } from './probability-comparison.ts';
5
6
  export declare class BrierScore implements Scorer<ProbabilityComparison> {
6
7
  #private;
7
- constructor({ positiveLabel }?: {
8
- readonly positiveLabel?: Label;
8
+ constructor({ positiveLabel, form, }?: {
9
+ readonly positiveLabel?: undefined;
10
+ readonly form?: undefined;
11
+ } | {
12
+ readonly positiveLabel: Label;
13
+ readonly form?: BrierForm;
9
14
  });
10
15
  score(comparison: ProbabilityComparison): Ratio;
11
16
  mean(comparisons: readonly ProbabilityComparison[], weights?: readonly number[]): number;
@@ -1 +1 @@
1
- {"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AASzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAGlD,EAAE,aAAa,EAAE,GAAE;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,CAAA;KAAO;IAItE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAyCV"}
1
+ {"version":3,"file":"brier-score.d.ts","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,YAAY,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAUzE,qBAAa,UAAW,YAAW,MAAM,CAAC,qBAAqB,CAAC;;gBAIlD,EACV,aAAa,EACb,IAA0B,GAC3B,GACG;QAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAE,GACjE;QAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAA;KAAO;IAKrE,KAAK,CAAC,UAAU,EAAE,qBAAqB,GAAG,KAAK;IAM/C,IAAI,CACF,WAAW,EAAE,SAAS,qBAAqB,EAAE,EAC7C,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM;CAsCV"}
@@ -1,17 +1,21 @@
1
1
  import gsumpw from '@stdlib/blas-ext-base-gsumpw';
2
2
  import { Ratio } from "../aggregation/ratio.js";
3
+ import { TwoColumnForm } from "./brier-form.js";
3
4
  import { compareLabels } from "./label.js";
4
5
  // scikit-learn 1.9.1's brier_score_loss. For one answer, the sum over the
5
6
  // classes of (1 when the class is the expected label, else 0, minus the
6
7
  // stated probability) squared, with numpy's pairwise sum over the classes in
7
8
  // their sorted order; times 0.5 for two classes. With a positive label, the
8
9
  // answer gives the probability p of that label only, and the two classes are
9
- // 1 - p and p, as scikit-learn reads a one-dimensional y_proba. Its ratio is
10
- // the score over 1, and mean gives the score of a whole run.
10
+ // 1 - p and p, as scikit-learn reads a one-dimensional y_proba; the form
11
+ // (TwoColumnForm by default) gives the score of p. Its ratio is the score
12
+ // over 1, and mean gives the score of a whole run.
11
13
  export class BrierScore {
12
14
  #positiveLabel;
13
- constructor({ positiveLabel } = {}) {
15
+ #form;
16
+ constructor({ positiveLabel, form = new TwoColumnForm(), } = {}) {
14
17
  this.#positiveLabel = positiveLabel;
18
+ this.#form = form;
15
19
  }
16
20
  score(comparison) {
17
21
  return new Ratio(this.#value(comparison), 1);
@@ -32,10 +36,7 @@ export class BrierScore {
32
36
  const p = probabilities.get(this.#positiveLabel);
33
37
  if (p === undefined)
34
38
  throw new RangeError(`The answer gives no probability of the positive label ${JSON.stringify(this.#positiveLabel)}.`);
35
- const y = expected === this.#positiveLabel ? 1 : 0;
36
- const negative = 1 - y - (1 - p);
37
- const positive = y - p;
38
- return 0.5 * (negative * negative + positive * positive);
39
+ return this.#form.of(p, expected === this.#positiveLabel ? 1 : 0);
39
40
  }
40
41
  if (!probabilities.has(expected))
41
42
  throw new RangeError(`The answer gives no probability of the expected label ${JSON.stringify(expected)}.`);
@@ -1 +1 @@
1
- {"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAE3C,YAAY,EAAE,aAAa,KAAyC,EAAE;QACpE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YACvB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,MAAM,CAAC,GAAG,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;CACF"}
1
+ {"version":3,"file":"brier-score.js","sourceRoot":"","sources":["../../src/classification/brier-score.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,8BAA8B,CAAC;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,OAAO,EAAkB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAc,aAAa,EAAE,MAAM,YAAY,CAAC;AAGvD,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,yEAAyE;AACzE,0EAA0E;AAC1E,mDAAmD;AACnD,MAAM,OAAO,UAAU;IACZ,cAAc,CAAoB;IAClC,KAAK,CAAY;IAE1B,YAAY,EACV,aAAa,EACb,IAAI,GAAG,IAAI,aAAa,EAAE,MAGuC,EAAE;QACnE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAiC;QACrC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IACzE,iDAAiD;IACjD,IAAI,CACF,WAA6C,EAC7C,OAA2B;QAE3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,KAAK,SAAS;YACvB,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAClC,MAAM,IAAI,UAAU,CAClB,oCAAoC,MAAM,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,UAAU,CAClF,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CACzB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAyB;QACvD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,SAAS;gBACjB,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAChG,CAAC;YACJ,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,IAAI,UAAU,CAClB,yDAAyD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CACrF,CAAC;QACJ,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,CAAC;aAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC9D,OAAO,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACL,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;CACF"}
@@ -16,6 +16,7 @@ export { MacroAverage } from './macro-average.ts';
16
16
  export { MicroAverage } from './micro-average.ts';
17
17
  export { WeightedAverage } from './weighted-average.ts';
18
18
  export { BrierScore } from './brier-score.ts';
19
+ export { type BrierForm, SquaredErrorForm, TwoColumnForm, } from './brier-form.ts';
19
20
  export { ProbabilityComparer } from './probability-comparer.ts';
20
21
  export { ProbabilityComparison } from './probability-comparison.ts';
21
22
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EACL,KAAK,SAAS,EACd,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
@@ -14,6 +14,7 @@ export { MacroAverage } from "./macro-average.js";
14
14
  export { MicroAverage } from "./micro-average.js";
15
15
  export { WeightedAverage } from "./weighted-average.js";
16
16
  export { BrierScore } from "./brier-score.js";
17
+ export { SquaredErrorForm, TwoColumnForm, } from "./brier-form.js";
17
18
  export { ProbabilityComparer } from "./probability-comparer.js";
18
19
  export { ProbabilityComparison } from "./probability-comparison.js";
19
20
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/classification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAEL,gBAAgB,EAChB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zukhruf/evals",
3
- "version": "0.8.5",
3
+ "version": "0.8.6",
4
4
  "description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -63,8 +63,8 @@
63
63
  "access": "public"
64
64
  },
65
65
  "dependencies": {
66
- "@zukhruf/python": "^0.8.5",
67
- "@zukhruf/statistics": "^0.8.5"
66
+ "@zukhruf/python": "^0.8.6",
67
+ "@zukhruf/statistics": "^0.8.6"
68
68
  },
69
69
  "peerDependencies": {
70
70
  "@cfworker/json-schema": "4.1.4",
@@ -85,7 +85,7 @@
85
85
  "devDependencies": {
86
86
  "@cfworker/json-schema": "4.1.4",
87
87
  "@stdlib/blas-ext-base-gsumpw": "^0.3.1",
88
- "@zukhruf/testing": "^0.8.5",
88
+ "@zukhruf/testing": "^0.8.6",
89
89
  "munkres": "^2.1.1",
90
90
  "promptfoo": "0.125.1",
91
91
  "python-random": "1.0.0"