@zukhruf/evals 0.8.25 → 0.8.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -143,7 +143,7 @@ const brier = paired.verdict(baseline, change, 'brier', {
143
143
  });
144
144
  ```
145
145
 
146
- A run holds the scorecards of one model. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
146
+ A run holds the scorecards of one model. It reads only their ratios, as `Totals` does, so a host that scores without a scorer of this package adds `{ ratios }` alone. A second `add` of a test id is one more epoch of that test: startup-credits adds three runs of one model as three epochs of one `Run`. To test many names, call `verdict` once for each name, and correct the p-values, such as with `holm` of `@zukhruf/mteb`. To test one name, use its p-value as it is.
147
147
 
148
148
  ```ts
149
149
  // @zukhruf/evals: two runs side by side
@@ -155,7 +155,11 @@ import type {
155
155
  /** The scorecards of one model on many tests. A test can have many epochs. */
156
156
  export class Run {
157
157
  /** Adds one epoch of a test. Another weight for a test id that the run has is a RangeError. */
158
- add(testId: string, scorecard: Scorecard, weight: number): void;
158
+ add(
159
+ testId: string,
160
+ scorecard: Pick<Scorecard, 'ratios'>,
161
+ weight: number,
162
+ ): void;
159
163
  }
160
164
 
161
165
  export type Verdict = {
@@ -1,7 +1,7 @@
1
1
  import type { Scorecard } from './scorecard.ts';
2
2
  import { TestEpochs } from './test-epochs.ts';
3
3
  export declare class Run {
4
- add(testId: string, scorecard: Scorecard, weight: number): void;
4
+ add(testId: string, scorecard: Pick<Scorecard, 'ratios'>, weight: number): void;
5
5
  }
6
6
  export declare function runTests(run: Run): ReadonlyMap<string, TestEpochs>;
7
7
  //# sourceMappingURL=run.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,qBAAa,GAAG;IACd,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;CAWhE;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAElE"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,qBAAa,GAAG;IACd,GAAG,CACD,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EACpC,MAAM,EAAE,MAAM,GACb,IAAI;CAWR;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAElE"}
@@ -1 +1 @@
1
- {"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,6EAA6E;AAC7E,4EAA4E;AAC5E,MAAM,UAAU,GAAG,IAAI,OAAO,EAAgC,CAAC;AAC/D,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC3B,UAAU,CAAC,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;AAEvD,0EAA0E;AAC1E,+BAA+B;AAC/B,MAAM,OAAO,GAAG;IACd,GAAG,CAAC,MAAc,EAAE,SAAoB,EAAE,MAAc;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAC5C,MAAM,EACN,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAC7B,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YACxB,MAAM,IAAI,UAAU,CAClB,SAAS,MAAM,oBAAoB,IAAI,CAAC,MAAM,SAAS,MAAM,kCAAkC,CAChG,CAAC;QACJ,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,GAAQ;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC"}
1
+ {"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/scoring/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,6EAA6E;AAC7E,4EAA4E;AAC5E,MAAM,UAAU,GAAG,IAAI,OAAO,EAAgC,CAAC;AAC/D,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC3B,UAAU,CAAC,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;AAEvD,0EAA0E;AAC1E,+BAA+B;AAC/B,MAAM,OAAO,GAAG;IACd,GAAG,CACD,MAAc,EACd,SAAoC,EACpC,MAAc;QAEd,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAC5C,MAAM,EACN,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAC7B,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YACxB,MAAM,IAAI,UAAU,CAClB,SAAS,MAAM,oBAAoB,IAAI,CAAC,MAAM,SAAS,MAAM,kCAAkC,CAChG,CAAC;QACJ,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,GAAQ;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC"}
@@ -4,7 +4,7 @@ export declare class TestEpochs {
4
4
  #private;
5
5
  readonly weight: number;
6
6
  constructor(weight: number);
7
- add(scorecard: Scorecard): void;
7
+ add(scorecard: Pick<Scorecard, 'ratios'>): void;
8
8
  gives(name: string): boolean;
9
9
  mean(name: string): Ratio;
10
10
  }
@@ -1 +1 @@
1
- {"version":3,"file":"test-epochs.d.ts","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,qBAAa,UAAU;;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAGZ,MAAM,EAAE,MAAM;IAI1B,GAAG,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAI/B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAO5B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;CAS1B"}
1
+ {"version":3,"file":"test-epochs.d.ts","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,qBAAa,UAAU;;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAGZ,MAAM,EAAE,MAAM;IAI1B,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,IAAI;IAI/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAO5B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;CAS1B"}
@@ -1 +1 @@
1
- {"version":3,"file":"test-epochs.js","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,gEAAgE;AAChE,MAAM,OAAO,UAAU;IACZ,MAAM,CAAS;IACf,WAAW,GAAgB,EAAE,CAAC;IAEvC,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,SAAoB;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,6BAA6B;IAC7B,IAAI,CAAC,IAAY;QACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,CACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAClB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAC7D,CACF,CAAC,KAAK,CAAC;QACR,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAClC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;CACF"}
1
+ {"version":3,"file":"test-epochs.js","sourceRoot":"","sources":["../../src/scoring/test-epochs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,gEAAgE;AAChE,MAAM,OAAO,UAAU;IACZ,MAAM,CAAS;IACf,WAAW,GAAgC,EAAE,CAAC;IAEvD,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,SAAoC;QACtC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,6BAA6B;IAC7B,IAAI,CAAC,IAAY;QACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,CACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAClB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAC7D,CACF,CAAC,KAAK,CAAC;QACR,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QAClC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;CACF"}
@@ -2,7 +2,7 @@ import { RatioOfSums } from '../aggregation/ratio-of-sums.ts';
2
2
  import type { Scorecard } from './scorecard.ts';
3
3
  export declare class Totals {
4
4
  #private;
5
- add(scorecard: Scorecard, weight: number): void;
5
+ add(scorecard: Pick<Scorecard, 'ratios'>, weight: number): void;
6
6
  get(name: string): RatioOfSums;
7
7
  names(): IterableIterator<string>;
8
8
  }
@@ -1 +1 @@
1
- {"version":3,"file":"totals.d.ts","sourceRoot":"","sources":["../../src/scoring/totals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAIhD,qBAAa,MAAM;;IAGjB,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAQ/C,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAO9B,KAAK,IAAI,gBAAgB,CAAC,MAAM,CAAC;CAGlC"}
1
+ {"version":3,"file":"totals.d.ts","sourceRoot":"","sources":["../../src/scoring/totals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAIhD,qBAAa,MAAM;;IAGjB,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAQ/D,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAO9B,KAAK,IAAI,gBAAgB,CAAC,MAAM,CAAC;CAGlC"}
@@ -1 +1 @@
1
- {"version":3,"file":"totals.js","sourceRoot":"","sources":["../../src/scoring/totals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,2EAA2E;AAC3E,aAAa;AACb,MAAM,OAAO,MAAM;IACR,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEhD,GAAG,CAAC,SAAoB,EAAE,MAAc;QACtC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,CAAC,KAAK;iBACP,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;iBAClD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAY;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,SAAS;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC;QAC/D,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}
1
+ {"version":3,"file":"totals.js","sourceRoot":"","sources":["../../src/scoring/totals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,2EAA2E;AAC3E,aAAa;AACb,MAAM,OAAO,MAAM;IACR,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEhD,GAAG,CAAC,SAAoC,EAAE,MAAc;QACtD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,CAAC,KAAK;iBACP,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;iBAClD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAY;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,SAAS;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC;QAC/D,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zukhruf/evals",
3
- "version": "0.8.25",
3
+ "version": "0.8.27",
4
4
  "description": "Scores the answers of an eval against their labels, with one import path for each shape of answer: a core of ratios and totals, ./structured for structured values, ./classification for labels as scikit-learn scores them, and ./promptfoo.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -63,8 +63,8 @@
63
63
  "access": "public"
64
64
  },
65
65
  "dependencies": {
66
- "@zukhruf/python": "^0.8.25",
67
- "@zukhruf/statistics": "^0.8.25"
66
+ "@zukhruf/python": "^0.8.27",
67
+ "@zukhruf/statistics": "^0.8.27"
68
68
  },
69
69
  "peerDependencies": {
70
70
  "@cfworker/json-schema": "4.1.4",
@@ -80,9 +80,9 @@
80
80
  },
81
81
  "devDependencies": {
82
82
  "@cfworker/json-schema": "4.1.4",
83
- "@zukhruf/fs": "^0.8.25",
83
+ "@zukhruf/fs": "^0.8.27",
84
84
  "@zukhruf/random": "*",
85
- "@zukhruf/testing": "^0.8.25",
85
+ "@zukhruf/testing": "^0.8.27",
86
86
  "munkres": "^2.1.1",
87
87
  "promptfoo": "0.125.1",
88
88
  "python-random": "1.0.0"