@namzu/sdk 62.13.0 → 62.15.0

package/dist/eval/report-stats.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE"}
+{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIpD"}

package/dist/eval/report-stats.js

@@ -181,4 +181,14 @@ export function describeSplits(splits) {
     }
     return lines;
 }
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export function clarifyDeclared(line) {
+    return line.startsWith('no held-out cases')
+        ? `${line} (declared by the suite; the split above is the hashed one)`
+        : line;
+}
 //# sourceMappingURL=report-stats.js.map

package/dist/eval/report-stats.js.map

@@ -1 +1 @@
-{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}
+{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAC1C,CAAC,CAAC,GAAG,IAAI,6DAA6D;QACtE,CAAC,CAAC,IAAI,CAAA;AACR,CAAC"}

package/dist/eval/run-options.d.ts

@@ -1,3 +1,5 @@
+import type { SettingsSnapshot } from './settings.js';
+import type { SplitFilter } from './split.js';
 /**
  * Settings an operator gives a whole eval run, handed to every
  * `runExperiment` that executes inside it.
@@ -22,9 +24,32 @@ export interface EvalRunOptions {
     readonly commit?: string;
     /** Version of the suite being run, when known. */
     readonly suiteVersion?: string;
+    /**
+     * The settings the run was given. Written into every report. A suite shapes
+     * itself by them through {@link currentEvalSettings}; one that does not is
+     * reported as not having read them.
+     */
+    readonly settings?: SettingsSnapshot;
+    /** Which tasks run. Absent means all of them. */
+    readonly split?: SplitFilter;
+    /** Set by {@link currentEvalSettings} when a suite asks for the settings. Owned by the runner. */
+    readonly settingsUse?: {
+        read: boolean;
+        keys?: Set<string>;
+    };
 }
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export declare const READ_ALL = "*";
 /** Run `work` with `options` in force for every experiment it starts. */
 export declare function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T;
 /** The options in force here, or an empty set outside any scope. */
 export declare function currentEvalRunOptions(): EvalRunOptions;
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export declare function currentEvalSettings(): Readonly<Record<string, unknown>>;
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export declare function evalSetting(path: string): unknown;
 //# sourceMappingURL=run-options.d.ts.map

package/dist/eval/run-options.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAcD,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD"}
+{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IACpC,iDAAiD;IACjD,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAA;IAC5B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;KAAE,CAAA;CAC5D;AAED,4FAA4F;AAC5F,eAAO,MAAM,QAAQ,MAAM,CAAA;AAoB3B,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAQvE;AAED,qGAAqG;AACrG,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAYjD"}

package/dist/eval/run-options.js

@@ -1,4 +1,11 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export const READ_ALL = '*';
+function markRead(use, key) {
+    const keys = use.keys ?? new Set();
+    keys.add(key);
+    use.keys = keys;
+}
 const STORE_KEY = Symbol.for('namzu.eval.runOptions');
 /**
  * One store per process, found through a global symbol, so a suite that
@@ -17,4 +24,33 @@ export function withEvalRunOptions(options, work) {
 export function currentEvalRunOptions() {
     return store().getStore() ?? {};
 }
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export function currentEvalSettings() {
+    const options = currentEvalRunOptions();
+    if (options.settingsUse) {
+        options.settingsUse.read = true;
+        // The whole object was asked for: every key counts as read.
+        markRead(options.settingsUse, READ_ALL);
+    }
+    return options.settings?.values ?? {};
+}
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export function evalSetting(path) {
+    const options = currentEvalRunOptions();
+    let at = options.settings?.values ?? {};
+    if (options.settingsUse) {
+        options.settingsUse.read = true;
+        markRead(options.settingsUse, path);
+    }
+    for (const part of path.split('.')) {
+        if (typeof at !== 'object' || at === null || Array.isArray(at))
+            return undefined;
+        at = at[part];
+    }
+    return at;
+}
 //# sourceMappingURL=run-options.js.map

package/dist/eval/run-options.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4BpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC"}
+{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAwCpD,4FAA4F;AAC5F,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAA;AAE3B,SAAS,QAAQ,CAAC,GAA0C,EAAE,GAAW;IACxE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,IAAI,GAAG,EAAU,CAAA;IAC1C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,GAAG,CAAC,IAAI,GAAG,IAAI,CAAA;AAChB,CAAC;AAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IAClC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAC/B,4DAA4D;QAC5D,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAA;AACtC,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,IAAI,EAAE,GAAY,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAA;IAChD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAC/B,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;IACpC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAAE,OAAO,SAAS,CAAA;QAChF,EAAE,GAAI,EAA8B,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,EAAE,CAAA;AACV,CAAC"}

package/dist/eval/run-summary.d.ts

@@ -1,5 +1,7 @@
 import type { RunMode } from './disclosure.js';
 import { type RunStats } from './report-stats.js';
+import type { SettingsSnapshot } from './settings.js';
+import type { SplitFilter } from './split.js';
 import type { ExperimentReport } from './types.js';
 /** What produced a whole run, across every suite in it. */
 export interface RunDisclosure {
@@ -15,6 +17,10 @@ export interface RunDisclosure {
     readonly runs: number;
     /** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
     readonly mode: RunMode | 'mixed';
+    /** The settings the run was given. Absent from a run written before reports named them. */
+    readonly settings?: SettingsSnapshot;
+    /** Which tasks ran. Absent from a run written before there were splits. */
+    readonly split?: SplitFilter;
 }
 /** The numbers for every case of every suite taken together. */
 export interface RunSummary extends Omit<RunStats, 'caseSummaries' | 'reliability'> {
@@ -31,6 +37,8 @@ export declare function summarizeRun(entries: ReadonlyArray<{
     commit: string;
     generatedAt: string;
     sdkVersion: string;
+    settings?: SettingsSnapshot;
+    split?: SplitFilter;
 }): RunSummary;
 export declare function formatRunSummary(summary: RunSummary): string;
 //# sourceMappingURL=run-summary.d.ts.map

package/dist/eval/run-summary.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAKb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;CAChC;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB,GACC,UAAU,CAmCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAuB5D"}
+{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAMb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;IAChC,2FAA2F;IAC3F,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IACpC,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAA;CAC5B;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB,GACC,UAAU,CAqCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CA8B5D"}

package/dist/eval/run-summary.js

@@ -1,5 +1,5 @@
 import { UNKNOWN } from './disclosure.js';
-import { describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
+import { clarifyDeclared, describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
 export function summarizeRun(entries, context) {
     // A case is named by its suite and its id: two suites may reuse a name.
     const cases = entries.flatMap(({ suite, report }) => groupRuns(report.cases ?? [], (name) => `${suite}\u0000${name}`));
@@ -30,6 +30,8 @@ export function summarizeRun(entries, context) {
             cases: stats.cases,
             runs: stats.runs,
             mode,
+            ...(context.settings ? { settings: context.settings } : {}),
+            ...(context.split ? { split: context.split } : {}),
         },
     };
 }
@@ -49,8 +51,16 @@ export function formatRunSummary(summary) {
         const r = summary.reliability;
         lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
     }
-    for (const line of describeSplits(summary.splits))
-        lines.push(`  ${line}`);
+    for (const line of describeSplits(summary.splits)) {
+        if (d.split === 'held-out' && line.startsWith('no held-out cases'))
+            continue;
+        lines.push(`  ${d.split ? clarifyDeclared(line) : line}`);
+    }
+    if (d.settings) {
+        lines.push(`  settings: "${d.settings.name}", sha256 ${d.settings.digest.slice(0, 12)}`);
+    }
+    if (d.split)
+        lines.push(`  split: ${d.split}`);
     lines.push(`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`);
     return lines.join('\n');
 }

package/dist/eval/run-summary.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AA0B1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAMC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;SACJ;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAgC1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAQC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;YACJ,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;YAAE,SAAQ;QAC5E,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAA;IACzF,CAAC;IACD,IAAI,CAAC,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;IAC9C,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/settings.d.ts

@@ -0,0 +1,74 @@
+/**
+ * The settings an eval run ran under, as a named, canonical, hashed record.
+ *
+ * Two reports of one suite say little about a harness setting unless each says
+ * exactly which settings it ran under. The snapshot is that statement: a name a
+ * person chose (or `default`), a digest of the values, and the values. The
+ * digest is taken over the values alone, in a canonical form, so it is the same
+ * whatever order the keys were written in and whatever the snapshot is called.
+ *
+ * Secrets never enter one. {@link buildSettingsSnapshot} refuses a value that
+ * names a credential or looks like one, so a report written from a snapshot
+ * cannot carry a key even if a caller passed it by mistake.
+ */
+export interface SettingsSnapshot {
+    /** What the person called it; the file's name when it came from a file. */
+    readonly name: string;
+    /** SHA-256 of the canonical JSON of {@link values}, in hex. */
+    readonly digest: string;
+    /** How the digest was made, so a later version can change it without a silent mismatch. */
+    readonly algorithm: typeof SETTINGS_DIGEST_ALGORITHM;
+    /** The settings that differ from the defaults: whatever was stated. Empty for `default`. */
+    readonly values: Readonly<Record<string, unknown>>;
+}
+export declare const SETTINGS_DIGEST_ALGORITHM = "sha256-canonical-json-1";
+/** The name of the snapshot of a run that was given no settings. */
+export declare const DEFAULT_SETTINGS_NAME = "default";
+/** What a suite that read the settings says about it, written into its report. */
+export interface ReportSettings extends SettingsSnapshot {
+    /**
+     * Whether the suite asked for the settings while it ran. A suite that never
+     * does cannot have been shaped by them, so a difference between two snapshots
+     * says nothing about its numbers.
+     */
+    readonly read: boolean;
+    /**
+     * The dotted paths the suite asked for, or `*` when it asked for the whole
+     * object. Absent from a report that predates it, which is read as every key.
+     */
+    readonly readKeys?: readonly string[];
+}
+/**
+ * Canonical JSON: keys sorted at every level, no whitespace, array order kept.
+ * A value JSON cannot carry (`undefined` inside an array, a non-finite number,
+ * a function) is refused, because a digest over a value that cannot be written
+ * back would name something nobody can reproduce. An `undefined` object field
+ * is the same as an absent one and is dropped.
+ */
+export declare function canonicalJson(value: unknown): string;
+/**
+ * The path of the first thing in `values` that is, or names, a credential, or
+ * undefined when there is none. The path never includes the secret itself.
+ */
+export declare function findSecretIn(values: unknown, path?: string): string | undefined;
+/** A snapshot name is plain: it is printed in a comparison and used as a file label. */
+export declare function validateSettingsName(name: string): string;
+/**
+ * Build a snapshot. Throws an {@link EvalUsageError} naming the path of a
+ * secret, a value JSON cannot carry, or a name that is not plain. A snapshot
+ * called `default` must have no values: `default` is the name of "no settings
+ * stated", and a snapshot with values under it would say the opposite.
+ */
+export declare function buildSettingsSnapshot(name: string, values?: Readonly<Record<string, unknown>>): SettingsSnapshot;
+/** The snapshot of a run given no settings. */
+export declare function defaultSettingsSnapshot(): SettingsSnapshot;
+export interface SettingDifference {
+    /** Dotted path of the setting; a list is one value. */
+    readonly key: string;
+    /** Absent on the side that does not state it. */
+    readonly a?: unknown;
+    readonly b?: unknown;
+}
+/** What differs between two snapshots' values, key by key, sorted by key. */
+export declare function diffSettings(a: Readonly<Record<string, unknown>>, b: Readonly<Record<string, unknown>>): SettingDifference[];
+//# sourceMappingURL=settings.d.ts.map

package/dist/eval/settings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settings.d.ts","sourceRoot":"","sources":["../../src/eval/settings.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAChC,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,+DAA+D;IAC/D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,2FAA2F;IAC3F,QAAQ,CAAC,SAAS,EAAE,OAAO,yBAAyB,CAAA;IACpD,4FAA4F;IAC5F,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CAClD;AAED,eAAO,MAAM,yBAAyB,4BAA4B,CAAA;AAElE,oEAAoE;AACpE,eAAO,MAAM,qBAAqB,YAAY,CAAA;AAE9C,kFAAkF;AAClF,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACvD;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACrC;AAMD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CA4BpD;AAoBD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,SAAK,GAAG,MAAM,GAAG,SAAS,CAuB3E;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOzD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAM,GAC5C,gBAAgB,CAsBlB;AAED,+CAA+C;AAC/C,wBAAgB,uBAAuB,IAAI,gBAAgB,CAE1D;AAED,MAAM,WAAW,iBAAiB;IACjC,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,iDAAiD;IACjD,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;CACpB;AAWD,6EAA6E;AAC7E,wBAAgB,YAAY,CAC3B,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACpC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAClC,iBAAiB,EAAE,CAoBrB"}

package/dist/eval/settings.js

@@ -0,0 +1,161 @@
+import { createHash } from 'node:crypto';
+import { EvalUsageError } from './bootstrap.js';
+export const SETTINGS_DIGEST_ALGORITHM = 'sha256-canonical-json-1';
+/** The name of the snapshot of a run that was given no settings. */
+export const DEFAULT_SETTINGS_NAME = 'default';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Canonical JSON: keys sorted at every level, no whitespace, array order kept.
+ * A value JSON cannot carry (`undefined` inside an array, a non-finite number,
+ * a function) is refused, because a digest over a value that cannot be written
+ * back would name something nobody can reproduce. An `undefined` object field
+ * is the same as an absent one and is dropped.
+ */
+export function canonicalJson(value) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean') {
+        return JSON.stringify(value);
+    }
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value)) {
+            throw new EvalUsageError(`a setting must be a finite number; received ${String(value)}`);
+        }
+        return JSON.stringify(value);
+    }
+    if (Array.isArray(value)) {
+        return `[${value
+            .map((item) => {
+            if (item === undefined)
+                throw new EvalUsageError('a setting list cannot hold undefined');
+            return canonicalJson(item);
+        })
+            .join(',')}]`;
+    }
+    if (isRecord(value)) {
+        const parts = [];
+        for (const key of Object.keys(value).sort()) {
+            const item = value[key];
+            if (item === undefined)
+                continue;
+            parts.push(`${JSON.stringify(key)}:${canonicalJson(item)}`);
+        }
+        return `{${parts.join(',')}}`;
+    }
+    throw new EvalUsageError(`a setting must be plain JSON; received ${typeof value}`);
+}
+/** Names that hold a credential. `maxResponseTokens` and `tokenBudget` are not among them. */
+const SECRET_KEY = /^(?:token|secret|password|passwd|credentials?|authorization|cookie|bearer)$|api[-_]?key|private[-_]?key|secret|passw(?:or)?d|(?:access|auth|refresh|bearer|session|id)[-_]?token|client[-_]?secret/i;
+/** Shapes of the keys and tokens providers hand out. */
+const SECRET_VALUE = [
+    // A password written into a URL: scheme://user:password@host
+    /[A-Za-z][A-Za-z0-9+.-]*:\/\/[^/\s:@]+:[^/\s@]+@/,
+    /\bsk-[A-Za-z0-9_-]{16,}/,
+    /\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{20,}/,
+    /\bxox[abprs]-[A-Za-z0-9-]{10,}/,
+    /\bAKIA[0-9A-Z]{16}\b/,
+    /\bAIza[0-9A-Za-z_-]{30,}/,
+    /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i,
+    /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/,
+    /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+];
+/**
+ * The path of the first thing in `values` that is, or names, a credential, or
+ * undefined when there is none. The path never includes the secret itself.
+ */
+export function findSecretIn(values, path = '') {
+    if (typeof values === 'string') {
+        return SECRET_VALUE.some((pattern) => pattern.test(values)) ? path || '(value)' : undefined;
+    }
+    if (Array.isArray(values)) {
+        for (const [index, item] of values.entries()) {
+            const hit = findSecretIn(item, `${path}[${index}]`);
+            if (hit !== undefined)
+                return hit;
+        }
+        return undefined;
+    }
+    if (isRecord(values)) {
+        for (const key of Object.keys(values).sort()) {
+            const here = path === '' ? key : `${path}.${key}`;
+            // A setting that names the environment variable holding a credential
+            // (`apiKeyEnv`) holds a name, not a secret, but is still refused: the
+            // snapshot has no business carrying where a credential lives.
+            if (SECRET_KEY.test(key))
+                return here;
+            const hit = findSecretIn(values[key], here);
+            if (hit !== undefined)
+                return hit;
+        }
+    }
+    return undefined;
+}
+/** A snapshot name is plain: it is printed in a comparison and used as a file label. */
+export function validateSettingsName(name) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(name)) {
+        throw new EvalUsageError(`a settings name is 1 to 64 letters, digits, dots, dashes or underscores, starting with a letter or digit; received ${JSON.stringify(name)}`);
+    }
+    return name;
+}
+/**
+ * Build a snapshot. Throws an {@link EvalUsageError} naming the path of a
+ * secret, a value JSON cannot carry, or a name that is not plain. A snapshot
+ * called `default` must have no values: `default` is the name of "no settings
+ * stated", and a snapshot with values under it would say the opposite.
+ */
+export function buildSettingsSnapshot(name, values = {}) {
+    validateSettingsName(name);
+    if (!isRecord(values))
+        throw new EvalUsageError('settings must be an object of settings');
+    const secret = findSecretIn(values);
+    if (secret !== undefined) {
+        throw new EvalUsageError(`settings "${secret}" is, or names, a credential. A credential never goes into an eval settings snapshot, because the snapshot is written into the report.`);
+    }
+    const canonical = canonicalJson(values);
+    if (name === DEFAULT_SETTINGS_NAME && canonical !== '{}') {
+        throw new EvalUsageError(`the name "${DEFAULT_SETTINGS_NAME}" is for a run with no settings stated; give these settings another name`);
+    }
+    return {
+        name,
+        digest: createHash('sha256').update(canonical).digest('hex'),
+        algorithm: SETTINGS_DIGEST_ALGORITHM,
+        // A copy through the canonical form, so what is stored is what was hashed.
+        values: JSON.parse(canonical),
+    };
+}
+/** The snapshot of a run given no settings. */
+export function defaultSettingsSnapshot() {
+    return buildSettingsSnapshot(DEFAULT_SETTINGS_NAME, {});
+}
+function leaves(value, path, into) {
+    if (isRecord(value) && Object.keys(value).length > 0) {
+        for (const key of Object.keys(value))
+            leaves(value[key], path === '' ? key : `${path}.${key}`, into);
+        return;
+    }
+    into.set(path, value);
+}
+/** What differs between two snapshots' values, key by key, sorted by key. */
+export function diffSettings(a, b) {
+    const left = new Map();
+    const right = new Map();
+    leaves(a, '', left);
+    leaves(b, '', right);
+    left.delete('');
+    right.delete('');
+    const keys = [...new Set([...left.keys(), ...right.keys()])].sort();
+    const out = [];
+    for (const key of keys) {
+        const inA = left.has(key);
+        const inB = right.has(key);
+        if (inA && inB && canonicalJson(left.get(key)) === canonicalJson(right.get(key)))
+            continue;
+        out.push({
+            key,
+            ...(inA ? { a: left.get(key) } : {}),
+            ...(inB ? { b: right.get(key) } : {}),
+        });
+    }
+    return out;
+}
+//# sourceMappingURL=settings.js.map

package/dist/eval/settings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settings.js","sourceRoot":"","sources":["../../src/eval/settings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AA0B/C,MAAM,CAAC,MAAM,yBAAyB,GAAG,yBAAyB,CAAA;AAElE,oEAAoE;AACpE,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAA;AAiB9C,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC3C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/E,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,cAAc,CAAC,+CAA+C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACzF,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,KAAK;aACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,cAAc,CAAC,sCAAsC,CAAC,CAAA;YACxF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAA;QAC3B,CAAC,CAAC;aACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IACf,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;YACvB,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAQ;YAChC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC5D,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IAC9B,CAAC;IACD,MAAM,IAAI,cAAc,CAAC,0CAA0C,OAAO,KAAK,EAAE,CAAC,CAAA;AACnF,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,GACf,qMAAqM,CAAA;AAEtM,wDAAwD;AACxD,MAAM,YAAY,GAAG;IACpB,6DAA6D;IAC7D,iDAAiD;IACjD,yBAAyB;IACzB,wDAAwD;IACxD,gCAAgC;IAChC,sBAAsB;IACtB,0BAA0B;IAC1B,qCAAqC;IACrC,iEAAiE;IACjE,oCAAoC;CACpC,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAe,EAAE,IAAI,GAAG,EAAE;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;IAC5F,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,CAAA;YACnD,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAA;QAClC,CAAC;QACD,OAAO,SAAS,CAAA;IACjB,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAA;YACjD,qEAAqE;YACrE,sEAAsE;YACtE,8DAA8D;YAC9D,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAA;YACrC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;YAC3C,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAA;QAClC,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAChD,IAAI,CAAC,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,cAAc,CACvB,sHAAsH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAC5I,CAAA;IACF,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACpC,IAAY,EACZ,SAA4C,EAAE;IAE9C,oBAAoB,CAAC,IAAI,CAAC,CAAA;IAC1B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,wCAAwC,CAAC,CAAA;IACzF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;IACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,cAAc,CACvB,aAAa,MAAM,wIAAwI,CAC3J,CAAA;IACF,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IACvC,IAAI,IAAI,KAAK,qBAAqB,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QAC1D,MAAM,IAAI,cAAc,CACvB,aAAa,qBAAqB,0EAA0E,CAC5G,CAAA;IACF,CAAC;IACD,OAAO;QACN,IAAI;QACJ,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC5D,SAAS,EAAE,yBAAyB;QACpC,2EAA2E;QAC3E,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAA4B;KACxD,CAAA;AACF,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,uBAAuB;IACtC,OAAO,qBAAqB,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAA;AACxD,CAAC;AAUD,SAAS,MAAM,CAAC,KAAc,EAAE,IAAY,EAAE,IAA0B;IACvE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,IAAI,CAAC,CAAA;QAC/D,OAAM;IACP,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;AACtB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAC3B,CAAoC,EACpC,CAAoC;IAEpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAA;IACxC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACnB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,CAAA;IACpB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACf,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAChB,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACnE,MAAM,GAAG,GAAwB,EAAE,CAAA;IACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC1B,IAAI,GAAG,IAAI,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAAE,SAAQ;QAC1F,GAAG,CAAC,IAAI,CAAC;YACR,GAAG;YACH,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrC,CAAC,CAAA;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC"}

package/dist/eval/split.d.ts

@@ -0,0 +1,48 @@
+import type { EvalSplit } from './types.js';
+/**
+ * Which tasks a harness setting may be chosen on, and which it is judged on.
+ *
+ * A setting tuned against some tasks and then judged on the same tasks has been
+ * shown to fit them, and nothing more. Every task belongs to one of two splits:
+ * `tune`, the tasks to choose settings on, and `held-out`, the tasks to judge
+ * the choice on. The split of a task is a function of its identity and a fixed
+ * seed, so the same task is in the same split on every machine, in every run,
+ * whoever runs it. A suite may state a task's split itself, and that wins.
+ */
+export type TaskSplit = 'tune' | 'held-out';
+/** What `namzu eval --split` selects. */
+export type SplitFilter = TaskSplit | 'all';
+export declare const SPLIT_FILTERS: readonly SplitFilter[];
+/** The seed of the assignment. Fixed in the kernel and written into every report. */
+export declare const TASK_SPLIT_SEED = "namzu-eval-split-1";
+/** The share of tasks assigned to `held-out`, written into every report. */
+export declare const HELD_OUT_FRACTION = 0.5;
+export declare const TASK_SPLIT_ALGORITHM = "sha256(seed, suite, task) as a fraction; held-out when below the share";
+/** The id of a task: its suite and its case name, which is unique within the suite. */
+export declare function taskId(suite: string, name: string): string;
+/** The split a seeded hash assigns to a task, whatever a suite declares. */
+export declare function hashedSplit(suite: string, name: string): TaskSplit;
+/**
+ * The split of a task. A declared split wins; `development`, the name a case
+ * declares for the tasks a harness was shaped against, is `tune`.
+ */
+export declare function assignSplit(suite: string, name: string, declared?: EvalSplit): {
+    split: TaskSplit;
+    declared: boolean;
+};
+export declare function validateSplitFilter(value: string | undefined): SplitFilter;
+/** The tasks a suite ran, and how they were assigned. Written into the suite's report. */
+export interface ReportTaskSplit {
+    /** What the run asked for. */
+    readonly requested: SplitFilter;
+    readonly seed: typeof TASK_SPLIT_SEED;
+    /** The share of tasks assigned to `held-out` when a suite does not say. */
+    readonly heldOutFraction: number;
+    /** The tasks that ran, in the suite's order, each with its split. */
+    readonly tasks: readonly {
+        readonly id: string;
+        readonly split: TaskSplit;
+        readonly declared: boolean;
+    }[];
+}
+//# sourceMappingURL=split.d.ts.map

package/dist/eval/split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../src/eval/split.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAA;AAE3C,yCAAyC;AACzC,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,KAAK,CAAA;AAE3C,eAAO,MAAM,aAAa,EAAE,SAAS,WAAW,EAAgC,CAAA;AAEhF,qFAAqF;AACrF,eAAO,MAAM,eAAe,uBAAuB,CAAA;AAEnD,4EAA4E;AAC5E,eAAO,MAAM,iBAAiB,MAAM,CAAA;AAEpC,eAAO,MAAM,oBAAoB,2EACwC,CAAA;AAEzE,uFAAuF;AACvF,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,CAOlE;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,SAAS,GAClB;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAIzC;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,CAQ1E;AAED,0FAA0F;AAC1F,MAAM,WAAW,eAAe;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAA;IAC/B,QAAQ,CAAC,IAAI,EAAE,OAAO,eAAe,CAAA;IACrC,2EAA2E;IAC3E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,SAAS;QACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;QACzB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;KAC1B,EAAE,CAAA;CACH"}

package/dist/eval/split.js

@@ -0,0 +1,41 @@
+import { createHash } from 'node:crypto';
+import { EvalUsageError } from './bootstrap.js';
+export const SPLIT_FILTERS = ['tune', 'held-out', 'all'];
+/** The seed of the assignment. Fixed in the kernel and written into every report. */
+export const TASK_SPLIT_SEED = 'namzu-eval-split-1';
+/** The share of tasks assigned to `held-out`, written into every report. */
+export const HELD_OUT_FRACTION = 0.5;
+export const TASK_SPLIT_ALGORITHM = 'sha256(seed, suite, task) as a fraction; held-out when below the share';
+/** The id of a task: its suite and its case name, which is unique within the suite. */
+export function taskId(suite, name) {
+    return `${suite}/${name}`;
+}
+/** The split a seeded hash assigns to a task, whatever a suite declares. */
+export function hashedSplit(suite, name) {
+    const digest = createHash('sha256')
+        .update(`${TASK_SPLIT_SEED}\u0000${suite}\u0000${name}`)
+        .digest();
+    // Six bytes: 48 bits, exact in a double.
+    const fraction = digest.readUIntBE(0, 6) / 2 ** 48;
+    return fraction < HELD_OUT_FRACTION ? 'held-out' : 'tune';
+}
+/**
+ * The split of a task. A declared split wins; `development`, the name a case
+ * declares for the tasks a harness was shaped against, is `tune`.
+ */
+export function assignSplit(suite, name, declared) {
+    if (declared === 'held-out')
+        return { split: 'held-out', declared: true };
+    if (declared === 'development')
+        return { split: 'tune', declared: true };
+    return { split: hashedSplit(suite, name), declared: false };
+}
+export function validateSplitFilter(value) {
+    if (value === undefined)
+        return 'all';
+    if (!SPLIT_FILTERS.includes(value)) {
+        throw new EvalUsageError(`split must be "tune", "held-out" or "all"; received ${JSON.stringify(value)}`);
+    }
+    return value;
+}
+//# sourceMappingURL=split.js.map

package/dist/eval/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/eval/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAkB/C,MAAM,CAAC,MAAM,aAAa,GAA2B,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;AAEhF,qFAAqF;AACrF,MAAM,CAAC,MAAM,eAAe,GAAG,oBAAoB,CAAA;AAEnD,4EAA4E;AAC5E,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAEpC,MAAM,CAAC,MAAM,oBAAoB,GAChC,wEAAwE,CAAA;AAEzE,uFAAuF;AACvF,MAAM,UAAU,MAAM,CAAC,KAAa,EAAE,IAAY;IACjD,OAAO,GAAG,KAAK,IAAI,IAAI,EAAE,CAAA;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,IAAY;IACtD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;SACjC,MAAM,CAAC,GAAG,eAAe,SAAS,KAAK,SAAS,IAAI,EAAE,CAAC;SACvD,MAAM,EAAE,CAAA;IACV,yCAAyC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;IAClD,OAAO,QAAQ,GAAG,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAA;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAC1B,KAAa,EACb,IAAY,EACZ,QAAoB;IAEpB,IAAI,QAAQ,KAAK,UAAU;QAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IACzE,IAAI,QAAQ,KAAK,aAAa;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IACxE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAA;AAC5D,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAyB;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IACrC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAoB,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,cAAc,CACvB,uDAAuD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC9E,CAAA;IACF,CAAC;IACD,OAAO,KAAoB,CAAA;AAC5B,CAAC"}