@namzu/sdk 62.14.0 → 62.15.0

package/dist/eval/experiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AAWxD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,8DAA8D;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IAC3C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAE9E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAC7C,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACvD,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAYnB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,qEAAqE;IACrE,wEAAwE;IACxE,MAAM,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,QAAQ,EAAE,CAAC;QACX,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;KAC5C,CAAC,CAAC,CAAA;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAA;IACnF,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAoB;QAClC,SAAS;QACT,IAAI,EAAE,eAAe;QACrB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACpB,CAAC,CAAC;KACH,CAAA;IAED,8DAA8D;IAC9D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IACpC,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAEvE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EACtC,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;QACD,QAAQ,EAAE;YACT,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,uBAAuB,EAAE,CAAC;YAClD,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,IAAI,KAAK;YACxC,QAAQ,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;SACvD;QACD,SAAS;KACT,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,mEAAmE;YACnE,uEAAuE;YACvE,IAAI,MAAM,CAAC,SAAS,EAAE,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;gBACrF,SAAQ;YACT,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAA;QACzB,KAAK,CAAC,IAAI,CACT,gBAAgB,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kCAAkC,EAAE,CACnJ,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAA;QAC1B,KAAK,CAAC,IAAI,CACT,YAAY,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CACvF,CAAA;IACF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/index.d.ts

@@ -11,13 +11,17 @@ export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL,
 export type { BootstrapMethod, CaseIntervals, CaseTally, DifferenceIntervals, Interval, PairedTally, } from './bootstrap.js';
 export type { ReportDisclosure, RoleDisclosure, RoutingDisclosure, RunMode, SuiteFacts, } from './disclosure.js';
 export { sdkVersion } from './disclosure.js';
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { currentEvalRunOptions, currentEvalSettings, evalSetting, withEvalRunOptions, } from './run-options.js';
+export { DEFAULT_SETTINGS_NAME, SETTINGS_DIGEST_ALGORITHM, buildSettingsSnapshot, canonicalJson, defaultSettingsSnapshot, diffSettings, findSecretIn, validateSettingsName, } from './settings.js';
+export type { ReportSettings, SettingDifference, SettingsSnapshot } from './settings.js';
+export { HELD_OUT_FRACTION, SPLIT_FILTERS, TASK_SPLIT_SEED, assignSplit, hashedSplit, taskId, validateSplitFilter, } from './split.js';
+export type { ReportTaskSplit, SplitFilter, TaskSplit } from './split.js';
 export type { EvalRunOptions } from './run-options.js';
 export { formatRunSummary, summarizeRun } from './run-summary.js';
 export type { RunDisclosure, RunSummary } from './run-summary.js';
 export type { RepeatReliability, SplitSummary } from './report-stats.js';
 export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
-export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSuite, SideNumbers, } from './compare.js';
+export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSettings, LoadedSuite, SettingsComparison, SideSplit, SnapshotRef, WithheldReason, SideNumbers, } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 export type { ScoreUncertainty } from './uncertainty.js';
 export type { CaseRepeatSummary, CaseResult, EvalSplit, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EACN,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,YAAY,EACZ,YAAY,EACZ,oBAAoB,GACpB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACxF,OAAO,EACN,iBAAiB,EACjB,aAAa,EACb,eAAe,EACf,WAAW,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACzE,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,cAAc,EACd,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}

package/dist/eval/index.js

@@ -5,7 +5,9 @@ export { judgeScorer } from './judge.js';
 export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
 export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
 export { sdkVersion } from './disclosure.js';
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { currentEvalRunOptions, currentEvalSettings, evalSetting, withEvalRunOptions, } from './run-options.js';
+export { DEFAULT_SETTINGS_NAME, SETTINGS_DIGEST_ALGORITHM, buildSettingsSnapshot, canonicalJson, defaultSettingsSnapshot, diffSettings, findSecretIn, validateSettingsName, } from './settings.js';
+export { HELD_OUT_FRACTION, SPLIT_FILTERS, TASK_SPLIT_SEED, assignSplit, hashedSplit, taskId, validateSplitFilter, } from './split.js';
 export { formatRunSummary, summarizeRun } from './run-summary.js';
 export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAa3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EACN,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,YAAY,EACZ,YAAY,EACZ,oBAAoB,GACpB,MAAM,eAAe,CAAA;AAEtB,OAAO,EACN,iBAAiB,EACjB,aAAa,EACb,eAAe,EACf,WAAW,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAkB3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/eval/report-stats.d.ts

@@ -74,4 +74,10 @@ export declare function describeIntervals(args: {
 }): string[];
 /** The line that says which splits a suite has, and says so plainly when none is held out. */
 export declare function describeSplits(splits: RunStats['splits']): string[];
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export declare function clarifyDeclared(line: string): string;
 //# sourceMappingURL=report-stats.d.ts.map

package/dist/eval/report-stats.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE"}
+{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIpD"}

package/dist/eval/report-stats.js

@@ -181,4 +181,14 @@ export function describeSplits(splits) {
     }
     return lines;
 }
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export function clarifyDeclared(line) {
+    return line.startsWith('no held-out cases')
+        ? `${line} (declared by the suite; the split above is the hashed one)`
+        : line;
+}
 //# sourceMappingURL=report-stats.js.map

package/dist/eval/report-stats.js.map

@@ -1 +1 @@
-{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}
+{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAC1C,CAAC,CAAC,GAAG,IAAI,6DAA6D;QACtE,CAAC,CAAC,IAAI,CAAA;AACR,CAAC"}

package/dist/eval/run-options.d.ts

@@ -1,3 +1,5 @@
+import type { SettingsSnapshot } from './settings.js';
+import type { SplitFilter } from './split.js';
 /**
  * Settings an operator gives a whole eval run, handed to every
  * `runExperiment` that executes inside it.
@@ -22,9 +24,32 @@ export interface EvalRunOptions {
     readonly commit?: string;
     /** Version of the suite being run, when known. */
     readonly suiteVersion?: string;
+    /**
+     * The settings the run was given. Written into every report. A suite shapes
+     * itself by them through {@link currentEvalSettings}; one that does not is
+     * reported as not having read them.
+     */
+    readonly settings?: SettingsSnapshot;
+    /** Which tasks run. Absent means all of them. */
+    readonly split?: SplitFilter;
+    /** Set by {@link currentEvalSettings} when a suite asks for the settings. Owned by the runner. */
+    readonly settingsUse?: {
+        read: boolean;
+        keys?: Set<string>;
+    };
 }
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export declare const READ_ALL = "*";
 /** Run `work` with `options` in force for every experiment it starts. */
 export declare function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T;
 /** The options in force here, or an empty set outside any scope. */
 export declare function currentEvalRunOptions(): EvalRunOptions;
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export declare function currentEvalSettings(): Readonly<Record<string, unknown>>;
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export declare function evalSetting(path: string): unknown;
 //# sourceMappingURL=run-options.d.ts.map

package/dist/eval/run-options.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAcD,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD"}
+{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IACpC,iDAAiD;IACjD,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAA;IAC5B,kGAAkG;IAClG,QAAQ,CAAC,WAAW,CAAC,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;KAAE,CAAA;CAC5D;AAED,4FAA4F;AAC5F,eAAO,MAAM,QAAQ,MAAM,CAAA;AAoB3B,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAQvE;AAED,qGAAqG;AACrG,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAYjD"}

package/dist/eval/run-options.js

@@ -1,4 +1,11 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export const READ_ALL = '*';
+function markRead(use, key) {
+    const keys = use.keys ?? new Set();
+    keys.add(key);
+    use.keys = keys;
+}
 const STORE_KEY = Symbol.for('namzu.eval.runOptions');
 /**
  * One store per process, found through a global symbol, so a suite that
@@ -17,4 +24,33 @@ export function withEvalRunOptions(options, work) {
 export function currentEvalRunOptions() {
     return store().getStore() ?? {};
 }
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export function currentEvalSettings() {
+    const options = currentEvalRunOptions();
+    if (options.settingsUse) {
+        options.settingsUse.read = true;
+        // The whole object was asked for: every key counts as read.
+        markRead(options.settingsUse, READ_ALL);
+    }
+    return options.settings?.values ?? {};
+}
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export function evalSetting(path) {
+    const options = currentEvalRunOptions();
+    let at = options.settings?.values ?? {};
+    if (options.settingsUse) {
+        options.settingsUse.read = true;
+        markRead(options.settingsUse, path);
+    }
+    for (const part of path.split('.')) {
+        if (typeof at !== 'object' || at === null || Array.isArray(at))
+            return undefined;
+        at = at[part];
+    }
+    return at;
+}
 //# sourceMappingURL=run-options.js.map

package/dist/eval/run-options.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4BpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC"}
+{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAwCpD,4FAA4F;AAC5F,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAA;AAE3B,SAAS,QAAQ,CAAC,GAA0C,EAAE,GAAW;IACxE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,IAAI,GAAG,EAAU,CAAA;IAC1C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,GAAG,CAAC,IAAI,GAAG,IAAI,CAAA;AAChB,CAAC;AAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IAClC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAC/B,4DAA4D;QAC5D,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAA;AACtC,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,IAAI,EAAE,GAAY,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAA;IAChD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAA;QAC/B,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;IACpC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAAE,OAAO,SAAS,CAAA;QAChF,EAAE,GAAI,EAA8B,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,EAAE,CAAA;AACV,CAAC"}

package/dist/eval/run-summary.d.ts

@@ -1,5 +1,7 @@
 import type { RunMode } from './disclosure.js';
 import { type RunStats } from './report-stats.js';
+import type { SettingsSnapshot } from './settings.js';
+import type { SplitFilter } from './split.js';
 import type { ExperimentReport } from './types.js';
 /** What produced a whole run, across every suite in it. */
 export interface RunDisclosure {
@@ -15,6 +17,10 @@ export interface RunDisclosure {
     readonly runs: number;
     /** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
     readonly mode: RunMode | 'mixed';
+    /** The settings the run was given. Absent from a run written before reports named them. */
+    readonly settings?: SettingsSnapshot;
+    /** Which tasks ran. Absent from a run written before there were splits. */
+    readonly split?: SplitFilter;
 }
 /** The numbers for every case of every suite taken together. */
 export interface RunSummary extends Omit<RunStats, 'caseSummaries' | 'reliability'> {
@@ -31,6 +37,8 @@ export declare function summarizeRun(entries: ReadonlyArray<{
     commit: string;
     generatedAt: string;
     sdkVersion: string;
+    settings?: SettingsSnapshot;
+    split?: SplitFilter;
 }): RunSummary;
 export declare function formatRunSummary(summary: RunSummary): string;
 //# sourceMappingURL=run-summary.d.ts.map

package/dist/eval/run-summary.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAKb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;CAChC;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB,GACC,UAAU,CAmCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAuB5D"}
+{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAMb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;IAChC,2FAA2F;IAC3F,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IACpC,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAA;CAC5B;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB,GACC,UAAU,CAqCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CA8B5D"}

package/dist/eval/run-summary.js

@@ -1,5 +1,5 @@
 import { UNKNOWN } from './disclosure.js';
-import { describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
+import { clarifyDeclared, describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
 export function summarizeRun(entries, context) {
     // A case is named by its suite and its id: two suites may reuse a name.
     const cases = entries.flatMap(({ suite, report }) => groupRuns(report.cases ?? [], (name) => `${suite}\u0000${name}`));
@@ -30,6 +30,8 @@ export function summarizeRun(entries, context) {
             cases: stats.cases,
             runs: stats.runs,
             mode,
+            ...(context.settings ? { settings: context.settings } : {}),
+            ...(context.split ? { split: context.split } : {}),
         },
     };
 }
@@ -49,8 +51,16 @@ export function formatRunSummary(summary) {
         const r = summary.reliability;
         lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
     }
-    for (const line of describeSplits(summary.splits))
-        lines.push(`  ${line}`);
+    for (const line of describeSplits(summary.splits)) {
+        if (d.split === 'held-out' && line.startsWith('no held-out cases'))
+            continue;
+        lines.push(`  ${d.split ? clarifyDeclared(line) : line}`);
+    }
+    if (d.settings) {
+        lines.push(`  settings: "${d.settings.name}", sha256 ${d.settings.digest.slice(0, 12)}`);
+    }
+    if (d.split)
+        lines.push(`  split: ${d.split}`);
     lines.push(`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`);
     return lines.join('\n');
 }

package/dist/eval/run-summary.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AA0B1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAMC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;SACJ;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAgC1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAQC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;YACJ,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;YAAE,SAAQ;QAC5E,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAA;IACzF,CAAC;IACD,IAAI,CAAC,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;IAC9C,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/settings.d.ts

@@ -0,0 +1,74 @@
+/**
+ * The settings an eval run ran under, as a named, canonical, hashed record.
+ *
+ * Two reports of one suite say little about a harness setting unless each says
+ * exactly which settings it ran under. The snapshot is that statement: a name a
+ * person chose (or `default`), a digest of the values, and the values. The
+ * digest is taken over the values alone, in a canonical form, so it is the same
+ * whatever order the keys were written in and whatever the snapshot is called.
+ *
+ * Secrets never enter one. {@link buildSettingsSnapshot} refuses a value that
+ * names a credential or looks like one, so a report written from a snapshot
+ * cannot carry a key even if a caller passed it by mistake.
+ */
+export interface SettingsSnapshot {
+    /** What the person called it; the file's name when it came from a file. */
+    readonly name: string;
+    /** SHA-256 of the canonical JSON of {@link values}, in hex. */
+    readonly digest: string;
+    /** How the digest was made, so a later version can change it without a silent mismatch. */
+    readonly algorithm: typeof SETTINGS_DIGEST_ALGORITHM;
+    /** The settings that differ from the defaults: whatever was stated. Empty for `default`. */
+    readonly values: Readonly<Record<string, unknown>>;
+}
+export declare const SETTINGS_DIGEST_ALGORITHM = "sha256-canonical-json-1";
+/** The name of the snapshot of a run that was given no settings. */
+export declare const DEFAULT_SETTINGS_NAME = "default";
+/** What a suite that read the settings says about it, written into its report. */
+export interface ReportSettings extends SettingsSnapshot {
+    /**
+     * Whether the suite asked for the settings while it ran. A suite that never
+     * does cannot have been shaped by them, so a difference between two snapshots
+     * says nothing about its numbers.
+     */
+    readonly read: boolean;
+    /**
+     * The dotted paths the suite asked for, or `*` when it asked for the whole
+     * object. Absent from a report that predates it, which is read as every key.
+     */
+    readonly readKeys?: readonly string[];
+}
+/**
+ * Canonical JSON: keys sorted at every level, no whitespace, array order kept.
+ * A value JSON cannot carry (`undefined` inside an array, a non-finite number,
+ * a function) is refused, because a digest over a value that cannot be written
+ * back would name something nobody can reproduce. An `undefined` object field
+ * is the same as an absent one and is dropped.
+ */
+export declare function canonicalJson(value: unknown): string;
+/**
+ * The path of the first thing in `values` that is, or names, a credential, or
+ * undefined when there is none. The path never includes the secret itself.
+ */
+export declare function findSecretIn(values: unknown, path?: string): string | undefined;
+/** A snapshot name is plain: it is printed in a comparison and used as a file label. */
+export declare function validateSettingsName(name: string): string;
+/**
+ * Build a snapshot. Throws an {@link EvalUsageError} naming the path of a
+ * secret, a value JSON cannot carry, or a name that is not plain. A snapshot
+ * called `default` must have no values: `default` is the name of "no settings
+ * stated", and a snapshot with values under it would say the opposite.
+ */
+export declare function buildSettingsSnapshot(name: string, values?: Readonly<Record<string, unknown>>): SettingsSnapshot;
+/** The snapshot of a run given no settings. */
+export declare function defaultSettingsSnapshot(): SettingsSnapshot;
+export interface SettingDifference {
+    /** Dotted path of the setting; a list is one value. */
+    readonly key: string;
+    /** Absent on the side that does not state it. */
+    readonly a?: unknown;
+    readonly b?: unknown;
+}
+/** What differs between two snapshots' values, key by key, sorted by key. */
+export declare function diffSettings(a: Readonly<Record<string, unknown>>, b: Readonly<Record<string, unknown>>): SettingDifference[];
+//# sourceMappingURL=settings.d.ts.map

package/dist/eval/settings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settings.d.ts","sourceRoot":"","sources":["../../src/eval/settings.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAChC,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,+DAA+D;IAC/D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,2FAA2F;IAC3F,QAAQ,CAAC,SAAS,EAAE,OAAO,yBAAyB,CAAA;IACpD,4FAA4F;IAC5F,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CAClD;AAED,eAAO,MAAM,yBAAyB,4BAA4B,CAAA;AAElE,oEAAoE;AACpE,eAAO,MAAM,qBAAqB,YAAY,CAAA;AAE9C,kFAAkF;AAClF,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACvD;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACrC;AAMD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CA4BpD;AAoBD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,SAAK,GAAG,MAAM,GAAG,SAAS,CAuB3E;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOzD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAM,GAC5C,gBAAgB,CAsBlB;AAED,+CAA+C;AAC/C,wBAAgB,uBAAuB,IAAI,gBAAgB,CAE1D;AAED,MAAM,WAAW,iBAAiB;IACjC,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,iDAAiD;IACjD,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;IACpB,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;CACpB;AAWD,6EAA6E;AAC7E,wBAAgB,YAAY,CAC3B,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACpC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAClC,iBAAiB,EAAE,CAoBrB"}