@namzu/sdk 62.11.0 → 62.13.0

package/dist/eval/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AA6BnG,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SAClE,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AAmDD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC5D,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;KACjD,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CAAC,IAAc,EAAE,MAAc,EAAE,KAAY;IAC3D,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/disclosure.d.ts

@@ -0,0 +1,70 @@
+import type { EvalRunOptions } from './run-options.js';
+/**
+ * What a report says about how it was produced.
+ *
+ * A score without its provenance cannot be compared with another: two runs
+ * three points apart may differ by a model, a route, a reasoning effort or a
+ * harness version, and the number alone does not say which. Every fact here
+ * is either known or the literal string `unknown`. Nothing is guessed, and a
+ * report written before this existed has none of it and is read as unknown.
+ */
+export type RunMode = 
+/** The model is a script; nothing measures a vendor. */
+'scripted'
+/** A real provider answered. */
+ | 'real-model'
+/** No model was involved at all. */
+ | 'no-model' | 'unknown';
+export interface RoleDisclosure {
+    readonly provider: string;
+    readonly model: string;
+}
+/** The routing configuration, `off` when routing was not on. */
+export type RoutingDisclosure = 'off' | 'unknown' | Readonly<Record<string, unknown>>;
+/**
+ * What a suite declares about itself in its `runExperiment` config. Every
+ * field is optional; the ones left out are written `unknown`.
+ */
+export interface SuiteFacts {
+    readonly mode?: RunMode;
+    /** One entry per role that ran, such as `agent`, `judge`, `child`. */
+    readonly roles?: Readonly<Record<string, {
+        provider?: string;
+        model?: string;
+    }>>;
+    readonly routing?: RoutingDisclosure;
+    /** The reasoning effort the model ran at, or `not applicable`. */
+    readonly reasoningEffort?: string;
+    /** Version of the suite. The runner fills it from the suite's package when it can. */
+    readonly suiteVersion?: string;
+}
+export interface ReportDisclosure {
+    /** The program that ran the suites, for example the CLI's version. */
+    readonly harnessVersion: string;
+    readonly sdkVersion: string;
+    /** ISO time the experiment finished. */
+    readonly generatedAt: string;
+    /** Cases in the suite. */
+    readonly cases: number;
+    readonly repeats: number;
+    readonly seed: number;
+    readonly mode: RunMode;
+    /** `unknown` when the suite declared no roles; an empty object when no model ran. */
+    readonly roles: Readonly<Record<string, RoleDisclosure>> | 'unknown';
+    readonly routing: RoutingDisclosure;
+    readonly reasoningEffort: string;
+    readonly suiteVersion: string;
+    /** Source revision, when the runner could read one. */
+    readonly commit: string;
+}
+export declare const UNKNOWN = "unknown";
+/** The version of this SDK, read from its manifest; `unknown` when it cannot be read. */
+export declare function sdkVersion(): string;
+export declare function buildDisclosure(facts: SuiteFacts | undefined, counts: {
+    cases: number;
+    repeats: number;
+    seed: number;
+}, options: EvalRunOptions, generatedAt: string): ReportDisclosure;
+/** One plain line for a terminal. */
+export declare function describeDisclosure(d: ReportDisclosure): string;
+//# sourceMappingURL=disclosure.d.ts.map

package/dist/eval/disclosure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"disclosure.d.ts","sourceRoot":"","sources":["../../src/eval/disclosure.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAEtD;;;;;;;;GAQG;AACH,MAAM,MAAM,OAAO;AAClB,wDAAwD;AACtD,UAAU;AACZ,gCAAgC;GAC9B,YAAY;AACd,oCAAoC;GAClC,UAAU,GACV,SAAS,CAAA;AAEZ,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;AAErF;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAA;IAChF,QAAQ,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;IACpC,kEAAkE;IAClE,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,sFAAsF;IACtF,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAChC,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,wCAAwC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,qFAAqF;IACrF,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,GAAG,SAAS,CAAA;IACpE,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,eAAO,MAAM,OAAO,YAAY,CAAA;AAIhC,yFAAyF;AACzF,wBAAgB,UAAU,IAAI,MAAM,CAcnC;AAMD,wBAAgB,eAAe,CAC9B,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EACxD,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GACjB,gBAAgB,CAuBlB;AAED,qCAAqC;AACrC,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAc9D"}

package/dist/eval/disclosure.js

@@ -0,0 +1,64 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+export const UNKNOWN = 'unknown';
+let cachedSdkVersion;
+/** The version of this SDK, read from its manifest; `unknown` when it cannot be read. */
+export function sdkVersion() {
+    if (cachedSdkVersion !== undefined)
+        return cachedSdkVersion;
+    try {
+        // `src/eval/` and `dist/eval/` are both two levels below the package.
+        const path = fileURLToPath(new URL('../../package.json', import.meta.url));
+        const manifest = JSON.parse(readFileSync(path, 'utf8'));
+        cachedSdkVersion =
+            typeof manifest.version === 'string' && manifest.version.length > 0
+                ? manifest.version
+                : UNKNOWN;
+    }
+    catch {
+        cachedSdkVersion = UNKNOWN;
+    }
+    return cachedSdkVersion;
+}
+function text(value) {
+    return value !== undefined && value.length > 0 ? value : UNKNOWN;
+}
+export function buildDisclosure(facts, counts, options, generatedAt) {
+    let roles = UNKNOWN;
+    if (facts?.roles !== undefined) {
+        const declared = {};
+        for (const [role, who] of Object.entries(facts.roles)) {
+            declared[role] = { provider: text(who.provider), model: text(who.model) };
+        }
+        roles = declared;
+    }
+    return {
+        harnessVersion: text(options.harnessVersion),
+        sdkVersion: sdkVersion(),
+        generatedAt,
+        cases: counts.cases,
+        repeats: counts.repeats,
+        seed: counts.seed,
+        mode: facts?.mode ?? UNKNOWN,
+        roles,
+        routing: facts?.routing ?? UNKNOWN,
+        reasoningEffort: text(facts?.reasoningEffort),
+        suiteVersion: text(facts?.suiteVersion ?? options.suiteVersion),
+        commit: text(options.commit),
+    };
+}
+/** One plain line for a terminal. */
+export function describeDisclosure(d) {
+    const roles = d.roles === UNKNOWN
+        ? 'models unknown'
+        : Object.keys(d.roles).length === 0
+            ? 'no model'
+            : Object.entries(d.roles)
+                .map(([role, who]) => `${role} ${who.provider}/${who.model}`)
+                .join(', ');
+    const routing = d.routing === 'off' || d.routing === UNKNOWN
+        ? `routing ${d.routing}`
+        : `routing ${JSON.stringify(d.routing)}`;
+    return `ran as ${d.mode}: ${roles}; ${routing}; effort ${d.reasoningEffort}; suite ${d.suiteVersion}`;
+}
+//# sourceMappingURL=disclosure.js.map

package/dist/eval/disclosure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"disclosure.js","sourceRoot":"","sources":["../../src/eval/disclosure.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAgExC,MAAM,CAAC,MAAM,OAAO,GAAG,SAAS,CAAA;AAEhC,IAAI,gBAAoC,CAAA;AAExC,yFAAyF;AACzF,MAAM,UAAU,UAAU;IACzB,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAA;IAC3D,IAAI,CAAC;QACJ,sEAAsE;QACtE,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA0B,CAAA;QAChF,gBAAgB;YACf,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;gBAClE,CAAC,CAAC,QAAQ,CAAC,OAAO;gBAClB,CAAC,CAAC,OAAO,CAAA;IACZ,CAAC;IAAC,MAAM,CAAC;QACR,gBAAgB,GAAG,OAAO,CAAA;IAC3B,CAAC;IACD,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,SAAS,IAAI,CAAC,KAAyB;IACtC,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAA;AACjE,CAAC;AAED,MAAM,UAAU,eAAe,CAC9B,KAA6B,EAC7B,MAAwD,EACxD,OAAuB,EACvB,WAAmB;IAEnB,IAAI,KAAK,GAA8B,OAAO,CAAA;IAC9C,IAAI,KAAK,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAmC,EAAE,CAAA;QACnD,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAA;QAC1E,CAAC;QACD,KAAK,GAAG,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO;QACN,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAC5C,UAAU,EAAE,UAAU,EAAE;QACxB,WAAW;QACX,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,OAAO;QAC5B,KAAK;QACL,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,OAAO;QAClC,eAAe,EAAE,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC;QAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC;QAC/D,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC5B,CAAA;AACF,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,kBAAkB,CAAC,CAAmB;IACrD,MAAM,KAAK,GACV,CAAC,CAAC,KAAK,KAAK,OAAO;QAClB,CAAC,CAAC,gBAAgB;QAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;iBACtB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;iBAC5D,IAAI,CAAC,IAAI,CAAC,CAAA;IAChB,MAAM,OAAO,GACZ,CAAC,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO;QAC3C,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACxB,CAAC,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAA;IAC1C,OAAO,UAAU,CAAC,CAAC,IAAI,KAAK,KAAK,KAAK,OAAO,YAAY,CAAC,CAAC,eAAe,WAAW,CAAC,CAAC,YAAY,EAAE,CAAA;AACtG,CAAC"}

package/dist/eval/experiment.d.ts

@@ -1,3 +1,4 @@
+import { type SuiteFacts } from './disclosure.js';
 import type { CaseResult, EvalCase, EvalTurn, ExperimentReport, Scorer } from './types.js';
 export interface ExperimentConfig<TInput = unknown> {
     name: string;
@@ -29,7 +30,29 @@ export interface ExperimentConfig<TInput = unknown> {
     passThreshold?: number;
     /** Cases to run at once. Default 1 — deterministic ordering by default. */
     concurrency?: number;
+    /** Called once per run of a case; with repeats, once per repeat. */
     onCaseFinish?: (result: CaseResult) => void;
+    /**
+     * Times every case runs, from 1 to 100. Default 1.
+     *
+     * One run of a case says little about a model that does not answer the
+     * same way twice. The report keeps every run and says per case how many
+     * passed, and per suite how many cases passed every time. An
+     * `EvalRunOptions` scope opened by `namzu eval --repeat` wins over this.
+     */
+    repeats?: number;
+    /**
+     * Seed for the bootstrap intervals, an integer from 0 to 4294967295.
+     * Default 1, so the same cases give the same interval on every machine.
+     * Written into the report.
+     */
+    seed?: number;
+    /**
+     * What the suite knows about how it runs: scripted or a real model, which
+     * provider and model fill each role, routing, reasoning effort. Whatever is
+     * left out is written `unknown` in the report, never guessed.
+     */
+    disclosure?: SuiteFacts;
 }
 /**
  * Run a dataset and score it.

package/dist/eval/experiment.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAMnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;CAC3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA2C3B;AAwPD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA6D7D"}
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAUtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAsE3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA4F7D"}

package/dist/eval/experiment.js

@@ -1,4 +1,9 @@
+import { EvalUsageError, validateSeed } from './bootstrap.js';
+import { buildDisclosure, describeDisclosure } from './disclosure.js';
+import { describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
+import { currentEvalRunOptions } from './run-options.js';
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js';
+const MAX_REPEATS = 100;
 const MAX_CASE_TIMEOUT_MS = 2_147_483_647;
 /**
  * Run a dataset and score it.
@@ -17,25 +22,42 @@ export async function runExperiment(config) {
     const threshold = config.passThreshold ?? 1;
     const concurrency = Math.max(1, config.concurrency ?? 1);
     const timeoutMs = validateCaseTimeoutMs(config.timeoutMs);
-    const results = new Array(config.cases.length);
+    const options = currentEvalRunOptions();
+    const repeats = validateRepeats(options.repeats ?? config.repeats);
+    const seed = validateSeed(options.seed ?? config.seed);
+    for (const evalCase of config.cases)
+        validateSplit(evalCase);
+    const alike = duplicateName(config.cases.map((c) => c.name));
+    if (alike !== undefined)
+        throw new Error(duplicateMessage(config.name, alike));
+    // Case-major, so a case's repeats sit together in the report.
+    const total = config.cases.length * repeats;
+    const results = new Array(total);
     let cursor = 0;
     const worker = async () => {
         for (;;) {
             const index = cursor++;
-            const evalCase = config.cases[index];
-            if (!evalCase)
+            if (index >= total)
                 return;
+            const evalCase = config.cases[Math.floor(index / repeats)];
             const result = await evaluateCase(config, evalCase, threshold, timeoutMs);
+            result.repeat = (index % repeats) + 1;
+            if (evalCase.split)
+                result.split = evalCase.split;
             results[index] = result;
             config.onCaseFinish?.(result);
         }
     };
-    await Promise.all(Array.from({ length: Math.min(concurrency, config.cases.length) }, worker));
+    await Promise.all(Array.from({ length: Math.min(concurrency, total) }, worker));
     const settled = results.filter(Boolean);
     // An inconclusive case has no mean to contribute; averaging its zero in
     // would report a suite-wide regression caused by the harness.
     const scored = settled.filter((r) => r.status !== 'inconclusive');
     const mean = scored.length === 0 ? 0 : scored.reduce((sum, r) => sum + r.mean, 0) / scored.length;
+    const stats = statsOver(groupRuns(settled), seed);
+    // One number per CASE, so a case's repeats are not counted as independent
+    // evidence about the suite.
+    const caseMeans = stats.caseSummaries.flatMap((c) => (c.mean === null ? [] : [c.mean]));
     return {
         name: config.name,
         cases: settled,
@@ -43,14 +65,35 @@ export async function runExperiment(config) {
         // Over the same cases the mean is over. Computing spread across a
         // different denominator than the average it qualifies would produce
         // an interval that does not belong to the number beside it.
-        uncertainty: uncertaintyOf(scored.map((r) => r.mean)),
+        uncertainty: uncertaintyOf(caseMeans),
         passed: settled.filter((r) => r.status === 'passed').length,
         failed: settled.filter((r) => r.status === 'failed').length,
         inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
         byScorer: meanByScorer(settled),
         durationMs: Date.now() - startedAt,
+        repeats,
+        seed,
+        ...(stats.passRate === null ? {} : { passRate: stats.passRate }),
+        intervals: stats.intervals,
+        ...(repeats > 1 ? { caseSummaries: stats.caseSummaries, reliability: stats.reliability } : {}),
+        splits: stats.splits,
+        disclosure: buildDisclosure(config.disclosure, { cases: config.cases.length, repeats, seed }, options, new Date().toISOString()),
     };
 }
+function validateRepeats(value) {
+    if (value === undefined)
+        return 1;
+    if (!Number.isInteger(value) || value < 1 || value > MAX_REPEATS) {
+        throw new EvalUsageError(`repeats must be an integer from 1 to ${MAX_REPEATS}, or omitted; received ${String(value)}`);
+    }
+    return value;
+}
+function validateSplit(evalCase) {
+    const split = evalCase.split;
+    if (split !== undefined && split !== 'development' && split !== 'held-out') {
+        throw new RangeError(`case "${evalCase.name}" declares split ${JSON.stringify(split)}; a split is "development" or "held-out", or left out`);
+    }
+}
 function validateCaseTimeoutMs(value) {
     if (value === undefined)
         return undefined;
@@ -263,17 +306,46 @@ function derivedUncertainty(report) {
     return uncertaintyOf(report.cases.filter((c) => c.status !== 'inconclusive').map((c) => c.mean));
 }
 export function formatReport(report) {
+    const repeats = report.repeats ?? 1;
+    const caseCount = repeats > 1 ? report.cases.length / repeats : report.cases.length;
     const lines = [
-        `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
-        // On its own line and always printed, including when the interval is
-        // undefined. A mean printed alone is the thing that has been
-        // over-read: two experiments three points apart look like a difference, and
-        // at the n a hand-built suite has they are usually the same experiment
-        // twice. Computing the interval and not showing it would leave the
-        // reader exactly where they started.
-        `  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
-        '',
+        repeats > 1
+            ? `${report.name}: ${report.passed}/${report.cases.length} runs passed (${caseCount} cases x ${repeats} repeats, mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`
+            : `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
     ];
+    if (report.intervals) {
+        // The interval sits next to the number it belongs to. A mean printed
+        // alone is the thing that has been over-read: two experiments three
+        // points apart look like a difference, and at the n a hand-built
+        // suite has they are usually the same experiment twice.
+        for (const line of describeIntervals({
+            mean: report.cases.some((c) => c.status !== 'inconclusive') ? report.mean : null,
+            passRate: report.passRate ?? null,
+            intervals: report.intervals,
+        })) {
+            lines.push(`  ${line}`);
+        }
+    }
+    else {
+        // A report from before bootstrap intervals, or one built by hand: the
+        // older interval, still never omitted.
+        lines.push(`  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`);
+    }
+    if (report.reliability && repeats > 1) {
+        const r = report.reliability;
+        lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
+        for (const c of report.caseSummaries ?? []) {
+            if (!c.passedEveryRepeat)
+                lines.push(`    ${c.case}: ${c.passed} of ${c.repeats} passed`);
+        }
+    }
+    if (report.splits) {
+        for (const line of describeSplits(report.splits))
+            lines.push(`  ${line}`);
+    }
+    if (report.disclosure)
+        lines.push(`  ${describeDisclosure(report.disclosure)}`);
+    lines.push('');
     // Surfaced above the failures, because an inconclusive case means the
     // harness is broken and every number below it is measured over less
     // evidence than it looks like.
@@ -287,7 +359,7 @@ export function formatReport(report) {
     if (failures.length > 0) {
         lines.push('', 'Failures:');
         for (const failure of failures) {
-            lines.push(`  ✗ ${failure.case} (${failure.mean.toFixed(2)})`);
+            lines.push(`  ✗ ${failure.case}${repeats > 1 && failure.repeat ? ` [repeat ${failure.repeat} of ${repeats}]` : ''} (${failure.mean.toFixed(2)})`);
             if (failure.failedGates && failure.failedGates.length > 0) {
                 // First line under the case, because it says WHY this failed.
                 // A mean of 0.75 next to four scores leaves the reader

package/dist/eval/experiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAmCzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAEzD,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC5D,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpC,IAAI,CAAC,QAAQ;gBAAE,OAAM;YAErB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE7F,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAClC,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,KAAK,GAAa;QACvB,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;QAC3H,qEAAqE;QACrE,6DAA6D;QAC7D,4EAA4E;QAC5E,uEAAuE;QACvE,mEAAmE;QACnE,qCAAqC;QACrC,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE;QACzF,EAAE;KACF,CAAA;IAED,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YAC9D,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AAWxD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,8DAA8D;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IAC3C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAE9E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAC7C,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/from-turn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"from-turn.d.ts","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAE1C;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAczF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACtC,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,GACtC,OAAO,CAAC,QAAQ,CAAC,CAKnB"}
+{"version":3,"file":"from-turn.d.ts","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAE1C;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAkBzF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACtC,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,GACtC,OAAO,CAAC,QAAQ,CAAC,CAKnB"}

package/dist/eval/from-turn.js

@@ -1,3 +1,4 @@
+import { compareAnswerToOutcome } from '../runtime/query/turn/answer-claims.js';
 /**
  * Project a finished `Turn` into the shape scorers consume.
  *
@@ -9,6 +10,7 @@
  */
 export function evalTurnFromTurn(turn, opts = {}) {
     const steps = turn.steps ?? [];
+    const claimMismatch = compareAnswerToOutcome(turn.result, turn.outcome);
     return {
         output: turn.result ?? null,
         ...(turn.structuredOutput !== undefined ? { structuredOutput: turn.structuredOutput } : {}),
@@ -20,6 +22,9 @@ export function evalTurnFromTurn(turn, opts = {}) {
         totalCostUsd: turn.costInfo.totalCost,
         durationMs: opts.durationMs ?? (turn.endedAt ?? Date.now()) - turn.startedAt,
         ...(turn.lastError ? { error: turn.lastError } : {}),
+        ...(turn.outcome ? { outcome: turn.outcome } : {}),
+        ...(claimMismatch ? { claimMismatch } : {}),
+        ...(turn.handedBack ? { handedBack: turn.handedBack } : {}),
     };
 }
 /**

package/dist/eval/from-turn.js.map

@@ -1 +1 @@
-{"version":3,"file":"from-turn.js","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,OAAgC,EAAE;IAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA;IAC9B,OAAO;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,GAAG,CAAC,IAAI,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,KAAK;QACL,yDAAyD;QACzD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,WAAW;QACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QACrC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS;QAC5E,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,SAAwC;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IACjC,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IAChD,OAAO,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;AAC5E,CAAC"}
+{"version":3,"file":"from-turn.js","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAA;AAI/E;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,OAAgC,EAAE;IAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA;IAC9B,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;IACvE,OAAO;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,GAAG,CAAC,IAAI,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,KAAK;QACL,yDAAyD;QACzD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,WAAW;QACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QACrC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS;QAC5E,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,SAAwC;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IACjC,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IAChD,OAAO,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;AAC5E,CAAC"}

package/dist/eval/index.d.ts

@@ -6,8 +6,19 @@ export { formatReport, runExperiment } from './experiment.js';
 export type { ExperimentConfig } from './experiment.js';
 export { judgeScorer } from './judge.js';
 export type { JudgeScorerConfig } from './judge.js';
-export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
+export type { BootstrapMethod, CaseIntervals, CaseTally, DifferenceIntervals, Interval, PairedTally, } from './bootstrap.js';
+export type { ReportDisclosure, RoleDisclosure, RoutingDisclosure, RunMode, SuiteFacts, } from './disclosure.js';
+export { sdkVersion } from './disclosure.js';
+export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export type { EvalRunOptions } from './run-options.js';
+export { formatRunSummary, summarizeRun } from './run-summary.js';
+export type { RunDisclosure, RunSummary } from './run-summary.js';
+export type { RepeatReliability, SplitSummary } from './report-stats.js';
+export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
+export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSuite, SideNumbers, } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 export type { ScoreUncertainty } from './uncertainty.js';
-export type { CaseResult, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';
+export type { CaseRepeatSummary, CaseResult, EvalSplit, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,UAAU,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}

package/dist/eval/index.js

@@ -2,6 +2,11 @@ export { evalTurnFromQuery, evalTurnFromTurn } from './from-turn.js';
 export { compareHarnessTrials, reviewHarnessCandidate } from './harness-verification.js';
 export { formatReport, runExperiment } from './experiment.js';
 export { judgeScorer } from './judge.js';
-export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
+export { sdkVersion } from './disclosure.js';
+export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { formatRunSummary, summarizeRun } from './run-summary.js';
+export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 //# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAa3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/eval/report-stats.d.ts

@@ -0,0 +1,77 @@
+import { type CaseIntervals, type CaseTally } from './bootstrap.js';
+import type { CaseRepeatSummary, CaseResult, EvalSplit } from './types.js';
+/**
+ * The numbers a report carries about a set of runs, whether that set is one
+ * suite, one split of it, or every suite in a run.
+ *
+ * Cases are the unit. A case with several repeats is one case with several
+ * runs: the pass rate pools the runs, the intervals resample the cases, and
+ * reliability asks of each case whether it held up every time.
+ */
+export interface RepeatReliability {
+    /** Cases that ran. */
+    cases: number;
+    /** Cases whose every repeat passed. */
+    passedEveryRepeat: number;
+    /** `passedEveryRepeat / cases`; null when no case ran. */
+    fraction: number | null;
+}
+export interface SplitSummary {
+    cases: number;
+    runs: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    passRate: number | null;
+    mean: number | null;
+    intervals: CaseIntervals;
+}
+/** The runs of one case, with the key that pairs it with the same case elsewhere. */
+export interface CaseRuns {
+    readonly key: string;
+    readonly name: string;
+    readonly split?: EvalSplit;
+    readonly runs: readonly CaseResult[];
+}
+export interface RunStats {
+    cases: number;
+    runs: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    passRate: number | null;
+    mean: number | null;
+    intervals: CaseIntervals;
+    caseSummaries: CaseRepeatSummary[];
+    reliability: RepeatReliability;
+    splits: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>;
+}
+export declare const SPLIT_ORDER: readonly ["development", "held-out", "unassigned"];
+/** Group a flat list of runs by case name, in the order the cases first appear. */
+export declare function groupRuns(results: readonly CaseResult[], keyOf?: (name: string) => string): CaseRuns[];
+/** The first name that appears more than once, or undefined. */
+export declare function duplicateName(names: readonly string[]): string | undefined;
+export declare function duplicateMessage(suite: string, name: string): string;
+/**
+ * The first case in a list of runs that appears twice for the same repeat: a
+ * second run of a case is a repeat only when it says it is a later one.
+ */
+export declare function duplicateRun(runs: readonly Pick<CaseResult, 'case' | 'repeat'>[]): string | undefined;
+/** Status of a run, tolerating a report old enough to carry only `passed`. */
+export declare function statusOf(run: Pick<CaseResult, 'status' | 'passed'>): CaseResult['status'];
+export declare function tallyOf(runs: readonly CaseResult[]): CaseTally | null;
+/** True when the case has at least one judged run and every run passed. */
+export declare function passedEveryRun(runs: readonly CaseResult[]): boolean;
+export declare function statsOver(cases: readonly CaseRuns[], seed: number): RunStats;
+/**
+ * The intervals, as lines a person reads next to the numbers. A number whose
+ * interval is not computed is printed with the reason, never alone.
+ */
+export declare function describeIntervals(args: {
+    mean: number | null;
+    passRate: number | null;
+    intervals: CaseIntervals;
+}): string[];
+/** The line that says which splits a suite has, and says so plainly when none is held out. */
+export declare function describeSplits(splits: RunStats['splits']): string[];
+//# sourceMappingURL=report-stats.d.ts.map