@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAkB,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EACL,SAAS,GAGV,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAQ7D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC;AAEzC,OAAO,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;GAMG;AAEH,iFAAiF;AACjF,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACxC,yEAAyE;AACzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC;AACnC,2DAA2D;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,yEAAyE;AACzE,MAAM,CAAC,MAAM,cAAc,GACzB,wGAAwG,CAAC;AA+E3G,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAClC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE9D;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,CAO9B;IACC,OAAO,GAAG,CACR,UAAU,CAAC;QACT,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;KACjB,CAAC,EACF,EAAE,CACH,CAAC;AACJ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,cAAc,CAAC,KAAmB;IAChD,OAAO,GAAG,CACR,KAAK;SACF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC1E,CAAC,CAAC;SACD,IAAI,EAAE;SACN,IAAI,CAAC,GAAG,CAAC,CACb,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,WAAW,CAAC,MAAyB;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM;QACpB,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACpD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACjE,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAU,EACV,KAAa,EACb,EAA2B;IAE3B,MAAM,OAAO,GAAQ,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,UAAU,MAAM;QACnB,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAC9D,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,YAAY,CAAC,IAAmB;IAC9C,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAY,EAAE;QAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IACF,MAAM,YAAY,GAAG,CAAC,KAAa,EAAE,EAAE,CACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,IAAI,cAAc,CAAC;IAEjD;;;;OAIG;IACH,KAAK,UAAU,QAAQ,CAAC,KAAa,EAAE,GAAY;QACjD,MAAM,MAAM,GAAG,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACzD,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CACzB,CAAC;QACF,IAAI,GAAG,KAAK,KAAK;YAAE,OAAO,MAAM,CAAC;QACjC,IAAI,GAAG,KAAK,OAAO;YAAE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,8EAA8E;IAC9E,KAAK,UAAU,aAAa,CAC1B,SAAiB;QAEjB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,KAAK,UAAU,WAAW,CACxB,QAAgB,EAChB,KAAa,EACb,GAAY,EACZ,OAAe,EACf,IAKI,EAAE;QASN,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzD,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;YACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI;YACvD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC;YACnC,CAAC,CAAC,eAAe,KAAK,SAAS;gBAC7B,CAAC,CAAC,CAAC,CAAC,eAAe;gBACnB,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC;SACnC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,UAAU,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK;YACpB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,CACE,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAClB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvC,KAAK,CAAC,kBAAkB,CACtB,cAAc,CAAC;gBACb,KAAK,EAAE,KAAK,CAAC,GAAG;gBAChB,KAAK;gBACL,MAAM;gBACN,MAAM;gBACN,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,GAAG,CAAC;aACd,CAAC,EACF,KAAK,CACN,CACF,CACF,CACF,CACF,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,OAAO;YACP,MAAM;YACN,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC;SAChE,CAAC;IACJ,CAAC;IAED,sDAAsD;IACtD,KAAK,UAAU,SAAS,CAAC,IAWxB;QACC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QAClC,MAAM,eAAe,GACnB,IAAI,CAAC,eAAe,KAAK,SAAS;YAChC,CAAC,CAAC,IAAI,CAAC,eAAe;YACtB,CAAC,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClD,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE;gBAC/C,eAAe;gBACf,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;aAC3B,CAAC;YACF,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC;SACpC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,eAAe,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;YAChC,KAAK,EAAE,KAAK,CAAC,GAAG;YAChB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa;YACb,eAAe,EACb,IAAI,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU;YACxE,GAAG;YACH,OAAO;YACP,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,gBAAgB,EAAE,QAAQ,CAAC,OAAO;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;SAClC,CAAC,CAAC;QACH,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;IAC3B,CAAC;IAED,iEAAiE;IACjE,KAAK,UAAU,WAAW,CACxB,KAAe,EACf,CAAa,EACb,MAAe,EACf,MAA8B;QAE9B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,OAAO,KAAK,CAAC,KAAK,CAChB,KAAK,EACL,MAAM,EACN,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EACpC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAChC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,UAAU,aAAa,CAAC,GAAc,EAAE,KAAe;QAC1D,IAAI,CAAC,KAAK,CAAC,aAAa;YAAE,OAAO;QACjC,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CACzD,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CACtD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CACvC,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,OAAO;YACrB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBACvC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACtE,OAAO;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;oBACjC,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;iBAC5C,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,QAAQ,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EACjD,CAAC,EACD,KAAK,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC5C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC9D,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,UAAU,UAAU,CACvB,GAAc,EACd,KAAe,EACf,YAAoB;QAEpB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,MAAM,MAAM,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACjD,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;YACzC,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC,CAAC,CAAC;QACJ,OAAO;YACL,GAAG,YAAY,CACb,MAAM,EACN,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAC/B;YACD,YAAY;YACZ,aAAa,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;SACxD,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,OAAO,CAAC,KAAa,EAAE,IAAoB,EAAE;QAC1D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,6DAA6D;QAC7D,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO;QAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,UAAU,CAAC,CAAC;YAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACtE,CAAC;YACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CACvE,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAExE,IAAI,SAAS,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;YACpC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;YACpD,+DAA+D;YAC/D,IAAI,YAAY,GAAG,IAAqB,CAAC;YACzC,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,IAAI,iBAAiB,CAAC;YAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc;gBAClC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,GAAG,OAAO;gBAC5C,CAAC,CAAC,QAAQ,CAAC;YACb,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;gBACnE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,YAAY,EAAE,CAAC;oBAC1C,OAAO,EAAE,CAAC;oBACV,OAAO;gBACT,CAAC;gBACD,MAAM,WAAW,GAAG,cAAc,CAAC;oBACjC,KAAK,EAAE,KAAK,CAAC,GAAG;oBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,GAAG,CAAC,eAAe;oBAC3B,MAAM,EAAE,GAAG,CAAC,aAAa;oBACzB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM;iBACP,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK;oBACrB,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,MAAM,KAAK,CAAC,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;gBACvD,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,KAAK,CAAC,YAAY,CAAC;wBACvB,KAAK;wBACL,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,MAAM;wBACN,WAAW;wBACX,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;wBAC5D,8DAA8D;wBAC9D,6DAA6D;wBAC7D,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,EAAE,EAAE,OAAO,CAAC,EAAE;wBACd,MAAM,EAAE,IAAI;qBACb,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;gBACb,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;wBACpC,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,eAAe,EACZ,GAAG,CAAC,eAA0C,IAAI,IAAI;wBACzD,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;qBAC/B,CAAC,EACF,OAAO,CACR,CAAC;oBACF,MAAM,KAAK,CAAC,YAAY,CAAC;wBACvB,KAAK;wBACL,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,MAAM;wBACN,WAAW;wBACX,MAAM;wBACN,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC;wBACpD,OAAO,EAAE,IAAI;wBACb,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;wBACxB,MAAM,EAAE,KAAK;qBACd,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC3D,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC;wBAAE,YAAY,GAAG,OAAO,CAAC;oBACzD,MAAM,KAAK,CAAC,YAAY,CAAC;wBACvB,KAAK;wBACL,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,MAAM;wBACN,WAAW;wBACX,MAAM,EAAE,IAAI;wBACZ,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;wBAC7B,MAAM,EAAE,IAAI;wBACZ,OAAO,EAAE,IAAI;wBACb,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;wBACxB,MAAM,EAAE,KAAK;qBACd,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG;oBACX,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK;oBAChB,GAAG,EAAE,GAAG,GAAG,CAAC;oBACZ,YAAY,EAAE,SAAS;iBACxB,CAAC;gBACF,IACE,cAAc,CAAC;oBACb,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,IAAI,CAAC,MAAM,GAAG,OAAO;oBAC3B,GAAG;oBACH,YAAY,EAAE,CAAC,CAAC,YAAY;iBAC7B,CAAC;oBACF,IAAI,CAAC,WAAW;oBAChB,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;oBAExD,OAAO;gBACT,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE;oBAC3B,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE;oBACtB,KAAK,EAAE,YAAY;wBACjB,CAAC,CAAC,OAAO,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,yDAAyD,IAAI,CAAC,MAAM,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,sFAAsF;wBACtN,CAAC,CAAC,yBAAyB,IAAI,CAAC,MAAM,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,8EAA8E;iBACnJ,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE;gBAC3B,MAAM,EAAE,WAAW;gBACnB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,cAAc,EAAE,cAAc,CAAC,KAAK,CAAC;gBACrC,OAAO,EAAE,MAAM,UAAU,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;aACjD,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE;gBAC3B,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;aAClD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,UAAU,CAAC,KAAa;QACrC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CACzD,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CACrD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CACvC,CAAC;QACF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;QACpD,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CACrC,KAAK,CAAC,eAAe,CACnB,CAAC,CAAC,EAAE,EACJ,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CACtE,CACF,CAAC;QACF,MAAM,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CACjE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CACnB,CAAC;QACF,MAAM,eAAe,GAAG,OAAO,CAAC,IAAI,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CACxD,CAAC;QACF,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE;YAC3B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACpE,OAAO,EAAE,MAAM,UAAU,CACvB,GAAG,EACH,KAAK,EACL,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,SAAS,CAC7C;SACF,CAAC,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,KAAK,GAAG,EAAE;QACvD,KAAK,MAAM,CAAC,IAAI,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;YAC7C,MAAM,EAAE,WAAW;YACnB,KAAK;SACN,CAAC;YACA,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,UAAU,eAAe,CAAC,QAAgB;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,SAAS,CAAC;YAC9B,KAAK,EAAE,KAAK,CAAC,GAAG;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,GAAG,EAAE,OAAO;YACZ,SAAS,EAAE,SAAS;YACpB,eAAe,EAAE,KAAK,CAAC,eAAe;SACvC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,EAAE,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,aAAa,CAAC,QAAgB;QAC3C,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,EAAE;YAAE,MAAM,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,wDAAwD;IACxD,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,QAAQ;gBAAE,SAAS;YAC7C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CACtB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACnD,IAAI,CACL,CAAC;YACF,IAAI,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC;gBAC3C,MAAM,KAAK,CAAC,SAAS,CACnB,CAAC,CAAC,EAAE,EACJ;oBACE,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC;oBACzB,KAAK,EACH,+FAA+F;iBAClG,EACD,EAAE,MAAM,EAAE,SAAS,EAAE,CACtB,CAAC;QACN,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,UAAU,OAAO,CAAC,QAAgB,EAAE,CAAS,EAAE,CAAS;QAC3D,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,IACE,CAAC,EAAE;YACH,CAAC,EAAE;YACH,CAAC,KAAK,CAAC;YACP,EAAE,CAAC,KAAK,KAAK,KAAK,CAAC,GAAG;YACtB,EAAE,CAAC,KAAK,KAAK,KAAK,CAAC,GAAG;YACtB,EAAE,CAAC,MAAM,KAAK,WAAW;YACzB,EAAE,CAAC,MAAM,KAAK,WAAW;YAEzB,OAAO,IAAI,CAAC;QACd,MAAM,SAAS,GAAG,KAAK,EAAE,CAAY,EAAwB,EAAE,CAC7D,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;YACzC,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC,CAAC,CAAC;QACN,MAAM,UAAU,GAAG,WAAW,CAC5B,MAAM,SAAS,CAAC,EAAE,CAAC,EACnB,MAAM,SAAS,CAAC,EAAE,CAAC,EACnB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAC/B,CAAC;QACF,8CAA8C;QAC9C,MAAM,OAAO,GAAG;YACd,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,mBAAmB,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,GAAG,GAAG;YAC/D,EAAE,CAAC,cAAc,KAAK,EAAE,CAAC,cAAc;gBACrC,yFAAyF;YAC3F,EAAE,CAAC,aAAa,KAAK,EAAE,CAAC,aAAa;gBACnC,2DAA2D;YAC7D,EAAE,CAAC,aAAa,KAAK,EAAE,CAAC,aAAa;gBACnC,2BAA2B,EAAE,CAAC,aAAa,QAAQ,EAAE,CAAC,aAAa,GAAG;SACzE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,KAAK,CAAC,QAAgB;QACnC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU;YACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;YAC3B,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC;SAC3B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CACzD,KAAK,CAAC,SAAS,CAChB,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CACnD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAC/C,CAAC;QACF,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;QACpD,uEAAuE;QACvE,8CAA8C;QAC9C,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,MAAqB,EAAE,EAAE,CAC1D,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,MAAgC,CAAC,IAAI,IAAI,CAAC;QAC1E,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,MAAqB,EAAE,EAAE,CACrD,GAAG,KAAK,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,CAAY,EAAE,EAAE,CACjC,CAAC,CAAC,cAAc,KAAK,OAAO,IAAI,CAAC,CAAC,aAAa,KAAK,MAAM,CAAC;QAE7D,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,uEAAuE;QACvE,qDAAqD;QACrD,MAAM,UAAU,GAAG,CAAC,CAAY,EAAE,EAAE,CAClC,OAAO,KAAK,SAAS;YACrB,CAAC,CAAC,cAAc,KAAK,OAAO,CAAC,cAAc;YAC3C,CAAC,CAAC,aAAa,KAAK,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC/C,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE;YACjC,OAAO,CAAC,GAAG,CACT,CAAC,CAAC,EAAE,EACJ,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1C,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;gBACzC,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC,CAAC,CACJ,CAAC;QACJ,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,IAAI,GAAG;YACX,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpB,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YACH,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5B,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YACH,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,CAAC,eAAe;aAC1B,CAAC,CAAC;SACJ,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,GAAG,CACN,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAClB;YACE,KAAK;YACL,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,IAAI,IAAI;SACvD,CACb,CACF,CACF,CAAC;QAEF,MAAM,IAAI,GAAe,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACjC,MAAM,OAAO,GACX,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACjE,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,IAAI,SAAS,CAAC;YAClC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,wEAAwE;YACxE,MAAM,MAAM,GACV,CAAC;gBACD,OAAO;gBACP,QAAQ,CAAC,CAAC,CAAC;gBACX,QAAQ,CAAC,OAAO,CAAC;gBACjB,UAAU,CAAC,CAAC,CAAC;gBACb,GAAG,KAAK,OAAO;gBACb,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC;qBAC3D,OAAO;gBACZ,CAAC,CAAC,IAAI,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK,CAAC;YACjD,OAAO;gBACL,GAAG;gBACH,KAAK;gBACL,MAAM;gBACN,OAAO;gBACP,KAAK,EAAE,OAAO;oBACZ,CAAC,CAAC,GAAG,KAAK,MAAM,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,YAAY;oBAClE,CAAC,CAAC,KAAK;gBACT,IAAI,EAAE,GAAG,KAAK,OAAO;gBACrB,QAAQ,EAAE,CAAC,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,aAAa;gBACnD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;gBACzB,MAAM,EACJ,OAAO,EAAE,MAAM,KAAK,QAAQ;oBAC1B,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBAClD,CAAC,CAAC,IAAI;gBACV,gBAAgB,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI;gBAC9C,IAAI,EACF,CAAC,IAAI,CAAC;oBACJ,CAAC,CAAC;wBACE,KAAK,EAAE,CAAC,CAAC,EAAE;wBACX,EAAE,EAAE,CAAC,CAAC,SAAS;wBACf,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM;wBACzC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;wBACpC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM;wBAC9B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;wBACpB,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE;wBAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI;wBACtB,aAAa,EAAE,CAAC,CAAC,aAAa,IAAI,IAAI;wBACtC,KAAK,EAAE,CAAC,KAAK;wBACb,aAAa,EAAE,CAAC,CAAC,MAAM;wBACvB,aAAa,EAAE,CAAC,CAAC,aAAa,IAAI,IAAI;qBACvC;oBACH,CAAC,CAAC,IAAI;gBACV,MAAM;gBACN,gEAAgE;gBAChE,WAAW,EACT,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;oBACtB,KAAK;oBACL,QAAQ,CAAC,CAAC,CAAC;oBACX,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,MAAM,CAAC;gBAC5C,QAAQ,EAAE,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACzC,YAAY,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACrC,eAAe,EACb,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC9B,CAAC,OAAO;oBACR,KAAK,KAAK,KAAK,CAAC,KAAK;oBACrB,KAAK,KAAK,KAAK,CAAC,aAAa;oBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,QAAQ;oBACnE,kEAAkE;oBAClE,2CAA2C;oBAC3C,KAAK;oBACL,QAAQ,CAAC,CAAC,CAAC;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,sEAAsE;QACtE,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/B,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YACvC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACjC,CAAC;QACF,4EAA4E;QAC5E,MAAM,WAAW,GAAuD,EAAE,CAAC;QAC3E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa;gBACtE,SAAS;YACX,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC/D,CAAC,CAAC,MAAM,EAAE,CAAC;gBACX,IAAI,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;oBAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACtC,CAAC;QACH,CAAC;QACD,OAAO;YACL,KAAK;YACL,KAAK;YACL,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,WAAW;YACX,yDAAyD;YACzD,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChE,IAAI;YACJ,SAAS,EAAE,OAAO,EAAE,EAAE,IAAI,IAAI;SAC/B,CAAC;IACJ,CAAC;IAED,OAAO;QACL,eAAe;QACf,aAAa;QACb,KAAK;QACL,YAAY;QACZ,QAAQ;QACR,WAAW;QACX,SAAS;QACT,OAAO;QACP,WAAW;QACX,UAAU;QACV,aAAa;QACb,aAAa;QACb,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/settings.d.ts

@@ -0,0 +1,157 @@
+import type { ReasoningEffort } from "./models.js";
+import type { Route } from "./runner.js";
+/**
+ * Versioned model settings. Each AI feature's settings (its model,
+ * fallback, reasoning setting, and output limit, with the instructions they
+ * ran with) are numbered versions that never change. Changing a setting
+ * makes a new version live; rolling back makes an earlier one live again;
+ * every switch is logged with who, when, why, and its evidence. A feature
+ * never changed runs on its default from code. Optional: use it, or keep
+ * your own routing and pass `route` to the runner yourself.
+ */
+/**
+ * A feature's model settings: live model, fallback, reasoning setting, and output limit.
+ */
+export type Settings = {
+    model: string;
+    fallbackModel: string | null;
+    /** Null: the model's own default. */
+    reasoningEffort: ReasoningEffort | null;
+    /** Null: the call's own default. */
+    maxOutputTokens: number | null;
+};
+/**
+ * One numbered version of a feature's settings, with the instructions it ran with. Never changed.
+ */
+export type SettingsVersion = Settings & {
+    id: string;
+    operation: string;
+    /** 1, 2, 3, … per feature. */
+    number: number;
+    /** The instructions' hash when this version was made. */
+    promptVersion: string | null;
+    createdAt: Date;
+};
+/**
+ * CHANGE: a person changed a setting. PROMOTE: a model was made live or
+ * the fallback from a test, with the test as evidence. ROLLBACK: an
+ * earlier version made live again. DEPLOY: the instructions changed in code
+ * under the live settings.
+ */
+export type ChangeKind = "CHANGE" | "PROMOTE" | "ROLLBACK" | "DEPLOY";
+/**
+ * One switch in a feature's history: its kind, the versions it went from and to, who, why,
+ * and the evidence.
+ */
+export type SettingsChange = {
+    id: string;
+    operation: string;
+    kind: ChangeKind;
+    fromVersionId: string | null;
+    toVersionId: string;
+    reason: string | null;
+    /** Who made it; null for the system (a deploy, the first default). */
+    by: string | null;
+    /** Why it's safe: the tests compared, the verdict. */
+    evidence: unknown;
+    createdAt: Date;
+};
+/** One atomic switch: maybe a new version, then live, then the log. */
+export type SettingsSwitch = {
+    /** Make a new version with these settings (numbered next). */
+    create?: {
+        settings: Settings;
+        promptVersion: string | null;
+    };
+    /** Or make this existing version live. */
+    versionId?: string;
+    kind: ChangeKind;
+    reason: string | null;
+    by: string | null;
+    evidence?: unknown;
+    /**
+     * The live version this switch was decided against (null: none). The
+     * store refuses the switch if another one got in first.
+     */
+    expectLive: string | null;
+};
+/** Where versions live. `@crowandraven/evalkeeper-prisma` and `memoryStore` implement it. */
+export interface SettingsStore {
+    liveVersion(operation: string): Promise<SettingsVersion | null>;
+    /** Newest first. */
+    listVersions(operation: string): Promise<SettingsVersion[]>;
+    /** Newest first. */
+    listChanges(operation: string, opts?: {
+        limit?: number;
+    }): Promise<SettingsChange[]>;
+    /**
+     * Applies a switch atomically. Throws SettingsConflict when the live
+     * version is no longer `expectLive`.
+     */
+    switchTo(operation: string, s: SettingsSwitch): Promise<SettingsVersion>;
+}
+/** Another change landed first: reload and decide again. */
+export declare class SettingsConflict extends Error {
+    constructor(operation: string);
+}
+/** Whether two settings are the same in every field. */
+export declare function sameSettings(a: Settings, b: Settings): boolean;
+/** Just the settings of a version. */
+export declare function settingsOf(v: Settings): Settings;
+/** What changed between two settings, in plain words. */
+export declare function describeChange(from: Settings | null, to: Settings, label?: (model: string) => string): string[];
+/** A switch with the versions it went from and to, for a history page. */
+export type HistoryEntry = SettingsChange & {
+    to: SettingsVersion;
+    from: SettingsVersion | null;
+};
+/**
+ * Versioned settings: each feature's settings as numbered versions, with every change,
+ * promotion, rollback, and deploy logged. Its `route` is the runner's route.
+ *
+ * @example
+ * ```ts
+ * const settings = createSettings({
+ *   store: prismaSettingsStore(prisma),
+ *   defaults: (op) => DEFAULTS[op],
+ *   promptVersion: (op) => PROMPT_VERSIONS[op],
+ * });
+ * await settings.update("ticket_triage", { model: "claude-haiku-5-5" }, {
+ *   by: admin.name,
+ *   reason: "About as accurate at half the cost",
+ *   evidence: { runId },
+ * });
+ * await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
+ * ```
+ */
+export declare function createSettings(opts: {
+    store: SettingsStore;
+    /** Each feature's settings in code, used until it's first changed. */
+    defaults: (operation: string) => Settings;
+    /** The instructions' current hash, so a version records what it ran. */
+    promptVersion?: (operation: string) => string | Promise<string>;
+}): {
+    current: (operation: string) => Promise<{
+        settings: Settings;
+        version: SettingsVersion | null;
+    }>;
+    route: (operation: string) => Promise<Route>;
+    change: (operation: string, next: Settings, by: {
+        by: string | null;
+        reason: string | null;
+        evidence?: unknown;
+    }) => Promise<SettingsVersion | null>;
+    update: (operation: string, patch: Partial<Settings>, by: {
+        by: string | null;
+        reason: string | null;
+        evidence?: unknown;
+    }) => Promise<SettingsVersion | null>;
+    rollBack: (operation: string, versionId: string, by: {
+        by: string | null;
+        reason: string;
+    }) => Promise<SettingsVersion>;
+    syncPrompt: (operation: string) => Promise<SettingsVersion | null>;
+    history: (operation: string, limit?: number) => Promise<HistoryEntry[]>;
+};
+/** What createSettings returns. */
+export type VersionedSettings = ReturnType<typeof createSettings>;

package/dist/settings.js

@@ -0,0 +1,183 @@
+/** Another change landed first: reload and decide again. */
+export class SettingsConflict extends Error {
+    constructor(operation) {
+        super(`The settings for ${operation} changed meanwhile. Reload and try again.`);
+        this.name = "SettingsConflict";
+    }
+}
+/** Whether two settings are the same in every field. */
+export function sameSettings(a, b) {
+    return (a.model === b.model &&
+        a.fallbackModel === b.fallbackModel &&
+        a.reasoningEffort === b.reasoningEffort &&
+        a.maxOutputTokens === b.maxOutputTokens);
+}
+/** Just the settings of a version. */
+export function settingsOf(v) {
+    return {
+        model: v.model,
+        fallbackModel: v.fallbackModel,
+        reasoningEffort: v.reasoningEffort,
+        maxOutputTokens: v.maxOutputTokens,
+    };
+}
+/** What changed between two settings, in plain words. */
+export function describeChange(from, to, label = (m) => m) {
+    if (!from)
+        return [`Live: ${label(to.model)}`];
+    const out = [];
+    if (from.model !== to.model)
+        out.push(`Live: ${label(from.model)} → ${label(to.model)}`);
+    if (from.fallbackModel !== to.fallbackModel)
+        out.push(`Fallback: ${from.fallbackModel ? label(from.fallbackModel) : "none"} → ${to.fallbackModel ? label(to.fallbackModel) : "none"}`);
+    if (from.reasoningEffort !== to.reasoningEffort)
+        out.push(`Reasoning: ${from.reasoningEffort ?? "default"} → ${to.reasoningEffort ?? "default"}`);
+    if (from.maxOutputTokens !== to.maxOutputTokens)
+        out.push(`Output limit: ${from.maxOutputTokens ?? "default"} → ${to.maxOutputTokens ?? "default"}`);
+    return out;
+}
+/**
+ * Versioned settings: each feature's settings as numbered versions, with every change,
+ * promotion, rollback, and deploy logged. Its `route` is the runner's route.
+ *
+ * @example
+ * ```ts
+ * const settings = createSettings({
+ *   store: prismaSettingsStore(prisma),
+ *   defaults: (op) => DEFAULTS[op],
+ *   promptVersion: (op) => PROMPT_VERSIONS[op],
+ * });
+ * await settings.update("ticket_triage", { model: "claude-haiku-5-5" }, {
+ *   by: admin.name,
+ *   reason: "About as accurate at half the cost",
+ *   evidence: { runId },
+ * });
+ * await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
+ * ```
+ */
+export function createSettings(opts) {
+    const { store } = opts;
+    const prompt = async (op) => opts.promptVersion ? await opts.promptVersion(op) : null;
+    /** The live settings: the live version's, else the default from code. */
+    async function current(operation) {
+        const version = await store.liveVersion(operation);
+        return {
+            settings: version ? settingsOf(version) : opts.defaults(operation),
+            version,
+        };
+    }
+    /** The runner's route for a feature, from its live settings. */
+    async function route(operation) {
+        const { settings } = await current(operation);
+        return {
+            model: settings.model,
+            fallbackModel: settings.fallbackModel,
+            reasoningEffort: settings.reasoningEffort,
+        };
+    }
+    /**
+     * Makes new settings live as a new version. The first change to a
+     * feature first records its default as version 1, so there's always
+     * something to roll back to. With `evidence`, it's a promotion. Returns
+     * null when nothing changed.
+     */
+    async function change(operation, next, by) {
+        const promptVersion = await prompt(operation);
+        let live = await store.liveVersion(operation);
+        if (!live) {
+            const initial = opts.defaults(operation);
+            if (sameSettings(initial, next))
+                return null;
+            live = await store.switchTo(operation, {
+                create: { settings: initial, promptVersion },
+                kind: "CHANGE",
+                reason: "The default from code, before the first change",
+                by: null,
+                expectLive: null,
+            });
+        }
+        else if (sameSettings(settingsOf(live), next))
+            return null;
+        return store.switchTo(operation, {
+            create: { settings: settingsOf(next), promptVersion },
+            kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
+            reason: by.reason,
+            by: by.by,
+            evidence: by.evidence,
+            expectLive: live.id,
+        });
+    }
+    /** Changes only some settings (e.g. the fallback), keeping the rest. */
+    async function update(operation, patch, by) {
+        const { settings } = await current(operation);
+        return change(operation, { ...settings, ...patch }, by);
+    }
+    /** Makes an earlier version live again. */
+    async function rollBack(operation, versionId, by) {
+        const versions = await store.listVersions(operation);
+        const to = versions.find((v) => v.id === versionId);
+        if (!to)
+            throw new Error("No such version for this feature");
+        const live = await store.liveVersion(operation);
+        if (live?.id === to.id)
+            return to;
+        return store.switchTo(operation, {
+            versionId: to.id,
+            kind: "ROLLBACK",
+            reason: by.reason,
+            by: by.by,
+            expectLive: live?.id ?? null,
+        });
+    }
+    /**
+     * Instructions live in code, so a deploy can change them under live
+     * settings. When it has, records a version with the same settings and the
+     * new instructions, so the history says what changed and when. Call it
+     * where the feature's settings are read (or after each deploy).
+     */
+    async function syncPrompt(operation) {
+        const live = await store.liveVersion(operation);
+        const now = await prompt(operation);
+        if (!live || !now || now === live.promptVersion)
+            return live;
+        try {
+            return await store.switchTo(operation, {
+                create: { settings: settingsOf(live), promptVersion: now },
+                kind: "DEPLOY",
+                reason: live.promptVersion
+                    ? `Instructions changed (${live.promptVersion} → ${now})`
+                    : `Instructions recorded (${now})`,
+                by: null,
+                expectLive: live.id,
+            });
+        }
+        catch (e) {
+            // Another request recorded the same deploy first.
+            if (e instanceof SettingsConflict)
+                return store.liveVersion(operation);
+            throw e;
+        }
+    }
+    /** Every switch, newest first, with the versions it went from and to. */
+    async function history(operation, limit = 50) {
+        const [changes, versions] = await Promise.all([
+            store.listChanges(operation, { limit }),
+            store.listVersions(operation),
+        ]);
+        const byId = new Map(versions.map((v) => [v.id, v]));
+        return changes.flatMap((c) => {
+            const to = byId.get(c.toVersionId);
+            if (!to)
+                return [];
+            return [
+                {
+                    ...c,
+                    to,
+                    from: c.fromVersionId ? (byId.get(c.fromVersionId) ?? null) : null,
+                },
+            ];
+        });
+    }
+    return { current, route, change, update, rollBack, syncPrompt, history };
+}
+//# sourceMappingURL=settings.js.map

package/dist/settings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAkGA,4DAA4D;AAC5D,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,SAAiB;QAC3B,KAAK,CACH,oBAAoB,SAAS,2CAA2C,CACzE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,UAAU,YAAY,CAAC,CAAW,EAAE,CAAW;IACnD,OAAO,CACL,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;QACnB,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;QACnC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe;QACvC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe,CACxC,CAAC;AACJ,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,UAAU,CAAC,CAAW;IACpC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC;AACJ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,cAAc,CAC5B,IAAqB,EACrB,EAAY,EACZ,QAAmC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK;QACzB,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC,aAAa;QACzC,GAAG,CAAC,IAAI,CACN,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAChI,CAAC;IACJ,IAAI,IAAI,CAAC,eAAe,KAAK,EAAE,CAAC,eAAe;QAC7C,GAAG,CAAC,IAAI,CACN,cAAc,IAAI,CAAC,eAAe,IAAI,SAAS,MAAM,EAAE,CAAC,eAAe,IAAI,SAAS,EAAE,CACvF,CAAC;IACJ,IAAI,IAAI,CAAC,eAAe,KAAK,EAAE,CAAC,eAAe;QAC7C,GAAG,CAAC,IAAI,CACN,iBAAiB,IAAI,CAAC,eAAe,IAAI,SAAS,MAAM,EAAE,CAAC,eAAe,IAAI,SAAS,EAAE,CAC1F,CAAC;IACJ,OAAO,GAAG,CAAC;AACb,CAAC;AAQD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,cAAc,CAAC,IAM9B;IACC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACvB,MAAM,MAAM,GAAG,KAAK,EAAE,EAAU,EAAE,EAAE,CAClC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE3D,yEAAyE;IACzE,KAAK,UAAU,OAAO,CACpB,SAAiB;QAEjB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACnD,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAClE,OAAO;SACR,CAAC;IACJ,CAAC;IAED,gEAAgE;IAChE,KAAK,UAAU,KAAK,CAAC,SAAiB;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,eAAe,EAAE,QAAQ,CAAC,eAAe;SAC1C,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,UAAU,MAAM,CACnB,SAAiB,EACjB,IAAc,EACd,EAAoE;QAEpE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzC,IAAI,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC7C,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;gBACrC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC5C,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,gDAAgD;gBACxD,EAAE,EAAE,IAAI;gBACR,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7D,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC/B,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE;YACrD,IAAI,EAAE,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACtD,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,UAAU,EAAE,IAAI,CAAC,EAAE;SACpB,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,KAAK,UAAU,MAAM,CACnB,SAAiB,EACjB,KAAwB,EACxB,EAAoE;QAEpE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,2CAA2C;IAC3C,KAAK,UAAU,QAAQ,CACrB,SAAiB,EACjB,SAAiB,EACjB,EAAyC;QAEzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC/B,SAAS,EAAE,EAAE,CAAC,EAAE;YAChB,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,UAAU,EAAE,IAAI,EAAE,EAAE,IAAI,IAAI;SAC7B,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,UAAU,UAAU,CACvB,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAC7D,IAAI,CAAC;YACH,OAAO,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;gBACrC,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE;gBAC1D,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI,CAAC,aAAa;oBACxB,CAAC,CAAC,yBAAyB,IAAI,CAAC,aAAa,MAAM,GAAG,GAAG;oBACzD,CAAC,CAAC,0BAA0B,GAAG,GAAG;gBACpC,EAAE,EAAE,IAAI;gBACR,UAAU,EAAE,IAAI,CAAC,EAAE;aACpB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,CAAC,YAAY,gBAAgB;gBAAE,OAAO,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,KAAK,UAAU,OAAO,CACpB,SAAiB,EACjB,KAAK,GAAG,EAAE;QAEV,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC5C,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC;YACvC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC;SAC9B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE;gBAAE,OAAO,EAAE,CAAC;YACnB,OAAO;gBACL;oBACE,GAAG,CAAC;oBACJ,EAAE;oBACF,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;iBACnE;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AAC3E,CAAC"}

package/dist/stableJson.d.ts

@@ -0,0 +1,11 @@
+/**
+ * JSON with object keys sorted, so a case's input compares equal however
+ * its keys were ordered (Postgres jsonb reorders them on save).
+ */
+export declare function stableJson(value: unknown): string;
+/** Drafted cases not yet in the answer key. */
+export declare function newStarters<T extends {
+    input: unknown;
+}>(starter: T[] | undefined, existing: {
+    input: unknown;
+}[]): T[];

package/dist/stableJson.js

@@ -0,0 +1,15 @@
+/**
+ * JSON with object keys sorted, so a case's input compares equal however
+ * its keys were ordered (Postgres jsonb reorders them on save).
+ */
+export function stableJson(value) {
+    return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v)
+        ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0))
+        : v);
+}
+/** Drafted cases not yet in the answer key. */
+export function newStarters(starter, existing) {
+    const have = new Set(existing.map((c) => stableJson(c.input)));
+    return (starter ?? []).filter((c) => !have.has(stableJson(c.input)));
+}
+//# sourceMappingURL=stableJson.js.map

package/dist/stableJson.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stableJson.js","sourceRoot":"","sources":["../src/stableJson.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAU,EAAE,EAAE,CAChD,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,CAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7D,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3B,CACF;QACH,CAAC,CAAC,CAAC,CACN,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,WAAW,CACzB,OAAwB,EACxB,QAA8B;IAE9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC"}

package/dist/stats.d.ts

@@ -0,0 +1,98 @@
+import type { CheckResult } from "./types.js";
+/**
+ * Summary numbers for a run: pass rates per check with a 95% range, so a
+ * small dataset doesn't look more certain than it is. Pure; no database.
+ */
+/** A pass rate with its 95% range; null when nothing was measured. */
+export type Rate = {
+    passed: number;
+    /** Outputs the check applied to. */
+    n: number;
+    rate: number | null;
+    /** 95% Wilson interval. */
+    low: number | null;
+    high: number | null;
+};
+/** Wilson score interval: honest for small n and rates near 0 or 1. */
+export declare function wilson(passed: number, n: number, z?: number): Rate;
+/** One graded answer, as the metrics read it. */
+export type ScoredOutput = {
+    caseId: string;
+    repeat: number;
+    /** null when the call failed. */
+    checks: CheckResult[] | null;
+    costUsd: number;
+    ms: number;
+    reused: boolean;
+};
+/**
+ * A test's numbers: answers right, partial credit, each check's pass rate, consistency,
+ * cost, time, and reuse.
+ */
+export type SuiteMetrics = {
+    cases: number;
+    outputs: number;
+    failed: number;
+    /** Every applicable check passed. Failed calls count as not passing. */
+    allPass: Rate;
+    /**
+     * Partial credit summed over answers (e.g. items right of items
+     * expected), where the suite gives it; the headline when present.
+     */
+    units: {
+        right: number;
+        total: number;
+    } | null;
+    byCheck: Record<string, Rate>;
+    /**
+     * Of cases run more than once, the share whose all-pass result was the
+     * same every time. Low: the model is inconsistent on these cases.
+     */
+    consistency: Rate | null;
+    /** What grading's judge spent, apart from the model under test (runner). */
+    judgeCostUsd?: number;
+    /** The error most failed answers share (runner). */
+    failureReason?: string | null;
+    /**
+     * Per check, the mean of each measure its answers carried (brier,
+     * logLoss, …), and, where answers carried a confidence, the calibration
+     * error. Only checks with measures appear.
+     */
+    measures?: Record<string, CheckMeasures>;
+    /** What this run spent: reused outputs cost nothing. */
+    costUsd: number;
+    /** What one answer costs on this model, reused or not. */
+    costPerAnswer: number | null;
+    /** How long one answer takes on this model. */
+    avgMs: number;
+    reused: number;
+};
+/**
+ * Whether an answer passed every check that applied. A failed answer (null) didn't.
+ */
+export declare function allPassed(checks: CheckResult[] | null): boolean;
+/** Sums each answer's partial credit; null when the suite gives none. */
+export declare function sumUnits(outputs: {
+    checks: CheckResult[] | null;
+}[]): {
+    right: number;
+    total: number;
+} | null;
+/** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */
+export declare function answerScore(checks: CheckResult[] | null): number;
+/** A test's numbers from its graded answers. */
+/** One check's measures across a test. */
+export type CheckMeasures = {
+    /** Answers that carried measures. */
+    n: number;
+    /** The mean of each measure, over the answers that carried it. */
+    mean: Record<string, number>;
+    /** Expected calibration error, from confidence and correctness; null without both. */
+    calibrationError: number | null;
+};
+/** One check's measures across a test's answers; null when none carried any. */
+export declare function measuresFor(outputs: ScoredOutput[], key: string): CheckMeasures | null;
+/**
+ *
+ */
+export declare function suiteMetrics(outputs: ScoredOutput[], checkKeys: string[]): SuiteMetrics;

package/dist/stats.js

@@ -0,0 +1,119 @@
+import { calibrationError } from "./decisions.js";
+/** Wilson score interval: honest for small n and rates near 0 or 1. */
+export function wilson(passed, n, z = 1.96) {
+    if (n === 0)
+        return { passed, n, rate: null, low: null, high: null };
+    const p = passed / n;
+    const denom = 1 + (z * z) / n;
+    const center = (p + (z * z) / (2 * n)) / denom;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
+    return {
+        passed,
+        n,
+        rate: p,
+        low: Math.max(0, center - half),
+        high: Math.min(1, center + half),
+    };
+}
+/**
+ * Whether an answer passed every check that applied. A failed answer (null) didn't.
+ */
+export function allPassed(checks) {
+    return checks !== null && checks.every((c) => c.pass !== false);
+}
+/** Sums each answer's partial credit; null when the suite gives none. */
+export function sumUnits(outputs) {
+    let right = 0;
+    let total = 0;
+    let any = false;
+    for (const o of outputs)
+        for (const c of o.checks ?? [])
+            if (c.units) {
+                any = true;
+                right += c.units.right;
+                total += c.units.total;
+            }
+    return any ? { right, total } : null;
+}
+/** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */
+export function answerScore(checks) {
+    if (!checks)
+        return 0;
+    const u = checks.find((c) => c.units)?.units;
+    if (u)
+        return u.total ? u.right / u.total : 1;
+    return allPassed(checks) ? 1 : 0;
+}
+/** One check's measures across a test's answers; null when none carried any. */
+export function measuresFor(outputs, key) {
+    const all = outputs
+        .map((o) => o.checks?.find((c) => c.key === key)?.measures)
+        .filter((m) => m !== undefined);
+    if (!all.length)
+        return null;
+    const sums = new Map();
+    for (const m of all)
+        for (const [name, v] of Object.entries(m)) {
+            const s = sums.get(name) ?? { total: 0, n: 0 };
+            s.total += v;
+            s.n++;
+            sums.set(name, s);
+        }
+    const pairs = all.flatMap((m) => m.confidence !== undefined && m.correct !== undefined
+        ? [{ confidence: m.confidence, correct: m.correct === 1 }]
+        : []);
+    return {
+        n: all.length,
+        mean: Object.fromEntries([...sums].map(([k, s]) => [k, s.total / s.n])),
+        calibrationError: calibrationError(pairs),
+    };
+}
+/**
+ *
+ */
+export function suiteMetrics(outputs, checkKeys) {
+    const byCheck = {};
+    for (const key of checkKeys) {
+        const applied = outputs
+            .map((o) => o.checks?.find((c) => c.key === key))
+            .filter((c) => c !== undefined && c.pass !== null);
+        byCheck[key] = wilson(applied.filter((c) => c.pass).length, applied.length);
+    }
+    const measures = {};
+    for (const key of checkKeys) {
+        const m = measuresFor(outputs, key);
+        if (m)
+            measures[key] = m;
+    }
+    const byCase = new Map();
+    for (const o of outputs)
+        byCase.set(o.caseId, [
+            ...(byCase.get(o.caseId) ?? []),
+            allPassed(o.checks),
+        ]);
+    const repeated = [...byCase.values()].filter((r) => r.length > 1);
+    const called = outputs.filter((o) => !o.reused);
+    // Answers with a recorded time (older reused rows have none).
+    const timed = outputs.filter((o) => o.checks && o.ms > 0);
+    return {
+        cases: byCase.size,
+        outputs: outputs.length,
+        failed: outputs.filter((o) => !o.checks).length,
+        allPass: wilson(outputs.filter((o) => allPassed(o.checks)).length, outputs.length),
+        units: sumUnits(outputs),
+        byCheck,
+        ...(Object.keys(measures).length ? { measures } : {}),
+        consistency: repeated.length
+            ? wilson(repeated.filter((r) => r.every((x) => x === r[0])).length, repeated.length)
+            : null,
+        costUsd: called.reduce((t, o) => t + o.costUsd, 0),
+        costPerAnswer: timed.length
+            ? timed.reduce((t, o) => t + o.costUsd, 0) / timed.length
+            : null,
+        avgMs: timed.length
+            ? Math.round(timed.reduce((t, o) => t + o.ms, 0) / timed.length)
+            : 0,
+        reused: outputs.length - called.length,
+    };
+}
+//# sourceMappingURL=stats.js.map