@cogitator-ai/evals 0.9.0 → 0.11.0

package/dist/eval-suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyErC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAG3B,MAAM,cAAc,GAAG,IAAI,KAAK,CAAe,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,cAAc,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzD,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACrF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,cAAc,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;wBACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;wBAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;wBACjC,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAA0B;QAE1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QACD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;YAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1D,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;YAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;QAExD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACtE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAapE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0IrC;;;GAGG;AACH,MAAM,WAAY,SAAQ,KAAK;IAGlB;IACA;IAHX,YACE,KAAc,EACL,KAAuB,EACvB,KAA8B;QAEvC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAH7B,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAyB;QAGvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,6EAA6E;IACrE,eAAe,CAAC,OAAmB;QACzC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,sBAAsB,MAAM,qDAAqD,CAC1G,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,mBAAmB,CAAC,oCAAoC,IAAI,CAAC,MAAM,mBAAmB,CAAC,UAAU,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAmB,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAuB,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACzE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAChF,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnF,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzC,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,SAAuC,CAAC;QAC5C,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAEvF,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;YAC9E,MAAM,KAAK,GAAG,WAAW,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5B,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,SAAS,CACV,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;SAC5B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAwB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAqB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAc,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,KAA8B,CAAC;QACnC,MAAM,KAAK,GAA2E,EAAE,CAAC;QAEzF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,WAAW,GAAqB,EAAE,CAAC;YACzC,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;oBACjD,KAAK;oBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM;oBACN,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;iBACzE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,SAAS;oBAAE,MAAM,KAAK,CAAC;gBACxC,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,KAAK,CACN,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBAClD,CAAC,CAAE,SAAS,CAAC,SAA4B;gBACzC,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,SAAS;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YACjC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;gBAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzD,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;gBAAE,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACjF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;gBAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,MAAM;gBACN,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;aACnE,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,gBAAgB,IAAI,EAAE,SAAS,EAAE,CAAC;YACtC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9C,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,GAAG,KAAK;SACT,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,eAAe,CAAC,OAA2B;QACjD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,GAA0B,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACvE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU;gBAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACxF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAwB;IACxC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,UAKrB;IACC,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACrE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,OAAgB;IACtC,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,kFAAkF;AAClF,mGAAmG;AACnG,SAAS,SAAS,CAAC,MAA0B;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,YAAY,CAAC,CAAmB;IACvC,OAAO;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -1,32 +1,37 @@
 export declare const VERSION: string;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
-export type { EvalTarget, EvalTargetContext, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
+export type { EvalTarget, EvalTargetContext, EvalTurn, RetrievalToolResult, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
 export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ComparisonResult, } from './eval-comparison.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
-export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
-export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export type { LLMMetricFn, JudgeContext, JudgeMessage } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
+export type { PassAtKOptions, PassAtKMetricFn } from './metrics/statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
+export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCallsScore, TrajectoryOrder, } from './metrics/tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
+export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
-export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
-export type { CustomMetricConfig } from './metrics/custom.js';
+export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
+export type { CustomMetricConfig, CustomMetricData } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
 export { maxErrorRate } from './assertions/index.js';
 export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
-export type { ReporterType, ReporterOptions } from './reporters/index.js';
+export type { ReporterType, ReporterOptions, ReportedCase } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export type { TTestResult } from './stats/t-test.js';
 export { mcnemarsTest } from './stats/mcnemar.js';
 export type { McNemarResult } from './stats/mcnemar.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
-export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
+export type { EvalCase, ExpectedToolCall, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
 export { createRunEvalTool, evalTools } from './tools.js';
 export type { EvalTool } from './tools.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -7,9 +7,12 @@ export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
-export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
+export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAYlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/matchers.d.ts

@@ -0,0 +1,68 @@
+import type { TrajectoryOrder } from './metrics/tool-calls.js';
+/** The matchers `evalMatchers` adds to Vitest's `expect`, typed for Vitest 5 */
+export interface EvalMatchers<R = unknown> {
+    /**
+     * The case result called the tool `name`, over any of its turns, with arguments holding `args`
+     * when given (other arguments may be there too)
+     */
+    toHaveCalledTool(name: string, args?: Record<string, unknown>): R;
+    /**
+     * The case result called the tools in `names`. `order: 'exact'` (default): those calls and
+     * no others, in this order. `'in-order'`: in this order, other calls may come between.
+     * `'any-order'`: each of them, in any order, other calls allowed
+     */
+    toFollowTrajectory(names: readonly string[], options?: {
+        order?: TrajectoryOrder;
+    }): R;
+    /**
+     * The case result retrieved a passage with this id or this text, or whose text contains this
+     * one or is contained in it when the contained text has at least three words
+     */
+    toHaveRetrieved(idOrText: string): R;
+    /**
+     * A case result scored at least `min` on the metric, or a suite result's mean of the metric
+     * is at least `min`
+     */
+    toScoreAtLeast(metricName: string, min: number): R;
+}
+declare module 'vitest' {
+    interface Assertion<R extends void | Promise<void> = void, T = unknown> extends EvalMatchers<R> {
+    }
+    interface AsymmetricMatchersContaining extends EvalMatchers {
+    }
+}
+/** The part of Vitest's matcher context the eval matchers use */
+interface MatcherContext {
+    isNot: boolean;
+}
+interface MatcherResult {
+    pass: boolean;
+    message: () => string;
+    actual?: unknown;
+    expected?: unknown;
+}
+/**
+ * Vitest matchers for eval results. Register them once, in a test file or a setup file:
+ *
+ * @example
+ * ```ts
+ * import { expect } from 'vitest';
+ * import { evalMatchers } from '@cogitator-ai/evals/matchers';
+ *
+ * expect.extend(evalMatchers);
+ *
+ * const { results } = await suite.run();
+ * expect(results[0]).toHaveCalledTool('get_weather', { city: 'Paris' });
+ * expect(results[0]).toFollowTrajectory(['search', 'answer'], { order: 'in-order' });
+ * ```
+ */
+export declare const evalMatchers: {
+    toHaveCalledTool(this: MatcherContext, received: unknown, name: string, args?: Record<string, unknown>): MatcherResult;
+    toFollowTrajectory(this: MatcherContext, received: unknown, names: readonly string[], options?: {
+        order?: TrajectoryOrder;
+    }): MatcherResult;
+    toHaveRetrieved(this: MatcherContext, received: unknown, idOrText: string): MatcherResult;
+    toScoreAtLeast(this: MatcherContext, received: unknown, metricName: string, min: number): MatcherResult;
+};
+export {};
+//# sourceMappingURL=matchers.d.ts.map

package/dist/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAS5D,gFAAgF;AAChF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,OAAO;IACvC;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAClE;;;;OAIG;IACH,kBAAkB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,eAAe,CAAA;KAAE,GAAG,CAAC,CAAC;IACvF;;;OAGG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC;CACpD;AAED,OAAO,QAAQ,QAAQ,CAAC;IAEtB,UAAU,SAAS,CAAC,CAAC,SAAS,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,OAAO,CAAE,SAAQ,YAAY,CAAC,CAAC,CAAC;KAAG;IAElG,UAAU,4BAA6B,SAAQ,YAAY;KAAG;CAC/D;AAED,iEAAiE;AACjE,UAAU,cAAc;IACtB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,UAAU,aAAa;IACrB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AA4BD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,YAAY;2BAEf,cAAc,YACV,OAAO,QACX,MAAM,SACL,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,aAAa;6BAgBR,cAAc,YACV,OAAO,SACV,SAAS,MAAM,EAAE,YACf;QAAE,KAAK,CAAC,EAAE,eAAe,CAAA;KAAE,GACnC,aAAa;0BAmBM,cAAc,YAAY,OAAO,YAAY,MAAM,GAAG,aAAa;yBAejF,cAAc,YACV,OAAO,cACL,MAAM,OACb,MAAM,GACV,aAAa;CAUjB,CAAC"}

package/dist/matchers.js

@@ -0,0 +1,112 @@
+import { trajectoryScore } from './metrics/tool-calls.js';
+import { isSubset, longestCommonSubsequence, multisetOverlap, passageMatches, } from './metrics/matching.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function caseResultOf(received, matcher) {
+    if (isRecord(received) && isRecord(received.case) && typeof received.output === 'string') {
+        return received;
+    }
+    throw new TypeError(`${matcher} expects a case result, one of suite.run()'s results`);
+}
+function callText(name, args) {
+    return args === undefined ? name : `${name}(${JSON.stringify(args)})`;
+}
+function callsText(result) {
+    const calls = result.toolCalls ?? [];
+    return calls.length === 0
+        ? 'no tool calls'
+        : calls.map((c) => callText(c.name, c.arguments)).join(', ');
+}
+function negation(context) {
+    return context.isNot ? 'not ' : '';
+}
+/**
+ * Vitest matchers for eval results. Register them once, in a test file or a setup file:
+ *
+ * @example
+ * ```ts
+ * import { expect } from 'vitest';
+ * import { evalMatchers } from '@cogitator-ai/evals/matchers';
+ *
+ * expect.extend(evalMatchers);
+ *
+ * const { results } = await suite.run();
+ * expect(results[0]).toHaveCalledTool('get_weather', { city: 'Paris' });
+ * expect(results[0]).toFollowTrajectory(['search', 'answer'], { order: 'in-order' });
+ * ```
+ */
+export const evalMatchers = {
+    toHaveCalledTool(received, name, args) {
+        const result = caseResultOf(received, 'toHaveCalledTool');
+        const calls = result.toolCalls ?? [];
+        const pass = calls.some((call) => call.name === name && (args === undefined || isSubset(args, call.arguments)));
+        return {
+            pass,
+            message: () => `expected the case ${negation(this)}to call ${callText(name, args)}, it made ${callsText(result)}`,
+            actual: calls.map((c) => ({ name: c.name, arguments: c.arguments })),
+            expected: { name, ...(args !== undefined && { arguments: args }) },
+        };
+    },
+    toFollowTrajectory(received, names, options = {}) {
+        const result = caseResultOf(received, 'toFollowTrajectory');
+        const order = options.order ?? 'exact';
+        const actual = (result.toolCalls ?? []).map((c) => c.name);
+        const pass = order === 'exact'
+            ? trajectoryScore(names, actual, 'exact') === 1
+            : order === 'in-order'
+                ? longestCommonSubsequence(names, actual) === names.length
+                : multisetOverlap(names, actual) === names.length;
+        return {
+            pass,
+            message: () => `expected the tool calls ${negation(this)}to follow [${names.join(', ')}] (${order}), they were [${actual.join(', ')}]`,
+            actual,
+            expected: [...names],
+        };
+    },
+    toHaveRetrieved(received, idOrText) {
+        const result = caseResultOf(received, 'toHaveRetrieved');
+        const retrieved = result.retrieved ?? [];
+        const pass = retrieved.some((p) => passageMatches(p, idOrText));
+        const shown = retrieved.map((p) => p.id ?? JSON.stringify(p.content.slice(0, 60)));
+        return {
+            pass,
+            message: () => `expected the case ${negation(this)}to retrieve ${JSON.stringify(idOrText)}, it retrieved ${retrieved.length === 0 ? 'nothing' : `${retrieved.length} passage(s): ${shown.join(', ')}`}`,
+        };
+    },
+    toScoreAtLeast(received, metricName, min) {
+        const { score, of } = metricScoreOf(received, metricName);
+        const pass = score >= min;
+        return {
+            pass,
+            message: () => `expected ${of} ${negation(this)}to be at least ${min}, it is ${score}`,
+            actual: score,
+            expected: min,
+        };
+    },
+};
+/** The score a matcher compares: a case result's score of the metric, or a suite's mean of it */
+function metricScoreOf(received, metricName) {
+    if (isRecord(received) && isRecord(received.aggregated)) {
+        const aggregated = received.aggregated;
+        const metric = aggregated[metricName];
+        if (!metric) {
+            throw new Error(`toScoreAtLeast: the suite has no metric '${metricName}', it has ${namesText(Object.keys(aggregated))}`);
+        }
+        return { score: metric.mean, of: `the mean of ${metricName}` };
+    }
+    const result = caseResultOf(received, 'toScoreAtLeast');
+    const scores = result.scores ?? [];
+    const found = scores.find((s) => s.name === metricName);
+    if (!found) {
+        throw new Error(`toScoreAtLeast: the case has no score for '${metricName}', it has ${namesText(scores.map((s) => s.name))}`);
+    }
+    return {
+        score: found.score,
+        of: `the ${metricName} score of the case${found.details ? ` (${found.details})` : ''}`,
+    };
+}
+function namesText(names) {
+    return names.length === 0 ? 'none' : names.map((n) => `'${n}'`).join(', ');
+}
+//# sourceMappingURL=matchers.js.map

package/dist/matchers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,QAAQ,EACR,wBAAwB,EACxB,eAAe,EACf,cAAc,GACf,MAAM,oBAAoB,CAAC;AA8C5B,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,QAAiB,EAAE,OAAe;IACtD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzF,OAAO,QAAqC,CAAC;IAC/C,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,GAAG,OAAO,sDAAsD,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAA8B;IAC5D,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,MAAsB;IACvC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,OAAuB;IACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,gBAAgB,CAEd,QAAiB,EACjB,IAAY,EACZ,IAA8B;QAE9B,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CACrB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CACvF,CAAC;QACF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,qBAAqB,QAAQ,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,SAAS,CAAC,MAAM,CAAC,EAAE;YACpG,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YACpE,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE;SACnE,CAAC;IACJ,CAAC;IAED,kBAAkB,CAEhB,QAAiB,EACjB,KAAwB,EACxB,UAAuC,EAAE;QAEzC,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,IAAI,GACR,KAAK,KAAK,OAAO;YACf,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;YAC/C,CAAC,CAAC,KAAK,KAAK,UAAU;gBACpB,CAAC,CAAC,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM;gBAC1D,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC;QACxD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,2BAA2B,QAAQ,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,iBAAiB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACzH,MAAM;YACN,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC;SACrB,CAAC;IACJ,CAAC;IAED,eAAe,CAAuB,QAAiB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACnF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,qBAAqB,QAAQ,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,kBACxE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,gBAAgB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAC1F,EAAE;SACL,CAAC;IACJ,CAAC;IAED,cAAc,CAEZ,QAAiB,EACjB,UAAkB,EAClB,GAAW;QAEX,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC;QAC1B,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,kBAAkB,GAAG,WAAW,KAAK,EAAE;YACtF,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,GAAG;SACd,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,iGAAiG;AACjG,SAAS,aAAa,CAAC,QAAiB,EAAE,UAAkB;IAC1D,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,MAAM,UAAU,GAAG,QAAQ,CAAC,UAA2C,CAAC;QACxE,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,4CAA4C,UAAU,aAAa,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CACxG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,eAAe,UAAU,EAAE,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACxD,MAAM,MAAM,GAA4B,MAAqC,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3F,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACxD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,8CAA8C,UAAU,aAAa,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAC5G,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,EAAE,EAAE,OAAO,UAAU,qBAAqB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;KACvF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7E,CAAC"}

package/dist/metrics/custom.d.ts

@@ -1,12 +1,7 @@
-import type { MetricFn } from './types.js';
+import type { MetricFn, EvalCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from './types.js';
 export interface CustomMetricConfig {
     name: string;
-    evaluate: (data: {
-        input: string;
-        output: string;
-        expected?: string;
-        context?: Record<string, unknown>;
-    }) => Promise<{
+    evaluate: (data: CustomMetricData) => Promise<{
         score: number;
         details?: string;
     }> | {
@@ -14,6 +9,23 @@ export interface CustomMetricConfig {
         details?: string;
     };
 }
+/** What a custom metric's `evaluate` gets about a case */
+export interface CustomMetricData {
+    input: string;
+    output: string;
+    expected?: string;
+    context?: Record<string, unknown>;
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the run retrieved */
+    retrieved?: readonly Passage[];
+    /** Each turn of a multi-turn case */
+    turns?: readonly EvalTurnResult[];
+    /** The whole case result, the case itself included */
+    result: EvalCaseResult;
+}
 /**
  * A metric from an `evaluate` function. Its score is clamped to 0 - 1. When `evaluate` throws or
  * returns a score that is not a finite number (such as NaN from a division by zero), the metric

package/dist/metrics/custom.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,SAAS,CAAC;AAGxD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CAAC,IAAI,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CAsC3D"}
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACR,MAAM,SAAS,CAAC;AAGjB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CACR,IAAI,EAAE,gBAAgB,KACnB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACzF;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,sDAAsD;IACtD,MAAM,EAAE,cAAc,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CA2C3D"}

package/dist/metrics/custom.js

@@ -12,6 +12,11 @@ export function metric(config) {
                 output: result.output,
                 expected: result.case.expected,
                 context: result.case.context,
+                toolCalls: result.toolCalls,
+                toolResults: result.toolResults,
+                retrieved: result.retrieved,
+                turns: result.turns,
+                result,
             });
             const invalid = nonFiniteScoreError(score);
             if (invalid) {

package/dist/metrics/custom.js.map

@@ -1 +1 @@
-{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAY9C;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;gBACrC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AA2B9C;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;gBAC5B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;gBACrC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -5,6 +5,7 @@ interface MatchOptions extends MetricOptions {
 }
 export declare function exactMatch(opts?: MatchOptions): MetricFn;
 export declare function similarity(opts?: MatchOptions): MetricFn;
+export declare function tokenF1(opts?: MatchOptions): MetricFn;
 export declare function contains(opts?: MatchOptions): MetricFn;
 /** Scores the fraction of keywords found in the output. */
 export declare function containsAll(keywords: string[], opts?: MatchOptions): MetricFn;

package/dist/metrics/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAkCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AA8BD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAaD,wBAAgB,OAAO,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CA+CrD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}

package/dist/metrics/deterministic.js

@@ -55,10 +55,7 @@ function levenshteinDistance(s1, s2) {
         const char1 = s1[i - 1];
         for (let j = 1; j <= n; j++) {
             const cost = char1 === s2[j - 1] ? 0 : 1;
-            curr[j] = Math.min(prev[j] + 1, // deletion
-            curr[j - 1] + 1, // insertion
-            prev[j - 1] + cost // substitution
-            );
+            curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
         }
         const temp = prev;
         prev = curr;
@@ -93,6 +90,54 @@ export function similarity(opts) {
         };
     }));
 }
+function tokenize(text, caseSensitive) {
+    const normalized = (caseSensitive ? text : text.toLowerCase())
+        .replace(/[!-/:-@[-`{-~]/g, '')
+        .replace(/\b(a|an|the)\b/gi, ' ');
+    return normalized.split(/\s+/).filter(Boolean);
+}
+function roundToTwo(value) {
+    return Number(value.toFixed(2));
+}
+export function tokenF1(opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'tokenF1';
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expected;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected value provided' };
+        }
+        const output = result.output.trim();
+        const target = expected.trim();
+        const actualTokens = tokenize(output, caseSensitive);
+        const expectedTokens = tokenize(target, caseSensitive);
+        if (actualTokens.length === 0 && expectedTokens.length === 0) {
+            return { name, score: 1 };
+        }
+        const counts = new Map();
+        for (const token of expectedTokens) {
+            counts.set(token, (counts.get(token) ?? 0) + 1);
+        }
+        let overlap = 0;
+        for (const token of actualTokens) {
+            const left = counts.get(token) ?? 0;
+            if (left > 0) {
+                overlap++;
+                counts.set(token, left - 1);
+            }
+        }
+        const precision = actualTokens.length > 0 ? overlap / actualTokens.length : 0;
+        const recall = expectedTokens.length > 0 ? overlap / expectedTokens.length : 0;
+        const score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+        return {
+            name,
+            score,
+            details: score < 1
+                ? `expected "${target}", got "${output}" (precision: ${roundToTwo(precision)}, recall: ${roundToTwo(recall)})`
+                : undefined,
+        };
+    }));
+}
 export function contains(opts) {
     const caseSensitive = opts?.caseSensitive ?? false;
     const name = opts?.name ?? 'contains';

package/dist/metrics/deterministic.js.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY;YAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,aAAsB;IACpD,MAAM,UAAU,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;SAC3D,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,SAAS,CAAC;IAErC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEvD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3F,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC;gBACP,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,iBAAiB,UAAU,CAAC,SAAS,CAAC,aAAa,UAAU,CAAC,MAAM,CAAC,GAAG;gBAC9G,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/index.d.ts

@@ -1,8 +1,12 @@
-export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
+export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, } from './types.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
-export type { CustomMetricConfig } from './custom.js';
-export { latency, cost, tokenUsage } from './statistical.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
+export type { CustomMetricConfig, CustomMetricData } from './custom.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './statistical.js';
+export type { PassAtKOptions, PassAtKMetricFn } from './statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './tool-calls.js';
+export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCallsScore, TrajectoryOrder, } from './tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages } from './retrieval.js';
+export { faithfulness, relevance, coherence, helpfulness, contextRelevance, llmMetric, bindJudgeContext, } from './llm-judge.js';
 export type { JudgeContext, LLMMetricFn } from './llm-judge.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,GACR,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACrE,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACvF,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}