@cogitator-ai/evals 0.9.0 → 0.10.0

package/dist/metrics/llm-judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI,SAAS,CAAC;IAC1F,MAAM,KAAK,GACT,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,MAAM,KAAK,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;YAC3F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4D,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAM,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,UAAU,CAAC,KAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtF,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,WAAW;QACX,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;KAC5E,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,SAAS,CAAC,KAAiB,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAY,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAW,EAAE,OAAe;IACvE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,KAAc;IACzD,OAAO,CAAC,WAAW,CACjB,6BAA6B,MAAM,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1G,EAAE,IAAI,EAAE,6BAA6B,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,SAAS,QAAQ,CAAC,WAAwB,EAAE,KAAa;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,iBAAiB;QACjB,kBAAkB;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,SAAS,IAAI,IAAI;QAC7B,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;;YAC7E,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACZ,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,MAAM,YAAY,GAChB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACzG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,qaAAqa,CACta,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AA6BxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI,SAAS,CAAC;IAC1F,MAAM,KAAK,GACT,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,MAAM,KAAK,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;YAC3F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4D,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAM,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,UAAU,CAAC,KAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtF,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,WAAW;QACX,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;KAC5E,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,SAAS,CAAC,KAAiB,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAY,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAW,EAAE,OAAe;IACvE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,KAAc;IACzD,OAAO,CAAC,WAAW,CACjB,6BAA6B,MAAM,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1G,EAAE,IAAI,EAAE,6BAA6B,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,SAAS,QAAQ,CAAC,WAAwB,EAAE,KAAa;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,iBAAiB;QACjB,kBAAkB;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,SAAS,IAAI,IAAI;QAC7B,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;;YAC7E,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACZ,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAY,EACZ,YAAoB,EACpB,OAAkD;IAElD,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO;QAAE,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE9F,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,QAA4B;IAChD,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;SAC3E,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,MAAsB,EAAE,eAAwB;IACrE,IAAI,eAAe,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,OAAO,wBAAwB,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,OAAO,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;QAC/C,CAAC,CAAC,aAAa,WAAW,CAAC,OAAO,CAAC,EAAE;QACrC,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAsB;IACxC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM;SACnB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACZ,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,KAAK,gBAAgB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC;IACtF,OAAO,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,gEAAgE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAChJ,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB,EAAE,eAAe,GAAG,KAAK;IACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;IACnE,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACzF,IAAI,MAAM,IAAI,OAAO;gBAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;YAEvD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,6bAA6b,EAC7b,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAC1D,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACnD,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,kBAAkB,EAChC,yZAAyZ,EACzZ,CAAC,MAAM,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,2CAA2C,EAAE,CAAC;QACzF,OAAO,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,EAAE,CAAC;IAC3D,CAAC,CACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}

package/dist/metrics/matching.d.ts

@@ -0,0 +1,28 @@
+import type { Passage } from './types.js';
+/** Whether two JSON-like values are equal, objects key by key and arrays item by item */
+export declare function deepEqual(a: unknown, b: unknown): boolean;
+/**
+ * Whether `actual` holds everything `expected` has: every key of an expected object is in the
+ * actual one with a matching value, arrays match item by item with the same length, and other
+ * values are equal
+ */
+export declare function isSubset(expected: unknown, actual: unknown): boolean;
+/** Text compared without case, punctuation or runs of whitespace */
+export declare function normalizeText(text: string): string;
+/**
+ * Whether a retrieved passage is the expected one: the same id, the same words once case and
+ * punctuation are set aside, or texts where one holds the other's words in a row when that one
+ * has at least three words, so a short id or a single word never matches every passage that
+ * happens to hold it
+ */
+export declare function passageMatches(passage: Passage, expected: string): boolean;
+/**
+ * A largest one-to-one matching between `left` and `right` items, `matches` telling which pairs
+ * may match (Kuhn's augmenting paths): the index of the right item each matched left item took
+ */
+export declare function maxMatching<L, R>(left: readonly L[], right: readonly R[], matches: (l: L, r: R) => boolean): Map<number, number>;
+/** The length of the longest common subsequence of two sequences */
+export declare function longestCommonSubsequence(a: readonly string[], b: readonly string[]): number;
+/** How many items two multisets share */
+export declare function multisetOverlap(a: readonly string[], b: readonly string[]): number;
+//# sourceMappingURL=matching.d.ts.map

package/dist/metrics/matching.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matching.d.ts","sourceRoot":"","sources":["../../src/metrics/matching.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAMvC,yFAAyF;AACzF,wBAAgB,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,GAAG,OAAO,CAWzD;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAYpE;AAED,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMlD;AAcD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQ1E;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,EAC9B,IAAI,EAAE,SAAS,CAAC,EAAE,EAClB,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,OAAO,GAC/B,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrB;AAED,oEAAoE;AACpE,wBAAgB,wBAAwB,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAU3F;AAED,yCAAyC;AACzC,wBAAgB,eAAe,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAYlF"}

package/dist/metrics/matching.js

@@ -0,0 +1,124 @@
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/** Whether two JSON-like values are equal, objects key by key and arrays item by item */
+export function deepEqual(a, b) {
+    if (Object.is(a, b))
+        return true;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        return a.length === b.length && a.every((item, i) => deepEqual(item, b[i]));
+    }
+    if (isRecord(a) && isRecord(b)) {
+        const keys = Object.keys(a);
+        if (keys.length !== Object.keys(b).length)
+            return false;
+        return keys.every((key) => Object.hasOwn(b, key) && deepEqual(a[key], b[key]));
+    }
+    return false;
+}
+/**
+ * Whether `actual` holds everything `expected` has: every key of an expected object is in the
+ * actual one with a matching value, arrays match item by item with the same length, and other
+ * values are equal
+ */
+export function isSubset(expected, actual) {
+    if (Array.isArray(expected) && Array.isArray(actual)) {
+        return (expected.length === actual.length && expected.every((item, i) => isSubset(item, actual[i])));
+    }
+    if (isRecord(expected) && isRecord(actual)) {
+        return Object.entries(expected).every(([key, value]) => Object.hasOwn(actual, key) && isSubset(value, actual[key]));
+    }
+    return Object.is(expected, actual);
+}
+/** Text compared without case, punctuation or runs of whitespace */
+export function normalizeText(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+/** Texts shorter than this many words match only as a whole, never as a part of another */
+const MIN_CONTAINED_WORDS = 3;
+function wordCount(text) {
+    return text === '' ? 0 : text.split(' ').length;
+}
+/** Whether `inner` is a run of whole words of `outer`, both normalized */
+function containsWords(outer, inner) {
+    return ` ${outer} `.includes(` ${inner} `);
+}
+/**
+ * Whether a retrieved passage is the expected one: the same id, the same words once case and
+ * punctuation are set aside, or texts where one holds the other's words in a row when that one
+ * has at least three words, so a short id or a single word never matches every passage that
+ * happens to hold it
+ */
+export function passageMatches(passage, expected) {
+    if (passage.id !== undefined && passage.id === expected)
+        return true;
+    const content = normalizeText(passage.content);
+    const target = normalizeText(expected);
+    if (content === '' || target === '')
+        return false;
+    if (content === target)
+        return true;
+    if (wordCount(target) >= MIN_CONTAINED_WORDS && containsWords(content, target))
+        return true;
+    return wordCount(content) >= MIN_CONTAINED_WORDS && containsWords(target, content);
+}
+/**
+ * A largest one-to-one matching between `left` and `right` items, `matches` telling which pairs
+ * may match (Kuhn's augmenting paths): the index of the right item each matched left item took
+ */
+export function maxMatching(left, right, matches) {
+    const owner = new Array(right.length).fill(-1);
+    const edges = left.map((l) => right.flatMap((r, j) => (matches(l, r) ? [j] : [])));
+    const augment = (i, seen) => {
+        for (const j of edges[i]) {
+            if (seen[j])
+                continue;
+            seen[j] = true;
+            if (owner[j] === -1 || augment(owner[j], seen)) {
+                owner[j] = i;
+                return true;
+            }
+        }
+        return false;
+    };
+    for (let i = 0; i < left.length; i++)
+        augment(i, new Array(right.length).fill(false));
+    const pairs = new Map();
+    owner.forEach((i, j) => {
+        if (i !== -1)
+            pairs.set(i, j);
+    });
+    return pairs;
+}
+/** The length of the longest common subsequence of two sequences */
+export function longestCommonSubsequence(a, b) {
+    let prev = new Array(b.length + 1).fill(0);
+    for (const item of a) {
+        const curr = new Array(b.length + 1).fill(0);
+        for (let j = 1; j <= b.length; j++) {
+            curr[j] = item === b[j - 1] ? prev[j - 1] + 1 : Math.max(prev[j], curr[j - 1]);
+        }
+        prev = curr;
+    }
+    return prev[b.length];
+}
+/** How many items two multisets share */
+export function multisetOverlap(a, b) {
+    const counts = new Map();
+    for (const item of a)
+        counts.set(item, (counts.get(item) ?? 0) + 1);
+    let shared = 0;
+    for (const item of b) {
+        const left = counts.get(item) ?? 0;
+        if (left > 0) {
+            counts.set(item, left - 1);
+            shared++;
+        }
+    }
+    return shared;
+}
+//# sourceMappingURL=matching.js.map

package/dist/metrics/matching.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matching.js","sourceRoot":"","sources":["../../src/metrics/matching.ts"],"names":[],"mappings":"AAEA,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,SAAS,CAAC,CAAU,EAAE,CAAU;IAC9C,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACxD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,QAAiB,EAAE,MAAe;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACrD,OAAO,CACL,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAC5F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CACnC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC;SACjC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,2FAA2F;AAC3F,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAClD,CAAC;AAED,0EAA0E;AAC1E,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa;IACjD,OAAO,IAAI,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,OAAgB,EAAE,QAAgB;IAC/D,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,IAAI,OAAO,CAAC,EAAE,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACrE,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,OAAO,KAAK,EAAE,IAAI,MAAM,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,mBAAmB,IAAI,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5F,OAAO,SAAS,CAAC,OAAO,CAAC,IAAI,mBAAmB,IAAI,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACrF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,IAAkB,EAClB,KAAmB,EACnB,OAAgC;IAEhC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,IAAe,EAAW,EAAE;QACtD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,CAAC,CAAC;gBAAE,SAAS;YACtB,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACb,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC,EAAE,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE/F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,wBAAwB,CAAC,CAAoB,EAAE,CAAoB;IACjF,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACxB,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,eAAe,CAAC,CAAoB,EAAE,CAAoB;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,CAAC;QAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC3B,MAAM,EAAE,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/metrics/retrieval.d.ts

@@ -0,0 +1,21 @@
+import type { MetricFn, MetricOptions, Passage } from './types.js';
+/**
+ * The passages of a tool result shaped like the `rag_search` tool's of `@cogitator-ai/rag`:
+ * `{ results: [{ content, chunkId?, score?, source?, ... }] }`, each result's `id` (or else its
+ * `chunkId`) as the passage id. Undefined for any other result.
+ */
+export declare function ragSearchPassages(toolResult: {
+    name: string;
+    result: unknown;
+}): Passage[] | undefined;
+/** The share of the expected passages that some retrieved passage matches */
+export declare function retrievalRecall(opts?: MetricOptions): MetricFn;
+/** The share of the retrieved passages that match an expected one */
+export declare function retrievalPrecision(opts?: MetricOptions): MetricFn;
+/**
+ * The reciprocal rank of the first retrieved passage that matches an expected one: 1 when it
+ * came first, 1/2 second and so on, 0 when none matches. Passages rank in the order they came,
+ * one tool result after the other.
+ */
+export declare function retrievalMRR(opts?: MetricOptions): MetricFn;
+//# sourceMappingURL=retrieval.d.ts.map

package/dist/metrics/retrieval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval.d.ts","sourceRoot":"","sources":["../../src/metrics/retrieval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAuBhE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,EAAE,GAAG,SAAS,CAOxB;AA6CD,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAW9D;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAWjE;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa3D"}

package/dist/metrics/retrieval.js

@@ -0,0 +1,108 @@
+import { passageMatches } from './matching.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function passageOf(item) {
+    if (!isRecord(item) || typeof item.content !== 'string')
+        return undefined;
+    const id = typeof item.id === 'string'
+        ? item.id
+        : typeof item.chunkId === 'string'
+            ? item.chunkId
+            : undefined;
+    return {
+        ...(id !== undefined && { id }),
+        content: item.content,
+        ...(typeof item.score === 'number' && Number.isFinite(item.score) && { score: item.score }),
+        ...(typeof item.source === 'string' && { source: item.source }),
+    };
+}
+/**
+ * The passages of a tool result shaped like the `rag_search` tool's of `@cogitator-ai/rag`:
+ * `{ results: [{ content, chunkId?, score?, source?, ... }] }`, each result's `id` (or else its
+ * `chunkId`) as the passage id. Undefined for any other result.
+ */
+export function ragSearchPassages(toolResult) {
+    const payload = toolResult.result;
+    if (!isRecord(payload) || !Array.isArray(payload.results))
+        return undefined;
+    return payload.results.flatMap((item) => {
+        const passage = passageOf(item);
+        return passage ? [passage] : [];
+    });
+}
+function createMetricFn(name, fn) {
+    fn.metricName = name;
+    return fn;
+}
+/**
+ * A retrieval metric over a case's `retrieved` passages and its `expectedPassages`. A case
+ * expecting no passages scores 1 when nothing was retrieved and 0 otherwise, and so does a case
+ * expecting passages when nothing was retrieved.
+ */
+function retrievalMetric(name, compute) {
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expectedPassages;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected passages provided' };
+        }
+        const retrieved = result.retrieved ?? [];
+        if (expected.length === 0 || retrieved.length === 0) {
+            const score = expected.length === retrieved.length ? 1 : 0;
+            return {
+                name,
+                score,
+                details: score === 1
+                    ? undefined
+                    : retrieved.length === 0
+                        ? 'nothing was retrieved'
+                        : 'no passages were expected',
+            };
+        }
+        const { score, details } = compute({ expected, retrieved });
+        return { name, score, ...(details !== undefined && { details }) };
+    }));
+}
+/** The share of the expected passages that some retrieved passage matches */
+export function retrievalRecall(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalRecall', ({ expected, retrieved }) => {
+        const missing = expected.filter((e) => !retrieved.some((p) => passageMatches(p, e)));
+        return {
+            score: (expected.length - missing.length) / expected.length,
+            details: missing.length > 0
+                ? `not retrieved: ${missing.map((e) => JSON.stringify(e)).join(', ')}`
+                : undefined,
+        };
+    });
+}
+/** The share of the retrieved passages that match an expected one */
+export function retrievalPrecision(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalPrecision', ({ expected, retrieved }) => {
+        const relevant = retrieved.filter((p) => expected.some((e) => passageMatches(p, e))).length;
+        return {
+            score: relevant / retrieved.length,
+            details: relevant < retrieved.length
+                ? `${retrieved.length - relevant} of ${retrieved.length} retrieved passages were not expected`
+                : undefined,
+        };
+    });
+}
+/**
+ * The reciprocal rank of the first retrieved passage that matches an expected one: 1 when it
+ * came first, 1/2 second and so on, 0 when none matches. Passages rank in the order they came,
+ * one tool result after the other.
+ */
+export function retrievalMRR(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalMRR', ({ expected, retrieved }) => {
+        const rank = retrieved.findIndex((p) => expected.some((e) => passageMatches(p, e))) + 1;
+        return {
+            score: rank === 0 ? 0 : 1 / rank,
+            details: rank === 0
+                ? 'no retrieved passage was expected'
+                : rank > 1
+                    ? `first expected passage at rank ${rank}`
+                    : undefined,
+        };
+    });
+}
+//# sourceMappingURL=retrieval.js.map

package/dist/metrics/retrieval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval.js","sourceRoot":"","sources":["../../src/metrics/retrieval.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,SAAS,CAAC,IAAa;IAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC1E,MAAM,EAAE,GACN,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ;QACzB,CAAC,CAAC,IAAI,CAAC,EAAE;QACT,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAChC,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,SAAS,CAAC;IAClB,OAAO;QACL,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,EAAE,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,GAAG,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3F,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAGjC;IACC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5E,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD;;;;GAIG;AACH,SAAS,eAAe,CACtB,IAAY,EACZ,OAA2E;IAE3E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;QACtE,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,OAAO;gBACL,IAAI;gBACJ,KAAK;gBACL,OAAO,EACL,KAAK,KAAK,CAAC;oBACT,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;wBACtB,CAAC,CAAC,uBAAuB;wBACzB,CAAC,CAAC,2BAA2B;aACpC,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,IAAoB;IAClD,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,iBAAiB,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QAClF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;YACL,KAAK,EAAE,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;YAC3D,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,kBAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,kBAAkB,CAAC,IAAoB;IACrD,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,oBAAoB,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QACrF,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5F,OAAO;YACL,KAAK,EAAE,QAAQ,GAAG,SAAS,CAAC,MAAM;YAClC,OAAO,EACL,QAAQ,GAAG,SAAS,CAAC,MAAM;gBACzB,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,QAAQ,OAAO,SAAS,CAAC,MAAM,uCAAuC;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QAC/E,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxF,OAAO;YACL,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;YAChC,OAAO,EACL,IAAI,KAAK,CAAC;gBACR,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,IAAI,GAAG,CAAC;oBACR,CAAC,CAAC,kCAAkC,IAAI,EAAE;oBAC1C,CAAC,CAAC,SAAS;SAClB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/metrics/statistical.d.ts

@@ -5,4 +5,32 @@ export declare function latency(): StatisticalMetricFn;
 export declare function cost(): StatisticalMetricFn;
 /** Total tokens per case, for cases that report usage. */
 export declare function tokenUsage(): StatisticalMetricFn;
+export interface PassAtKOptions {
+    /** Trials that may be drawn, a positive integer up to the suite's `trials` */
+    k: number;
+    /** The score at which a trial passes. Default: 1 */
+    threshold?: number;
+}
+/** A `passAtK()` metric: the suite checks its metric and its `k` against its own setup */
+export type PassAtKMetricFn = StatisticalMetricFn & {
+    readonly passAtK: {
+        readonly metric: string;
+        readonly k: number;
+        readonly threshold: number;
+    };
+};
+/** Whether a statistical metric is a `passAtK()` one */
+export declare function isPassAtKMetric(metric: StatisticalMetricFn): metric is PassAtKMetricFn;
+/**
+ * The chance that at least one of `k` trials drawn from `n`, `c` of which passed, passes:
+ * 1 - C(n - c, k) / C(n, k), computed as a product so large counts do not overflow
+ */
+export declare function passAtKEstimate(n: number, c: number, k: number): number;
+/**
+ * pass@k of a per-case metric over the suite's trials: for each case, the unbiased estimate of
+ * the chance that at least one of `k` trials scores `threshold` or more, averaged over the cases
+ * the metric scored at least one trial of. A trial that errored counts as a failure. Named
+ * `passAtK(<metric>,<k>)`. The suite must run at least `k` trials and have the metric.
+ */
+export declare function passAtK(metricName: string, opts: PassAtKOptions): PassAtKMetricFn;
 //# sourceMappingURL=statistical.d.ts.map

package/dist/metrics/statistical.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBrF,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD"}
+{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBlG,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,CAAC,EAAE,MAAM,CAAC;IACV,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,0FAA0F;AAC1F,MAAM,MAAM,eAAe,GAAG,mBAAmB,GAAG;IAClD,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/F,CAAC;AAEF,wDAAwD;AACxD,wBAAgB,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,IAAI,eAAe,CAEtF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKvE;AAID;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,eAAe,CA4CjF"}

package/dist/metrics/statistical.js

@@ -75,4 +75,65 @@ export function tokenUsage() {
         };
     });
 }
+/** Whether a statistical metric is a `passAtK()` one */
+export function isPassAtKMetric(metric) {
+    return 'passAtK' in metric && typeof metric.passAtK === 'object';
+}
+/**
+ * The chance that at least one of `k` trials drawn from `n`, `c` of which passed, passes:
+ * 1 - C(n - c, k) / C(n, k), computed as a product so large counts do not overflow
+ */
+export function passAtKEstimate(n, c, k) {
+    if (n - c < k)
+        return 1;
+    let allFail = 1;
+    for (let i = n - c + 1; i <= n; i++)
+        allFail *= 1 - k / i;
+    return 1 - allFail;
+}
+/**
+ * pass@k of a per-case metric over the suite's trials: for each case, the unbiased estimate of
+ * the chance that at least one of `k` trials scores `threshold` or more, averaged over the cases
+ * the metric scored at least one trial of. A trial that errored counts as a failure. Named
+ * `passAtK(<metric>,<k>)`. The suite must run at least `k` trials and have the metric.
+ */
+export function passAtK(metricName, opts) {
+    const { k, threshold = 1 } = opts;
+    if (!Number.isInteger(k) || k < 1) {
+        throw new Error(`passAtK k must be a positive integer, got ${k}`);
+    }
+    if (!Number.isFinite(threshold)) {
+        throw new Error(`passAtK threshold must be a finite number, got ${threshold}`);
+    }
+    const name = `passAtK(${metricName},${k})`;
+    const fn = createStatisticalFn(name, (results) => {
+        const values = [];
+        for (const result of results) {
+            const trials = result.trials ?? [result];
+            if (trials.length < k) {
+                throw new Error(`${name} needs at least ${k} trials per case, a case has ${trials.length}: run the suite with trials: ${k} or more`);
+            }
+            const scores = trials.map((trial) => trial.error === undefined ? trial.scores?.find((s) => s.name === metricName) : undefined);
+            if (scores.every((s) => s === undefined))
+                continue;
+            const passed = scores.filter((s) => s !== undefined && s.score >= threshold).length;
+            values.push(passAtKEstimate(trials.length, passed, k));
+        }
+        const metadata = { metric: metricName, k, threshold, cases: values.length };
+        if (values.length === 0) {
+            return {
+                name,
+                score: Number.NaN,
+                details: `no case has a trial scored on ${metricName}`,
+                metadata,
+            };
+        }
+        return { name, score: mean(values), values, metadata };
+    });
+    Object.defineProperty(fn, 'passAtK', {
+        value: Object.freeze({ metric: metricName, k, threshold }),
+        writable: false,
+    });
+    return fn;
+}
 //# sourceMappingURL=statistical.js.map

package/dist/metrics/statistical.js.map

@@ -1 +1 @@
-{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAcD,wDAAwD;AACxD,MAAM,UAAU,eAAe,CAAC,MAA2B;IACzD,OAAO,SAAS,IAAI,MAAM,IAAI,OAAQ,MAA0B,CAAC,OAAO,KAAK,QAAQ,CAAC;AACxF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC7D,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,OAAO,CAAC;AACrB,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,UAAkB,EAAE,IAAoB;IAC9D,MAAM,EAAE,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,UAAU,IAAI,CAAC,GAAG,CAAC;IAE3C,MAAM,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,CAAC,OAAyB,EAAE,EAAE;QACjE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAA2B,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,mBAAmB,CAAC,gCAAgC,MAAM,CAAC,MAAM,gCAAgC,CAAC,UAAU,CACpH,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CACzF,CAAC;YACF,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;gBAAE,SAAS;YACnD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,MAAM,CAAC;YACpF,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,OAAO,EAAE,iCAAiC,UAAU,EAAE;gBACtD,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;QAC1D,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IACH,OAAO,EAAqB,CAAC;AAC/B,CAAC"}

package/dist/metrics/tool-calls.d.ts

@@ -0,0 +1,43 @@
+import type { ExpectedToolCall } from '../schema.js';
+import type { EvalToolCall, MetricFn, MetricOptions } from './types.js';
+/** How the arguments of an expected tool call are compared with the actual call's */
+export type ToolArgumentsMatch = 'ignore' | 'subset' | 'exact';
+/** Which figure of the tool call match `toolCalls()` scores */
+export type ToolCallsScore = 'f1' | 'recall' | 'precision';
+/** How strictly `trajectory()` compares the order of the tool calls */
+export type TrajectoryOrder = 'exact' | 'in-order' | 'any-order';
+export interface ToolCallsOptions extends MetricOptions {
+    /**
+     * `'subset'` (default): the expected arguments must all be in the call with equal values,
+     * others may be there too. `'exact'`: the arguments must be equal. `'ignore'`: only the name
+     * counts. An expected call without `arguments` matches on its name alone
+     */
+    arguments?: ToolArgumentsMatch;
+    /** `'f1'` (default), `'recall'` (expected calls made) or `'precision'` (calls that were expected) */
+    score?: ToolCallsScore;
+}
+export interface TrajectoryOptions extends MetricOptions {
+    /**
+     * `'exact'` (default): 1 when the tool names are the expected ones in the same order, else 0.
+     * `'in-order'`: the longest common subsequence over the longer of the two sequences.
+     * `'any-order'`: the names both share, as multisets, over the longer of the two
+     */
+    order?: TrajectoryOrder;
+}
+/** Whether an actual call is the expected one, its arguments compared as `mode` says */
+export declare function toolCallMatches(expected: ExpectedToolCall, actual: EvalToolCall, mode?: ToolArgumentsMatch): boolean;
+/**
+ * Compares the tool calls of the run, over every turn, with the case's `expectedToolCalls` as
+ * multisets: each expected call pairs with at most one actual call. Scores the F1 of the match
+ * by default, or its recall or precision. A case expecting no calls scores 1 when the agent made
+ * none and 0 otherwise, and so does a case expecting calls when the agent made none.
+ */
+export declare function toolCalls(opts?: ToolCallsOptions): MetricFn;
+/** How well a sequence of tool names follows the expected one, 0 to 1 */
+export declare function trajectoryScore(expected: readonly string[], actual: readonly string[], order: TrajectoryOrder): number;
+/**
+ * Compares the sequence of tools the agent called, over every turn, with the names of the
+ * case's `expectedToolCalls`. Arguments are not looked at: pair it with `toolCalls()` for them.
+ */
+export declare function trajectory(opts?: TrajectoryOptions): MetricFn;
+//# sourceMappingURL=tool-calls.d.ts.map

package/dist/metrics/tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../src/metrics/tool-calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AASrE,qFAAqF;AACrF,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/D,+DAA+D;AAC/D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE3D,uEAAuE;AACvE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD;;;;OAIG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,qGAAqG;IACrG,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAwBD,wFAAwF;AACxF,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE,kBAA6B,GAClC,OAAO,CAMT;AAYD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,QAAQ,CAqC3D;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,EAAE,eAAe,GACrB,MAAM,CAWR;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,QAAQ,CAsB7D"}