@smthrs/evals 1.0.0-rc.6 → 1.0.0-rc.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +7 -3
  2. package/dist/cjs/Baseline.d.ts +3 -1
  3. package/dist/cjs/Baseline.d.ts.map +1 -1
  4. package/dist/cjs/Baseline.js +3 -1
  5. package/dist/cjs/Baseline.js.map +2 -2
  6. package/dist/cjs/Regression.d.ts.map +1 -1
  7. package/dist/cjs/Regression.js +3 -2
  8. package/dist/cjs/Regression.js.map +2 -2
  9. package/dist/cjs/Runner.d.ts.map +1 -1
  10. package/dist/cjs/Runner.js +26 -3
  11. package/dist/cjs/Runner.js.map +2 -2
  12. package/dist/cjs/Trials.d.ts.map +1 -1
  13. package/dist/cjs/Trials.js +4 -4
  14. package/dist/cjs/Trials.js.map +2 -2
  15. package/dist/esm/Baseline.d.ts +3 -1
  16. package/dist/esm/Baseline.d.ts.map +1 -1
  17. package/dist/esm/Baseline.js +7 -2
  18. package/dist/esm/Baseline.js.map +1 -1
  19. package/dist/esm/Regression.d.ts.map +1 -1
  20. package/dist/esm/Regression.js +7 -2
  21. package/dist/esm/Regression.js.map +1 -1
  22. package/dist/esm/Runner.d.ts.map +1 -1
  23. package/dist/esm/Runner.js +29 -4
  24. package/dist/esm/Runner.js.map +1 -1
  25. package/dist/esm/Trials.d.ts.map +1 -1
  26. package/dist/esm/Trials.js +5 -4
  27. package/dist/esm/Trials.js.map +1 -1
  28. package/docs/README.md +1 -3
  29. package/docs/installation.md +4 -18
  30. package/docs/troubleshooting.md +5 -0
  31. package/package.json +4 -4
  32. package/src/Baseline.ts +9 -2
  33. package/src/Regression.ts +9 -2
  34. package/src/Runner.ts +33 -3
  35. package/src/Trials.ts +5 -4
package/dist/esm/Runner.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Runner.js","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,MAAM,MAAM,wBAAwB,CAAA;AAEhD,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAuD,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAEjD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AA0JrC;;;;;;;;;GASG;AACH,MAAM,OAAO,MAAO,SAAQ,OAAO,CAAC,OAAO,EAA4B,CAAC,oBAAoB,CAAC;CAAG;AAEhG,yDAAyD;AACzD,MAAM,eAAe,GAAG,IAAI,CAAA;AAE5B,MAAM,aAAa,GAAG,CAAC,MAAc,EAAU,EAAE,CAC/C,MAAM,CAAC,MAAM,IAAI,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,aAAa,CAAA;AAE9F,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAA;AAE1E,+EAA+E;AAC/E,6EAA6E;AAC7E,yDAAyD;AACzD,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA;AAEtE,MAAM,KAAK,GAAG,CAAC,OAAgB,EAAU,EAAE,CACzC,WAAW,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;AAElF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAE,MAAc,EAAE,EAAU,EAAe,EAAE;IACtF,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,cAAc;QACpB,MAAM;QACN,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,6EAA6E;AAC7E,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,UAAU,IAAI,IAAI,CAAA;AAE7D,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,+EAA+E;AAC/E,uEAAuE;AACvE,kDAAkD;AAClD,MAAM,SAAS,GAAG,CAAC,SAAe,EAAE,KAA2B,EAAa,EAAE;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,MAAM,YAAY,SAAS;QAChC,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YAClG,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC7C,KAAK,EAAE,MAAM;SACd,CAAC;QACF,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAClG,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,KAAK,EAAE,MAAM;SACd,CAAC,CAAA;AACN,CAAC,CAAA;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,MAAM,OAAO,GAAG,CAAC,QAA6B,EAAE,SAAe,EAA6B,EAAE,CAC5F,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CACjE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAc,EAAE,CAAC,CAAC;IACrC,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,SAAS,EAAE,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;IACjF,YAAY,EAAE,EAAE;CACjB,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAC/G,EACD,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CACvE,CAAA;AAEH,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,cAAc,GAAG,CAAC,SAAe,EAAE,OAAgB,EAAsC,EAAE;IAC/F,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;IAC/F,OAAO,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,CAAC,WAAW,CAAC,EAAE,CAAA;AACvF,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,SAAe,EAAE,SAAoB,EAAE,OAAgB,EAAgB,EAAE,CAAC,CAAC;IAC7F,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,OAAO,EAAE,SAAS,CAAC,OAAO;IAC1B,OAAO,EAAE;QACP,GAAG,eAAe,CAAC;YACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACvE,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B;IACD,KAAK,EAAE;QACL,KAAK,EAAE,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,GAAG,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACvF,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B;CACF,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,CAAC,OAAqB,EAAE,MAAwB,EAAE,EAAU,EAAe,EAAE;IAClG,MAAM,MAAM,GAAG,MAKd,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAChD,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,2CAA2C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5B,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAC5B,OAAO,YAAY,CACjB,OAAO,EACP,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC;YACvB,CAAC,CAAC,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sDAAsD,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;IAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,qCAAqC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACnG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO;QACb,KAAK;QACL,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,CAAC,GAAa,EAAmC,EAAE,CACvE,GAAG,CAAC,KAAK,CAAC,IAAI,CACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAA+B,EAAE,CAAC,CAAC;IACnD,GAAG,GAAG,CAAC,WAAW;IAClB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,MAAM,CAAC,KAAK;IACnB,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACjE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3D,EAAE,EAAE,GAAG,CAAC,EAAE;CACX,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAC1D,CACF,CAAA;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAqB,EAAE;IAC/C,MAAM,kBAAkB,GAAG,CACzB,IAA6B,EAC7B,OAAuD,EACZ,EAAE,CAC7C,MAAM,CAAC,OAAO,CACZ,IAAI,EACJ,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EACxG,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,EAAE,CAC3C,CAAA;IACH,OAAO;QACL,kBAAkB;QAClB,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAC1B,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;KAC/G,CAAA;AACH,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAwB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;AASnF,MAAM,SAAS,GAAG,CAChB,KAAgC,EAChC,KAAY,EACZ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAgB,EAIlD,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2EAA2E;IAC3E,2EAA2E;IAC3E,wBAAwB;IACxB,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QACrD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;QACtC,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAE,CAAA;QACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,CACtD,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,MAAM;YACpC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;YACxE,CAAC,CAAC,EAAE,CACP,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnC,UAAU,EACV,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,CAC5B,QAAQ,CAAC,MAAM,CACb,OAAO,CAAC,OAAO,CAAC,QAAQ,EACxB,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAC7B,CAAC,IAAI,CACJ,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EACxD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,IAAI,SAAS,CAAC;QACZ,IAAI,EAAE,eAAe;QACrB,OAAO,EAAE,sCAAsC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACvE,IAAI,EAAE,YAAY,YAAY,YAAY;QAC1C,KAAK;KACN,CAAC,CACH,CACF,EACH,EAAE,WAAW,EAAE,CAChB,CAAA;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAA2B,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAoB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,QAAQ,CAChB,KAAK,CAAC,IAAI,EACV,KAAK,EACL,QAAQ,EACR,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAC5B,GAAG,KAAK,EAAE,CACX;QACD,WAAW,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACxF,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxE,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC,CAAA;IACH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;AAC3B,CAAC,CAAC,CAAA;AAEJ,MAAM,aAAa,GAAG,CACpB,IAA6B,EAC7B,KAA2B,EACqB,EAAE,CAClD,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;AAE9E,MAAM,eAAe,GAAG,CACtB,IAA6B,EAC7B,MAAwB,EACxB,kBAAuE,EACvE,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAC3G,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAC1B,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,QAAQ;QAC/B,WAAW;KACZ,CAAC,CAAC,CACJ,CACF,CACF,CACF,CAAA;IACD,IAAI,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,oCAAoC,YAAY,CAAC,MAAM,gBAAgB,IAAI,CAAC,MAAM,uEAAuE;YAC3J,IAAI,EAAE,oBAAoB;SAC3B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC,CAAC,CAAA;IAC5F,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAA;IACtD,IAAI,OAA0E,CAAA;IAC9E,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACxD,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EACL,wDAAwD,MAAM,CAAC,QAAQ,aAAa,QAAQ,oBAAoB,KAAK,EAAE;gBACzH,IAAI,EAAE,sBAAsB,KAAK,GAAG;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAA;QAC1E,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAA;IACrD,CAAC;IACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAE,CAAC,QAAQ,CAAA;QACvD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,sDAAsD,OAAO,CAAC,QAAQ,qBAAqB,OAAO,CAAC,KAAK,SAAS,eAAe,cAAc,eAAe,aAAa;YAC5K,IAAI,EAAE,sBAAsB,OAAO,CAAC,KAAK,GAAG;SAC7C,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,CAAA;AACzD,CAAC,CAAC,CAAA;AAEJ,MAAM,cAAc,GAAG,CACrB,IAA6B,EAC7B,QAAqC,EACrC,MAAwB,EACxB,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,aAAa,EAAE,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QAC/E,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAE,CAAA;YACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAE,CAAA;YAC/B,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EACL,0CAA0C,KAAK,CAAC,IAAI,UAAU,MAAM,CAAC,IAAI,yBAAyB,GAAG,CAAC,WAAW,CAAC,aAAa,gBAC7H,KAAK,CAAC,MAAM,CAAC,OAAO,CACtB,iGAAiG;gBACnG,IAAI,EAAE,UAAU;aACjB,CAAC,CACH,CAAA;QACH,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CACtF,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CACzD,CAAA;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,yBAAyB,OAAO,CAAC,MAAM,qBAAqB,IAAI,CAAC,MAAM,6EAA6E;YACtJ,IAAI,EAAE,UAAU;SACjB,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAC,CAAA;AAEJ,MAAM,KAAK,GAAG,CACZ,KAAgC,EAChC,KAAY,EACZ,MAAwB,EACxB,OAAqB,EACgC,EAAE,CACvD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACvC,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAA;IACpD,MAAM,QAAQ,GAAG,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAA;IACrF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,KAAK,SAAS;QACrD,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;QACjD,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAA;IAC9D,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAA;IACtB,MAAM,YAAY,GAAuB,EAAE,CAAA;IAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAA;QAC9B,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,KAAK,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAClG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,QAAQ,KAAK,oBAAoB;oBACxC,CAAC,CAAC,qCACA,IAAI,CAAC,KAAK,CAAE,CAAC,QACf,oBAAoB,MAAM,CAAC,aAAa,iBAAiB,MAAM,CAAC,SAAS,eAAe,KAAK,sBAC3F,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,GAAG;oBAChC,CAAC,CAAC,6CAA6C,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,aAAa,eAAe,KAAK,eACnH,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,8CAA8C;gBAC7E,IAAI,EAAE,GAAG,QAAQ,IAAI,KAAK,GAAG;aAC9B,CAAC,CACH,CAAA;QACH,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAA;IACpD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAA;;YAChE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAChC,GAAG,UAAU;QACb,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE;KAChD,CAAC,CAAC,CAAA;AACL,CAAC,CAAC,CAAA;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,CACjB,KAAY,EACZ,OAAmB,EACgC,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,YAAY,CAAA;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,8CAA8C;YACvD,IAAI,EAAE,eAAe;SACtB,CAAC,CACH,CAAA;IACH,CAAC;IACD,IACE,CAAC,gDAAgD,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAClE,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAAE,EAC/C,CAAC;QACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EACL,+FAA+F,OAAO,CAAC,EAAE,GAAG;YAC9G,IAAI,EAAE,YAAY;SACnB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;IAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAA;IACxB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,oFAAoF;YAC7F,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAA;IACrB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAA;IAC9C,2EAA2E;IAC3E,mFAAmF;IACnF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,EAAE,CAC7D,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE;YAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,OAAO,EAAE,KAAK;SACf,CAAC,CAAA;QACF,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;YACxC,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,EAAE;YACF,KAAK;YACL,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;SACjE,CAAC,CAAA;IACJ,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAmC,EAAE,CAAA;IACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACrC,KAAK,CAAC,KAAK,CAAE,CACd,CAAA;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACtC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,IAAI,WAAW,CAAC,KAAK,KAAK,CAAC,CAAC,CACrG,CAAA;QACD,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAA;QACjC,OAAO;YACL,GAAG,QAAQ,CAAC,CAAC,CAAE;YACf,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,KAAK;YACrE,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;YACjE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;SACpC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,OAAO;QACL,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;QACpE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrC,CAAC;KACF,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAA;AAEnG;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,SAAS,GAAwB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE;IACpE,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAA;IACjH,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE,CAC3B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC;SACzD,CAAC,CAAC,CAAC;KACP,CAAA;AACH,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n & {\n readonly case: string\n readonly scorer: string\n readonly scorerName?: string | undefined\n readonly stepKey: string\n readonly at: string\n }\n & (\n | {\n readonly kind: \"score\"\n readonly score: number\n readonly reason?: string | undefined\n readonly meta?: unknown\n }\n | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n readonly case: string\n readonly stepKey: string\n readonly binding: Binding\n readonly input: {\n readonly input: unknown\n readonly output: unknown\n readonly groundTruth?: unknown\n readonly context?: unknown\n readonly latencyMs?: number\n }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n readonly runBatch: (\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n readonly runBatchCorrelated?:\n | ((\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n readonly identity: string\n readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n readonly case: string\n readonly execution?: Execution | undefined\n readonly error?: EvalError | undefined\n readonly observations: ReadonlyArray<Observation>\n readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n readonly runId: string\n readonly suite: string\n readonly cases: ReadonlyArray<CaseResult>\n readonly observations: ReadonlyArray<Observation>\n readonly trials?: Trials.Aggregate | undefined\n readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n readonly scorer?: ScoreBatchRunner | undefined\n readonly runId: string\n readonly sampleId?: string | undefined\n readonly trials?: number | undefined\n readonly k?: number | undefined\n readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n return {\n case: request.case,\n scorer: scorerKeyOf(request.binding),\n scorerName: scorerNameOf(request.binding),\n stepKey: request.stepKey,\n kind: \"inconclusive\",\n reason,\n at\n }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n const reason = Cause.squash(cause)\n return reason instanceof EvalError\n ? new EvalError({\n code: reason.code,\n message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n path: reason.path ?? casePath(suiteCase.name),\n cause: reason\n })\n : new EvalError({\n code: \"executor\",\n message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n path: casePath(suiteCase.name),\n cause: reason\n })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n Effect.map((execution): CaseResult => ({\n case: suiteCase.name,\n execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n observations: []\n })),\n Effect.catchCause((cause) =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n ),\n Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n case: suiteCase.name,\n stepKey: execution.stepKey,\n binding: {\n ...structuredClone({\n sampling: binding.sampling,\n ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n ...(binding.context === undefined ? {} : { context: binding.context })\n }),\n scorer: binding.scorer,\n appliesTo: binding.appliesTo\n },\n input: {\n input: structuredClone(suiteCase.input),\n output: execution.output,\n ...(groundTruthFor(suiteCase, binding)),\n ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n latencyMs: execution.latencyMs\n }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n const source = result as {\n readonly kind: unknown\n readonly score?: unknown\n readonly reason?: unknown\n readonly meta?: unknown\n }\n const kind = source.kind\n if (kind !== \"score\" && kind !== \"inconclusive\") {\n return inconclusive(\n request,\n boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n at\n )\n }\n const reason = source.reason\n if (kind === \"inconclusive\") {\n return inconclusive(\n request,\n typeof reason === \"string\" && reason.length > 0\n ? boundedReason(reason)\n : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n at\n )\n }\n const score = source.score\n if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n return inconclusive(\n request,\n boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n at\n )\n }\n const meta = source.meta\n return {\n case: request.case,\n scorer: scorerKeyOf(request.binding),\n scorerName: scorerNameOf(request.binding),\n stepKey: request.stepKey,\n kind: \"score\",\n score,\n ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n ...(meta === undefined ? {} : { meta }),\n at\n }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n job.score.pipe(\n Effect.flatMap(Scorer.validate),\n Effect.map((result): ScoreStore.ScoreObservation => ({\n ...job.observation,\n kind: \"score\",\n score: result.score,\n ...(result.reason === undefined ? {} : { reason: result.reason }),\n ...(result.meta === undefined ? {} : { meta: result.meta }),\n at: job.at\n })),\n Effect.catchCause((cause) =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n )\n )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n const runBatchCorrelated = (\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n Effect.forEach(\n jobs,\n (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n { concurrency: options?.concurrency ?? 1 }\n )\n return {\n runBatchCorrelated,\n runBatch: (jobs, options) =>\n runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n readonly at: string\n readonly runId: string\n readonly sampleId: string\n readonly concurrency: number\n}\n\nconst buildJobs = (\n cases: ReadonlyArray<CaseResult>,\n suite: Suite,\n { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n EvalError\n> =>\n Effect.gen(function*() {\n // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n // two arrays are index-aligned by construction; nothing here has to look a\n // case back up by name.\n const candidates = cases.flatMap((caseResult, index) => {\n const execution = caseResult.execution\n if (execution === undefined) return []\n const suiteCase = suite.cases[index]!\n return suite.bindings.flatMap((binding, bindingIndex) =>\n binding.appliesTo === execution.target\n ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n : []\n )\n })\n const sampled = yield* Effect.forEach(\n candidates,\n ({ request, bindingIndex }) =>\n Sampling.decide(\n request.binding.sampling,\n request.stepKey,\n scorerKeyOf(request.binding)\n ).pipe(\n Effect.map((selected) => selected ? request : undefined),\n Effect.mapError((cause) =>\n new EvalError({\n code: \"invalid_suite\",\n message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n path: `bindings[${bindingIndex}].sampling`,\n cause\n })\n )\n ),\n { concurrency }\n )\n const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n identity: tupleKey(\n suite.name,\n runId,\n sampleId,\n request.case,\n request.stepKey,\n scorerKeyOf(request.binding),\n `${index}`\n ),\n observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n at: Date.parse(at)\n }))\n return { requests, jobs }\n })\n\nconst batchFallback = (\n jobs: ReadonlyArray<ScoreJob>,\n cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n jobs: ReadonlyArray<ScoreJob>,\n scorer: ScoreBatchRunner,\n runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n Effect.gen(function*() {\n const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n Effect.catchCause((cause) =>\n batchFallback(jobs, cause).pipe(\n Effect.map((observations) =>\n observations.map((observation, index) => ({\n identity: jobs[index]!.identity,\n observation\n }))\n )\n )\n )\n )\n if (batchResults.length !== jobs.length) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n path: \"runBatchCorrelated\"\n })\n )\n }\n const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n const byIdentity = new Map<string, ScoreObservation>()\n let unknown: { readonly identity: string; readonly index: number } | undefined\n for (const [index, result] of batchResults.entries()) {\n const jobIndex = jobIndexByIdentity.get(result.identity)\n if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n path: `runBatchCorrelated[${index}]`\n })\n )\n }\n if (jobIndex === undefined) unknown = { identity: result.identity, index }\n byIdentity.set(result.identity, result.observation)\n }\n if (unknown !== undefined) {\n const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n const missingIdentity = jobs[missingJobIndex]!.identity\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n path: `runBatchCorrelated[${unknown.index}]`\n })\n )\n }\n return jobs.map((job) => byIdentity.get(job.identity)!)\n })\n\nconst scoreOrderOnly = (\n jobs: ReadonlyArray<ScoreJob>,\n requests: ReadonlyArray<ScoreRequest>,\n scorer: ScoreBatchRunner,\n { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n Effect.gen(function*() {\n const pairIndexes = new Map<string, number>()\n for (const [index, job] of jobs.entries()) {\n const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n const previous = pairIndexes.get(pair)\n if (previous !== undefined) {\n const first = requests[previous]!\n const second = requests[index]!\n return yield* Effect.fail(\n new EvalError({\n code: \"ambiguous_score_job\",\n message:\n `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n label(second.binding)\n }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n path: \"runBatch\"\n })\n )\n }\n pairIndexes.set(pair, index)\n }\n const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n Effect.catchCause((cause) => batchFallback(jobs, cause))\n )\n if (results.length !== jobs.length) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n path: \"runBatch\"\n })\n )\n }\n return results\n })\n\nconst score = (\n cases: ReadonlyArray<CaseResult>,\n suite: Suite,\n scorer: ScoreBatchRunner,\n context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n Effect.gen(function*() {\n const { requests, jobs } = yield* buildJobs(cases, suite, context)\n if (requests.length === 0) return cases\n const runBatchCorrelated = scorer.runBatchCorrelated\n const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n const results = yield* runBatchCorrelated === undefined\n ? scoreOrderOnly(jobs, requests, scorer, context)\n : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n const { at } = context\n const observations: Array<Observation> = []\n for (const [index, request] of requests.entries()) {\n const result = results[index]!\n if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message: protocol === \"runBatchCorrelated\"\n ? `Correlated scorer batch identity '${\n jobs[index]!.identity\n }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n scorerKeyOf(request.binding)\n }' at step '${request.stepKey}'`\n : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n scorerKeyOf(request.binding)\n }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n path: `${protocol}[${index}]`\n })\n )\n }\n observations.push(observationFor(request, result, at))\n }\n const byCase = new Map<string, Array<Observation>>()\n for (const observation of observations) {\n const bucket = byCase.get(observation.case)\n if (bucket === undefined) byCase.set(observation.case, [observation])\n else bucket.push(observation)\n }\n return cases.map((caseResult) => ({\n ...caseResult,\n observations: byCase.get(caseResult.case) ?? []\n }))\n })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n suite: Suite,\n options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n Effect.gen(function*() {\n const executor = yield* CaseExecutor\n const injected = yield* Effect.serviceOption(Runner)\n const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n const atMillis = Date.parse(options.at)\n if (options.runId.trim().length === 0) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_run_options\",\n message: \"Deterministic runs require a non-empty runId\",\n path: \"options.runId\"\n })\n )\n }\n if (\n !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n !Number.isFinite(atMillis) ||\n new Date(atMillis).toISOString() !== options.at\n ) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_run_options\",\n message:\n `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n path: \"options.at\"\n })\n )\n }\n const n = options.trials ?? suite.trials ?? 1\n const k = options.k ?? 1\n if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_trials\",\n message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n path: \"options.trials\"\n })\n )\n }\n const at = options.at\n const runId = options.runId\n const sampleId = options.sampleId ?? \"default\"\n // Await each batch before starting the next: concurrency within a trial is\n // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n Effect.gen(function*() {\n const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n runCase(executor, suiteCase), {\n concurrency: suite.concurrency,\n discard: false\n })\n return yield* score(cases, suite, scorer, {\n concurrency: suite.concurrency,\n at,\n runId,\n sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n })\n }), { concurrency: 1 })\n const outcomes: Record<string, Array<boolean>> = {}\n const cases = suite.cases.map((suiteCase, index) => {\n const attempts = batches.map((batch) =>\n batch[index]!\n )\n const passed = attempts.map((attempt) =>\n attempt.error === undefined && attempt.observations.length > 0 &&\n attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n )\n outcomes[suiteCase.name] = passed\n return {\n ...attempts[0]!,\n error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n observations: attempts.flatMap((attempt) => attempt.observations),\n trials: Trials.summarize(passed, k)\n }\n })\n return {\n runId,\n suite: suite.name,\n cases,\n observations: cases.flatMap((caseResult) => caseResult.observations),\n trials: Trials.aggregate(outcomes, k),\n k\n }\n }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n return {\n runBatch: () => Effect.fail(unavailable),\n runBatchCorrelated: (jobs) =>\n Effect.succeed(jobs.map((job) => ({\n identity: job.identity,\n observation: ScorerRunner.inconclusive(job, unavailable)\n })))\n }\n})\n"]}
1
+ {"version":3,"file":"Runner.js","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,MAAM,MAAM,wBAAwB,CAAA;AAEhD,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAuD,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAEjD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AA0JrC;;;;;;;;;GASG;AACH,MAAM,OAAO,MAAO,SAAQ,OAAO,CAAC,OAAO,EAA4B,CAAC,oBAAoB,CAAC;CAAG;AAEhG,yDAAyD;AACzD,MAAM,eAAe,GAAG,IAAI,CAAA;AAE5B,MAAM,aAAa,GAAG,CAAC,MAAc,EAAU,EAAE,CAC/C,MAAM,CAAC,MAAM,IAAI,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,aAAa,CAAA;AAE9F,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAA;AAE1E,+EAA+E;AAC/E,6EAA6E;AAC7E,yDAAyD;AACzD,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA;AAEtE,MAAM,KAAK,GAAG,CAAC,OAAgB,EAAU,EAAE,CACzC,WAAW,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;AAElF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAE,MAAc,EAAE,EAAU,EAAe,EAAE;IACtF,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,cAAc;QACpB,MAAM;QACN,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,6EAA6E;AAC7E,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,UAAU,IAAI,IAAI,CAAA;AAE7D,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,+EAA+E;AAC/E,uEAAuE;AACvE,kDAAkD;AAClD,MAAM,SAAS,GAAG,CAAC,SAAe,EAAE,KAA2B,EAAa,EAAE;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,MAAM,YAAY,SAAS;QAChC,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YAClG,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC7C,KAAK,EAAE,MAAM;SACd,CAAC;QACF,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAClG,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,KAAK,EAAE,MAAM;SACd,CAAC,CAAA;AACN,CAAC,CAAA;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,2EAA2E;AAC3E,6EAA6E;AAC7E,yCAAyC;AACzC,MAAM,kBAAkB,GAAG,CAAC,SAAe,EAAE,SAAoB,EAAyB,EAAE;IAC1F,MAAM,KAAK,GACT,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAA;IACtE,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,IAAI,SAAS,CAAC;YACnB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,qDAAqD,OAAO,KAAK,CAAC,OAAO,GAAG;YACrF,IAAI,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU;SAC5C,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACpG,OAAO,IAAI,SAAS,CAAC;YACnB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,2EACP,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,SAC/E,GAAG;YACH,IAAI,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;SAC9C,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAC,QAA6B,EAAE,SAAe,EAA6B,EAAE,CAC5F,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CACjE,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;IAC3B,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAC1D,OAAO,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;AACrF,CAAC,CAAC,EACF,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAc,EAAE,CAAC,CAAC;IACrC,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,SAAS,EAAE,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;IACjF,YAAY,EAAE,EAAE;CACjB,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAC/G,EACD,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CACvE,CAAA;AAEH,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,cAAc,GAAG,CAAC,SAAe,EAAE,OAAgB,EAAsC,EAAE;IAC/F,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;IAC/F,OAAO,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,CAAC,WAAW,CAAC,EAAE,CAAA;AACvF,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,SAAe,EAAE,SAAoB,EAAE,OAAgB,EAAgB,EAAE,CAAC,CAAC;IAC7F,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,OAAO,EAAE,SAAS,CAAC,OAAO;IAC1B,OAAO,EAAE;QACP,GAAG,eAAe,CAAC;YACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACvE,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B;IACD,KAAK,EAAE;QACL,KAAK,EAAE,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,GAAG,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACvF,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B;CACF,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,CAAC,OAAqB,EAAE,MAAwB,EAAE,EAAU,EAAe,EAAE;IAClG,MAAM,MAAM,GAAG,MAKd,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAChD,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,2CAA2C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5B,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAC5B,OAAO,YAAY,CACjB,OAAO,EACP,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC;YACvB,CAAC,CAAC,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sDAAsD,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;IAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,qCAAqC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACnG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO;QACb,KAAK;QACL,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,CAAC,GAAa,EAAmC,EAAE,CACvE,GAAG,CAAC,KAAK,CAAC,IAAI,CACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAA+B,EAAE,CAAC,CAAC;IACnD,GAAG,GAAG,CAAC,WAAW;IAClB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,MAAM,CAAC,KAAK;IACnB,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACjE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3D,EAAE,EAAE,GAAG,CAAC,EAAE;CACX,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAC1D,CACF,CAAA;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAqB,EAAE;IAC/C,MAAM,kBAAkB,GAAG,CACzB,IAA6B,EAC7B,OAAuD,EACZ,EAAE,CAC7C,MAAM,CAAC,OAAO,CACZ,IAAI,EACJ,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EACxG,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,EAAE,CAC3C,CAAA;IACH,OAAO;QACL,kBAAkB;QAClB,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAC1B,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;KAC/G,CAAA;AACH,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAwB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;AASnF,MAAM,SAAS,GAAG,CAChB,KAAgC,EAChC,KAAY,EACZ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAgB,EAIlD,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2EAA2E;IAC3E,2EAA2E;IAC3E,wBAAwB;IACxB,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QACrD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;QACtC,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAE,CAAA;QACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,CACtD,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,MAAM;YACpC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;YACxE,CAAC,CAAC,EAAE,CACP,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnC,UAAU,EACV,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,CAC5B,QAAQ,CAAC,MAAM,CACb,OAAO,CAAC,OAAO,CAAC,QAAQ,EACxB,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAC7B,CAAC,IAAI,CACJ,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EACxD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,IAAI,SAAS,CAAC;QACZ,IAAI,EAAE,eAAe;QACrB,OAAO,EAAE,sCAAsC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACvE,IAAI,EAAE,YAAY,YAAY,YAAY;QAC1C,KAAK;KACN,CAAC,CACH,CACF,EACH,EAAE,WAAW,EAAE,CAChB,CAAA;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAA2B,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAoB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,QAAQ,CAChB,KAAK,CAAC,IAAI,EACV,KAAK,EACL,QAAQ,EACR,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAC5B,GAAG,KAAK,EAAE,CACX;QACD,WAAW,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACxF,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxE,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC,CAAA;IACH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;AAC3B,CAAC,CAAC,CAAA;AAEJ,MAAM,aAAa,GAAG,CACpB,IAA6B,EAC7B,KAA2B,EACqB,EAAE,CAClD,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;AAE9E,MAAM,eAAe,GAAG,CACtB,IAA6B,EAC7B,MAAwB,EACxB,kBAAuE,EACvE,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAC3G,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAC1B,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,QAAQ;QAC/B,WAAW;KACZ,CAAC,CAAC,CACJ,CACF,CACF,CACF,CAAA;IACD,IAAI,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,oCAAoC,YAAY,CAAC,MAAM,gBAAgB,IAAI,CAAC,MAAM,uEAAuE;YAC3J,IAAI,EAAE,oBAAoB;SAC3B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC,CAAC,CAAA;IAC5F,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAA;IACtD,IAAI,OAA0E,CAAA;IAC9E,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACxD,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EACL,wDAAwD,MAAM,CAAC,QAAQ,aAAa,QAAQ,oBAAoB,KAAK,EAAE;gBACzH,IAAI,EAAE,sBAAsB,KAAK,GAAG;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAA;QAC1E,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAA;IACrD,CAAC;IACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAE,CAAC,QAAQ,CAAA;QACvD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,sDAAsD,OAAO,CAAC,QAAQ,qBAAqB,OAAO,CAAC,KAAK,SAAS,eAAe,cAAc,eAAe,aAAa;YAC5K,IAAI,EAAE,sBAAsB,OAAO,CAAC,KAAK,GAAG;SAC7C,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,CAAA;AACzD,CAAC,CAAC,CAAA;AAEJ,MAAM,cAAc,GAAG,CACrB,IAA6B,EAC7B,QAAqC,EACrC,MAAwB,EACxB,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,aAAa,EAAE,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QAC/E,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAE,CAAA;YACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAE,CAAA;YAC/B,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EACL,0CAA0C,KAAK,CAAC,IAAI,UAAU,MAAM,CAAC,IAAI,yBAAyB,GAAG,CAAC,WAAW,CAAC,aAAa,gBAC7H,KAAK,CAAC,MAAM,CAAC,OAAO,CACtB,iGAAiG;gBACnG,IAAI,EAAE,UAAU;aACjB,CAAC,CACH,CAAA;QACH,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CACtF,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CACzD,CAAA;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,yBAAyB,OAAO,CAAC,MAAM,qBAAqB,IAAI,CAAC,MAAM,6EAA6E;YACtJ,IAAI,EAAE,UAAU;SACjB,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAC,CAAA;AAEJ,MAAM,KAAK,GAAG,CACZ,KAAgC,EAChC,KAAY,EACZ,MAAwB,EACxB,OAAqB,EACgC,EAAE,CACvD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACvC,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAA;IACpD,MAAM,QAAQ,GAAG,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAA;IACrF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,KAAK,SAAS;QACrD,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;QACjD,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAA;IAC9D,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAA;IACtB,MAAM,YAAY,GAAuB,EAAE,CAAA;IAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAA;QAC9B,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,KAAK,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAClG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,QAAQ,KAAK,oBAAoB;oBACxC,CAAC,CAAC,qCACA,IAAI,CAAC,KAAK,CAAE,CAAC,QACf,oBAAoB,MAAM,CAAC,aAAa,iBAAiB,MAAM,CAAC,SAAS,eAAe,KAAK,sBAC3F,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,GAAG;oBAChC,CAAC,CAAC,6CAA6C,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,aAAa,eAAe,KAAK,eACnH,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,8CAA8C;gBAC7E,IAAI,EAAE,GAAG,QAAQ,IAAI,KAAK,GAAG;aAC9B,CAAC,CACH,CAAA;QACH,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAA;IACpD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAA;;YAChE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAChC,GAAG,UAAU;QACb,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE;KAChD,CAAC,CAAC,CAAA;AACL,CAAC,CAAC,CAAA;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,CACjB,KAAY,EACZ,OAAmB,EACgC,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,YAAY,CAAA;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,8CAA8C;YACvD,IAAI,EAAE,eAAe;SACtB,CAAC,CACH,CAAA;IACH,CAAC;IACD,IACE,CAAC,gDAAgD,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAClE,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAAE,EAC/C,CAAC;QACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EACL,+FAA+F,OAAO,CAAC,EAAE,GAAG;YAC9G,IAAI,EAAE,YAAY;SACnB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;IAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAA;IACxB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,oFAAoF;YAC7F,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAA;IACrB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAA;IAC9C,2EAA2E;IAC3E,mFAAmF;IACnF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,EAAE,CAC7D,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE;YAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,OAAO,EAAE,KAAK;SACf,CAAC,CAAA;QACF,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;YACxC,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,EAAE;YACF,KAAK;YACL,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;SACjE,CAAC,CAAA;IACJ,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;IACzB,MAAM,QAAQ,GAA4C,EAAE,CAAA;IAC5D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACrC,KAAK,CAAC,KAAK,CAAE,CACd,CAAA;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACtC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,IAAI,WAAW,CAAC,KAAK,KAAK,CAAC,CAAC,CACrG,CAAA;QACD,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QACvC,OAAO;YACL,GAAG,QAAQ,CAAC,CAAC,CAAE;YACf,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,KAAK;YACrE,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;YACjE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;SACpC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,OAAO;QACL,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;QACpE,gFAAgF;QAChF,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;KACF,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAA;AAEnG;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,SAAS,GAAwB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE;IACpE,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAA;IACjH,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE,CAC3B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC;SACzD,CAAC,CAAC,CAAC;KACP,CAAA;AACH,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n & {\n readonly case: string\n readonly scorer: string\n readonly scorerName?: string | undefined\n readonly stepKey: string\n readonly at: string\n }\n & (\n | {\n readonly kind: \"score\"\n readonly score: number\n readonly reason?: string | undefined\n readonly meta?: unknown\n }\n | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n readonly case: string\n readonly stepKey: string\n readonly binding: Binding\n readonly input: {\n readonly input: unknown\n readonly output: unknown\n readonly groundTruth?: unknown\n readonly context?: unknown\n readonly latencyMs?: number\n }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n readonly runBatch: (\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n readonly runBatchCorrelated?:\n | ((\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n readonly identity: string\n readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n readonly case: string\n readonly execution?: Execution | undefined\n readonly error?: EvalError | undefined\n readonly observations: ReadonlyArray<Observation>\n readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n readonly runId: string\n readonly suite: string\n readonly cases: ReadonlyArray<CaseResult>\n readonly observations: ReadonlyArray<Observation>\n readonly trials?: Trials.Aggregate | undefined\n readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n readonly scorer?: ScoreBatchRunner | undefined\n readonly runId: string\n readonly sampleId?: string | undefined\n readonly trials?: number | undefined\n readonly k?: number | undefined\n readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n return {\n case: request.case,\n scorer: scorerKeyOf(request.binding),\n scorerName: scorerNameOf(request.binding),\n stepKey: request.stepKey,\n kind: \"inconclusive\",\n reason,\n at\n }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n const reason = Cause.squash(cause)\n return reason instanceof EvalError\n ? new EvalError({\n code: reason.code,\n message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n path: reason.path ?? casePath(suiteCase.name),\n cause: reason\n })\n : new EvalError({\n code: \"executor\",\n message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n path: casePath(suiteCase.name),\n cause: reason\n })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\n// An executor written in plain JavaScript can return any shape, so the two\n// fields every scorer and baseline reads are checked here, where the failure\n// can still name the case and the field.\nconst malformedExecution = (suiteCase: Case, execution: Execution): EvalError | undefined => {\n const shape: { readonly stepKey?: unknown; readonly latencyMs?: unknown } =\n typeof execution === \"object\" && execution !== null ? execution : {}\n if (typeof shape.stepKey !== \"string\") {\n return new EvalError({\n code: \"executor\",\n message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,\n path: `${casePath(suiteCase.name)}.stepKey`\n })\n }\n if (typeof shape.latencyMs !== \"number\" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {\n return new EvalError({\n code: \"executor\",\n message: `executor returned a latencyMs that is not a finite non-negative number (${\n typeof shape.latencyMs === \"number\" ? String(shape.latencyMs) : typeof shape.latencyMs\n })`,\n path: `${casePath(suiteCase.name)}.latencyMs`\n })\n }\n return undefined\n}\n\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n Effect.flatMap((execution) => {\n const malformed = malformedExecution(suiteCase, execution)\n return malformed === undefined ? Effect.succeed(execution) : Effect.fail(malformed)\n }),\n Effect.map((execution): CaseResult => ({\n case: suiteCase.name,\n execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n observations: []\n })),\n Effect.catchCause((cause) =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n ),\n Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n case: suiteCase.name,\n stepKey: execution.stepKey,\n binding: {\n ...structuredClone({\n sampling: binding.sampling,\n ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n ...(binding.context === undefined ? {} : { context: binding.context })\n }),\n scorer: binding.scorer,\n appliesTo: binding.appliesTo\n },\n input: {\n input: structuredClone(suiteCase.input),\n output: execution.output,\n ...(groundTruthFor(suiteCase, binding)),\n ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n latencyMs: execution.latencyMs\n }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n const source = result as {\n readonly kind: unknown\n readonly score?: unknown\n readonly reason?: unknown\n readonly meta?: unknown\n }\n const kind = source.kind\n if (kind !== \"score\" && kind !== \"inconclusive\") {\n return inconclusive(\n request,\n boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n at\n )\n }\n const reason = source.reason\n if (kind === \"inconclusive\") {\n return inconclusive(\n request,\n typeof reason === \"string\" && reason.length > 0\n ? boundedReason(reason)\n : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n at\n )\n }\n const score = source.score\n if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n return inconclusive(\n request,\n boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n at\n )\n }\n const meta = source.meta\n return {\n case: request.case,\n scorer: scorerKeyOf(request.binding),\n scorerName: scorerNameOf(request.binding),\n stepKey: request.stepKey,\n kind: \"score\",\n score,\n ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n ...(meta === undefined ? {} : { meta }),\n at\n }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n job.score.pipe(\n Effect.flatMap(Scorer.validate),\n Effect.map((result): ScoreStore.ScoreObservation => ({\n ...job.observation,\n kind: \"score\",\n score: result.score,\n ...(result.reason === undefined ? {} : { reason: result.reason }),\n ...(result.meta === undefined ? {} : { meta: result.meta }),\n at: job.at\n })),\n Effect.catchCause((cause) =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n )\n )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n const runBatchCorrelated = (\n jobs: ReadonlyArray<ScoreJob>,\n options?: { readonly concurrency?: number | undefined }\n ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n Effect.forEach(\n jobs,\n (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n { concurrency: options?.concurrency ?? 1 }\n )\n return {\n runBatchCorrelated,\n runBatch: (jobs, options) =>\n runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n readonly at: string\n readonly runId: string\n readonly sampleId: string\n readonly concurrency: number\n}\n\nconst buildJobs = (\n cases: ReadonlyArray<CaseResult>,\n suite: Suite,\n { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n EvalError\n> =>\n Effect.gen(function*() {\n // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n // two arrays are index-aligned by construction; nothing here has to look a\n // case back up by name.\n const candidates = cases.flatMap((caseResult, index) => {\n const execution = caseResult.execution\n if (execution === undefined) return []\n const suiteCase = suite.cases[index]!\n return suite.bindings.flatMap((binding, bindingIndex) =>\n binding.appliesTo === execution.target\n ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n : []\n )\n })\n const sampled = yield* Effect.forEach(\n candidates,\n ({ request, bindingIndex }) =>\n Sampling.decide(\n request.binding.sampling,\n request.stepKey,\n scorerKeyOf(request.binding)\n ).pipe(\n Effect.map((selected) => selected ? request : undefined),\n Effect.mapError((cause) =>\n new EvalError({\n code: \"invalid_suite\",\n message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n path: `bindings[${bindingIndex}].sampling`,\n cause\n })\n )\n ),\n { concurrency }\n )\n const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n identity: tupleKey(\n suite.name,\n runId,\n sampleId,\n request.case,\n request.stepKey,\n scorerKeyOf(request.binding),\n `${index}`\n ),\n observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n at: Date.parse(at)\n }))\n return { requests, jobs }\n })\n\nconst batchFallback = (\n jobs: ReadonlyArray<ScoreJob>,\n cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n Cause.hasInterrupts(cause)\n ? Effect.interrupt\n : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n jobs: ReadonlyArray<ScoreJob>,\n scorer: ScoreBatchRunner,\n runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n Effect.gen(function*() {\n const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n Effect.catchCause((cause) =>\n batchFallback(jobs, cause).pipe(\n Effect.map((observations) =>\n observations.map((observation, index) => ({\n identity: jobs[index]!.identity,\n observation\n }))\n )\n )\n )\n )\n if (batchResults.length !== jobs.length) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n path: \"runBatchCorrelated\"\n })\n )\n }\n const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n const byIdentity = new Map<string, ScoreObservation>()\n let unknown: { readonly identity: string; readonly index: number } | undefined\n for (const [index, result] of batchResults.entries()) {\n const jobIndex = jobIndexByIdentity.get(result.identity)\n if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n path: `runBatchCorrelated[${index}]`\n })\n )\n }\n if (jobIndex === undefined) unknown = { identity: result.identity, index }\n byIdentity.set(result.identity, result.observation)\n }\n if (unknown !== undefined) {\n const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n const missingIdentity = jobs[missingJobIndex]!.identity\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n path: `runBatchCorrelated[${unknown.index}]`\n })\n )\n }\n return jobs.map((job) => byIdentity.get(job.identity)!)\n })\n\nconst scoreOrderOnly = (\n jobs: ReadonlyArray<ScoreJob>,\n requests: ReadonlyArray<ScoreRequest>,\n scorer: ScoreBatchRunner,\n { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n Effect.gen(function*() {\n const pairIndexes = new Map<string, number>()\n for (const [index, job] of jobs.entries()) {\n const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n const previous = pairIndexes.get(pair)\n if (previous !== undefined) {\n const first = requests[previous]!\n const second = requests[index]!\n return yield* Effect.fail(\n new EvalError({\n code: \"ambiguous_score_job\",\n message:\n `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n label(second.binding)\n }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n path: \"runBatch\"\n })\n )\n }\n pairIndexes.set(pair, index)\n }\n const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n Effect.catchCause((cause) => batchFallback(jobs, cause))\n )\n if (results.length !== jobs.length) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message:\n `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n path: \"runBatch\"\n })\n )\n }\n return results\n })\n\nconst score = (\n cases: ReadonlyArray<CaseResult>,\n suite: Suite,\n scorer: ScoreBatchRunner,\n context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n Effect.gen(function*() {\n const { requests, jobs } = yield* buildJobs(cases, suite, context)\n if (requests.length === 0) return cases\n const runBatchCorrelated = scorer.runBatchCorrelated\n const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n const results = yield* runBatchCorrelated === undefined\n ? scoreOrderOnly(jobs, requests, scorer, context)\n : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n const { at } = context\n const observations: Array<Observation> = []\n for (const [index, request] of requests.entries()) {\n const result = results[index]!\n if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n return yield* Effect.fail(\n new EvalError({\n code: \"scorer_protocol\",\n message: protocol === \"runBatchCorrelated\"\n ? `Correlated scorer batch identity '${\n jobs[index]!.identity\n }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n scorerKeyOf(request.binding)\n }' at step '${request.stepKey}'`\n : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n scorerKeyOf(request.binding)\n }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n path: `${protocol}[${index}]`\n })\n )\n }\n observations.push(observationFor(request, result, at))\n }\n const byCase = new Map<string, Array<Observation>>()\n for (const observation of observations) {\n const bucket = byCase.get(observation.case)\n if (bucket === undefined) byCase.set(observation.case, [observation])\n else bucket.push(observation)\n }\n return cases.map((caseResult) => ({\n ...caseResult,\n observations: byCase.get(caseResult.case) ?? []\n }))\n })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n suite: Suite,\n options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n Effect.gen(function*() {\n const executor = yield* CaseExecutor\n const injected = yield* Effect.serviceOption(Runner)\n const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n const atMillis = Date.parse(options.at)\n if (options.runId.trim().length === 0) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_run_options\",\n message: \"Deterministic runs require a non-empty runId\",\n path: \"options.runId\"\n })\n )\n }\n if (\n !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n !Number.isFinite(atMillis) ||\n new Date(atMillis).toISOString() !== options.at\n ) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_run_options\",\n message:\n `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n path: \"options.at\"\n })\n )\n }\n const n = options.trials ?? suite.trials ?? 1\n const k = options.k ?? 1\n if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n return yield* Effect.fail(\n new EvalError({\n code: \"invalid_trials\",\n message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n path: \"options.trials\"\n })\n )\n }\n const at = options.at\n const runId = options.runId\n const sampleId = options.sampleId ?? \"default\"\n // Await each batch before starting the next: concurrency within a trial is\n // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n Effect.gen(function*() {\n const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n runCase(executor, suiteCase), {\n concurrency: suite.concurrency,\n discard: false\n })\n return yield* score(cases, suite, scorer, {\n concurrency: suite.concurrency,\n at,\n runId,\n sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n })\n }), { concurrency: 1 })\n const outcomes: Array<[string, ReadonlyArray<boolean>]> = []\n const cases = suite.cases.map((suiteCase, index) => {\n const attempts = batches.map((batch) =>\n batch[index]!\n )\n const passed = attempts.map((attempt) =>\n attempt.error === undefined && attempt.observations.length > 0 &&\n attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n )\n outcomes.push([suiteCase.name, passed])\n return {\n ...attempts[0]!,\n error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n observations: attempts.flatMap((attempt) => attempt.observations),\n trials: Trials.summarize(passed, k)\n }\n })\n return {\n runId,\n suite: suite.name,\n cases,\n observations: cases.flatMap((caseResult) => caseResult.observations),\n // fromEntries defines own properties, so a case named `__proto__` stays a case.\n trials: Trials.aggregate(Object.fromEntries(outcomes), k),\n k\n }\n }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n return {\n runBatch: () => Effect.fail(unavailable),\n runBatchCorrelated: (jobs) =>\n Effect.succeed(jobs.map((job) => ({\n identity: job.identity,\n observation: ScorerRunner.inconclusive(job, unavailable)\n })))\n }\n})\n"]}
package/dist/esm/Trials.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAgB9F,CAAA"}
1
+ {"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAiB9F,CAAA"}
package/dist/esm/Trials.js CHANGED
@@ -97,12 +97,13 @@ const mean = (values) => values.length === 0 ? 0 : values.reduce((total, value)
97
97
  export const aggregate = (cases, k) => {
98
98
  if (!Number.isInteger(k) || k < 1)
99
99
  throw invalid(`k must be a positive integer, received ${k}`, "k");
100
- const perCase = {};
101
- for (const [name, outcomes] of Object.entries(cases)) {
100
+ const entries = Object.entries(cases).map(([name, outcomes]) => {
102
101
  if (outcomes.length === 0)
103
102
  throw invalid(`Case '${name}' has no runs`, `cases['${name}']`);
104
- perCase[name] = summarize(outcomes, Math.min(k, outcomes.length));
105
- }
103
+ return [name, summarize(outcomes, Math.min(k, outcomes.length))];
104
+ });
105
+ // fromEntries defines own properties, so a case named `__proto__` stays a case.
106
+ const perCase = Object.fromEntries(entries);
106
107
  const summaries = Object.values(perCase);
107
108
  return {
108
109
  cases: summaries.length,
package/dist/esm/Trials.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Trials.js","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAuC1C,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAAY,EAAa,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AAEtH,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAQ,EAAE;IACzD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,kDAAkD,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IAC5G,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,CAAC,2CAA2C,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,kCAAkC,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;AACtH,CAAC,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAU,EAAE;IACjE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IACjB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAA;IACvB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,IAAI,CAAA;AACjB,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAU,EAAE;IAClE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IACjB,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AACzB,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,QAAgC,EAAE,CAAS,EAAW,EAAE;IAChF,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAA;IACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAA;IAC9C,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,CAAA;IACvB,OAAO;QACL,CAAC;QACD,MAAM;QACN,IAAI;QACJ,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9B,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAChC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;KACtD,CAAA;AACH,CAAC,CAAA;AAED,MAAM,IAAI,GAAG,CAAC,MAA6B,EAAU,EAAE,CACrD,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;AAE7F;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAuD,EAAE,CAAS,EAAa,EAAE;IACzG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,0CAA0C,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IACpG,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,SAAS,IAAI,eAAe,EAAE,UAAU,IAAI,IAAI,CAAC,CAAA;QAC1F,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IACnE,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACxC,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,MAAM;QACvB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1D,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO;KACR,CAAA;AACH,CAAC,CAAA","sourcesContent":["/**\n * Repeated-run statistics for live model evaluations.\n *\n * A live case runs `n` times and passes `c` of them. {@link passAtK} is the\n * chance that at least one of `k` runs passes, the capability a best-of-k\n * caller sees; {@link passHatK} is the chance that all `k` pass, the\n * consistency a user sees on every attempt. Both are exact for sampling `k` of\n * the `n` recorded runs without replacement.\n *\n * Every function is pure and throws an `EvalError` with code `invalid_trials`\n * for counts outside the documented bounds.\n *\n * @since 0.1.0\n */\n\nimport { EvalError } from \"./EvalError.ts\"\n\n/**\n * Repeated-run statistics for one case.\n *\n * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,\n * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Summary {\n readonly n: number\n readonly passes: number\n readonly rate: number\n readonly passAt1: number\n readonly passAtK: number\n readonly passHatK: number\n readonly stderr: number\n}\n\n/**\n * Repeated-run statistics across cases.\n *\n * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is\n * the fraction of cases whose every run passed.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Aggregate {\n readonly cases: number\n readonly passAt1: number\n readonly passAtK: number\n readonly passHatK: number\n readonly allPass: number\n readonly perCase: Readonly<Record<string, Summary>>\n}\n\nconst invalid = (message: string, path: string): EvalError => new EvalError({ code: \"invalid_trials\", message, path })\n\nconst validate = (n: number, c: number, k: number): void => {\n if (!Number.isInteger(n) || n < 1) throw invalid(`Run count must be a positive integer, received ${n}`, \"n\")\n if (!Number.isInteger(c) || c < 0 || c > n) {\n throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, \"c\")\n }\n if (!Number.isInteger(k) || k < 1 || k > n) throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, \"k\")\n}\n\n/**\n * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,\n * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is\n * computed as a product, so large `n` neither overflows nor loses precision.\n *\n * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with\n * `0 <= c <= n` and `1 <= k <= n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passAtK = (n: number, c: number, k: number): number => {\n validate(n, c, k)\n if (n - c < k) return 1\n let fail = 1\n for (let i = n - c + 1; i <= n; i++) fail *= 1 - k / i\n return 1 - fail\n}\n\n/**\n * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`\n * without replacement pass, the consistency measure of tau-bench.\n *\n * Throws `invalid_trials` under the same bounds as {@link passAtK}.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passHatK = (n: number, c: number, k: number): number => {\n validate(n, c, k)\n let all = 1\n for (let i = 0; i < k; i++) all *= (c - i) / (n - i)\n return Math.max(0, all)\n}\n\n/**\n * Summarizes one case's run outcomes at `k`.\n *\n * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const summarize = (outcomes: ReadonlyArray<boolean>, k: number): Summary => {\n const n = outcomes.length\n const passes = outcomes.filter(Boolean).length\n validate(n, passes, k)\n const rate = passes / n\n return {\n n,\n passes,\n rate,\n passAt1: rate,\n passAtK: passAtK(n, passes, k),\n passHatK: passHatK(n, passes, k),\n stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)\n }\n}\n\nconst mean = (values: ReadonlyArray<number>): number =>\n values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length\n\n/**\n * Summarizes every case at `k` and averages over cases.\n *\n * A case with fewer than `k` runs is summarized at `k = n`, its own run\n * count, so a short case still contributes rather than failing the whole\n * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case\n * with no runs or a `k` that is not a positive integer.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {\n if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, \"k\")\n const perCase: Record<string, Summary> = {}\n for (const [name, outcomes] of Object.entries(cases)) {\n if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)\n perCase[name] = summarize(outcomes, Math.min(k, outcomes.length))\n }\n const summaries = Object.values(perCase)\n return {\n cases: summaries.length,\n passAt1: mean(summaries.map((summary) => summary.passAt1)),\n passAtK: mean(summaries.map((summary) => summary.passAtK)),\n passHatK: mean(summaries.map((summary) => summary.passHatK)),\n allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),\n perCase\n }\n}\n"]}
1
+ {"version":3,"file":"Trials.js","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAuC1C,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAAY,EAAa,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AAEtH,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAQ,EAAE;IACzD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,kDAAkD,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IAC5G,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,CAAC,2CAA2C,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,kCAAkC,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;AACtH,CAAC,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAU,EAAE;IACjE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IACjB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAA;IACvB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,IAAI,CAAA;AACjB,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS,EAAU,EAAE;IAClE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IACjB,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AACzB,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,QAAgC,EAAE,CAAS,EAAW,EAAE;IAChF,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAA;IACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAA;IAC9C,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,CAAA;IACvB,OAAO;QACL,CAAC;QACD,MAAM;QACN,IAAI;QACJ,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9B,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAChC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;KACtD,CAAA;AACH,CAAC,CAAA;AAED,MAAM,IAAI,GAAG,CAAC,MAA6B,EAAU,EAAE,CACrD,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;AAE7F;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAuD,EAAE,CAAS,EAAa,EAAE;IACzG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,OAAO,CAAC,0CAA0C,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IACpG,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAqB,EAAE;QAChF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,SAAS,IAAI,eAAe,EAAE,UAAU,IAAI,IAAI,CAAC,CAAA;QAC1F,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAClE,CAAC,CAAC,CAAA;IACF,gFAAgF;IAChF,MAAM,OAAO,GAA4B,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACxC,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,MAAM;QACvB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1D,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO;KACR,CAAA;AACH,CAAC,CAAA","sourcesContent":["/**\n * Repeated-run statistics for live model evaluations.\n *\n * A live case runs `n` times and passes `c` of them. {@link passAtK} is the\n * chance that at least one of `k` runs passes, the capability a best-of-k\n * caller sees; {@link passHatK} is the chance that all `k` pass, the\n * consistency a user sees on every attempt. Both are exact for sampling `k` of\n * the `n` recorded runs without replacement.\n *\n * Every function is pure and throws an `EvalError` with code `invalid_trials`\n * for counts outside the documented bounds.\n *\n * @since 0.1.0\n */\n\nimport { EvalError } from \"./EvalError.ts\"\n\n/**\n * Repeated-run statistics for one case.\n *\n * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,\n * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Summary {\n readonly n: number\n readonly passes: number\n readonly rate: number\n readonly passAt1: number\n readonly passAtK: number\n readonly passHatK: number\n readonly stderr: number\n}\n\n/**\n * Repeated-run statistics across cases.\n *\n * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is\n * the fraction of cases whose every run passed.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Aggregate {\n readonly cases: number\n readonly passAt1: number\n readonly passAtK: number\n readonly passHatK: number\n readonly allPass: number\n readonly perCase: Readonly<Record<string, Summary>>\n}\n\nconst invalid = (message: string, path: string): EvalError => new EvalError({ code: \"invalid_trials\", message, path })\n\nconst validate = (n: number, c: number, k: number): void => {\n if (!Number.isInteger(n) || n < 1) throw invalid(`Run count must be a positive integer, received ${n}`, \"n\")\n if (!Number.isInteger(c) || c < 0 || c > n) {\n throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, \"c\")\n }\n if (!Number.isInteger(k) || k < 1 || k > n) throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, \"k\")\n}\n\n/**\n * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,\n * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is\n * computed as a product, so large `n` neither overflows nor loses precision.\n *\n * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with\n * `0 <= c <= n` and `1 <= k <= n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passAtK = (n: number, c: number, k: number): number => {\n validate(n, c, k)\n if (n - c < k) return 1\n let fail = 1\n for (let i = n - c + 1; i <= n; i++) fail *= 1 - k / i\n return 1 - fail\n}\n\n/**\n * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`\n * without replacement pass, the consistency measure of tau-bench.\n *\n * Throws `invalid_trials` under the same bounds as {@link passAtK}.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passHatK = (n: number, c: number, k: number): number => {\n validate(n, c, k)\n let all = 1\n for (let i = 0; i < k; i++) all *= (c - i) / (n - i)\n return Math.max(0, all)\n}\n\n/**\n * Summarizes one case's run outcomes at `k`.\n *\n * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const summarize = (outcomes: ReadonlyArray<boolean>, k: number): Summary => {\n const n = outcomes.length\n const passes = outcomes.filter(Boolean).length\n validate(n, passes, k)\n const rate = passes / n\n return {\n n,\n passes,\n rate,\n passAt1: rate,\n passAtK: passAtK(n, passes, k),\n passHatK: passHatK(n, passes, k),\n stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)\n }\n}\n\nconst mean = (values: ReadonlyArray<number>): number =>\n values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length\n\n/**\n * Summarizes every case at `k` and averages over cases.\n *\n * A case with fewer than `k` runs is summarized at `k = n`, its own run\n * count, so a short case still contributes rather than failing the whole\n * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case\n * with no runs or a `k` that is not a positive integer.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {\n if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, \"k\")\n const entries = Object.entries(cases).map(([name, outcomes]): [string, Summary] => {\n if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)\n return [name, summarize(outcomes, Math.min(k, outcomes.length))]\n })\n // fromEntries defines own properties, so a case named `__proto__` stays a case.\n const perCase: Record<string, Summary> = Object.fromEntries(entries)\n const summaries = Object.values(perCase)\n return {\n cases: summaries.length,\n passAt1: mean(summaries.map((summary) => summary.passAt1)),\n passAtK: mean(summaries.map((summary) => summary.passAtK)),\n passHatK: mean(summaries.map((summary) => summary.passHatK)),\n allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),\n perCase\n }\n}\n"]}
package/docs/README.md CHANGED
@@ -36,9 +36,7 @@ Every stage is an `Effect` value, and nothing in a run reads a clock or
36
36
  generates an identifier: `runId` and `at` come from the caller, so two runs over
37
37
  the same inputs produce byte-identical observations.
38
38
 
39
- The package is at 1.0.0-rc.0 and is not on the npm registry yet, so you get it
40
- by working in a clone of the
41
- [Smithers repository](https://github.com/smithersai/smithers).
39
+ Prereleases are published to npm under the `next` tag;
42
40
  [Installation](./installation.md) has the steps.
43
41
 
44
42
  ## A working example
package/docs/installation.md CHANGED
@@ -1,27 +1,13 @@
1
1
  ---
2
2
  title: "Installation"
3
- description: "Add @smthrs/evals to a workspace package, plus its runtime requirements and entry points."
3
+ description: "Install @smthrs/evals from npm, plus its runtime requirements and entry points."
4
4
  ---
5
5
 
6
- The package is at 1.0.0-rc.0 and is not on the npm registry yet. It is a
7
- workspace package of https://github.com/smithersai/smithers, so use it from a
8
- package in a clone of that repository:
6
+ Prereleases are published to npm under the `next` tag. Install the package
7
+ with its `effect` peer dependency:
9
8
 
10
9
  ```bash
11
- git clone https://github.com/smithersai/smithers
12
- cd smithers
13
- pnpm install
14
- ```
15
-
16
- Then depend on it through the workspace protocol in your package's
17
- `package.json` and run `pnpm install` again:
18
-
19
- ```json
20
- {
21
- "dependencies": {
22
- "@smthrs/evals": "workspace:*"
23
- }
24
- }
10
+ pnpm add @smthrs/evals@next effect@^4.0.4
25
11
  ```
26
12
 
27
13
  ## Runtime requirements
package/docs/troubleshooting.md CHANGED
@@ -91,6 +91,11 @@ The target flow failed for a case, or no executor was available.
91
91
  the typed error, locates it with the executor's own `path` or with
92
92
  `cases['<name>']`, and produces no observations for the case. The rest of
93
93
  the suite still ran. Read the retained `cause`.
94
+ - `Target failed for case '<name>': executor returned a stepKey that is not a
95
+ string` or `... a latencyMs that is not a finite non-negative number`: the
96
+ executor succeeded but returned a malformed execution. `path` is
97
+ `cases['<name>'].stepKey` or `cases['<name>'].latencyMs`. Return the step
98
+ key as a string and the latency in milliseconds as a number of 0 or more.
94
99
  - `No executor is available for case '<name>'`: the run was provided
95
100
  `CaseExecutor.makeNoop()` or `CaseExecutor.layerNoop`. Wire a real executor.
96
101
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@smthrs/evals",
3
3
  "type": "module",
4
- "version": "1.0.0-rc.6",
4
+ "version": "1.0.0-rc.8",
5
5
  "license": "MIT",
6
6
  "smthrs": {
7
7
  "group": "agent"
@@ -165,8 +165,8 @@
165
165
  "tag": "next"
166
166
  },
167
167
  "dependencies": {
168
- "@smthrs/core": "1.0.0-rc.6",
169
- "@smthrs/scorers": "1.0.0-rc.6"
168
+ "@smthrs/core": "1.0.0-rc.8",
169
+ "@smthrs/scorers": "1.0.0-rc.8"
170
170
  },
171
171
  "peerDependencies": {
172
172
  "effect": "^4.0.4"
@@ -190,7 +190,7 @@
190
190
  "vitest": "5.0.0"
191
191
  },
192
192
  "scripts": {
193
- "lint": "eslint src --max-warnings=0 && dprint check",
193
+ "lint": "eslint src examples --max-warnings=0 && dprint check",
194
194
  "format": "dprint fmt",
195
195
  "build": "node scripts/build.mjs",
196
196
  "check": "tsc -b tsconfig.json && tsc -p tsconfig.test.json --noEmit",
package/src/Baseline.ts CHANGED
@@ -219,7 +219,9 @@ export const make = (
219
219
  *
220
220
  * Records are ordered by an injective encoding of `(suite, case, scorer,
221
221
  * stepKey)`, so two records whose names differ only in where a separator falls
222
- * still sort apart. The result ends with a newline and never throws: a
222
+ * still sort apart. Records that share that tuple, one per trial of a repeated
223
+ * run, are ordered by score and then by `scorerName`, so the bytes never
224
+ * depend on the order trials finished in. The result ends with a newline and never throws: a
223
225
  * validated baseline holds only strings and finite numbers.
224
226
  *
225
227
  * @category serialization
@@ -229,7 +231,12 @@ export const write = (baseline: Baseline): string => {
229
231
  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)
230
232
  const records = baseline.records
231
233
  .map((record) => [sortKey(record), record] as const)
232
- .sort(([left], [right]) => CanonicalJson.compareText(left, right))
234
+ .sort(([leftKey, left], [rightKey, right]) =>
235
+ CanonicalJson.compareText(leftKey, rightKey) ||
236
+ left.score - right.score ||
237
+ CanonicalJson.compareText(left.scorerName ?? "", right.scorerName ?? "") ||
238
+ Number(left.scorerName !== undefined) - Number(right.scorerName !== undefined)
239
+ )
233
240
  .map(([, record]) => record)
234
241
  return CanonicalJson.stringify({
235
242
  version: baseline.version,
package/src/Regression.ts CHANGED
@@ -104,6 +104,12 @@ type Score = Extract<Observation, { readonly kind: "score" }>
104
104
  const key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)
105
105
  const validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0
106
106
 
107
+ // Decimal scores are not exactly representable, so 0.8 - 0.7 is
108
+ // 0.10000000000000009. A move counts only when it clears a positive tolerance
109
+ // by more than a few ulps; a zero tolerance still reports every move.
110
+ const exceeds = (move: number, tolerance: number): boolean =>
111
+ tolerance === 0 ? move > 0 : move - tolerance > 8 * Number.EPSILON * Math.max(1, tolerance)
112
+
107
113
  const groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {
108
114
  const groups = new Map<string, Array<A>>()
109
115
  for (const item of items) {
@@ -278,7 +284,8 @@ export const compare = (
278
284
  // dead: a score is validated into [0, 1], so nothing can drop below 0.
279
285
  if (observation.stepKey === baselineRecord.stepKey) {
280
286
  if (
281
- Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative
287
+ exceeds(Math.abs(delta), absolute) &&
288
+ exceeds(Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12), relative)
282
289
  ) {
283
290
  nondeterminism.push({
284
291
  case: observation.case,
@@ -289,7 +296,7 @@ export const compare = (
289
296
  })
290
297
  }
291
298
  } else if (
292
- delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative
299
+ delta < 0 && exceeds(-delta, absolute) && exceeds(-delta / Math.max(baselineRecord.score, 1e-12), relative)
293
300
  ) {
294
301
  regressions.push({
295
302
  case: observation.case,
package/src/Runner.ts CHANGED
@@ -248,8 +248,37 @@ const caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {
248
248
  // A step key is a runtime value the target returns, so unlike a suite name it
249
249
  // is flattened rather than rejected: it reaches baselines, reports and the
250
250
  // gate summary, and a control character in any of them corrupts a CI log line.
251
+ // An executor written in plain JavaScript can return any shape, so the two
252
+ // fields every scorer and baseline reads are checked here, where the failure
253
+ // can still name the case and the field.
254
+ const malformedExecution = (suiteCase: Case, execution: Execution): EvalError | undefined => {
255
+ const shape: { readonly stepKey?: unknown; readonly latencyMs?: unknown } =
256
+ typeof execution === "object" && execution !== null ? execution : {}
257
+ if (typeof shape.stepKey !== "string") {
258
+ return new EvalError({
259
+ code: "executor",
260
+ message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,
261
+ path: `${casePath(suiteCase.name)}.stepKey`
262
+ })
263
+ }
264
+ if (typeof shape.latencyMs !== "number" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {
265
+ return new EvalError({
266
+ code: "executor",
267
+ message: `executor returned a latencyMs that is not a finite non-negative number (${
268
+ typeof shape.latencyMs === "number" ? String(shape.latencyMs) : typeof shape.latencyMs
269
+ })`,
270
+ path: `${casePath(suiteCase.name)}.latencyMs`
271
+ })
272
+ }
273
+ return undefined
274
+ }
275
+
251
276
  const runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>
252
277
  Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(
278
+ Effect.flatMap((execution) => {
279
+ const malformed = malformedExecution(suiteCase, execution)
280
+ return malformed === undefined ? Effect.succeed(execution) : Effect.fail(malformed)
281
+ }),
253
282
  Effect.map((execution): CaseResult => ({
254
283
  case: suiteCase.name,
255
284
  execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },
@@ -709,7 +738,7 @@ export const run = (
709
738
  sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))
710
739
  })
711
740
  }), { concurrency: 1 })
712
- const outcomes: Record<string, Array<boolean>> = {}
741
+ const outcomes: Array<[string, ReadonlyArray<boolean>]> = []
713
742
  const cases = suite.cases.map((suiteCase, index) => {
714
743
  const attempts = batches.map((batch) =>
715
744
  batch[index]!
@@ -718,7 +747,7 @@ export const run = (
718
747
  attempt.error === undefined && attempt.observations.length > 0 &&
719
748
  attempt.observations.every((observation) => observation.kind === "score" && observation.score === 1)
720
749
  )
721
- outcomes[suiteCase.name] = passed
750
+ outcomes.push([suiteCase.name, passed])
722
751
  return {
723
752
  ...attempts[0]!,
724
753
  error: attempts.find((attempt) => attempt.error !== undefined)?.error,
@@ -731,7 +760,8 @@ export const run = (
731
760
  suite: suite.name,
732
761
  cases,
733
762
  observations: cases.flatMap((caseResult) => caseResult.observations),
734
- trials: Trials.aggregate(outcomes, k),
763
+ // fromEntries defines own properties, so a case named `__proto__` stays a case.
764
+ trials: Trials.aggregate(Object.fromEntries(outcomes), k),
735
765
  k
736
766
  }
737
767
  }).pipe(Effect.withSpan("Eval.run", { attributes: { suite: suite.name, runId: options.runId } }))
package/src/Trials.ts CHANGED
@@ -137,11 +137,12 @@ const mean = (values: ReadonlyArray<number>): number =>
137
137
  */
138
138
  export const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {
139
139
  if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, "k")
140
- const perCase: Record<string, Summary> = {}
141
- for (const [name, outcomes] of Object.entries(cases)) {
140
+ const entries = Object.entries(cases).map(([name, outcomes]): [string, Summary] => {
142
141
  if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)
143
- perCase[name] = summarize(outcomes, Math.min(k, outcomes.length))
144
- }
142
+ return [name, summarize(outcomes, Math.min(k, outcomes.length))]
143
+ })
144
+ // fromEntries defines own properties, so a case named `__proto__` stays a case.
145
+ const perCase: Record<string, Summary> = Object.fromEntries(entries)
145
146
  const summaries = Object.values(perCase)
146
147
  return {
147
148
  cases: summaries.length,