@bradtaylorsf/alpha-loop 1.4.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +136 -3
  2. package/dist/cli.js +73 -6
  3. package/dist/cli.js.map +1 -1
  4. package/dist/commands/eval.d.ts +59 -4
  5. package/dist/commands/eval.js +370 -55
  6. package/dist/commands/eval.js.map +1 -1
  7. package/dist/commands/evolve.d.ts +43 -4
  8. package/dist/commands/evolve.js +444 -66
  9. package/dist/commands/evolve.js.map +1 -1
  10. package/dist/commands/init.js +3 -7
  11. package/dist/commands/init.js.map +1 -1
  12. package/dist/commands/plan.d.ts +11 -0
  13. package/dist/commands/plan.js +298 -0
  14. package/dist/commands/plan.js.map +1 -0
  15. package/dist/commands/roadmap.d.ts +9 -0
  16. package/dist/commands/roadmap.js +202 -0
  17. package/dist/commands/roadmap.js.map +1 -0
  18. package/dist/commands/triage.d.ts +9 -0
  19. package/dist/commands/triage.js +226 -0
  20. package/dist/commands/triage.js.map +1 -0
  21. package/dist/commands/vision.js +1 -0
  22. package/dist/commands/vision.js.map +1 -1
  23. package/dist/lib/config.d.ts +20 -0
  24. package/dist/lib/config.js +55 -0
  25. package/dist/lib/config.js.map +1 -1
  26. package/dist/lib/eval-checks.d.ts +11 -1
  27. package/dist/lib/eval-checks.js +39 -0
  28. package/dist/lib/eval-checks.js.map +1 -1
  29. package/dist/lib/eval-fixtures.d.ts +55 -0
  30. package/dist/lib/eval-fixtures.js +172 -0
  31. package/dist/lib/eval-fixtures.js.map +1 -0
  32. package/dist/lib/eval-runner.d.ts +26 -2
  33. package/dist/lib/eval-runner.js +202 -17
  34. package/dist/lib/eval-runner.js.map +1 -1
  35. package/dist/lib/eval-skill-bridge.d.ts +53 -0
  36. package/dist/lib/eval-skill-bridge.js +121 -0
  37. package/dist/lib/eval-skill-bridge.js.map +1 -0
  38. package/dist/lib/eval-swebench.d.ts +68 -0
  39. package/dist/lib/eval-swebench.js +274 -0
  40. package/dist/lib/eval-swebench.js.map +1 -0
  41. package/dist/lib/eval.d.ts +9 -1
  42. package/dist/lib/eval.js +27 -7
  43. package/dist/lib/eval.js.map +1 -1
  44. package/dist/lib/github.d.ts +46 -0
  45. package/dist/lib/github.js +179 -0
  46. package/dist/lib/github.js.map +1 -1
  47. package/dist/lib/pipeline.js +44 -2
  48. package/dist/lib/pipeline.js.map +1 -1
  49. package/dist/lib/planning.d.ts +91 -0
  50. package/dist/lib/planning.js +315 -0
  51. package/dist/lib/planning.js.map +1 -0
  52. package/dist/lib/prompts.d.ts +79 -0
  53. package/dist/lib/prompts.js +151 -2
  54. package/dist/lib/prompts.js.map +1 -1
  55. package/dist/lib/score.d.ts +24 -2
  56. package/dist/lib/score.js +162 -3
  57. package/dist/lib/score.js.map +1 -1
  58. package/package.json +2 -1
@@ -1 +1 @@
1
- {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAQ1C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAmB,EACnB,YAAoB,EACpB,OAAqC;IAErC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjB,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;AACvE,CAAC;AAgDD,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,EAAE;IACR,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,EAAE;IACT,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,QAAQ;IACpB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,OAAO;IACnB,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,KAAK;IACpB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,IAAI;IACjB,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,EAAE;IAChB,OAAO,EAAE,mBAAmB;IAC5B,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,KAAK;IACf,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,IAAI;IACjB,qBAAqB,EAAE,KAAK;IAC5B,uBAAuB,EAAE,KAAK;IAC9B,OAAO,EAAE;QACP,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAChD,mBAAmB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACjD,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAChD,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KAC3C;CACF,CAAC;AAEF,gEAAgE;AAChE,MAAM,YAAY,GAAiC;IACjD,SAAS,EAAE,WAAW;IACtB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,YAAY;IACnB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;CAC5B,CAAC;AAEF,2CAA2C;AAC3C,MAAM,WAAW,GAAiC;IAChD,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,wBAAwB,EAAE,uBAAuB;IACjD,0BAA0B,EAAE,yBAAyB;CACtD,CAAC;AAEF,SAAS,MAAM,CAAC,KAAa,EAAE,OAAgB;IAC7C,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,2BAA2B,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,2CAA2C;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC1D,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5C,qCAAqC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACvD,IAAI,GAAG;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAChE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAuC,CAAC;QAC1D,IAAI,EAAE,CAAC,MAAM,KAAK,KAAK;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC7D,IAAI,EAAE,CAAC,aAAa,KAAK,KAAK;YAAE,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACxE,CAAC;IAED,4DAA4D;IAC5D,IAAI,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;YACvF,MAAM,CAAC,GAAG,KAAgC,CAAC;YAC3C,IAAI,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAA2B;IACpD,MAAM,UAAU,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;IAClC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,sFAAsF;IACtF,oFAAoF;IACpF,MAAM,aAAa,GAAG;QACpB,GAAG,QAAQ,CAAC,OAAO;QACnB,GAAG,UAAU,CAAC,OAAO;QACrB,GAAG,SAAS,EAAE,OAAO;KACtB,CAAC;IAEF,MAAM,MAAM,GAAW;QACrB,GAAG,QAAQ;QACX,GAAG,UAAU;QACb,GAAG,UAAU;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;QACZ,OAAO,EAAE,aAAa;KACvB,CAAC;IAEF,kCAAkC;IAClC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAU,CAAC;IAC9D,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAoC,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,wBAAwB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAoB1C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAmB,EACnB,YAAoB,EACpB,OAAqC;IAErC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjB,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;AACvE,CAAC;AAkDD,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,EAAE;IACR,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,EAAE;IACT,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,QAAQ;IACpB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,OAAO;IACnB,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,KAAK;IACpB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,IAAI;IACjB,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,EAAE;IAChB,OAAO,EAAE,mBAAmB;IAC5B,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,KAAK;IACf,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,IAAI;IACjB,qBAAqB,EAAE,KAAK;IAC5B,uBAAuB,EAAE,KAAK;IAC9B,OAAO,EAAE;QACP,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAChD,mBAAmB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACjD,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAChD,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAC1C,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QACvC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAC5C,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAC5C,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;KAClD;IACD,QAAQ,EAAE,EAAE;CACb,CAAC;AAEF,gEAAgE;AAChE,MAAM,YAAY,GAAiC;IACjD,SAAS,EAAE,WAAW;IACtB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,YAAY;IACnB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,UAAU;CACrB,CAAC;AAEF,2CAA2C;AAC3C,MAAM,WAAW,GAAiC;IAChD,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,wBAAwB,EAAE,uBAAuB;IACjD,0BAA0B,EAAE,yBAAyB;CACtD,CAAC;AAEF,SAAS,MAAM,CAAC,KAAa,EAAE,OAAgB;IAC7C,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,2BAA2B,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,2CAA2C;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC1D,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5C,qCAAqC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACvD,IAAI,GAAG;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAChE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,IAAI,MAAM,CAAC,QAAQ,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,QAAmC,CAAC;QAC/D,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,MAAM,UAAU,GAAuB,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACtG,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,CAAC,GAAG,KAAgC,CAAC;gBAC3C,MAAM,OAAO,GAAe,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;YAChE,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAuC,CAAC;QAC1D,IAAI,EAAE,CAAC,MAAM,KAAK,KAAK;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC7D,IAAI,EAAE,CAAC,aAAa,KAAK,KAAK;YAAE,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACxE,CAAC;IAED,4DAA4D;IAC5D,IAAI,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;YACvF,MAAM,CAAC,GAAG,KAAgC,CAAC;YAC3C,IAAI,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAA2B;IACpD,MAAM,UAAU,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;IAClC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,sFAAsF;IACtF,oFAAoF;IACpF,MAAM,aAAa,GAAG;QACpB,GAAG,QAAQ,CAAC,OAAO;QACnB,GAAG,UAAU,CAAC,OAAO;QACrB,GAAG,SAAS,EAAE,OAAO;KACtB,CAAC;IAEF,4EAA4E;IAC5E,MAAM,cAAc,GAAmB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;QACvB,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,EAAE,CAAC;QACzC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,EAAE,CAAC;KAC1C,CAA0B,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,cAAc,CAAC,IAAI,CAAC,GAAG;YACrB,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAW;QACrB,GAAG,QAAQ;QACX,GAAG,UAAU;QACb,GAAG,UAAU;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;QACZ,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,cAAc;KACzB,CAAC;IAEF,kCAAkC;IAClC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAU,CAAC;IAC9D,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAoC,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,wBAAwB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAc,EACd,IAAsB;IAEtB,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,YAAY,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,KAAK,QAAQ;QACrC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC;QACtC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACjB,MAAM,KAAK,GAAG,YAAY,EAAE,KAAK,IAAI,aAAa,CAAC;IACnD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC"}
@@ -55,8 +55,18 @@ export type LlmJudgeCheck = BaseCheck & {
55
55
  rubric: string;
56
56
  min_score: number;
57
57
  };
58
+ /** Contains any — pass if ANY value in the array is found (skill-creator compat). */
59
+ export type ContainsAnyCheck = BaseCheck & {
60
+ type: 'contains_any';
61
+ values: string[];
62
+ };
63
+ /** Not contains — fail if ANY forbidden keyword is found (skill-creator compat). */
64
+ export type NotContainsCheck = BaseCheck & {
65
+ type: 'not_contains';
66
+ values: string[];
67
+ };
58
68
  /** Union of all check types. */
59
- export type CheckDefinition = TestPassCheck | FileExistsCheck | GrepCheck | HttpCheck | DiffSizeCheck | KeywordPresentCheck | KeywordAbsentCheck | LlmJudgeCheck;
69
+ export type CheckDefinition = TestPassCheck | FileExistsCheck | GrepCheck | HttpCheck | DiffSizeCheck | KeywordPresentCheck | KeywordAbsentCheck | LlmJudgeCheck | ContainsAnyCheck | NotContainsCheck;
60
70
  /** Context passed to check runners. */
61
71
  export type CheckContext = {
62
72
  /** Working directory (worktree root). */
@@ -33,6 +33,10 @@ export async function runCheck(check, ctx) {
33
33
  return runKeywordAbsentCheck(check, ctx);
34
34
  case 'llm_judge':
35
35
  return runLlmJudgeCheck(check, ctx);
36
+ case 'contains_any':
37
+ return runContainsAnyCheck(check, ctx);
38
+ case 'not_contains':
39
+ return runNotContainsCheck(check, ctx);
36
40
  default:
37
41
  return { passed: false, score: 0, detail: `Unknown check type: ${check.type}` };
38
42
  }
@@ -198,6 +202,30 @@ Respond with ONLY a single number (1-5) on the first line, followed by a brief e
198
202
  };
199
203
  }
200
204
  }
205
+ async function runContainsAnyCheck(check, ctx) {
206
+ const output = ctx.output ?? '';
207
+ const found = check.values.filter((v) => output.includes(v));
208
+ const passed = found.length > 0;
209
+ return {
210
+ passed,
211
+ score: passed ? 1 : 0,
212
+ detail: passed
213
+ ? `Found matching value(s): ${found.join(', ')}`
214
+ : `None of the expected values found: ${check.values.join(', ')}`,
215
+ };
216
+ }
217
+ async function runNotContainsCheck(check, ctx) {
218
+ const output = ctx.output ?? '';
219
+ const present = check.values.filter((v) => output.includes(v));
220
+ const passed = present.length === 0;
221
+ return {
222
+ passed,
223
+ score: passed ? 1 : 0,
224
+ detail: passed
225
+ ? `No forbidden values found`
226
+ : `Forbidden values present: ${present.join(', ')}`,
227
+ };
228
+ }
201
229
  /**
202
230
  * Parse check definitions from a checks.yaml object.
203
231
  */
@@ -220,6 +248,7 @@ export function parseChecks(raw) {
220
248
  type: 'http',
221
249
  method: String(c.method ?? 'GET'),
222
250
  path: String(c.path ?? '/'),
251
+ port: c.port != null ? Number(c.port) : undefined,
223
252
  expect_status: Number(c.expect_status ?? 200),
224
253
  expect_body_contains: c.expect_body_contains ? String(c.expect_body_contains) : undefined,
225
254
  };
@@ -246,6 +275,16 @@ export function parseChecks(raw) {
246
275
  rubric: String(c.rubric ?? ''),
247
276
  min_score: Number(c.min_score ?? 3),
248
277
  };
278
+ case 'contains_any':
279
+ return {
280
+ type: 'contains_any',
281
+ values: Array.isArray(c.values) ? c.values.map(String) : [],
282
+ };
283
+ case 'not_contains':
284
+ return {
285
+ type: 'not_contains',
286
+ values: Array.isArray(c.values) ? c.values.map(String) : [],
287
+ };
249
288
  default:
250
289
  return { type };
251
290
  }
@@ -1 +1 @@
1
- {"version":3,"file":"eval-checks.js","sourceRoot":"","sources":["../../src/lib/eval-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AA8FlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAsB,EAAE,GAAiB;IACtE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uBAAwB,KAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,GAAiB;IAK1E,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,KAAK,CAAC,IAAI,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,mCAAmC;AAEnC,KAAK,UAAU,gBAAgB,CAAC,MAAqB,EAAE,GAAiB;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,UAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IACrC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAsB,EAAE,GAAiB;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,kBAAkB,KAAK,CAAC,IAAI,EAAE;KAC/H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,oBAAoB,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;QACxC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE;gBAClE,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,cAAc,KAAK,CAAC,aAAa,iBAAiB,SAAS,EAAE;SAC9G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;IAEhC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,SAAS,UAAU,WAAW,SAAS,QAAQ;YACrD,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,KAA0B,EAAE,GAAiB;IACjF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,uBAAuB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpD,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAyB,EAAE,GAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAClE,MAAM,MAAM,GAAG;;;EAGf,KAAK,CAAC,MAAM;;;EAGZ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;4FAEyD,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,KAAK,EAAE,QAAQ;YACf,KAAK;YACL,MAAM;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,eAAe,IAAI,KAAK,CAAC,SAAS,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,eAAe,GAAG,CAAC;YAC1B,MAAM,EAAE,oBAAoB,eAAe,YAAY,KAAK,CAAC,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC1G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxF,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAmB,CAAC;YAChD,KAAK,aAAa;gBAChB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAqB,CAAC;YAChF,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAe,CAAC;YACrG,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;oBAC3B,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC;oBAC7C,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7E,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;oBAChE,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;iBAChD,CAAC;YACrB,KAAK,iBAAiB;gBACpB,OAAO;oBACL,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC3C,CAAC;YAC3B,KAAK,gBAAgB;gBACnB,OAAO;oBACL,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC5C,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;iBACnB,CAAC;YACrB;gBACE,OAAO,EAAE,IAAI,EAAkC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"eval-checks.js","sourceRoot":"","sources":["../../src/lib/eval-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AA4GlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAsB,EAAE,GAAiB;IACtE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,cAAc;YACjB,OAAO,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,KAAK,cAAc;YACjB,OAAO,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uBAAwB,KAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,GAAiB;IAK1E,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,KAAK,CAAC,IAAI,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,mCAAmC;AAEnC,KAAK,UAAU,gBAAgB,CAAC,MAAqB,EAAE,GAAiB;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,UAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IACrC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAsB,EAAE,GAAiB;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,kBAAkB,KAAK,CAAC,IAAI,EAAE;KAC/H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,oBAAoB,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;QACxC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE;gBAClE,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,cAAc,KAAK,CAAC,aAAa,iBAAiB,SAAS,EAAE;SAC9G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;IAEhC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,SAAS,UAAU,WAAW,SAAS,QAAQ;YACrD,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,KAA0B,EAAE,GAAiB;IACjF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,uBAAuB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpD,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAyB,EAAE,GAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAClE,MAAM,MAAM,GAAG;;;EAGf,KAAK,CAAC,MAAM;;;EAGZ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;4FAEyD,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,KAAK,EAAE,QAAQ;YACf,KAAK;YACL,MAAM;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,eAAe,IAAI,KAAK,CAAC,SAAS,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,eAAe,GAAG,CAAC;YAC1B,MAAM,EAAE,oBAAoB,eAAe,YAAY,KAAK,CAAC,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC1G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxF,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAuB,EAAE,GAAiB;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,4BAA4B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAChD,CAAC,CAAC,sCAAsC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACpE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAuB,EAAE,GAAiB;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,2BAA2B;YAC7B,CAAC,CAAC,6BAA6B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACtD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAmB,CAAC;YAChD,KAAK,aAAa;gBAChB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAqB,CAAC;YAChF,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAe,CAAC;YACrG,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;oBAC3B,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;oBACjD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC;oBAC7C,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7E,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;oBAChE,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;iBAChD,CAAC;YACrB,KAAK,iBAAiB;gBACpB,OAAO;oBACL,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC3C,CAAC;YAC3B,KAAK,gBAAgB;gBACnB,OAAO;oBACL,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC5C,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;iBACnB,CAAC;YACrB,KAAK,cAAc;gBACjB,OAAO;oBACL,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBACxC,CAAC;YACxB,KAAK,cAAc;gBACjB,OAAO;oBACL,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBACxC,CAAC;YACxB;gBACE,OAAO,EAAE,IAAI,EAAkC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,55 @@
1
+ /** Configuration for a single fixture within the monorepo. */
2
+ export type FixtureEntry = {
3
+ /** Subdirectory path within the monorepo (e.g. 'fixtures/ts-api'). */
4
+ path: string;
5
+ /** Setup command to run after extraction (e.g. 'pnpm install'). */
6
+ setup?: string;
7
+ /** Test command for this fixture (e.g. 'pnpm test'). */
8
+ test?: string;
9
+ /** Start/dev command (e.g. 'pnpm dev'). */
10
+ start?: string;
11
+ };
12
+ /** Top-level fixture repo configuration from config.yaml. */
13
+ export type FixtureConfig = {
14
+ /** Git remote URL or local path. */
15
+ url: string;
16
+ /** Pinned commit hash for reproducibility. */
17
+ commit: string;
18
+ /** Named fixtures within the monorepo. */
19
+ fixtures: Record<string, FixtureEntry>;
20
+ };
21
+ /** SWE-bench repo entry from config.yaml. */
22
+ export type SwebenchRepoConfig = {
23
+ base_commits: Record<string, string>;
24
+ };
25
+ /** Full eval config loaded from .alpha-loop/evals/config.yaml. */
26
+ export type EvalConfig = {
27
+ fixture_repo?: FixtureConfig;
28
+ swebench_repos?: Record<string, SwebenchRepoConfig>;
29
+ };
30
+ /**
31
+ * Load eval fixture configuration from .alpha-loop/evals/config.yaml.
32
+ */
33
+ export declare function loadEvalConfig(evalDir: string): EvalConfig;
34
+ /**
35
+ * Clone the fixture monorepo (or reuse a cached clone at the pinned commit).
36
+ * Returns the path to the cached repo directory.
37
+ */
38
+ export declare function cloneOrCacheFixtureRepo(config: FixtureConfig, projectDir: string): string;
39
+ /**
40
+ * Extract a specific fixture subdirectory from the monorepo into an isolated target directory.
41
+ */
42
+ export declare function extractFixture(repoDir: string, fixtureName: string, fixtureConfig: FixtureConfig, targetDir: string): string;
43
+ /**
44
+ * Run the fixture's setup command (install dependencies, etc.).
45
+ */
46
+ export declare function setupFixture(targetDir: string, entry: FixtureEntry): void;
47
+ /**
48
+ * Clean up an extracted fixture directory.
49
+ */
50
+ export declare function cleanupFixture(targetDir: string): void;
51
+ /**
52
+ * Resolve a fixture name (e.g. 'ts-api') to its FixtureEntry from config.
53
+ * Returns null if no fixture_repo is configured or fixture not found.
54
+ */
55
+ export declare function resolveFixture(evalConfig: EvalConfig, fixtureName: string): FixtureEntry | null;
@@ -0,0 +1,172 @@
1
+ /**
2
+ * Eval Fixture Monorepo Manager — clone, cache, and extract fixture codebases.
3
+ *
4
+ * Fixtures live in a single monorepo (e.g. alpha-loop-evals) with multiple
5
+ * small codebases as subdirectories. The eval runner clones the monorepo once,
6
+ * then copies individual fixtures into isolated worktrees for each eval run.
7
+ */
8
+ import { existsSync, mkdirSync, cpSync, rmSync, readFileSync } from 'node:fs';
9
+ import { join, resolve } from 'node:path';
10
+ import { parse as parseYaml } from 'yaml';
11
+ import { exec } from './shell.js';
12
+ import { log } from './logger.js';
13
+ /**
14
+ * Load eval fixture configuration from .alpha-loop/evals/config.yaml.
15
+ */
16
+ export function loadEvalConfig(evalDir) {
17
+ const configPath = join(evalDir, 'config.yaml');
18
+ if (!existsSync(configPath))
19
+ return {};
20
+ try {
21
+ const raw = readFileSync(configPath, 'utf-8');
22
+ const parsed = parseYaml(raw);
23
+ if (!parsed || typeof parsed !== 'object')
24
+ return {};
25
+ const result = {};
26
+ // Parse fixture_repo
27
+ const fr = parsed.fixture_repo;
28
+ if (fr && typeof fr === 'object') {
29
+ const fixtures = {};
30
+ const fixturesRaw = (fr.fixtures ?? {});
31
+ for (const [name, entry] of Object.entries(fixturesRaw)) {
32
+ if (entry && typeof entry === 'object') {
33
+ fixtures[name] = {
34
+ path: String(entry.path ?? ''),
35
+ setup: entry.setup ? String(entry.setup) : undefined,
36
+ test: entry.test ? String(entry.test) : undefined,
37
+ start: entry.start ? String(entry.start) : undefined,
38
+ };
39
+ }
40
+ }
41
+ result.fixture_repo = {
42
+ url: String(fr.url ?? ''),
43
+ commit: String(fr.commit ?? 'main'),
44
+ fixtures,
45
+ };
46
+ }
47
+ // Parse swebench_repos
48
+ const sr = parsed.swebench_repos;
49
+ if (sr && typeof sr === 'object') {
50
+ result.swebench_repos = {};
51
+ for (const [repo, config] of Object.entries(sr)) {
52
+ if (config && typeof config === 'object') {
53
+ const commits = (config.base_commits ?? {});
54
+ result.swebench_repos[repo] = {
55
+ base_commits: Object.fromEntries(Object.entries(commits).map(([k, v]) => [k, String(v)])),
56
+ };
57
+ }
58
+ }
59
+ }
60
+ return result;
61
+ }
62
+ catch (err) {
63
+ log.warn(`Failed to load eval config: ${err instanceof Error ? err.message : err}`);
64
+ return {};
65
+ }
66
+ }
67
+ /**
68
+ * Clone the fixture monorepo (or reuse a cached clone at the pinned commit).
69
+ * Returns the path to the cached repo directory.
70
+ */
71
+ export function cloneOrCacheFixtureRepo(config, projectDir) {
72
+ const cacheDir = resolve(projectDir, '.worktrees', 'eval-fixtures');
73
+ // Check if cache exists and is at the right commit
74
+ if (existsSync(join(cacheDir, '.git'))) {
75
+ const currentCommit = exec('git rev-parse HEAD', { cwd: cacheDir, timeout: 5000 });
76
+ if (currentCommit.exitCode === 0 && currentCommit.stdout.startsWith(config.commit)) {
77
+ log.info(`Using cached fixture repo at ${config.commit.slice(0, 8)}`);
78
+ return cacheDir;
79
+ }
80
+ // Wrong commit — reset to correct one
81
+ log.info(`Updating fixture repo to ${config.commit.slice(0, 8)}...`);
82
+ const fetch = exec('git fetch origin', { cwd: cacheDir, timeout: 120_000 });
83
+ if (fetch.exitCode !== 0) {
84
+ // Fetch failed, re-clone
85
+ rmSync(cacheDir, { recursive: true, force: true });
86
+ }
87
+ else {
88
+ const checkout = exec(`git checkout "${config.commit}"`, { cwd: cacheDir, timeout: 30_000 });
89
+ if (checkout.exitCode === 0)
90
+ return cacheDir;
91
+ // Checkout failed, re-clone
92
+ rmSync(cacheDir, { recursive: true, force: true });
93
+ }
94
+ }
95
+ // Clone fresh
96
+ mkdirSync(cacheDir, { recursive: true });
97
+ rmSync(cacheDir, { recursive: true, force: true });
98
+ const url = config.url.includes('://')
99
+ ? config.url
100
+ : `https://github.com/${config.url}.git`;
101
+ log.info(`Cloning fixture repo from ${url}...`);
102
+ const cloneResult = exec(`git clone "${url}" "${cacheDir}"`, { timeout: 120_000 });
103
+ if (cloneResult.exitCode !== 0) {
104
+ throw new Error(`Failed to clone fixture repo: ${cloneResult.stderr}`);
105
+ }
106
+ // Checkout the pinned commit
107
+ const checkout = exec(`git checkout "${config.commit}"`, { cwd: cacheDir, timeout: 30_000 });
108
+ if (checkout.exitCode !== 0) {
109
+ throw new Error(`Failed to checkout commit ${config.commit}: ${checkout.stderr}`);
110
+ }
111
+ return cacheDir;
112
+ }
113
+ /**
114
+ * Extract a specific fixture subdirectory from the monorepo into an isolated target directory.
115
+ */
116
+ export function extractFixture(repoDir, fixtureName, fixtureConfig, targetDir) {
117
+ const entry = fixtureConfig.fixtures[fixtureName];
118
+ if (!entry) {
119
+ throw new Error(`Unknown fixture '${fixtureName}'. Available: ${Object.keys(fixtureConfig.fixtures).join(', ')}`);
120
+ }
121
+ const srcDir = join(repoDir, entry.path);
122
+ if (!existsSync(srcDir)) {
123
+ throw new Error(`Fixture path not found: ${srcDir}`);
124
+ }
125
+ // Clean and create target
126
+ if (existsSync(targetDir)) {
127
+ rmSync(targetDir, { recursive: true, force: true });
128
+ }
129
+ mkdirSync(targetDir, { recursive: true });
130
+ // Copy fixture files
131
+ cpSync(srcDir, targetDir, { recursive: true });
132
+ // Initialize git in the extracted fixture so eval runner can track changes
133
+ exec('git init', { cwd: targetDir, timeout: 10_000 });
134
+ exec('git add -A', { cwd: targetDir, timeout: 10_000 });
135
+ exec('git commit -m "Initial fixture state" --allow-empty', { cwd: targetDir, timeout: 10_000 });
136
+ return targetDir;
137
+ }
138
+ /**
139
+ * Run the fixture's setup command (install dependencies, etc.).
140
+ */
141
+ export function setupFixture(targetDir, entry) {
142
+ if (!entry.setup)
143
+ return;
144
+ log.info(`Running fixture setup: ${entry.setup}`);
145
+ const result = exec(entry.setup, { cwd: targetDir, timeout: 300_000 });
146
+ if (result.exitCode !== 0) {
147
+ log.warn(`Fixture setup warning: ${result.stderr.slice(0, 500)}`);
148
+ }
149
+ }
150
+ /**
151
+ * Clean up an extracted fixture directory.
152
+ */
153
+ export function cleanupFixture(targetDir) {
154
+ try {
155
+ if (existsSync(targetDir)) {
156
+ rmSync(targetDir, { recursive: true, force: true });
157
+ }
158
+ }
159
+ catch {
160
+ /* non-fatal */
161
+ }
162
+ }
163
+ /**
164
+ * Resolve a fixture name (e.g. 'ts-api') to its FixtureEntry from config.
165
+ * Returns null if no fixture_repo is configured or fixture not found.
166
+ */
167
+ export function resolveFixture(evalConfig, fixtureName) {
168
+ if (!evalConfig.fixture_repo)
169
+ return null;
170
+ return evalConfig.fixture_repo.fixtures[fixtureName] ?? null;
171
+ }
172
+ //# sourceMappingURL=eval-fixtures.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval-fixtures.js","sourceRoot":"","sources":["../../src/lib/eval-fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAmClC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAA4B,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAErD,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,qBAAqB;QACrB,MAAM,EAAE,GAAG,MAAM,CAAC,YAAmD,CAAC;QACtE,IAAI,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAiC,EAAE,CAAC;YAClD,MAAM,WAAW,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAA4C,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACvC,QAAQ,CAAC,IAAI,CAAC,GAAG;wBACf,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;wBACpD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;wBACjD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;qBACrD,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,MAAM,CAAC,YAAY,GAAG;gBACpB,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;gBACzB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,CAAC;gBACnC,QAAQ;aACT,CAAC;QACJ,CAAC;QAED,uBAAuB;QACvB,MAAM,EAAE,GAAG,MAAM,CAAC,cAAqE,CAAC;QACxF,IAAI,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,CAAC,cAAc,GAAG,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;gBAChD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACzC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAA2B,CAAC;oBACtE,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG;wBAC5B,YAAY,EAAE,MAAM,CAAC,WAAW,CAC9B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CACxD;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACpF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAqB,EAAE,UAAkB;IAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;IAEpE,mDAAmD;IACnD,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnF,GAAG,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,sCAAsC;QACtC,GAAG,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5E,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACzB,yBAAyB;YACzB,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7F,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,QAAQ,CAAC;YAC7C,4BAA4B;YAC5B,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,cAAc;IACd,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QACpC,CAAC,CAAC,MAAM,CAAC,GAAG;QACZ,CAAC,CAAC,sBAAsB,MAAM,CAAC,GAAG,MAAM,CAAC;IAE3C,GAAG,CAAC,IAAI,CAAC,6BAA6B,GAAG,KAAK,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,GAAG,MAAM,QAAQ,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnF,IAAI,WAAW,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7F,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,aAA4B,EAAE,SAAiB;IAClH,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,oBAAoB,WAAW,iBAAiB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,0BAA0B;IAC1B,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,qBAAqB;IACrB,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE/C,2EAA2E;IAC3E,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,qDAAqD,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAEjG,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB,EAAE,KAAmB;IACjE,IAAI,CAAC,KAAK,CAAC,KAAK;QAAE,OAAO;IAEzB,GAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAsB,EAAE,WAAmB;IACxE,IAAI,CAAC,UAAU,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;AAC/D,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { Config } from './config.js';
1
+ import type { Config, PipelineConfig, PipelineStepName } from './config.js';
2
2
  import type { CheckDefinition } from './eval-checks.js';
3
3
  import type { EvalCase, EvalSuiteResult } from './eval.js';
4
4
  /** Options for running the eval suite. */
@@ -7,6 +7,8 @@ export type EvalRunOptions = {
7
7
  caseId?: string;
8
8
  /** Verbose output. */
9
9
  verbose?: boolean;
10
+ /** Pipeline config overrides to merge into config before running. */
11
+ configOverrides?: PipelineConfig;
10
12
  };
11
13
  /** Extended eval case with parsed check definitions. */
12
14
  export type EvalCaseWithChecks = EvalCase & {
@@ -20,10 +22,32 @@ export type EvalCaseWithChecks = EvalCase & {
20
22
  export declare function runEvalSuite(cases: EvalCaseWithChecks[], config: Config, options?: EvalRunOptions): Promise<EvalSuiteResult>;
21
23
  /**
22
24
  * Prepare a fixture directory for an e2e eval case.
23
- * Clones the repo at the specified ref into a worktree.
25
+ *
26
+ * Supports three modes:
27
+ * 1. Monorepo fixture — fixtureRepo matches a name in config.yaml fixture_repo.fixtures
28
+ * 2. Remote/local repo — fixtureRepo is a GitHub owner/repo or local path
29
+ * 3. Current project — fixtureRepo is empty, uses the current project as fixture
24
30
  */
25
31
  export declare function prepareFixture(evalCase: EvalCase, projectDir: string): string;
26
32
  /**
27
33
  * Snapshot the current harness state (prompts + skills + config) as a hash.
28
34
  */
29
35
  export declare function snapshotHarness(config: Config): string;
36
+ export type CostEstimate = {
37
+ steps: Array<{
38
+ step: PipelineStepName;
39
+ model: string;
40
+ inputTokens: number;
41
+ outputTokens: number;
42
+ costPerCase: number;
43
+ }>;
44
+ totalPerCase: number;
45
+ totalForSuite: number;
46
+ caseCount: number;
47
+ };
48
+ /**
49
+ * Estimate the cost of running an eval suite with a given config.
50
+ * Uses average token counts from previous runs if available,
51
+ * otherwise falls back to default estimates.
52
+ */
53
+ export declare function estimateRunCost(caseCount: number, config: Config): CostEstimate;