@bradtaylorsf/alpha-loop 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -3
- package/dist/cli.js +73 -6
- package/dist/cli.js.map +1 -1
- package/dist/commands/eval.d.ts +59 -4
- package/dist/commands/eval.js +370 -55
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/evolve.d.ts +43 -4
- package/dist/commands/evolve.js +444 -66
- package/dist/commands/evolve.js.map +1 -1
- package/dist/commands/init.js +3 -7
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/plan.d.ts +11 -0
- package/dist/commands/plan.js +298 -0
- package/dist/commands/plan.js.map +1 -0
- package/dist/commands/roadmap.d.ts +9 -0
- package/dist/commands/roadmap.js +202 -0
- package/dist/commands/roadmap.js.map +1 -0
- package/dist/commands/triage.d.ts +9 -0
- package/dist/commands/triage.js +226 -0
- package/dist/commands/triage.js.map +1 -0
- package/dist/commands/vision.js +1 -0
- package/dist/commands/vision.js.map +1 -1
- package/dist/lib/config.d.ts +20 -0
- package/dist/lib/config.js +55 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/eval-checks.d.ts +11 -1
- package/dist/lib/eval-checks.js +39 -0
- package/dist/lib/eval-checks.js.map +1 -1
- package/dist/lib/eval-fixtures.d.ts +55 -0
- package/dist/lib/eval-fixtures.js +172 -0
- package/dist/lib/eval-fixtures.js.map +1 -0
- package/dist/lib/eval-runner.d.ts +26 -2
- package/dist/lib/eval-runner.js +202 -17
- package/dist/lib/eval-runner.js.map +1 -1
- package/dist/lib/eval-skill-bridge.d.ts +53 -0
- package/dist/lib/eval-skill-bridge.js +121 -0
- package/dist/lib/eval-skill-bridge.js.map +1 -0
- package/dist/lib/eval-swebench.d.ts +68 -0
- package/dist/lib/eval-swebench.js +274 -0
- package/dist/lib/eval-swebench.js.map +1 -0
- package/dist/lib/eval.d.ts +9 -1
- package/dist/lib/eval.js +27 -7
- package/dist/lib/eval.js.map +1 -1
- package/dist/lib/github.d.ts +46 -0
- package/dist/lib/github.js +179 -0
- package/dist/lib/github.js.map +1 -1
- package/dist/lib/pipeline.js +44 -2
- package/dist/lib/pipeline.js.map +1 -1
- package/dist/lib/planning.d.ts +91 -0
- package/dist/lib/planning.js +315 -0
- package/dist/lib/planning.js.map +1 -0
- package/dist/lib/prompts.d.ts +79 -0
- package/dist/lib/prompts.js +151 -2
- package/dist/lib/prompts.js.map +1 -1
- package/dist/lib/score.d.ts +24 -2
- package/dist/lib/score.js +162 -3
- package/dist/lib/score.js.map +1 -1
- package/package.json +2 -1
package/dist/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAoB1C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAmB,EACnB,YAAoB,EACpB,OAAqC;IAErC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjB,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;AACvE,CAAC;AAkDD,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,EAAE;IACR,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,EAAE;IACT,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,QAAQ;IACpB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,OAAO;IACnB,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,KAAK;IACpB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,IAAI;IACjB,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,EAAE;IAChB,OAAO,EAAE,mBAAmB;IAC5B,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,KAAK;IACf,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,IAAI;IACjB,qBAAqB,EAAE,KAAK;IAC5B,uBAAuB,EAAE,KAAK;IAC9B,OAAO,EAAE;QACP,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAChD,mBAAmB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACjD,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAChD,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAC1C,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QACvC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAC5C,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAC5C,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;KAClD;IACD,QAAQ,EAAE,EAAE;CACb,CAAC;AAEF,gEAAgE;AAChE,MAAM,YAAY,GAAiC;IACjD,SAAS,EAAE,WAAW;IACtB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,YAAY;IACnB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,UAAU;CACrB,CAAC;AAEF,2CAA2C;AAC3C,MAAM,WAAW,GAAiC;IAChD,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,wBAAwB,EAAE,uBAAuB;IACjD,0BAA0B,EAAE,yBAAyB;CACtD,CAAC;AAEF,SAAS,MAAM,CAAC,KAAa,EAAE,OAAgB;IAC7C,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,2BAA2B,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,2CAA2C;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC1D,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5C,qCAAqC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACvD,IAAI,GAAG;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAChE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,IAAI,MAAM,CAAC,QAAQ,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,QAAmC,CAAC;QAC/D,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,MAAM,UAAU,GAAuB,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACtG,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,CAAC,GAAG,KAAgC,CAAC;gBAC3C,MAAM,OAAO,GAAe,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;YAChE,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAuC,CAAC;QAC1D,IAAI,EAAE,CAAC,MAAM,KAAK,KAAK;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC7D,IAAI,EAAE,CAAC,aAAa,KAAK,KAAK;YAAE,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACxE,CAAC;IAED,4DAA4D;IAC5D,IAAI,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;YACvF,MAAM,CAAC,GAAG,KAAgC,CAAC;YAC3C,IAAI,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAA2B;IACpD,MAAM,UAAU,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;IAClC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,sFAAsF;IACtF,oFAAoF;IACpF,MAAM,aAAa,GAAG;QACpB,GAAG,QAAQ,CAAC,OAAO;QACnB,GAAG,UAAU,CAAC,OAAO;QACrB,GAAG,SAAS,EAAE,OAAO;KACtB,CAAC;IAEF,4EAA4E;IAC5E,MAAM,cAAc,GAAmB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;QACvB,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,EAAE,CAAC;QACzC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,EAAE,CAAC;KAC1C,CAA0B,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,cAAc,CAAC,IAAI,CAAC,GAAG;YACrB,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAW;QACrB,GAAG,QAAQ;QACX,GAAG,UAAU;QACb,GAAG,UAAU;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;QACZ,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,cAAc;KACzB,CAAC;IAEF,kCAAkC;IAClC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAU,CAAC;IAC9D,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAoC,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,wBAAwB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAc,EACd,IAAsB;IAEtB,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,YAAY,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,KAAK,QAAQ;QACrC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC;QACtC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACjB,MAAM,KAAK,GAAG,YAAY,EAAE,KAAK,IAAI,aAAa,CAAC;IACnD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -55,8 +55,18 @@ export type LlmJudgeCheck = BaseCheck & {
|
|
|
55
55
|
rubric: string;
|
|
56
56
|
min_score: number;
|
|
57
57
|
};
|
|
58
|
+
/** Contains any — pass if ANY value in the array is found (skill-creator compat). */
|
|
59
|
+
export type ContainsAnyCheck = BaseCheck & {
|
|
60
|
+
type: 'contains_any';
|
|
61
|
+
values: string[];
|
|
62
|
+
};
|
|
63
|
+
/** Not contains — fail if ANY forbidden keyword is found (skill-creator compat). */
|
|
64
|
+
export type NotContainsCheck = BaseCheck & {
|
|
65
|
+
type: 'not_contains';
|
|
66
|
+
values: string[];
|
|
67
|
+
};
|
|
58
68
|
/** Union of all check types. */
|
|
59
|
-
export type CheckDefinition = TestPassCheck | FileExistsCheck | GrepCheck | HttpCheck | DiffSizeCheck | KeywordPresentCheck | KeywordAbsentCheck | LlmJudgeCheck;
|
|
69
|
+
export type CheckDefinition = TestPassCheck | FileExistsCheck | GrepCheck | HttpCheck | DiffSizeCheck | KeywordPresentCheck | KeywordAbsentCheck | LlmJudgeCheck | ContainsAnyCheck | NotContainsCheck;
|
|
60
70
|
/** Context passed to check runners. */
|
|
61
71
|
export type CheckContext = {
|
|
62
72
|
/** Working directory (worktree root). */
|
package/dist/lib/eval-checks.js
CHANGED
|
@@ -33,6 +33,10 @@ export async function runCheck(check, ctx) {
|
|
|
33
33
|
return runKeywordAbsentCheck(check, ctx);
|
|
34
34
|
case 'llm_judge':
|
|
35
35
|
return runLlmJudgeCheck(check, ctx);
|
|
36
|
+
case 'contains_any':
|
|
37
|
+
return runContainsAnyCheck(check, ctx);
|
|
38
|
+
case 'not_contains':
|
|
39
|
+
return runNotContainsCheck(check, ctx);
|
|
36
40
|
default:
|
|
37
41
|
return { passed: false, score: 0, detail: `Unknown check type: ${check.type}` };
|
|
38
42
|
}
|
|
@@ -198,6 +202,30 @@ Respond with ONLY a single number (1-5) on the first line, followed by a brief e
|
|
|
198
202
|
};
|
|
199
203
|
}
|
|
200
204
|
}
|
|
205
|
+
async function runContainsAnyCheck(check, ctx) {
|
|
206
|
+
const output = ctx.output ?? '';
|
|
207
|
+
const found = check.values.filter((v) => output.includes(v));
|
|
208
|
+
const passed = found.length > 0;
|
|
209
|
+
return {
|
|
210
|
+
passed,
|
|
211
|
+
score: passed ? 1 : 0,
|
|
212
|
+
detail: passed
|
|
213
|
+
? `Found matching value(s): ${found.join(', ')}`
|
|
214
|
+
: `None of the expected values found: ${check.values.join(', ')}`,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
async function runNotContainsCheck(check, ctx) {
|
|
218
|
+
const output = ctx.output ?? '';
|
|
219
|
+
const present = check.values.filter((v) => output.includes(v));
|
|
220
|
+
const passed = present.length === 0;
|
|
221
|
+
return {
|
|
222
|
+
passed,
|
|
223
|
+
score: passed ? 1 : 0,
|
|
224
|
+
detail: passed
|
|
225
|
+
? `No forbidden values found`
|
|
226
|
+
: `Forbidden values present: ${present.join(', ')}`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
201
229
|
/**
|
|
202
230
|
* Parse check definitions from a checks.yaml object.
|
|
203
231
|
*/
|
|
@@ -220,6 +248,7 @@ export function parseChecks(raw) {
|
|
|
220
248
|
type: 'http',
|
|
221
249
|
method: String(c.method ?? 'GET'),
|
|
222
250
|
path: String(c.path ?? '/'),
|
|
251
|
+
port: c.port != null ? Number(c.port) : undefined,
|
|
223
252
|
expect_status: Number(c.expect_status ?? 200),
|
|
224
253
|
expect_body_contains: c.expect_body_contains ? String(c.expect_body_contains) : undefined,
|
|
225
254
|
};
|
|
@@ -246,6 +275,16 @@ export function parseChecks(raw) {
|
|
|
246
275
|
rubric: String(c.rubric ?? ''),
|
|
247
276
|
min_score: Number(c.min_score ?? 3),
|
|
248
277
|
};
|
|
278
|
+
case 'contains_any':
|
|
279
|
+
return {
|
|
280
|
+
type: 'contains_any',
|
|
281
|
+
values: Array.isArray(c.values) ? c.values.map(String) : [],
|
|
282
|
+
};
|
|
283
|
+
case 'not_contains':
|
|
284
|
+
return {
|
|
285
|
+
type: 'not_contains',
|
|
286
|
+
values: Array.isArray(c.values) ? c.values.map(String) : [],
|
|
287
|
+
};
|
|
249
288
|
default:
|
|
250
289
|
return { type };
|
|
251
290
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-checks.js","sourceRoot":"","sources":["../../src/lib/eval-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AA8FlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAsB,EAAE,GAAiB;IACtE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uBAAwB,KAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,GAAiB;IAK1E,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,KAAK,CAAC,IAAI,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,mCAAmC;AAEnC,KAAK,UAAU,gBAAgB,CAAC,MAAqB,EAAE,GAAiB;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,UAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IACrC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAsB,EAAE,GAAiB;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,kBAAkB,KAAK,CAAC,IAAI,EAAE;KAC/H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,oBAAoB,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;QACxC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE;gBAClE,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,cAAc,KAAK,CAAC,aAAa,iBAAiB,SAAS,EAAE;SAC9G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;IAEhC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,SAAS,UAAU,WAAW,SAAS,QAAQ;YACrD,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,KAA0B,EAAE,GAAiB;IACjF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,uBAAuB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpD,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAyB,EAAE,GAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAClE,MAAM,MAAM,GAAG;;;EAGf,KAAK,CAAC,MAAM;;;EAGZ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;4FAEyD,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,KAAK,EAAE,QAAQ;YACf,KAAK;YACL,MAAM;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,eAAe,IAAI,KAAK,CAAC,SAAS,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,eAAe,GAAG,CAAC;YAC1B,MAAM,EAAE,oBAAoB,eAAe,YAAY,KAAK,CAAC,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC1G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxF,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAmB,CAAC;YAChD,KAAK,aAAa;gBAChB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAqB,CAAC;YAChF,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAe,CAAC;YACrG,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;oBAC3B,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC;oBAC7C,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7E,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;oBAChE,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;iBAChD,CAAC;YACrB,KAAK,iBAAiB;gBACpB,OAAO;oBACL,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC3C,CAAC;YAC3B,KAAK,gBAAgB;gBACnB,OAAO;oBACL,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC5C,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;iBACnB,CAAC;YACrB;gBACE,OAAO,EAAE,IAAI,EAAkC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"eval-checks.js","sourceRoot":"","sources":["../../src/lib/eval-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AA4GlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAsB,EAAE,GAAiB;IACtE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,cAAc;YACjB,OAAO,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,KAAK,cAAc;YACjB,OAAO,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uBAAwB,KAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,GAAiB;IAK1E,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,KAAK,CAAC,IAAI,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,mCAAmC;AAEnC,KAAK,UAAU,gBAAgB,CAAC,MAAqB,EAAE,GAAiB;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,UAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IACrC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAsB,EAAE,GAAiB;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,kBAAkB,KAAK,CAAC,IAAI,EAAE;KAC/H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,oBAAoB,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;QACxC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE;gBAClE,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,cAAc,KAAK,CAAC,aAAa,iBAAiB,SAAS,EAAE;SAC9G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;IAEhC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,SAAS,UAAU,WAAW,SAAS,QAAQ;YACrD,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,KAA0B,EAAE,GAAiB;IACjF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,uBAAuB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpD,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAyB,EAAE,GAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAClE,MAAM,MAAM,GAAG;;;EAGf,KAAK,CAAC,MAAM;;;EAGZ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;4FAEyD,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,KAAK,EAAE,QAAQ;YACf,KAAK;YACL,MAAM;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,eAAe,IAAI,KAAK,CAAC,SAAS,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,eAAe,GAAG,CAAC;YAC1B,MAAM,EAAE,oBAAoB,eAAe,YAAY,KAAK,CAAC,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC1G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxF,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAuB,EAAE,GAAiB;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,4BAA4B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAChD,CAAC,CAAC,sCAAsC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACpE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAuB,EAAE,GAAiB;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,2BAA2B;YAC7B,CAAC,CAAC,6BAA6B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACtD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAmB,CAAC;YAChD,KAAK,aAAa;gBAChB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAqB,CAAC;YAChF,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAe,CAAC;YACrG,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;oBAC3B,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;oBACjD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC;oBAC7C,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7E,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;oBAChE,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;iBAChD,CAAC;YACrB,KAAK,iBAAiB;gBACpB,OAAO;oBACL,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC3C,CAAC;YAC3B,KAAK,gBAAgB;gBACnB,OAAO;oBACL,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC5C,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;iBACnB,CAAC;YACrB,KAAK,cAAc;gBACjB,OAAO;oBACL,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBACxC,CAAC;YACxB,KAAK,cAAc;gBACjB,OAAO;oBACL,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBACxC,CAAC;YACxB;gBACE,OAAO,EAAE,IAAI,EAAkC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** Configuration for a single fixture within the monorepo. */
|
|
2
|
+
export type FixtureEntry = {
|
|
3
|
+
/** Subdirectory path within the monorepo (e.g. 'fixtures/ts-api'). */
|
|
4
|
+
path: string;
|
|
5
|
+
/** Setup command to run after extraction (e.g. 'pnpm install'). */
|
|
6
|
+
setup?: string;
|
|
7
|
+
/** Test command for this fixture (e.g. 'pnpm test'). */
|
|
8
|
+
test?: string;
|
|
9
|
+
/** Start/dev command (e.g. 'pnpm dev'). */
|
|
10
|
+
start?: string;
|
|
11
|
+
};
|
|
12
|
+
/** Top-level fixture repo configuration from config.yaml. */
|
|
13
|
+
export type FixtureConfig = {
|
|
14
|
+
/** Git remote URL or local path. */
|
|
15
|
+
url: string;
|
|
16
|
+
/** Pinned commit hash for reproducibility. */
|
|
17
|
+
commit: string;
|
|
18
|
+
/** Named fixtures within the monorepo. */
|
|
19
|
+
fixtures: Record<string, FixtureEntry>;
|
|
20
|
+
};
|
|
21
|
+
/** SWE-bench repo entry from config.yaml. */
|
|
22
|
+
export type SwebenchRepoConfig = {
|
|
23
|
+
base_commits: Record<string, string>;
|
|
24
|
+
};
|
|
25
|
+
/** Full eval config loaded from .alpha-loop/evals/config.yaml. */
|
|
26
|
+
export type EvalConfig = {
|
|
27
|
+
fixture_repo?: FixtureConfig;
|
|
28
|
+
swebench_repos?: Record<string, SwebenchRepoConfig>;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Load eval fixture configuration from .alpha-loop/evals/config.yaml.
|
|
32
|
+
*/
|
|
33
|
+
export declare function loadEvalConfig(evalDir: string): EvalConfig;
|
|
34
|
+
/**
|
|
35
|
+
* Clone the fixture monorepo (or reuse a cached clone at the pinned commit).
|
|
36
|
+
* Returns the path to the cached repo directory.
|
|
37
|
+
*/
|
|
38
|
+
export declare function cloneOrCacheFixtureRepo(config: FixtureConfig, projectDir: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* Extract a specific fixture subdirectory from the monorepo into an isolated target directory.
|
|
41
|
+
*/
|
|
42
|
+
export declare function extractFixture(repoDir: string, fixtureName: string, fixtureConfig: FixtureConfig, targetDir: string): string;
|
|
43
|
+
/**
|
|
44
|
+
* Run the fixture's setup command (install dependencies, etc.).
|
|
45
|
+
*/
|
|
46
|
+
export declare function setupFixture(targetDir: string, entry: FixtureEntry): void;
|
|
47
|
+
/**
|
|
48
|
+
* Clean up an extracted fixture directory.
|
|
49
|
+
*/
|
|
50
|
+
export declare function cleanupFixture(targetDir: string): void;
|
|
51
|
+
/**
|
|
52
|
+
* Resolve a fixture name (e.g. 'ts-api') to its FixtureEntry from config.
|
|
53
|
+
* Returns null if no fixture_repo is configured or fixture not found.
|
|
54
|
+
*/
|
|
55
|
+
export declare function resolveFixture(evalConfig: EvalConfig, fixtureName: string): FixtureEntry | null;
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval Fixture Monorepo Manager — clone, cache, and extract fixture codebases.
|
|
3
|
+
*
|
|
4
|
+
* Fixtures live in a single monorepo (e.g. alpha-loop-evals) with multiple
|
|
5
|
+
* small codebases as subdirectories. The eval runner clones the monorepo once,
|
|
6
|
+
* then copies individual fixtures into isolated worktrees for each eval run.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, mkdirSync, cpSync, rmSync, readFileSync } from 'node:fs';
|
|
9
|
+
import { join, resolve } from 'node:path';
|
|
10
|
+
import { parse as parseYaml } from 'yaml';
|
|
11
|
+
import { exec } from './shell.js';
|
|
12
|
+
import { log } from './logger.js';
|
|
13
|
+
/**
|
|
14
|
+
* Load eval fixture configuration from .alpha-loop/evals/config.yaml.
|
|
15
|
+
*/
|
|
16
|
+
export function loadEvalConfig(evalDir) {
|
|
17
|
+
const configPath = join(evalDir, 'config.yaml');
|
|
18
|
+
if (!existsSync(configPath))
|
|
19
|
+
return {};
|
|
20
|
+
try {
|
|
21
|
+
const raw = readFileSync(configPath, 'utf-8');
|
|
22
|
+
const parsed = parseYaml(raw);
|
|
23
|
+
if (!parsed || typeof parsed !== 'object')
|
|
24
|
+
return {};
|
|
25
|
+
const result = {};
|
|
26
|
+
// Parse fixture_repo
|
|
27
|
+
const fr = parsed.fixture_repo;
|
|
28
|
+
if (fr && typeof fr === 'object') {
|
|
29
|
+
const fixtures = {};
|
|
30
|
+
const fixturesRaw = (fr.fixtures ?? {});
|
|
31
|
+
for (const [name, entry] of Object.entries(fixturesRaw)) {
|
|
32
|
+
if (entry && typeof entry === 'object') {
|
|
33
|
+
fixtures[name] = {
|
|
34
|
+
path: String(entry.path ?? ''),
|
|
35
|
+
setup: entry.setup ? String(entry.setup) : undefined,
|
|
36
|
+
test: entry.test ? String(entry.test) : undefined,
|
|
37
|
+
start: entry.start ? String(entry.start) : undefined,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
result.fixture_repo = {
|
|
42
|
+
url: String(fr.url ?? ''),
|
|
43
|
+
commit: String(fr.commit ?? 'main'),
|
|
44
|
+
fixtures,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
// Parse swebench_repos
|
|
48
|
+
const sr = parsed.swebench_repos;
|
|
49
|
+
if (sr && typeof sr === 'object') {
|
|
50
|
+
result.swebench_repos = {};
|
|
51
|
+
for (const [repo, config] of Object.entries(sr)) {
|
|
52
|
+
if (config && typeof config === 'object') {
|
|
53
|
+
const commits = (config.base_commits ?? {});
|
|
54
|
+
result.swebench_repos[repo] = {
|
|
55
|
+
base_commits: Object.fromEntries(Object.entries(commits).map(([k, v]) => [k, String(v)])),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return result;
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
log.warn(`Failed to load eval config: ${err instanceof Error ? err.message : err}`);
|
|
64
|
+
return {};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Clone the fixture monorepo (or reuse a cached clone at the pinned commit).
|
|
69
|
+
* Returns the path to the cached repo directory.
|
|
70
|
+
*/
|
|
71
|
+
export function cloneOrCacheFixtureRepo(config, projectDir) {
|
|
72
|
+
const cacheDir = resolve(projectDir, '.worktrees', 'eval-fixtures');
|
|
73
|
+
// Check if cache exists and is at the right commit
|
|
74
|
+
if (existsSync(join(cacheDir, '.git'))) {
|
|
75
|
+
const currentCommit = exec('git rev-parse HEAD', { cwd: cacheDir, timeout: 5000 });
|
|
76
|
+
if (currentCommit.exitCode === 0 && currentCommit.stdout.startsWith(config.commit)) {
|
|
77
|
+
log.info(`Using cached fixture repo at ${config.commit.slice(0, 8)}`);
|
|
78
|
+
return cacheDir;
|
|
79
|
+
}
|
|
80
|
+
// Wrong commit — reset to correct one
|
|
81
|
+
log.info(`Updating fixture repo to ${config.commit.slice(0, 8)}...`);
|
|
82
|
+
const fetch = exec('git fetch origin', { cwd: cacheDir, timeout: 120_000 });
|
|
83
|
+
if (fetch.exitCode !== 0) {
|
|
84
|
+
// Fetch failed, re-clone
|
|
85
|
+
rmSync(cacheDir, { recursive: true, force: true });
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
const checkout = exec(`git checkout "${config.commit}"`, { cwd: cacheDir, timeout: 30_000 });
|
|
89
|
+
if (checkout.exitCode === 0)
|
|
90
|
+
return cacheDir;
|
|
91
|
+
// Checkout failed, re-clone
|
|
92
|
+
rmSync(cacheDir, { recursive: true, force: true });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Clone fresh
|
|
96
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
97
|
+
rmSync(cacheDir, { recursive: true, force: true });
|
|
98
|
+
const url = config.url.includes('://')
|
|
99
|
+
? config.url
|
|
100
|
+
: `https://github.com/${config.url}.git`;
|
|
101
|
+
log.info(`Cloning fixture repo from ${url}...`);
|
|
102
|
+
const cloneResult = exec(`git clone "${url}" "${cacheDir}"`, { timeout: 120_000 });
|
|
103
|
+
if (cloneResult.exitCode !== 0) {
|
|
104
|
+
throw new Error(`Failed to clone fixture repo: ${cloneResult.stderr}`);
|
|
105
|
+
}
|
|
106
|
+
// Checkout the pinned commit
|
|
107
|
+
const checkout = exec(`git checkout "${config.commit}"`, { cwd: cacheDir, timeout: 30_000 });
|
|
108
|
+
if (checkout.exitCode !== 0) {
|
|
109
|
+
throw new Error(`Failed to checkout commit ${config.commit}: ${checkout.stderr}`);
|
|
110
|
+
}
|
|
111
|
+
return cacheDir;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Extract a specific fixture subdirectory from the monorepo into an isolated target directory.
|
|
115
|
+
*/
|
|
116
|
+
export function extractFixture(repoDir, fixtureName, fixtureConfig, targetDir) {
|
|
117
|
+
const entry = fixtureConfig.fixtures[fixtureName];
|
|
118
|
+
if (!entry) {
|
|
119
|
+
throw new Error(`Unknown fixture '${fixtureName}'. Available: ${Object.keys(fixtureConfig.fixtures).join(', ')}`);
|
|
120
|
+
}
|
|
121
|
+
const srcDir = join(repoDir, entry.path);
|
|
122
|
+
if (!existsSync(srcDir)) {
|
|
123
|
+
throw new Error(`Fixture path not found: ${srcDir}`);
|
|
124
|
+
}
|
|
125
|
+
// Clean and create target
|
|
126
|
+
if (existsSync(targetDir)) {
|
|
127
|
+
rmSync(targetDir, { recursive: true, force: true });
|
|
128
|
+
}
|
|
129
|
+
mkdirSync(targetDir, { recursive: true });
|
|
130
|
+
// Copy fixture files
|
|
131
|
+
cpSync(srcDir, targetDir, { recursive: true });
|
|
132
|
+
// Initialize git in the extracted fixture so eval runner can track changes
|
|
133
|
+
exec('git init', { cwd: targetDir, timeout: 10_000 });
|
|
134
|
+
exec('git add -A', { cwd: targetDir, timeout: 10_000 });
|
|
135
|
+
exec('git commit -m "Initial fixture state" --allow-empty', { cwd: targetDir, timeout: 10_000 });
|
|
136
|
+
return targetDir;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Run the fixture's setup command (install dependencies, etc.).
|
|
140
|
+
*/
|
|
141
|
+
export function setupFixture(targetDir, entry) {
|
|
142
|
+
if (!entry.setup)
|
|
143
|
+
return;
|
|
144
|
+
log.info(`Running fixture setup: ${entry.setup}`);
|
|
145
|
+
const result = exec(entry.setup, { cwd: targetDir, timeout: 300_000 });
|
|
146
|
+
if (result.exitCode !== 0) {
|
|
147
|
+
log.warn(`Fixture setup warning: ${result.stderr.slice(0, 500)}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Clean up an extracted fixture directory.
|
|
152
|
+
*/
|
|
153
|
+
export function cleanupFixture(targetDir) {
|
|
154
|
+
try {
|
|
155
|
+
if (existsSync(targetDir)) {
|
|
156
|
+
rmSync(targetDir, { recursive: true, force: true });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
/* non-fatal */
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Resolve a fixture name (e.g. 'ts-api') to its FixtureEntry from config.
|
|
165
|
+
* Returns null if no fixture_repo is configured or fixture not found.
|
|
166
|
+
*/
|
|
167
|
+
export function resolveFixture(evalConfig, fixtureName) {
|
|
168
|
+
if (!evalConfig.fixture_repo)
|
|
169
|
+
return null;
|
|
170
|
+
return evalConfig.fixture_repo.fixtures[fixtureName] ?? null;
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=eval-fixtures.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-fixtures.js","sourceRoot":"","sources":["../../src/lib/eval-fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAmClC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAA4B,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAErD,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,qBAAqB;QACrB,MAAM,EAAE,GAAG,MAAM,CAAC,YAAmD,CAAC;QACtE,IAAI,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAiC,EAAE,CAAC;YAClD,MAAM,WAAW,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAA4C,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACvC,QAAQ,CAAC,IAAI,CAAC,GAAG;wBACf,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;wBACpD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;wBACjD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;qBACrD,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,MAAM,CAAC,YAAY,GAAG;gBACpB,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;gBACzB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,CAAC;gBACnC,QAAQ;aACT,CAAC;QACJ,CAAC;QAED,uBAAuB;QACvB,MAAM,EAAE,GAAG,MAAM,CAAC,cAAqE,CAAC;QACxF,IAAI,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,CAAC,cAAc,GAAG,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;gBAChD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACzC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAA2B,CAAC;oBACtE,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG;wBAC5B,YAAY,EAAE,MAAM,CAAC,WAAW,CAC9B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CACxD;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACpF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAqB,EAAE,UAAkB;IAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;IAEpE,mDAAmD;IACnD,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnF,GAAG,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,sCAAsC;QACtC,GAAG,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5E,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACzB,yBAAyB;YACzB,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7F,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,QAAQ,CAAC;YAC7C,4BAA4B;YAC5B,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,cAAc;IACd,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QACpC,CAAC,CAAC,MAAM,CAAC,GAAG;QACZ,CAAC,CAAC,sBAAsB,MAAM,CAAC,GAAG,MAAM,CAAC;IAE3C,GAAG,CAAC,IAAI,CAAC,6BAA6B,GAAG,KAAK,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,GAAG,MAAM,QAAQ,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnF,IAAI,WAAW,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7F,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,aAA4B,EAAE,SAAiB;IAClH,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,oBAAoB,WAAW,iBAAiB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,0BAA0B;IAC1B,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,qBAAqB;IACrB,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE/C,2EAA2E;IAC3E,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,qDAAqD,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAEjG,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB,EAAE,KAAmB;IACjE,IAAI,CAAC,KAAK,CAAC,KAAK;QAAE,OAAO;IAEzB,GAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAsB,EAAE,WAAmB;IACxE,IAAI,CAAC,UAAU,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;AAC/D,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Config } from './config.js';
|
|
1
|
+
import type { Config, PipelineConfig, PipelineStepName } from './config.js';
|
|
2
2
|
import type { CheckDefinition } from './eval-checks.js';
|
|
3
3
|
import type { EvalCase, EvalSuiteResult } from './eval.js';
|
|
4
4
|
/** Options for running the eval suite. */
|
|
@@ -7,6 +7,8 @@ export type EvalRunOptions = {
|
|
|
7
7
|
caseId?: string;
|
|
8
8
|
/** Verbose output. */
|
|
9
9
|
verbose?: boolean;
|
|
10
|
+
/** Pipeline config overrides to merge into config before running. */
|
|
11
|
+
configOverrides?: PipelineConfig;
|
|
10
12
|
};
|
|
11
13
|
/** Extended eval case with parsed check definitions. */
|
|
12
14
|
export type EvalCaseWithChecks = EvalCase & {
|
|
@@ -20,10 +22,32 @@ export type EvalCaseWithChecks = EvalCase & {
|
|
|
20
22
|
export declare function runEvalSuite(cases: EvalCaseWithChecks[], config: Config, options?: EvalRunOptions): Promise<EvalSuiteResult>;
|
|
21
23
|
/**
|
|
22
24
|
* Prepare a fixture directory for an e2e eval case.
|
|
23
|
-
*
|
|
25
|
+
*
|
|
26
|
+
* Supports three modes:
|
|
27
|
+
* 1. Monorepo fixture — fixtureRepo matches a name in config.yaml fixture_repo.fixtures
|
|
28
|
+
* 2. Remote/local repo — fixtureRepo is a GitHub owner/repo or local path
|
|
29
|
+
* 3. Current project — fixtureRepo is empty, uses the current project as fixture
|
|
24
30
|
*/
|
|
25
31
|
export declare function prepareFixture(evalCase: EvalCase, projectDir: string): string;
|
|
26
32
|
/**
|
|
27
33
|
* Snapshot the current harness state (prompts + skills + config) as a hash.
|
|
28
34
|
*/
|
|
29
35
|
export declare function snapshotHarness(config: Config): string;
|
|
36
|
+
export type CostEstimate = {
|
|
37
|
+
steps: Array<{
|
|
38
|
+
step: PipelineStepName;
|
|
39
|
+
model: string;
|
|
40
|
+
inputTokens: number;
|
|
41
|
+
outputTokens: number;
|
|
42
|
+
costPerCase: number;
|
|
43
|
+
}>;
|
|
44
|
+
totalPerCase: number;
|
|
45
|
+
totalForSuite: number;
|
|
46
|
+
caseCount: number;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Estimate the cost of running an eval suite with a given config.
|
|
50
|
+
* Uses average token counts from previous runs if available,
|
|
51
|
+
* otherwise falls back to default estimates.
|
|
52
|
+
*/
|
|
53
|
+
export declare function estimateRunCost(caseCount: number, config: Config): CostEstimate;
|