agentic-engineering-harness 0.4.16 → 0.5.1

package/dist/evals/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGhE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC9I,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAC3C,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;QACrB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC3C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QACrD,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QAChE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QAC/C,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;KAC/C,CAAC,CAAC,QAAQ,EAAE;IACb,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;CACjR,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc,EAAE,WAAoB;IAChH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,2BAA2B,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAClI,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,6BAA6B,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAC5I,IAAI,GAAG,CAAC,QAAQ,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxD,MAAM,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAChI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,UAAU,IAAI,WAAW,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC7H,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACxG,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QACtG,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAoB,CAAC;QAClH,MAAM,IAAI,GAAiD;YACzD,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,QAAQ,CAAC,EAAE;YACnB,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;YACN,eAAe,EAAE,SAAS,CAAC,QAAQ;YACnC,OAAO,EAAE,GAAG,EAAE,OAAO;YACrB,MAAM;YACN,SAAS;YACT,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAe,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1G,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/E,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpE,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACjE,OAAO,MAAM,CAAC;IAChB,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,+BAA+B,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IACzG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc;IAC9F,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;IAChE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,MAAM,KAAK,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,IAAI,cAAc,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAChG,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAa,CAAC;AACnF,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB,EAAE,IAAa;IACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,IAAI,SAAS,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1G,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,IAAI,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAkB,EAAE,SAAiB;IACtE,OAAO,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AAC9F,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,IAAI,CAAC;QAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AACzF,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,IAAY,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AACvF,SAAS,IAAI,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGhE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC9I,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAC3C,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;QACrB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC3C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QACrD,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QAChE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QAC/C,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;KAC/C,CAAC,CAAC,QAAQ,EAAE;IACb,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;CACjR,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc,EAAE,WAAoB;IAChH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,2BAA2B,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAClI,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,6BAA6B,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAC5I,IAAI,GAAG,CAAC,QAAQ,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxD,MAAM,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAChI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,UAAU,IAAI,WAAW,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC7H,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACxG,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QACtG,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAoB,CAAC;QAClH,MAAM,IAAI,GAAiD;YACzD,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,QAAQ,CAAC,EAAE;YACnB,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;YACN,eAAe,EAAE,SAAS,CAAC,QAAQ;YACnC,OAAO,EAAE,GAAG,EAAE,OAAO;YACrB,MAAM;YACN,SAAS;YACT,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAe,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1G,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/E,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpE,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACjE,OAAO,MAAM,CAAC;IAChB,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,+BAA+B,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IACzG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc;IAC9F,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;IAChE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,gBAAgB,CAAC,EAAE,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,MAAM,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC;IAClM,CAAC;IACD,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,IAAI,cAAc,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAChG,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAa,CAAC;AACnF,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB,EAAE,IAAa;IACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,IAAI,SAAS,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1G,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,IAAI,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAkB,EAAE,SAAiB;IACtE,OAAO,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AAC9F,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,IAAI,CAAC;QAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AACzF,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,IAAY,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AACvF,SAAS,IAAI,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC"}

package/dist/evals/statistics.d.ts

@@ -0,0 +1,38 @@
+import type { HarnessProjectConfig } from "../core/types.js";
+import type { EvalResult } from "./types.js";
+export interface ConfidenceInterval {
+    low: number;
+    high: number;
+    level: number;
+}
+export interface MetricSummary {
+    count: number;
+    mean: number;
+    median: number;
+    standardDeviation: number;
+    confidence: ConfidenceInterval;
+}
+export interface VariantStatistics {
+    variant: string;
+    runs: number;
+    passRate: number;
+    passRateConfidence: ConfidenceInterval;
+    score: MetricSummary;
+    durationMs?: MetricSummary;
+    totalTokens?: MetricSummary;
+    costUsd?: MetricSummary;
+    repairs?: MetricSummary;
+    humanInterventions?: MetricSummary;
+}
+export interface EvalDashboard {
+    version: 1;
+    caseId: string;
+    generatedAt: string;
+    confidenceLevel: number;
+    variants: VariantStatistics[];
+}
+export declare function runRepeatedEval(root: string, config: HarnessProjectConfig, caseId: string, variant: string | undefined, runs?: number): Promise<EvalDashboard>;
+export declare function buildEvalDashboard(root: string, config: HarnessProjectConfig, caseId: string): Promise<EvalDashboard>;
+export declare function aggregateVariant(variant: string, results: EvalResult[], level?: number): VariantStatistics;
+export declare function summarize(values: number[], level?: number): MetricSummary;
+export declare function wilson(successes: number, count: number, level?: number): ConfidenceInterval;

package/dist/evals/statistics.js

@@ -0,0 +1,88 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { runEvalCase, compareEvalCase } from "./runner.js";
+export async function runRepeatedEval(root, config, caseId, variant, runs) {
+    const count = runs ?? config.evals?.defaultRuns ?? 5;
+    if (count < 2)
+        throw new Error("Repeated evals require at least two runs to estimate variance.");
+    for (let index = 0; index < count; index += 1)
+        await runEvalCase(root, config, caseId, variant);
+    return buildEvalDashboard(root, config, caseId);
+}
+export async function buildEvalDashboard(root, config, caseId) {
+    const results = await compareEvalCase(root, config, caseId);
+    const confidenceLevel = config.evals?.confidenceLevel ?? 0.95;
+    const groups = new Map();
+    for (const result of results) {
+        const items = groups.get(result.variant) ?? [];
+        items.push(result);
+        groups.set(result.variant, items);
+    }
+    const variants = [...groups.entries()].map(([variant, items]) => aggregateVariant(variant, items, confidenceLevel)).sort((a, b) => b.passRate - a.passRate || b.score.mean - a.score.mean || a.variant.localeCompare(b.variant));
+    const dashboard = { version: 1, caseId, generatedAt: new Date().toISOString(), confidenceLevel, variants };
+    const output = path.resolve(root, config.evals?.resultsDir ?? ".harness/evals/results", caseId, "dashboard.json");
+    await fs.mkdir(path.dirname(output), { recursive: true });
+    await fs.writeFile(output, `${JSON.stringify(dashboard, null, 2)}\n`);
+    return dashboard;
+}
+export function aggregateVariant(variant, results, level = 0.95) {
+    const passes = results.filter((result) => result.status === "PASS").length;
+    const metric = (select) => { const values = results.map(select).filter((value) => Number.isFinite(value)); return values.length ? summarize(values, level) : undefined; };
+    return {
+        variant,
+        runs: results.length,
+        passRate: results.length ? passes / results.length : 0,
+        passRateConfidence: wilson(passes, results.length, level),
+        score: summarize(results.map((result) => result.score), level),
+        durationMs: metric((result) => result.metrics?.durationMs),
+        totalTokens: metric((result) => result.metrics?.usage.totalTokens),
+        costUsd: metric((result) => result.metrics?.usage.costUsd),
+        repairs: metric((result) => result.metrics?.repairCount),
+        humanInterventions: metric((result) => result.metrics?.humanInterventions)
+    };
+}
+export function summarize(values, level = 0.95) {
+    const sorted = [...values].sort((a, b) => a - b);
+    const count = sorted.length;
+    const mean = sorted.reduce((sum, value) => sum + value, 0) / count;
+    const variance = count > 1 ? sorted.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (count - 1) : 0;
+    const standardDeviation = Math.sqrt(variance);
+    const median = count % 2 ? sorted[(count - 1) / 2] : (sorted[count / 2 - 1] + sorted[count / 2]) / 2;
+    const z = inverseNormal(0.5 + level / 2);
+    const margin = count > 1 ? z * standardDeviation / Math.sqrt(count) : 0;
+    return { count, mean, median, standardDeviation, confidence: { low: mean - margin, high: mean + margin, level } };
+}
+export function wilson(successes, count, level = 0.95) {
+    if (!count)
+        return { low: 0, high: 1, level };
+    const z = inverseNormal(0.5 + level / 2);
+    const p = successes / count;
+    const z2 = z * z;
+    const denominator = 1 + z2 / count;
+    const center = (p + z2 / (2 * count)) / denominator;
+    const margin = z * Math.sqrt((p * (1 - p) + z2 / (4 * count)) / count) / denominator;
+    return { low: Math.max(0, center - margin), high: Math.min(1, center + margin), level };
+}
+// Peter J. Acklam's rational approximation, sufficient for confidence reporting.
+function inverseNormal(p) {
+    if (!(p > 0 && p < 1))
+        throw new Error("Normal quantile requires 0 < p < 1.");
+    const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
+    const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
+    const c = [-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968, 2.938163982698783];
+    const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
+    const low = 0.02425;
+    const high = 1 - low;
+    if (p < low) {
+        const q = Math.sqrt(-2 * Math.log(p));
+        return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+    }
+    if (p > high) {
+        const q = Math.sqrt(-2 * Math.log(1 - p));
+        return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+    }
+    const q = p - 0.5;
+    const r = q * q;
+    return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
+}
+//# sourceMappingURL=statistics.js.map

package/dist/evals/statistics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/evals/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAmB3D,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc,EAAE,OAA2B,EAAE,IAAa;IAC1I,MAAM,KAAK,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC,CAAC;IACrD,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACjG,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,CAAC;QAAE,MAAM,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAChG,OAAO,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,MAA4B,EAAE,MAAc;IACjG,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,eAAe,GAAG,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,IAAI,CAAC;IAC9D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAAC,CAAC;IACxI,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACjO,MAAM,SAAS,GAAkB,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;IAC1H,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,wBAAwB,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAClH,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACjI,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,OAAqB,EAAE,KAAK,GAAG,IAAI;IACnF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,MAAM,GAAG,CAAC,MAAkD,EAA6B,EAAE,GAAG,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAClQ,OAAO;QACL,OAAO;QACP,IAAI,EAAE,OAAO,CAAC,MAAM;QACpB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC;QACzD,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;QAC9D,UAAU,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC;QAC1D,WAAW,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC;QAClE,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC;QAC1D,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC;QACxD,kBAAkB,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC;KAC3E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAgB,EAAE,KAAK,GAAG,IAAI;IACtD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAAC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAAC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;IAClJ,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1J,MAAM,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACrG,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,GAAG,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;AACpH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,SAAiB,EAAE,KAAa,EAAE,KAAK,GAAG,IAAI;IACnE,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IAC9C,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;IAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IAC5H,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC;IAAC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,WAAW,CAAC;IAC1I,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;AAC1F,CAAC;AAED,iFAAiF;AACjF,SAAS,aAAa,CAAC,CAAS;IAC9B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC9E,MAAM,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IAC/H,MAAM,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,iBAAiB,CAAC,CAAC;IAC7G,MAAM,CAAC,GAAG,CAAC,CAAC,oBAAoB,EAAE,CAAC,kBAAkB,EAAE,CAAC,iBAAiB,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IACrI,MAAM,CAAC,GAAG,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IAC3F,MAAM,GAAG,GAAG,OAAO,CAAC;IAAC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC;IAC1C,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAAC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAAC,CAAC;IAC7L,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QAAC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAAC,CAAC;IACnM,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5L,CAAC"}

package/dist/evidence/graph.d.ts

@@ -0,0 +1,48 @@
+import type { NormalizedFinding, PlannerOutput } from "../agents/outputContracts.js";
+import type { DeliveryFinalizationResult } from "../delivery/finalize.js";
+import type { HarnessProjectConfig, TaskContract, ValidationCheck, ValidationReport, WorkerSession } from "../core/types.js";
+export type EvidenceNodeType = "run" | "requirement" | "task" | "file" | "check" | "finding" | "agent-session" | "commit" | "pull-request";
+export interface EvidenceNode {
+    id: string;
+    type: EvidenceNodeType;
+    label: string;
+    data?: Record<string, unknown>;
+}
+export interface EvidenceEdge {
+    from: string;
+    to: string;
+    type: "contains" | "implemented-by" | "changed" | "validated-by" | "reported-by" | "located-in" | "produced" | "finalized-as" | "delivered-by";
+}
+export interface RequirementCoverage {
+    requirementId: string;
+    implementation: boolean;
+    validation: boolean;
+    passingValidators: string[];
+    missingValidators: string[];
+    files: string[];
+    tasks: string[];
+    complete: boolean;
+}
+export interface RequirementEvidenceGraph {
+    version: 1;
+    taskId: string;
+    createdAt: string;
+    nodes: EvidenceNode[];
+    edges: EvidenceEdge[];
+    requirements: RequirementCoverage[];
+    complete: boolean;
+    reasons: string[];
+    sha256: string;
+}
+export declare function buildRequirementEvidenceGraph(input: {
+    root: string;
+    stateRoot?: string;
+    config: HarnessProjectConfig;
+    contract: TaskContract;
+    report: ValidationReport;
+    plan?: PlannerOutput;
+    findings?: NormalizedFinding[];
+    sessions?: WorkerSession[];
+    delivery?: DeliveryFinalizationResult;
+}): Promise<RequirementEvidenceGraph>;
+export declare function evidenceValidationCheck(graph: RequirementEvidenceGraph, config: HarnessProjectConfig): ValidationCheck;

package/dist/evidence/graph.js

@@ -0,0 +1,94 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { minimatch } from "minimatch";
+export async function buildRequirementEvidenceGraph(input) {
+    const stateRoot = path.resolve(input.stateRoot ?? input.root);
+    const runId = `run:${input.contract.task.id}`;
+    const nodes = new Map();
+    const edges = [];
+    addNode(nodes, { id: runId, type: "run", label: input.contract.task.title, data: { mode: input.contract.mode ?? "spec", status: input.report.status } });
+    const requirements = normalizedRequirements(input.contract);
+    const planTasks = input.plan?.tasks?.length ? input.plan.tasks : [{ id: input.contract.task.id, summary: input.contract.task.title, agent: input.contract.routing?.agent ?? "implementation-worker", scope: input.contract.scope?.allowed ?? ["**"], dependencies: [], acceptance: requirements.map((item) => item.id), risk: input.contract.routing?.risk ?? "low" }];
+    for (const requirement of requirements) {
+        const id = `req:${requirement.id}`;
+        addNode(nodes, { id, type: "requirement", label: requirement.description || requirement.id });
+        addEdge(edges, runId, id, "contains");
+    }
+    for (const task of planTasks) {
+        const id = `task:${task.id}`;
+        addNode(nodes, { id, type: "task", label: task.summary, data: { agent: task.agent, scope: task.scope, dependencies: task.dependencies } });
+        addEdge(edges, runId, id, "contains");
+        for (const req of task.acceptance)
+            if (requirements.some((item) => item.id === req))
+                addEdge(edges, `req:${req}`, id, "implemented-by");
+    }
+    for (const file of input.report.changedFiles) {
+        const id = `file:${file}`;
+        addNode(nodes, { id, type: "file", label: file });
+        addEdge(edges, runId, id, "contains");
+        for (const task of planTasks)
+            if (task.scope.some((scope) => matches(file, scope)))
+                addEdge(edges, `task:${task.id}`, id, "changed");
+    }
+    for (const check of input.report.checks) {
+        const id = `check:${check.id}`;
+        addNode(nodes, { id, type: "check", label: check.id, data: { category: check.category, status: check.status, message: check.message } });
+        addEdge(edges, runId, id, "contains");
+        for (const requirement of requirements)
+            if (requirement.validators.includes(check.id))
+                addEdge(edges, `req:${requirement.id}`, id, "validated-by");
+    }
+    for (const [index, finding] of (input.findings ?? []).entries()) {
+        const id = `finding:${finding.id || index}`;
+        addNode(nodes, { id, type: "finding", label: finding.id, data: { severity: finding.severity, category: finding.category, evidence: finding.evidence } });
+        addEdge(edges, runId, id, "contains");
+        const fileId = `file:${finding.location.file}`;
+        if (!nodes.has(fileId))
+            addNode(nodes, { id: fileId, type: "file", label: finding.location.file });
+        addEdge(edges, id, fileId, "located-in");
+    }
+    for (const [index, session] of (input.sessions ?? []).entries()) {
+        const id = `session:${session.id ?? `${session.logicalAgent ?? session.provider}-${index}`}`;
+        addNode(nodes, { id, type: "agent-session", label: session.logicalAgent ?? session.provider, data: { runtime: session.runtime, model: session.model, exitCode: session.exitCode } });
+        addEdge(edges, runId, id, "produced");
+    }
+    if (input.delivery?.commitSha) {
+        const id = `commit:${input.delivery.commitSha}`;
+        addNode(nodes, { id, type: "commit", label: input.delivery.commitSha });
+        addEdge(edges, runId, id, "finalized-as");
+    }
+    if (input.delivery?.pullRequest?.number) {
+        const id = `pr:${input.delivery.pullRequest.number}`;
+        addNode(nodes, { id, type: "pull-request", label: `PR #${input.delivery.pullRequest.number}`, data: { url: input.delivery.pullRequest.url } });
+        addEdge(edges, runId, id, "delivered-by");
+        if (input.delivery.commitSha)
+            addEdge(edges, `commit:${input.delivery.commitSha}`, id, "delivered-by");
+    }
+    const coverage = requirements.map((requirement) => coverageFor(requirement, nodes, edges));
+    const reasons = [];
+    for (const item of coverage) {
+        if (!item.implementation)
+            reasons.push(`${item.requirementId} has no changed implementation evidence.`);
+        if (!item.validation)
+            reasons.push(`${item.requirementId} lacks PASS evidence for: ${item.missingValidators.join(", ") || "a declared validator"}.`);
+    }
+    const payload = { version: 1, taskId: input.contract.task.id, createdAt: new Date().toISOString(), nodes: [...nodes.values()].sort((a, b) => a.id.localeCompare(b.id)), edges: uniqueEdges(edges), requirements: coverage, complete: coverage.every((item) => item.complete), reasons };
+    const graph = { ...payload, sha256: digest(payload) };
+    const output = path.resolve(stateRoot, input.config.evidence?.outputDir ?? ".harness/evidence", `${input.contract.task.id}.json`);
+    await fs.mkdir(path.dirname(output), { recursive: true });
+    await fs.writeFile(output, `${JSON.stringify(graph, null, 2)}\n`);
+    return graph;
+}
+export function evidenceValidationCheck(graph, config) { const strict = config.evidence?.requireComplete === true; return { id: "evidence.requirement-coverage", category: "evidence", status: graph.complete ? "PASS" : strict ? "FAIL" : "WARN", message: graph.complete ? `Requirement evidence graph complete for ${graph.requirements.length} requirement(s).` : `Requirement evidence graph is incomplete: ${graph.reasons.join("; ")}`, details: { sha256: graph.sha256, requirements: graph.requirements, strict } }; }
+function normalizedRequirements(contract) { if (contract.requirements?.length)
+    return contract.requirements.map((item) => ({ id: item.id, description: item.description ?? item.id, validators: [...new Set([...(item.validators ?? []), ...(item.validator ? [item.validator] : [])])] })); const validators = [...(contract.verification?.validators ?? []).map((item) => item.id), ...(contract.verification?.commands ?? []).map((item) => item.id)]; return (contract.quick?.acceptance ?? []).map((description, index) => ({ id: `QUICK-AC-${index + 1}`, description, validators })); }
+function coverageFor(requirement, nodes, edges) { const req = `req:${requirement.id}`; const taskIds = edges.filter((edge) => edge.from === req && edge.type === "implemented-by").map((edge) => edge.to); const files = [...new Set(taskIds.flatMap((task) => edges.filter((edge) => edge.from === task && edge.type === "changed").map((edge) => edge.to.replace(/^file:/, ""))))].sort(); const validators = requirement.validators; const passingValidators = validators.filter((id) => nodes.get(`check:${id}`)?.data?.status === "PASS"); const missingValidators = validators.filter((id) => !passingValidators.includes(id)); const implementation = taskIds.length > 0 && files.length > 0; const validation = validators.length > 0 && missingValidators.length === 0; return { requirementId: requirement.id, implementation, validation, passingValidators, missingValidators, files, tasks: taskIds.map((id) => id.replace(/^task:/, "")), complete: implementation && validation }; }
+function addNode(nodes, node) { if (!nodes.has(node.id))
+    nodes.set(node.id, node); }
+function addEdge(edges, from, to, type) { edges.push({ from, to, type }); }
+function uniqueEdges(edges) { const seen = new Set(); return edges.filter((edge) => { const key = `${edge.from}\0${edge.type}\0${edge.to}`; if (seen.has(key))
+    return false; seen.add(key); return true; }).sort((a, b) => `${a.from}:${a.type}:${a.to}`.localeCompare(`${b.from}:${b.type}:${b.to}`)); }
+function matches(file, scope) { return scope === "**" || minimatch(file, scope, { dot: true }) || (Boolean(scope.split(/[?*\[]/, 1)[0]) && file.startsWith(scope.split(/[?*\[]/, 1)[0].replace(/\/+$/, ""))); }
+function digest(value) { return crypto.createHash("sha256").update(JSON.stringify(value)).digest("hex"); }
+//# sourceMappingURL=graph.js.map

package/dist/evidence/graph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph.js","sourceRoot":"","sources":["../../src/evidence/graph.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAWtC,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAAqP;IACvS,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAAC,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IAAC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAwB,CAAC;IAAC,MAAM,KAAK,GAAmB,EAAE,CAAC;IAC9L,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzJ,MAAM,YAAY,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,IAAI,uBAAuB,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;IACvW,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,OAAO,WAAW,CAAC,EAAE,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,CAAC,WAAW,IAAI,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;IAAC,CAAC;IACrN,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,QAAQ,IAAI,CAAC,EAAE,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAAC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU;YAAE,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC;gBAAE,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAAC,CAAC;IAC3X,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,QAAQ,IAAI,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,SAAS;YAAE,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,CAAC,KAAK,EAAE,QAAQ,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;IAAC,CAAC;IAC5S,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,SAAS,KAAK,CAAC,EAAE,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAAC,KAAK,MAAM,WAAW,IAAI,YAAY;YAAE,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,OAAO,CAAC,KAAK,EAAE,OAAO,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;IAAC,CAAC;IACjZ,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,WAAW,OAAO,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAAC,MAAM,MAAM,GAAG,QAAQ,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAAC,CAAC;IAChf,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,WAAW,OAAO,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;IAAC,CAAC;IAC/X,IAAI,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,UAAU,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;IAAC,CAAC;IACvM,IAAI,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;QAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS;YAAE,OAAO,CAAC,KAAK,EAAE,UAAU,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;IAAC,CAAC;IACrY,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAAC,MAAM,OAAO,GAAa,EAAE,CAAC;IACzH,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,0CAA0C,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,6BAA6B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAAC,CAAC;IAC/R,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,CAAU,EAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACjS,MAAM,KAAK,GAA6B,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAAC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,IAAI,mBAAmB,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAAC,OAAO,KAAK,CAAC;AACjW,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAA+B,EAAE,MAA4B,IAAqB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,eAAe,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,2CAA2C,KAAK,CAAC,YAAY,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,6CAA6C,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAChkB,SAAS,sBAAsB,CAAC,QAAsB,IAAsE,IAAI,QAAQ,CAAC,YAAY,EAAE,MAAM;IAAE,OAAO,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,YAAY,KAAK,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9oB,SAAS,WAAW,CAAC,WAAiD,EAAE,KAAgC,EAAE,KAAqB,IAAyB,MAAM,GAAG,GAAG,OAAO,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC;AACziC,SAAS,OAAO,CAAC,KAAgC,EAAE,IAAkB,IAAU,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;IAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACnI,SAAS,OAAO,CAAC,KAAqB,EAAE,IAAY,EAAE,EAAU,EAAE,IAA0B,IAAU,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACvI,SAAS,WAAW,CAAC,KAAqB,IAAoB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAAE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACjV,SAAS,OAAO,CAAC,IAAY,EAAE,KAAa,IAAa,OAAO,KAAK,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxO,SAAS,MAAM,CAAC,KAAc,IAAY,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC"}

package/dist/mcp/benchmark.d.ts

@@ -0,0 +1,24 @@
+import type { HarnessProjectConfig } from "../core/types.js";
+export interface McpBenchmarkResult {
+    server: string;
+    type: "local" | "remote";
+    available: boolean;
+    baselineConfigTokens: number;
+    permissionSurfaceScore: number;
+    staleDataRisk: "low" | "medium" | "high";
+    latencyMs?: {
+        median: number;
+        min: number;
+        max: number;
+        samples: number[];
+    };
+    notes: string[];
+}
+export interface McpBenchmarkReport {
+    version: 1;
+    generatedAt: string;
+    results: McpBenchmarkResult[];
+    packs: Record<string, string[]>;
+}
+export declare function benchmarkMcpCatalog(root: string, config: HarnessProjectConfig, names?: string[]): Promise<McpBenchmarkReport>;
+export declare function resolveMcpPack(config: HarnessProjectConfig, pack: string): string[];

package/dist/mcp/benchmark.js

@@ -0,0 +1,82 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { commandExists } from "../utils/process.js";
+export async function benchmarkMcpCatalog(root, config, names) {
+    const catalog = config.mcp?.servers ?? {};
+    const selected = names?.length ? names : Object.keys(catalog).filter((name) => catalog[name].enabled !== false);
+    const repetitions = config.mcp?.benchmark?.repetitions ?? 3;
+    const results = [];
+    for (const name of selected) {
+        const server = catalog[name];
+        if (!server)
+            throw new Error(`Unknown MCP server '${name}'.`);
+        results.push(await benchmarkOne(root, name, server, repetitions));
+    }
+    const packs = {};
+    for (const [name, pack] of Object.entries(config.mcp?.packs ?? {}))
+        if (pack.enabled !== false)
+            packs[name] = pack.servers.filter((server) => Boolean(catalog[server]));
+    const report = { version: 1, generatedAt: new Date().toISOString(), results: results.sort((a, b) => a.server.localeCompare(b.server)), packs };
+    const output = path.resolve(root, config.mcp?.benchmark?.resultsDir ?? ".harness/mcp-benchmarks", `${Date.now()}.json`);
+    await fs.mkdir(path.dirname(output), { recursive: true });
+    await fs.writeFile(output, `${JSON.stringify(report, null, 2)}\n`);
+    return report;
+}
+export function resolveMcpPack(config, pack) { const value = config.mcp?.packs?.[pack]; if (!value || value.enabled === false)
+    throw new Error(`Unknown or disabled MCP pack '${pack}'.`); const catalog = config.mcp?.servers ?? {}; const missing = value.servers.filter((name) => !catalog[name]); if (missing.length)
+    throw new Error(`MCP pack '${pack}' references unknown servers: ${missing.join(", ")}`); return [...new Set(value.servers)]; }
+async function benchmarkOne(root, name, server, repetitions) {
+    const notes = [];
+    const serialized = JSON.stringify({ name, ...server });
+    const baselineConfigTokens = Math.ceil(serialized.length / 4);
+    const samples = [];
+    let available = false;
+    if (server.type === "local") {
+        const command = server.command?.[0];
+        available = Boolean(command && await commandExists(command, root));
+        if (!available)
+            notes.push(`local command '${command ?? "<missing>"}' unavailable`);
+        else
+            for (let index = 0; index < repetitions; index += 1) {
+                const started = performance.now();
+                await commandExists(command, root);
+                samples.push(performance.now() - started);
+            }
+    }
+    else {
+        for (let index = 0; index < repetitions; index += 1) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), Math.min(server.timeoutMs ?? 10_000, 10_000));
+            const started = performance.now();
+            try {
+                const response = await fetch(server.url, { method: "HEAD", headers: resolvedHeaders(server.headers), signal: controller.signal });
+                samples.push(performance.now() - started);
+                available = available || response.status < 500;
+                if (response.status === 401 || response.status === 403)
+                    notes.push(`probe reached server but authentication is required (HTTP ${response.status})`);
+            }
+            catch (error) {
+                notes.push(`probe failed: ${String(error)}`);
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        }
+    }
+    const permissionSurfaceScore = permissionSurface(server);
+    const staleDataRisk = staleRisk(name, server);
+    if (baselineConfigTokens > 500)
+        notes.push("large MCP configuration footprint; measure runtime tool-schema overhead before broad enablement");
+    return { server: name, type: server.type, available, baselineConfigTokens, permissionSurfaceScore, staleDataRisk, latencyMs: samples.length ? summarizeLatency(samples) : undefined, notes: [...new Set(notes)] };
+}
+function permissionSurface(server) { let score = server.type === "remote" ? 2 : 1; score += Object.keys(server.environment ?? {}).length; score += Object.keys(server.headers ?? {}).length; if (server.oauth)
+    score += 2; if (server.command?.some((item) => /docker|podman|kubectl|terraform|gh|aws|gcloud|az/i.test(item)))
+    score += 3; if (/write|admin|mutation/i.test(server.description ?? ""))
+    score += 4; return score; }
+function staleRisk(name, server) { const text = `${name} ${server.description ?? ""}`.toLowerCase(); if (/database|cluster|kubernetes|production|inventory|cloud/.test(text))
+    return "high"; if (/docs|documentation|issue|sentry|observability|search/.test(text))
+    return "medium"; return "low"; }
+function summarizeLatency(samples) { const sorted = [...samples].sort((a, b) => a - b); const middle = Math.floor(sorted.length / 2); const median = sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; return { median, min: sorted[0], max: sorted.at(-1), samples }; }
+function resolvedHeaders(headers) { const result = new Headers(); for (const [name, value] of Object.entries(headers ?? {}))
+    result.set(name, value.replace(/\{env:([^}]+)\}/g, (_, env) => process.env[env] ?? "")); return result; }
+//# sourceMappingURL=benchmark.js.map

package/dist/mcp/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/mcp/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAcpD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY,EAAE,MAA4B,EAAE,KAAgB;IACpG,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;IAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC;IAAC,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,WAAW,IAAI,CAAC,CAAC;IACxN,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAAC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,IAAI,CAAC,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAAC,CAAC;IAChM,MAAM,KAAK,GAA6B,EAAE,CAAC;IAAC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC;QAAE,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACpN,MAAM,MAAM,GAAuB,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;IACnK,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,IAAI,yBAAyB,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AACxQ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAA4B,EAAE,IAAY,IAAc,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK;IAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM;IAAE,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,iCAAiC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAEhe,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY,EAAE,MAAuB,EAAE,WAAmB;IAClG,MAAM,KAAK,GAAa,EAAE,CAAC;IAAC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAAC,MAAM,OAAO,GAAa,EAAE,CAAC;IAAC,IAAI,SAAS,GAAG,KAAK,CAAC;IACvM,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAAC,SAAS,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QACxG,IAAI,CAAC,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,IAAI,WAAW,eAAe,CAAC,CAAC;;YAC/E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;gBAAC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAAC,MAAM,aAAa,CAAC,OAAQ,EAAE,IAAI,CAAC,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YAAC,CAAC;IAClL,CAAC;SAAM,CAAC;QACN,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YAAC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;YAAC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChL,IAAI,CAAC;gBAAC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;gBAAC,SAAS,GAAG,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;gBAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;oBAAE,KAAK,CAAC,IAAI,CAAC,6DAA6D,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAAC,CAAC;YAC3X,OAAO,KAAK,EAAE,CAAC;gBAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAAC,CAAC;oBACvD,CAAC;gBAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAAC,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,IAAI,oBAAoB,GAAG,GAAG;QAAE,KAAK,CAAC,IAAI,CAAC,iGAAiG,CAAC,CAAC;IACvP,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AACpN,CAAC;AACD,SAAS,iBAAiB,CAAC,MAAuB,IAAY,IAAI,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK;IAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mDAAmD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;IAAE,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC;AAC3b,SAAS,SAAS,CAAC,IAAY,EAAE,MAAuB,IAA+B,MAAM,IAAI,GAAG,GAAG,IAAI,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,wDAAwD,CAAC,IAAI,CAAC,IAAI,CAAC;IAAE,OAAO,MAAM,CAAC,CAAC,IAAI,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC;IAAE,OAAO,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC;AACxV,SAAS,gBAAgB,CAAC,OAAiB,IAAqE,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AAClX,SAAS,eAAe,CAAC,OAAgC,IAAa,MAAM,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC"}

package/dist/paseo/start.d.ts

@@ -0,0 +1,47 @@
+import type { ResolvedAgentTopology } from "../agents/types.js";
+import { loadResolvedAgentTopology } from "../agents/config.js";
+import type { HarnessProjectConfig } from "../core/types.js";
+import { setupToolchain } from "../toolchain/setup.js";
+import { commandExists, runProcess } from "../utils/process.js";
+export declare const PASEO_BOOTSTRAP_VERSION = 1;
+export interface PaseoStartOptions {
+    autoSetup?: boolean;
+    webUi?: boolean;
+    forceNew?: boolean;
+    leadAgent?: string;
+    title?: string;
+    aehCommand?: string;
+}
+export interface PaseoLeadState {
+    version: 1;
+    bootstrapVersion: number;
+    projectRoot: string;
+    projectName: string;
+    agentId: string;
+    title: string;
+    leadAgent: string;
+    provider: string;
+    model: string;
+    createdAt: string;
+}
+export interface PaseoStartResult {
+    daemonStarted: boolean;
+    session: "created" | "reused";
+    agentId: string;
+    title: string;
+    leadAgent: string;
+    provider: string;
+    model: string;
+    stateFile: string;
+    bootstrapFile: string;
+}
+interface PaseoStartDeps {
+    run: typeof runProcess;
+    commandExists: typeof commandExists;
+    setupToolchain: typeof setupToolchain;
+    loadTopology: typeof loadResolvedAgentTopology;
+}
+export declare function startPaseoHarness(root: string, config: HarnessProjectConfig, options?: PaseoStartOptions, deps?: PaseoStartDeps): Promise<PaseoStartResult>;
+export declare function buildPaseoLeadBootstrap(projectName: string, projectRoot: string, aehCommand: string): string;
+export declare function resolveLeadAgent(topology: ResolvedAgentTopology, configured?: string): string;
+export {};