agent-hitch 0.2.7 → 0.2.8

package/dist/src/evals/retry-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry-state.js","sourceRoot":"","sources":["../../../src/evals/retry-state.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAExG,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACjH,OAAO,EAAE,qBAAqB,EAAE,kCAAkC,EAAE,MAAM,0BAA0B,CAAC;AAiCrG,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,aAAqB,EAAE,MAAc;IAC5E,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,EAAE,IAAI,CAAC,CAAC;IACjG,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,KAQjD;IACC,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,cAAc,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6DAA6D,CAAC,CAAC;IACjJ,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACvF,OAAO,cAAc,CAAC;QACpB,GAAG,KAAK;QACR,cAAc;QACd,WAAW,EAAE,gBAAgB;QAC7B,WAAW,EAAE,SAAS,CAAC,OAAO;QAC9B,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,KAQjD;IACC,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,WAAW,GAAG,cAAc,CAAC,cAAc,CAAC,CAAC;IACnD,OAAO,cAAc,CAAC;QACpB,GAAG,KAAK;QACR,cAAc;QACd,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,SAAS,IAAI,oBAAoB,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;KACzF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAM7C;IACC,OAAO,WAAW,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE;QAChE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,KAAK,KAAK,CAAC,UAAU,CAAC,CAAC;QACnG,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACzF,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAoB,CAAC;QAC7D,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACxF,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,QAAQ,CAAC,KAAK,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5J,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,cAAc,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,2BAA2B,CAAC,EAAE,CAAC;QACpJ,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,SAAS,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QAC3B,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC9G,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAyB,EAAE,KAA8B,EAAE,MAAc;IACxG,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;IAC1F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAoB,CAAC;IAC9C,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;QACnH,MAAM,IAAI,SAAS,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,sDAAsD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACxG,MAAM,IAAI,GAAG,kCAAkC,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACnI,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,MAAM,EAAE,CAAC,CAAC;IACvG,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAY7B;IACC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACzJ,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IAC5H,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,2BAA2B,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/I,MAAM,QAAQ,GAAoB;QAChC,WAAW,EAAE,kBAAkB,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE;QACtF,OAAO,EAAE,MAAM;QACf,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ;QACxC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,2BAA2B,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxG,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS;KACtB,CAAC;IACF,OAAO,WAAW,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE;QAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/F,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAChG,MAAM,IAAI,SAAS,CAAC,qCAAqC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACtD,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,kBAAkB,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACtI,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,aAAqB,EACrB,MAAc,EACd,SAAiG;IAEjG,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,aAAa,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,QAAQ,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;QAClH,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,mBAAmB,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;YACnH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,EAAE,IAAI,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC,EAAE,EAAE,WAAW,EAAE,yBAAyB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAc,EAAE,cAAuB;IACzE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;IAC3H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;WACvH,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;WAClH,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,KAAK,cAAc;WACjE,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QACvH,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,CAAC,CAAC;IAC1J,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACvI,OAAO;QACL,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAoB;QACrF,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE,6BAA6B,CAAC;KACxF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACzH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,QAAQ,GAAG,CAAC,gBAAgB,EAAE,eAAe,EAAE,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACrJ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;WACpF,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;WACnG,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,gBAAgB;WAChI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;IAChJ,MAAM,cAAc,GAAG,0BAA0B,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IACzE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,cAAc,EAAE,UAAU,EAAE,mBAAmB,CAAC,CAAC;WACjH,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC5J,KAAK,MAAM,KAAK,IAAI,QAAQ;QAAE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,kBAAkB,KAAK,aAAa,CAAC,CAAC;IAClL,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;QAAE,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;IACpG,IAAI,MAAM,CAAC,WAAW,KAAK,gBAAgB,IAAI,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,kDAAkD,CAAC,CAAC;IACjK,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACnG,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,cAAwB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAqB;QAChI,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,aAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK;QAC9F,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE,2BAA2B,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE,2BAA2B,CAAC;KACvJ,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,cAAuC;IAC7D,IAAI,cAAc,CAAC,YAAY,KAAK,eAAe;QAAE,OAAO,eAAe,CAAC;IAC5E,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc;QAAE,OAAO,cAAc,CAAC;IAC1E,IAAI,cAAc,CAAC,YAAY,KAAK,mBAAmB;QAAE,OAAO,mBAAmB,CAAC;IACpF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAyB;IAClD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC;IACpH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAA0B,EAAE,EAAwB;IAC7E,OAAO,IAAI,KAAK,SAAS,IAAI,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,SAAS,CAAC;WAC9D,IAAI,KAAK,SAAS,IAAI,CAAC,EAAE,KAAK,UAAU,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,WAAW,CAAC,CAAC;AAC3F,CAAC;AAED,SAAS,kBAAkB,CAAC,SAA4B;IACtD,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;WAC5G,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;AAC9H,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IACjH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAmB,KAAc,EAAE,OAAqB;IACtE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAU,CAAC,CAAC;AACnE,CAAC"}

package/dist/src/evals/scheduler-metrics.js

@@ -0,0 +1,95 @@
+import { performance } from "node:perf_hooks";
+export class EvalSchedulerMetrics {
+    startedAt = performance.now();
+    lastTransitionAt = this.startedAt;
+    active = 0;
+    weightedActiveMs = 0;
+    maxActive = 0;
+    finalSingleTailStartedAt;
+    finalSingleTailMs = 0;
+    initialWorkMs = 0;
+    retryWorkMs = 0;
+    verifierWorkMs = 0;
+    resourceBlockedMs = 0;
+    collisionBlockedMs = 0;
+    backoffBlockedMs = 0;
+    running = new Map();
+    startWork(workId, kind) {
+        if (this.running.has(workId))
+            return;
+        const now = performance.now();
+        this.transition(now);
+        this.active += 1;
+        this.maxActive = Math.max(this.maxActive, this.active);
+        if (this.active === 1)
+            this.finalSingleTailStartedAt = now;
+        else
+            this.finalSingleTailStartedAt = undefined;
+        this.running.set(workId, { kind, startedAt: now });
+    }
+    finishWork(workId) {
+        const entry = this.running.get(workId);
+        if (!entry)
+            return 0;
+        const now = performance.now();
+        this.transition(now);
+        const duration = Math.max(0, now - entry.startedAt);
+        if (entry.kind === "initial")
+            this.initialWorkMs += duration;
+        else
+            this.retryWorkMs += duration;
+        this.running.delete(workId);
+        this.active = Math.max(0, this.active - 1);
+        if (this.active === 1)
+            this.finalSingleTailStartedAt = now;
+        else if (this.active === 0 && this.finalSingleTailStartedAt !== undefined) {
+            this.finalSingleTailMs = now - this.finalSingleTailStartedAt;
+        }
+        return Math.max(1, Math.round(duration));
+    }
+    addVerifier(durationMs) {
+        if (Number.isFinite(durationMs) && durationMs >= 0)
+            this.verifierWorkMs += durationMs;
+    }
+    addBlocked(kind, durationMs) {
+        if (!Number.isFinite(durationMs) || durationMs < 0)
+            return;
+        if (kind === "resource")
+            this.resourceBlockedMs += durationMs;
+        else if (kind === "collision")
+            this.collisionBlockedMs += durationMs;
+        else
+            this.backoffBlockedMs += durationMs;
+    }
+    summary(input) {
+        const now = performance.now();
+        this.transition(now);
+        const makespan = Math.max(1, now - this.startedAt);
+        const physical = this.initialWorkMs + this.retryWorkMs;
+        return {
+            policy: input.prioritized ? "critical-path-lpt-v1" : "fifo-compat",
+            makespan_ms: Math.round(makespan),
+            physical_work_ms: Math.round(physical),
+            initial_work_ms: Math.round(this.initialWorkMs),
+            retry_work_ms: Math.round(this.retryWorkMs),
+            verifier_work_ms: Math.round(this.verifierWorkMs),
+            max_active: this.maxActive,
+            effective_parallelism: rounded(this.weightedActiveMs / makespan),
+            slot_utilization: rounded(this.weightedActiveMs / (makespan * Math.max(1, input.maxParallelism))),
+            single_active_tail_ms: Math.round(this.active === 1 && this.finalSingleTailStartedAt !== undefined
+                ? now - this.finalSingleTailStartedAt : this.finalSingleTailMs),
+            resource_blocked_ms: Math.round(this.resourceBlockedMs),
+            collision_blocked_ms: Math.round(this.collisionBlockedMs),
+            backoff_blocked_ms: Math.round(this.backoffBlockedMs),
+            verifier_skipped: input.verifierSkipped,
+        };
+    }
+    transition(now) {
+        this.weightedActiveMs += Math.max(0, now - this.lastTransitionAt) * this.active;
+        this.lastTransitionAt = now;
+    }
+}
+function rounded(value) {
+    return Math.round(value * 1_000) / 1_000;
+}
+//# sourceMappingURL=scheduler-metrics.js.map

package/dist/src/evals/scheduler-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler-metrics.js","sourceRoot":"","sources":["../../../src/evals/scheduler-metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,OAAO,oBAAoB;IACd,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACvC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;IAClC,MAAM,GAAG,CAAC,CAAC;IACX,gBAAgB,GAAG,CAAC,CAAC;IACrB,SAAS,GAAG,CAAC,CAAC;IACd,wBAAwB,CAAqB;IAC7C,iBAAiB,GAAG,CAAC,CAAC;IACtB,aAAa,GAAG,CAAC,CAAC;IAClB,WAAW,GAAG,CAAC,CAAC;IAChB,cAAc,GAAG,CAAC,CAAC;IACnB,iBAAiB,GAAG,CAAC,CAAC;IACtB,kBAAkB,GAAG,CAAC,CAAC;IACvB,gBAAgB,GAAG,CAAC,CAAC;IACZ,OAAO,GAAG,IAAI,GAAG,EAA4D,CAAC;IAE/F,SAAS,CAAC,MAAc,EAAE,IAAyB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QACrC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC;;YACtD,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC;;YACxD,IAAI,CAAC,WAAW,IAAI,QAAQ,CAAC;QAClC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC;aACtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,wBAAwB,KAAK,SAAS,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,GAAG,GAAG,GAAG,IAAI,CAAC,wBAAwB,CAAC;QAC/D,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,UAAkB;QAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC;YAAE,IAAI,CAAC,cAAc,IAAI,UAAU,CAAC;IACxF,CAAC;IAED,UAAU,CAAC,IAA0C,EAAE,UAAkB;QACvE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;YAAE,OAAO;QAC3D,IAAI,IAAI,KAAK,UAAU;YAAE,IAAI,CAAC,iBAAiB,IAAI,UAAU,CAAC;aACzD,IAAI,IAAI,KAAK,WAAW;YAAE,IAAI,CAAC,kBAAkB,IAAI,UAAU,CAAC;;YAChE,IAAI,CAAC,gBAAgB,IAAI,UAAU,CAAC;IAC3C,CAAC;IAED,OAAO,CAAC,KAAgF;QACtF,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC;QACvD,OAAO;YACL,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa;YAClE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;YACjC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;YACtC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC;YAC/C,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;YAC3C,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;YACjD,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,qBAAqB,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;YAChE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YACjG,qBAAqB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,wBAAwB,KAAK,SAAS;gBAChG,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACjE,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACvD,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;YACzD,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;YACrD,gBAAgB,EAAE,KAAK,CAAC,eAAe;SACxC,CAAC;IACJ,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAChF,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;AAC3C,CAAC"}

package/dist/src/evals/scheduler-trace-replay.js

@@ -0,0 +1,107 @@
+export function replaySchedulerTrace(work, options) {
+    if (!Number.isSafeInteger(options.slots) || options.slots < 1)
+        throw new TypeError("trace replay slots are invalid");
+    const chains = traceChains(work);
+    const replay = options.retryScheduling === "batch-v1"
+        ? replayBatch(chains, options.slots, options.policy)
+        : replayImmediate(chains, options.slots, options.policy);
+    const physicalWorkMs = work.reduce((sum, item) => sum + item.duration_ms, 0);
+    return {
+        makespan_ms: replay.makespanMs,
+        physical_work_ms: physicalWorkMs,
+        max_active: replay.maxActive,
+        slot_utilization: rounded(physicalWorkMs / (Math.max(1, replay.makespanMs) * options.slots)),
+        runnable_idle_ms: 0,
+    };
+}
+function traceChains(work) {
+    const byTask = new Map();
+    const ids = new Set();
+    for (const item of work) {
+        validateTraceWork(item);
+        if (ids.has(item.work_id))
+            throw new TypeError(`trace work identity is duplicated: ${item.work_id}`);
+        ids.add(item.work_id);
+        const stages = byTask.get(item.task_id) ?? [];
+        stages.push(item);
+        byTask.set(item.task_id, stages);
+    }
+    const chains = [...byTask.entries()].map(([taskId, stages]) => ({
+        taskId,
+        stages: stages.sort((left, right) => left.retry_index - right.retry_index),
+    }));
+    for (const chain of chains) {
+        if (chain.stages[0]?.execution_kind !== "initial" || chain.stages[0]?.retry_index !== 0
+            || chain.stages.some((stage, index) => stage.retry_index !== index
+                || (index > 0 && stage.execution_kind !== "physical-infrastructure-retry"))) {
+            throw new TypeError(`trace work chain is invalid: ${chain.taskId}`);
+        }
+    }
+    return chains.sort((left, right) => compareBytes(left.taskId, right.taskId));
+}
+function replayBatch(chains, slots, policy) {
+    let now = 0;
+    let maxActive = 0;
+    const stages = Math.max(0, ...chains.map((chain) => chain.stages.length));
+    for (let index = 0; index < stages; index += 1) {
+        const ready = chains.flatMap((chain) => chain.stages[index]
+            ? [{ chain, index, remainingMs: remaining(chain, index) }]
+            : []);
+        maxActive = Math.max(maxActive, Math.min(slots, ready.length));
+        now = scheduleReady(ready, slots, policy, now);
+    }
+    return { makespanMs: now, maxActive };
+}
+function replayImmediate(chains, slots, policy) {
+    const ready = chains.map((chain) => ({ chain, index: 0, remainingMs: remaining(chain, 0) }));
+    const running = [];
+    let now = 0;
+    let maxActive = 0;
+    while (ready.length > 0 || running.length > 0) {
+        ready.sort((left, right) => compareReady(left, right, policy));
+        while (running.length < slots && ready.length > 0) {
+            const item = ready.shift();
+            running.push({ ...item, completedAt: now + (item.chain.stages[item.index]?.duration_ms ?? 0) });
+        }
+        maxActive = Math.max(maxActive, running.length);
+        running.sort((left, right) => left.completedAt - right.completedAt || compareBytes(left.chain.taskId, right.chain.taskId));
+        const completed = running.shift();
+        if (!completed)
+            break;
+        now = completed.completedAt;
+        const next = completed.index + 1;
+        if (completed.chain.stages[next])
+            ready.push({ chain: completed.chain, index: next, remainingMs: remaining(completed.chain, next) });
+    }
+    return { makespanMs: now, maxActive };
+}
+function scheduleReady(ready, slots, policy, start) {
+    ready.sort((left, right) => compareReady(left, right, policy));
+    const available = Array.from({ length: slots }, () => start);
+    for (const item of ready) {
+        const earliest = Math.min(...available);
+        const slot = available.indexOf(earliest);
+        available[slot] = earliest + (item.chain.stages[item.index]?.duration_ms ?? 0);
+    }
+    return Math.max(start, ...available);
+}
+function compareReady(left, right, policy) {
+    if (policy === "critical-path-v1" && left.remainingMs !== right.remainingMs)
+        return right.remainingMs - left.remainingMs;
+    const leftStage = left.chain.stages[left.index];
+    const rightStage = right.chain.stages[right.index];
+    return leftStage.queued_order - rightStage.queued_order || compareBytes(leftStage.work_id, rightStage.work_id);
+}
+function remaining(chain, index) {
+    return chain.stages.slice(index).reduce((sum, stage) => sum + stage.duration_ms, 0);
+}
+function validateTraceWork(item) {
+    if (!item.work_id || !item.task_id || !Number.isSafeInteger(item.duration_ms) || item.duration_ms < 1
+        || !Number.isSafeInteger(item.queued_order) || item.queued_order < 0
+        || !Number.isSafeInteger(item.retry_index) || item.retry_index < 0
+        || (item.execution_kind === "initial") !== (item.retry_index === 0))
+        throw new TypeError("scheduler trace work is invalid");
+}
+function rounded(value) { return Math.round(value * 1_000) / 1_000; }
+function compareBytes(left, right) { return Buffer.compare(Buffer.from(left), Buffer.from(right)); }
+//# sourceMappingURL=scheduler-trace-replay.js.map

package/dist/src/evals/scheduler-trace-replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler-trace-replay.js","sourceRoot":"","sources":["../../../src/evals/scheduler-trace-replay.ts"],"names":[],"mappings":"AAiBA,MAAM,UAAU,oBAAoB,CAClC,IAAqC,EACrC,OAIC;IAED,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,gCAAgC,CAAC,CAAC;IACrH,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,KAAK,UAAU;QACnD,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC;QACpD,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC7E,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,UAAU;QAC9B,gBAAgB,EAAE,cAAc;QAChC,UAAU,EAAE,MAAM,CAAC,SAAS;QAC5B,gBAAgB,EAAE,OAAO,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5F,gBAAgB,EAAE,CAAC;KACpB,CAAC;AACJ,CAAC;AAkBD,SAAS,WAAW,CAAC,IAAqC;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkC,CAAC;IACzD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9D,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;KAC3E,CAAC,CAAC,CAAC;IACJ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,KAAK,CAAC;eAClF,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,KAAK;mBAC7D,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,+BAA+B,CAAC,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,SAAS,CAAC,gCAAgC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,WAAW,CAAC,MAA6B,EAAE,KAAa,EAAE,MAAsC;IACvG,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;YACzD,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;YAC1D,CAAC,CAAC,EAAE,CAAC,CAAC;QACR,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/D,GAAG,GAAG,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,SAAS,eAAe,CAAC,MAA6B,EAAE,KAAa,EAAE,MAAsC;IAC3G,MAAM,KAAK,GAAiB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3G,MAAM,OAAO,GAAgD,EAAE,CAAC;IAChE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAgB,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3H,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,CAAC,SAAS;YAAE,MAAM;QACtB,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC;QAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QACjC,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IACvI,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,KAAmB,EAAE,KAAa,EAAE,MAAsC,EAAE,KAAa;IAC9G,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,IAAgB,EAAE,KAAiB,EAAE,MAAsC;IAC/F,IAAI,MAAM,KAAK,kBAAkB,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACzH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAyB,CAAC;IACxE,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAyB,CAAC;IAC3E,OAAO,SAAS,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,IAAI,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;AACjH,CAAC;AAED,SAAS,SAAS,CAAC,KAAiB,EAAE,KAAa;IACjD,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,iBAAiB,CAAC,IAA0B;IACnD,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC;WAChG,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC;WACjE,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC;WAC/D,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;AAChI,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,IAAY,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AACrF,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa,IAAY,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/service.js

@@ -7,14 +7,13 @@ import { ensureControllerRuntime, writeRuntimeReference } from "../controller-ru
 import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
 import { EvalEventSink } from "./events.js";
 import { createEvalProgress, mergeEvalProgressTrial, writeEvalProgress } from "./progress.js";
-import { infrastructureFailureTrials, runInfrastructureRetries } from "./infrastructure-retry.js";
+import { runInfrastructureRetries } from "./infrastructure-retry.js";
 import { newEvalId, resolveLocalDatasetTaskIds, validateEvalId, validateEvalRequest } from "./request.js";
-import { invalidTrialSlots } from "./rerun-slots.js";
 import { prepareEvalDirectory } from "./directory.js";
 import { buildEvalExecutionPlan, DEFAULT_EVAL_TRIAL_RESOURCES } from "./execution-plan.js";
 import { planLocalEvalInputs } from "./local-eval-planning.js";
 import { resolvedImageMapping } from "./environment-image-planning.js";
-import { assertBackendTrialSet, attemptDirectoryName, localSourceBackendFailure, summarizeTrialRefs, transportSummary } from "./result-helpers.js";
+import { assertBackendTrialSet, attemptDirectoryName, localSourceBackendFailure, transportSummary } from "./result-helpers.js";
 import { executePlannedHarborTasks } from "./planned-execution.js";
 import { assertEvalResumeState, executionPlanWorkState, loadEvalResumeState } from "./resume-state.js";
 import { modelCaptureDegradationEvent, resolveEvalModelCapturePlan } from "./model-capture-plan.js";
@@ -25,7 +24,11 @@ import { prepareEvalArtifactAssignments, preparedArtifactPlanFields } from "./ev
 import { startEvalModelCaptureRuntime } from "./model-capture-runtime.js";
 import { recoverPromotedEvalTrialPublications } from "./trial-publication-recovery.js";
 import { emitEvalPlanLifecycle } from "./eval-lifecycle-events.js";
-export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent, trialBundleGraceMs, precreated = false, normalizedRequest, maxConcurrentOverride, executionResources, executionResourceSource = "operator-default", executionStrategy = "legacy-attempt-shards", executionWorker, modelCapturePlan, workItemAdmission, remoteWorkExecutor, resumeExisting = false, onControlPhase, onWorkItemState, dockerResourceReaper, environmentBuildMode = "backend", environmentImageResolver, environmentImageBuilder, environmentImageManifestLoader, harborArtifactBuilder }) {
+import { materializeEvalPlan, writeEvalPlanningCheckpoint } from "./eval-logical-plan.js";
+import { planTaskSchedulingHints, schedulingHintsFromPlan } from "./duration-estimator.js";
+import { buildCompletedEvalResult } from "./eval-result-builder.js";
+import { loadBenchmarkAdapterManifest } from "./benchmark-adapter-manifest.js";
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent, trialBundleGraceMs, precreated = false, replaceTerminal = false, normalizedRequest, maxConcurrentOverride, executionResources, executionResourceSource = "operator-default", executionStrategy = "legacy-attempt-shards", executionWorker, modelCapturePlan, workItemAdmission, remoteWorkExecutor, resumeExisting = false, onControlPhase, onWorkItemState, onWorkItemQueued, evolutionBaselineDurations, dockerResourceReaper, environmentBuildMode = "backend", environmentImageResolver, environmentImageBuilder, environmentImageManifestLoader, harborArtifactBuilder }) {
     if (!root)
         throw invalidInput("a Hitch state root is required for eval");
     evalId = validateEvalId(evalId);
@@ -35,14 +38,16 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
     }
     const normalized = maxConcurrentOverride === undefined ? persistedRequest : { ...persistedRequest, max_concurrent: maxConcurrentOverride };
     const evalsDirectory = await ensureDir(statePaths(root).evals);
-    const evalDirectory = await prepareEvalDirectory({ evalsDirectory, evalId, request: persistedRequest, precreated });
+    const evalDirectory = await prepareEvalDirectory({ evalsDirectory, evalId, request: persistedRequest, precreated, replaceTerminal });
     let startedAt = new Date();
     const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
     await sink.open();
     let result;
     let trialRefs = [];
     let progress = null;
+    let schedulerSummary;
     let captureRuntime;
+    let failureStage = "planning";
     try {
         const planningStartedAt = Date.now();
         sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
@@ -50,6 +55,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         await onControlPhase?.("planning");
         const requestedReference = parseHarnessReference(normalized.harness_ref);
         const resolvedRevision = await resolveHarness(normalized.harness_ref, { root, env });
+        failureStage = "preparing";
         await onControlPhase?.("preparing");
         let localTransport;
         if (resolvedRevision.source.type === "git" && resolvedRevision.source.registered !== true) {
@@ -84,16 +90,11 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             reference: runtimeRefFile,
         });
         const localTaskIds = await resolveLocalDatasetTaskIds(normalized.dataset);
+        const standardDataset = localTaskIds !== null && await loadBenchmarkAdapterManifest(normalized.dataset) !== null;
         const resume = resumeExisting ? await loadEvalResumeState(evalDirectory) : null;
         await withEnvironmentImageReferenceLock(root, () => beginEvalEnvironmentImagePlanning(evalDirectory, evalId));
         const localPlanning = await planLocalEvalInputs({ root, dataset: normalized.dataset, taskIds: localTaskIds, defaultResources: executionResources ?? DEFAULT_EVAL_TRIAL_RESOURCES, defaultSource: executionResourceSource, benchmarkId: normalized.benchmark_id, benchmarkRevision: normalized.benchmark_revision, buildMode: environmentBuildMode, harborTaskResourceInspector: path.join(controllerRuntime.directory, "payload", "integrations", "harbor", "hitch_harbor_task_resources.py"), ...(environmentImageResolver ? { resolver: environmentImageResolver } : {}), ...(environmentImageBuilder ? { builder: environmentImageBuilder } : {}), ...(resume ? { resumePlan: resume.executionPlan } : {}), ...(harborExecutable ? { harborExecutable } : {}), env, ...(signal ? { signal } : {}) });
         await withEnvironmentImageReferenceLock(root, () => writeEvalEnvironmentImageReferences(evalDirectory, evalId, localPlanning.environmentImages));
-        const selectedArtifactBuilder = harborArtifactBuilder ?? (env.NODE_ENV === "test" && env.HITCH_TEST_HOST_ARTIFACT_BUILDER === "1" ? prepareHostHarborArtifactForTest : prepareHarborArtifact);
-        const prepared = await prepareEvalArtifactAssignments({ builder: selectedArtifactBuilder, root, resolvedRevision, requestedReference, controllerRuntime, ...(localTransport ? { localTransport } : {}), taskRuntimeContracts: localPlanning.taskRuntimeContracts, sink, env, ...(signal ? { signal } : {}) });
-        const preparedAssignments = prepared.assignments;
-        const preparedArtifact = prepared.primary;
-        const preparedArtifacts = prepared.artifactsById;
-        const multipleRuntimeContracts = preparedAssignments.length > 1;
         const taskResources = localPlanning.taskResources;
         const plannedTasks = localTaskIds?.length ?? null;
         const plannedTrials = plannedTasks === null ? null : plannedTasks * normalized.attempts;
@@ -102,26 +103,18 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         if (resume)
             startedAt = new Date(resume.progress.started_at);
         const capture = resolveEvalModelCapturePlan({ requested: modelCapturePlan, resumed: resume?.executionPlan.model_capture, resuming: Boolean(resume) });
-        const activeCaptureRuntime = await startEvalModelCaptureRuntime({ plan: capture.plan, evalId, evalDirectory, env });
-        captureRuntime = activeCaptureRuntime;
-        capture.plan = activeCaptureRuntime.plan;
-        const plan = {
+        const multipleRuntimeContracts = localPlanning.taskRuntimeContracts.length > 1;
+        const logicalPlan = {
             schema_version: SCHEMA_VERSION,
+            kind: "eval-logical-plan",
             eval_id: evalId,
             backend: "harbor",
-            candidate: {
-                id: "candidate-1",
-                requested_harness_ref: normalized.harness_ref,
-                harness_ref: lockedHarnessRef(resolvedRevision),
-                harness_id: resolvedRevision.harness_id,
-                revision_identity: resolvedRevision.identity,
-                model: normalized.model || null,
-            },
+            candidate: { id: "candidate-1", requested_harness_ref: normalized.harness_ref, harness_ref: lockedHarnessRef(resolvedRevision), harness_id: resolvedRevision.harness_id, revision_identity: resolvedRevision.identity, model: normalized.model || null },
             dataset: normalized.dataset,
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
             attempts: normalized.attempts,
-            attempt_execution: (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null
+            attempt_execution: (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts || standardDataset) && localTaskIds !== null
                 ? "harbor-task-slots-v1"
                 : "harbor-attempt-shards-v1",
             max_concurrent: normalized.max_concurrent,
@@ -129,14 +122,49 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             infrastructure_retries: normalized.infrastructure_retries,
             infrastructure_retry_backoff_ms: normalized.infrastructure_retry_backoff_ms,
             ...(localTaskIds === null ? {} : { tasks: localTaskIds }),
-            controller_runtime: {
-                runtime_id: controllerRuntime.runtime_id,
-                manifest_digest: controllerRuntime.manifest_digest,
-            },
-            ...preparedArtifactPlanFields(preparedAssignments),
+            controller_runtime: { runtime_id: controllerRuntime.runtime_id, manifest_digest: controllerRuntime.manifest_digest },
             ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
             created_at: typeof resume?.plan.created_at === "string" ? resume.plan.created_at : new Date().toISOString(),
         };
+        progress = resume?.progress ?? createEvalProgress({
+            evalId,
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+            plannedTasks,
+            plannedTrials,
+            startedAt: startedAt.toISOString(),
+        });
+        if (!resume)
+            await Promise.all([
+                atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision),
+                writeEvalPlanningCheckpoint(evalDirectory, logicalPlan, progress),
+            ]);
+        const selectedArtifactBuilder = harborArtifactBuilder ?? (env.NODE_ENV === "test" && env.HITCH_TEST_HOST_ARTIFACT_BUILDER === "1" ? prepareHostHarborArtifactForTest : prepareHarborArtifact);
+        const prepared = await prepareEvalArtifactAssignments({ builder: selectedArtifactBuilder, root, resolvedRevision, requestedReference, controllerRuntime, ...(localTransport ? { localTransport } : {}), taskRuntimeContracts: localPlanning.taskRuntimeContracts, sink, env, ...(signal ? { signal } : {}) });
+        const preparedAssignments = prepared.assignments;
+        const preparedArtifact = prepared.primary;
+        const preparedArtifacts = prepared.artifactsById;
+        const activeCaptureRuntime = await startEvalModelCaptureRuntime({ plan: capture.plan, evalId, evalDirectory, env });
+        captureRuntime = activeCaptureRuntime;
+        capture.plan = activeCaptureRuntime.plan;
+        const plan = materializeEvalPlan(logicalPlan, capture.persist ? capture.plan : undefined, preparedArtifactPlanFields(preparedAssignments));
+        const taskSlotPlanning = (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts || standardDataset) && localTaskIds !== null;
+        const taskScheduling = taskSlotPlanning
+            ? resume
+                ? schedulingHintsFromPlan(resume.executionPlan)
+                : await planTaskSchedulingHints({
+                    root,
+                    dataset: normalized.dataset,
+                    taskIds: localTaskIds,
+                    benchmarkId: normalized.benchmark_id,
+                    benchmarkRevision: normalized.benchmark_revision,
+                    provider: executionWorker?.provider ?? "local-docker",
+                    model: normalized.model,
+                    requestTimeoutMs: normalized.timeout_ms,
+                    infrastructureRetries: normalized.infrastructure_retries,
+                    ...(evolutionBaselineDurations ? { evolutionBaselineDurations } : {}),
+                })
+            : undefined;
         const expectedExecutionPlan = buildEvalExecutionPlan({
             evalId,
             request: normalized,
@@ -161,14 +189,14 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             ...(localPlanning.environmentImageFallbacks.length > 0 ? { environmentImageFallbacks: localPlanning.environmentImageFallbacks } : {}),
             ...(executionWorker ? { provider: executionWorker.provider } : {}),
             modelCapture: capture.persist ? capture.plan : null,
-            ...((executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null ? { workItemMode: "task-slots" } : {}),
-            createdAt: plan.created_at,
+            ...(taskSlotPlanning ? { workItemMode: "task-slots" } : {}),
+            ...(taskScheduling ? { taskScheduling } : {}),
+            createdAt: logicalPlan.created_at,
         });
         if (resume)
             assertEvalResumeState({ state: resume, expectedPlan: plan, expectedExecutionPlan, expectedResolutionIdentity: resolvedRevision.identity, plannedTasks, plannedTrials });
         else
             await Promise.all([
-                atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision),
                 atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan),
                 atomicWriteJSON(path.join(evalDirectory, "execution-plan.json"), expectedExecutionPlan),
             ]);
@@ -176,17 +204,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         const captureDegradation = modelCaptureDegradationEvent(executionPlan.model_capture);
         if (captureDegradation)
             sink.emit(captureDegradation);
-        progress = resume?.progress ?? createEvalProgress({
-            evalId,
-            benchmarkId: normalized.benchmark_id,
-            benchmarkRevision: normalized.benchmark_revision,
-            plannedTasks,
-            plannedTrials,
-            startedAt: startedAt.toISOString(),
-        });
-        if (!resume)
-            await writeEvalProgress(evalDirectory, progress);
-        else
+        if (resume)
             progress = (await recoverPromotedEvalTrialPublications({ root, evalDirectory, plan: executionPlan, progress, sink })).progress;
         sink.emit({
             type: "eval.planned",
@@ -198,7 +216,18 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             work_items: executionPlan.work_items.length,
             membership: executionPlan.membership,
         });
+        sink.emit({
+            type: "eval.execution-strategy.verified",
+            execution_strategy: taskSlotPlanning ? "local-task-slots-v1" : "legacy-attempt-shards",
+            attempt_execution: plan.attempt_execution,
+            controller_runtime_id: controllerRuntime.runtime_id,
+            harness_revision_identity: resolvedRevision.identity,
+            membership: executionPlan.membership,
+            planned_trials: plannedTrials,
+            work_items: executionPlan.work_items.length,
+        });
         emitEvalPlanLifecycle(sink, executionPlan, planningStartedAt);
+        failureStage = "running";
         await onControlPhase?.("running", executionPlanWorkState(executionPlan, progress));
         if (localTransport) {
             await verifyLocalGitTransport(localTransport, {
@@ -213,7 +242,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         }
         const backendRuns = [];
         const infrastructureRetryRuns = [];
-        const plannedTaskExecution = (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null;
+        const plannedTaskExecution = (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts || standardDataset) && localTaskIds !== null;
         if (plannedTaskExecution) {
             const execution = await executePlannedHarborTasks({
                 evalId,
@@ -239,6 +268,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                 },
                 ...(workItemAdmission ? { admission: workItemAdmission } : {}),
                 ...(onWorkItemState ? { onWorkItemState } : {}),
+                ...(onWorkItemQueued ? { onWorkItemQueued } : {}),
                 ...(dockerResourceReaper ? { dockerResourceReaper } : {}),
                 ...(environmentImageManifestLoader ? { environmentImageManifestLoader } : {}),
                 ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
@@ -253,6 +283,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                 leaseId: entry.leaseId,
             })));
             infrastructureRetryRuns.push(...execution.infrastructureRetryRuns);
+            schedulerSummary = execution.schedulerSummary;
         }
         else
             for (let logicalAttempt = 1; logicalAttempt <= normalized.attempts; logicalAttempt += 1) {
@@ -326,7 +357,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                                 benchmarkId: normalized.benchmark_id,
                                 benchmarkRevision: normalized.benchmark_revision,
                                 runtimeId: controllerRuntime.runtime_id,
-                                env,
+                                env, ...(signal ? { signal } : {}),
                                 modelCapturePlan: activeCaptureRuntime.plan,
                                 ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
                                 requireCompleteMarker: true,
@@ -354,7 +385,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                     benchmarkId: normalized.benchmark_id,
                     benchmarkRevision: normalized.benchmark_revision,
                     runtimeId: controllerRuntime.runtime_id,
-                    env,
+                    env, ...(signal ? { signal } : {}),
                     modelCapturePlan: activeCaptureRuntime.plan,
                     ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
                     rawResult: backendRun.rawResult,
@@ -400,100 +431,17 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             }
         if (progress === null)
             throw new Error("eval progress was not initialized");
+        failureStage = "finalizing";
         await onControlPhase?.("finalizing");
         trialRefs = progress.trials;
-        const cancelled = signal?.aborted === true;
-        const localSourceFailure = localTransport
-            ? backendRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
-                || infrastructureRetryRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
-            : false;
-        const expectedBackendRuns = plannedTaskExecution ? executionPlan.work_items.length : normalized.attempts;
-        const backendsSucceeded = backendRuns.every(({ run }) => run.backend.process_exit_code === 0 && run.rawResult !== null)
-            && (plannedTaskExecution ? progress.trials.length === plannedTrials : backendRuns.length === expectedBackendRuns);
-        const invalidTrials = localTaskIds === null
-            ? trialRefs.filter((trial) => trial.observation_status !== "valid").map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }))
-            : invalidTrialSlots(localTaskIds, normalized.attempts, progress);
-        const infrastructureFailures = infrastructureFailureTrials(trialRefs);
-        const infrastructureFailureSlots = infrastructureFailures.map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }));
-        const verifierRetriesExhausted = normalized.infrastructure_retries > 0 && infrastructureFailures.some((trial) => trial.invalid_reason === "verifier_infrastructure_failure");
-        const infrastructureErrorCode = verifierRetriesExhausted || (normalized.infrastructure_retries > 0 && infrastructureRetryRuns.length > 0)
-            ? "eval_infrastructure_retries_exhausted"
-            : "eval_has_infrastructure_failures";
-        const succeeded = !cancelled && !localSourceFailure && backendsSucceeded && invalidTrials.length === 0;
-        const singleBackend = backendRuns.length === 1 ? backendRuns[0].run : undefined;
-        result = {
-            schema_version: SCHEMA_VERSION,
-            eval_id: evalId,
-            status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
-            exit_code: cancelled ? 9 : succeeded ? 0 : 13,
-            ...(singleBackend ? { backend: singleBackend.backend, backend_summary: singleBackend.summary } : {}),
-            ...(plannedTaskExecution ? {
-                backend_work_items: backendRuns.map(({ attempt, workId, tasks, leaseId, run }) => ({
-                    work_id: workId,
-                    lease_id: leaseId,
-                    attempt,
-                    tasks,
-                    backend: run.backend,
-                    backend_summary: run.summary,
-                })),
-            } : normalized.attempts > 1 ? {
-                backend_runs: backendRuns.map(({ attempt, run }) => ({
-                    attempt,
-                    backend: run.backend,
-                    backend_summary: run.summary,
-                })),
-            } : {}),
-            infrastructure_retry_policy: {
-                max_retries: normalized.infrastructure_retries,
-                backoff_ms: normalized.infrastructure_retry_backoff_ms,
-                verifier_execution: "same_trial_verifier_only",
-                candidate_rerun_on_verifier_failure: false,
-            },
-            ...(infrastructureRetryRuns.length > 0 ? {
-                infrastructure_retry_runs: infrastructureRetryRuns.map(({ attempt, retry, tasks, triggers, refs, run, leaseId, workId }) => ({
-                    execution_kind: "physical-infrastructure-retry", ...(leaseId ? { lease_id: leaseId } : {}), ...(workId ? { work_id: workId } : {}), attempt,
-                    retry,
-                    tasks,
-                    trigger_trials: triggers,
-                    trials: refs,
-                    backend: run.backend,
-                    backend_summary: run.summary,
-                })),
-            } : {}),
-            candidate: plan.candidate,
-            dataset: normalized.dataset,
-            benchmark_id: normalized.benchmark_id,
-            benchmark_revision: normalized.benchmark_revision,
-            generation: progress.generation,
-            trials: trialRefs,
-            summary: summarizeTrialRefs(trialRefs),
-            ...preparedArtifactPlanFields(preparedAssignments),
-            ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
-            ...(succeeded ? {} : {
-                error: {
-                    code: cancelled
-                        ? "cancelled"
-                        : localSourceFailure
-                            ? "local_source_materialize_failed"
-                            : !backendsSucceeded
-                                ? "harbor_failed"
-                                : infrastructureFailures.length > 0
-                                    ? infrastructureErrorCode
-                                    : "eval_has_invalid_tasks",
-                    message: cancelled
-                        ? "eval was cancelled"
-                        : localSourceFailure
-                            ? "Harbor rejected the transported local Git source before candidate execution"
-                            : !backendsSucceeded
-                                ? `Harbor work items completed ${backendRuns.length}/${expectedBackendRuns}`
-                                : infrastructureFailures.length > 0
-                                    ? `${infrastructureErrorCode === "eval_infrastructure_retries_exhausted" ? "infrastructure retries exhausted" : "verifier infrastructure failure"}: ${infrastructureFailureSlots.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`
-                                    : `eval has invalid or missing trials: ${invalidTrials.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`,
-                },
-            }),
-            started_at: startedAt.toISOString(),
-            completed_at: new Date().toISOString(),
-        };
+        result = buildCompletedEvalResult({
+            evalId, request: normalized, plannedTaskExecution, plannedTrials,
+            executionWorkItems: executionPlan.work_items.length, localTaskIds, backendRuns,
+            infrastructureRetryRuns, candidate: plan.candidate, progress,
+            ...(schedulerSummary ? { schedulerSummary } : {}), preparedAssignments,
+            ...(localTransport ? { localTransport } : {}), startedAt,
+            cancelled: signal?.aborted === true,
+        });
     }
     catch (error) {
         trialRefs = progress?.trials ?? trialRefs;
@@ -507,6 +455,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                 code: signal?.aborted ? "cancelled" : typed ? error.code : "internal_error",
                 message: safeDiagnosticMessage(error, credentialValuesFromEnv(normalized.pass_env, env)),
             },
+            failure_stage: failureStage,
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
             ...(progress === null ? {} : { generation: progress.generation }),