agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/request.js.map

@@ -1 +1 @@
-{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAC7D,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,wBAAwB,EAAE,iCAAiC;QAC7E,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC3D,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,sBAAsB,IAAI,8BAA8B,EAC9D,wBAAwB,CACzB,CAAC;IACF,MAAM,0BAA0B,GAAG,iBAAiB,CAClD,KAAK,CAAC,+BAA+B,IAAI,uCAAuC,EAChF,iCAAiC,CAClC,CAAC;IACF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,sBAAsB,EAAE,qBAAqB;QAC7C,+BAA+B,EAAE,0BAA0B;QAC3D,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,IAAY;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IAC9G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAC7D,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,wBAAwB,EAAE,iCAAiC;QAC7E,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC3D,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,sBAAsB,IAAI,8BAA8B,EAC9D,wBAAwB,CACzB,CAAC;IACF,MAAM,0BAA0B,GAAG,iBAAiB,CAClD,KAAK,CAAC,+BAA+B,IAAI,uCAAuC,EAChF,iCAAiC,CAClC,CAAC;IACF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,EAAE,YAAY,CAAC,CAAC;IACxH,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACvB,MAAM,gCAAgC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACpH,CAAC;IACD,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,sBAAsB,EAAE,qBAAqB;QAC7C,+BAA+B,EAAE,0BAA0B;QAC3D,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,EAAE,YAAY,EAAE,kBAAkB,EAAE,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACpF,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,CAAC;AAC9C,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,OAAe;IAKpD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC,KAAK,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO;oBACL,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE;oBACnC,kBAAkB,EAAE,QAAQ,CAAC,cAAc;oBAC3C,QAAQ;iBACT,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACzF,QAAQ,EAAE,IAAI;aACf,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AACrF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,IAAY;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IAC9G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAmC,MAAM,iCAAiC,CAAC;AAChH,OAAO,EAAE,gCAAgC,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/src/evals/rerun-inputs.js

@@ -0,0 +1,108 @@
+import path from "node:path";
+import { loadPreparedArtifact, preparedArtifactDirectory } from "../artifacts/index.js";
+import { validateLocalGitTransportManifest, verifyLocalGitTransport } from "../backends/index.js";
+import { HitchError, readJSON } from "../foundation/index.js";
+import { loadHarborArtifact } from "./harbor-artifact-builder.js";
+import { validateEvalRequest } from "./request.js";
+import { frozenRerunBenchmark } from "./verifier-only-rerun.js";
+export async function loadRerunResolvedRevision(evalDirectory, plan) {
+    const resolution = await readJSON(path.join(evalDirectory, "resolution.json"));
+    if (!resolution || resolution.identity !== requiredString(plan.candidate.revision_identity, "candidate revision identity")
+        || resolution.harness_id !== requiredString(plan.candidate.harness_id, "candidate harness id")) {
+        throw unavailable("eval resolution identity changed");
+    }
+    return resolution;
+}
+export async function loadRerunPreparedArtifacts(root, plan) {
+    const artifacts = new Map();
+    for (const summary of plan.preparedArtifacts) {
+        const artifact = await loadRerunPreparedArtifactSummary(root, summary);
+        artifacts.set(artifact.artifact_id, artifact);
+    }
+    if (artifacts.size === 0)
+        throw unavailable("eval has no prepared artifact assignments");
+    return artifacts;
+}
+async function loadRerunPreparedArtifactSummary(root, summary) {
+    const artifactId = requiredString(summary.artifact_id, "prepared artifact id");
+    if (summary.storage === "harbor-artifact-cache-v2") {
+        return loadHarborArtifact(root, {
+            artifact_id: artifactId,
+            artifact_integrity: requiredString(summary.artifact_integrity, "prepared artifact integrity"),
+            entrypoint_integrity: requiredString(summary.entrypoint_integrity, "prepared artifact entrypoint integrity"),
+            harness_id: requiredString(summary.harness_id, "prepared artifact harness id"),
+            revision_identity: requiredString(summary.revision_identity, "prepared artifact revision identity"),
+            platform: requiredString(summary.platform, "prepared artifact platform"),
+        });
+    }
+    const artifact = await loadPreparedArtifact(preparedArtifactDirectory(root, artifactId), {
+        artifact_id: artifactId,
+        artifact_integrity: requiredString(summary.artifact_integrity, "prepared artifact integrity"),
+        entrypoint_integrity: requiredString(summary.entrypoint_integrity, "prepared artifact entrypoint integrity"),
+        harness_id: requiredString(summary.harness_id, "prepared artifact harness id"),
+        revision_identity: requiredString(summary.revision_identity, "prepared artifact revision identity"),
+        platform: requiredString(summary.platform, "prepared artifact platform"),
+    });
+    return {
+        directory: preparedArtifactDirectory(root, artifact.artifact_id),
+        artifact_id: artifact.artifact_id,
+        artifact_integrity: requiredString(artifact.artifact_integrity, "artifact integrity"),
+        entrypoint_integrity: requiredString(artifact.entrypoint_integrity, "artifact entrypoint integrity"),
+        harness_id: artifact.harness_id,
+        revision_identity: artifact.revision_identity,
+        adapter_version: artifact.adapter_version,
+        recipe_version: artifact.recipe_version,
+        platform: artifact.platform,
+        node_version: artifact.toolchain.node || process.version,
+        source_type: artifact.source_type,
+    };
+}
+export async function loadRerunLocalTransport(evalDirectory, plan, resolution, env, signal) {
+    if (plan.localSourceTransport === undefined)
+        return undefined;
+    const directory = path.join(evalDirectory, "local-source");
+    const manifestPath = path.join(directory, "manifest.json");
+    const use = {
+        directory,
+        manifestPath,
+        payloadPath: path.join(directory, "payload.pack"),
+        resolutionPath: path.join(directory, "resolution.json"),
+        manifest: validateLocalGitTransportManifest(await readJSON(manifestPath)),
+    };
+    await verifyLocalGitTransport(use, {
+        expected: {
+            harnessId: resolution.harness_id,
+            resolutionIdentity: resolution.identity,
+            commit: requiredString(resolution.revision.commit, "local source commit"),
+        },
+        env: env ?? process.env,
+        ...(signal ? { signal } : {}),
+    });
+    return use;
+}
+function requiredString(value, label) {
+    if (typeof value !== "string" || value.length === 0)
+        throw unavailable(`${label} is missing`);
+    return value;
+}
+function unavailable(message) {
+    return new HitchError(message, { code: "eval_rerun_unavailable", exitCode: 2 });
+}
+export async function loadPersistedRerunRequest(value, evalDirectory) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw unavailable("eval request is invalid");
+    const persisted = value;
+    const input = Object.fromEntries(Object.entries(persisted).filter(([key]) => key !== "benchmark_id" && key !== "benchmark_revision"));
+    let request = await validateEvalRequest(input);
+    const benchmark = await frozenRerunBenchmark(evalDirectory);
+    if (benchmark) {
+        if (request.dataset !== benchmark.tasks)
+            throw unavailable("compiled dataset path changed");
+        request = { ...request, benchmark_id: benchmark.id, benchmark_revision: benchmark.revision };
+    }
+    if (persisted.benchmark_id !== request.benchmark_id || persisted.benchmark_revision !== request.benchmark_revision) {
+        throw unavailable("eval dataset identity changed since the original run");
+    }
+    return request;
+}
+//# sourceMappingURL=rerun-inputs.js.map

package/dist/src/evals/rerun-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun-inputs.js","sourceRoot":"","sources":["../../../src/evals/rerun-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAExF,OAAO,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAElG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAQhE,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,aAAqB,EAAE,IAAoB;IACzF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACjG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,QAAQ,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;WACrH,UAAU,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,sBAAsB,CAAC,EAAE,CAAC;QACjG,MAAM,WAAW,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAY,EAAE,IAAoB;IACjF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAqC,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,gCAAgC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,2CAA2C,CAAC,CAAC;IACzF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,gCAAgC,CAAC,IAAY,EAAE,OAAgC;IAC5F,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,OAAO,KAAK,0BAA0B,EAAE,CAAC;QACnD,OAAO,kBAAkB,CAAC,IAAI,EAAE;YAC9B,WAAW,EAAE,UAAU;YACvB,kBAAkB,EAAE,cAAc,CAAC,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;YAC7F,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,oBAAoB,EAAE,wCAAwC,CAAC;YAC5G,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,8BAA8B,CAAC;YAC9E,iBAAiB,EAAE,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;YACnG,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,4BAA4B,CAAC;SACzE,CAAC,CAAC;IACL,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE;QACvF,WAAW,EAAE,UAAU;QACvB,kBAAkB,EAAE,cAAc,CAAC,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;QAC7F,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,oBAAoB,EAAE,wCAAwC,CAAC;QAC5G,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,8BAA8B,CAAC;QAC9E,iBAAiB,EAAE,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;QACnG,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,4BAA4B,CAAC;KACzE,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,yBAAyB,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC;QAChE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,kBAAkB,EAAE,cAAc,CAAC,QAAQ,CAAC,kBAAkB,EAAE,oBAAoB,CAAC;QACrF,oBAAoB,EAAE,cAAc,CAAC,QAAQ,CAAC,oBAAoB,EAAE,+BAA+B,CAAC;QACpG,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO;QACxD,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,aAAqB,EACrB,IAAoB,EACpB,UAA4B,EAC5B,GAAkC,EAClC,MAA+B;IAE/B,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAyB;QAChC,SAAS;QACT,YAAY;QACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC;QACjD,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC;QACvD,QAAQ,EAAE,iCAAiC,CAAC,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;KAC1E,CAAC;IACF,MAAM,uBAAuB,CAAC,GAAG,EAAE;QACjC,QAAQ,EAAE;YACR,SAAS,EAAE,UAAU,CAAC,UAAU;YAChC,kBAAkB,EAAE,UAAU,CAAC,QAAQ;YACvC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;SAC1E;QACD,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG;QACvB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAC9F,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,KAAc,EAAE,aAAqB;IACnF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,yBAAyB,CAAC,CAAC;IAC9G,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,cAAc,IAAI,GAAG,KAAK,oBAAoB,CAAC,CAAC,CAAC;IACtI,IAAI,OAAO,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,KAAK;YAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;QAC5F,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,EAAE,kBAAkB,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC;IAC/F,CAAC;IACD,IAAI,SAAS,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACnH,MAAM,WAAW,CAAC,sDAAsD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/evals/rerun-slots.js

@@ -3,7 +3,7 @@ import { evalTrialKey } from "./progress.js";
 export function selectRerunTasks(tasks, progress, selector, attempts = 1) {
     return uniqueTasks(selectRerunTrialSlots(tasks, attempts, progress, selector));
 }
-export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
+export function selectRerunTrialSlots(tasks, attempts, progress, selector, options = {}) {
     const planned = new Set(tasks);
     if (planned.size !== tasks.length || tasks.some((task) => typeof task !== "string" || task.length === 0)) {
         throw unavailable("eval task plan is invalid");
@@ -12,7 +12,7 @@ export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
         throw unavailable("eval attempts plan is invalid");
     const invalid = invalidTrialSlots(tasks, attempts, progress);
     if (selector.mode === "invalid")
-        return rejectVerifierOnlyReruns(progress, invalid);
+        return options.allowVerifierFailures ? invalid : rejectVerifierOnlyReruns(progress, invalid);
     const requested = [...new Set(selector.taskNames)].sort();
     if (requested.length === 0)
         throw invalidInput("eval rerun requires at least one --task");
@@ -24,7 +24,8 @@ export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
         }
     }
     const requestedSet = new Set(requested);
-    return rejectVerifierOnlyReruns(progress, invalid.filter((slot) => requestedSet.has(slot.task_id)));
+    const selected = invalid.filter((slot) => requestedSet.has(slot.task_id));
+    return options.allowVerifierFailures ? selected : rejectVerifierOnlyReruns(progress, selected);
 }
 function rejectVerifierOnlyReruns(progress, slots) {
     const selected = new Set(slots.map(slotKey));

package/dist/src/evals/rerun-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"rerun-slots.js","sourceRoot":"","sources":["../../../src/evals/rerun-slots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAW7C,MAAM,UAAU,gBAAgB,CAC9B,KAAwB,EACxB,QAAwB,EACxB,QAAuB,EACvB,QAAQ,GAAG,CAAC;IAEZ,OAAO,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAwB,EACxB,QAAgB,EAChB,QAAwB,EACxB,QAAuB;IAEvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,OAAO,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAwB,EAAE,KAAsB;IAChF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;WACnF,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,IAAI,KAAK,CAAC,cAAc,KAAK,yBAAyB,CAAC,CAAC,CAAC;IACzH,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,IAAI,UAAU,CAClB,mFAAmF,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC,EACvM,EAAE,IAAI,EAAE,sCAAsC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAC9D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAwB,EAAE,QAAgB,EAAE,QAAwB;IACpG,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,qCAAqC,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAwB,EACxB,IAAoD,EACpD,OAAoB,EACpB,MAAc;IAEd,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClH,MAAM,WAAW,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,WAAW,CAAC,8CAA8C,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,OAAO,GAAG,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,OAAO,CAAC,GAAG,KAAK,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA+B;IACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}
+{"version":3,"file":"rerun-slots.js","sourceRoot":"","sources":["../../../src/evals/rerun-slots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAW7C,MAAM,UAAU,gBAAgB,CAC9B,KAAwB,EACxB,QAAwB,EACxB,QAAuB,EACvB,QAAQ,GAAG,CAAC;IAEZ,OAAO,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAwB,EACxB,QAAgB,EAChB,QAAwB,EACxB,QAAuB,EACvB,UAA+C,EAAE;IAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9H,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAwB,EAAE,KAAsB;IAChF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;WACnF,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,IAAI,KAAK,CAAC,cAAc,KAAK,yBAAyB,CAAC,CAAC,CAAC;IACzH,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,IAAI,UAAU,CAClB,mFAAmF,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC,EACvM,EAAE,IAAI,EAAE,sCAAsC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAC9D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAwB,EAAE,QAAgB,EAAE,QAAwB;IACpG,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,qCAAqC,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAwB,EACxB,IAAoD,EACpD,OAAoB,EACpB,MAAc;IAEd,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClH,MAAM,WAAW,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,WAAW,CAAC,8CAA8C,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,OAAO,GAAG,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,OAAO,CAAC,GAAG,KAAK,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA+B;IACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}

package/dist/src/evals/rerun-types.js

@@ -0,0 +1,45 @@
+import { HitchError, invalidInput } from "../foundation/index.js";
+export const EVAL_RERUN_TYPES = [
+    "candidate-restart",
+    "candidate-resume",
+    "trajectory-replay",
+    "verifier-only",
+    "collect-only",
+];
+export function parseEvalRerunType(value) {
+    if (typeof value !== "string" || !EVAL_RERUN_TYPES.includes(value)) {
+        throw invalidInput(`eval rerun --type must be one of: ${EVAL_RERUN_TYPES.join(", ")}`);
+    }
+    return value;
+}
+export function evalRerunSemantics(type) {
+    switch (type) {
+        case "candidate-restart":
+            return { candidate_action: "restart", conversation_source: "original-instruction", sandbox_source: "clean", candidate_executes: true };
+        case "candidate-resume":
+            return { candidate_action: "resume", conversation_source: "native-session", sandbox_source: "checkpoint", candidate_executes: true };
+        case "trajectory-replay":
+            return { candidate_action: "replay", conversation_source: "canonical-trajectory", sandbox_source: "checkpoint", candidate_executes: true };
+        case "verifier-only":
+            return { candidate_action: "none", conversation_source: "none", sandbox_source: "recorded-artifacts", candidate_executes: false };
+        case "collect-only":
+            return { candidate_action: "none", conversation_source: "none", sandbox_source: "none", candidate_executes: false };
+    }
+}
+export function assertEvalRerunTypeSupported(type) {
+    if (type === "candidate-restart" || type === "collect-only" || type === "verifier-only")
+        return;
+    const unavailable = {
+        "candidate-resume": {
+            code: "eval_candidate_resume_unavailable",
+            message: "candidate-resume requires both a restorable sandbox checkpoint and adapter-native session resume; the current Harbor execution does not retain both",
+        },
+        "trajectory-replay": {
+            code: "eval_trajectory_replay_unavailable",
+            message: "trajectory-replay requires a verified trajectory, a restorable sandbox checkpoint, and explicit adapter replay support; trajectory evidence alone cannot restore tool or process state",
+        },
+    };
+    const failure = unavailable[type];
+    throw new HitchError(failure.message, { code: failure.code, exitCode: 2 });
+}
+//# sourceMappingURL=rerun-types.js.map

package/dist/src/evals/rerun-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun-types.js","sourceRoot":"","sources":["../../../src/evals/rerun-types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAElE,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,mBAAmB;IACnB,kBAAkB;IAClB,mBAAmB;IACnB,eAAe;IACf,cAAc;CACN,CAAC;AAyDX,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAsB,CAAC,EAAE,CAAC;QACpF,MAAM,YAAY,CAAC,qCAAqC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,KAAsB,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAmB;IACpD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,mBAAmB;YACtB,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,cAAc,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;QACzI,KAAK,kBAAkB;YACrB,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;QACvI,KAAK,mBAAmB;YACtB,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;QAC7I,KAAK,eAAe;YAClB,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC;QACpI,KAAK,cAAc;YACjB,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC;IACxH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,IAAmB;IAC9D,IAAI,IAAI,KAAK,mBAAmB,IAAI,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,eAAe;QAAE,OAAO;IAChG,MAAM,WAAW,GAA8H;QAC7I,kBAAkB,EAAE;YAClB,IAAI,EAAE,mCAAmC;YACzC,OAAO,EAAE,qJAAqJ;SAC/J;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,oCAAoC;YAC1C,OAAO,EAAE,wLAAwL;SAClM;KACF,CAAC;IACF,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7E,CAAC"}