agent-hitch 0.2.4 → 0.2.6

package/dist/src/evals/infrastructure-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEnF,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;CAC1B,CAAC,CAAC;AAEH,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,0CAA0C;AAC1C,MAAM,sCAAsC,GAAG,IAAI,GAAG,CAAC;IACrD,wBAAwB;CACzB,CAAC,CAAC;AA+BH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC9G,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,+BAA+B,GAAG,KAAK,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,aAAa,EACrB,wBAAwB,EACxB,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;YACjC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,gBAAgB;YAChB,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,SAAS;YACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;YAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;YACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;YACtH,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;gBACzD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;wBACnC,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;wBACpC,kBAAkB;wBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;wBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;wBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;wBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;wBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;wBAC/C,qBAAqB,EAAE,IAAI;wBAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;qBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;oBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;oBACnB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,uBAAuB;wBAAE,OAAO,KAAK,CAAC;oBAC3D,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;YAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,kBAAkB;YAClB,eAAe,EAAE,OAAO,CAAC,cAAc;YACvC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;YACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;YACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAC/C,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,YAAY;YAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;YAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACvH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sCAAsC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}

package/dist/src/evals/progress.js

@@ -42,6 +42,9 @@ export function mergeEvalProgressTrial(progress, trial, now = new Date().toISOSt
     if (progress.trials.some((item) => item.run_id === parsed.run_id)) {
         throw new TypeError(`eval progress run identity conflict: ${parsed.run_id}`);
     }
+    if (progress.trials.some((item) => evalTrialKey(item) === evalTrialKey(parsed))) {
+        throw new TypeError(`eval progress logical trial conflict: ${parsed.task_id} attempt ${parsed.attempt}`);
+    }
     const trials = [...progress.trials, parsed].sort((left, right) => left.task_id.localeCompare(right.task_id)
         || left.attempt - right.attempt
         || left.trial_id.localeCompare(right.trial_id));

package/dist/src/evals/progress.js.map

@@ -1 +1 @@
-{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../src/evals/progress.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,GAAG,qBAAqB,CAAC;AACtC,MAAM,MAAM,GAAG,oBAAoB,CAAC;AAEpC,MAAM,UAAU,kBAAkB,CAAC,KAOlC;IACC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,iBAAiB;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACvH,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9G,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;QACzC,cAAc,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3C,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClE,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,aAAqB;IAC1D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9F,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,aAAqB,EAAE,QAAwB;IACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAwB,EAAE,KAAqB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IACpH,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzI,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACtG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkD;IAC7E,OAAO,GAAG,KAAK,CAAC,OAAO,SAAS,KAAK,CAAC,OAAO,EAAE,CAAC;AAClD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,+BAA+B,CAC7C,QAAwB,EACxB,KAAqB,EACrB,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAE9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,kBAAkB,KAAK,OAAO;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACvG,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IAC5E,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACpG,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACnJ,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACzG,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC9D,OAAO,MAAM,CAAC,kBAAkB,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,kBAAkB;WAC3E,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;IACnG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9I,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAU,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAK,OAAkB,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,IAAI,aAAa,CAAC,CAAC;IAC/I,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,uBAAuB,KAAK,EAAE,CAAC,CAAC,CAAC;IACtG,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;WACpE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACrF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACnI,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC/H,MAAM,aAAa,GAAG,OAAkC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACpF,IAAI,MAAM,CAAC,cAAc,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAI,MAAM,CAAC,cAAyB,EAAE,CAAC;QACxF,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAI,MAAM,CAAC,aAAwB,EAAE,CAAC;QAC3H,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,IAAI,aAAa,CAAC,YAAY,KAAK,KAAK;WACrF,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,GAAG,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAChI,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACvF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,MAAM,CAAC,UAAoB;QACvC,aAAa,EAAE,MAAM,CAAC,aAA8B;QACpD,cAAc,EAAE,MAAM,CAAC,cAA+B;QACtD,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,KAAa;IAC/C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IACnH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ;WACpD,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC9D,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO;WACnD,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAK,KAAK,CAAC,OAAkB,GAAG,CAAC;WACrE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,4BAA4B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,mBAAmB,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC7H,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAiB;QAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAgB,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAA6B,EAAE,CAAC;QAChH,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAwB,EAAE,CAAC;KAClG,CAAC;AACJ,CAAC"}
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../src/evals/progress.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,GAAG,qBAAqB,CAAC;AACtC,MAAM,MAAM,GAAG,oBAAoB,CAAC;AAEpC,MAAM,UAAU,kBAAkB,CAAC,KAOlC;IACC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,iBAAiB;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACvH,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9G,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;QACzC,cAAc,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3C,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClE,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,aAAqB;IAC1D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9F,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,aAAqB,EAAE,QAAwB;IACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAwB,EAAE,KAAqB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IACpH,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzI,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACtG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkD;IAC7E,OAAO,GAAG,KAAK,CAAC,OAAO,SAAS,KAAK,CAAC,OAAO,EAAE,CAAC;AAClD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,+BAA+B,CAC7C,QAAwB,EACxB,KAAqB,EACrB,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAE9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,kBAAkB,KAAK,OAAO;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACvG,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IAC5E,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACpG,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACnJ,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACzG,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC9D,OAAO,MAAM,CAAC,kBAAkB,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,kBAAkB;WAC3E,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;IACnG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9I,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAU,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAK,OAAkB,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,IAAI,aAAa,CAAC,CAAC;IAC/I,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,uBAAuB,KAAK,EAAE,CAAC,CAAC,CAAC;IACtG,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;WACpE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACrF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACnI,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC/H,MAAM,aAAa,GAAG,OAAkC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACpF,IAAI,MAAM,CAAC,cAAc,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAI,MAAM,CAAC,cAAyB,EAAE,CAAC;QACxF,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAI,MAAM,CAAC,aAAwB,EAAE,CAAC;QAC3H,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,IAAI,aAAa,CAAC,YAAY,KAAK,KAAK;WACrF,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,GAAG,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAChI,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACvF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,MAAM,CAAC,UAAoB;QACvC,aAAa,EAAE,MAAM,CAAC,aAA8B;QACpD,cAAc,EAAE,MAAM,CAAC,cAA+B;QACtD,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,KAAa;IAC/C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IACnH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ;WACpD,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC9D,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO;WACnD,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAK,KAAK,CAAC,OAAkB,GAAG,CAAC;WACrE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,4BAA4B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,mBAAmB,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC7H,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAiB;QAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAgB,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAA6B,EAAE,CAAC;QAChH,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAwB,EAAE,CAAC;KAClG,CAAC;AACJ,CAAC"}

package/dist/src/evals/request.js

@@ -1,5 +1,7 @@
 export const DEFAULT_EVAL_TIMEOUT_MS = 15 * 60 * 1_000;
 export const DEFAULT_EVAL_SETUP_TIMEOUT_MS = 30 * 60 * 1_000;
+export const DEFAULT_INFRASTRUCTURE_RETRIES = 1;
+export const DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS = 1_000;
 const HARBOR_EVAL_BYPASS_ARG = {
     codex: "--dangerously-bypass-approvals-and-sandbox",
     opencode: "--dangerously-skip-permissions",
@@ -17,7 +19,8 @@ export async function validateEvalRequest(input) {
         throw invalidInput("eval request must be a JSON object");
     const allowed = new Set([
         "schema_version", "backend", "dataset", "harness_ref", "model", "attempts",
-        "max_concurrent", "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
+        "max_concurrent", "infrastructure_retries", "infrastructure_retry_backoff_ms",
+        "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
     ]);
     const unexpected = Object.keys(input).find((field) => !allowed.has(field));
     if (unexpected)
@@ -38,6 +41,8 @@ export async function validateEvalRequest(input) {
     const benchmark = await resolveBenchmarkReference(input.dataset.trim());
     const attempts = positiveInteger(input.attempts ?? 1, "attempts");
     const maxConcurrent = positiveInteger(input.max_concurrent ?? 4, "max_concurrent");
+    const infrastructureRetries = nonNegativeInteger(input.infrastructure_retries ?? DEFAULT_INFRASTRUCTURE_RETRIES, "infrastructure_retries");
+    const infrastructureRetryBackoff = nonNegativeNumber(input.infrastructure_retry_backoff_ms ?? DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS, "infrastructure_retry_backoff_ms");
     const timeout = nonNegativeNumber(input.timeout_ms ?? DEFAULT_EVAL_TIMEOUT_MS, "timeout_ms");
     const setupTimeout = nonNegativeNumber(input.setup_timeout_ms ?? DEFAULT_EVAL_SETUP_TIMEOUT_MS, "setup_timeout_ms");
     if (input.model !== undefined && typeof input.model !== "string")
@@ -60,6 +65,8 @@ export async function validateEvalRequest(input) {
         model: typeof input.model === "string" ? input.model : "",
         attempts,
         max_concurrent: maxConcurrent,
+        infrastructure_retries: infrastructureRetries,
+        infrastructure_retry_backoff_ms: infrastructureRetryBackoff,
         timeout_ms: timeout,
         setup_timeout_ms: setupTimeout,
         agent_args: agentArgs,
@@ -130,6 +137,12 @@ function positiveInteger(value, name) {
         throw invalidInput(`${name} must be a positive integer`);
     return parsed;
 }
+function nonNegativeInteger(value, name) {
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed) || parsed < 0)
+        throw invalidInput(`${name} must be a non-negative integer`);
+    return parsed;
+}
 function nonNegativeNumber(value, name) {
     const parsed = Number(value);
     if (!Number.isFinite(parsed) || parsed < 0)

package/dist/src/evals/request.js.map

@@ -1 +1 @@
-{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAgBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC7E,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAC7D,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,wBAAwB,EAAE,iCAAiC;QAC7E,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC3D,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,sBAAsB,IAAI,8BAA8B,EAC9D,wBAAwB,CACzB,CAAC;IACF,MAAM,0BAA0B,GAAG,iBAAiB,CAClD,KAAK,CAAC,+BAA+B,IAAI,uCAAuC,EAChF,iCAAiC,CAClC,CAAC;IACF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,sBAAsB,EAAE,qBAAqB;QAC7C,+BAA+B,EAAE,0BAA0B;QAC3D,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,IAAY;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IAC9G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/src/evals/rerun-slots.js

@@ -0,0 +1,103 @@
+import { HitchError, invalidInput } from "../foundation/index.js";
+import { evalTrialKey } from "./progress.js";
+export function selectRerunTasks(tasks, progress, selector, attempts = 1) {
+    return uniqueTasks(selectRerunTrialSlots(tasks, attempts, progress, selector));
+}
+export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
+    const planned = new Set(tasks);
+    if (planned.size !== tasks.length || tasks.some((task) => typeof task !== "string" || task.length === 0)) {
+        throw unavailable("eval task plan is invalid");
+    }
+    if (!Number.isSafeInteger(attempts) || attempts < 1)
+        throw unavailable("eval attempts plan is invalid");
+    const invalid = invalidTrialSlots(tasks, attempts, progress);
+    if (selector.mode === "invalid")
+        return rejectVerifierOnlyReruns(progress, invalid);
+    const requested = [...new Set(selector.taskNames)].sort();
+    if (requested.length === 0)
+        throw invalidInput("eval rerun requires at least one --task");
+    for (const task of requested) {
+        if (!planned.has(task))
+            throw new HitchError(`eval task is not in the plan: ${task}`, { code: "eval_rerun_unknown_task", exitCode: 2 });
+        if (!invalid.some((slot) => slot.task_id === task)) {
+            throw new HitchError(`eval task is already valid: ${task}`, { code: "eval_task_already_valid", exitCode: 2 });
+        }
+    }
+    const requestedSet = new Set(requested);
+    return rejectVerifierOnlyReruns(progress, invalid.filter((slot) => requestedSet.has(slot.task_id)));
+}
+function rejectVerifierOnlyReruns(progress, slots) {
+    const selected = new Set(slots.map(slotKey));
+    const verifierOnly = progress.trials.filter((trial) => selected.has(evalTrialKey(trial))
+        && (trial.invalid_reason === "verifier_infrastructure_failure" || trial.invalid_reason === "verifier_result_missing"));
+    if (verifierOnly.length === 0)
+        return slots;
+    throw new HitchError(`verifier-only retry is unavailable after the original trial environment closed: ${verifierOnly.map((trial) => `${trial.task_id}#${trial.attempt}`).join(", ")}; the Candidate Agent will not be rerun`, { code: "eval_verifier_only_rerun_unavailable", exitCode: 2 });
+}
+export function invalidTrialSlots(tasks, attempts, progress) {
+    const byKey = new Map();
+    for (const trial of progress.trials) {
+        const key = evalTrialKey(trial);
+        if (byKey.has(key))
+            throw unavailable(`eval has duplicate logical trial: ${trial.task_id} attempt ${trial.attempt}`);
+        byKey.set(key, trial);
+    }
+    const invalid = [];
+    for (const task of tasks) {
+        for (let attempt = 1; attempt <= attempts; attempt += 1) {
+            if (byKey.get(slotKey({ task_id: task, attempt }))?.observation_status !== "valid") {
+                invalid.push({ task_id: task, attempt });
+            }
+        }
+    }
+    return sortSlots(invalid);
+}
+export function validateProgressPlan(progress, plan, request, evalId) {
+    if (progress.eval_id !== evalId)
+        throw unavailable("eval progress identity is invalid");
+    if (progress.benchmark_id !== request.benchmark_id || progress.benchmark_revision !== request.benchmark_revision) {
+        throw unavailable("eval progress benchmark identity changed");
+    }
+    if (progress.planned_tasks !== plan.tasks.length || progress.planned_trials !== plan.tasks.length * plan.attempts) {
+        throw unavailable("eval progress does not match the frozen task/attempt plan");
+    }
+    const planned = new Set(plan.tasks);
+    for (const trial of progress.trials) {
+        if (trial.attempt > plan.attempts || !planned.has(trial.task_id)) {
+            throw unavailable(`eval progress contains an unplanned trial: ${trial.task_id} attempt ${trial.attempt}`);
+        }
+    }
+    invalidTrialSlots(plan.tasks, plan.attempts, progress);
+}
+export function slotKey(slot) {
+    return `${slot.task_id}\u0000${slot.attempt}`;
+}
+export function sortSlots(slots) {
+    return [...slots]
+        .map((slot) => ({ task_id: slot.task_id, attempt: slot.attempt }))
+        .sort((left, right) => left.task_id.localeCompare(right.task_id) || left.attempt - right.attempt);
+}
+export function uniqueTasks(slots) {
+    return [...new Set(slots.map((slot) => slot.task_id))].sort();
+}
+export function groupSlotsByAttempt(slots) {
+    const grouped = new Map();
+    for (const slot of slots) {
+        const current = grouped.get(slot.attempt) ?? [];
+        current.push(slot);
+        grouped.set(slot.attempt, current);
+    }
+    return [...grouped.entries()]
+        .sort(([left], [right]) => left - right)
+        .map(([attempt, values]) => [attempt, sortSlots(values)]);
+}
+export function attemptDirectoryName(attempt) {
+    return `attempt-${String(attempt).padStart(4, "0")}`;
+}
+export function formatSlot(slot) {
+    return `${slot.task_id}#${slot.attempt}`;
+}
+function unavailable(message) {
+    return new HitchError(message, { code: "eval_rerun_unavailable", exitCode: 2 });
+}
+//# sourceMappingURL=rerun-slots.js.map

package/dist/src/evals/rerun-slots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun-slots.js","sourceRoot":"","sources":["../../../src/evals/rerun-slots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAW7C,MAAM,UAAU,gBAAgB,CAC9B,KAAwB,EACxB,QAAwB,EACxB,QAAuB,EACvB,QAAQ,GAAG,CAAC;IAEZ,OAAO,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAwB,EACxB,QAAgB,EAChB,QAAwB,EACxB,QAAuB;IAEvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,OAAO,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAwB,EAAE,KAAsB;IAChF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;WACnF,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,IAAI,KAAK,CAAC,cAAc,KAAK,yBAAyB,CAAC,CAAC,CAAC;IACzH,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,IAAI,UAAU,CAClB,mFAAmF,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC,EACvM,EAAE,IAAI,EAAE,sCAAsC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAC9D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAwB,EAAE,QAAgB,EAAE,QAAwB;IACpG,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,qCAAqC,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAwB,EACxB,IAAoD,EACpD,OAAoB,EACpB,MAAc;IAEd,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClH,MAAM,WAAW,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,WAAW,CAAC,8CAA8C,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,OAAO,GAAG,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,OAAO,CAAC,GAAG,KAAK,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA+B;IACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}