@lesliechan721/agent-skill-eval 0.1.0-beta.27 → 0.1.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codex-eval-app-server.d.ts +6 -0
- package/dist/codex-eval-app-server.js +2 -2
- package/dist/codex-eval-app-server.js.map +1 -1
- package/dist/codex-workflow-evaluation.d.ts +76 -0
- package/dist/codex-workflow-evaluation.js +263 -0
- package/dist/codex-workflow-evaluation.js.map +1 -0
- package/dist/codex-workflow.d.ts +5 -0
- package/dist/codex-workflow.js +170 -66
- package/dist/codex-workflow.js.map +1 -1
- package/dist/eval-run.js +1 -0
- package/dist/eval-run.js.map +1 -1
- package/dist/providers.d.ts +2 -0
- package/dist/providers.js +4 -0
- package/dist/providers.js.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codex-workflow-evaluation.js","sourceRoot":"","sources":["../src/codex-workflow-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAG5B,OAAO,EAAE,8BAA8B,EAAE,MAAM,gBAAgB,CAAA;AAwD/D,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,KAAgC,CAAA;AACzC,CAAC;AAED,SAAS,IAAI,CAAC,KAA8B,EAAE,OAAiB,EAAE,KAAa;IAC5E,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAA;IACxD,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,KAAc,EAAE,KAAa,EAAE,GAAG,GAAG,MAAM;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kCAAkC,CAAC,CAAA;IAC7D,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,mCAAmC,EAAE,KAAK,CAAC,CAAA;IACpE,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;WACxE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;IAC5E,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,KAAK,CAAC,KAAc,EAAE,KAAa,EAAE,GAAW;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kCAAkC,GAAG,WAAW,CAAC,CAAA;IAC3E,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,CAAA;IACjF,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,sBAAsB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAClF,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAiB,EAAE;QACvF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAA;QAC/C,IAAI,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,kBAAkB,CAAC,CAAA;QACtD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,CAAA;IAC3G,CAAC,CAAC,CAAA;IACF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,yBAAyB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAuB,EAAE;QAC9G,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,wBAAwB,CAAC,CAAA;QACpD,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,CAAC,EAAE,wBAAwB,CAAC,CAAA;QAChF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACrF,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;QAC7E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,EAAE,GAAG,CAAC;YACxC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC;YACzC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC;YAC/C,cAAc;SACf,CAAA;IACH,CAAC,CAAC,CAAA;IACF,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI;QAC5B,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;KAC7C,EAAE,CAAC;QACX,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,kBAAkB,CAAC,CAAA;IACpG,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IAC3E,OAAO;QACL,SAAS,EAAE,MAAM,EAAE,YAAY;QAC/B,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,wBAAwB,CAAC,EAAE,CAAC;KAC/G,CAAA;AACH,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,qBAAqB,CAAC,SAAiB,EAAE,KAAe;IACtE,IAAI,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;IAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAY,EAAE;QAClC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACd,IAAI,MAAM,GAAG,SAAS,CAAA;QACtB,IAAI,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;gBAChC,IAAI,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE;oBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAA;YACvF,CAAC;YACD,MAAM,IAAI,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,SAAS,EAAE,CAAC;gBACrE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAA;YAC5C,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAA;YAC/E,IAAI,KAAa,CAAA;YACjB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,OAAO,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG,SAAS,EAAE,CAAC;oBAC9E,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAA;gBAC5C,CAAC;gBACD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAClC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;gBACxD,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;YAClC,CAAC;oBAAS,CAAC;gBACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAClB,CAAC;YACD,SAAS,IAAI,KAAK,CAAC,MAAM,CAAA;YACzB,OAAO;gBACL,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAClF,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;aAC/D,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAG,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,EAAE,CAAA;QAC5G,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,uBAAuB,CAAC,QAAyB,EAAE,KAAwB;IACzF,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAA;IACnF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAuB,EAAE;QACpD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;eAC7E,IAAgC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;QACrD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAA4B,CAAA;YAChD,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;mBACzD,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAyC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnG,CAAC;QACH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,qDAAqD,EAAE,CAAA;IACtG,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,6EAA6E;AAC7E,MAAM,OAAO,kBAAkB;IAWlB;IACQ;IACA;IAZV,YAAY,GAAkB,EAAE,CAAA;IAChC,SAAS,GAAiB,EAAE,CAAA;IAC5B,SAAS,GAAmB,EAAE,CAAA;IACvC,gBAAgB,GAAG,CAAC,CAAA;IACZ,OAAO,GAAG,CAAC,CAAA;IACX,IAAI,GAAG,CAAC,CAAA;IACR,OAAO,GAAG,KAAK,CAAA;IACf,gBAAgB,GAAG,CAAC,CAAA;IAE5B,YACW,OAAkC,EAC1B,SAAiB,EACjB,KAAoB;QAF5B,YAAO,GAAP,OAAO,CAA2B;QAC1B,cAAS,GAAT,SAAS,CAAQ;QACjB,UAAK,GAAL,KAAK,CAAe;QAErC,IAAI,CAAC,OAAO,EAAE,CAAA;IAChB,CAAC;IAEO,OAAO;QACb,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QACtC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;YAC/F,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IAClC,CAAC;IAED,OAAO,CAAC,KAAc,EAAE,QAAuB;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;QACrD,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAA;QAC7B,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,KAAK,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;QACpE,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,IAAI,CAAC,OAAO,CAAA;QAC9B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;QAC9E,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,IAAI,kBAAkB;QACpB,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAiB,EAAE,eAAuB;QACpD,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAA;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAE,CAAA;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;YACvE,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;QAC/H,CAAC,CAAC,CAAA;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACnF,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,CAAA;QAC9E,MAAM,QAAQ,GAAiB;YAC7B,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ;YAChD,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;YAC5C,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,gDAAgD,CAAC,CAAC,CAAC,mCAAmC;SAC7G,CAAA;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YAClC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,8CAA8C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;YACrH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC;YAC5D,UAAU,EAAE;gBACV,uGAAuG,IAAI,CAAC,IAAI,EAAE;gBAClH,iIAAiI;gBACjI,gHAAgH;gBAChH,2IAA2I;gBAC3I,uGAAuG;aACxG;YACD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACrC,kBAAkB,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;SAChI,CAAC,CAAC,CAAA;QACH,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAA;QACtB,MAAM,KAAK,GAAG,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAmC,CAAC,CAAC,CAAC,EAAE,CAAA;QACjG,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;QACzE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,EAAE,UAAU,CAAA;QAC5C,MAAM,OAAO,GAAG,OAAO,UAAU,KAAK,QAAQ,IAAI,CAAC,SAAS;YAC1D,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAAiC,CAAC,EAAE,KAAK,UAAU,CAAC;YACvH,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAA;QAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,mFAAmF;QACnF,IAAI,CAAC,OAAO,EAAE,CAAA;QACd,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACnE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3I,QAAQ,CAAC,MAAM,GAAG,kEAAkE,CAAA;YACpF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAA;QAC7B,QAAQ,CAAC,MAAM,GAAG,4EAA4E,CAAA;QAC9F,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC7B,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,uBAAuB,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAA;QAClH,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;QACd,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,UAAoB,EAAE,CAAA;IAChE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAiB,EAAE,UAAmB;QACjD,IAAI,CAAC,OAAO,EAAE,CAAA;QACd,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC;YAC5D,UAAU,EAAE;gBACV,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC1E,6HAA6H;gBAC7H,sHAAsH;gBACtH,8IAA8I;gBAC9I,oKAAoK;gBACpK,kJAAkJ;gBAClJ,0KAA0K;aAC3K;YACD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;YACvC,kBAAkB,EAAE;gBAClB,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS;gBACpE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACnE;SACF,CAAC,CAAC,CAAA;QACH,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAClE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;oBAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;gBACzF,QAAQ,EAAE,oBAAoB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,sBAAsB;aAClG,CAAC,CAAA;QACJ,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC;YAC9D,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY;YAClE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACnE,CAAA;IACH,CAAC;CACF"}
|
package/dist/codex-workflow.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { type CodexEvalProcessManagementEvidence } from './codex-eval-app-server
|
|
|
2
2
|
import { type CodexEvalRuntimeConfig } from './codex-eval-runtime-config.js';
|
|
3
3
|
import type { ProcessTreeDependencies } from './process-tree.js';
|
|
4
4
|
import { type EvalProviderRunInput, type EvalProviderRunResult } from './custom-process.js';
|
|
5
|
+
import { type WorkflowEvaluationRequest } from './codex-workflow-evaluation.js';
|
|
5
6
|
export declare const CODEX_WORKFLOW_ARTIFACTS: readonly [{
|
|
6
7
|
readonly path: "workflow-evidence.json";
|
|
7
8
|
readonly role: "evidence";
|
|
@@ -29,6 +30,7 @@ export interface CodexWorkflowRequest {
|
|
|
29
30
|
model: string | null;
|
|
30
31
|
developerInstructions: string;
|
|
31
32
|
roles: CodexWorkflowRole[];
|
|
33
|
+
evaluation?: WorkflowEvaluationRequest;
|
|
32
34
|
multiAgent: {
|
|
33
35
|
protocol: 'v1' | 'v2';
|
|
34
36
|
maxDepth: number;
|
|
@@ -120,6 +122,7 @@ export interface CodexWorkflowEvidence {
|
|
|
120
122
|
cleanup: {
|
|
121
123
|
isolatedCodexHomeRemoved: true;
|
|
122
124
|
};
|
|
125
|
+
evaluation?: Record<string, unknown>;
|
|
123
126
|
}
|
|
124
127
|
export interface CodexWorkflowExecutionResult {
|
|
125
128
|
evidence: CodexWorkflowEvidence;
|
|
@@ -136,6 +139,7 @@ export interface CodexWorkflowTransportInput {
|
|
|
136
139
|
sourceEnv: NodeJS.ProcessEnv;
|
|
137
140
|
timeoutMs: number;
|
|
138
141
|
processManagement: CodexEvalProcessManagementEvidence;
|
|
142
|
+
protectedEvalRoots?: readonly string[];
|
|
139
143
|
signal?: AbortSignal;
|
|
140
144
|
}
|
|
141
145
|
export interface CodexWorkflowTransport {
|
|
@@ -152,6 +156,7 @@ export interface CodexWorkflowExecutorOptions {
|
|
|
152
156
|
processTree?: ProcessTreeDependencies;
|
|
153
157
|
processManagement?: CodexEvalProcessManagementEvidence;
|
|
154
158
|
transport?: CodexWorkflowTransport;
|
|
159
|
+
protectedEvalRoots?: readonly string[];
|
|
155
160
|
}
|
|
156
161
|
export declare function parseCodexWorkflowRequest(value: unknown, allowedModels: readonly string[], workspace: string): Promise<CodexWorkflowRequest>;
|
|
157
162
|
export declare function codexWorkflowAdapterEnvironment(sourceEnv: NodeJS.ProcessEnv, evalRoot: string, phase: 'prepare' | 'grade'): NodeJS.ProcessEnv;
|
package/dist/codex-workflow.js
CHANGED
|
@@ -2,12 +2,14 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import process from 'node:process';
|
|
4
4
|
import fs from 'fs-extra';
|
|
5
|
-
import { CodexEvalAttemptControl, EvalAppServerSession, extractAppServerThreadId, } from './codex-eval-app-server.js';
|
|
5
|
+
import { CodexEvalAttemptControl, EvalAppServerSession, extractAppServerThreadId, assertHostReadContainmentAvailable, hostReadContainmentProfile, } from './codex-eval-app-server.js';
|
|
6
6
|
import { applyCodexEvalProviderEnvironment, createCodexEvalTemporaryRoot, materializeCodexEvalRuntimeHome, redactCodexEvalValue, } from './codex-eval-runtime-config.js';
|
|
7
7
|
import { applyCodexEvalGitIsolation } from './codex-eval-isolation.js';
|
|
8
8
|
import { runProviderProcess } from './process.js';
|
|
9
9
|
import { assertProviderRunPaths } from './run-paths.js';
|
|
10
10
|
import { parseEvalProviderGrade, } from './custom-process.js';
|
|
11
|
+
import { createCodexGrader, parseConversationGraderOutput } from './providers.js';
|
|
12
|
+
import { parseWorkflowEvaluation, normalizeWorkflowChecks, WorkflowEvaluation, } from './codex-workflow-evaluation.js';
|
|
11
13
|
const MAX_ADAPTER_OUTPUT_BYTES = 4 * 1024 * 1024;
|
|
12
14
|
const MAX_REQUEST_TEXT_BYTES = 200_000;
|
|
13
15
|
const MAX_ROLE_COUNT = 32;
|
|
@@ -71,7 +73,7 @@ async function assertWorkspaceRoleFile(workspace, relative) {
|
|
|
71
73
|
export async function parseCodexWorkflowRequest(value, allowedModels, workspace) {
|
|
72
74
|
if (!isRecord(value))
|
|
73
75
|
throw new Error('workflow prepare stdout must be one JSON object.');
|
|
74
|
-
exactKeys(value, ['version', 'prompt', 'sandbox', 'model', 'developerInstructions', 'roles', 'multiAgent'], [], 'workflow prepare stdout');
|
|
76
|
+
exactKeys(value, ['version', 'prompt', 'sandbox', 'model', 'developerInstructions', 'roles', 'multiAgent'], ['evaluation'], 'workflow prepare stdout');
|
|
75
77
|
if (value.version !== 1)
|
|
76
78
|
throw new Error('workflow prepare stdout version must be 1.');
|
|
77
79
|
if (value.sandbox !== 'read-only' && value.sandbox !== 'workspace-write') {
|
|
@@ -126,6 +128,7 @@ export async function parseCodexWorkflowRequest(value, allowedModels, workspace)
|
|
|
126
128
|
model: value.model,
|
|
127
129
|
developerInstructions: boundedString(value.developerInstructions, 'workflow request developerInstructions', MAX_REQUEST_TEXT_BYTES),
|
|
128
130
|
roles,
|
|
131
|
+
...(value.evaluation === undefined ? {} : { evaluation: parseWorkflowEvaluation(value.evaluation) }),
|
|
129
132
|
multiAgent: {
|
|
130
133
|
protocol: multiAgent.protocol,
|
|
131
134
|
maxDepth: multiAgent.maxDepth,
|
|
@@ -1462,6 +1465,9 @@ export async function normalizeCodexWorkflowRollouts(options) {
|
|
|
1462
1465
|
};
|
|
1463
1466
|
}
|
|
1464
1467
|
function sandboxPolicy(request) {
|
|
1468
|
+
// Opt-in evaluations use one outer OS profile; nesting Seatbelt is unreliable.
|
|
1469
|
+
if (request.evaluation)
|
|
1470
|
+
return { type: 'externalSandbox', networkAccess: 'enabled' };
|
|
1465
1471
|
return request.sandbox === 'read-only'
|
|
1466
1472
|
? { type: 'readOnly', networkAccess: true }
|
|
1467
1473
|
: {
|
|
@@ -1487,6 +1493,26 @@ const defaultTransport = {
|
|
|
1487
1493
|
let codexHomeRemoved = false;
|
|
1488
1494
|
const startedAt = Date.now();
|
|
1489
1495
|
const frames = [];
|
|
1496
|
+
const judgeUsage = [];
|
|
1497
|
+
let judgeNumber = 0;
|
|
1498
|
+
const runGrader = input.request.evaluation ? createCodexGrader({
|
|
1499
|
+
sourceEnv: input.sourceEnv,
|
|
1500
|
+
codexRuntime: input.codexRuntime,
|
|
1501
|
+
codexBin: input.codexBin,
|
|
1502
|
+
...(input.request.model ? { model: input.request.model } : {}),
|
|
1503
|
+
processManagement: input.processManagement,
|
|
1504
|
+
}) : undefined;
|
|
1505
|
+
const evaluation = input.request.evaluation ? new WorkflowEvaluation(input.request.evaluation, input.input.workspacePath, async (prompt) => {
|
|
1506
|
+
const outputDir = path.join(input.input.outputDir, 'workflow-judges', String(++judgeNumber));
|
|
1507
|
+
await fs.ensureDir(outputDir);
|
|
1508
|
+
const result = await runGrader({
|
|
1509
|
+
prompt, outputDir, timeoutMs: control.remaining('workflow semantic check'), signal: control.signal,
|
|
1510
|
+
});
|
|
1511
|
+
if (!result.mechanicalPassed)
|
|
1512
|
+
throw new Error('workflow semantic grader did not complete its read-only checks.');
|
|
1513
|
+
judgeUsage.push(result.usage);
|
|
1514
|
+
return parseConversationGraderOutput(result.output);
|
|
1515
|
+
}) : undefined;
|
|
1490
1516
|
let removeHandler;
|
|
1491
1517
|
let removeFailureHandler;
|
|
1492
1518
|
try {
|
|
@@ -1498,6 +1524,15 @@ const defaultTransport = {
|
|
|
1498
1524
|
}),
|
|
1499
1525
|
]);
|
|
1500
1526
|
const before = await snapshotWorkspace(input.input.workspacePath);
|
|
1527
|
+
let containment;
|
|
1528
|
+
if (evaluation) {
|
|
1529
|
+
containment = hostReadContainmentProfile({
|
|
1530
|
+
forbiddenHostRoots: [input.input.fixturesPath, input.input.outputDir],
|
|
1531
|
+
writableRoots: [temporaryRoot.root, input.input.workspacePath],
|
|
1532
|
+
unwritableRoots: input.request.sandbox === 'read-only' ? [input.input.workspacePath] : [],
|
|
1533
|
+
}) + '\n' + [...new Set([input.evalRoot, ...(input.protectedEvalRoots ?? [])])].map((root) => `(deny file-read* (require-all (subpath ${JSON.stringify(root)}) (require-not (subpath ${JSON.stringify(input.input.workspacePath)}))))`).join('\n');
|
|
1534
|
+
await assertHostReadContainmentAvailable(containment);
|
|
1535
|
+
}
|
|
1501
1536
|
session = new EvalAppServerSession({
|
|
1502
1537
|
codexBin: input.codexBin,
|
|
1503
1538
|
cwd: input.input.workspacePath,
|
|
@@ -1507,6 +1542,7 @@ const defaultTransport = {
|
|
|
1507
1542
|
isolation: 'standard',
|
|
1508
1543
|
processManagement: input.processManagement,
|
|
1509
1544
|
configOverrides: workflowConfig(input.request, input.input.workspacePath),
|
|
1545
|
+
...(containment ? { hostReadContainmentProfile: containment } : {}),
|
|
1510
1546
|
});
|
|
1511
1547
|
await session.initialize(control.rpcTimeout('initialize'));
|
|
1512
1548
|
const threadResult = await session.sendRequest('thread/start', {
|
|
@@ -1522,11 +1558,8 @@ const defaultTransport = {
|
|
|
1522
1558
|
let turnStatus = null;
|
|
1523
1559
|
let resolveTurn;
|
|
1524
1560
|
let rejectTurn;
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
rejectTurn = reject;
|
|
1528
|
-
});
|
|
1529
|
-
void turnDone.catch(() => undefined);
|
|
1561
|
+
let lastMessageOrdinal = 0;
|
|
1562
|
+
let frameQueue = Promise.resolve();
|
|
1530
1563
|
removeFailureHandler = session.onFailure((error) => rejectTurn?.(error));
|
|
1531
1564
|
removeHandler = session.onFrame((frame) => {
|
|
1532
1565
|
const isolatedPaths = [
|
|
@@ -1537,68 +1570,112 @@ const defaultTransport = {
|
|
|
1537
1570
|
frames.push(sanitizeTraceValue(frame, input.codexRuntime, isolatedPaths, input.request.multiAgent.protocol === 'v2'));
|
|
1538
1571
|
const method = typeof frame.method === 'string' ? frame.method : '';
|
|
1539
1572
|
const params = isRecord(frame.params) ? frame.params : {};
|
|
1540
|
-
|
|
1573
|
+
// Capture on receipt, before asynchronous semantic matching. All threads count.
|
|
1574
|
+
const ordinal = evaluation && ['item/completed', 'item/tool/requestUserInput', 'turn/completed'].includes(method)
|
|
1575
|
+
? evaluation.observe(sanitizeTraceValue(frame, input.codexRuntime, isolatedPaths, input.request.multiAgent.protocol === 'v2'), codexWorkflowFrameThreadId(frame)) : 0;
|
|
1576
|
+
frameQueue = frameQueue.then(async () => {
|
|
1577
|
+
if (method === 'turn/completed') {
|
|
1578
|
+
if (!codexWorkflowFrameIsForRoot(frame, rootThreadId))
|
|
1579
|
+
return;
|
|
1580
|
+
const turn = isRecord(params.turn) ? params.turn : {};
|
|
1581
|
+
turnStatus = typeof turn.status === 'string' ? turn.status : 'unknown';
|
|
1582
|
+
resolveTurn?.();
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1541
1585
|
if (!codexWorkflowFrameIsForRoot(frame, rootThreadId))
|
|
1542
1586
|
return;
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1587
|
+
if (method === 'item/completed' && isRecord(params.item)) {
|
|
1588
|
+
const item = params.item;
|
|
1589
|
+
if (item.type === 'agentMessage'
|
|
1590
|
+
&& item.phase === 'final_answer'
|
|
1591
|
+
&& typeof item.text === 'string') {
|
|
1592
|
+
finalMessage = item.text;
|
|
1593
|
+
lastMessageOrdinal = ordinal;
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
if (method === 'thread/tokenUsage/updated')
|
|
1597
|
+
usage = params.tokenUsage ?? usage;
|
|
1598
|
+
if (method === 'item/tool/requestUserInput') {
|
|
1599
|
+
if (!evaluation)
|
|
1600
|
+
throw new Error('codex-workflow root requested interactive user input.');
|
|
1601
|
+
if ((typeof frame.id !== 'string' && typeof frame.id !== 'number') || !Array.isArray(params.questions)) {
|
|
1602
|
+
throw new Error('workflow user-input request lacks its actual id or questions.');
|
|
1603
|
+
}
|
|
1604
|
+
const reply = await evaluation.reply({ questions: params.questions }, ordinal);
|
|
1605
|
+
const answers = Object.fromEntries(params.questions.filter(isRecord).map((question) => [
|
|
1606
|
+
String(question.id), { answers: [reply && reply.questionId === question.id
|
|
1607
|
+
? reply.text : 'No user decision is available for this question. Return control without dependent changes.'] },
|
|
1608
|
+
]));
|
|
1609
|
+
await session.respond(frame.id, { answers });
|
|
1610
|
+
}
|
|
1611
|
+
if (method === 'error')
|
|
1612
|
+
rejectTurn?.(new Error(`codex-workflow app-server error: ${JSON.stringify(params)}`));
|
|
1613
|
+
});
|
|
1614
|
+
return frameQueue;
|
|
1615
|
+
});
|
|
1616
|
+
let nextPrompt = input.request.evaluation?.setupPrompt ?? input.request.prompt;
|
|
1617
|
+
let setup = input.request.evaluation?.setupPrompt !== undefined;
|
|
1618
|
+
while (true) {
|
|
1619
|
+
const turnDone = new Promise((resolve, reject) => {
|
|
1620
|
+
resolveTurn = resolve;
|
|
1621
|
+
rejectTurn = reject;
|
|
1622
|
+
});
|
|
1623
|
+
void turnDone.catch(() => undefined);
|
|
1624
|
+
finalMessage = null;
|
|
1625
|
+
turnStatus = null;
|
|
1626
|
+
await session.sendRequest('turn/start', {
|
|
1627
|
+
threadId: rootThreadId,
|
|
1628
|
+
input: [{ type: 'text', text: nextPrompt, text_elements: [] }],
|
|
1629
|
+
cwd: input.input.workspacePath,
|
|
1630
|
+
approvalPolicy: 'never',
|
|
1631
|
+
sandboxPolicy: sandboxPolicy(input.request),
|
|
1632
|
+
...(input.request.model ? { model: input.request.model } : {}),
|
|
1633
|
+
}, control.rpcTimeout('turn/start'));
|
|
1634
|
+
let abortTurnWait;
|
|
1635
|
+
try {
|
|
1636
|
+
await Promise.race([
|
|
1637
|
+
turnDone,
|
|
1638
|
+
new Promise((_resolve, reject) => {
|
|
1639
|
+
abortTurnWait = () => reject(control.stopError('turn wait'));
|
|
1640
|
+
if (control.signal.aborted)
|
|
1641
|
+
abortTurnWait();
|
|
1642
|
+
else
|
|
1643
|
+
control.signal.addEventListener('abort', abortTurnWait, { once: true });
|
|
1644
|
+
}),
|
|
1645
|
+
]);
|
|
1547
1646
|
}
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
const item = params.item;
|
|
1552
|
-
if (item.type === 'agentMessage'
|
|
1553
|
-
&& item.phase === 'final_answer'
|
|
1554
|
-
&& typeof item.text === 'string')
|
|
1555
|
-
finalMessage = item.text;
|
|
1647
|
+
finally {
|
|
1648
|
+
if (abortTurnWait)
|
|
1649
|
+
control.signal.removeEventListener('abort', abortTurnWait);
|
|
1556
1650
|
}
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
if (
|
|
1560
|
-
|
|
1651
|
+
await frameQueue;
|
|
1652
|
+
control.throwIfStopped('turn wait');
|
|
1653
|
+
if (turnStatus !== 'completed') {
|
|
1654
|
+
throw new Error(`codex-workflow root turn ended with status ${String(turnStatus)}.`);
|
|
1561
1655
|
}
|
|
1562
|
-
if (
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
}
|
|
1586
|
-
finally {
|
|
1587
|
-
if (abortTurnWait)
|
|
1588
|
-
control.signal.removeEventListener('abort', abortTurnWait);
|
|
1589
|
-
}
|
|
1590
|
-
control.throwIfStopped('turn wait');
|
|
1591
|
-
if (turnStatus !== 'completed') {
|
|
1592
|
-
throw new Error(`codex-workflow root turn ended with status ${String(turnStatus)}.`);
|
|
1593
|
-
}
|
|
1594
|
-
if (!finalMessage) {
|
|
1595
|
-
const diagnostic = codexWorkflowMissingFinalMessageDiagnostic({
|
|
1596
|
-
frames,
|
|
1597
|
-
rootThreadId,
|
|
1598
|
-
turnStatus,
|
|
1599
|
-
usage,
|
|
1600
|
-
});
|
|
1601
|
-
throw new Error(`codex-workflow root turn did not produce a final message: ${JSON.stringify(diagnostic)}.`);
|
|
1656
|
+
if (!finalMessage) {
|
|
1657
|
+
const diagnostic = codexWorkflowMissingFinalMessageDiagnostic({
|
|
1658
|
+
frames,
|
|
1659
|
+
rootThreadId,
|
|
1660
|
+
turnStatus,
|
|
1661
|
+
usage,
|
|
1662
|
+
});
|
|
1663
|
+
throw new Error(`codex-workflow root turn did not produce a final message: ${JSON.stringify(diagnostic)}.`);
|
|
1664
|
+
}
|
|
1665
|
+
if (setup) {
|
|
1666
|
+
setup = false;
|
|
1667
|
+
evaluation.measurementStart = evaluation.observe({ method: 'evaluation/measurementStart' }, rootThreadId);
|
|
1668
|
+
nextPrompt = input.request.prompt;
|
|
1669
|
+
continue;
|
|
1670
|
+
}
|
|
1671
|
+
if (evaluation?.hasPendingDecision) {
|
|
1672
|
+
const reply = await evaluation.reply({ text: finalMessage }, lastMessageOrdinal);
|
|
1673
|
+
if (reply) {
|
|
1674
|
+
nextPrompt = reply.text;
|
|
1675
|
+
continue;
|
|
1676
|
+
}
|
|
1677
|
+
}
|
|
1678
|
+
break;
|
|
1602
1679
|
}
|
|
1603
1680
|
const isolatedPaths = [
|
|
1604
1681
|
{ path: codexHome, marker: '[codex-home]' },
|
|
@@ -1635,7 +1712,7 @@ const defaultTransport = {
|
|
|
1635
1712
|
appServerProcessTreeCleanupConfirmed: true,
|
|
1636
1713
|
},
|
|
1637
1714
|
usage,
|
|
1638
|
-
finalMessage,
|
|
1715
|
+
finalMessage: finalMessage,
|
|
1639
1716
|
workspaceDiff: workspaceDiff(before, after),
|
|
1640
1717
|
skillLoads: rollout.skillLoads,
|
|
1641
1718
|
threadGraph: rollout.threadGraph,
|
|
@@ -1643,6 +1720,17 @@ const defaultTransport = {
|
|
|
1643
1720
|
leafCalls: rollout.leafCalls,
|
|
1644
1721
|
cleanup: { isolatedCodexHomeRemoved: true },
|
|
1645
1722
|
};
|
|
1723
|
+
if (evaluation) {
|
|
1724
|
+
// Subject processes are stopped before final snapshots and independent grading.
|
|
1725
|
+
evidence.evaluation = await evaluation.finish(evidence, rollout.trace);
|
|
1726
|
+
const totals = [usage, ...judgeUsage].map(standardUsage);
|
|
1727
|
+
if (totals.some((value) => value === null))
|
|
1728
|
+
throw new Error('workflow evaluation lacks required judge usage evidence.');
|
|
1729
|
+
evidence.usage = {
|
|
1730
|
+
inputTokens: totals.reduce((sum, value) => sum + value.inputTokens, 0),
|
|
1731
|
+
outputTokens: totals.reduce((sum, value) => sum + value.outputTokens, 0),
|
|
1732
|
+
};
|
|
1733
|
+
}
|
|
1646
1734
|
return {
|
|
1647
1735
|
evidence: session.scrub(redactCodexEvalValue(evidence, input.codexRuntime)),
|
|
1648
1736
|
trace: session.scrub(redactCodexEvalValue([
|
|
@@ -1650,7 +1738,7 @@ const defaultTransport = {
|
|
|
1650
1738
|
...rollout.trace,
|
|
1651
1739
|
], input.codexRuntime)),
|
|
1652
1740
|
modelOutput: redactCodexEvalValue(finalMessage, input.codexRuntime),
|
|
1653
|
-
usage: redactCodexEvalValue(usage, input.codexRuntime),
|
|
1741
|
+
usage: redactCodexEvalValue(evidence.usage, input.codexRuntime),
|
|
1654
1742
|
};
|
|
1655
1743
|
}
|
|
1656
1744
|
finally {
|
|
@@ -1772,6 +1860,7 @@ export async function executeCodexWorkflowProvider(options) {
|
|
|
1772
1860
|
sourceEnv,
|
|
1773
1861
|
timeoutMs: control.remaining('Codex execution'),
|
|
1774
1862
|
processManagement,
|
|
1863
|
+
...(options.protectedEvalRoots ? { protectedEvalRoots: options.protectedEvalRoots } : {}),
|
|
1775
1864
|
signal: control.signal,
|
|
1776
1865
|
});
|
|
1777
1866
|
}
|
|
@@ -1815,6 +1904,21 @@ export async function executeCodexWorkflowProvider(options) {
|
|
|
1815
1904
|
throw error;
|
|
1816
1905
|
}
|
|
1817
1906
|
const grade = parseEvalProviderGrade(gradeValue, 'workflow grade stdout');
|
|
1907
|
+
if (request.evaluation) {
|
|
1908
|
+
const results = sanitized.evidence.evaluation;
|
|
1909
|
+
const required = [
|
|
1910
|
+
...request.evaluation.checks,
|
|
1911
|
+
...(request.evaluation.interactions.length ? [{ name: 'user-decisions', criterion: 'All required user decisions were delivered.' }] : []),
|
|
1912
|
+
];
|
|
1913
|
+
const complete = normalizeWorkflowChecks(required, {
|
|
1914
|
+
passed: true, score: null, details: results,
|
|
1915
|
+
}).every((check) => check.verdict === 'pass');
|
|
1916
|
+
if (!complete) {
|
|
1917
|
+
grade.passed = false;
|
|
1918
|
+
grade.score = 0;
|
|
1919
|
+
grade.details = { ...grade.details, workflowEvidenceSatisfied: false };
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1818
1922
|
const afterGrade = await artifactDigests(paths.outputDir);
|
|
1819
1923
|
if (JSON.stringify(beforeGrade) !== JSON.stringify(afterGrade)) {
|
|
1820
1924
|
throw new Error('workflow grade modified Engine-owned attempt artifacts.');
|