@vercel/agent-eval 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1 -1
- package/dist/lib/agents/claude-code.js +1 -1
- package/dist/lib/agents/claude-code.js.map +1 -1
- package/dist/lib/agents/codex.js +1 -1
- package/dist/lib/agents/codex.js.map +1 -1
- package/dist/lib/agents/shared.d.ts +1 -1
- package/dist/lib/agents/shared.d.ts.map +1 -1
- package/dist/lib/agents/shared.js +36 -4
- package/dist/lib/agents/shared.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -18,7 +18,7 @@ const program = new Command();
|
|
|
18
18
|
program
|
|
19
19
|
.name('agent-eval')
|
|
20
20
|
.description('Framework for testing AI coding agents in isolated sandboxes')
|
|
21
|
-
.version('0.0.
|
|
21
|
+
.version('0.0.3');
|
|
22
22
|
/**
|
|
23
23
|
* Resolve config path shorthand.
|
|
24
24
|
* - "cc" -> "experiments/cc.ts"
|
|
@@ -149,7 +149,7 @@ IMPORTANT: Do not run npm, pnpm, yarn, or any package manager commands. Dependen
|
|
|
149
149
|
}
|
|
150
150
|
// Upload test files for validation
|
|
151
151
|
await sandbox.uploadFiles(testFiles);
|
|
152
|
-
// Create vitest config for EVAL.ts
|
|
152
|
+
// Create vitest config for EVAL.ts/tsx
|
|
153
153
|
await createVitestConfig(sandbox);
|
|
154
154
|
// Capture the Claude Code transcript
|
|
155
155
|
const transcript = await captureTranscript(sandbox);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claude-code.js","sourceRoot":"","sources":["../../../src/lib/agents/claude-code.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAAC,OAAuB;IACtD,IAAI,CAAC;QACH,6DAA6D;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC9C,4DAA4D;QAC5D,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAChD,MAAM,gBAAgB,GAAG,sBAAsB,WAAW,EAAE,CAAC;QAE7D,oDAAoD;QACpD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC,SAAS,gBAAgB,gCAAgC,CAC1D,CAAC;QAEF,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3D,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAE,kBAAkB,EAAmC;IAC3F,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,aAAa;QAC1E,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,aAAa;QAEnF,eAAe;YACb,OAAO,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC;QACtF,CAAC;QAED,eAAe;YACb,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAwB;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,OAAO,GAA0B,IAAI,CAAC;YAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,sBAAsB;YACtB,MAAM,YAAY,GAAG,GAAG,EAAE;gBACxB,OAAO,GAAG,IAAI,CAAC;gBACf,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3B,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,sBAAsB;wBAC7B,QAAQ,EAAE,CAAC;qBACZ,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC;gBACH,6BAA6B;gBAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBACtD,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAE/D,8CAA8C;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACjC,CAAC;gBACJ,CAAC;gBAED,iBAAiB;gBACjB,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC;oBACpC,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;gBAE1C,iCAAiC;gBACjC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBAED,uBAAuB;gBACvB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnE,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,uBAAuB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,CAAC;gBAED,mCAAmC;gBACnC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE;oBACjD,SAAS;oBACT,IAAI;oBACJ,2BAA2B;iBAC5B,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBACtE,CAAC;gBAED,kCAAkC;gBAClC,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAEjC,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;;6LAEwI,CAAC;gBAExL,kDAAkD;gBAClD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,QAAQ,EACR,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,gCAAgC,EAAE,cAAc,CAAC,EACvF;oBACE,GAAG,EAAE,kBAAkB;wBACrB,CAAC,CAAC;4BACE,2CAA2C;4BAC3C,kBAAkB,EAAE,UAAU,CAAC,OAAO;4BACtC,oBAAoB,EAAE,OAAO,CAAC,MAAM;4BACpC,iBAAiB,EAAE,EAAE;yBACtB;wBACH,CAAC,CAAC;4BACE,uBAAuB;4BACvB,iBAAiB,EAAE,OAAO,CAAC,MAAM;yBAClC;iBACN,CACF,CAAC;gBAEF,WAAW,GAAG,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;gBAExD,IAAI,YAAY,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAChC,kFAAkF;oBAClF,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACvE,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,UAAU,IAAI,gCAAgC,YAAY,CAAC,QAAQ,EAAE;wBAC5E,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAErC,
|
|
1
|
+
{"version":3,"file":"claude-code.js","sourceRoot":"","sources":["../../../src/lib/agents/claude-code.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAAC,OAAuB;IACtD,IAAI,CAAC;QACH,6DAA6D;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC9C,4DAA4D;QAC5D,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAChD,MAAM,gBAAgB,GAAG,sBAAsB,WAAW,EAAE,CAAC;QAE7D,oDAAoD;QACpD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC,SAAS,gBAAgB,gCAAgC,CAC1D,CAAC;QAEF,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3D,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAE,kBAAkB,EAAmC;IAC3F,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,aAAa;QAC1E,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,aAAa;QAEnF,eAAe;YACb,OAAO,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC;QACtF,CAAC;QAED,eAAe;YACb,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAwB;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,OAAO,GAA0B,IAAI,CAAC;YAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,sBAAsB;YACtB,MAAM,YAAY,GAAG,GAAG,EAAE;gBACxB,OAAO,GAAG,IAAI,CAAC;gBACf,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3B,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,sBAAsB;wBAC7B,QAAQ,EAAE,CAAC;qBACZ,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC;gBACH,6BAA6B;gBAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBACtD,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAE/D,8CAA8C;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACjC,CAAC;gBACJ,CAAC;gBAED,iBAAiB;gBACjB,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC;oBACpC,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;gBAE1C,iCAAiC;gBACjC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBAED,uBAAuB;gBACvB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnE,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,uBAAuB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,CAAC;gBAED,mCAAmC;gBACnC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE;oBACjD,SAAS;oBACT,IAAI;oBACJ,2BAA2B;iBAC5B,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBACtE,CAAC;gBAED,kCAAkC;gBAClC,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAEjC,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;;6LAEwI,CAAC;gBAExL,kDAAkD;gBAClD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,QAAQ,EACR,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,gCAAgC,EAAE,cAAc,CAAC,EACvF;oBACE,GAAG,EAAE,kBAAkB;wBACrB,CAAC,CAAC;4BACE,2CAA2C;4BAC3C,kBAAkB,EAAE,UAAU,CAAC,OAAO;4BACtC,oBAAoB,EAAE,OAAO,CAAC,MAAM;4BACpC,iBAAiB,EAAE,EAAE;yBACtB;wBACH,CAAC,CAAC;4BACE,uBAAuB;4BACvB,iBAAiB,EAAE,OAAO,CAAC,MAAM;yBAClC;iBACN,CACF,CAAC;gBAEF,WAAW,GAAG,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;gBAExD,IAAI,YAAY,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAChC,kFAAkF;oBAClF,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACvE,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,UAAU,IAAI,gCAAgC,YAAY,CAAC,QAAQ,EAAE;wBAC5E,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAErC,uCAAuC;gBACvC,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAElC,qCAAqC;gBACrC,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAEpD,yBAAyB;gBACzB,MAAM,iBAAiB,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;gBAE9E,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBAE5D,OAAO;oBACL,OAAO,EAAE,iBAAiB,CAAC,SAAS;oBACpC,MAAM,EAAE,WAAW;oBACnB,UAAU;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,UAAU,EAAE,iBAAiB,CAAC,IAAI;oBAClC,cAAc,EAAE,iBAAiB,CAAC,OAAO;oBACzC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,cAAc;iBACf,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,6BAA6B;gBAC7B,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,EAAE,SAAS;qBAC9B,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,WAAW;oBACnB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,SAAS,EAAE,OAAO,EAAE,SAAS;iBAC9B,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAC5D,CAAC;gBACD,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACF,CAAC"}
|
package/dist/lib/agents/codex.js
CHANGED
|
@@ -193,7 +193,7 @@ IMPORTANT: Do not run npm, pnpm, yarn, or any package manager commands. Dependen
|
|
|
193
193
|
}
|
|
194
194
|
// Upload test files for validation
|
|
195
195
|
await sandbox.uploadFiles(testFiles);
|
|
196
|
-
// Create vitest config for EVAL.ts
|
|
196
|
+
// Create vitest config for EVAL.ts/tsx
|
|
197
197
|
await createVitestConfig(sandbox);
|
|
198
198
|
// Extract transcript from the Codex JSON output (--json flag outputs JSONL)
|
|
199
199
|
const transcript = extractTranscriptFromOutput(agentOutput);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codex.js","sourceRoot":"","sources":["../../../src/lib/agents/codex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,aAAa,GACd,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,SAAS,2BAA2B,CAAC,MAAc;IACjD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8DAA8D;IAC9D,2DAA2D;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,KAAa,EAAE,kBAA2B,EAAE,MAAe;IACtF,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC;IAElE,IAAI,kBAAkB,EAAE,CAAC;QACvB,OAAO;;;;;cAKG,UAAU,CAAC,aAAa;aACzB,UAAU,CAAC,YAAY;;;;;WAKzB,SAAS;CACnB,CAAC;IACA,CAAC;SAAM,CAAC;QACN,wDAAwD;QACxD,6DAA6D;QAC7D,OAAO;;;;;cAKG,aAAa,CAAC,OAAO;aACtB,MAAM;;;;;WAKR,SAAS;CACnB,CAAC;IACA,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAE,kBAAkB,EAAmC;IACtF,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,OAAO;QAC9D,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,cAAc;QAErF,eAAe;YACb,OAAO,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC;QACnF,CAAC;QAED,eAAe;YACb,OAAO,sBAAsB,CAAC;QAChC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAwB;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,OAAO,GAA0B,IAAI,CAAC;YAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,sBAAsB;YACtB,MAAM,YAAY,GAAG,GAAG,EAAE;gBACxB,OAAO,GAAG,IAAI,CAAC;gBACf,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3B,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,sBAAsB;wBAC7B,QAAQ,EAAE,CAAC;qBACZ,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC;gBACH,6BAA6B;gBAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBACtD,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAE/D,8CAA8C;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACjC,CAAC;gBACJ,CAAC;gBAED,iBAAiB;gBACjB,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC;oBACpC,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;gBAE1C,iCAAiC;gBACjC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBAED,uBAAuB;gBACvB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnE,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,uBAAuB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,CAAC;gBAED,6BAA6B;gBAC7B,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE;oBACjD,SAAS;oBACT,IAAI;oBACJ,eAAe;iBAChB,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,gDAAgD;gBAChD,MAAM,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;gBAC5C,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,CAAC,KAAK,EACb,kBAAkB,EAClB,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAChD,CAAC;gBACF,MAAM,OAAO,CAAC,QAAQ,CAAC;EAC3B,aAAa;IACX,CAAC,CAAC;gBAEA,kCAAkC;gBAClC,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAEjC,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;;6LAEwI,CAAC;gBAExL,8DAA8D;gBAC9D,yFAAyF;gBACzF,6FAA6F;gBAC7F,oEAAoE;gBACpE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,CAC1C,OAAO,EACP;oBACE,MAAM;oBACN,4CAA4C;oBAC5C,QAAQ;oBACR,uBAAuB;oBACvB,cAAc;iBACf,EACD;oBACE,GAAG,EAAE,kBAAkB;wBACrB,CAAC,CAAC;4BACE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM;yBAC1C;wBACH,CAAC,CAAC;4BACE,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM;yBAC7C;iBACN,CACF,CAAC;gBAEF,WAAW,GAAG,WAAW,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;gBAEtD,IAAI,WAAW,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC/B,kFAAkF;oBAClF,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACvE,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,UAAU,IAAI,8BAA8B,WAAW,CAAC,QAAQ,EAAE;wBACzE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAErC,
|
|
1
|
+
{"version":3,"file":"codex.js","sourceRoot":"","sources":["../../../src/lib/agents/codex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,aAAa,GACd,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,SAAS,2BAA2B,CAAC,MAAc;IACjD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8DAA8D;IAC9D,2DAA2D;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,KAAa,EAAE,kBAA2B,EAAE,MAAe;IACtF,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC;IAElE,IAAI,kBAAkB,EAAE,CAAC;QACvB,OAAO;;;;;cAKG,UAAU,CAAC,aAAa;aACzB,UAAU,CAAC,YAAY;;;;;WAKzB,SAAS;CACnB,CAAC;IACA,CAAC;SAAM,CAAC;QACN,wDAAwD;QACxD,6DAA6D;QAC7D,OAAO;;;;;cAKG,aAAa,CAAC,OAAO;aACtB,MAAM;;;;;WAKR,SAAS;CACnB,CAAC;IACA,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAE,kBAAkB,EAAmC;IACtF,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,OAAO;QAC9D,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,cAAc;QAErF,eAAe;YACb,OAAO,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC;QACnF,CAAC;QAED,eAAe;YACb,OAAO,sBAAsB,CAAC;QAChC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAwB;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,OAAO,GAA0B,IAAI,CAAC;YAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,sBAAsB;YACtB,MAAM,YAAY,GAAG,GAAG,EAAE;gBACxB,OAAO,GAAG,IAAI,CAAC;gBACf,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3B,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,sBAAsB;wBAC7B,QAAQ,EAAE,CAAC;qBACZ,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC;gBACH,6BAA6B;gBAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBACtD,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAE/D,8CAA8C;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACjC,CAAC;gBACJ,CAAC;gBAED,iBAAiB;gBACjB,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC;oBACpC,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;gBAE1C,iCAAiC;gBACjC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBAED,uBAAuB;gBACvB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnE,IAAI,aAAa,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,uBAAuB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,CAAC;gBAED,6BAA6B;gBAC7B,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE;oBACjD,SAAS;oBACT,IAAI;oBACJ,eAAe;iBAChB,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,gDAAgD;gBAChD,MAAM,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;gBAC5C,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,CAAC,KAAK,EACb,kBAAkB,EAClB,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAChD,CAAC;gBACF,MAAM,OAAO,CAAC,QAAQ,CAAC;EAC3B,aAAa;IACX,CAAC,CAAC;gBAEA,kCAAkC;gBAClC,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAEjC,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;;6LAEwI,CAAC;gBAExL,8DAA8D;gBAC9D,yFAAyF;gBACzF,6FAA6F;gBAC7F,oEAAoE;gBACpE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,CAC1C,OAAO,EACP;oBACE,MAAM;oBACN,4CAA4C;oBAC5C,QAAQ;oBACR,uBAAuB;oBACvB,cAAc;iBACf,EACD;oBACE,GAAG,EAAE,kBAAkB;wBACrB,CAAC,CAAC;4BACE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM;yBAC1C;wBACH,CAAC,CAAC;4BACE,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM;yBAC7C;iBACN,CACF,CAAC;gBAEF,WAAW,GAAG,WAAW,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;gBAEtD,IAAI,WAAW,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;oBAC/B,kFAAkF;oBAClF,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACvE,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,UAAU,IAAI,8BAA8B,WAAW,CAAC,QAAQ,EAAE;wBACzE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;qBAC7B,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAErC,uCAAuC;gBACvC,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAElC,4EAA4E;gBAC5E,MAAM,UAAU,GAAG,2BAA2B,CAAC,WAAW,CAAC,CAAC;gBAE5D,yBAAyB;gBACzB,MAAM,iBAAiB,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;gBAE9E,0BAA0B;gBAC1B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBAE5D,OAAO;oBACL,OAAO,EAAE,iBAAiB,CAAC,SAAS;oBACpC,MAAM,EAAE,WAAW;oBACnB,UAAU;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,UAAU,EAAE,iBAAiB,CAAC,IAAI;oBAClC,cAAc,EAAE,iBAAiB,CAAC,OAAO;oBACzC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,cAAc;iBACf,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,6BAA6B;gBAC7B,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAChC,SAAS,EAAE,OAAO,EAAE,SAAS;qBAC9B,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,WAAW;oBACnB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,SAAS,EAAE,OAAO,EAAE,SAAS;iBAC9B,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAC5D,CAAC;gBACD,IAAI,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/B,cAAc,GAAG,IAAI,CAAC;oBACtB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACF,CAAC"}
|
|
@@ -20,7 +20,7 @@ export declare function runValidation(sandbox: SandboxManager, scripts: string[]
|
|
|
20
20
|
*/
|
|
21
21
|
export declare function captureGeneratedFiles(sandbox: SandboxManager): Promise<Record<string, string>>;
|
|
22
22
|
/**
|
|
23
|
-
* Create vitest config for running EVAL.ts.
|
|
23
|
+
* Create vitest config for running EVAL.ts or EVAL.tsx.
|
|
24
24
|
*/
|
|
25
25
|
export declare function createVitestConfig(sandbox: SandboxManager): Promise<void>;
|
|
26
26
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AA+BD;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,cAAc,EACvB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,iBAAiB,CAAC,CAmC5B;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CA2BpG;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAe/E;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;;;CAIb,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,gBAAgB;;CAEnB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;;CAGhB,CAAC"}
|
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared utilities for agent implementations.
|
|
3
3
|
*/
|
|
4
|
+
/**
|
|
5
|
+
* Detect which eval file exists in the sandbox (EVAL.ts or EVAL.tsx).
|
|
6
|
+
* Returns the filename if found, or 'EVAL.ts' as fallback.
|
|
7
|
+
*/
|
|
8
|
+
async function detectEvalFile(sandbox) {
|
|
9
|
+
try {
|
|
10
|
+
// Check for EVAL.tsx first (prefer JSX if both exist)
|
|
11
|
+
const tsxResult = await sandbox.runCommand('test', ['-f', 'EVAL.tsx']);
|
|
12
|
+
if (tsxResult.exitCode === 0) {
|
|
13
|
+
return 'EVAL.tsx';
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
// Ignore errors
|
|
18
|
+
}
|
|
19
|
+
try {
|
|
20
|
+
// Check for EVAL.ts
|
|
21
|
+
const tsResult = await sandbox.runCommand('test', ['-f', 'EVAL.ts']);
|
|
22
|
+
if (tsResult.exitCode === 0) {
|
|
23
|
+
return 'EVAL.ts';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// Ignore errors
|
|
28
|
+
}
|
|
29
|
+
// Default to EVAL.ts (will fail later if it doesn't exist)
|
|
30
|
+
return 'EVAL.ts';
|
|
31
|
+
}
|
|
4
32
|
/**
|
|
5
33
|
* Run validation scripts in the sandbox.
|
|
6
34
|
*/
|
|
@@ -9,8 +37,10 @@ export async function runValidation(sandbox, scripts) {
|
|
|
9
37
|
allPassed: true,
|
|
10
38
|
scripts: {},
|
|
11
39
|
};
|
|
12
|
-
//
|
|
13
|
-
const
|
|
40
|
+
// Detect which eval file exists (EVAL.ts or EVAL.tsx)
|
|
41
|
+
const evalFile = await detectEvalFile(sandbox);
|
|
42
|
+
// Always run vitest for the eval file (explicitly specify the file)
|
|
43
|
+
const testResult = await sandbox.runCommand('npx', ['vitest', 'run', evalFile]);
|
|
14
44
|
results.test = {
|
|
15
45
|
success: testResult.exitCode === 0,
|
|
16
46
|
output: testResult.stdout + testResult.stderr,
|
|
@@ -60,15 +90,17 @@ export async function captureGeneratedFiles(sandbox) {
|
|
|
60
90
|
return files;
|
|
61
91
|
}
|
|
62
92
|
/**
|
|
63
|
-
* Create vitest config for running EVAL.ts.
|
|
93
|
+
* Create vitest config for running EVAL.ts or EVAL.tsx.
|
|
64
94
|
*/
|
|
65
95
|
export async function createVitestConfig(sandbox) {
|
|
96
|
+
// Detect which eval file exists
|
|
97
|
+
const evalFile = await detectEvalFile(sandbox);
|
|
66
98
|
await sandbox.writeFiles({
|
|
67
99
|
'vitest.config.ts': `
|
|
68
100
|
import { defineConfig } from 'vitest/config';
|
|
69
101
|
export default defineConfig({
|
|
70
102
|
test: {
|
|
71
|
-
include: ['
|
|
103
|
+
include: ['${evalFile}'],
|
|
72
104
|
globals: false,
|
|
73
105
|
},
|
|
74
106
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAcH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAuB,EACvB,OAAiB;IAEjB,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAcH;;;GAGG;AACH,KAAK,UAAU,cAAc,CAAC,OAAuB;IACnD,IAAI,CAAC;QACH,sDAAsD;QACtD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;QACvE,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;IAED,IAAI,CAAC;QACH,oBAAoB;QACpB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;QACrE,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;IAED,2DAA2D;IAC3D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAuB,EACvB,OAAiB;IAEjB,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,sDAAsD;IACtD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,oEAAoE;IACpE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,CAAC,IAAI,GAAG;QACb,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;KAC9C,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAC1B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,YAAY,CAAC,QAAQ,KAAK,CAAC;YACpC,MAAM,EAAE,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM;SAClD,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAuB;IACjE,MAAM,KAAK,GAA2B,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,wBAAwB;QACxB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC,+HAA+H,CAChI,CAAC;QAEF,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM;aAChC,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACjD,KAAK,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wCAAwC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAuB;IAC9D,gCAAgC;IAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,kBAAkB,EAAE;;;;iBAIP,QAAQ;;;;CAIxB;KACE,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO,EAAE,8BAA8B;IACvC,aAAa,EAAE,iCAAiC;IAChD,YAAY,EAAE,oBAAoB;CAC1B,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,YAAY,EAAE,mBAAmB;CACzB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,2BAA2B;IACpC,YAAY,EAAE,gBAAgB;CACtB,CAAC"}
|