@vercel/agent-eval 0.0.9 → 0.0.12

package/dist/lib/runner.js

@@ -62,7 +62,7 @@ export async function runExperiment(options) {
         let timeoutId;
         const agentResult = await Promise.race([
             agent.run(fixture.path, {
-                prompt: fixture.prompt,
+                prompt: config.editPrompt ? config.editPrompt(fixture.prompt) : fixture.prompt,
                 model: config.model,
                 timeout: timeoutMs,
                 apiKey,
@@ -161,15 +161,25 @@ export async function runExperiment(options) {
  */
 export async function runSingleEval(fixture, options) {
     const agent = getAgent(options.agent ?? 'vercel-ai-gateway/claude-code');
-    const agentResult = await agent.run(fixture.path, {
-        prompt: fixture.prompt,
-        model: options.model,
-        timeout: options.timeout * 1000,
-        apiKey: options.apiKey,
-        setup: options.setup,
-        scripts: options.scripts,
-        sandbox: options.sandbox,
-    });
-    return agentResultToEvalRunData(agentResult);
+    const models = Array.isArray(options.model) ? options.model : [options.model];
+    const prompt = options.editPrompt ? options.editPrompt(fixture.prompt) : fixture.prompt;
+    const results = [];
+    for (const model of models) {
+        const agentResult = await agent.run(fixture.path, {
+            prompt,
+            model,
+            timeout: options.timeout * 1000,
+            apiKey: options.apiKey,
+            setup: options.setup,
+            scripts: options.scripts,
+            sandbox: options.sandbox,
+        });
+        results.push(agentResultToEvalRunData(agentResult));
+    }
+    // TODO: remove this on the next major and return an array directly...it's just here to prevent breaking changes
+    if (!Array.isArray(options.model)) {
+        return results[0];
+    }
+    return results;
 }
 //# sourceMappingURL=runner.js.map

package/dist/lib/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,cAAc,CAAC;AAwCtB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,gCAAgC,QAAQ,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,QAAQ,CAAC,CAAC;IAE/G,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEtD,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,4EAA4E;QAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,GAAG,CAAC,eAAe,OAAO,CAAC,IAAI,kBAAkB,QAAQ,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC9F,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,gCAAgC;IAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAE5D,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;KACf,CAAC,CAAC;IAEH,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAE3C,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OASC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;QAChD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;QAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,OAAO,wBAAwB,CAAC,WAAW,CAAC,CAAC;AAC/C,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,cAAc,CAAC;AAwCtB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,gCAAgC,QAAQ,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,QAAQ,CAAC,CAAC;IAE/G,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAC9E,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEtD,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,4EAA4E;QAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,GAAG,CAAC,eAAe,OAAO,CAAC,IAAI,kBAAkB,QAAQ,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC9F,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,gCAAgC;IAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAE5D,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;KACf,CAAC,CAAC;IAEH,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAE3C,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OAUC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAa,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAExF,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACjD,MAAM;YACN,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;YAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACxB,CAAC,CAAC;QAEA,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,gHAAgH;IAChH,IAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC,CAAC,CAA2D,CAAC;IAC3E,CAAC;IAED,OAAO,OAAiE,CAAC;AAC3E,CAAC"}

package/dist/lib/types.d.ts

@@ -50,8 +50,10 @@ export type SandboxBackend = 'vercel' | 'docker';
 export interface ExperimentConfig {
     /** Which AI agent to use */
     agent: AgentType;
-    /** Which AI model the agent should use. Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
-    model?: ModelTier;
+    /** Which AI model the agent should use. Can be a single model or array of models to test.
+     * If an array is provided, the experiment will run on each model.
+     * Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
+    model?: ModelTier | ModelTier[];
     /** Which evals to run. Can be a string, array, or filter function. @default '*' (all evals) */
     evals?: string | string[] | EvalFilter;
     /** How many times to run each eval. @default 1 */
@@ -66,11 +68,28 @@ export interface ExperimentConfig {
     setup?: SetupFunction;
     /** Sandbox backend to use. @default 'auto' (Vercel if token present, else Docker) */
     sandbox?: SandboxBackend | 'auto';
+    /** Optional function to modify the prompt before running the experiment. @default undefined */
+    editPrompt?: (prompt: string) => string;
 }
 /**
  * Resolved experiment config with all defaults applied.
  */
 export interface ResolvedExperimentConfig {
+    agent: AgentType;
+    model: ModelTier | ModelTier[];
+    evals: string | string[] | EvalFilter;
+    runs: number;
+    earlyExit: boolean;
+    scripts: string[];
+    timeout: number;
+    setup?: SetupFunction;
+    sandbox: SandboxBackend | 'auto';
+    editPrompt?: (prompt: string) => string;
+}
+/**
+ * Resolved experiment config with all defaults applied.
+ */
+export interface RunnableExperimentConfig {
     agent: AgentType;
     model: ModelTier;
     evals: string | string[] | EvalFilter;
@@ -80,6 +99,7 @@ export interface ResolvedExperimentConfig {
     timeout: number;
     setup?: SetupFunction;
     sandbox: SandboxBackend | 'auto';
+    editPrompt?: (prompt: string) => string;
 }
 /**
  * Required files for a valid eval fixture.
@@ -115,8 +135,10 @@ export interface EvalRunResult {
     error?: string;
     /** Duration in seconds */
     duration: number;
-    /** Path to transcript file (relative to run directory) */
+    /** Path to parsed transcript file (relative to run directory) */
     transcriptPath?: string;
+    /** Path to raw transcript file (relative to run directory) */
+    transcriptRawPath?: string;
     /** Paths to output files (relative to run directory) */
     outputPaths?: {
         /** Path to EVAL.ts test output */

package/dist/lib/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB,+HAA+H;IAC/H,KAAK,CAAC,EAAE,SAAS,CAAC;IAElB,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;;6FAEyF;IACzF,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}

package/dist/lib/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqGH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2HH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@vercel/agent-eval",
-  "version": "0.0.9",
+  "version": "0.0.12",
   "description": "Framework for testing AI coding agents in isolated sandboxes",
   "repository": {
     "type": "git",
-    "url": "https://github.com/vercel-labs/agent-eval.git"
+    "url": "git+https://github.com/vercel-labs/agent-eval.git"
   },
   "type": "module",
   "main": "dist/index.js",
@@ -24,6 +24,7 @@
     "test:integration:docker": "SANDBOX_BACKEND=docker INTEGRATION_TEST=1 vitest run src/integration.test.ts",
     "test:integration:vercel": "SANDBOX_BACKEND=vercel INTEGRATION_TEST=1 vitest run src/integration.test.ts",
     "lint": "eslint src/",
+    "release": "changeset publish",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
@@ -61,6 +62,7 @@
   ],
   "license": "MIT",
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "provenance": true
   }
 }

package/README.md

@@ -1,474 +0,0 @@
-# @vercel/agent-eval
-
-Test AI coding agents on your framework. Measure what actually works.
-
-## Why?
-
-You're building a frontend framework and want AI agents to work well with it. But how do you know if:
-- Your documentation helps agents write correct code?
-- Adding an MCP server improves agent success rates?
-- Sonnet performs as well as Opus for your use cases?
-- Your latest API changes broke agent compatibility?
-
-**This framework gives you answers.** Run controlled experiments, measure pass rates, compare techniques.
-
-## Quick Start
-
-```bash
-# Create a new eval project
-npx @vercel/agent-eval init my-framework-evals
-cd my-framework-evals
-
-# Install dependencies
-npm install
-
-# Add your API keys
-cp .env.example .env
-# Edit .env with your AI_GATEWAY_API_KEY and VERCEL_TOKEN
-
-# Preview what will run (no API calls, no cost)
-npx @vercel/agent-eval cc --dry
-
-# Run the evals
-npx @vercel/agent-eval cc
-```
-
-## A/B Testing AI Techniques
-
-The real power is comparing different approaches. Create multiple experiment configs:
-
-### Control: Baseline Agent
-
-```typescript
-// experiments/control.ts
-import type { ExperimentConfig } from 'agent-eval';
-
-const config: ExperimentConfig = {
-  agent: 'vercel-ai-gateway/claude-code',
-  model: 'opus',
-  runs: 10,        // Multiple runs for statistical significance
-  earlyExit: false, // Run all attempts to measure reliability
-};
-
-export default config;
-```
-
-### Treatment: Agent with MCP Server
-
-```typescript
-// experiments/with-mcp.ts
-import type { ExperimentConfig } from 'agent-eval';
-
-const config: ExperimentConfig = {
-  agent: 'vercel-ai-gateway/claude-code',
-  model: 'opus',
-  runs: 10,
-  earlyExit: false,
-
-  setup: async (sandbox) => {
-    // Install your framework's MCP server
-    await sandbox.runCommand('npm', ['install', '-g', '@myframework/mcp-server']);
-
-    // Configure Claude to use it
-    await sandbox.writeFiles({
-      '.claude/settings.json': JSON.stringify({
-        mcpServers: {
-          myframework: { command: 'myframework-mcp' }
-        }
-      })
-    });
-  },
-};
-
-export default config;
-```
-
-### Run Both & Compare
-
-```bash
-# Preview first
-npx @vercel/agent-eval control --dry
-npx @vercel/agent-eval with-mcp --dry
-
-# Run experiments
-npx @vercel/agent-eval control
-npx @vercel/agent-eval with-mcp
-```
-
-**Compare results:**
-```
-Control (baseline):     7/10 passed (70%)
-With MCP:              9/10 passed (90%)
-```
-
-## Creating Evals for Your Framework
-
-Each eval tests one specific task an agent should be able to do with your framework.
-
-### Example: Testing Component Creation
-
-```
-evals/
-  create-button-component/
-    PROMPT.md           # Task for the agent
-    EVAL.ts             # Tests to verify success (or EVAL.tsx for JSX)
-    package.json        # Your framework as a dependency
-    src/                # Starter code
-```
-
-### EVAL.ts vs EVAL.tsx
-
-Use **EVAL.tsx** when your tests require JSX syntax (React Testing Library, component testing):
-```typescript
-// EVAL.tsx - use when testing React components
-import { test, expect } from 'vitest';
-import { render, screen } from '@testing-library/react';
-import { Button } from './src/components/Button';
-
-test('Button renders with label', () => {
-  render(<Button label="Click me" onClick={() => {}} />);
-  expect(screen.getByText('Click me')).toBeDefined();
-});
-```
-
-Use **EVAL.ts** for tests that don't need JSX:
-```typescript
-// EVAL.ts - use for file checks, build tests, etc.
-import { test, expect } from 'vitest';
-import { existsSync } from 'fs';
-
-test('Button component exists', () => {
-  expect(existsSync('src/components/Button.tsx')).toBe(true);
-});
-```
-
-> **Note:** You only need one eval file per fixture. Choose `.tsx` if any test needs JSX, otherwise use `.ts`.
-
-**PROMPT.md** - What you want the agent to do:
-```markdown
-Create a Button component using MyFramework.
-
-Requirements:
-- Export a Button component from src/components/Button.tsx
-- Accept `label` and `onClick` props
-- Use the framework's styling system for hover states
-```
-
-**EVAL.ts** (or **EVAL.tsx**) - How you verify it worked:
-```typescript
-import { test, expect } from 'vitest';
-import { readFileSync, existsSync } from 'fs';
-import { execSync } from 'child_process';
-
-test('Button component exists', () => {
-  expect(existsSync('src/components/Button.tsx')).toBe(true);
-});
-
-test('has required props', () => {
-  const content = readFileSync('src/components/Button.tsx', 'utf-8');
-  expect(content).toContain('label');
-  expect(content).toContain('onClick');
-});
-
-test('project builds', () => {
-  execSync('npm run build', { stdio: 'pipe' });
-});
-```
-
-**package.json** - Include your framework:
-```json
-{
-  "name": "create-button-component",
-  "type": "module",
-  "scripts": { "build": "tsc" },
-  "dependencies": {
-    "myframework": "^2.0.0"
-  }
-}
-```
-
-## Experiment Ideas
-
-| Experiment | Control | Treatment |
-|------------|---------|-----------|
-| MCP impact | No MCP | With MCP server |
-| Model comparison | Haiku | Sonnet / Opus |
-| Documentation | Minimal docs | Rich examples |
-| System prompt | Default | Framework-specific |
-| Tool availability | Read/write only | + custom tools |
-
-## Configuration Reference
-
-### Agent Selection
-
-Choose your agent and authentication method:
-
-```typescript
-// Vercel AI Gateway (recommended - unified billing & observability)
-agent: 'vercel-ai-gateway/claude-code'  // Claude Code via AI Gateway
-agent: 'vercel-ai-gateway/codex'        // OpenAI Codex via AI Gateway
-agent: 'vercel-ai-gateway/opencode'     // OpenCode via AI Gateway
-agent: 'vercel-ai-gateway/ai-sdk-harness' // Simple AI SDK harness (any model)
-
-// Direct API (uses provider keys directly)
-agent: 'claude-code'  // requires ANTHROPIC_API_KEY
-agent: 'codex'        // requires OPENAI_API_KEY
-```
-
-See the Environment Variables section below for setup instructions.
-
-### OpenCode Model Configuration
-
-OpenCode uses Vercel AI Gateway exclusively. Models **must** be specified with the `vercel/{provider}/{model}` format:
-
-```typescript
-// Anthropic models
-model: 'vercel/anthropic/claude-sonnet-4'
-model: 'vercel/anthropic/claude-opus-4'
-
-// Minimax models
-model: 'vercel/minimax/minimax-m2.1'
-model: 'vercel/minimax/minimax-m2.1-lightning'
-
-// Moonshot AI (Kimi) models
-model: 'vercel/moonshotai/kimi-k2'
-model: 'vercel/moonshotai/kimi-k2-thinking'
-
-// OpenAI models
-model: 'vercel/openai/gpt-4o'
-model: 'vercel/openai/o3'
-```
-
-> **Important:** The `vercel/` prefix is required. OpenCode's config sets up a `vercel` provider, so the model string must start with `vercel/` to route through Vercel AI Gateway correctly. Using just `anthropic/claude-sonnet-4` (without the `vercel/` prefix) will fail with a "provider not found" error.
-
-Under the hood, the agent creates an `opencode.json` config file that configures the Vercel provider:
-
-```json
-{
-  "provider": {
-    "vercel": {
-      "options": {
-        "apiKey": "{env:AI_GATEWAY_API_KEY}"
-      }
-    }
-  },
-  "permission": {
-    "write": "allow",
-    "edit": "allow",
-    "bash": "allow"
-  }
-}
-```
-
-And runs: `opencode run "<prompt>" --model {provider}/{model} --format json`
-
-### AI SDK Harness Model Configuration
-
-The AI SDK harness (`vercel-ai-gateway/ai-sdk-harness`) is a lightweight agent that works with **any model** available on Vercel AI Gateway. Unlike OpenCode, it uses the standard `{provider}/{model}` format without a `vercel/` prefix:
-
-```typescript
-// Anthropic models
-model: 'anthropic/claude-sonnet-4'
-model: 'anthropic/claude-opus-4'
-
-// Moonshot AI (Kimi) models
-model: 'moonshotai/kimi-k2.5'
-model: 'moonshotai/kimi-k2-thinking'
-
-// Minimax models
-model: 'minimax/minimax-m2.1'
-
-// OpenAI models
-model: 'openai/gpt-4o'
-```
-
-The AI SDK harness includes these tools: `readFile`, `writeFile`, `editFile`, `listFiles`, `glob`, `grep`, and `bash`. It's ideal for evaluating models that may not be fully compatible with OpenCode.
-
-### Full Configuration
-
-```typescript
-import type { ExperimentConfig } from 'agent-eval';
-
-const config: ExperimentConfig = {
-  // Required: which agent and authentication to use
-  agent: 'vercel-ai-gateway/claude-code',
-
-  // Model to use (defaults vary by agent)
-  // - claude-code: 'opus'
-  // - codex: 'openai/gpt-5.2-codex'
-  // - opencode: 'vercel/anthropic/claude-sonnet-4' (note: vercel/ prefix required)
-  // - ai-sdk-harness: 'anthropic/claude-sonnet-4' (works with any AI Gateway model)
-  model: 'opus',
-
-  // How many times to run each eval
-  runs: 10,
-
-  // Stop after first success? (false for reliability measurement)
-  earlyExit: false,
-
-  // npm scripts that must pass after agent finishes
-  scripts: ['build', 'lint'],
-
-  // Timeout per run in seconds (default: 600)
-  timeout: 600,
-
-  // Filter which evals to run (pick one)
-  evals: '*',                                // all (default)
-  // evals: ['specific-eval'],              // by name
-  // evals: (name) => name.startsWith('api-'), // by function
-
-  // Setup function for environment configuration
-  setup: async (sandbox) => {
-    await sandbox.writeFiles({ '.env': 'API_KEY=test' });
-    await sandbox.runCommand('npm', ['run', 'setup']);
-  },
-};
-
-export default config;
-```
-
-## CLI Commands
-
-### `init <name>`
-
-Create a new eval project:
-```bash
-npx @vercel/agent-eval init my-evals
-```
-
-### `<experiment>`
-
-Run an experiment:
-```bash
-npx @vercel/agent-eval cc
-```
-
-**Dry run** - preview without executing (no API calls, no cost):
-```bash
-npx @vercel/agent-eval cc --dry
-
-# Output:
-# Found 5 valid fixture(s), will run 5:
-#   - create-button
-#   - add-routing
-#   - setup-state
-#   - ...
-# Running 5 eval(s) x 10 run(s) = 50 total runs
-# Agent: claude-code, Model: opus, Timeout: 300s
-# [DRY RUN] Would execute evals here
-```
-
-## Results
-
-Results are saved to `results/<experiment>/<timestamp>/`:
-
-```
-results/
-  with-mcp/
-    2026-01-27T10-30-00Z/
-      experiment.json       # Config and summary
-      create-button/
-        summary.json        # { totalRuns: 10, passedRuns: 9, passRate: "90%" }
-        run-1/
-          result.json       # Individual run result
-          transcript.jsonl  # Agent conversation
-          outputs/          # Test/script output
-```
-
-### Analyzing Results
-
-```bash
-# Quick comparison
-cat results/control/*/experiment.json | jq '.evals[] | {name, passRate}'
-cat results/with-mcp/*/experiment.json | jq '.evals[] | {name, passRate}'
-```
-
-| Pass Rate | Interpretation |
-|-----------|----------------|
-| 90-100%   | Agent handles this reliably |
-| 70-89%    | Usually works, room for improvement |
-| 50-69%    | Unreliable, needs investigation |
-| < 50%     | Task too hard or prompt needs work |
-
-## Environment Variables
-
-Every run requires **two things**: an API key for the agent and a token for the Vercel sandbox. The exact variables depend on which authentication mode you use.
-
-| Variable | Required when | Description |
-|---|---|---|
-| `AI_GATEWAY_API_KEY` | `agent: 'vercel-ai-gateway/...'` | Vercel AI Gateway key — works for all agents (claude-code, codex, opencode) |
-| `ANTHROPIC_API_KEY` | `agent: 'claude-code'` | Direct Anthropic API key (`sk-ant-...`) |
-| `OPENAI_API_KEY` | `agent: 'codex'` | Direct OpenAI API key (`sk-proj-...`) |
-| `VERCEL_TOKEN` | Always (pick one) | Vercel personal access token — for local dev |
-| `VERCEL_OIDC_TOKEN` | Always (pick one) | Vercel OIDC token — for CI/CD pipelines |
-
-> **Note:** OpenCode only supports Vercel AI Gateway (`vercel-ai-gateway/opencode`). There is no direct API option for OpenCode.
-
-> You always need **one agent key** + **one sandbox token**.
-
-### Vercel AI Gateway (Recommended)
-
-Use `vercel-ai-gateway/` prefixed agents. One key for all models.
-
-```bash
-# Agent access — get yours at https://vercel.com/dashboard -> AI Gateway
-AI_GATEWAY_API_KEY=your-ai-gateway-api-key
-
-# Sandbox access — create at https://vercel.com/account/tokens
-VERCEL_TOKEN=your-vercel-token
-# OR for CI/CD:
-# VERCEL_OIDC_TOKEN=your-oidc-token
-```
-
-### Direct API Keys (Alternative)
-
-Remove the `vercel-ai-gateway/` prefix and use provider keys directly:
-
-```bash
-# For agent: 'claude-code'
-ANTHROPIC_API_KEY=sk-ant-...
-
-# For agent: 'codex'
-OPENAI_API_KEY=sk-proj-...
-
-# Sandbox access is still required
-VERCEL_TOKEN=your-vercel-token
-```
-
-### `.env` Setup
-
-The `init` command generates a `.env.example` file. Copy it and fill in your keys:
-
-```bash
-cp .env.example .env
-```
-
-The framework loads `.env` automatically via [dotenv](https://github.com/motdotla/dotenv).
-
-### Vercel Employees
-
-**To get the environment variables, link to `vercel-labs/agent-eval` on Vercel:**
-
-```bash
-# Link to the vercel-labs/agent-eval project
-vc link vercel-labs/agent-eval
-
-# Pull environment variables
-vc env pull
-```
-
-This writes a `.env.local` file with all the required environment variables (AI_GATEWAY_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY, VERCEL_OIDC_TOKEN) — no manual key setup needed. The framework automatically loads from both `.env` and `.env.local`.
-
-## Tips
-
-**Start with `--dry`**: Always preview before running to verify your config and avoid unexpected costs.
-
-**Use multiple runs**: Single runs don't tell you reliability. Use `runs: 10` and `earlyExit: false` for meaningful data.
-
-**Isolate variables**: Change one thing at a time between experiments. Don't compare "Opus with MCP" to "Haiku without MCP".
-
-**Test incrementally**: Start with simple tasks, add complexity as you learn what works.
-
-## License
-
-MIT

package/dist/lib/agents/ai-sdk-agent.d.ts

@@ -1,10 +0,0 @@
-/**
- * AI SDK Agent - A simple coding agent using the Vercel AI SDK.
- * Works with any model available on Vercel AI Gateway.
- */
-import type { Agent } from './types.js';
-/**
- * Create AI SDK agent with Vercel AI Gateway authentication.
- */
-export declare function createAiSdkAgent(): Agent;
-//# sourceMappingURL=ai-sdk-agent.d.ts.map

package/dist/lib/agents/ai-sdk-agent.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-sdk-agent.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/ai-sdk-agent.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAmC,MAAM,YAAY,CAAC;AA8QzE;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,KAAK,CAiMxC"}